feat(tests): add dashboards with indexed unique screens and all screens

tests/conftest.py

@@ -269,13 +269,13 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) -
         if missing and ui_tests.list_missing():
             session.exitstatus = pytest.ExitCode.TESTS_FAILED
         ui_tests.write_fixtures_suggestion(missing)
-        testreport.index()
+        testreport.generate_reports()
     elif test_ui == "record":
         if exitstatus == pytest.ExitCode.OK:
             ui_tests.write_fixtures(missing)
         else:
             ui_tests.write_fixtures_suggestion(missing, only_passed_tests=True)
-        testreport.index()
+        testreport.generate_reports()
 
 
 def pytest_terminal_summary(
@@ -390,6 +390,9 @@ def pytest_runtest_teardown(item: pytest.Item) -> None:
 
     Dumps the current UI test report HTML.
     """
+    # Not calling `testreport.generate_reports()` not to generate
+    # the `all_screens` report, as would take a lot of time.
+    # That will be generated in `pytest_sessionfinish`.
     if item.session.config.getoption("ui"):
         testreport.index()
 

tests/ui_tests/__init__.py

@@ -219,4 +219,4 @@ def main() -> None:
         except Failed:
             print("FAILED:", record.name)
 
-    testreport.index()
+    testreport.generate_reports()

tests/ui_tests/reporting/html.py

@@ -4,6 +4,7 @@ from itertools import zip_longest
 from pathlib import Path
 from typing import Dict, List, Optional
 
+from dominate import document
 from dominate.tags import a, i, img, table, td, th, tr
 
 
@@ -25,31 +26,37 @@ def report_links(
                 td(a(test.name, href=path))
 
 
-def write(fixture_test_path: Path, doc, filename: str) -> Path:
+def write(fixture_test_path: Path, doc: document, filename: str) -> Path:
     (fixture_test_path / filename).write_text(doc.render())
     return fixture_test_path / filename
 
 
-def image(src: Path, image_width: Optional[int] = None) -> None:
+def image_column(src: Path, image_width: Optional[int] = None) -> None:
+    """Put image into table as one cell."""
     with td():
         if src:
-            # open image file
+            image_raw(src, image_width)
-            image = src.read_bytes()
-            # encode image as base64
-            image = base64.b64encode(image)
-            # convert output to str
-            image = image.decode()
-            # img(src=src.relative_to(fixture_test_path))
-            img(
-                src="data:image/png;base64, " + image,
-                style=f"width: {image_width}px; image-rendering: pixelated;"
-                if image_width
-                else "",
-            )
         else:
             i("missing")
 
 
+def image_raw(src: Path, image_width: Optional[int] = None) -> None:
+    """Display image on the screen"""
+    # open image file
+    image = src.read_bytes()
+    # encode image as base64
+    image = base64.b64encode(image)
+    # convert output to str
+    image = image.decode()
+    # img(src=src.relative_to(fixture_test_path))
+    img(
+        src="data:image/png;base64, " + image,
+        style=f"width: {image_width}px; image-rendering: pixelated;"
+        if image_width
+        else "",
+    )
+
+
 def diff_table(
     left_screens: List[Path],
     right_screens: List[Path],
@@ -61,5 +68,5 @@ def diff_table(
         else:
             background = "red"
         with tr(bgcolor=background):
-            image(left, image_width)
+            image_column(left, image_width)
-            image(right, image_width)
+            image_column(right, image_width)

tests/ui_tests/reporting/report_master_diff.py

@@ -62,7 +62,7 @@ def removed(screens_path: Path, test_name: str) -> Path:
 
             for screen in screens:
                 with tr():
-                    html.image(screen)
+                    html.image_column(screen)
 
     return html.write(REPORTS_PATH / "removed", doc, test_name + ".html")
 
@@ -85,7 +85,7 @@ def added(screens_path: Path, test_name: str) -> Path:
 
             for screen in screens:
                 with tr():
-                    html.image(screen)
+                    html.image_column(screen)
 
     return html.write(REPORTS_PATH / "added", doc, test_name + ".html")
 

tests/ui_tests/reporting/testreport.py

@@ -1,18 +1,21 @@
+import hashlib
 import shutil
+from collections import defaultdict
 from datetime import datetime
 from distutils.dir_util import copy_tree
 from pathlib import Path
-from typing import Dict
+from typing import Dict, List, Set
 
 import dominate
 import dominate.tags as t
-from dominate.tags import div, h1, h2, hr, p, strong, table, th, tr
+from dominate.tags import a, div, h1, h2, hr, p, span, strong, table, th, tr
 from dominate.util import text
 
 from . import download, html
 
 HERE = Path(__file__).resolve().parent
 REPORTS_PATH = HERE / "reports" / "test"
+RECORDED_SCREENS_PATH = Path(__file__).resolve().parent.parent / "screens"
 
 STYLE = (HERE / "testreport.css").read_text()
 SCRIPT = (HERE / "testreport.js").read_text()
@@ -22,9 +25,21 @@ SCREENSHOTS_WIDTH_PX_TO_DISPLAY = {
     "TR": 128 * 2,  # original is 128px
 }
 
+# These two html files are referencing each other
+ALL_SCREENS = "all_screens.html"
+ALL_UNIQUE_SCREENS = "all_unique_screens.html"
+
 ACTUAL_HASHES: Dict[str, str] = {}
 
 
+def _image_width(test_name: str) -> int:
+    """Return the width of the image to display for the given test name.
+
+    Is model-specific. Model is at the beginning of each test-case.
+    """
+    return SCREENSHOTS_WIDTH_PX_TO_DISPLAY[test_name[:2]]
+
+
 def document(
     title: str, actual_hash: str = None, index: bool = False
 ) -> dominate.document:
@@ -63,7 +78,7 @@ def _header(test_name: str, expected_hash: str, actual_hash: str) -> None:
 
 
 def clear_dir() -> None:
-    # delete and create the reports dir to clear previous entries
+    """Delete and create the reports dir to clear previous entries."""
     shutil.rmtree(REPORTS_PATH, ignore_errors=True)
     REPORTS_PATH.mkdir()
     (REPORTS_PATH / "failed").mkdir()
@@ -71,6 +86,7 @@ def clear_dir() -> None:
 
 
 def index() -> Path:
+    """Generate index.html with all the test results - lists of failed and passed tests."""
     passed_tests = list((REPORTS_PATH / "passed").iterdir())
     failed_tests = list((REPORTS_PATH / "failed").iterdir())
 
@@ -112,9 +128,124 @@ def index() -> Path:
     return html.write(REPORTS_PATH, doc, "index.html")
 
 
+def all_screens(test_case_dirs: List[Path]) -> Path:
+    """Generate an HTML file for all the screens from the current test run.
+
+    Shows all test-cases at one place.
+    """
+    title = "All test cases"
+    doc = dominate.document(title=title)
+
+    with doc:
+        h1("All test cases")
+        hr()
+
+        count = 0
+        for test_case_dir in test_case_dirs:
+            test_case_name = test_case_dir.name
+            h2(test_case_name, id=test_case_name)
+            actual_dir = test_case_dir / "actual"
+            for png in sorted(actual_dir.rglob("*.png")):
+                # Including link to each image to see where else it occurs.
+                png_hash = _img_hash(png)
+                with a(href=f"{ALL_UNIQUE_SCREENS}#{png_hash}"):
+                    html.image_raw(png, _image_width(test_case_name))
+                count += 1
+
+        h2(f"{count} screens from {len(test_case_dirs)} testcases.")
+
+    return html.write(REPORTS_PATH, doc, ALL_SCREENS)
+
+
+def all_unique_screens(test_case_dirs: List[Path]) -> Path:
+    """Generate an HTML file with all the unique screens from the current test run."""
+    title = "All unique screens"
+    doc = dominate.document(title=title)
+
+    with doc:
+        h1("All unique screens")
+        hr()
+
+        screen_hashes: Dict[str, List[Path]] = defaultdict(list)
+        hash_images: Dict[str, Path] = {}
+
+        # Adding all unique images onto the page
+        for test_case_dir in test_case_dirs:
+            actual_dir = test_case_dir / "actual"
+            for png in sorted(actual_dir.rglob("*.png")):
+                png_hash = _img_hash(png)
+                if png_hash not in screen_hashes:
+                    # Adding link to the appropriate hash, where other testcases
+                    # with the same hash (screen) are listed.
+                    with a(href=f"#{png_hash}"):
+                        with span(id=png_hash[:8]):
+                            html.image_raw(png, _image_width(test_case_dir.name))
+
+                screen_hashes[png_hash].append(test_case_dir)
+                hash_images[png_hash] = png
+
+        # Adding all screen hashes together with links to testcases having these screens.
+        for png_hash, test_cases in screen_hashes.items():
+            h2(png_hash)
+            with div(id=png_hash):
+                # Showing the exact image as well (not magnifying it)
+                with a(href=f"#{png_hash[:8]}"):
+                    html.image_raw(hash_images[png_hash])
+                for case in test_cases:
+                    # Adding link to each test-case
+                    with a(href=f"{ALL_SCREENS}#{case.name}"):
+                        p(case.name.split("/")[-1])
+
+        h2(f"{len(screen_hashes)} unique screens from {len(test_case_dirs)} testcases.")
+
+    return html.write(REPORTS_PATH, doc, ALL_UNIQUE_SCREENS)
+
+
+def generate_reports() -> None:
+    """Generate HTML reports for the test."""
+    index()
+
+    # To only get screens from the last running test-cases,
+    # we need to get the list of all directories with screenshots.
+    current_testcases = _get_testcases_dirs()
+    all_screens(current_testcases)
+    all_unique_screens(current_testcases)
+
+
+def _img_hash(img: Path) -> str:
+    """Return the hash of the image."""
+    content = img.read_bytes()
+    return hashlib.md5(content).hexdigest()
+
+
+def _get_testcases_dirs() -> List[Path]:
+    """Get the list of test-cases dirs that the current test was running."""
+    current_testcases = _get_all_current_testcases()
+    all_test_cases_dirs = [
+        case
+        for case in (RECORDED_SCREENS_PATH).iterdir()
+        if case.name in current_testcases
+    ]
+    return sorted(all_test_cases_dirs)
+
+
+def _get_all_current_testcases() -> Set[str]:
+    """Get names of all current test-cases.
+
+    Equals to the names of HTML files in the reports dir.
+    """
+    passed_tests = list((REPORTS_PATH / "passed").glob("*.html"))
+    failed_tests = list((REPORTS_PATH / "failed").glob("*.html"))
+    return {test.stem for test in (passed_tests + failed_tests)}
+
+
 def failed(
     fixture_test_path: Path, test_name: str, actual_hash: str, expected_hash: str
 ) -> Path:
+    """Generate an HTML file for a failed test-case.
+
+    Compares the actual screenshots to the expected ones.
+    """
     ACTUAL_HASHES[test_name] = actual_hash
 
     doc = document(title=test_name, actual_hash=actual_hash)
@@ -155,19 +286,24 @@ def failed(
             html.diff_table(
                 recorded_screens,
                 actual_screens,
-                SCREENSHOTS_WIDTH_PX_TO_DISPLAY[test_name[:2]],
+                _image_width(test_name),
             )
 
     return html.write(REPORTS_PATH / "failed", doc, test_name + ".html")
 
 
 def passed(fixture_test_path: Path, test_name: str, actual_hash: str) -> Path:
+    """Generate an HTML file for a passed test-case."""
     copy_tree(str(fixture_test_path / "actual"), str(fixture_test_path / "recorded"))
 
     return recorded(fixture_test_path / "actual", test_name, actual_hash)
 
 
 def recorded(fixture_test_path: Path, test_name: str, actual_hash: str) -> Path:
+    """Generate an HTML file for a passed test-case.
+
+    Shows all the screens from it in exact order.
+    """
     doc = document(title=test_name)
     actual_screens = sorted(fixture_test_path.iterdir())
 
@@ -180,6 +316,6 @@ def recorded(fixture_test_path: Path, test_name: str, actual_hash: str) -> Path:
 
             for screen in actual_screens:
                 with tr():
-                    html.image(screen, SCREENSHOTS_WIDTH_PX_TO_DISPLAY[test_name[:2]])
+                    html.image_column(screen, _image_width(test_name))
 
     return html.write(REPORTS_PATH / "passed", doc, test_name + ".html")