diff --git a/tests/ui_tests/reporting/master_diff.py b/tests/ui_tests/reporting/master_diff.py index 55352be51..e3bbd1537 100644 --- a/tests/ui_tests/reporting/master_diff.py +++ b/tests/ui_tests/reporting/master_diff.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from pathlib import Path from typing import Any -from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr +from dominate.tags import a, br, h1, h2, hr, i, p, table, td, th, tr from ..common import ( SCREENS_DIR, @@ -138,19 +138,13 @@ def added(screens_path: Path, test_name: str) -> Path: return html.write(MASTERDIFF_PATH / "added", doc, test_name + ".html") -def diff( - master_screens_path: Path, - current_screens_path: Path, +def create_testcase_html_diff_file( + zipped_screens: list[tuple[str | None, str | None]], test_name: str, master_hash: str, current_hash: str, ) -> Path: doc = document(title=test_name, model=test_name[:2]) - master_screens, master_hashes = screens_and_hashes(master_screens_path) - current_screens, current_hashes = screens_and_hashes(current_screens_path) - html.store_images(master_screens, master_hashes) - html.store_images(current_screens, current_hashes) - with doc: h1(test_name) p("This UI test differs from master.", style="color: grey; font-weight: bold;") @@ -168,9 +162,7 @@ def diff( th("Master") th("Current branch") - html.diff_table( - screens_diff(master_hashes, current_hashes), MASTERDIFF_PATH / "diff" - ) + html.diff_table(zipped_screens, MASTERDIFF_PATH / "diff") return html.write(MASTERDIFF_PATH / "diff", doc, test_name + ".html") @@ -249,28 +241,72 @@ def create_reports() -> None: continue added(screen_path, test_name) + # Holding unique screen differences, connected with a certain testcase + # Used for diff report + unique_differing_screens: dict[tuple[str | None, str | None], str] = {} + for test_name, (master_hash, current_hash) in diff_tests.items(): with tmpdir() as master_root: - master_screens = master_root / "downloaded" - master_screens.mkdir() + master_screens_path = master_root / "downloaded" + master_screens_path.mkdir() try: - download.fetch_recorded(master_hash, master_screens) + download.fetch_recorded(master_hash, master_screens_path) except RuntimeError as e: print("WARNING:", e) - current_screens = _get_screen_path(test_name) - if not current_screens: - current_screens = master_root / "empty_current_screens" - current_screens.mkdir() - - diff( - master_screens, - current_screens, + current_screens_path = _get_screen_path(test_name) + if not current_screens_path: + current_screens_path = master_root / "empty_current_screens" + current_screens_path.mkdir() + + # Saving all the images to a common directory + # They will be referenced from the HTML files + master_screens, master_hashes = screens_and_hashes(master_screens_path) + current_screens, current_hashes = screens_and_hashes(current_screens_path) + html.store_images(master_screens, master_hashes) + html.store_images(current_screens, current_hashes) + + # List of tuples of master and current screens + # Useful for both testcase HTML report and the differing screen report + zipped_screens = list(screens_diff(master_hashes, current_hashes)) + + # Create testcase HTML report + create_testcase_html_diff_file( + zipped_screens, test_name, master_hash, current_hash, ) + # Save differing screens for differing screens report + for master, current in zipped_screens: + if master != current: + unique_differing_screens[(master, current)] = test_name + + differing_screens_report(unique_differing_screens) + + +def differing_screens_report( + unique_differing_screens: dict[tuple[str | None, str | None], str] +) -> None: + doc = document(title="Master differing screens") + with doc: + with table(border=1, width=600): + with tr(): + th("Expected") + th("Actual") + th("Testcase (link)") + + for (master, current), testcase in unique_differing_screens.items(): + with tr(bgcolor="red"): + html.image_column(master, MASTERDIFF_PATH) + html.image_column(current, MASTERDIFF_PATH) + with td(): + with a(href=f"diff/{testcase}.html"): + i(testcase) + + html.write(MASTERDIFF_PATH, doc, "differing_screens.html") + def main() -> None: create_dirs()