From 1fa9a58fa5348f8ece5825588b8605c7b9a64fb3 Mon Sep 17 00:00:00 2001 From: grdddj Date: Wed, 18 Jan 2023 15:08:59 +0100 Subject: [PATCH] feat(tests): generate UI report with all unique differing screens [no changelog] --- tests/ui_tests/common.py | 10 +++++++ tests/ui_tests/reporting/testreport.py | 40 +++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/ui_tests/common.py b/tests/ui_tests/common.py index 396c3e3d6..1950806f4 100644 --- a/tests/ui_tests/common.py +++ b/tests/ui_tests/common.py @@ -270,6 +270,9 @@ class TestResult: json.dumps(metadata, indent=2, sort_keys=True) + "\n" ) + def succeeded_in_ui_comparison(self) -> bool: + return self.actual_hash == self.expected_hash + @classmethod def load(cls, testdir: Path) -> Self: metadata = json.loads((testdir / "metadata.json").read_text()) @@ -294,6 +297,13 @@ class TestResult: continue yield cls.load(testdir) + @classmethod + def recent_ui_failures(cls) -> t.Iterator[Self]: + """Returning just the results that resulted in UI failure.""" + for result in cls.recent_results(): + if not result.succeeded_in_ui_comparison(): + yield result + def store_recorded(self) -> None: self.expected_hash = self.actual_hash shutil.rmtree(self.test.recorded_dir, ignore_errors=True) diff --git a/tests/ui_tests/reporting/testreport.py b/tests/ui_tests/reporting/testreport.py index f22744d72..7fc47dba2 100644 --- a/tests/ui_tests/reporting/testreport.py +++ b/tests/ui_tests/reporting/testreport.py @@ -7,7 +7,7 @@ from pathlib import Path import dominate import dominate.tags as t -from dominate.tags import a, div, h1, h2, hr, p, span, strong, table, th, tr +from dominate.tags import a, div, h1, h2, hr, i, p, span, strong, table, td, th, tr from dominate.util import text from ..common import UI_TESTS_DIR, TestCase, TestResult @@ -243,6 +243,43 @@ def screen_text_report() -> None: html.write(TESTREPORT_PATH, doc, "screen_text.html") +def differing_screens() -> None: + """Creating an HTML page showing all the unique screens that got changed.""" + unique_diffs: set[tuple[str | None, str | None]] = set() + + def already_included(left: str | None, right: str | None) -> bool: + return (left, right) in unique_diffs + + def include(left: str | None, right: str | None) -> None: + unique_diffs.add((left, right)) + + # Only going through tests failed in UI comparison, + # there are no differing screens in UI-passed tests. + recent_ui_failures = list(TestResult.recent_ui_failures()) + + model = recent_ui_failures[0].test.model if recent_ui_failures else None + doc = document(title="Differing screens", model=model) + with doc: + with table(border=1, width=600): + with tr(): + th("Expected") + th("Actual") + th("Testcase (link)") + + for ui_failure in recent_ui_failures: + for recorded, actual in ui_failure.diff_lines(): + if recorded != actual and not already_included(recorded, actual): + include(recorded, actual) + with tr(bgcolor="red"): + html.image_column(recorded, TESTREPORT_PATH) + html.image_column(actual, TESTREPORT_PATH) + with td(): + with a(href=f"failed/{ui_failure.test.id}.html"): + i(ui_failure.test.id) + + html.write(TESTREPORT_PATH, doc, "differing_screens.html") + + def generate_reports(do_screen_text: bool = False) -> None: """Generate HTML reports for the test.""" html.set_image_dir(IMAGES_PATH) @@ -251,6 +288,7 @@ def generate_reports(do_screen_text: bool = False) -> None: all_unique_screens() if do_screen_text: screen_text_report() + differing_screens() def _copy_deduplicated(test: TestCase) -> None: