From 20c9d81018bc805315a87f4c71e9aeb91340872e Mon Sep 17 00:00:00 2001 From: grdddj Date: Tue, 27 Jun 2023 14:36:11 +0200 Subject: [PATCH] feat(tests): running master-diff report after each UI test [no changelog] --- ci/prepare_ui_artifacts.py | 5 +- core/Makefile | 12 +- tests/conftest.py | 10 +- tests/ui_tests/.gitignore | 1 + tests/ui_tests/__init__.py | 3 +- tests/ui_tests/common.py | 20 +- tests/ui_tests/reporting/common.py | 239 ++++++++++++++++++++++++ tests/ui_tests/reporting/master_diff.py | 197 +------------------ tests/ui_tests/reporting/testreport.py | 85 +++++---- 9 files changed, 336 insertions(+), 236 deletions(-) create mode 100644 tests/ui_tests/reporting/common.py diff --git a/ci/prepare_ui_artifacts.py b/ci/prepare_ui_artifacts.py index f5fc8f753..cd97555be 100644 --- a/ci/prepare_ui_artifacts.py +++ b/ci/prepare_ui_artifacts.py @@ -5,10 +5,11 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT)) # Needed for setup purposes, filling the FILE_HASHES dict -from tests.ui_tests.common import TestResult, _hash_files, get_fixtures # isort:skip +from tests.ui_tests.common import TestResult, _hash_files # isort:skip +from tests.ui_tests.common import get_current_fixtures # isort:skip -FIXTURES = get_fixtures() +FIXTURES = get_current_fixtures() for result in TestResult.recent_results(): if not result.passed or result.expected_hash != result.actual_hash: diff --git a/core/Makefile b/core/Makefile index 0db292abd..fddf4de11 100644 --- a/core/Makefile +++ b/core/Makefile @@ -115,26 +115,28 @@ test_emu_click: ## run click tests $(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests $(TESTOPTS) test_emu_click_ui: ## run click tests with UI testing - $(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests --ui=test --ui-check-missing $(TESTOPTS) + $(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests $(TESTOPTS) \ + --ui=test --ui-check-missing --do-master-diff test_emu_persistence: ## run persistence tests $(PYTEST) $(TESTPATH)/persistence_tests $(TESTOPTS) test_emu_persistence_ui: ## run persistence tests with UI testing - $(PYTEST) $(TESTPATH)/persistence_tests --ui=test --ui-check-missing $(TESTOPTS) + $(PYTEST) $(TESTPATH)/persistence_tests $(TESTOPTS) \ + --ui=test --ui-check-missing --do-master-diff test_emu_ui: ## run ui integration tests $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \ - --ui=test --ui-check-missing --record-text-layout + --ui=test --ui-check-missing --record-text-layout --do-master-diff test_emu_ui_multicore: ## run ui integration tests using multiple cores $(PYTEST) -n $(MULTICORE) $(TESTPATH)/device_tests $(TESTOPTS) --timeout $(PYTEST_TIMEOUT) \ - --ui=test --ui-check-missing --record-text-layout \ + --ui=test --ui-check-missing --record-text-layout --do-master-diff \ --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM) test_emu_ui_record: ## record and hash screens for ui integration tests $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \ - --ui=record --ui-check-missing + --ui=record --ui-check-missing --do-master-diff test_emu_ui_record_multicore: ## quickly record all screens make test_emu_ui_multicore || echo "All errors are recorded in fixtures.json" diff --git a/tests/conftest.py b/tests/conftest.py index 639333ef1..cdc0bce4b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -290,6 +290,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) - test_ui, # type: ignore bool(session.config.getoption("ui_check_missing")), bool(session.config.getoption("record_text_layout")), + bool(session.config.getoption("do_master_diff")), ) @@ -342,7 +343,14 @@ def pytest_addoption(parser: "Parser") -> None: action="store_true", default=False, help="Saving debugging traces for each screen change. " - "Will generate a report with text from all test-cases. ", + "Will generate a report with text from all test-cases.", + ) + parser.addoption( + "--do-master-diff", + action="store_true", + default=False, + help="Generating a master-diff report. " + "This shows all unique differing screens compared to master.", ) diff --git a/tests/ui_tests/.gitignore b/tests/ui_tests/.gitignore index 5e3fedad2..192127612 100644 --- a/tests/ui_tests/.gitignore +++ b/tests/ui_tests/.gitignore @@ -3,3 +3,4 @@ *.zip *.txt fixtures.suggestion.json +reporting/master_cache diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py index 5a3762bb6..ebe621125 100644 --- a/tests/ui_tests/__init__.py +++ b/tests/ui_tests/__init__.py @@ -168,11 +168,12 @@ def sessionfinish( test_ui: str, check_missing: bool, record_text_layout: bool, + do_master_diff: bool, ) -> pytest.ExitCode: if not _should_write_ui_report(exitstatus): return exitstatus - testreport.generate_reports(record_text_layout) + testreport.generate_reports(record_text_layout, do_master_diff) if test_ui == "test" and check_missing and list_missing(): common.write_fixtures( TestResult.recent_results(), diff --git a/tests/ui_tests/common.py b/tests/ui_tests/common.py index 43e4c0b8f..1406dc794 100644 --- a/tests/ui_tests/common.py +++ b/tests/ui_tests/common.py @@ -37,7 +37,7 @@ FixturesType = t.NewType("FixturesType", "dict[str, dict[str, dict[str, str]]]") FIXTURES: FixturesType = FixturesType({}) -def get_fixtures() -> FixturesType: +def get_current_fixtures() -> FixturesType: global FIXTURES if not FIXTURES and FIXTURES_FILE.exists(): FIXTURES = FixturesType(json.loads(FIXTURES_FILE.read_text())) @@ -60,7 +60,7 @@ def prepare_fixtures( missing_tests: set[TestCase] = set() # merge with previous fixtures - fixtures = deepcopy(get_fixtures()) + fixtures = deepcopy(get_current_fixtures()) for (model, group), new_content in grouped_tests.items(): # for every model/group, update the data with the new content current_content = fixtures.setdefault(model, {}).setdefault(group, {}) @@ -163,6 +163,20 @@ def _get_test_name_and_group(node_id: str) -> tuple[str, str]: return shortened_name, group_name +def get_screen_path(test_name: str) -> Path | None: + path = SCREENS_DIR / test_name / "actual" + if path.exists(): + return path + path = SCREENS_DIR / test_name / "recorded" + if path.exists(): + print( + f"WARNING: no actual screens for {test_name}, recording may be outdated: {path}" + ) + return path + print(f"WARNING: missing screens for {test_name}. Did the test run?") + return None + + def screens_diff( expected_hashes: list[str], actual_hashes: list[str] ) -> t.Iterator[tuple[str | None, str | None]]: @@ -258,7 +272,7 @@ class TestResult: def __post_init__(self) -> None: if self.expected_hash is None: self.expected_hash = ( - get_fixtures() + get_current_fixtures() .get(self.test.model, {}) .get(self.test.group, {}) .get(self.test.fixtures_name) diff --git a/tests/ui_tests/reporting/common.py b/tests/ui_tests/reporting/common.py new file mode 100644 index 000000000..b78e1ab1d --- /dev/null +++ b/tests/ui_tests/reporting/common.py @@ -0,0 +1,239 @@ +from pathlib import Path +from typing import Any + +import dominate +import dominate.tags as t +from dominate.tags import a, h1, hr, i, p, table, td, th, tr + +from ..common import ( + UI_TESTS_DIR, + FixturesType, + get_screen_path, + screens_and_hashes, + screens_diff, +) +from . import download, html + +HERE = Path(__file__).resolve().parent + +STYLE = (HERE / "testreport.css").read_text() +SCRIPT = (HERE / "testreport.js").read_text() +GIF_SCRIPT = (HERE / "create-gif.js").read_text() + +REPORTS_PATH = UI_TESTS_DIR / "reports" + +# Saving the master screens on disk not to fetch them all the time +MASTER_CACHE_DIR = HERE / "master_cache" +if not MASTER_CACHE_DIR.exists(): + MASTER_CACHE_DIR.mkdir() + + +def generate_master_diff_report( + diff_tests: dict[str, tuple[str, str]], base_dir: Path +) -> None: + unique_differing_screens = _get_unique_differing_screens(diff_tests, base_dir) + _differing_screens_report(unique_differing_screens, base_dir) + + +def get_diff( + current: FixturesType, print_to_console: bool = False +) -> tuple[dict[str, str], dict[str, str], dict[str, tuple[str, str]]]: + master = _get_preprocessed_master_fixtures() + + removed = {} + added = {} + diff = {} + + for model in master.keys() | current.keys(): + master_groups = master.get(model, {}) + current_groups = current.get(model, {}) + for group in master_groups.keys() | current_groups.keys(): + master_tests = master_groups.get(group, {}) + current_tests = current_groups.get(group, {}) + + def testname(test: str) -> str: + assert test.startswith(model + "_") + test = test[len(model) + 1 :] + return f"{model}-{group}-{test}" + + # removed items + removed_here = { + testname(test): master_tests[test] + for test in (master_tests.keys() - current_tests.keys()) + } + # added items + added_here = { + testname(test): current_tests[test] + for test in (current_tests.keys() - master_tests.keys()) + } + # create the diff from items in both branches + diff_here = {} + for master_test, master_hash in master_tests.items(): + full_test_name = testname(master_test) + if full_test_name in removed_here: + continue + if current_tests.get(master_test) == master_hash: + continue + diff_here[full_test_name] = ( + master_tests[master_test], + current_tests[master_test], + ) + + removed.update(removed_here) + added.update(added_here) + diff.update(diff_here) + if print_to_console: + print(f"{model} {group}") + print(f" removed: {len(removed_here)}") + print(f" added: {len(added_here)}") + print(f" diff: {len(diff_here)}") + + return removed, added, diff + + +def document( + title: str, + actual_hash: str | None = None, + index: bool = False, + model: str | None = None, +) -> dominate.document: + doc = dominate.document(title=title) + style = t.style() + style.add_raw_string(STYLE) + script = t.script() + script.add_raw_string(GIF_SCRIPT) + script.add_raw_string(SCRIPT) + doc.head.add(style, script) + + if actual_hash is not None: + doc.body["data-actual-hash"] = actual_hash + + if index: + doc.body["data-index"] = True + + if model: + doc.body["class"] = f"model-{model}" + + return doc + + +def _preprocess_master_compat(master_fixtures: dict[str, Any]) -> FixturesType: + if all(isinstance(v, str) for v in master_fixtures.values()): + # old format, convert to new format + new_fixtures = {} + for key, val in master_fixtures.items(): + model, _test = key.split("_", maxsplit=1) + groups_by_model = new_fixtures.setdefault(model, {}) + default_group = groups_by_model.setdefault("device_tests", {}) + default_group[key] = val + return FixturesType(new_fixtures) + else: + return FixturesType(master_fixtures) + + +def _get_preprocessed_master_fixtures() -> FixturesType: + return _preprocess_master_compat(download.fetch_fixtures_master()) + + +def _create_testcase_html_diff_file( + zipped_screens: list[tuple[str | None, str | None]], + test_name: str, + master_hash: str, + current_hash: str, + base_dir: Path, +) -> Path: + doc = document(title=test_name, model=test_name[:2]) + with doc: + h1(test_name) + p("This UI test differs from master.", style="color: grey; font-weight: bold;") + with table(): + with tr(): + td("Master:") + td(master_hash, style="color: red;") + with tr(): + td("Current:") + td(current_hash, style="color: green;") + hr() + + with table(border=1, width=600): + with tr(): + th("Master") + th("Current branch") + + html.diff_table(zipped_screens, base_dir / "diff") + + return html.write(base_dir / "diff", doc, test_name + ".html") + + +def _differing_screens_report( + unique_differing_screens: dict[tuple[str | None, str | None], str], base_dir: Path +) -> None: + try: + model = next(iter(unique_differing_screens.values()))[:2] + except StopIteration: + model = "" + + doc = document(title="Master differing screens", model=model) + with doc: + with table(border=1, width=600): + with tr(): + th("Expected") + th("Actual") + th("Testcase (link)") + + for (master, current), testcase in unique_differing_screens.items(): + with tr(bgcolor="red"): + html.image_column(master, base_dir) + html.image_column(current, base_dir) + with td(): + with a(href=f"diff/{testcase}.html"): + i(testcase) + + html.write(base_dir, doc, "master_diff.html") + + +def _get_unique_differing_screens( + diff_tests: dict[str, tuple[str, str]], base_dir: Path +) -> dict[tuple[str | None, str | None], str]: + + # Holding unique screen differences, connected with a certain testcase + # Used for diff report + unique_differing_screens: dict[tuple[str | None, str | None], str] = {} + + for test_name, (master_hash, current_hash) in diff_tests.items(): + # Downloading master recordings only if we do not have them already + master_screens_path = MASTER_CACHE_DIR / master_hash + if not master_screens_path.exists(): + master_screens_path.mkdir() + try: + download.fetch_recorded(master_hash, master_screens_path) + except RuntimeError as e: + print("WARNING:", e) + + current_screens_path = get_screen_path(test_name) + if not current_screens_path: + current_screens_path = MASTER_CACHE_DIR / "empty_current_screens" + current_screens_path.mkdir() + + # Saving all the images to a common directory + # They will be referenced from the HTML files + master_screens, master_hashes = screens_and_hashes(master_screens_path) + current_screens, current_hashes = screens_and_hashes(current_screens_path) + html.store_images(master_screens, master_hashes) + html.store_images(current_screens, current_hashes) + + # List of tuples of master and current screens + # Useful for both testcase HTML report and the differing screen report + zipped_screens = list(screens_diff(master_hashes, current_hashes)) + + # Create testcase HTML report + _create_testcase_html_diff_file( + zipped_screens, test_name, master_hash, current_hash, base_dir + ) + + # Save differing screens for differing screens report + for master, current in zipped_screens: + if master != current: + unique_differing_screens[(master, current)] = test_name + + return unique_differing_screens diff --git a/tests/ui_tests/reporting/master_diff.py b/tests/ui_tests/reporting/master_diff.py index 330bcc88d..8755cfbc3 100644 --- a/tests/ui_tests/reporting/master_diff.py +++ b/tests/ui_tests/reporting/master_diff.py @@ -4,92 +4,17 @@ import shutil import tempfile from contextlib import contextmanager from pathlib import Path -from typing import Any -from dominate.tags import a, br, h1, h2, hr, i, p, table, td, th, tr +from dominate.tags import br, h1, h2, hr, i, p, table, th, tr -from ..common import ( - SCREENS_DIR, - FixturesType, - get_fixtures, - screens_and_hashes, - screens_diff, -) +from ..common import get_current_fixtures, get_screen_path, screens_and_hashes from . import download, html -from .testreport import REPORTS_PATH, document +from .common import REPORTS_PATH, document, generate_master_diff_report, get_diff MASTERDIFF_PATH = REPORTS_PATH / "master_diff" IMAGES_PATH = MASTERDIFF_PATH / "images" -def _preprocess_master_compat(master_fixtures: dict[str, Any]) -> FixturesType: - if all(isinstance(v, str) for v in master_fixtures.values()): - # old format, convert to new format - new_fixtures = {} - for key, val in master_fixtures.items(): - model, _test = key.split("_", maxsplit=1) - groups_by_model = new_fixtures.setdefault(model, {}) - default_group = groups_by_model.setdefault("device_tests", {}) - default_group[key] = val - return FixturesType(new_fixtures) - else: - return FixturesType(master_fixtures) - - -def get_diff() -> tuple[dict[str, str], dict[str, str], dict[str, tuple[str, str]]]: - master = _preprocess_master_compat(download.fetch_fixtures_master()) - current = get_fixtures() - - removed = {} - added = {} - diff = {} - - for model in master.keys() | current.keys(): - master_groups = master.get(model, {}) - current_groups = current.get(model, {}) - for group in master_groups.keys() | current_groups.keys(): - master_tests = master_groups.get(group, {}) - current_tests = current_groups.get(group, {}) - - def testname(test: str) -> str: - assert test.startswith(model + "_") - test = test[len(model) + 1 :] - return f"{model}-{group}-{test}" - - # removed items - removed_here = { - testname(test): master_tests[test] - for test in (master_tests.keys() - current_tests.keys()) - } - # added items - added_here = { - testname(test): current_tests[test] - for test in (current_tests.keys() - master_tests.keys()) - } - # create the diff from items in both branches - diff_here = {} - for master_test, master_hash in master_tests.items(): - full_test_name = testname(master_test) - if full_test_name in removed_here: - continue - if current_tests.get(master_test) == master_hash: - continue - diff_here[full_test_name] = ( - master_tests[master_test], - current_tests[master_test], - ) - - removed.update(removed_here) - added.update(added_here) - diff.update(diff_here) - print(f"{model} {group}") - print(f" removed: {len(removed_here)}") - print(f" added: {len(added_here)}") - print(f" diff: {len(diff_here)}") - - return removed, added, diff - - def removed(screens_path: Path, test_name: str) -> Path: doc = document(title=test_name, model=test_name[:2]) screens, hashes = screens_and_hashes(screens_path) @@ -138,35 +63,6 @@ def added(screens_path: Path, test_name: str) -> Path: return html.write(MASTERDIFF_PATH / "added", doc, test_name + ".html") -def create_testcase_html_diff_file( - zipped_screens: list[tuple[str | None, str | None]], - test_name: str, - master_hash: str, - current_hash: str, -) -> Path: - doc = document(title=test_name, model=test_name[:2]) - with doc: - h1(test_name) - p("This UI test differs from master.", style="color: grey; font-weight: bold;") - with table(): - with tr(): - td("Master:") - td(master_hash, style="color: red;") - with tr(): - td("Current:") - td(current_hash, style="color: green;") - hr() - - with table(border=1, width=600): - with tr(): - th("Master") - th("Current branch") - - html.diff_table(zipped_screens, MASTERDIFF_PATH / "diff") - - return html.write(MASTERDIFF_PATH / "diff", doc, test_name + ".html") - - def index() -> Path: removed = list((MASTERDIFF_PATH / "removed").iterdir()) added = list((MASTERDIFF_PATH / "added").iterdir()) @@ -176,7 +72,7 @@ def index() -> Path: doc = document(title=title) with doc: - h1("UI changes from master") + h1(title) hr() h2("Removed:", style="color: red;") @@ -208,22 +104,9 @@ def create_dirs() -> None: IMAGES_PATH.mkdir(exist_ok=True) -def _get_screen_path(test_name: str) -> Path | None: - path = SCREENS_DIR / test_name / "actual" - if path.exists(): - return path - path = SCREENS_DIR / test_name / "recorded" - if path.exists(): - print( - f"WARNING: no actual screens for {test_name}, recording may be outdated: {path}" - ) - return path - print(f"WARNING: missing screens for {test_name}. Did the test run?") - return None - - def create_reports() -> None: - removed_tests, added_tests, diff_tests = get_diff() + current = get_current_fixtures() + removed_tests, added_tests, diff_tests = get_diff(current, print_to_console=True) @contextmanager def tmpdir(): @@ -240,76 +123,12 @@ def create_reports() -> None: removed(temp_dir, test_name) for test_name, test_hash in added_tests.items(): - screen_path = _get_screen_path(test_name) + screen_path = get_screen_path(test_name) if not screen_path: continue added(screen_path, test_name) - # Holding unique screen differences, connected with a certain testcase - # Used for diff report - unique_differing_screens: dict[tuple[str | None, str | None], str] = {} - - for test_name, (master_hash, current_hash) in diff_tests.items(): - with tmpdir() as master_root: - master_screens_path = master_root / "downloaded" - master_screens_path.mkdir() - try: - download.fetch_recorded(master_hash, master_screens_path) - except RuntimeError as e: - print("WARNING:", e) - - current_screens_path = _get_screen_path(test_name) - if not current_screens_path: - current_screens_path = master_root / "empty_current_screens" - current_screens_path.mkdir() - - # Saving all the images to a common directory - # They will be referenced from the HTML files - master_screens, master_hashes = screens_and_hashes(master_screens_path) - current_screens, current_hashes = screens_and_hashes(current_screens_path) - html.store_images(master_screens, master_hashes) - html.store_images(current_screens, current_hashes) - - # List of tuples of master and current screens - # Useful for both testcase HTML report and the differing screen report - zipped_screens = list(screens_diff(master_hashes, current_hashes)) - - # Create testcase HTML report - create_testcase_html_diff_file( - zipped_screens, - test_name, - master_hash, - current_hash, - ) - - # Save differing screens for differing screens report - for master, current in zipped_screens: - if master != current: - unique_differing_screens[(master, current)] = test_name - - differing_screens_report(unique_differing_screens) - - -def differing_screens_report( - unique_differing_screens: dict[tuple[str | None, str | None], str] -) -> None: - doc = document(title="Master differing screens") - with doc: - with table(border=1, width=600): - with tr(): - th("Expected") - th("Actual") - th("Testcase (link)") - - for (master, current), testcase in unique_differing_screens.items(): - with tr(bgcolor="red"): - html.image_column(master, MASTERDIFF_PATH) - html.image_column(current, MASTERDIFF_PATH) - with td(): - with a(href=f"diff/{testcase}.html"): - i(testcase) - - html.write(MASTERDIFF_PATH, doc, "differing_screens.html") + generate_master_diff_report(diff_tests, MASTERDIFF_PATH) def main() -> None: diff --git a/tests/ui_tests/reporting/testreport.py b/tests/ui_tests/reporting/testreport.py index 2f72de39d..335cafc80 100644 --- a/tests/ui_tests/reporting/testreport.py +++ b/tests/ui_tests/reporting/testreport.py @@ -10,50 +10,19 @@ import dominate.tags as t from dominate.tags import a, div, h1, h2, hr, i, p, span, strong, table, td, th, tr from dominate.util import text -from ..common import UI_TESTS_DIR, TestCase, TestResult +from ..common import FixturesType, TestCase, TestResult from . import download, html +from .common import REPORTS_PATH, document, generate_master_diff_report, get_diff -HERE = Path(__file__).resolve().parent -REPORTS_PATH = UI_TESTS_DIR / "reports" TESTREPORT_PATH = REPORTS_PATH / "test" IMAGES_PATH = TESTREPORT_PATH / "images" SCREEN_TEXT_FILE = TESTREPORT_PATH / "screen_text.txt" -STYLE = (HERE / "testreport.css").read_text() -SCRIPT = (HERE / "testreport.js").read_text() -GIF_SCRIPT = (HERE / "create-gif.js").read_text() - # These two html files are referencing each other ALL_SCREENS = "all_screens.html" ALL_UNIQUE_SCREENS = "all_unique_screens.html" -def document( - title: str, - actual_hash: str | None = None, - index: bool = False, - model: str | None = None, -) -> dominate.document: - doc = dominate.document(title=title) - style = t.style() - style.add_raw_string(STYLE) - script = t.script() - script.add_raw_string(GIF_SCRIPT) - script.add_raw_string(SCRIPT) - doc.head.add(style, script) - - if actual_hash is not None: - doc.body["data-actual-hash"] = actual_hash - - if index: - doc.body["data-index"] = True - - if model: - doc.body["class"] = f"model-{model}" - - return doc - - def _header(test_name: str, expected_hash: str | None, actual_hash: str) -> None: h1(test_name) with div(): @@ -85,6 +54,7 @@ def setup(main_runner: bool) -> None: (TESTREPORT_PATH / "failed").mkdir() (TESTREPORT_PATH / "passed").mkdir() (TESTREPORT_PATH / "new").mkdir() + (TESTREPORT_PATH / "diff").mkdir() IMAGES_PATH.mkdir(parents=True) html.set_image_dir(IMAGES_PATH) @@ -282,15 +252,60 @@ def differing_screens() -> None: html.write(TESTREPORT_PATH, doc, "differing_screens.html") -def generate_reports(do_screen_text: bool = False) -> None: +def _get_current_results() -> FixturesType: + current: FixturesType = {} # type: ignore + for res in TestResult.recent_results(): + model = res.test.model + group = res.test.group + fixtures_name = res.test.fixtures_name + actual_hash = res.actual_hash + if model not in current: + current[model] = {} + if group not in current[model]: + current[model][group] = {} + current[model][group][fixtures_name] = actual_hash + return current + + +def master_diff() -> None: + """Creating an HTML page showing all screens differing from master.""" + current = _get_current_results() + _removed_tests, _added_tests, diff_tests = get_diff(current) + generate_master_diff_report(diff_tests, TESTREPORT_PATH) + + +def master_index() -> Path: + """Shows all the differing tests from master.""" + diff = list((TESTREPORT_PATH / "diff").iterdir()) + + title = "UI changes from master" + doc = document(title=title) + + with doc: + h1(title) + hr() + + h2("Differs:", style="color: grey;") + i("UI fixtures that have been modified:") + html.report_links(diff, TESTREPORT_PATH) + + return html.write(TESTREPORT_PATH, doc, "master_index.html") + + +def generate_reports( + do_screen_text: bool = False, do_master_diff: bool = False +) -> None: """Generate HTML reports for the test.""" html.set_image_dir(IMAGES_PATH) index() all_screens() all_unique_screens() + differing_screens() if do_screen_text: screen_text_report() - differing_screens() + if do_master_diff: + master_diff() + master_index() def _copy_deduplicated(test: TestCase) -> None: