feat(tests): running master-diff report after each UI test

[no changelog]

ci/prepare_ui_artifacts.py

@@ -5,10 +5,11 @@ from pathlib import Path
 ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(ROOT))
 # Needed for setup purposes, filling the FILE_HASHES dict
-from tests.ui_tests.common import TestResult, _hash_files, get_fixtures  # isort:skip
+from tests.ui_tests.common import TestResult, _hash_files  # isort:skip
+from tests.ui_tests.common import get_current_fixtures  # isort:skip
 
 
-FIXTURES = get_fixtures()
+FIXTURES = get_current_fixtures()
 
 for result in TestResult.recent_results():
     if not result.passed or result.expected_hash != result.actual_hash:

core/Makefile

@@ -115,26 +115,28 @@ test_emu_click: ## run click tests
 	$(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests $(TESTOPTS)
 
 test_emu_click_ui: ## run click tests with UI testing
-	$(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests --ui=test --ui-check-missing $(TESTOPTS)
+	$(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests $(TESTOPTS) \
+		--ui=test --ui-check-missing --do-master-diff
 
 test_emu_persistence: ## run persistence tests
 	$(PYTEST) $(TESTPATH)/persistence_tests $(TESTOPTS)
 
 test_emu_persistence_ui: ## run persistence tests with UI testing
-	$(PYTEST) $(TESTPATH)/persistence_tests --ui=test --ui-check-missing $(TESTOPTS)
+	$(PYTEST) $(TESTPATH)/persistence_tests $(TESTOPTS) \
+		--ui=test --ui-check-missing --do-master-diff
 
 test_emu_ui: ## run ui integration tests
 	$(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \
-		--ui=test --ui-check-missing --record-text-layout
+		--ui=test --ui-check-missing --record-text-layout --do-master-diff
 
 test_emu_ui_multicore: ## run ui integration tests using multiple cores
 	$(PYTEST) -n $(MULTICORE) $(TESTPATH)/device_tests $(TESTOPTS) --timeout $(PYTEST_TIMEOUT) \
-		--ui=test --ui-check-missing --record-text-layout \
+		--ui=test --ui-check-missing --record-text-layout --do-master-diff \
 		--control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM)
 
 test_emu_ui_record: ## record and hash screens for ui integration tests
 	$(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \
-		--ui=record --ui-check-missing
+		--ui=record --ui-check-missing --do-master-diff
 
 test_emu_ui_record_multicore: ## quickly record all screens
 	make test_emu_ui_multicore || echo "All errors are recorded in fixtures.json"

tests/conftest.py

@@ -290,6 +290,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) -
             test_ui,  # type: ignore
             bool(session.config.getoption("ui_check_missing")),
             bool(session.config.getoption("record_text_layout")),
+            bool(session.config.getoption("do_master_diff")),
         )
 
 
@@ -344,6 +345,13 @@ def pytest_addoption(parser: "Parser") -> None:
         help="Saving debugging traces for each screen change. "
         "Will generate a report with text from all test-cases.",
     )
+    parser.addoption(
+        "--do-master-diff",
+        action="store_true",
+        default=False,
+        help="Generating a master-diff report. "
+        "This shows all unique differing screens compared to master.",
+    )
 
 
 def pytest_configure(config: "Config") -> None:

tests/ui_tests/.gitignore

@@ -3,3 +3,4 @@
 *.zip
 *.txt
 fixtures.suggestion.json
+reporting/master_cache

tests/ui_tests/__init__.py

@@ -168,11 +168,12 @@ def sessionfinish(
     test_ui: str,
     check_missing: bool,
     record_text_layout: bool,
+    do_master_diff: bool,
 ) -> pytest.ExitCode:
     if not _should_write_ui_report(exitstatus):
         return exitstatus
 
-    testreport.generate_reports(record_text_layout)
+    testreport.generate_reports(record_text_layout, do_master_diff)
     if test_ui == "test" and check_missing and list_missing():
         common.write_fixtures(
             TestResult.recent_results(),

tests/ui_tests/common.py

@@ -37,7 +37,7 @@ FixturesType = t.NewType("FixturesType", "dict[str, dict[str, dict[str, str]]]")
 FIXTURES: FixturesType = FixturesType({})
 
 
-def get_fixtures() -> FixturesType:
+def get_current_fixtures() -> FixturesType:
     global FIXTURES
     if not FIXTURES and FIXTURES_FILE.exists():
         FIXTURES = FixturesType(json.loads(FIXTURES_FILE.read_text()))
@@ -60,7 +60,7 @@ def prepare_fixtures(
     missing_tests: set[TestCase] = set()
 
     # merge with previous fixtures
-    fixtures = deepcopy(get_fixtures())
+    fixtures = deepcopy(get_current_fixtures())
     for (model, group), new_content in grouped_tests.items():
         # for every model/group, update the data with the new content
         current_content = fixtures.setdefault(model, {}).setdefault(group, {})
@@ -163,6 +163,20 @@ def _get_test_name_and_group(node_id: str) -> tuple[str, str]:
     return shortened_name, group_name
 
 
+def get_screen_path(test_name: str) -> Path | None:
+    path = SCREENS_DIR / test_name / "actual"
+    if path.exists():
+        return path
+    path = SCREENS_DIR / test_name / "recorded"
+    if path.exists():
+        print(
+            f"WARNING: no actual screens for {test_name}, recording may be outdated: {path}"
+        )
+        return path
+    print(f"WARNING: missing screens for {test_name}. Did the test run?")
+    return None
+
+
 def screens_diff(
     expected_hashes: list[str], actual_hashes: list[str]
 ) -> t.Iterator[tuple[str | None, str | None]]:
@@ -258,7 +272,7 @@ class TestResult:
     def __post_init__(self) -> None:
         if self.expected_hash is None:
             self.expected_hash = (
-                get_fixtures()
+                get_current_fixtures()
                 .get(self.test.model, {})
                 .get(self.test.group, {})
                 .get(self.test.fixtures_name)

tests/ui_tests/reporting/common.py

@@ -0,0 +1,239 @@
+from pathlib import Path
+from typing import Any
+
+import dominate
+import dominate.tags as t
+from dominate.tags import a, h1, hr, i, p, table, td, th, tr
+
+from ..common import (
+    UI_TESTS_DIR,
+    FixturesType,
+    get_screen_path,
+    screens_and_hashes,
+    screens_diff,
+)
+from . import download, html
+
+HERE = Path(__file__).resolve().parent
+
+STYLE = (HERE / "testreport.css").read_text()
+SCRIPT = (HERE / "testreport.js").read_text()
+GIF_SCRIPT = (HERE / "create-gif.js").read_text()
+
+REPORTS_PATH = UI_TESTS_DIR / "reports"
+
+# Saving the master screens on disk not to fetch them all the time
+MASTER_CACHE_DIR = HERE / "master_cache"
+if not MASTER_CACHE_DIR.exists():
+    MASTER_CACHE_DIR.mkdir()
+
+
+def generate_master_diff_report(
+    diff_tests: dict[str, tuple[str, str]], base_dir: Path
+) -> None:
+    unique_differing_screens = _get_unique_differing_screens(diff_tests, base_dir)
+    _differing_screens_report(unique_differing_screens, base_dir)
+
+
+def get_diff(
+    current: FixturesType, print_to_console: bool = False
+) -> tuple[dict[str, str], dict[str, str], dict[str, tuple[str, str]]]:
+    master = _get_preprocessed_master_fixtures()
+
+    removed = {}
+    added = {}
+    diff = {}
+
+    for model in master.keys() | current.keys():
+        master_groups = master.get(model, {})
+        current_groups = current.get(model, {})
+        for group in master_groups.keys() | current_groups.keys():
+            master_tests = master_groups.get(group, {})
+            current_tests = current_groups.get(group, {})
+
+            def testname(test: str) -> str:
+                assert test.startswith(model + "_")
+                test = test[len(model) + 1 :]
+                return f"{model}-{group}-{test}"
+
+            # removed items
+            removed_here = {
+                testname(test): master_tests[test]
+                for test in (master_tests.keys() - current_tests.keys())
+            }
+            # added items
+            added_here = {
+                testname(test): current_tests[test]
+                for test in (current_tests.keys() - master_tests.keys())
+            }
+            # create the diff from items in both branches
+            diff_here = {}
+            for master_test, master_hash in master_tests.items():
+                full_test_name = testname(master_test)
+                if full_test_name in removed_here:
+                    continue
+                if current_tests.get(master_test) == master_hash:
+                    continue
+                diff_here[full_test_name] = (
+                    master_tests[master_test],
+                    current_tests[master_test],
+                )
+
+            removed.update(removed_here)
+            added.update(added_here)
+            diff.update(diff_here)
+            if print_to_console:
+                print(f"{model} {group}")
+                print(f"  removed: {len(removed_here)}")
+                print(f"  added: {len(added_here)}")
+                print(f"  diff: {len(diff_here)}")
+
+    return removed, added, diff
+
+
+def document(
+    title: str,
+    actual_hash: str | None = None,
+    index: bool = False,
+    model: str | None = None,
+) -> dominate.document:
+    doc = dominate.document(title=title)
+    style = t.style()
+    style.add_raw_string(STYLE)
+    script = t.script()
+    script.add_raw_string(GIF_SCRIPT)
+    script.add_raw_string(SCRIPT)
+    doc.head.add(style, script)
+
+    if actual_hash is not None:
+        doc.body["data-actual-hash"] = actual_hash
+
+    if index:
+        doc.body["data-index"] = True
+
+    if model:
+        doc.body["class"] = f"model-{model}"
+
+    return doc
+
+
+def _preprocess_master_compat(master_fixtures: dict[str, Any]) -> FixturesType:
+    if all(isinstance(v, str) for v in master_fixtures.values()):
+        # old format, convert to new format
+        new_fixtures = {}
+        for key, val in master_fixtures.items():
+            model, _test = key.split("_", maxsplit=1)
+            groups_by_model = new_fixtures.setdefault(model, {})
+            default_group = groups_by_model.setdefault("device_tests", {})
+            default_group[key] = val
+        return FixturesType(new_fixtures)
+    else:
+        return FixturesType(master_fixtures)
+
+
+def _get_preprocessed_master_fixtures() -> FixturesType:
+    return _preprocess_master_compat(download.fetch_fixtures_master())
+
+
+def _create_testcase_html_diff_file(
+    zipped_screens: list[tuple[str | None, str | None]],
+    test_name: str,
+    master_hash: str,
+    current_hash: str,
+    base_dir: Path,
+) -> Path:
+    doc = document(title=test_name, model=test_name[:2])
+    with doc:
+        h1(test_name)
+        p("This UI test differs from master.", style="color: grey; font-weight: bold;")
+        with table():
+            with tr():
+                td("Master:")
+                td(master_hash, style="color: red;")
+            with tr():
+                td("Current:")
+                td(current_hash, style="color: green;")
+        hr()
+
+        with table(border=1, width=600):
+            with tr():
+                th("Master")
+                th("Current branch")
+
+            html.diff_table(zipped_screens, base_dir / "diff")
+
+    return html.write(base_dir / "diff", doc, test_name + ".html")
+
+
+def _differing_screens_report(
+    unique_differing_screens: dict[tuple[str | None, str | None], str], base_dir: Path
+) -> None:
+    try:
+        model = next(iter(unique_differing_screens.values()))[:2]
+    except StopIteration:
+        model = ""
+
+    doc = document(title="Master differing screens", model=model)
+    with doc:
+        with table(border=1, width=600):
+            with tr():
+                th("Expected")
+                th("Actual")
+                th("Testcase (link)")
+
+            for (master, current), testcase in unique_differing_screens.items():
+                with tr(bgcolor="red"):
+                    html.image_column(master, base_dir)
+                    html.image_column(current, base_dir)
+                    with td():
+                        with a(href=f"diff/{testcase}.html"):
+                            i(testcase)
+
+    html.write(base_dir, doc, "master_diff.html")
+
+
+def _get_unique_differing_screens(
+    diff_tests: dict[str, tuple[str, str]], base_dir: Path
+) -> dict[tuple[str | None, str | None], str]:
+
+    # Holding unique screen differences, connected with a certain testcase
+    # Used for diff report
+    unique_differing_screens: dict[tuple[str | None, str | None], str] = {}
+
+    for test_name, (master_hash, current_hash) in diff_tests.items():
+        # Downloading master recordings only if we do not have them already
+        master_screens_path = MASTER_CACHE_DIR / master_hash
+        if not master_screens_path.exists():
+            master_screens_path.mkdir()
+            try:
+                download.fetch_recorded(master_hash, master_screens_path)
+            except RuntimeError as e:
+                print("WARNING:", e)
+
+        current_screens_path = get_screen_path(test_name)
+        if not current_screens_path:
+            current_screens_path = MASTER_CACHE_DIR / "empty_current_screens"
+            current_screens_path.mkdir()
+
+        # Saving all the images to a common directory
+        # They will be referenced from the HTML files
+        master_screens, master_hashes = screens_and_hashes(master_screens_path)
+        current_screens, current_hashes = screens_and_hashes(current_screens_path)
+        html.store_images(master_screens, master_hashes)
+        html.store_images(current_screens, current_hashes)
+
+        # List of tuples of master and current screens
+        # Useful for both testcase HTML report and the differing screen report
+        zipped_screens = list(screens_diff(master_hashes, current_hashes))
+
+        # Create testcase HTML report
+        _create_testcase_html_diff_file(
+            zipped_screens, test_name, master_hash, current_hash, base_dir
+        )
+
+        # Save differing screens for differing screens report
+        for master, current in zipped_screens:
+            if master != current:
+                unique_differing_screens[(master, current)] = test_name
+
+    return unique_differing_screens

tests/ui_tests/reporting/master_diff.py

@@ -4,92 +4,17 @@ import shutil
 import tempfile
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any
 
-from dominate.tags import a, br, h1, h2, hr, i, p, table, td, th, tr
+from dominate.tags import br, h1, h2, hr, i, p, table, th, tr
 
-from ..common import (
+from ..common import get_current_fixtures, get_screen_path, screens_and_hashes
-    SCREENS_DIR,
-    FixturesType,
-    get_fixtures,
-    screens_and_hashes,
-    screens_diff,
-)
 from . import download, html
-from .testreport import REPORTS_PATH, document
+from .common import REPORTS_PATH, document, generate_master_diff_report, get_diff
 
 MASTERDIFF_PATH = REPORTS_PATH / "master_diff"
 IMAGES_PATH = MASTERDIFF_PATH / "images"
 
 
-def _preprocess_master_compat(master_fixtures: dict[str, Any]) -> FixturesType:
-    if all(isinstance(v, str) for v in master_fixtures.values()):
-        # old format, convert to new format
-        new_fixtures = {}
-        for key, val in master_fixtures.items():
-            model, _test = key.split("_", maxsplit=1)
-            groups_by_model = new_fixtures.setdefault(model, {})
-            default_group = groups_by_model.setdefault("device_tests", {})
-            default_group[key] = val
-        return FixturesType(new_fixtures)
-    else:
-        return FixturesType(master_fixtures)
-
-
-def get_diff() -> tuple[dict[str, str], dict[str, str], dict[str, tuple[str, str]]]:
-    master = _preprocess_master_compat(download.fetch_fixtures_master())
-    current = get_fixtures()
-
-    removed = {}
-    added = {}
-    diff = {}
-
-    for model in master.keys() | current.keys():
-        master_groups = master.get(model, {})
-        current_groups = current.get(model, {})
-        for group in master_groups.keys() | current_groups.keys():
-            master_tests = master_groups.get(group, {})
-            current_tests = current_groups.get(group, {})
-
-            def testname(test: str) -> str:
-                assert test.startswith(model + "_")
-                test = test[len(model) + 1 :]
-                return f"{model}-{group}-{test}"
-
-            # removed items
-            removed_here = {
-                testname(test): master_tests[test]
-                for test in (master_tests.keys() - current_tests.keys())
-            }
-            # added items
-            added_here = {
-                testname(test): current_tests[test]
-                for test in (current_tests.keys() - master_tests.keys())
-            }
-            # create the diff from items in both branches
-            diff_here = {}
-            for master_test, master_hash in master_tests.items():
-                full_test_name = testname(master_test)
-                if full_test_name in removed_here:
-                    continue
-                if current_tests.get(master_test) == master_hash:
-                    continue
-                diff_here[full_test_name] = (
-                    master_tests[master_test],
-                    current_tests[master_test],
-                )
-
-            removed.update(removed_here)
-            added.update(added_here)
-            diff.update(diff_here)
-            print(f"{model} {group}")
-            print(f"  removed: {len(removed_here)}")
-            print(f"  added: {len(added_here)}")
-            print(f"  diff: {len(diff_here)}")
-
-    return removed, added, diff
-
-
 def removed(screens_path: Path, test_name: str) -> Path:
     doc = document(title=test_name, model=test_name[:2])
     screens, hashes = screens_and_hashes(screens_path)
@@ -138,35 +63,6 @@ def added(screens_path: Path, test_name: str) -> Path:
     return html.write(MASTERDIFF_PATH / "added", doc, test_name + ".html")
 
 
-def create_testcase_html_diff_file(
-    zipped_screens: list[tuple[str | None, str | None]],
-    test_name: str,
-    master_hash: str,
-    current_hash: str,
-) -> Path:
-    doc = document(title=test_name, model=test_name[:2])
-    with doc:
-        h1(test_name)
-        p("This UI test differs from master.", style="color: grey; font-weight: bold;")
-        with table():
-            with tr():
-                td("Master:")
-                td(master_hash, style="color: red;")
-            with tr():
-                td("Current:")
-                td(current_hash, style="color: green;")
-        hr()
-
-        with table(border=1, width=600):
-            with tr():
-                th("Master")
-                th("Current branch")
-
-            html.diff_table(zipped_screens, MASTERDIFF_PATH / "diff")
-
-    return html.write(MASTERDIFF_PATH / "diff", doc, test_name + ".html")
-
-
 def index() -> Path:
     removed = list((MASTERDIFF_PATH / "removed").iterdir())
     added = list((MASTERDIFF_PATH / "added").iterdir())
@@ -176,7 +72,7 @@ def index() -> Path:
     doc = document(title=title)
 
     with doc:
-        h1("UI changes from master")
+        h1(title)
         hr()
 
         h2("Removed:", style="color: red;")
@@ -208,22 +104,9 @@ def create_dirs() -> None:
     IMAGES_PATH.mkdir(exist_ok=True)
 
 
-def _get_screen_path(test_name: str) -> Path | None:
-    path = SCREENS_DIR / test_name / "actual"
-    if path.exists():
-        return path
-    path = SCREENS_DIR / test_name / "recorded"
-    if path.exists():
-        print(
-            f"WARNING: no actual screens for {test_name}, recording may be outdated: {path}"
-        )
-        return path
-    print(f"WARNING: missing screens for {test_name}. Did the test run?")
-    return None
-
-
 def create_reports() -> None:
-    removed_tests, added_tests, diff_tests = get_diff()
+    current = get_current_fixtures()
+    removed_tests, added_tests, diff_tests = get_diff(current, print_to_console=True)
 
     @contextmanager
     def tmpdir():
@@ -240,76 +123,12 @@ def create_reports() -> None:
             removed(temp_dir, test_name)
 
     for test_name, test_hash in added_tests.items():
-        screen_path = _get_screen_path(test_name)
+        screen_path = get_screen_path(test_name)
         if not screen_path:
             continue
         added(screen_path, test_name)
 
-    # Holding unique screen differences, connected with a certain testcase
+    generate_master_diff_report(diff_tests, MASTERDIFF_PATH)
-    # Used for diff report
-    unique_differing_screens: dict[tuple[str | None, str | None], str] = {}
-
-    for test_name, (master_hash, current_hash) in diff_tests.items():
-        with tmpdir() as master_root:
-            master_screens_path = master_root / "downloaded"
-            master_screens_path.mkdir()
-            try:
-                download.fetch_recorded(master_hash, master_screens_path)
-            except RuntimeError as e:
-                print("WARNING:", e)
-
-            current_screens_path = _get_screen_path(test_name)
-            if not current_screens_path:
-                current_screens_path = master_root / "empty_current_screens"
-                current_screens_path.mkdir()
-
-            # Saving all the images to a common directory
-            # They will be referenced from the HTML files
-            master_screens, master_hashes = screens_and_hashes(master_screens_path)
-            current_screens, current_hashes = screens_and_hashes(current_screens_path)
-            html.store_images(master_screens, master_hashes)
-            html.store_images(current_screens, current_hashes)
-
-            # List of tuples of master and current screens
-            # Useful for both testcase HTML report and the differing screen report
-            zipped_screens = list(screens_diff(master_hashes, current_hashes))
-
-            # Create testcase HTML report
-            create_testcase_html_diff_file(
-                zipped_screens,
-                test_name,
-                master_hash,
-                current_hash,
-            )
-
-            # Save differing screens for differing screens report
-            for master, current in zipped_screens:
-                if master != current:
-                    unique_differing_screens[(master, current)] = test_name
-
-    differing_screens_report(unique_differing_screens)
-
-
-def differing_screens_report(
-    unique_differing_screens: dict[tuple[str | None, str | None], str]
-) -> None:
-    doc = document(title="Master differing screens")
-    with doc:
-        with table(border=1, width=600):
-            with tr():
-                th("Expected")
-                th("Actual")
-                th("Testcase (link)")
-
-            for (master, current), testcase in unique_differing_screens.items():
-                with tr(bgcolor="red"):
-                    html.image_column(master, MASTERDIFF_PATH)
-                    html.image_column(current, MASTERDIFF_PATH)
-                    with td():
-                        with a(href=f"diff/{testcase}.html"):
-                            i(testcase)
-
-    html.write(MASTERDIFF_PATH, doc, "differing_screens.html")
 
 
 def main() -> None:

tests/ui_tests/reporting/testreport.py

@@ -10,50 +10,19 @@ import dominate.tags as t
 from dominate.tags import a, div, h1, h2, hr, i, p, span, strong, table, td, th, tr
 from dominate.util import text
 
-from ..common import UI_TESTS_DIR, TestCase, TestResult
+from ..common import FixturesType, TestCase, TestResult
 from . import download, html
+from .common import REPORTS_PATH, document, generate_master_diff_report, get_diff
 
-HERE = Path(__file__).resolve().parent
-REPORTS_PATH = UI_TESTS_DIR / "reports"
 TESTREPORT_PATH = REPORTS_PATH / "test"
 IMAGES_PATH = TESTREPORT_PATH / "images"
 SCREEN_TEXT_FILE = TESTREPORT_PATH / "screen_text.txt"
 
-STYLE = (HERE / "testreport.css").read_text()
-SCRIPT = (HERE / "testreport.js").read_text()
-GIF_SCRIPT = (HERE / "create-gif.js").read_text()
-
 # These two html files are referencing each other
 ALL_SCREENS = "all_screens.html"
 ALL_UNIQUE_SCREENS = "all_unique_screens.html"
 
 
-def document(
-    title: str,
-    actual_hash: str | None = None,
-    index: bool = False,
-    model: str | None = None,
-) -> dominate.document:
-    doc = dominate.document(title=title)
-    style = t.style()
-    style.add_raw_string(STYLE)
-    script = t.script()
-    script.add_raw_string(GIF_SCRIPT)
-    script.add_raw_string(SCRIPT)
-    doc.head.add(style, script)
-
-    if actual_hash is not None:
-        doc.body["data-actual-hash"] = actual_hash
-
-    if index:
-        doc.body["data-index"] = True
-
-    if model:
-        doc.body["class"] = f"model-{model}"
-
-    return doc
-
-
 def _header(test_name: str, expected_hash: str | None, actual_hash: str) -> None:
     h1(test_name)
     with div():
@@ -85,6 +54,7 @@ def setup(main_runner: bool) -> None:
         (TESTREPORT_PATH / "failed").mkdir()
         (TESTREPORT_PATH / "passed").mkdir()
         (TESTREPORT_PATH / "new").mkdir()
+        (TESTREPORT_PATH / "diff").mkdir()
         IMAGES_PATH.mkdir(parents=True)
 
     html.set_image_dir(IMAGES_PATH)
@@ -282,15 +252,60 @@ def differing_screens() -> None:
     html.write(TESTREPORT_PATH, doc, "differing_screens.html")
 
 
-def generate_reports(do_screen_text: bool = False) -> None:
+def _get_current_results() -> FixturesType:
+    current: FixturesType = {}  # type: ignore
+    for res in TestResult.recent_results():
+        model = res.test.model
+        group = res.test.group
+        fixtures_name = res.test.fixtures_name
+        actual_hash = res.actual_hash
+        if model not in current:
+            current[model] = {}
+        if group not in current[model]:
+            current[model][group] = {}
+        current[model][group][fixtures_name] = actual_hash
+    return current
+
+
+def master_diff() -> None:
+    """Creating an HTML page showing all screens differing from master."""
+    current = _get_current_results()
+    _removed_tests, _added_tests, diff_tests = get_diff(current)
+    generate_master_diff_report(diff_tests, TESTREPORT_PATH)
+
+
+def master_index() -> Path:
+    """Shows all the differing tests from master."""
+    diff = list((TESTREPORT_PATH / "diff").iterdir())
+
+    title = "UI changes from master"
+    doc = document(title=title)
+
+    with doc:
+        h1(title)
+        hr()
+
+        h2("Differs:", style="color: grey;")
+        i("UI fixtures that have been modified:")
+        html.report_links(diff, TESTREPORT_PATH)
+
+    return html.write(TESTREPORT_PATH, doc, "master_index.html")
+
+
+def generate_reports(
+    do_screen_text: bool = False, do_master_diff: bool = False
+) -> None:
     """Generate HTML reports for the test."""
     html.set_image_dir(IMAGES_PATH)
     index()
     all_screens()
     all_unique_screens()
+    differing_screens()
     if do_screen_text:
         screen_text_report()
-    differing_screens()
+    if do_master_diff:
+        master_diff()
+        master_index()
 
 
 def _copy_deduplicated(test: TestCase) -> None: