tests/ui: create report what differs from master

ci/posttest.yml

@@ -19,3 +19,19 @@ core unix coverage posttest:
     - core/.coverage.*
     - core/htmlcov
     expire_in: 1 week
+
+core unix ui changes:
+  stage: posttest
+  extends: .core_job
+  except:
+    - master
+  dependencies:
+    - core device ui test
+  script:
+    - cd tests/ui_tests
+    - pipenv run python reporting/report_master_diff.py
+  artifacts:
+    name: core-unix-ui-changes
+    paths:
+    - tests/ui_tests/reporting/reports/master_diff
+    expire_in: 1 week

ci/test.yml

@@ -25,7 +25,8 @@ core device ui test:
     name: core-device-ui-test
     paths:
       - ci/ui_test_records/
-      - tests/ui_tests/reports/
+      - tests/ui_tests/reporting/reports/test/
+      - tests/ui_tests/screens/
       - tests/junit.xml
       - tests/trezor.log
     when: always

tests/conftest.py

@@ -25,7 +25,7 @@ from trezorlib.transport import enumerate_devices, get_transport
 
 from . import ui_tests
 from .device_handler import BackgroundDeviceHandler
-from .ui_tests import report
+from .ui_tests.reporting import report_test
 
 
 def get_device():
@@ -149,7 +149,7 @@ def client(request):
 def pytest_sessionstart(session):
     ui_tests.read_fixtures()
     if session.config.getoption("ui") == "test":
-        report.clear_dir()
+        report_test.clear_dir()
 
 
 def _should_write_ui_report(exitstatus):
@@ -166,7 +166,7 @@ def pytest_sessionfinish(session, exitstatus):
     if session.config.getoption("ui") == "test":
         if session.config.getoption("ui_check_missing") and ui_tests.list_missing():
             session.exitstatus = pytest.ExitCode.TESTS_FAILED
-        report.index()
+        report_test.index()
     if session.config.getoption("ui") == "record":
         ui_tests.write_fixtures(session.config.getoption("ui_check_missing"))
 
@@ -191,7 +191,7 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
             println()
 
     if _should_write_ui_report(exitstatus):
-        println(f"UI tests summary: {report.REPORTS_PATH / 'index.html'}")
+        println(f"UI tests summary: {report_test.REPORTS_PATH / 'index.html'}")
 
 
 def pytest_addoption(parser):

tests/ui_tests/.gitignore

@@ -1,4 +1,3 @@
 *.png
 *.html
 *.zip
-reports/

tests/ui_tests/__init__.py

@@ -7,7 +7,7 @@ from pathlib import Path
 
 import pytest
 
-from . import report
+from .reporting import report_test
 
 UI_TESTS_DIR = Path(__file__).parent.resolve()
 HASH_FILE = UI_TESTS_DIR / "fixtures.json"
@@ -61,7 +61,7 @@ def _process_tested(fixture_test_path, test_name):
     _rename_records(actual_path)
 
     if actual_hash != expected_hash:
-        file_path = report.failed(
+        file_path = report_test.failed(
             fixture_test_path, test_name, actual_hash, expected_hash
         )
 
@@ -71,7 +71,7 @@ def _process_tested(fixture_test_path, test_name):
             )
         )
     else:
-        report.passed(fixture_test_path, test_name, actual_hash)
+        report_test.passed(fixture_test_path, test_name, actual_hash)
 
 
 @contextmanager

tests/ui_tests/download.py

@@ -1,24 +0,0 @@
-import urllib.error
-import urllib.request
-import zipfile
-
-RECORDS_WEBSITE = "https://firmware.corp.sldev.cz/ui_tests/"
-
-
-def fetch_recorded(recorded_hash, recorded_path):
-    zip_src = RECORDS_WEBSITE + recorded_hash + ".zip"
-    zip_dest = recorded_path / "recorded.zip"
-
-    try:
-        urllib.request.urlretrieve(zip_src, zip_dest)
-    except urllib.error.HTTPError:
-        raise RuntimeError("No such recorded collection was found on '%s'." % zip_src)
-    except urllib.error.URLError:
-        raise RuntimeError(
-            "Server firmware.corp.sldev.cz could not be found. Are you on VPN?"
-        )
-
-    with zipfile.ZipFile(zip_dest, "r") as z:
-        z.extractall(recorded_path)
-
-    zip_dest.unlink()

tests/ui_tests/reporting/download.py

@@ -0,0 +1,42 @@
+import json
+import pathlib
+import urllib.error
+import urllib.request
+import zipfile
+from typing import Dict
+
+import requests
+
+RECORDS_WEBSITE = "https://firmware.corp.sldev.cz/ui_tests/"
+FIXTURES_MASTER = "https://raw.githubusercontent.com/trezor/trezor-firmware/master/tests/ui_tests/fixtures.json"
+FIXTURES_CURRENT = pathlib.Path(__file__).parent / "../fixtures.json"
+
+
+def fetch_recorded(hash, path):
+    zip_src = RECORDS_WEBSITE + hash + ".zip"
+    zip_dest = path / "recorded.zip"
+
+    try:
+        urllib.request.urlretrieve(zip_src, zip_dest)
+    except urllib.error.HTTPError:
+        raise RuntimeError("No such recorded collection was found on '%s'." % zip_src)
+    except urllib.error.URLError:
+        raise RuntimeError(
+            "Server firmware.corp.sldev.cz could not be found. Are you on VPN?"
+        )
+
+    with zipfile.ZipFile(zip_dest, "r") as z:
+        z.extractall(path)
+
+    zip_dest.unlink()
+
+
+def fetch_fixtures_master() -> Dict[str, str]:
+    r = requests.get(FIXTURES_MASTER)
+    r.raise_for_status()
+    return r.json()
+
+
+def fetch_fixtures_current() -> Dict[str, str]:
+    with open(FIXTURES_CURRENT) as f:
+        return json.loads(f.read())

tests/ui_tests/reporting/html.py

@@ -0,0 +1,49 @@
+import base64
+import filecmp
+from itertools import zip_longest
+
+from dominate.tags import a, i, img, table, td, th, tr
+
+
+def report_links(tests, reports_path):
+    if not tests:
+        i("None!")
+        return
+    with table(border=1):
+        with tr():
+            th("Link to report")
+        for test in sorted(tests):
+            with tr():
+                path = test.relative_to(reports_path)
+                td(a(test.name, href=path))
+
+
+def write(fixture_test_path, doc, filename):
+    (fixture_test_path / filename).write_text(doc.render())
+    return fixture_test_path / filename
+
+
+def image(src):
+    with td():
+        if src:
+            # open image file
+            image = src.read_bytes()
+            # encode image as base64
+            image = base64.b64encode(image)
+            # convert output to str
+            image = image.decode()
+            # img(src=src.relative_to(fixture_test_path))
+            img(src="data:image/png;base64, " + image)
+        else:
+            i("missing")
+
+
+def diff_table(left_screens, right_screens):
+    for left, right in zip_longest(left_screens, right_screens):
+        if left and right and filecmp.cmp(right, left):
+            background = "white"
+        else:
+            background = "red"
+        with tr(bgcolor=background):
+            image(left)
+            image(right)

tests/ui_tests/reporting/report_master_diff.py

@@ -0,0 +1,184 @@
+import shutil
+import tempfile
+from pathlib import Path
+
+import dominate
+from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr
+
+# These are imported directly because this script is run directly, isort gets confused by that.
+import download  # isort:skip
+import html  # isort:skip
+
+REPORTS_PATH = Path(__file__).parent.resolve() / "reports" / "master_diff"
+RECORDED_SCREENS_PATH = Path(__file__).parent.parent.resolve() / "screens"
+
+
+def get_diff():
+    master = download.fetch_fixtures_master()
+    current = download.fetch_fixtures_current()
+
+    # removed items
+    removed = {test: master[test] for test in (master.keys() - current.keys())}
+    # added items
+    added = {test: current[test] for test in (current.keys() - master.keys())}
+    # items in both branches
+    same = master.items() - removed.items() - added.items()
+    # create the diff
+    diff = dict()
+    for master_test, master_hash in same:
+        if current.get(master_test) == master_hash:
+            continue
+        diff[master_test] = master[master_test], current[master_test]
+
+    return removed, added, diff
+
+
+def removed(screens_path, test_name):
+    doc = dominate.document(title=test_name)
+    screens = sorted(screens_path.iterdir())
+
+    with doc:
+        h1(test_name)
+        p(
+            "This UI test has been removed from fixtures.json.",
+            style="color: red; font-weight: bold;",
+        )
+        hr()
+
+        with table(border=1):
+            with tr():
+                th("Removed files")
+
+            for screen in screens:
+                with tr():
+                    html.image(screen)
+
+    return html.write(REPORTS_PATH / "removed", doc, test_name + ".html")
+
+
+def added(screens_path, test_name):
+    doc = dominate.document(title=test_name)
+    screens = sorted(screens_path.iterdir())
+
+    with doc:
+        h1(test_name)
+        p(
+            "This UI test has been added to fixtures.json.",
+            style="color: green; font-weight: bold;",
+        )
+        hr()
+
+        with table(border=1):
+            with tr():
+                th("Added files")
+
+            for screen in screens:
+                with tr():
+                    html.image(screen)
+
+    return html.write(REPORTS_PATH / "added", doc, test_name + ".html")
+
+
+def diff(
+    master_screens_path, current_screens_path, test_name, master_hash, current_hash
+):
+    doc = dominate.document(title=test_name)
+    master_screens = sorted(master_screens_path.iterdir())
+    current_screens = sorted(current_screens_path.iterdir())
+
+    with doc:
+        h1(test_name)
+        p("This UI test differs from master.", style="color: grey; font-weight: bold;")
+        with table():
+            with tr():
+                td("Master:")
+                td(master_hash, style="color: red;")
+            with tr():
+                td("Current:")
+                td(current_hash, style="color: green;")
+        hr()
+
+        with table(border=1, width=600):
+            with tr():
+                th("Master")
+                th("Current branch")
+
+            html.diff_table(master_screens, current_screens)
+
+    return html.write(REPORTS_PATH / "diff", doc, test_name + ".html")
+
+
+def index():
+    removed = list((REPORTS_PATH / "removed").iterdir())
+    added = list((REPORTS_PATH / "added").iterdir())
+    diff = list((REPORTS_PATH / "diff").iterdir())
+
+    title = "UI changes from master"
+    doc = dominate.document(title=title)
+
+    with doc:
+        h1("UI changes from master")
+        hr()
+
+        h2("Removed:", style="color: red;")
+        i("UI fixtures that have been removed:")
+        html.report_links(removed, REPORTS_PATH)
+        br()
+        hr()
+
+        h2("Added:", style="color: green;")
+        i("UI fixtures that have been added:")
+        html.report_links(added, REPORTS_PATH)
+        br()
+        hr()
+
+        h2("Differs:", style="color: grey;")
+        i("UI fixtures that have been modified:")
+        html.report_links(diff, REPORTS_PATH)
+
+    return html.write(REPORTS_PATH, doc, "index.html")
+
+
+def create_dirs():
+    # delete the reports dir to clear previous entries and create folders
+    shutil.rmtree(REPORTS_PATH, ignore_errors=True)
+    REPORTS_PATH.mkdir()
+    (REPORTS_PATH / "removed").mkdir()
+    (REPORTS_PATH / "added").mkdir()
+    (REPORTS_PATH / "diff").mkdir()
+
+
+def create_reports():
+    removed_tests, added_tests, diff_tests = get_diff()
+
+    with tempfile.TemporaryDirectory(prefix="trezor-records-") as temp_dir:
+        temp_dir = Path(temp_dir)
+
+        for test_name, test_hash in removed_tests.items():
+            download.fetch_recorded(test_hash, temp_dir)
+            removed(temp_dir, test_name)
+
+        for test_name, test_hash in added_tests.items():
+            path = RECORDED_SCREENS_PATH / test_name / "actual"
+            if not path.exists():
+                raise RuntimeError("Folder does not exist, has it been recorded?", path)
+            added(path, test_name)
+
+        for test_name, (master_hash, current_hash) in diff_tests.items():
+            master_screens = temp_dir
+            download.fetch_recorded(master_hash, master_screens)
+
+            current_screens = RECORDED_SCREENS_PATH / test_name / "actual"
+            if not current_screens.exists():
+                raise RuntimeError(
+                    "Folder does not exist, has it been recorded?", current_screens
+                )
+            diff(
+                master_screens, current_screens, test_name, master_hash, current_hash,
+            )
+
+
+if __name__ == "__main__":
+    create_dirs()
+    create_reports()
+    index()

tests/ui_tests/reporting/report_test.py

@@ -1,33 +1,15 @@
-import base64
-import filecmp
 import shutil
 from datetime import datetime
 from distutils.dir_util import copy_tree
-from itertools import zip_longest
 from pathlib import Path
 
 import dominate
-from dominate.tags import a, div, h1, h2, hr, i, img, p, strong, table, td, th, tr
+from dominate.tags import div, h1, h2, hr, p, strong, table, th, tr
 from dominate.util import text
 
-from . import download
+from . import download, html
 
-REPORTS_PATH = Path(__file__).parent.resolve() / "reports"
-
-
-def _image(src):
-    with td():
-        if src:
-            # open image file
-            image = src.read_bytes()
-            # encode image as base64
-            image = base64.b64encode(image)
-            # convert output to str
-            image = image.decode()
-            # img(src=src.relative_to(fixture_test_path))
-            img(src="data:image/png;base64, " + image)
-        else:
-            i("missing")
+REPORTS_PATH = Path(__file__).parent.resolve() / "reports" / "test"
 
 
 def _header(test_name, expected_hash, actual_hash):
@@ -48,24 +30,6 @@ def _header(test_name, expected_hash, actual_hash):
     hr()
 
 
-def _write(fixture_test_path, doc, filename):
-    (fixture_test_path / filename).write_text(doc.render())
-    return fixture_test_path / filename
-
-
-def _report_links(tests):
-    if not tests:
-        i("None!")
-        return
-    with table(border=1):
-        with tr():
-            th("Link to report")
-        for test in sorted(tests):
-            with tr():
-                path = test.relative_to(REPORTS_PATH)
-                td(a(test.name, href=path))
-
-
 def clear_dir():
     # delete and create the reports dir to clear previous entries
     shutil.rmtree(REPORTS_PATH, ignore_errors=True)
@@ -90,12 +54,12 @@ def index():
         hr()
 
         h2("Failed", style="color: red;")
-        _report_links(failed_tests)
+        html.report_links(failed_tests, REPORTS_PATH)
 
         h2("Passed", style="color: green;")
-        _report_links(passed_tests)
+        html.report_links(passed_tests, REPORTS_PATH)
 
-    return _write(REPORTS_PATH, doc, "index.html")
+    return html.write(REPORTS_PATH, doc, "index.html")
 
 
 def failed(fixture_test_path, test_name, actual_hash, expected_hash):
@@ -128,16 +92,9 @@ def failed(fixture_test_path, test_name, actual_hash, expected_hash):
                 th("Expected")
                 th("Actual")
 
-            for recorded, actual in zip_longest(recorded_screens, actual_screens):
-                if recorded and actual and filecmp.cmp(actual, recorded):
-                    background = "white"
-                else:
-                    background = "red"
-                with tr(bgcolor=background):
-                    _image(recorded)
-                    _image(actual)
+            html.diff_table(recorded_screens, actual_screens)
 
-    return _write(REPORTS_PATH / "failed", doc, test_name + ".html")
+    return html.write(REPORTS_PATH / "failed", doc, test_name + ".html")
 
 
 def passed(fixture_test_path, test_name, actual_hash):
@@ -156,6 +113,6 @@ def passed(fixture_test_path, test_name, actual_hash):
 
             for screen in actual_screens:
                 with tr():
-                    _image(screen)
+                    html.image(screen)
 
-    return _write(REPORTS_PATH / "passed", doc, test_name + ".html")
+    return html.write(REPORTS_PATH / "passed", doc, test_name + ".html")