tests/ui: success html file

ci/test.yml

@@ -54,7 +54,8 @@ core unix device ui test:
     paths:
       - trezor.log
       - ci/ui_test_records/
-      - tests/ui_tests/fixtures/*/diff.html
+      - tests/ui_tests/fixtures/*/failure_diff.html
+      - tests/ui_tests/fixtures/*/success.html
       - tests/junit.xml
     when: always
     expire_in: 1 week

tests/ui_tests/__init__.py

@@ -2,12 +2,11 @@ import hashlib
 import re
 import shutil
 from contextlib import contextmanager
-from distutils.dir_util import copy_tree
 from pathlib import Path
 
 import pytest
 
-from . import html
+from . import report
 
 
 def _get_test_dirname(node):
@@ -67,21 +66,21 @@ def _process_tested(fixture_test_path, test_name):
     _rename_records(actual_path)
 
     if actual_hash != expected_hash:
-        diff_file = html.diff_file(
+        file_path = report.failure(
             fixture_test_path, test_name, actual_hash, expected_hash
         )
 
+        if (fixture_test_path / "success.html").exists():
+            (fixture_test_path / "success.html").unlink()
         pytest.fail(
             "Hash of {} differs.\nExpected:  {}\nActual:    {}\nDiff file: {}".format(
-                test_name, expected_hash, actual_hash, diff_file
+                test_name, expected_hash, actual_hash, file_path
             )
         )
     else:
-        copy_tree(
-            str(fixture_test_path / "actual"), str(fixture_test_path / "recorded")
-        )
-        if (fixture_test_path / "diff.html").exists():
-            (fixture_test_path / "diff.html").unlink()
+        report.success(fixture_test_path, test_name, actual_hash)
+        if (fixture_test_path / "failure_diff.html").exists():
+            (fixture_test_path / "failure_diff.html").unlink()
 
 
 @contextmanager

tests/ui_tests/html.py

@@ -1,65 +0,0 @@
-import base64
-import filecmp
-from itertools import zip_longest
-
-import dominate
-from dominate.tags import div, h1, hr, i, img, p, table, td, th, tr
-
-from . import download
-
-
-def _image(src):
-    with td():
-        if src:
-            # open image file
-            image = open(src, "rb")
-            # encode image as base64
-            image = base64.b64encode(image.read())
-            # convert output to str
-            image = image.decode()
-            # img(src=src.relative_to(fixture_test_path))
-            img(src="data:image/png;base64, " + image)
-        else:
-            i("missing")
-
-
-def diff_file(fixture_test_path, test_name, actual_hash, expected_hash):
-    doc = dominate.document(title=test_name)
-    recorded_path = fixture_test_path / "recorded"
-    actual_path = fixture_test_path / "actual"
-
-    if not recorded_path.exists():
-        recorded_path.mkdir()
-
-    download.fetch_recorded(expected_hash, recorded_path)
-
-    recorded = sorted(recorded_path.iterdir())
-    actual = sorted(actual_path.iterdir())
-
-    if not recorded:
-        return
-
-    with doc:
-        h1(test_name)
-        with div():
-            p("This test failed on UI comparison.")
-            p("Expected: ", expected_hash)
-            p("Actual: ", actual_hash)
-        hr()
-
-        with table(border=1, width=600):
-            with tr():
-                th("Expected")
-                th("Actual")
-
-            for r, a in zip_longest(recorded, actual):
-                if r and a and filecmp.cmp(a, r):
-                    background = "white"
-                else:
-                    background = "red"
-                with tr(bgcolor=background):
-                    _image(r)
-                    _image(a)
-
-    (fixture_test_path / "diff.html").write_text(doc.render())
-    return fixture_test_path / "diff.html"

tests/ui_tests/report.py

@@ -0,0 +1,104 @@
+import base64
+import filecmp
+from distutils.dir_util import copy_tree
+from itertools import zip_longest
+
+import dominate
+from dominate.tags import div, h1, hr, i, img, p, table, td, th, tr
+
+from . import download
+
+
+def _image(src):
+    with td():
+        if src:
+            # open image file
+            image = open(src, "rb")
+            # encode image as base64
+            image = base64.b64encode(image.read())
+            # convert output to str
+            image = image.decode()
+            # img(src=src.relative_to(fixture_test_path))
+            img(src="data:image/png;base64, " + image)
+        else:
+            i("missing")
+
+
+def _header(test_name, expected_hash, actual_hash):
+    h1(test_name)
+    with div():
+        if actual_hash == expected_hash:
+            p(
+                "This test succeeded on UI comparison.",
+                style="color: green; font-weight: bold;",
+            )
+        else:
+            p(
+                "This test failed on UI comparison.",
+                style="color: red; font-weight: bold;",
+            )
+        p("Expected: ", expected_hash)
+        p("Actual: ", actual_hash)
+    hr()
+
+
+def _write(fixture_test_path, doc, filename):
+    (fixture_test_path / filename).write_text(doc.render())
+    return fixture_test_path / filename
+
+
+def failure(fixture_test_path, test_name, actual_hash, expected_hash):
+    doc = dominate.document(title=test_name)
+    recorded_path = fixture_test_path / "recorded"
+    actual_path = fixture_test_path / "actual"
+
+    if not recorded_path.exists():
+        recorded_path.mkdir()
+
+    download.fetch_recorded(expected_hash, recorded_path)
+
+    recorded = sorted(recorded_path.iterdir())
+    actual = sorted(actual_path.iterdir())
+
+    if not recorded:
+        return
+
+    with doc:
+        _header(test_name, expected_hash, actual_hash)
+
+        with table(border=1, width=600):
+            with tr():
+                th("Expected")
+                th("Actual")
+
+            for r, a in zip_longest(recorded, actual):
+                if r and a and filecmp.cmp(a, r):
+                    background = "white"
+                else:
+                    background = "red"
+                with tr(bgcolor=background):
+                    _image(r)
+                    _image(a)
+
+    return _write(fixture_test_path, doc, "failure_diff.html")
+
+
+def success(fixture_test_path, test_name, actual_hash):
+    copy_tree(str(fixture_test_path / "actual"), str(fixture_test_path / "recorded"))
+
+    doc = dominate.document(title=test_name)
+    actual_path = fixture_test_path / "actual"
+    actual = sorted(actual_path.iterdir())
+
+    with doc:
+        _header(test_name, actual_hash, actual_hash)
+
+        with table(border=1):
+            with tr():
+                th("Recorded")
+
+            for a in actual:
+                with tr():
+                    _image(a)
+
+    return _write(fixture_test_path, doc, "success.html")