diff --git a/ci/test.yml b/ci/test.yml index 0002e0f499..61be0c3c80 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -54,7 +54,8 @@ core unix device ui test: paths: - trezor.log - ci/ui_test_records/ - - tests/ui_tests/fixtures/*/diff.html + - tests/ui_tests/fixtures/*/failure_diff.html + - tests/ui_tests/fixtures/*/success.html - tests/junit.xml when: always expire_in: 1 week diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py index 05364ba31e..262139dfbb 100644 --- a/tests/ui_tests/__init__.py +++ b/tests/ui_tests/__init__.py @@ -2,12 +2,11 @@ import hashlib import re import shutil from contextlib import contextmanager -from distutils.dir_util import copy_tree from pathlib import Path import pytest -from . import html +from . import report def _get_test_dirname(node): @@ -67,21 +66,21 @@ def _process_tested(fixture_test_path, test_name): _rename_records(actual_path) if actual_hash != expected_hash: - diff_file = html.diff_file( + file_path = report.failure( fixture_test_path, test_name, actual_hash, expected_hash ) + if (fixture_test_path / "success.html").exists(): + (fixture_test_path / "success.html").unlink() pytest.fail( "Hash of {} differs.\nExpected: {}\nActual: {}\nDiff file: {}".format( - test_name, expected_hash, actual_hash, diff_file + test_name, expected_hash, actual_hash, file_path ) ) else: - copy_tree( - str(fixture_test_path / "actual"), str(fixture_test_path / "recorded") - ) - if (fixture_test_path / "diff.html").exists(): - (fixture_test_path / "diff.html").unlink() + report.success(fixture_test_path, test_name, actual_hash) + if (fixture_test_path / "failure_diff.html").exists(): + (fixture_test_path / "failure_diff.html").unlink() @contextmanager diff --git a/tests/ui_tests/html.py b/tests/ui_tests/html.py deleted file mode 100644 index 95c4ec5c37..0000000000 --- a/tests/ui_tests/html.py +++ /dev/null @@ -1,65 +0,0 @@ -import base64 -import filecmp -from itertools import zip_longest - -import dominate -from dominate.tags import div, h1, hr, i, img, p, table, td, th, tr - -from . import download - - -def _image(src): - with td(): - if src: - # open image file - image = open(src, "rb") - # encode image as base64 - image = base64.b64encode(image.read()) - # convert output to str - image = image.decode() - # img(src=src.relative_to(fixture_test_path)) - img(src="data:image/png;base64, " + image) - else: - i("missing") - - -def diff_file(fixture_test_path, test_name, actual_hash, expected_hash): - doc = dominate.document(title=test_name) - recorded_path = fixture_test_path / "recorded" - actual_path = fixture_test_path / "actual" - - if not recorded_path.exists(): - recorded_path.mkdir() - - download.fetch_recorded(expected_hash, recorded_path) - - recorded = sorted(recorded_path.iterdir()) - actual = sorted(actual_path.iterdir()) - - if not recorded: - return - - with doc: - h1(test_name) - with div(): - p("This test failed on UI comparison.") - p("Expected: ", expected_hash) - p("Actual: ", actual_hash) - hr() - - with table(border=1, width=600): - with tr(): - th("Expected") - th("Actual") - - for r, a in zip_longest(recorded, actual): - if r and a and filecmp.cmp(a, r): - background = "white" - else: - background = "red" - with tr(bgcolor=background): - _image(r) - _image(a) - - (fixture_test_path / "diff.html").write_text(doc.render()) - return fixture_test_path / "diff.html" diff --git a/tests/ui_tests/report.py b/tests/ui_tests/report.py new file mode 100644 index 0000000000..734f6a8221 --- /dev/null +++ b/tests/ui_tests/report.py @@ -0,0 +1,104 @@ +import base64 +import filecmp +from distutils.dir_util import copy_tree +from itertools import zip_longest + +import dominate +from dominate.tags import div, h1, hr, i, img, p, table, td, th, tr + +from . import download + + +def _image(src): + with td(): + if src: + # open image file + image = open(src, "rb") + # encode image as base64 + image = base64.b64encode(image.read()) + # convert output to str + image = image.decode() + # img(src=src.relative_to(fixture_test_path)) + img(src="data:image/png;base64, " + image) + else: + i("missing") + + +def _header(test_name, expected_hash, actual_hash): + h1(test_name) + with div(): + if actual_hash == expected_hash: + p( + "This test succeeded on UI comparison.", + style="color: green; font-weight: bold;", + ) + else: + p( + "This test failed on UI comparison.", + style="color: red; font-weight: bold;", + ) + p("Expected: ", expected_hash) + p("Actual: ", actual_hash) + hr() + + +def _write(fixture_test_path, doc, filename): + (fixture_test_path / filename).write_text(doc.render()) + return fixture_test_path / filename + + +def failure(fixture_test_path, test_name, actual_hash, expected_hash): + doc = dominate.document(title=test_name) + recorded_path = fixture_test_path / "recorded" + actual_path = fixture_test_path / "actual" + + if not recorded_path.exists(): + recorded_path.mkdir() + + download.fetch_recorded(expected_hash, recorded_path) + + recorded = sorted(recorded_path.iterdir()) + actual = sorted(actual_path.iterdir()) + + if not recorded: + return + + with doc: + _header(test_name, expected_hash, actual_hash) + + with table(border=1, width=600): + with tr(): + th("Expected") + th("Actual") + + for r, a in zip_longest(recorded, actual): + if r and a and filecmp.cmp(a, r): + background = "white" + else: + background = "red" + with tr(bgcolor=background): + _image(r) + _image(a) + + return _write(fixture_test_path, doc, "failure_diff.html") + + +def success(fixture_test_path, test_name, actual_hash): + copy_tree(str(fixture_test_path / "actual"), str(fixture_test_path / "recorded")) + + doc = dominate.document(title=test_name) + actual_path = fixture_test_path / "actual" + actual = sorted(actual_path.iterdir()) + + with doc: + _header(test_name, actual_hash, actual_hash) + + with table(border=1): + with tr(): + th("Recorded") + + for a in actual: + with tr(): + _image(a) + + return _write(fixture_test_path, doc, "success.html")