From 89929c7a8c0724e816cb4cea6bbb164545b75e1a Mon Sep 17 00:00:00 2001 From: Tomas Susanka Date: Tue, 3 Mar 2020 14:50:57 +0000 Subject: [PATCH] tests/ui: create report what differs from master --- ci/posttest.yml | 16 ++ ci/test.yml | 3 +- tests/conftest.py | 8 +- tests/ui_tests/.gitignore | 1 - tests/ui_tests/__init__.py | 6 +- tests/ui_tests/download.py | 24 --- tests/ui_tests/reporting/download.py | 42 ++++ tests/ui_tests/reporting/html.py | 49 +++++ .../ui_tests/reporting/report_master_diff.py | 184 ++++++++++++++++++ .../{report.py => reporting/report_test.py} | 63 +----- tests/ui_tests/reporting/reports/.keep | 0 11 files changed, 310 insertions(+), 86 deletions(-) delete mode 100644 tests/ui_tests/download.py create mode 100644 tests/ui_tests/reporting/download.py create mode 100644 tests/ui_tests/reporting/html.py create mode 100644 tests/ui_tests/reporting/report_master_diff.py rename tests/ui_tests/{report.py => reporting/report_test.py} (62%) create mode 100644 tests/ui_tests/reporting/reports/.keep diff --git a/ci/posttest.yml b/ci/posttest.yml index 3404c237d..7e3456e2e 100644 --- a/ci/posttest.yml +++ b/ci/posttest.yml @@ -19,3 +19,19 @@ core unix coverage posttest: - core/.coverage.* - core/htmlcov expire_in: 1 week + +core unix ui changes: + stage: posttest + extends: .core_job + except: + - master + dependencies: + - core device ui test + script: + - cd tests/ui_tests + - pipenv run python reporting/report_master_diff.py + artifacts: + name: core-unix-ui-changes + paths: + - tests/ui_tests/reporting/reports/master_diff + expire_in: 1 week diff --git a/ci/test.yml b/ci/test.yml index 32dacba54..1bdd97ac0 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -25,7 +25,8 @@ core device ui test: name: core-device-ui-test paths: - ci/ui_test_records/ - - tests/ui_tests/reports/ + - tests/ui_tests/reporting/reports/test/ + - tests/ui_tests/screens/ - tests/junit.xml - tests/trezor.log when: always diff --git a/tests/conftest.py b/tests/conftest.py index 46b4bf9d3..86b36402a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,7 +25,7 @@ from trezorlib.transport import enumerate_devices, get_transport from . import ui_tests from .device_handler import BackgroundDeviceHandler -from .ui_tests import report +from .ui_tests.reporting import report_test def get_device(): @@ -149,7 +149,7 @@ def client(request): def pytest_sessionstart(session): ui_tests.read_fixtures() if session.config.getoption("ui") == "test": - report.clear_dir() + report_test.clear_dir() def _should_write_ui_report(exitstatus): @@ -166,7 +166,7 @@ def pytest_sessionfinish(session, exitstatus): if session.config.getoption("ui") == "test": if session.config.getoption("ui_check_missing") and ui_tests.list_missing(): session.exitstatus = pytest.ExitCode.TESTS_FAILED - report.index() + report_test.index() if session.config.getoption("ui") == "record": ui_tests.write_fixtures(session.config.getoption("ui_check_missing")) @@ -191,7 +191,7 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): println() if _should_write_ui_report(exitstatus): - println(f"UI tests summary: {report.REPORTS_PATH / 'index.html'}") + println(f"UI tests summary: {report_test.REPORTS_PATH / 'index.html'}") def pytest_addoption(parser): diff --git a/tests/ui_tests/.gitignore b/tests/ui_tests/.gitignore index 004c32299..58760efc8 100644 --- a/tests/ui_tests/.gitignore +++ b/tests/ui_tests/.gitignore @@ -1,4 +1,3 @@ *.png *.html *.zip -reports/ diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py index 59b5c0063..bbe4f2394 100644 --- a/tests/ui_tests/__init__.py +++ b/tests/ui_tests/__init__.py @@ -7,7 +7,7 @@ from pathlib import Path import pytest -from . import report +from .reporting import report_test UI_TESTS_DIR = Path(__file__).parent.resolve() HASH_FILE = UI_TESTS_DIR / "fixtures.json" @@ -61,7 +61,7 @@ def _process_tested(fixture_test_path, test_name): _rename_records(actual_path) if actual_hash != expected_hash: - file_path = report.failed( + file_path = report_test.failed( fixture_test_path, test_name, actual_hash, expected_hash ) @@ -71,7 +71,7 @@ def _process_tested(fixture_test_path, test_name): ) ) else: - report.passed(fixture_test_path, test_name, actual_hash) + report_test.passed(fixture_test_path, test_name, actual_hash) @contextmanager diff --git a/tests/ui_tests/download.py b/tests/ui_tests/download.py deleted file mode 100644 index f61856cfb..000000000 --- a/tests/ui_tests/download.py +++ /dev/null @@ -1,24 +0,0 @@ -import urllib.error -import urllib.request -import zipfile - -RECORDS_WEBSITE = "https://firmware.corp.sldev.cz/ui_tests/" - - -def fetch_recorded(recorded_hash, recorded_path): - zip_src = RECORDS_WEBSITE + recorded_hash + ".zip" - zip_dest = recorded_path / "recorded.zip" - - try: - urllib.request.urlretrieve(zip_src, zip_dest) - except urllib.error.HTTPError: - raise RuntimeError("No such recorded collection was found on '%s'." % zip_src) - except urllib.error.URLError: - raise RuntimeError( - "Server firmware.corp.sldev.cz could not be found. Are you on VPN?" - ) - - with zipfile.ZipFile(zip_dest, "r") as z: - z.extractall(recorded_path) - - zip_dest.unlink() diff --git a/tests/ui_tests/reporting/download.py b/tests/ui_tests/reporting/download.py new file mode 100644 index 000000000..d8f92da23 --- /dev/null +++ b/tests/ui_tests/reporting/download.py @@ -0,0 +1,42 @@ +import json +import pathlib +import urllib.error +import urllib.request +import zipfile +from typing import Dict + +import requests + +RECORDS_WEBSITE = "https://firmware.corp.sldev.cz/ui_tests/" +FIXTURES_MASTER = "https://raw.githubusercontent.com/trezor/trezor-firmware/master/tests/ui_tests/fixtures.json" +FIXTURES_CURRENT = pathlib.Path(__file__).parent / "../fixtures.json" + + +def fetch_recorded(hash, path): + zip_src = RECORDS_WEBSITE + hash + ".zip" + zip_dest = path / "recorded.zip" + + try: + urllib.request.urlretrieve(zip_src, zip_dest) + except urllib.error.HTTPError: + raise RuntimeError("No such recorded collection was found on '%s'." % zip_src) + except urllib.error.URLError: + raise RuntimeError( + "Server firmware.corp.sldev.cz could not be found. Are you on VPN?" + ) + + with zipfile.ZipFile(zip_dest, "r") as z: + z.extractall(path) + + zip_dest.unlink() + + +def fetch_fixtures_master() -> Dict[str, str]: + r = requests.get(FIXTURES_MASTER) + r.raise_for_status() + return r.json() + + +def fetch_fixtures_current() -> Dict[str, str]: + with open(FIXTURES_CURRENT) as f: + return json.loads(f.read()) diff --git a/tests/ui_tests/reporting/html.py b/tests/ui_tests/reporting/html.py new file mode 100644 index 000000000..8f793ef70 --- /dev/null +++ b/tests/ui_tests/reporting/html.py @@ -0,0 +1,49 @@ +import base64 +import filecmp +from itertools import zip_longest + +from dominate.tags import a, i, img, table, td, th, tr + + +def report_links(tests, reports_path): + if not tests: + i("None!") + return + with table(border=1): + with tr(): + th("Link to report") + for test in sorted(tests): + with tr(): + path = test.relative_to(reports_path) + td(a(test.name, href=path)) + + +def write(fixture_test_path, doc, filename): + (fixture_test_path / filename).write_text(doc.render()) + return fixture_test_path / filename + + +def image(src): + with td(): + if src: + # open image file + image = src.read_bytes() + # encode image as base64 + image = base64.b64encode(image) + # convert output to str + image = image.decode() + # img(src=src.relative_to(fixture_test_path)) + img(src="data:image/png;base64, " + image) + else: + i("missing") + + +def diff_table(left_screens, right_screens): + for left, right in zip_longest(left_screens, right_screens): + if left and right and filecmp.cmp(right, left): + background = "white" + else: + background = "red" + with tr(bgcolor=background): + image(left) + image(right) diff --git a/tests/ui_tests/reporting/report_master_diff.py b/tests/ui_tests/reporting/report_master_diff.py new file mode 100644 index 000000000..09f7abf64 --- /dev/null +++ b/tests/ui_tests/reporting/report_master_diff.py @@ -0,0 +1,184 @@ +import shutil +import tempfile +from pathlib import Path + +import dominate +from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr + +# These are imported directly because this script is run directly, isort gets confused by that. +import download # isort:skip +import html # isort:skip + +REPORTS_PATH = Path(__file__).parent.resolve() / "reports" / "master_diff" +RECORDED_SCREENS_PATH = Path(__file__).parent.parent.resolve() / "screens" + + +def get_diff(): + master = download.fetch_fixtures_master() + current = download.fetch_fixtures_current() + + # removed items + removed = {test: master[test] for test in (master.keys() - current.keys())} + # added items + added = {test: current[test] for test in (current.keys() - master.keys())} + # items in both branches + same = master.items() - removed.items() - added.items() + # create the diff + diff = dict() + for master_test, master_hash in same: + if current.get(master_test) == master_hash: + continue + diff[master_test] = master[master_test], current[master_test] + + return removed, added, diff + + +def removed(screens_path, test_name): + doc = dominate.document(title=test_name) + screens = sorted(screens_path.iterdir()) + + with doc: + h1(test_name) + p( + "This UI test has been removed from fixtures.json.", + style="color: red; font-weight: bold;", + ) + hr() + + with table(border=1): + with tr(): + th("Removed files") + + for screen in screens: + with tr(): + html.image(screen) + + return html.write(REPORTS_PATH / "removed", doc, test_name + ".html") + + +def added(screens_path, test_name): + doc = dominate.document(title=test_name) + screens = sorted(screens_path.iterdir()) + + with doc: + h1(test_name) + p( + "This UI test has been added to fixtures.json.", + style="color: green; font-weight: bold;", + ) + hr() + + with table(border=1): + with tr(): + th("Added files") + + for screen in screens: + with tr(): + html.image(screen) + + return html.write(REPORTS_PATH / "added", doc, test_name + ".html") + + +def diff( + master_screens_path, current_screens_path, test_name, master_hash, current_hash +): + doc = dominate.document(title=test_name) + master_screens = sorted(master_screens_path.iterdir()) + current_screens = sorted(current_screens_path.iterdir()) + + with doc: + h1(test_name) + p("This UI test differs from master.", style="color: grey; font-weight: bold;") + with table(): + with tr(): + td("Master:") + td(master_hash, style="color: red;") + with tr(): + td("Current:") + td(current_hash, style="color: green;") + hr() + + with table(border=1, width=600): + with tr(): + th("Master") + th("Current branch") + + html.diff_table(master_screens, current_screens) + + return html.write(REPORTS_PATH / "diff", doc, test_name + ".html") + + +def index(): + removed = list((REPORTS_PATH / "removed").iterdir()) + added = list((REPORTS_PATH / "added").iterdir()) + diff = list((REPORTS_PATH / "diff").iterdir()) + + title = "UI changes from master" + doc = dominate.document(title=title) + + with doc: + h1("UI changes from master") + hr() + + h2("Removed:", style="color: red;") + i("UI fixtures that have been removed:") + html.report_links(removed, REPORTS_PATH) + br() + hr() + + h2("Added:", style="color: green;") + i("UI fixtures that have been added:") + html.report_links(added, REPORTS_PATH) + br() + hr() + + h2("Differs:", style="color: grey;") + i("UI fixtures that have been modified:") + html.report_links(diff, REPORTS_PATH) + + return html.write(REPORTS_PATH, doc, "index.html") + + +def create_dirs(): + # delete the reports dir to clear previous entries and create folders + shutil.rmtree(REPORTS_PATH, ignore_errors=True) + REPORTS_PATH.mkdir() + (REPORTS_PATH / "removed").mkdir() + (REPORTS_PATH / "added").mkdir() + (REPORTS_PATH / "diff").mkdir() + + +def create_reports(): + removed_tests, added_tests, diff_tests = get_diff() + + with tempfile.TemporaryDirectory(prefix="trezor-records-") as temp_dir: + temp_dir = Path(temp_dir) + + for test_name, test_hash in removed_tests.items(): + download.fetch_recorded(test_hash, temp_dir) + removed(temp_dir, test_name) + + for test_name, test_hash in added_tests.items(): + path = RECORDED_SCREENS_PATH / test_name / "actual" + if not path.exists(): + raise RuntimeError("Folder does not exist, has it been recorded?", path) + added(path, test_name) + + for test_name, (master_hash, current_hash) in diff_tests.items(): + master_screens = temp_dir + download.fetch_recorded(master_hash, master_screens) + + current_screens = RECORDED_SCREENS_PATH / test_name / "actual" + if not current_screens.exists(): + raise RuntimeError( + "Folder does not exist, has it been recorded?", current_screens + ) + diff( + master_screens, current_screens, test_name, master_hash, current_hash, + ) + + +if __name__ == "__main__": + create_dirs() + create_reports() + index() diff --git a/tests/ui_tests/report.py b/tests/ui_tests/reporting/report_test.py similarity index 62% rename from tests/ui_tests/report.py rename to tests/ui_tests/reporting/report_test.py index 5d16f86ec..f57027b61 100644 --- a/tests/ui_tests/report.py +++ b/tests/ui_tests/reporting/report_test.py @@ -1,33 +1,15 @@ -import base64 -import filecmp import shutil from datetime import datetime from distutils.dir_util import copy_tree -from itertools import zip_longest from pathlib import Path import dominate -from dominate.tags import a, div, h1, h2, hr, i, img, p, strong, table, td, th, tr +from dominate.tags import div, h1, h2, hr, p, strong, table, th, tr from dominate.util import text -from . import download +from . import download, html -REPORTS_PATH = Path(__file__).parent.resolve() / "reports" - - -def _image(src): - with td(): - if src: - # open image file - image = src.read_bytes() - # encode image as base64 - image = base64.b64encode(image) - # convert output to str - image = image.decode() - # img(src=src.relative_to(fixture_test_path)) - img(src="data:image/png;base64, " + image) - else: - i("missing") +REPORTS_PATH = Path(__file__).parent.resolve() / "reports" / "test" def _header(test_name, expected_hash, actual_hash): @@ -48,24 +30,6 @@ def _header(test_name, expected_hash, actual_hash): hr() -def _write(fixture_test_path, doc, filename): - (fixture_test_path / filename).write_text(doc.render()) - return fixture_test_path / filename - - -def _report_links(tests): - if not tests: - i("None!") - return - with table(border=1): - with tr(): - th("Link to report") - for test in sorted(tests): - with tr(): - path = test.relative_to(REPORTS_PATH) - td(a(test.name, href=path)) - - def clear_dir(): # delete and create the reports dir to clear previous entries shutil.rmtree(REPORTS_PATH, ignore_errors=True) @@ -90,12 +54,12 @@ def index(): hr() h2("Failed", style="color: red;") - _report_links(failed_tests) + html.report_links(failed_tests, REPORTS_PATH) h2("Passed", style="color: green;") - _report_links(passed_tests) + html.report_links(passed_tests, REPORTS_PATH) - return _write(REPORTS_PATH, doc, "index.html") + return html.write(REPORTS_PATH, doc, "index.html") def failed(fixture_test_path, test_name, actual_hash, expected_hash): @@ -128,16 +92,9 @@ def failed(fixture_test_path, test_name, actual_hash, expected_hash): th("Expected") th("Actual") - for recorded, actual in zip_longest(recorded_screens, actual_screens): - if recorded and actual and filecmp.cmp(actual, recorded): - background = "white" - else: - background = "red" - with tr(bgcolor=background): - _image(recorded) - _image(actual) + html.diff_table(recorded_screens, actual_screens) - return _write(REPORTS_PATH / "failed", doc, test_name + ".html") + return html.write(REPORTS_PATH / "failed", doc, test_name + ".html") def passed(fixture_test_path, test_name, actual_hash): @@ -156,6 +113,6 @@ def passed(fixture_test_path, test_name, actual_hash): for screen in actual_screens: with tr(): - _image(screen) + html.image(screen) - return _write(REPORTS_PATH / "passed", doc, test_name + ".html") + return html.write(REPORTS_PATH / "passed", doc, test_name + ".html") diff --git a/tests/ui_tests/reporting/reports/.keep b/tests/ui_tests/reporting/reports/.keep new file mode 100644 index 000000000..e69de29bb