diff --git a/ci/test.yml b/ci/test.yml index 05220b62f2..be3711c913 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -16,12 +16,14 @@ core device ui test: after_script: - mv tests/ui_tests/reporting/reports/test/ test_ui_report - nix-shell --run "pipenv run python ci/prepare_ui_artifacts.py" + - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: - ci/ui_test_records/ - test_ui_report - tests/ui_tests/screens/ + - tests/ui_tests/fixtures.suggestion.json - tests/junit.xml - tests/trezor.log when: always diff --git a/tests/conftest.py b/tests/conftest.py index fde10e1daf..ca9f2f1f5c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -163,12 +163,14 @@ def pytest_sessionfinish(session, exitstatus): if not _should_write_ui_report(exitstatus): return + missing = session.config.getoption("ui_check_missing") if session.config.getoption("ui") == "test": - if session.config.getoption("ui_check_missing") and ui_tests.list_missing(): + if missing and ui_tests.list_missing(): session.exitstatus = pytest.ExitCode.TESTS_FAILED + ui_tests.write_fixtures_suggestion(missing) testreport.index() if session.config.getoption("ui") == "record": - ui_tests.write_fixtures(session.config.getoption("ui_check_missing")) + ui_tests.write_fixtures(missing) def pytest_terminal_summary(terminalreporter, exitstatus, config): @@ -180,7 +182,7 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): if ui_option and _should_write_ui_report(exitstatus) and missing_tests: println(f"{len(missing_tests)} expected UI tests did not run.") if config.getoption("ui_check_missing"): - println("List of missing tests follows:") + println("-------- List of missing tests follows: --------") for test in missing_tests: println("\t" + test) @@ -190,8 +192,15 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config): println("Removing missing tests from record.") println("") + if ui_option == "test" and _should_write_ui_report(exitstatus): + println("\n-------- Suggested fixtures.json diff: --------") + print("See", ui_tests.SUGGESTION_FILE) + println("") + if _should_write_ui_report(exitstatus): - println(f"UI tests summary: {testreport.REPORTS_PATH / 'index.html'}") + println("-------- UI tests summary: --------") + println(f"{testreport.REPORTS_PATH / 'index.html'}") + println("") def pytest_addoption(parser): diff --git a/tests/ui_tests/.gitignore b/tests/ui_tests/.gitignore index 58760efc84..047e341006 100644 --- a/tests/ui_tests/.gitignore +++ b/tests/ui_tests/.gitignore @@ -1,3 +1,4 @@ *.png *.html *.zip +fixtures.suggestion.json diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py index d4ec439747..b4c2e5770e 100644 --- a/tests/ui_tests/__init__.py +++ b/tests/ui_tests/__init__.py @@ -11,7 +11,9 @@ from .reporting import testreport UI_TESTS_DIR = Path(__file__).parent.resolve() HASH_FILE = UI_TESTS_DIR / "fixtures.json" -HASHES = {} +SUGGESTION_FILE = UI_TESTS_DIR / "fixtures.suggestion.json" +FILE_HASHES = {} +ACTUAL_HASHES = {} PROCESSED = set() @@ -29,7 +31,7 @@ def get_test_name(node_id): def _process_recorded(screen_path, test_name): # calculate hash - HASHES[test_name] = _hash_files(screen_path) + FILE_HASHES[test_name] = _hash_files(screen_path) _rename_records(screen_path) PROCESSED.add(test_name) @@ -50,13 +52,14 @@ def _hash_files(path): def _process_tested(fixture_test_path, test_name): - expected_hash = HASHES.get(test_name) + expected_hash = FILE_HASHES.get(test_name) if expected_hash is None: raise ValueError("Hash for '%s' not found in fixtures.json" % test_name) PROCESSED.add(test_name) actual_path = fixture_test_path / "actual" actual_hash = _hash_files(actual_path) + ACTUAL_HASHES[test_name] = actual_hash _rename_records(actual_path) @@ -103,20 +106,28 @@ def screen_recording(client, request): def list_missing(): - return set(HASHES.keys()) - PROCESSED + return set(FILE_HASHES.keys()) - PROCESSED def read_fixtures(): if not HASH_FILE.exists(): raise ValueError("File fixtures.json not found.") - global HASHES - HASHES = json.loads(HASH_FILE.read_text()) + global FILE_HASHES + FILE_HASHES = json.loads(HASH_FILE.read_text()) def write_fixtures(remove_missing: bool): - if remove_missing: - write = {i: HASHES[i] for i in PROCESSED} - else: - write = HASHES + HASH_FILE.write_text(_get_fixtures_content(FILE_HASHES, remove_missing)) - HASH_FILE.write_text(json.dumps(write, indent="", sort_keys=True) + "\n") + +def write_fixtures_suggestion(remove_missing: bool): + SUGGESTION_FILE.write_text(_get_fixtures_content(ACTUAL_HASHES, remove_missing)) + + +def _get_fixtures_content(fixtures: dict, remove_missing: bool): + if remove_missing: + fixtures = {i: fixtures[i] for i in PROCESSED} + else: + fixtures = fixtures + + return json.dumps(fixtures, indent="", sort_keys=True) + "\n"