test(core): print fixtures suggestion after the test

ci/test.yml

@@ -16,12 +16,14 @@ core device ui test:
   after_script:
     - mv tests/ui_tests/reporting/reports/test/ test_ui_report
     - nix-shell --run "pipenv run python ci/prepare_ui_artifacts.py"
+    - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
       - ci/ui_test_records/
       - test_ui_report
       - tests/ui_tests/screens/
+      - tests/ui_tests/fixtures.suggestion.json
       - tests/junit.xml
       - tests/trezor.log
     when: always

tests/conftest.py

@@ -163,12 +163,14 @@ def pytest_sessionfinish(session, exitstatus):
     if not _should_write_ui_report(exitstatus):
         return
 
+    missing = session.config.getoption("ui_check_missing")
     if session.config.getoption("ui") == "test":
-        if session.config.getoption("ui_check_missing") and ui_tests.list_missing():
+        if missing and ui_tests.list_missing():
             session.exitstatus = pytest.ExitCode.TESTS_FAILED
+        ui_tests.write_fixtures_suggestion(missing)
         testreport.index()
     if session.config.getoption("ui") == "record":
-        ui_tests.write_fixtures(session.config.getoption("ui_check_missing"))
+        ui_tests.write_fixtures(missing)
 
 
 def pytest_terminal_summary(terminalreporter, exitstatus, config):
@@ -180,7 +182,7 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
     if ui_option and _should_write_ui_report(exitstatus) and missing_tests:
         println(f"{len(missing_tests)} expected UI tests did not run.")
         if config.getoption("ui_check_missing"):
-            println("List of missing tests follows:")
+            println("-------- List of missing tests follows: --------")
             for test in missing_tests:
                 println("\t" + test)
 
@@ -190,8 +192,15 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config):
                 println("Removing missing tests from record.")
             println("")
 
+    if ui_option == "test" and _should_write_ui_report(exitstatus):
+        println("\n-------- Suggested fixtures.json diff: --------")
+        print("See", ui_tests.SUGGESTION_FILE)
+        println("")
+
     if _should_write_ui_report(exitstatus):
-        println(f"UI tests summary: {testreport.REPORTS_PATH / 'index.html'}")
+        println("-------- UI tests summary: --------")
+        println(f"{testreport.REPORTS_PATH / 'index.html'}")
+        println("")
 
 
 def pytest_addoption(parser):

tests/ui_tests/.gitignore

@@ -1,3 +1,4 @@
 *.png
 *.html
 *.zip
+fixtures.suggestion.json

tests/ui_tests/__init__.py

@@ -11,7 +11,9 @@ from .reporting import testreport
 
 UI_TESTS_DIR = Path(__file__).parent.resolve()
 HASH_FILE = UI_TESTS_DIR / "fixtures.json"
-HASHES = {}
+SUGGESTION_FILE = UI_TESTS_DIR / "fixtures.suggestion.json"
+FILE_HASHES = {}
+ACTUAL_HASHES = {}
 PROCESSED = set()
 
 
@@ -29,7 +31,7 @@ def get_test_name(node_id):
 
 def _process_recorded(screen_path, test_name):
     # calculate hash
-    HASHES[test_name] = _hash_files(screen_path)
+    FILE_HASHES[test_name] = _hash_files(screen_path)
     _rename_records(screen_path)
     PROCESSED.add(test_name)
 
@@ -50,13 +52,14 @@ def _hash_files(path):
 
 
 def _process_tested(fixture_test_path, test_name):
-    expected_hash = HASHES.get(test_name)
+    expected_hash = FILE_HASHES.get(test_name)
     if expected_hash is None:
         raise ValueError("Hash for '%s' not found in fixtures.json" % test_name)
     PROCESSED.add(test_name)
 
     actual_path = fixture_test_path / "actual"
     actual_hash = _hash_files(actual_path)
+    ACTUAL_HASHES[test_name] = actual_hash
 
     _rename_records(actual_path)
 
@@ -103,20 +106,28 @@ def screen_recording(client, request):
 
 
 def list_missing():
-    return set(HASHES.keys()) - PROCESSED
+    return set(FILE_HASHES.keys()) - PROCESSED
 
 
 def read_fixtures():
     if not HASH_FILE.exists():
         raise ValueError("File fixtures.json not found.")
-    global HASHES
-    HASHES = json.loads(HASH_FILE.read_text())
+    global FILE_HASHES
+    FILE_HASHES = json.loads(HASH_FILE.read_text())
 
 
 def write_fixtures(remove_missing: bool):
+    HASH_FILE.write_text(_get_fixtures_content(FILE_HASHES, remove_missing))
+
+
+def write_fixtures_suggestion(remove_missing: bool):
+    SUGGESTION_FILE.write_text(_get_fixtures_content(ACTUAL_HASHES, remove_missing))
+
+
+def _get_fixtures_content(fixtures: dict, remove_missing: bool):
     if remove_missing:
-        write = {i: HASHES[i] for i in PROCESSED}
+        fixtures = {i: fixtures[i] for i in PROCESSED}
     else:
-        write = HASHES
+        fixtures = fixtures
 
-    HASH_FILE.write_text(json.dumps(write, indent="", sort_keys=True) + "\n")
+    return json.dumps(fixtures, indent="", sort_keys=True) + "\n"