From 821f8aad8f6dae47c32906c2e8b0f87da894a101 Mon Sep 17 00:00:00 2001 From: grdddj Date: Fri, 14 Jul 2023 10:31:39 +0200 Subject: [PATCH] feat(tests): saving fixtures.results.json file with result hashes after each UI test [no changelog] --- ci/test.yml | 6 ++++ docs/ci/jobs.md | 64 +++++++++++++++++++------------------- tests/show_results.py | 4 +-- tests/ui_tests/.gitignore | 1 + tests/ui_tests/__init__.py | 12 +++++-- tests/ui_tests/common.py | 16 +++++++++- 6 files changed, 66 insertions(+), 37 deletions(-) diff --git a/ci/test.yml b/ci/test.yml index ce2b56416..983e17082 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -73,6 +73,7 @@ core device test: - test_ui_report - tests/ui_tests/screens/ - tests/ui_tests/fixtures.suggestion.json + - tests/ui_tests/fixtures.results.json - tests/junit.xml - tests/trezor.log - core/.coverage.* @@ -104,6 +105,7 @@ core device R test: - test_ui_report - tests/ui_tests/screens/ - tests/ui_tests/fixtures.suggestion.json + - tests/ui_tests/fixtures.results.json - tests/junit.xml - tests/trezor.log - core/.coverage.* @@ -316,6 +318,7 @@ core click test: - test_ui_report - tests/ui_tests/screens/ - tests/ui_tests/fixtures.suggestion.json + - tests/ui_tests/fixtures.results.json - tests/trezor.log - tests/junit.xml - core/.coverage.* @@ -347,6 +350,7 @@ core click R test: - test_ui_report - tests/ui_tests/screens/ - tests/ui_tests/fixtures.suggestion.json + - tests/ui_tests/fixtures.results.json - tests/trezor.log - tests/junit.xml reports: @@ -437,6 +441,7 @@ core persistence test: - test_ui_report - tests/ui_tests/screens/ - tests/ui_tests/fixtures.suggestion.json + - tests/ui_tests/fixtures.results.json - tests/trezor.log - tests/junit.xml - core/.coverage.* @@ -534,6 +539,7 @@ legacy device test: - test_ui_report - tests/ui_tests/screens/ - tests/ui_tests/fixtures.suggestion.json + - tests/ui_tests/fixtures.results.json - tests/junit.xml - tests/trezor.log when: always diff --git a/docs/ci/jobs.md b/docs/ci/jobs.md index 586313484..8cfee95f9 100644 --- a/docs/ci/jobs.md +++ b/docs/ci/jobs.md @@ -158,81 +158,81 @@ with the expected UI result. See artifacts for a comprehensive report of UI. See [docs/tests/ui-tests](../tests/ui-tests.md) for more info. -### [core device R test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L84) +### [core device R test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L85) -### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L115) +### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L117) -### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L134) +### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L136) Device tests excluding altcoins, only for BTC. -### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L154) +### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L156) -### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L175) +### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L177) Monero tests. -### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L195) +### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L197) -### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L218) +### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L220) Tests for U2F and HID. -### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L237) +### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L239) -### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L255) +### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L257) FIDO2 device tests. -### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L278) +### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L280) -### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L298) +### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L300) Click tests - UI. See [docs/tests/click-tests](../tests/click-tests.md) for more info. -### [core click R test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L329) +### [core click R test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L332) Click tests. See [docs/tests/click-tests](../tests/click-tests.md) for more info. -### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L357) +### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L361) -### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L378) +### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L382) Upgrade tests. See [docs/tests/upgrade-tests](../tests/upgrade-tests.md) for more info. -### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L397) +### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L401) -### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L419) +### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L423) Persistence tests - UI. -### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L448) +### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L453) -### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L466) +### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L471) -### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L485) +### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L490) -### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L517) +### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L522) Legacy device test - UI. -### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L544) +### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L550) -### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L556) +### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L562) -### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L576) +### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L582) -### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L591) +### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L597) -### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L610) +### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L616) -### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L631) +### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L637) -### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L651) +### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L657) -### [python support test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L670) +### [python support test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L676) -### [rust test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L679) +### [rust test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L685) -### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L689) +### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L695) -### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L713) +### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L719) -### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L737) +### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L743) --- ## TEST-HW stage - [test-hw.yml](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml) diff --git a/tests/show_results.py b/tests/show_results.py index 00ec77404..ab68bd789 100755 --- a/tests/show_results.py +++ b/tests/show_results.py @@ -13,7 +13,7 @@ from urllib.parse import unquote import click -from ui_tests.common import SCREENS_DIR, TestResult, write_fixtures +from ui_tests.common import SCREENS_DIR, TestResult, write_fixtures_complete from ui_tests.reporting import testreport # noqa: E402 ROOT = Path(__file__).resolve().parent.parent @@ -78,7 +78,7 @@ class NoCacheRequestHandler(http.server.SimpleHTTPRequestHandler): test_path = SCREENS_DIR / test_name result = TestResult.load(test_path) assert result.actual_hash == test_hash - write_fixtures([result]) + write_fixtures_complete([result]) self.send_response(200) self.send_header("Content-Type", "text/plain") diff --git a/tests/ui_tests/.gitignore b/tests/ui_tests/.gitignore index 192127612..6ff4503ef 100644 --- a/tests/ui_tests/.gitignore +++ b/tests/ui_tests/.gitignore @@ -3,4 +3,5 @@ *.zip *.txt fixtures.suggestion.json +fixtures.results.json reporting/master_cache diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py index ebe621125..f4a6b8ff8 100644 --- a/tests/ui_tests/__init__.py +++ b/tests/ui_tests/__init__.py @@ -14,6 +14,7 @@ from .common import SCREENS_DIR, UI_TESTS_DIR, TestCase, TestResult from .reporting import testreport FIXTURES_SUGGESTION_FILE = UI_TESTS_DIR / "fixtures.suggestion.json" +FIXTURES_RESULTS_FILE = UI_TESTS_DIR / "fixtures.results.json" def _process_recorded(result: TestResult) -> None: @@ -113,7 +114,7 @@ def update_fixtures(remove_missing: bool = False) -> int: for result in results: result.store_recorded() - common.write_fixtures(results, remove_missing=remove_missing) + common.write_fixtures_complete(results, remove_missing=remove_missing) return len(results) @@ -174,8 +175,15 @@ def sessionfinish( return exitstatus testreport.generate_reports(record_text_layout, do_master_diff) + + if test_ui == "test": + common.write_fixtures_only_new_results( + TestResult.recent_results(), + dest=FIXTURES_RESULTS_FILE, + ) + if test_ui == "test" and check_missing and list_missing(): - common.write_fixtures( + common.write_fixtures_complete( TestResult.recent_results(), remove_missing=True, dest=FIXTURES_SUGGESTION_FILE, diff --git a/tests/ui_tests/common.py b/tests/ui_tests/common.py index 1406dc794..ef460a9ce 100644 --- a/tests/ui_tests/common.py +++ b/tests/ui_tests/common.py @@ -77,11 +77,25 @@ def prepare_fixtures( return fixtures, missing_tests -def write_fixtures( +def write_fixtures_only_new_results( + results: t.Iterable[TestResult], + dest: Path, +) -> None: + """Generate new results file with only the tests that were actually run.""" + content: dict[str, dict[str, dict[str, str]]] = {} + for res in results: + model = content.setdefault(res.test.model, {}) + group = model.setdefault(res.test.group, {}) + group[res.test.fixtures_name] = res.actual_hash + dest.write_text(json.dumps(content, indent=0, sort_keys=True) + "\n") + + +def write_fixtures_complete( results: t.Iterable[TestResult], remove_missing: bool = False, dest: Path = FIXTURES_FILE, ) -> None: + """Generate new fixtures.json file with all the results, updated for the latest run.""" global FIXTURES content, _ = prepare_fixtures(results, remove_missing) dest.write_text(json.dumps(content, indent=0, sort_keys=True) + "\n")