From c153ffe4b3f15e130acb425ca2598af84c295ab4 Mon Sep 17 00:00:00 2001 From: Martin Milata Date: Thu, 31 Mar 2022 19:16:59 +0200 Subject: [PATCH] ci: ui2 device tests --- ci/build.yml | 15 +++ ci/posttest.yml | 2 +- ci/prepare_ui_artifacts.py | 3 + ci/test.yml | 46 +++++++- docs/ci/jobs.md | 100 ++++++++++-------- .../ui_tests/reporting/report_master_diff.py | 19 +++- 6 files changed, 129 insertions(+), 56 deletions(-) diff --git a/ci/build.yml b/ci/build.yml index 6f274da02..64442a47f 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -251,6 +251,21 @@ core unix frozen debug build: untracked: true expire_in: 1 week + +core unix frozen ui2 debug build: + stage: build + <<: *gitlab_caching + needs: [] + variables: + PYOPT: "0" + UI2: "1" + script: + - nix-shell --run "poetry run make -C core build_unix_frozen" + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" + untracked: true + expire_in: 1 week + core unix frozen debug asan build: stage: build <<: *gitlab_caching diff --git a/ci/posttest.yml b/ci/posttest.yml index 5170f6c1e..afde3fe59 100644 --- a/ci/posttest.yml +++ b/ci/posttest.yml @@ -37,7 +37,7 @@ unix ui changes: - core device test - legacy device test script: - - nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py" + - nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TT_ T1_" - mv tests/ui_tests/reporting/reports/master_diff/ . artifacts: name: core-unix-ui-changes diff --git a/ci/prepare_ui_artifacts.py b/ci/prepare_ui_artifacts.py index 0efa556a2..3f2589ce8 100644 --- a/ci/prepare_ui_artifacts.py +++ b/ci/prepare_ui_artifacts.py @@ -1,3 +1,4 @@ +import os import shutil import sys from pathlib import Path @@ -17,6 +18,8 @@ if len(sys.argv) > 1 and sys.argv[1].upper() == "T1": model = "T1" else: model = "TT" +if os.getenv("UI2") == "1": + model += "ui2" model_file_hashes = {k: v for k, v in FILE_HASHES.items() if k.startswith(f"{model}_")} for test_case, expected_hash in model_file_hashes.items(): diff --git a/ci/test.yml b/ci/test.yml index 7fadf07cf..bdf97c913 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -19,11 +19,17 @@ core unit test: - core unix regular build script: - nix-shell --run "poetry run make -C core test | ts -s" + +core unit ui2 test: + stage: test + <<: *gitlab_caching + needs: + - core unix frozen ui2 debug build + script: - nix-shell --run "poetry run make -C core clippy | ts -s" - - nix-shell --run "poetry run make -C core clean build_unix UI2=1 | ts -s" - nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s" -core unit test asan: +core unit asan test: stage: test <<: *gitlab_caching needs: @@ -40,7 +46,7 @@ core unit test asan: - nix-shell --run "poetry run make -C core clean build_unix UI2=1 | ts -s" - nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s" -core unit test t1: +core unit t1 test: stage: test <<: *gitlab_caching needs: @@ -84,6 +90,40 @@ core device test: reports: junit: tests/junit.xml +core device ui2 test: + stage: test + <<: *gitlab_caching + when: manual + needs: + - core unix frozen ui2 debug build + variables: + TREZOR_PROFILING: "1" + UI2: "1" + script: + - nix-shell --run "poetry run make -C core test_emu_ui | ts -s" + #- mv core/src/.coverage core/.coverage.test_emu + after_script: + - mv tests/ui_tests/reporting/reports/test/ test_ui_report + - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py TTui2 | ts -s" + - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json + - nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TTui2_" + - mv tests/ui_tests/reporting/reports/master_diff/ . + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" + paths: + - ci/ui_test_records/ + - test_ui_report + - tests/ui_tests/screens/ + - tests/ui_tests/fixtures.suggestion.json + - tests/junit.xml + - tests/trezor.log + #- core/.coverage.* + - master_diff + when: always + expire_in: 1 week + reports: + junit: tests/junit.xml + core device asan test: stage: test <<: *gitlab_caching diff --git a/docs/ci/jobs.md b/docs/ci/jobs.md index 53b810f4b..aa0626074 100644 --- a/docs/ci/jobs.md +++ b/docs/ci/jobs.md @@ -54,7 +54,7 @@ or contain `[no changelog]` in the commit message. ## BUILD stage - [build.yml](../../ci/build.yml) All builds are published as artifacts so they can be downloaded and used. -Consists of **27 jobs** below: +Consists of **28 jobs** below: ### [core fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L20) Build of Core into firmware. Regular version. @@ -102,122 +102,128 @@ Frozen version. That means you do not need any other files to run it, it is just a single binary file that you can execute directly. **Are you looking for a Trezor T emulator? This is most likely it.** -### [core unix frozen debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L254) +### [core unix frozen ui2 debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L255) -### [core unix frozen debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L270) +### [core unix frozen debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L269) -### [core unix frozen btconly debug t1 build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L291) +### [core unix frozen debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L285) -### [core macos frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L306) +### [core unix frozen btconly debug t1 build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L306) -### [crypto build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L331) +### [core macos frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L321) + +### [crypto build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L346) Build of our cryptographic library, which is then incorporated into the other builds. -### [legacy fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L360) +### [legacy fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L375) -### [legacy fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L376) +### [legacy fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L391) -### [legacy fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L393) +### [legacy fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L408) -### [legacy fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L412) +### [legacy fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L427) -### [legacy emu regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L433) +### [legacy emu regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L448) Regular version (not only Bitcoin) of above. **Are you looking for a Trezor One emulator? This is most likely it.** -### [legacy emu regular debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L448) +### [legacy emu regular debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L463) -### [legacy emu regular debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L466) +### [legacy emu regular debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L481) -### [legacy emu btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L491) +### [legacy emu btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L506) Build of Legacy into UNIX emulator. Use keyboard arrows to emulate button presses. Bitcoin-only version. -### [legacy emu btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L508) +### [legacy emu btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L523) --- ## TEST stage - [test.yml](../../ci/test.yml) All the tests run test cases on the freshly built emulators from the previous `BUILD` stage. -Consists of **32 jobs** below: +Consists of **34 jobs** below: ### [core unit test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L15) Python and rust unit tests, checking TT functionality. -### [core unit test asan](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L26) +### [core unit ui2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L23) + +### [core unit asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L32) -### [core unit test t1](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L43) +### [core unit t1 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L49) -### [core device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L58) +### [core device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L64) Device tests for Core. Running device tests and also comparing screens with the expected UI result. See artifacts for a comprehensive report of UI. See [docs/tests/ui-tests](../docs/tests/ui-tests.md) for more info. -### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L87) +### [core device ui2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L93) + +### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L127) -### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L104) +### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L144) Device tests excluding altcoins, only for BTC. -### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L124) +### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L164) -### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L144) +### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L184) Monero tests. -### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L163) +### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L203) -### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L185) +### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L225) Tests for U2F and HID. -### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L204) +### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L244) -### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L222) +### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L262) FIDO2 device tests. -### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L245) +### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L285) -### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L265) +### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L305) Click tests. See [docs/tests/click-tests](../docs/tests/click-tests.md) for more info. -### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L282) +### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L322) -### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L303) +### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L343) Upgrade tests. See [docs/tests/upgrade-tests](../docs/tests/upgrade-tests.md) for more info. -### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L322) +### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L362) -### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L344) +### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L384) Persistence tests. -### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L360) +### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L400) -### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L378) +### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L418) -### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L396) +### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L436) -### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L427) +### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L467) -### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L454) +### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L494) -### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L466) +### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L506) -### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L486) +### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L526) -### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L501) +### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L541) -### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L520) +### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L560) -### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L541) +### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L581) -### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L560) +### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L600) -### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L589) +### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L629) -### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L613) +### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L653) -### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L637) +### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L677) --- ## TEST-HW stage - [test-hw.yml](../../ci/test-hw.yml) diff --git a/tests/ui_tests/reporting/report_master_diff.py b/tests/ui_tests/reporting/report_master_diff.py index ca294a1a7..adc41b9bd 100644 --- a/tests/ui_tests/reporting/report_master_diff.py +++ b/tests/ui_tests/reporting/report_master_diff.py @@ -1,8 +1,9 @@ import shutil +import sys import tempfile from contextlib import contextmanager from pathlib import Path -from typing import Dict, Tuple +from typing import Dict, Sequence, Tuple import dominate from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr @@ -15,10 +16,18 @@ REPORTS_PATH = Path(__file__).resolve().parent / "reports" / "master_diff" RECORDED_SCREENS_PATH = Path(__file__).resolve().parent.parent / "screens" -def get_diff() -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]: +def get_diff( + test_prefixes: Sequence[str], +) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]: master = download.fetch_fixtures_master() current = download.fetch_fixtures_current() + def matches_prefix(name: str) -> bool: + any(name.startswith(prefix) for prefix in test_prefixes) + + master = {name: value for name, value in master.items() if matches_prefix(name)} + current = {name: value for name, value in current.items() if matches_prefix(name)} + # removed items removed = {test: master[test] for test in (master.keys() - current.keys())} # added items @@ -154,8 +163,8 @@ def create_dirs() -> None: (REPORTS_PATH / "diff").mkdir() -def create_reports() -> None: - removed_tests, added_tests, diff_tests = get_diff() +def create_reports(test_prefixes: Sequence[str]) -> None: + removed_tests, added_tests, diff_tests = get_diff(test_prefixes) @contextmanager def tmpdir(): @@ -193,5 +202,5 @@ def create_reports() -> None: if __name__ == "__main__": create_dirs() - create_reports() + create_reports(sys.argv[1:] or [""]) index()