ci: ui2 device tests

ci/build.yml

@@ -251,6 +251,21 @@ core unix frozen debug build:
     untracked: true
     expire_in: 1 week
 
+
+core unix frozen ui2 debug build:
+  stage: build
+  <<: *gitlab_caching
+  needs: []
+  variables:
+    PYOPT: "0"
+    UI2: "1"
+  script:
+    - nix-shell --run "poetry run make -C core build_unix_frozen"
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
+    untracked: true
+    expire_in: 1 week
+
 core unix frozen debug asan build:
   stage: build
   <<: *gitlab_caching

ci/posttest.yml

@@ -37,7 +37,7 @@ unix ui changes:
     - core device test
     - legacy device test
   script:
-    - nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py"
+    - nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TT_ T1_"
     - mv tests/ui_tests/reporting/reports/master_diff/ .
   artifacts:
     name: core-unix-ui-changes

ci/prepare_ui_artifacts.py

@@ -1,3 +1,4 @@
+import os
 import shutil
 import sys
 from pathlib import Path
@@ -17,6 +18,8 @@ if len(sys.argv) > 1 and sys.argv[1].upper() == "T1":
     model = "T1"
 else:
     model = "TT"
+if os.getenv("UI2") == "1":
+    model += "ui2"
 model_file_hashes = {k: v for k, v in FILE_HASHES.items() if k.startswith(f"{model}_")}
 
 for test_case, expected_hash in model_file_hashes.items():

ci/test.yml

@@ -19,11 +19,17 @@ core unit test:
     - core unix regular build
   script:
     - nix-shell --run "poetry run make -C core test | ts -s"
+
+core unit ui2 test:
+  stage: test
+  <<: *gitlab_caching
+  needs:
+    - core unix frozen ui2 debug build
+  script:
     - nix-shell --run "poetry run make -C core clippy | ts -s"
-    - nix-shell --run "poetry run make -C core clean build_unix UI2=1 | ts -s"
     - nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s"
 
-core unit test asan:
+core unit asan test:
   stage: test
   <<: *gitlab_caching
   needs:
@@ -40,7 +46,7 @@ core unit test asan:
     - nix-shell --run "poetry run make -C core clean build_unix UI2=1 | ts -s"
     - nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s"
 
-core unit test t1:
+core unit t1 test:
   stage: test
   <<: *gitlab_caching
   needs:
@@ -84,6 +90,40 @@ core device test:
     reports:
       junit: tests/junit.xml
 
+core device ui2 test:
+  stage: test
+  <<: *gitlab_caching
+  when: manual
+  needs:
+    - core unix frozen ui2 debug build
+  variables:
+    TREZOR_PROFILING: "1"
+    UI2: "1"
+  script:
+    - nix-shell --run "poetry run make -C core test_emu_ui | ts -s"
+    #- mv core/src/.coverage core/.coverage.test_emu
+  after_script:
+    - mv tests/ui_tests/reporting/reports/test/ test_ui_report
+    - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py TTui2 | ts -s"
+    - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json
+    - nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TTui2_"
+    - mv tests/ui_tests/reporting/reports/master_diff/ .
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
+    paths:
+      - ci/ui_test_records/
+      - test_ui_report
+      - tests/ui_tests/screens/
+      - tests/ui_tests/fixtures.suggestion.json
+      - tests/junit.xml
+      - tests/trezor.log
+      #- core/.coverage.*
+      - master_diff
+    when: always
+    expire_in: 1 week
+    reports:
+      junit: tests/junit.xml
+
 core device asan test:
   stage: test
   <<: *gitlab_caching

docs/ci/jobs.md

@@ -54,7 +54,7 @@ or contain `[no changelog]` in the commit message.
 ## BUILD stage - [build.yml](../../ci/build.yml)
 All builds are published as artifacts so they can be downloaded and used.
 
-Consists of **27 jobs** below:
+Consists of **28 jobs** below:
 
 ### [core fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L20)
 Build of Core into firmware. Regular version.
@@ -102,122 +102,128 @@ Frozen version. That means you do not need any other files to run it,
 it is just a single binary file that you can execute directly.
 **Are you looking for a Trezor T emulator? This is most likely it.**
 
-### [core unix frozen debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L254)
+### [core unix frozen ui2 debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L255)
 
-### [core unix frozen debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L270)
+### [core unix frozen debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L269)
 
-### [core unix frozen btconly debug t1 build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L291)
+### [core unix frozen debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L285)
 
-### [core macos frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L306)
+### [core unix frozen btconly debug t1 build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L306)
 
-### [crypto build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L331)
+### [core macos frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L321)
+
+### [crypto build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L346)
 Build of our cryptographic library, which is then incorporated into the other builds.
 
-### [legacy fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L360)
+### [legacy fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L375)
 
-### [legacy fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L376)
+### [legacy fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L391)
 
-### [legacy fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L393)
+### [legacy fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L408)
 
-### [legacy fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L412)
+### [legacy fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L427)
 
-### [legacy emu regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L433)
+### [legacy emu regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L448)
 Regular version (not only Bitcoin) of above.
 **Are you looking for a Trezor One emulator? This is most likely it.**
 
-### [legacy emu regular debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L448)
+### [legacy emu regular debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L463)
 
-### [legacy emu regular debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L466)
+### [legacy emu regular debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L481)
 
-### [legacy emu btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L491)
+### [legacy emu btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L506)
 Build of Legacy into UNIX emulator. Use keyboard arrows to emulate button presses.
 Bitcoin-only version.
 
-### [legacy emu btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L508)
+### [legacy emu btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L523)
 
 ---
 ## TEST stage - [test.yml](../../ci/test.yml)
 All the tests run test cases on the freshly built emulators from the previous `BUILD` stage.
 
-Consists of **32 jobs** below:
+Consists of **34 jobs** below:
 
 ### [core unit test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L15)
 Python and rust unit tests, checking TT functionality.
 
-### [core unit test asan](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L26)
+### [core unit ui2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L23)
 
-### [core unit test t1](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L43)
+### [core unit asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L32)
 
-### [core device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L58)
+### [core unit t1 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L49)
+
+### [core device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L64)
 Device tests for Core. Running device tests and also comparing screens
 with the expected UI result.
 See artifacts for a comprehensive report of UI.
 See [docs/tests/ui-tests](../docs/tests/ui-tests.md) for more info.
 
-### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L87)
+### [core device ui2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L93)
 
-### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L104)
+### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L127)
+
+### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L144)
 Device tests excluding altcoins, only for BTC.
 
-### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L124)
+### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L164)
 
-### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L144)
+### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L184)
 Monero tests.
 
-### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L163)
+### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L203)
 
-### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L185)
+### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L225)
 Tests for U2F and HID.
 
-### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L204)
+### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L244)
 
-### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L222)
+### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L262)
 FIDO2 device tests.
 
-### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L245)
+### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L285)
 
-### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L265)
+### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L305)
 Click tests.
 See [docs/tests/click-tests](../docs/tests/click-tests.md) for more info.
 
-### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L282)
+### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L322)
 
-### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L303)
+### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L343)
 Upgrade tests.
 See [docs/tests/upgrade-tests](../docs/tests/upgrade-tests.md) for more info.
 
-### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L322)
+### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L362)
 
-### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L344)
+### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L384)
 Persistence tests.
 
-### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L360)
+### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L400)
 
-### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L378)
+### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L418)
 
-### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L396)
+### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L436)
 
-### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L427)
+### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L467)
 
-### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L454)
+### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L494)
 
-### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L466)
+### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L506)
 
-### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L486)
+### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L526)
 
-### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L501)
+### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L541)
 
-### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L520)
+### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L560)
 
-### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L541)
+### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L581)
 
-### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L560)
+### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L600)
 
-### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L589)
+### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L629)
 
-### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L613)
+### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L653)
 
-### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L637)
+### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L677)
 
 ---
 ## TEST-HW stage - [test-hw.yml](../../ci/test-hw.yml)

tests/ui_tests/reporting/report_master_diff.py

@@ -1,8 +1,9 @@
 import shutil
+import sys
 import tempfile
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Dict, Tuple
+from typing import Dict, Sequence, Tuple
 
 import dominate
 from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr
@@ -15,10 +16,18 @@ REPORTS_PATH = Path(__file__).resolve().parent / "reports" / "master_diff"
 RECORDED_SCREENS_PATH = Path(__file__).resolve().parent.parent / "screens"
 
 
-def get_diff() -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
+def get_diff(
+    test_prefixes: Sequence[str],
+) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
     master = download.fetch_fixtures_master()
     current = download.fetch_fixtures_current()
 
+    def matches_prefix(name: str) -> bool:
+        any(name.startswith(prefix) for prefix in test_prefixes)
+
+    master = {name: value for name, value in master.items() if matches_prefix(name)}
+    current = {name: value for name, value in current.items() if matches_prefix(name)}
+
     # removed items
     removed = {test: master[test] for test in (master.keys() - current.keys())}
     # added items
@@ -154,8 +163,8 @@ def create_dirs() -> None:
     (REPORTS_PATH / "diff").mkdir()
 
 
-def create_reports() -> None:
+def create_reports(test_prefixes: Sequence[str]) -> None:
-    removed_tests, added_tests, diff_tests = get_diff()
+    removed_tests, added_tests, diff_tests = get_diff(test_prefixes)
 
     @contextmanager
     def tmpdir():
@@ -193,5 +202,5 @@ def create_reports() -> None:
 
 if __name__ == "__main__":
     create_dirs()
-    create_reports()
+    create_reports(sys.argv[1:] or [""])
     index()