diff --git a/ci/posttest.yml b/ci/posttest.yml index afde3fe59e..ed1ea9c2f3 100644 --- a/ci/posttest.yml +++ b/ci/posttest.yml @@ -13,6 +13,7 @@ core unix coverage posttest: COVERAGE_THRESHOLD: "78" <<: *gitlab_caching needs: + - core click test - core device test - core monero test - core u2f test @@ -34,11 +35,12 @@ unix ui changes: - master <<: *gitlab_caching needs: + - core click test - core device test - legacy device test script: - - nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TT_ T1_" - - mv tests/ui_tests/reporting/reports/master_diff/ . + - nix-shell --run "poetry run python -m tests.ui_tests.reporting master-diff" + - mv tests/ui_tests/reports/master_diff . artifacts: name: core-unix-ui-changes paths: diff --git a/ci/test.yml b/ci/test.yml index a84495c09b..aa03274672 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -60,11 +60,11 @@ core device test: TREZOR_PROFILING: 1 script: - nix-shell --run "poetry run make -C core test_emu_ui | ts -s" - - mv core/src/.coverage core/.coverage.test_emu after_script: - - mv tests/ui_tests/reporting/reports/test/ test_ui_report - - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py TT | ts -s" - - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json + - mv core/src/.coverage core/.coverage.test_emu + - mv tests/ui_tests/reports/test/ test_ui_report + - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py | ts -s" + - diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -266,11 +266,22 @@ core click test: <<: *gitlab_caching needs: - core unix frozen debug build + variables: + TREZOR_PROFILING: 1 script: - - nix-shell --run "poetry run make -C core test_emu_click | ts -s" + - nix-shell --run "poetry run make -C core test_emu_click_ui | ts -s" + after_script: + - mv core/src/.coverage core/.coverage.test_click + - mv tests/ui_tests/reports/test/ test_ui_report + - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py | ts -s" + - diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: + - ci/ui_test_records/ + - test_ui_report + - tests/ui_tests/screens/ + - tests/ui_tests/fixtures.suggestion.json - tests/trezor.log - tests/junit.xml reports: @@ -433,9 +444,9 @@ legacy device test: script: - nix-shell --run "poetry run make -C legacy test_emu_ui | ts -s" after_script: - - mv tests/ui_tests/reporting/reports/test/ test_ui_report - - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py T1 | ts -s" - - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json + - mv tests/ui_tests/reports/test/ test_ui_report + - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py | ts -s" + - diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: diff --git a/core/Makefile b/core/Makefile index 50e2eeb85b..f3d9fbbbb7 100644 --- a/core/Makefile +++ b/core/Makefile @@ -106,6 +106,9 @@ test_emu_fido2: ## run fido2 device tests test_emu_click: ## run click tests $(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests $(TESTOPTS) +test_emu_click_ui: ## run click tests with UI testing + $(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests --ui=test --ui-check-missing $(TESTOPTS) + test_emu_ui: ## run ui integration tests $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \ --ui=test --ui-check-missing diff --git a/docs/ci/jobs.md b/docs/ci/jobs.md index a392d4ca82..3c213ec777 100644 --- a/docs/ci/jobs.md +++ b/docs/ci/jobs.md @@ -177,46 +177,46 @@ FIDO2 device tests. Click tests. See [docs/tests/click-tests](../tests/click-tests.md) for more info. -### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L281) +### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L292) -### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L302) +### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L313) Upgrade tests. See [docs/tests/upgrade-tests](../tests/upgrade-tests.md) for more info. -### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L321) +### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L332) -### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L343) +### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L354) Persistence tests. -### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L359) +### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L370) -### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L377) +### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L388) -### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L395) +### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L406) -### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L426) +### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L437) -### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L453) +### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L464) -### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L465) +### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L476) -### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L485) +### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L496) -### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L500) +### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L511) -### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L519) +### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L530) -### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L540) +### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L551) -### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L559) +### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L570) -### [python support test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L578) +### [python support test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L589) -### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L588) +### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L599) -### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L612) +### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L623) -### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L636) +### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L647) --- ## TEST-HW stage - [test-hw.yml](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml) @@ -258,7 +258,7 @@ Consists of **2 jobs** below: ### [core unix coverage posttest](https://github.com/trezor/trezor-firmware/blob/master/ci/posttest.yml#L10) -### [unix ui changes](https://github.com/trezor/trezor-firmware/blob/master/ci/posttest.yml#L31) +### [unix ui changes](https://github.com/trezor/trezor-firmware/blob/master/ci/posttest.yml#L32) --- ## DEPLOY stage - [deploy.yml](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml)