diff --git a/ci/deploy.yml b/ci/deploy.yml index 8b141f0d7..9194b1b37 100644 --- a/ci/deploy.yml +++ b/ci/deploy.yml @@ -236,6 +236,7 @@ ui tests fixtures deploy: needs: - core click test - core device test + - core persistence test - legacy device test script: - echo "Deploying to $DEPLOY_PATH" diff --git a/ci/posttest.yml b/ci/posttest.yml index 3524f2a22..acfb6d7e7 100644 --- a/ci/posttest.yml +++ b/ci/posttest.yml @@ -14,6 +14,7 @@ core unix coverage posttest: <<: *gitlab_caching needs: - core click test + - core persistence test - core device test - core monero test - core u2f test @@ -36,6 +37,7 @@ unix ui changes: <<: *gitlab_caching needs: - core click test + - core persistence test - core device test - legacy device test script: diff --git a/ci/test.yml b/ci/test.yml index ed1eb096e..60abe3fd5 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -260,7 +260,7 @@ core fido2 asan test: expire_in: 1 week when: always -# Click tests. +# Click tests - UI. # See [docs/tests/click-tests](../tests/click-tests.md) for more info. core click test: stage: test @@ -352,18 +352,31 @@ core upgrade asan test: expire_in: 1 week when: always -# Persistence tests. +# Persistence tests - UI. core persistence test: stage: test <<: *gitlab_caching needs: - core unix frozen debug build + variables: + TREZOR_PROFILING: 1 script: - - $NIX_SHELL --run "poetry run pytest --junitxml=tests/junit.xml tests/persistence_tests | ts -s" + - $NIX_SHELL --run "poetry run make -C core test_emu_persistence_ui | ts -s" + after_script: + - mv core/src/.coverage core/.coverage.test_persistence + - mv tests/ui_tests/reports/test/ test_ui_report + - $NIX_SHELL --run "poetry run python ci/prepare_ui_artifacts.py | ts -s" + - diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: + - ci/ui_test_records/ + - test_ui_report + - tests/ui_tests/screens/ + - tests/ui_tests/fixtures.suggestion.json + - tests/trezor.log - tests/junit.xml + - core/.coverage.* reports: junit: tests/junit.xml expire_in: 1 week @@ -436,6 +449,7 @@ crypto test: # Legacy +# Legacy device test - UI. legacy device test: stage: test <<: *gitlab_caching diff --git a/core/Makefile b/core/Makefile index bbcb97afd..d1e3d51de 100644 --- a/core/Makefile +++ b/core/Makefile @@ -113,6 +113,12 @@ test_emu_click: ## run click tests test_emu_click_ui: ## run click tests with UI testing $(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests --ui=test --ui-check-missing $(TESTOPTS) +test_emu_persistence: ## run persistence tests + $(PYTEST) $(TESTPATH)/persistence_tests $(TESTOPTS) + +test_emu_persistence_ui: ## run persistence tests with UI testing + $(PYTEST) $(TESTPATH)/persistence_tests --ui=test --ui-check-missing $(TESTOPTS) + test_emu_ui: ## run ui integration tests $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \ --ui=test --ui-check-missing --record-text-layout diff --git a/docs/ci/jobs.md b/docs/ci/jobs.md index cc6eb05be..566937bf8 100644 --- a/docs/ci/jobs.md +++ b/docs/ci/jobs.md @@ -171,7 +171,7 @@ FIDO2 device tests. ### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L245) ### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L265) -Click tests. +Click tests - UI. See [docs/tests/click-tests](../tests/click-tests.md) for more info. ### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L294) @@ -183,37 +183,38 @@ See [docs/tests/upgrade-tests](../tests/upgrade-tests.md) for more info. ### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L334) ### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L356) -Persistence tests. +Persistence tests - UI. -### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L372) +### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L385) -### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L390) +### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L403) -### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L408) +### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L421) -### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L439) +### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L453) +Legacy device test - UI. -### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L466) +### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L480) -### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L478) +### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L492) -### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L498) +### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L512) -### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L513) +### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L527) -### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L532) +### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L546) -### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L553) +### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L567) -### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L572) +### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L586) -### [python support test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L591) +### [python support test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L605) -### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L601) +### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L615) -### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L625) +### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L639) -### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L649) +### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L663) --- ## TEST-HW stage - [test-hw.yml](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml) @@ -255,7 +256,7 @@ Consists of **2 jobs** below: ### [core unix coverage posttest](https://github.com/trezor/trezor-firmware/blob/master/ci/posttest.yml#L10) -### [unix ui changes](https://github.com/trezor/trezor-firmware/blob/master/ci/posttest.yml#L32) +### [unix ui changes](https://github.com/trezor/trezor-firmware/blob/master/ci/posttest.yml#L33) --- ## DEPLOY stage - [deploy.yml](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml) @@ -284,8 +285,8 @@ Consists of **13 jobs** below: ### [ui tests fixtures deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L229) -### [sync emulators to aws](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L252) +### [sync emulators to aws](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L253) -### [common sync](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L277) +### [common sync](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L278) ---