From 728c5e0c97cd03ec84b07af4cbf01168df93703e Mon Sep 17 00:00:00 2001 From: grdddj Date: Fri, 31 Mar 2023 12:58:34 +0200 Subject: [PATCH] TR-ci: new tests for TR --- ci/build.yml | 79 ++++++++++++++++++++++++++++++ ci/deploy.yml | 19 ++++++++ ci/test.yml | 61 +++++++++++++++++++++++ docs/ci/jobs.md | 125 ++++++++++++++++++++++++++++-------------------- 4 files changed, 231 insertions(+), 53 deletions(-) diff --git a/ci/build.yml b/ci/build.yml index edd0c245ba..4049717900 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -133,6 +133,37 @@ core fw btconly production build: - firmware-T2T1-btconly-production-*.*.*-$CI_COMMIT_SHORT_SHA.bin expire_in: 1 week +core fw R debug build: + stage: build + <<: *gitlab_caching + needs: [] + variables: + TREZOR_MODEL: "R" + PYOPT: "0" + script: + - nix-shell --run "poetry run make -C core build_firmware" + - cp core/build/firmware/firmware.bin trezor-fw-debug-tr-$CORE_VERSION-$CI_COMMIT_SHORT_SHA.bin + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" + paths: + - trezor-fw-*.*.*-$CI_COMMIT_SHORT_SHA.bin + expire_in: 1 week + +core fw R build: + stage: build + <<: *gitlab_caching + needs: [] + variables: + TREZOR_MODEL: "R" + script: + - nix-shell --run "poetry run make -C core build_firmware" + - cp core/build/firmware/firmware.bin trezor-fw-tr-$CORE_VERSION-$CI_COMMIT_SHORT_SHA.bin + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" + paths: + - trezor-fw-*.*.*-$CI_COMMIT_SHORT_SHA.bin + expire_in: 1 week + # Non-frozen emulator build. This means you still need Python files # present which get interpreted. core unix regular build: @@ -235,6 +266,54 @@ core unix frozen debug build: untracked: true expire_in: 1 week +core unix frozen R debug build: + stage: build + <<: *gitlab_caching + needs: [] + variables: + PYOPT: "0" + TREZOR_MODEL: "R" + script: + - nix-shell --run "poetry run make -C core build_unix_frozen" + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" + untracked: true + expire_in: 10 weeks + +core unix frozen R debug build arm: + image: nixos/nix + stage: build + <<: *gitlab_caching + needs: [] + variables: + PYOPT: "0" + TREZOR_MODEL: "R" + script: + - nix-shell --run "poetry run make -C core build_unix_frozen" + - mv core/build/unix/trezor-emu-core core/build/unix/trezor-emu-core-arm + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" + untracked: true + expire_in: 10 weeks + tags: + - docker_darwin_arm + +# Debugger build for gdb/lldb. +core unix R debugger build: + stage: build + <<: *gitlab_caching + needs: [] + variables: + TREZOR_EMULATOR_FROZEN: "1" + TREZOR_MODEL: "R" + script: + - nix-shell --run "poetry run make -C core build_unix_debug" + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" + paths: + - core/build/unix + expire_in: 1 week + core unix frozen debug asan build: stage: build <<: *gitlab_caching diff --git a/ci/deploy.yml b/ci/deploy.yml index 8b141f0d72..181c7f651a 100644 --- a/ci/deploy.yml +++ b/ci/deploy.yml @@ -247,6 +247,25 @@ ui tests fixtures deploy: tags: - deploy +ui tests R fixtures deploy: + stage: deploy + variables: + DEPLOY_PATH: "${DEPLOY_BASE_DIR}/ui_tests/" + BUCKET: "data.trezor.io" + GIT_SUBMODULE_STRATEGY: "none" + before_script: [] # no poetry + needs: + - core device R test + script: + - echo "Deploying to $DEPLOY_PATH" + - rsync --delete -va ci/ui_test_records/* "$DEPLOY_PATH" + - source ${AWS_DEPLOY_DATA} + - aws s3 sync $DEPLOY_PATH s3://$BUCKET/dev/firmware/ui_tests + # This "hack" is needed because aws does not have an easy option to generate autoindex. We fetch the one autogenerated by nginx on local server. + - wget https://firmware.corp.sldev.cz/ui_tests/ -O index.html && aws s3 cp index.html s3://$BUCKET/dev/firmware/ui_tests/ + tags: + - deploy + # sync to aws sync emulators to aws: diff --git a/ci/test.yml b/ci/test.yml index ed1eb096eb..d50e19e5a8 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -81,6 +81,37 @@ core device test: reports: junit: tests/junit.xml +core device R test: + stage: test + <<: *gitlab_caching + needs: + - core unix frozen R debug build + variables: + TREZOR_PROFILING: "1" + TREZOR_MODEL: "R" + script: + - nix-shell --run "poetry run make -C core test_emu_ui | ts -s" + after_script: + - mv tests/ui_tests/reporting/reports/test/ test_ui_report + - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py TR | ts -s" + - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json + - nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TR_" + - mv tests/ui_tests/reporting/reports/master_diff/ . + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" + paths: + - ci/ui_test_records/ + - test_ui_report + - tests/ui_tests/screens/ + - tests/ui_tests/fixtures.suggestion.json + - tests/junit.xml + - tests/trezor.log + - master_diff + when: always + expire_in: 4 weeks + reports: + junit: tests/junit.xml + core device asan test: stage: test <<: *gitlab_caching @@ -291,6 +322,36 @@ core click test: expire_in: 1 week when: always +# Click tests. +# See [docs/tests/click-tests](../tests/click-tests.md) for more info. +core click R test: + stage: test + <<: *gitlab_caching + needs: + - core unix frozen R debug build + variables: + TREZOR_PROFILING: 1 + script: + - nix-shell --run "poetry run make -C core test_emu_click_ui | ts -s" + after_script: + - mv core/src/.coverage core/.coverage.test_click + - mv tests/ui_tests/reports/test/ test_ui_report + - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py | ts -s" + - diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" + paths: + - ci/ui_test_records/ + - test_ui_report + - tests/ui_tests/screens/ + - tests/ui_tests/fixtures.suggestion.json + - tests/trezor.log + - tests/junit.xml + reports: + junit: tests/junit.xml + expire_in: 1 week + when: always + core click asan test: stage: test <<: *gitlab_caching diff --git a/docs/ci/jobs.md b/docs/ci/jobs.md index cc6eb05be9..fe201484c7 100644 --- a/docs/ci/jobs.md +++ b/docs/ci/jobs.md @@ -51,7 +51,7 @@ or contain `[no changelog]` in the commit message. ## BUILD stage - [build.yml](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml) All builds are published as artifacts so they can be downloaded and used. -Consists of **25 jobs** below: +Consists of **30 jobs** below: ### [core fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L20) Build of Core into firmware. Regular version. @@ -71,68 +71,79 @@ Build of Core into firmware. Bitcoin-only version. ### [core fw btconly production build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L117) -### [core unix regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L138) +### [core fw R debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L136) + +### [core fw R build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L152) + +### [core unix regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L169) Non-frozen emulator build. This means you still need Python files present which get interpreted. -### [core unix regular asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L150) +### [core unix regular asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L181) -### [core unix frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L169) +### [core unix frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L200) Build of Core into UNIX emulator. Something you can run on your laptop. Frozen version. That means you do not need any other files to run it, it is just a single binary file that you can execute directly. -### [core unix frozen btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L186) +### [core unix frozen btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L217) Build of Core into UNIX emulator. Something you can run on your laptop. Frozen version. That means you do not need any other files to run it, it is just a single binary file that you can execute directly. See [Emulator](../core/emulator/index.md) for more info. Debug mode enabled, Bitcoin-only version. -### [core unix frozen btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L202) +### [core unix frozen btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L233) -### [core unix frozen debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L225) +### [core unix frozen debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L256) Build of Core into UNIX emulator. Something you can run on your laptop. Frozen version. That means you do not need any other files to run it, it is just a single binary file that you can execute directly. **Are you looking for a Trezor T emulator? This is most likely it.** -### [core unix frozen debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L238) +### [core unix frozen R debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L269) -### [core unix frozen debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L254) +### [core unix frozen R debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L283) -### [core macos frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L276) +### [core unix R debugger build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L302) +Debugger build for gdb/lldb. -### [crypto build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L301) +### [core unix frozen debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L317) + +### [core unix frozen debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L333) + +### [core macos frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L355) + +### [crypto build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L380) Build of our cryptographic library, which is then incorporated into the other builds. -### [legacy fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L330) +### [legacy fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L409) -### [legacy fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L346) +### [legacy fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L425) -### [legacy fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L363) +### [legacy fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L442) -### [legacy fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L382) +### [legacy fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L461) -### [legacy emu regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L403) +### [legacy emu regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L482) Regular version (not only Bitcoin) of above. **Are you looking for a Trezor One emulator? This is most likely it.** -### [legacy emu regular debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L418) +### [legacy emu regular debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L497) -### [legacy emu regular debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L436) +### [legacy emu regular debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L515) -### [legacy emu btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L462) +### [legacy emu btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L541) Build of Legacy into UNIX emulator. Use keyboard arrows to emulate button presses. Bitcoin-only version. -### [legacy emu btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L479) +### [legacy emu btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L558) --- ## TEST stage - [test.yml](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml) All the tests run test cases on the freshly built emulators from the previous `BUILD` stage. -Consists of **33 jobs** below: +Consists of **35 jobs** below: ### [core unit python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L15) Python unit tests, checking core functionality. @@ -148,72 +159,78 @@ with the expected UI result. See artifacts for a comprehensive report of UI. See [docs/tests/ui-tests](../tests/ui-tests.md) for more info. -### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L84) +### [core device R test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L84) -### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L103) +### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L115) + +### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L134) Device tests excluding altcoins, only for BTC. -### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L123) +### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L154) -### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L144) +### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L175) Monero tests. -### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L163) +### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L194) -### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L185) +### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L216) Tests for U2F and HID. -### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L204) +### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L235) -### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L222) +### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L253) FIDO2 device tests. -### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L245) +### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L276) -### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L265) +### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L296) Click tests. See [docs/tests/click-tests](../tests/click-tests.md) for more info. -### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L294) +### [core click R test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L327) +Click tests. +See [docs/tests/click-tests](../tests/click-tests.md) for more info. -### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L315) +### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L355) + +### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L376) Upgrade tests. See [docs/tests/upgrade-tests](../tests/upgrade-tests.md) for more info. -### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L334) +### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L395) -### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L356) +### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L417) Persistence tests. -### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L372) +### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L433) -### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L390) +### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L451) -### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L408) +### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L469) -### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L439) +### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L500) -### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L466) +### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L527) -### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L478) +### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L539) -### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L498) +### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L559) -### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L513) +### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L574) -### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L532) +### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L593) -### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L553) +### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L614) -### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L572) +### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L633) -### [python support test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L591) +### [python support test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L652) -### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L601) +### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L662) -### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L625) +### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L686) -### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L649) +### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L710) --- ## TEST-HW stage - [test-hw.yml](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml) @@ -260,7 +277,7 @@ Consists of **2 jobs** below: --- ## DEPLOY stage - [deploy.yml](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml) -Consists of **13 jobs** below: +Consists of **14 jobs** below: ### [release core fw regular deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L5) @@ -284,8 +301,10 @@ Consists of **13 jobs** below: ### [ui tests fixtures deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L229) -### [sync emulators to aws](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L252) +### [ui tests R fixtures deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L250) -### [common sync](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L277) +### [sync emulators to aws](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L271) + +### [common sync](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L296) ---