diff --git a/Makefile b/Makefile index 1026b45b0..72ee6e849 100644 --- a/Makefile +++ b/Makefile @@ -119,6 +119,12 @@ protobuf: ## generate python protobuf headers protobuf_check: ## check that generated protobuf headers are up to date ./tools/build_protobuf --check -gen: mocks icons templates protobuf ## regeneate auto-generated files from sources +ci_docs: ## generate CI documentation + ./tools/generate_ci_docs.py -gen_check: mocks_check icons_check templates_check protobuf_check ## check validity of auto-generated files +ci_docs_check: ## check that generated CI documentation is up to date + ./tools/generate_ci_docs.py --check + +gen: mocks icons templates protobuf ci_docs ## regenerate auto-generated files from sources + +gen_check: mocks_check icons_check templates_check protobuf_check ci_docs_check ## check validity of auto-generated files diff --git a/ci/build.yml b/ci/build.yml index ca69a203a..b42994153 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -1,3 +1,5 @@ +# All builds are published as artifacts so they can be downloaded and used. + image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix # Caching @@ -13,6 +15,8 @@ variables: # Core +# Build of Core into firmware. Regular version. +# **Are you looking for Trezor T firmware build? This is most likely it.** core fw regular build: stage: build <<: *gitlab_caching @@ -31,6 +35,9 @@ core fw regular build: - trezor-fw-regular-*.*.*-$CI_COMMIT_SHORT_SHA.bin expire_in: 1 week +# Build of Core into firmware with enabled _debug_ mode. In debug mode you can +# upload mnemonic seed, use debug link etc. which enables device tests. Storage +# on the device gets wiped on every start in this firmware. core fw regular debug build: stage: build <<: *gitlab_caching @@ -66,6 +73,7 @@ core fw regular production build: - trezor-fw-regular-production-*.*.*-$CI_COMMIT_SHORT_SHA.bin expire_in: 1 week +# Build of Core into firmware. Bitcoin-only version. core fw btconly build: stage: build <<: *gitlab_caching @@ -141,6 +149,8 @@ core fw btconly t1 build: - trezor-fw-btconly-t1-*.*.*-$CI_COMMIT_SHORT_SHA.bin expire_in: 1 week +# Non-frozen emulator build. This means you still need Python files +# present which get interpreted. core unix regular build: stage: build <<: *gitlab_caching @@ -169,6 +179,9 @@ core unix regular asan build: - core/build/unix # most of it needed by test_rust expire_in: 1 week +# Build of Core into UNIX emulator. Something you can run on your laptop. +# Frozen version. That means you do not need any other files to run it, +# it is just a single binary file that you can execute directly. core unix frozen regular build: stage: build <<: *gitlab_caching @@ -181,6 +194,11 @@ core unix frozen regular build: - core/build/unix/trezor-emu-core expire_in: 1 week +# Build of Core into UNIX emulator. Something you can run on your laptop. +# Frozen version. That means you do not need any other files to run it, +# it is just a single binary file that you can execute directly. +# See [Emulator](../core/emulator/index.md) for more info. +# Debug mode enabled, Bitcoin-only version. core unix frozen btconly debug build: stage: build <<: *gitlab_caching @@ -216,6 +234,10 @@ core unix frozen btconly debug asan build: - core/build/unix/trezor-emu-core-bitcoinonly expire_in: 1 week +# Build of Core into UNIX emulator. Something you can run on your laptop. +# Frozen version. That means you do not need any other files to run it, +# it is just a single binary file that you can execute directly. +# **Are you looking for a Trezor T emulator? This is most likely it.** core unix frozen debug build: stage: build <<: *gitlab_caching @@ -305,6 +327,7 @@ core macos frozen regular build: # Crypto +# Build of our cryptographic library, which is then incorporated into the other builds. crypto build: stage: build <<: *gitlab_caching @@ -405,6 +428,8 @@ legacy fw btconly debug build: - trezor-fw-btconly-*.*.*-$CI_COMMIT_SHORT_SHA.bin expire_in: 1 week +# Regular version (not only Bitcoin) of above. +# **Are you looking for a Trezor One emulator? This is most likely it.** legacy emu regular debug build: stage: build <<: *gitlab_caching @@ -461,6 +486,8 @@ legacy emu regular debug build arm: tags: - docker_darwin_arm +# Build of Legacy into UNIX emulator. Use keyboard arrows to emulate button presses. +# Bitcoin-only version. legacy emu btconly debug build: stage: build <<: *gitlab_caching diff --git a/ci/environment.yml b/ci/environment.yml index 97080fbb5..8ae7adeda 100644 --- a/ci/environment.yml +++ b/ci/environment.yml @@ -1,3 +1,9 @@ +# Connected with creating the testing image for CI. + +# Environment job builds the `ci/Dockerfile` and pushes the built docker image +# into our GitLab registry. Since modifications of this Dockerfile are very rare +# this is a _manual_ job which needs to be triggered on GitLab. +# Almost all CI jobs run inside this docker image. .environment: stage: environment image: docker diff --git a/ci/prebuild.yml b/ci/prebuild.yml index c4f204b8f..80852b284 100644 --- a/ci/prebuild.yml +++ b/ci/prebuild.yml @@ -1,3 +1,5 @@ +# Static checks on the code. + image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix # Caching @@ -7,6 +9,10 @@ image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-en paths: - .venv/ +# Check the code for style correctness and perform some static code analysis. +# Biggest part is the python one - using `flake8`, `isort`, `black`, `pylint` and `pyright`, +# also checking Rust files by `rustfmt` and C files by `clang-format`. +# Changelogs formats are checked. style prebuild: stage: prebuild variables: @@ -15,30 +21,35 @@ style prebuild: script: - nix-shell --run "poetry run make style_check" +# Check validity of coin definitions and protobuf files. common prebuild: stage: prebuild <<: *gitlab_caching script: - nix-shell --run "poetry run make defs_check" +# Check validity of auto-generated files. gen prebuild: stage: prebuild <<: *gitlab_caching script: - nix-shell --run "poetry run make gen_check" +# Checking format of .editorconfig files. editor prebuild: stage: prebuild <<: *gitlab_caching script: - nix-shell --run "make editor_check" +# All .yml/.yaml files are checked for syntax validity and other correctness. yaml prebuild: stage: prebuild <<: *gitlab_caching script: - nix-shell --run "poetry run make yaml_check" +# Checking the format of release commit messages. release commit messages prebuild: stage: prebuild before_script: [] # nothing needed @@ -54,6 +65,8 @@ release commit messages prebuild: script: - nix-shell --run "ci/check_release_commit_messages.sh" +# Verifying that all commits changing some functionality have a changelog entry +# or contain `[no changelog]` in the commit message. changelog prebuild: stage: prebuild except: diff --git a/ci/test-hw.yml b/ci/test-hw.yml index 075a98a90..cee533374 100644 --- a/ci/test-hw.yml +++ b/ci/test-hw.yml @@ -9,6 +9,14 @@ image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-en # Hardware +# [Device tests](../docs/tests/device-tests.md) that run against an actual physical Trezor T. +# The device needs to have special bootloader, found in `core/embed/bootloader_ci`, that +# makes it possible to flash firmware without confirmation on the touchscreen. +# +# All hardware tests are run nightly on the `master` branch, as well as on push to branches +# with whitelisted prefix. If you want hardware tests ran on your branch, make sure its +# name starts with `hw/`. +# # Currently it's not possible to run all regular TT tests without getting into # a state where the micropython heap is too fragmented and allocations fail # (often manifesting as a stuck test case). For that reason some tests are @@ -42,6 +50,7 @@ hardware core regular device test: expire_in: 2 days when: always +# Also device tests on physical Trezor T but with Bitcoin-only firmware. hardware core btconly device test: stage: test only: @@ -98,6 +107,9 @@ hardware core monero test: expire_in: 2 days when: always +# [Device tests](../docs/tests/device-tests.md) executed on physical Trezor 1. +# This works thanks to [tpmb](https://github.com/mmahut/tpmb), which is a small arduino +# device capable of pushing an actual buttons on the device. hardware legacy regular device test: stage: test only: @@ -121,6 +133,7 @@ hardware legacy regular device test: expire_in: 2 days when: always +# Also device tests on physical Trezor 1 but with Bitcoin-only firmware. hardware legacy btconly device test: stage: test variables: diff --git a/ci/test.yml b/ci/test.yml index 5b6326d8d..05ac31eca 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -1,3 +1,5 @@ +# All the tests run test cases on the freshly built emulators from the previous `BUILD` stage. + image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix # Caching @@ -9,6 +11,7 @@ image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-en # Core +# Python and rust unit tests, checking TT functionality. core unit test: stage: test <<: *gitlab_caching @@ -46,6 +49,10 @@ core unit test t1: script: - nix-shell --run "poetry run make -C core test_rust | ts -s" +# Device tests for Core. Running device tests and also comparing screens +# with the expected UI result. +# See artifacts for a comprehensive report of UI. +# See [docs/tests/ui-tests](../docs/tests/ui-tests.md) for more info. core device test: stage: test <<: *gitlab_caching @@ -91,6 +98,7 @@ core device asan test: expire_in: 1 week when: always +# Device tests excluding altcoins, only for BTC. core btconly device test: stage: test <<: *gitlab_caching @@ -130,6 +138,7 @@ core btconly device asan test: expire_in: 1 week when: always +# Monero tests. core monero test: stage: test <<: *gitlab_caching @@ -170,6 +179,7 @@ core monero asan test: expire_in: 1 week when: always +# Tests for U2F and HID. core u2f test: stage: test <<: *gitlab_caching @@ -206,6 +216,7 @@ core u2f asan test: expire_in: 1 week when: always +# FIDO2 device tests. core fido2 test: stage: test <<: *gitlab_caching @@ -247,6 +258,8 @@ core fido2 asan test: expire_in: 1 week when: always +# Click tests. +# See [docs/tests/click-tests](../docs/tests/click-tests.md) for more info. core click test: stage: test <<: *gitlab_caching @@ -283,6 +296,8 @@ core click asan test: expire_in: 1 week when: always +# Upgrade tests. +# See [docs/tests/upgrade-tests](../docs/tests/upgrade-tests.md) for more info. core upgrade test: stage: test <<: *gitlab_caching @@ -323,6 +338,7 @@ core upgrade asan test: expire_in: 1 week when: always +# Persistence tests. core persistence test: stage: test <<: *gitlab_caching diff --git a/docs/ci/jobs.md b/docs/ci/jobs.md index 9341b73bf..3894b5038 100644 --- a/docs/ci/jobs.md +++ b/docs/ci/jobs.md @@ -1,98 +1,231 @@ -# List of GitLab CI Jobs +# CI pipeline +(Generated automatically by `tools/generate_ci_docs.py`. Do not edit by hand.) -## Environment +It consists of multiple stages below, each having one or more jobs +Latest CI pipeline of master branch can be seen at [https://gitlab.com/satoshilabs/trezor/trezor-firmware/-/pipelines/master/latest](https://gitlab.com/satoshilabs/trezor/trezor-firmware/-/pipelines/master/latest) -### environment +## ENVIRONMENT stage - [environment.yml](../../ci/environment.yml) +Connected with creating the testing image for CI. +Consists of **3 jobs** below: + +### [.environment](https://github.com/trezor/trezor-firmware/blob/master/ci/environment.yml#L7) Environment job builds the `ci/Dockerfile` and pushes the built docker image into our GitLab registry. Since modifications of this Dockerfile are very rare -this si a _manual_ job which needs to be triggered on GitLab. - +this is a _manual_ job which needs to be triggered on GitLab. Almost all CI jobs run inside this docker image. -## Build +### [environment manual](https://github.com/trezor/trezor-firmware/blob/master/ci/environment.yml#L31) -All builds are published as artifacts so you can download and use them. +### [environment scheduled](https://github.com/trezor/trezor-firmware/blob/master/ci/environment.yml#L35) -### core fw btconly build +--- +## PREBUILD stage - [prebuild.yml](../../ci/prebuild.yml) +Static checks on the code. -Build of Core into firmware. Bitcoin-only version. +Consists of **7 jobs** below: + +### [style prebuild](https://github.com/trezor/trezor-firmware/blob/master/ci/prebuild.yml#L16) +Check the code for style correctness and perform some static code analysis. +Biggest part is the python one - using `flake8`, `isort`, `black`, `pylint` and `pyright`, +also checking Rust files by `rustfmt` and C files by `clang-format`. +Changelogs formats are checked. + +### [common prebuild](https://github.com/trezor/trezor-firmware/blob/master/ci/prebuild.yml#L25) +Check validity of coin definitions and protobuf files. + +### [gen prebuild](https://github.com/trezor/trezor-firmware/blob/master/ci/prebuild.yml#L32) +Check validity of auto-generated files. + +### [editor prebuild](https://github.com/trezor/trezor-firmware/blob/master/ci/prebuild.yml#L39) +Checking format of .editorconfig files. + +### [yaml prebuild](https://github.com/trezor/trezor-firmware/blob/master/ci/prebuild.yml#L46) +All .yml/.yaml files are checked for syntax validity and other correctness. + +### [release commit messages prebuild](https://github.com/trezor/trezor-firmware/blob/master/ci/prebuild.yml#L53) +Checking the format of release commit messages. + +### [changelog prebuild](https://github.com/trezor/trezor-firmware/blob/master/ci/prebuild.yml#L70) +Verifying that all commits changing some functionality have a changelog entry +or contain `[no changelog]` in the commit message. -### core fw regular build +--- +## BUILD stage - [build.yml](../../ci/build.yml) +All builds are published as artifacts so they can be downloaded and used. -Build of Core into firmware. Regular version. **Are you looking for Trezor T firmware -build? This is most likely it.** +Consists of **27 jobs** below: -### core fw regular debug build +### [core fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L20) +Build of Core into firmware. Regular version. +**Are you looking for Trezor T firmware build? This is most likely it.** +### [core fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L41) Build of Core into firmware with enabled _debug_ mode. In debug mode you can upload mnemonic seed, use debug link etc. which enables device tests. Storage on the device gets wiped on every start in this firmware. -### core unix frozen btconly debug build +### [core fw regular production build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L54) -Build of Core into UNIX emulator. Something you can run on your laptop. +### [core fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L77) +Build of Core into firmware. Bitcoin-only version. -Frozen version. That means you do not need any other files to run it, it is just -a single binary file that you can execute directly. +### [core fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L94) -See [Emulator](../core/emulator/index.md) for more info. +### [core fw btconly production build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L117) -Debug mode enabled, Bitcoin-only version. +### [core fw btconly t1 build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L136) + +### [core unix regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L154) +Non-frozen emulator build. This means you still need Python files +present which get interpreted. -### core unix frozen debug build +### [core unix regular asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L166) + +### [core unix frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L185) +Build of Core into UNIX emulator. Something you can run on your laptop. +Frozen version. That means you do not need any other files to run it, +it is just a single binary file that you can execute directly. + +### [core unix frozen btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L202) +Build of Core into UNIX emulator. Something you can run on your laptop. +Frozen version. That means you do not need any other files to run it, +it is just a single binary file that you can execute directly. +See [Emulator](../core/emulator/index.md) for more info. +Debug mode enabled, Bitcoin-only version. -Same as above but regular version (not only Bitcoin). **Are you looking for a Trezor T -emulator? This is most likely it.** +### [core unix frozen btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L218) -### core unix frozen regular build +### [core unix frozen debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L241) +Build of Core into UNIX emulator. Something you can run on your laptop. +Frozen version. That means you do not need any other files to run it, +it is just a single binary file that you can execute directly. +**Are you looking for a Trezor T emulator? This is most likely it.** -Same as above but regular version (not only Bitcoin) without debug mode enabled. +### [core unix frozen debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L254) -### core unix regular build +### [core unix frozen debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L270) -Non-frozen emulator build. This means you still need Python files present which get -interpreted. +### [core unix frozen btconly debug t1 build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L291) -### crypto build +### [core macos frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L306) +### [crypto build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L331) Build of our cryptographic library, which is then incorporated into the other builds. -### legacy emu btconly build +### [legacy fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L360) -Build of Legacy into UNIX emulator. Use keyboard arrows to emulate button presses. +### [legacy fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L376) + +### [legacy fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L393) +### [legacy fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L412) + +### [legacy emu regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L433) +Regular version (not only Bitcoin) of above. +**Are you looking for a Trezor One emulator? This is most likely it.** + +### [legacy emu regular debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L448) + +### [legacy emu regular debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L466) + +### [legacy emu btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L491) +Build of Legacy into UNIX emulator. Use keyboard arrows to emulate button presses. Bitcoin-only version. -### legacy emu regular build +### [legacy emu btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L508) + +--- +## TEST stage - [test.yml](../../ci/test.yml) +All the tests run test cases on the freshly built emulators from the previous `BUILD` stage. + +Consists of **32 jobs** below: + +### [core unit test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L15) +Python and rust unit tests, checking TT functionality. + +### [core unit test asan](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L25) + +### [core unit test t1](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L41) + +### [core device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L56) +Device tests for Core. Running device tests and also comparing screens +with the expected UI result. +See artifacts for a comprehensive report of UI. +See [docs/tests/ui-tests](../docs/tests/ui-tests.md) for more info. + +### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L85) + +### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L102) +Device tests excluding altcoins, only for BTC. + +### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L122) -Regular version (not only Bitcoin) of above. **Are you looking for a Trezor One -emulator? This is most likely it.** +### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L142) +Monero tests. -### legacy fw btconly build +### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L161) -Build of Legacy into firmware. Bitcoin only. +### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L183) +Tests for U2F and HID. -### legacy fw debug build +### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L202) -Build of Legacy into firmware. Debug mode on. Storage on the device gets wiped on every -start in this firmware. +### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L220) +FIDO2 device tests. -### legacy fw regular build +### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L243) -Build of Legacy into firmware. Regular version. **Are you looking for Trezor One firmware -build? This is most likely it.** +### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L263) +Click tests. +See [docs/tests/click-tests](../docs/tests/click-tests.md) for more info. -## Test +### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L280) -### core device test +### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L301) +Upgrade tests. +See [docs/tests/upgrade-tests](../docs/tests/upgrade-tests.md) for more info. -Device tests with UI report for Core. See artifacts for a comprehensive report of UI. See [tests/ui-tests](../tests/ui-tests.html#reports) -for more info. +### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L320) -### hardware core regular device test +### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L342) +Persistence tests. -[Device tests](../tests/device-tests.md) that run against an actual physical Trezor T. +### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L358) + +### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L376) + +### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L394) + +### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L425) + +### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L452) + +### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L464) + +### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L484) + +### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L499) + +### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L518) + +### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L539) + +### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L558) + +### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L587) + +### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L610) + +### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L634) + +--- +## TEST-HW stage - [test-hw.yml](../../ci/test-hw.yml) + +Consists of **5 jobs** below: + +### [hardware core regular device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml#L25) +[Device tests](../docs/tests/device-tests.md) that run against an actual physical Trezor T. The device needs to have special bootloader, found in `core/embed/bootloader_ci`, that makes it possible to flash firmware without confirmation on the touchscreen. @@ -100,16 +233,63 @@ All hardware tests are run nightly on the `master` branch, as well as on push to with whitelisted prefix. If you want hardware tests ran on your branch, make sure its name starts with `hw/`. -### hardware core btconly device test +Currently it's not possible to run all regular TT tests without getting into +a state where the micropython heap is too fragmented and allocations fail +(often manifesting as a stuck test case). For that reason some tests are +skipped. +See also: https://github.com/trezor/trezor-firmware/issues/1371 +### [hardware core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml#L54) Also device tests on physical Trezor T but with Bitcoin-only firmware. -### hardware legacy regular device test +### [hardware core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml#L83) -[Device tests](../tests/device-tests.md) executed on physical Trezor 1. +### [hardware legacy regular device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml#L113) +[Device tests](../docs/tests/device-tests.md) executed on physical Trezor 1. This works thanks to [tpmb](https://github.com/mmahut/tpmb), which is a small arduino device capable of pushing an actual buttons on the device. -### hardware legacy btconly device test - +### [hardware legacy btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml#L137) Also device tests on physical Trezor 1 but with Bitcoin-only firmware. + +--- +## POSTTEST stage - [posttest.yml](../../ci/posttest.yml) + +Consists of **2 jobs** below: + +### [core unix coverage posttest](https://github.com/trezor/trezor-firmware/blob/master/ci/posttest.yml#L10) + +### [unix ui changes](https://github.com/trezor/trezor-firmware/blob/master/ci/posttest.yml#L31) + +--- +## DEPLOY stage - [deploy.yml](../../ci/deploy.yml) + +Consists of **13 jobs** below: + +### [release core fw regular deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L5) + +### [release core fw btconly deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L26) + +### [release core fw regular debug deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L47) + +### [release core fw btconly debug deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L68) + +### [release legacy fw regular deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L91) + +### [release legacy fw btconly deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L112) + +### [release legacy fw regular debug deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L133) + +### [release legacy fw btconly debug deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L154) + +### [release core unix debug deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L177) + +### [release legacy unix debug deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L202) + +### [ui tests fixtures deploy](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L229) + +### [sync emulators to aws](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L251) + +### [common sync](https://github.com/trezor/trezor-firmware/blob/master/ci/deploy.yml#L275) + +--- diff --git a/tools/generate_ci_docs.py b/tools/generate_ci_docs.py new file mode 100755 index 000000000..63f46f918 --- /dev/null +++ b/tools/generate_ci_docs.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +""" +Automatic generator of documentation about CI jobs. +Analyzes all .yml files connected with CI, takes the most important information +and writes it into a README file. + +Features: +- reads a job description from a comment above job definition +- includes a link to each file and also to job definition + +Usage: +- put comments (starting with "#") directly above the job definition in .yml file + +Running the script: +- `python generate_ci_docs.py` to generate the documentation +- `python generate_ci_docs.py --check` to check if documentation is up-to-date +""" + +import argparse +import filecmp +import os +import re +import sys +from collections import OrderedDict +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Any + +import yaml +from mako.template import Template + +parser = argparse.ArgumentParser() +parser.add_argument( + "--check", + action="store_true", + help="Check if there are no new changes in all CI .yml files", +) +args = parser.parse_args() + + +class DocsGenerator: + def __init__(self) -> None: + # Going to the root directory, so the relative + # locations of CI files are valid + os.chdir(Path(__file__).resolve().parent.parent) + + self.GITLAB_CI_FILE = ".gitlab-ci.yml" + self.DOC_FILE = "docs/ci/jobs.md" + + # Some keywords that are not job definitions and we should not care about them + self.NOT_JOBS = [ + "variables:", + "image:", + ".gitlab_caching:", + ] + + self.ALL_JOBS: dict[Path, dict[str, Any]] = OrderedDict() + + self.FILES = self.get_all_ci_files() + + def generate_docs(self) -> None: + """Whole pipeline of getting and saving the CI information.""" + for file in self.FILES: + self.ALL_JOBS[file] = { + "jobs": self.get_jobs_from_file(file), + "overall_description": self.get_overall_description_from_file(file), + } + + self.save_docs_into_file() + + def verify_docs(self) -> None: + """Checking if the docs are up-to-date with current CI .yml files. + + Creating a new doc file and comparing it against already existing one. + Exit with non-zero exit code when these files do not match. + """ + already_filled_doc_file = self.DOC_FILE + + with NamedTemporaryFile() as temp_file: + self.DOC_FILE = temp_file.name + + self.generate_docs() + if filecmp.cmp(already_filled_doc_file, self.DOC_FILE): + print("SUCCESS: Documentation is up-to-date!") + sys.exit(0) + else: + print("FAIL: Documentation is not up-to-date with CI .yml files!") + print(" Please run this script or `make gen`") + sys.exit(1) + + def get_all_ci_files(self) -> list[Path]: + """Loading all the CI files which are used in Gitlab.""" + if not os.path.exists(self.GITLAB_CI_FILE): + raise RuntimeError( + f"Main Gitlab CI file under {self.GITLAB_CI_FILE} does not exist!" + ) + + with open(self.GITLAB_CI_FILE, "r") as f: + gitlab_file_content = yaml.safe_load(f) + + all_ci_files = [Path(file) for file in gitlab_file_content["include"]] + + for file in all_ci_files: + if not file.exists(): + raise RuntimeError(f"File {file} does not exist!") + + return all_ci_files + + @staticmethod + def get_overall_description_from_file(file: Path) -> list[str]: + """Looking for comments at the very beginning of the file.""" + description_lines: list[str] = [] + with open(file, "r") as f: + for line in f: + if line.startswith("#"): + # Taking just the text - no hashes, no whitespace + description_lines.append(line.strip("# \n")) + else: + break + + return description_lines + + def get_jobs_from_file(self, file: Path) -> dict[str, dict[str, Any]]: + """Extract all jobs and their details from a certain file.""" + all_jobs: dict[str, dict[str, Any]] = OrderedDict() + + # Taking all the comments above a non-indented non-comment, which is + # always a job definition, unless defined in NOT_JOBS + with open(file, "r") as f: + comment_buffer: list[str] = [] + for index, line in enumerate(f): + if line.startswith("#"): + # Taking just the text - no hashes, no whitespace + comment_buffer.append(line.strip("# \n")) + else: + # regex: first character of a line is a word-character or a dot + if re.search(r"\A[\w\.]", line) and not any( + [line.startswith(not_job) for not_job in self.NOT_JOBS] + ): + job_name = line.rstrip(":\n") + all_jobs[job_name] = { + "description": comment_buffer, + "line_no": index + 1, + } + comment_buffer = [] + + return all_jobs + + def save_docs_into_file(self) -> None: + """Dump all the information into a documentation file.""" + + template_text = """ +# CI pipeline +(Generated automatically by `tools/generate_ci_docs.py`. Do not edit by hand.) + +It consists of multiple stages below, each having one or more jobs +Latest CI pipeline of master branch can be seen at [${latest_master}](${latest_master}) +<% + ## Needed because "##" is a comment in Mako templates + header_2 = "##" + header_3 = "###" +%> +% for file, file_info in all_jobs_items: +${header_2} ${file.stem.upper()} stage - [${file.name}](../../${file}) + % if file_info["overall_description"]: + % for stage_overall_description_line in file_info["overall_description"]: +${stage_overall_description_line} + % endfor + % endif + <% + job_amount = f"{len(file_info['jobs'])} job{'s' if len(file_info['jobs']) > 1 else ''}" + %> +Consists of **${job_amount}** below: + % for job_name, job_info in file_info["jobs"].items(): + <% + github_job_link = f"https://github.com/trezor/trezor-firmware/blob/master/{file}#L{job_info['line_no']}" + %> +${header_3} [${job_name}](${github_job_link}) + % if job_info["description"]: + % for job_description_line in job_info["description"]: +${job_description_line} + %endfor + % endif + % endfor + +--- +% endfor +""".strip() + + with open(self.DOC_FILE, "w") as doc_file: + doc_text: str = Template(template_text).render( + latest_master="https://gitlab.com/satoshilabs/trezor/trezor-firmware/-/pipelines/master/latest", + all_jobs_items=self.ALL_JOBS.items(), + ) + # Remove trailing whitespace coming from the template and include final newline + doc_file.writelines(line.rstrip() + "\n" for line in doc_text.splitlines()) + + +if __name__ == "__main__": + if args.check: + DocsGenerator().verify_docs() + else: + DocsGenerator().generate_docs()