mirror of
https://github.com/trezor/trezor-firmware.git
synced 2024-11-17 21:22:10 +00:00
ci: ui2 device tests
This commit is contained in:
Martin Milata
parent
1f612580f7
commit
c153ffe4b3
15
ci/build.yml
15
ci/build.yml
@ -251,6 +251,21 @@ core unix frozen debug build:
|
||||
untracked: true
|
||||
expire_in: 1 week
|
||||
|
||||
|
||||
core unix frozen ui2 debug build:
|
||||
stage: build
|
||||
<<: *gitlab_caching
|
||||
needs: []
|
||||
variables:
|
||||
PYOPT: "0"
|
||||
UI2: "1"
|
||||
script:
|
||||
- nix-shell --run "poetry run make -C core build_unix_frozen"
|
||||
artifacts:
|
||||
name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
|
||||
untracked: true
|
||||
expire_in: 1 week
|
||||
|
||||
core unix frozen debug asan build:
|
||||
stage: build
|
||||
<<: *gitlab_caching
|
||||
|
||||
@ -37,7 +37,7 @@ unix ui changes:
|
||||
- core device test
|
||||
- legacy device test
|
||||
script:
|
||||
- nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py"
|
||||
- nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TT_ T1_"
|
||||
- mv tests/ui_tests/reporting/reports/master_diff/ .
|
||||
artifacts:
|
||||
name: core-unix-ui-changes
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
@ -17,6 +18,8 @@ if len(sys.argv) > 1 and sys.argv[1].upper() == "T1":
|
||||
model = "T1"
|
||||
else:
|
||||
model = "TT"
|
||||
if os.getenv("UI2") == "1":
|
||||
model += "ui2"
|
||||
model_file_hashes = {k: v for k, v in FILE_HASHES.items() if k.startswith(f"{model}_")}
|
||||
|
||||
for test_case, expected_hash in model_file_hashes.items():
|
||||
|
||||
46
ci/test.yml
46
ci/test.yml
@ -19,11 +19,17 @@ core unit test:
|
||||
- core unix regular build
|
||||
script:
|
||||
- nix-shell --run "poetry run make -C core test | ts -s"
|
||||
|
||||
core unit ui2 test:
|
||||
stage: test
|
||||
<<: *gitlab_caching
|
||||
needs:
|
||||
- core unix frozen ui2 debug build
|
||||
script:
|
||||
- nix-shell --run "poetry run make -C core clippy | ts -s"
|
||||
- nix-shell --run "poetry run make -C core clean build_unix UI2=1 | ts -s"
|
||||
- nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s"
|
||||
|
||||
core unit test asan:
|
||||
core unit asan test:
|
||||
stage: test
|
||||
<<: *gitlab_caching
|
||||
needs:
|
||||
@ -40,7 +46,7 @@ core unit test asan:
|
||||
- nix-shell --run "poetry run make -C core clean build_unix UI2=1 | ts -s"
|
||||
- nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s"
|
||||
|
||||
core unit test t1:
|
||||
core unit t1 test:
|
||||
stage: test
|
||||
<<: *gitlab_caching
|
||||
needs:
|
||||
@ -84,6 +90,40 @@ core device test:
|
||||
reports:
|
||||
junit: tests/junit.xml
|
||||
|
||||
core device ui2 test:
|
||||
stage: test
|
||||
<<: *gitlab_caching
|
||||
when: manual
|
||||
needs:
|
||||
- core unix frozen ui2 debug build
|
||||
variables:
|
||||
TREZOR_PROFILING: "1"
|
||||
UI2: "1"
|
||||
script:
|
||||
- nix-shell --run "poetry run make -C core test_emu_ui | ts -s"
|
||||
#- mv core/src/.coverage core/.coverage.test_emu
|
||||
after_script:
|
||||
- mv tests/ui_tests/reporting/reports/test/ test_ui_report
|
||||
- nix-shell --run "poetry run python ci/prepare_ui_artifacts.py TTui2 | ts -s"
|
||||
- diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json
|
||||
- nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TTui2_"
|
||||
- mv tests/ui_tests/reporting/reports/master_diff/ .
|
||||
artifacts:
|
||||
name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
|
||||
paths:
|
||||
- ci/ui_test_records/
|
||||
- test_ui_report
|
||||
- tests/ui_tests/screens/
|
||||
- tests/ui_tests/fixtures.suggestion.json
|
||||
- tests/junit.xml
|
||||
- tests/trezor.log
|
||||
#- core/.coverage.*
|
||||
- master_diff
|
||||
when: always
|
||||
expire_in: 1 week
|
||||
reports:
|
||||
junit: tests/junit.xml
|
||||
|
||||
core device asan test:
|
||||
stage: test
|
||||
<<: *gitlab_caching
|
||||
|
||||
100
docs/ci/jobs.md
100
docs/ci/jobs.md
@ -54,7 +54,7 @@ or contain `[no changelog]` in the commit message.
|
||||
## BUILD stage - [build.yml](../../ci/build.yml)
|
||||
All builds are published as artifacts so they can be downloaded and used.
|
||||
|
||||
Consists of **27 jobs** below:
|
||||
Consists of **28 jobs** below:
|
||||
|
||||
### [core fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L20)
|
||||
Build of Core into firmware. Regular version.
|
||||
@ -102,122 +102,128 @@ Frozen version. That means you do not need any other files to run it,
|
||||
it is just a single binary file that you can execute directly.
|
||||
**Are you looking for a Trezor T emulator? This is most likely it.**
|
||||
|
||||
### [core unix frozen debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L254)
|
||||
### [core unix frozen ui2 debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L255)
|
||||
|
||||
### [core unix frozen debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L270)
|
||||
### [core unix frozen debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L269)
|
||||
|
||||
### [core unix frozen btconly debug t1 build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L291)
|
||||
### [core unix frozen debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L285)
|
||||
|
||||
### [core macos frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L306)
|
||||
### [core unix frozen btconly debug t1 build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L306)
|
||||
|
||||
### [crypto build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L331)
|
||||
### [core macos frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L321)
|
||||
|
||||
### [crypto build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L346)
|
||||
Build of our cryptographic library, which is then incorporated into the other builds.
|
||||
|
||||
### [legacy fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L360)
|
||||
### [legacy fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L375)
|
||||
|
||||
### [legacy fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L376)
|
||||
### [legacy fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L391)
|
||||
|
||||
### [legacy fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L393)
|
||||
### [legacy fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L408)
|
||||
|
||||
### [legacy fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L412)
|
||||
### [legacy fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L427)
|
||||
|
||||
### [legacy emu regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L433)
|
||||
### [legacy emu regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L448)
|
||||
Regular version (not only Bitcoin) of above.
|
||||
**Are you looking for a Trezor One emulator? This is most likely it.**
|
||||
|
||||
### [legacy emu regular debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L448)
|
||||
### [legacy emu regular debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L463)
|
||||
|
||||
### [legacy emu regular debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L466)
|
||||
### [legacy emu regular debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L481)
|
||||
|
||||
### [legacy emu btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L491)
|
||||
### [legacy emu btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L506)
|
||||
Build of Legacy into UNIX emulator. Use keyboard arrows to emulate button presses.
|
||||
Bitcoin-only version.
|
||||
|
||||
### [legacy emu btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L508)
|
||||
### [legacy emu btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L523)
|
||||
|
||||
---
|
||||
## TEST stage - [test.yml](../../ci/test.yml)
|
||||
All the tests run test cases on the freshly built emulators from the previous `BUILD` stage.
|
||||
|
||||
Consists of **32 jobs** below:
|
||||
Consists of **34 jobs** below:
|
||||
|
||||
### [core unit test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L15)
|
||||
Python and rust unit tests, checking TT functionality.
|
||||
|
||||
### [core unit test asan](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L26)
|
||||
### [core unit ui2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L23)
|
||||
|
||||
### [core unit test t1](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L43)
|
||||
### [core unit asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L32)
|
||||
|
||||
### [core device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L58)
|
||||
### [core unit t1 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L49)
|
||||
|
||||
### [core device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L64)
|
||||
Device tests for Core. Running device tests and also comparing screens
|
||||
with the expected UI result.
|
||||
See artifacts for a comprehensive report of UI.
|
||||
See [docs/tests/ui-tests](../docs/tests/ui-tests.md) for more info.
|
||||
|
||||
### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L87)
|
||||
### [core device ui2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L93)
|
||||
|
||||
### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L104)
|
||||
### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L127)
|
||||
|
||||
### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L144)
|
||||
Device tests excluding altcoins, only for BTC.
|
||||
|
||||
### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L124)
|
||||
### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L164)
|
||||
|
||||
### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L144)
|
||||
### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L184)
|
||||
Monero tests.
|
||||
|
||||
### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L163)
|
||||
### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L203)
|
||||
|
||||
### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L185)
|
||||
### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L225)
|
||||
Tests for U2F and HID.
|
||||
|
||||
### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L204)
|
||||
### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L244)
|
||||
|
||||
### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L222)
|
||||
### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L262)
|
||||
FIDO2 device tests.
|
||||
|
||||
### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L245)
|
||||
### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L285)
|
||||
|
||||
### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L265)
|
||||
### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L305)
|
||||
Click tests.
|
||||
See [docs/tests/click-tests](../docs/tests/click-tests.md) for more info.
|
||||
|
||||
### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L282)
|
||||
### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L322)
|
||||
|
||||
### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L303)
|
||||
### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L343)
|
||||
Upgrade tests.
|
||||
See [docs/tests/upgrade-tests](../docs/tests/upgrade-tests.md) for more info.
|
||||
|
||||
### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L322)
|
||||
### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L362)
|
||||
|
||||
### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L344)
|
||||
### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L384)
|
||||
Persistence tests.
|
||||
|
||||
### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L360)
|
||||
### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L400)
|
||||
|
||||
### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L378)
|
||||
### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L418)
|
||||
|
||||
### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L396)
|
||||
### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L436)
|
||||
|
||||
### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L427)
|
||||
### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L467)
|
||||
|
||||
### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L454)
|
||||
### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L494)
|
||||
|
||||
### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L466)
|
||||
### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L506)
|
||||
|
||||
### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L486)
|
||||
### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L526)
|
||||
|
||||
### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L501)
|
||||
### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L541)
|
||||
|
||||
### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L520)
|
||||
### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L560)
|
||||
|
||||
### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L541)
|
||||
### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L581)
|
||||
|
||||
### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L560)
|
||||
### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L600)
|
||||
|
||||
### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L589)
|
||||
### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L629)
|
||||
|
||||
### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L613)
|
||||
### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L653)
|
||||
|
||||
### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L637)
|
||||
### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L677)
|
||||
|
||||
---
|
||||
## TEST-HW stage - [test-hw.yml](../../ci/test-hw.yml)
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Dict, Tuple
|
||||
from typing import Dict, Sequence, Tuple
|
||||
|
||||
import dominate
|
||||
from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr
|
||||
@ -15,10 +16,18 @@ REPORTS_PATH = Path(__file__).resolve().parent / "reports" / "master_diff"
|
||||
RECORDED_SCREENS_PATH = Path(__file__).resolve().parent.parent / "screens"
|
||||
|
||||
|
||||
def get_diff() -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
|
||||
def get_diff(
|
||||
test_prefixes: Sequence[str],
|
||||
) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
|
||||
master = download.fetch_fixtures_master()
|
||||
current = download.fetch_fixtures_current()
|
||||
|
||||
def matches_prefix(name: str) -> bool:
|
||||
any(name.startswith(prefix) for prefix in test_prefixes)
|
||||
|
||||
master = {name: value for name, value in master.items() if matches_prefix(name)}
|
||||
current = {name: value for name, value in current.items() if matches_prefix(name)}
|
||||
|
||||
# removed items
|
||||
removed = {test: master[test] for test in (master.keys() - current.keys())}
|
||||
# added items
|
||||
@ -154,8 +163,8 @@ def create_dirs() -> None:
|
||||
(REPORTS_PATH / "diff").mkdir()
|
||||
|
||||
|
||||
def create_reports() -> None:
|
||||
removed_tests, added_tests, diff_tests = get_diff()
|
||||
def create_reports(test_prefixes: Sequence[str]) -> None:
|
||||
removed_tests, added_tests, diff_tests = get_diff(test_prefixes)
|
||||
|
||||
@contextmanager
|
||||
def tmpdir():
|
||||
@ -193,5 +202,5 @@ def create_reports() -> None:
|
||||
|
||||
if __name__ == "__main__":
|
||||
create_dirs()
|
||||
create_reports()
|
||||
create_reports(sys.argv[1:] or [""])
|
||||
index()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user