arno
/
trezor-firmware
mirror of https://github.com/trezor/trezor-firmware.git
1
0
Fork 0
Code Issues Releases Wiki Activity

ci: ui2 device tests

Browse Source
mmilata/monero-drop-python-test
Martin Milata 2 years ago
parent 1f612580f7
commit c153ffe4b3
6 changed files with 129 additions and 56 deletions
Show Stats Download Patch File Download Diff File

15
ci/build.yml
Unescape View File

@ -251,6 +251,21 @@ core unix frozen debug build:
untracked: true untracked: true
expire_in: 1 week expire_in: 1 week
core unix frozen ui2 debug build:
stage: build
<<: *gitlab_caching
needs: []
variables:
PYOPT: "0"
UI2: "1"
script:
- nix-shell --run "poetry run make -C core build_unix_frozen"
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
untracked: true
expire_in: 1 week
core unix frozen debug asan build: core unix frozen debug asan build:
stage: build stage: build
<<: *gitlab_caching <<: *gitlab_caching

2
ci/posttest.yml
Unescape View File

@ -37,7 +37,7 @@ unix ui changes:
- core device test - core device test
- legacy device test - legacy device test
script: script:
- nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py" - nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TT_ T1_"
- mv tests/ui_tests/reporting/reports/master_diff/ . - mv tests/ui_tests/reporting/reports/master_diff/ .
artifacts: artifacts:
name: core-unix-ui-changes name: core-unix-ui-changes

3
ci/prepare_ui_artifacts.py
Unescape View File

@ -1,3 +1,4 @@
import os
import shutil import shutil
import sys import sys
from pathlib import Path from pathlib import Path
@ -17,6 +18,8 @@ if len(sys.argv) > 1 and sys.argv[1].upper() == "T1":
model = "T1" model = "T1"
else: else:
model = "TT" model = "TT"
if os.getenv("UI2") == "1":
model += "ui2"
model_file_hashes = {k: v for k, v in FILE_HASHES.items() if k.startswith(f"{model}_")} model_file_hashes = {k: v for k, v in FILE_HASHES.items() if k.startswith(f"{model}_")}
for test_case, expected_hash in model_file_hashes.items(): for test_case, expected_hash in model_file_hashes.items():

46
ci/test.yml
Unescape View File

@ -19,11 +19,17 @@ core unit test:
- core unix regular build - core unix regular build
script: script:
- nix-shell --run "poetry run make -C core test | ts -s" - nix-shell --run "poetry run make -C core test | ts -s"
core unit ui2 test:
stage: test
<<: *gitlab_caching
needs:
- core unix frozen ui2 debug build
script:
- nix-shell --run "poetry run make -C core clippy | ts -s" - nix-shell --run "poetry run make -C core clippy | ts -s"
- nix-shell --run "poetry run make -C core clean build_unix UI2=1 | ts -s"
- nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s" - nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s"
core unit test asan: core unit asan test:
stage: test stage: test
<<: *gitlab_caching <<: *gitlab_caching
needs: needs:
@ -40,7 +46,7 @@ core unit test asan:
- nix-shell --run "poetry run make -C core clean build_unix UI2=1 | ts -s" - nix-shell --run "poetry run make -C core clean build_unix UI2=1 | ts -s"
- nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s" - nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s"
core unit test t1: core unit t1 test:
stage: test stage: test
<<: *gitlab_caching <<: *gitlab_caching
needs: needs:
@ -84,6 +90,40 @@ core device test:
reports: reports:
junit: tests/junit.xml junit: tests/junit.xml
core device ui2 test:
stage: test
<<: *gitlab_caching
when: manual
needs:
- core unix frozen ui2 debug build
variables:
TREZOR_PROFILING: "1"
UI2: "1"
script:
- nix-shell --run "poetry run make -C core test_emu_ui | ts -s"
#- mv core/src/.coverage core/.coverage.test_emu
after_script:
- mv tests/ui_tests/reporting/reports/test/ test_ui_report
- nix-shell --run "poetry run python ci/prepare_ui_artifacts.py TTui2 | ts -s"
- diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json
- nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TTui2_"
- mv tests/ui_tests/reporting/reports/master_diff/ .
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
paths:
- ci/ui_test_records/
- test_ui_report
- tests/ui_tests/screens/
- tests/ui_tests/fixtures.suggestion.json
- tests/junit.xml
- tests/trezor.log
#- core/.coverage.*
- master_diff
when: always
expire_in: 1 week
reports:
junit: tests/junit.xml
core device asan test: core device asan test:
stage: test stage: test
<<: *gitlab_caching <<: *gitlab_caching

100
docs/ci/jobs.md
Unescape View File

@ -54,7 +54,7 @@ or contain `[no changelog]` in the commit message.
## BUILD stage - [build.yml](../../ci/build.yml) ## BUILD stage - [build.yml](../../ci/build.yml)
All builds are published as artifacts so they can be downloaded and used. All builds are published as artifacts so they can be downloaded and used.
Consists of **27 jobs** below: Consists of **28 jobs** below:
### [core fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L20) ### [core fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L20)
Build of Core into firmware. Regular version. Build of Core into firmware. Regular version.
@ -102,122 +102,128 @@ Frozen version. That means you do not need any other files to run it,
it is just a single binary file that you can execute directly. it is just a single binary file that you can execute directly.
**Are you looking for a Trezor T emulator? This is most likely it.** **Are you looking for a Trezor T emulator? This is most likely it.**
### [core unix frozen debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L254) ### [core unix frozen ui2 debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L255)
### [core unix frozen debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L270) ### [core unix frozen debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L269)
### [core unix frozen btconly debug t1 build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L291) ### [core unix frozen debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L285)
### [core macos frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L306) ### [core unix frozen btconly debug t1 build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L306)
### [crypto build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L331) ### [core macos frozen regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L321)
### [crypto build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L346)
Build of our cryptographic library, which is then incorporated into the other builds. Build of our cryptographic library, which is then incorporated into the other builds.
### [legacy fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L360) ### [legacy fw regular build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L375)
### [legacy fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L376) ### [legacy fw regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L391)
### [legacy fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L393) ### [legacy fw btconly build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L408)
### [legacy fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L412) ### [legacy fw btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L427)
### [legacy emu regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L433) ### [legacy emu regular debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L448)
Regular version (not only Bitcoin) of above. Regular version (not only Bitcoin) of above.
**Are you looking for a Trezor One emulator? This is most likely it.** **Are you looking for a Trezor One emulator? This is most likely it.**
### [legacy emu regular debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L448) ### [legacy emu regular debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L463)
### [legacy emu regular debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L466) ### [legacy emu regular debug build arm](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L481)
### [legacy emu btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L491) ### [legacy emu btconly debug build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L506)
Build of Legacy into UNIX emulator. Use keyboard arrows to emulate button presses. Build of Legacy into UNIX emulator. Use keyboard arrows to emulate button presses.
Bitcoin-only version. Bitcoin-only version.
### [legacy emu btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L508) ### [legacy emu btconly debug asan build](https://github.com/trezor/trezor-firmware/blob/master/ci/build.yml#L523)
--- ---
## TEST stage - [test.yml](../../ci/test.yml) ## TEST stage - [test.yml](../../ci/test.yml)
All the tests run test cases on the freshly built emulators from the previous `BUILD` stage. All the tests run test cases on the freshly built emulators from the previous `BUILD` stage.
Consists of **32 jobs** below: Consists of **34 jobs** below:
### [core unit test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L15) ### [core unit test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L15)
Python and rust unit tests, checking TT functionality. Python and rust unit tests, checking TT functionality.
### [core unit test asan](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L26) ### [core unit ui2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L23)
### [core unit asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L32)
### [core unit test t1](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L43) ### [core unit t1 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L49)
### [core device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L58) ### [core device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L64)
Device tests for Core. Running device tests and also comparing screens Device tests for Core. Running device tests and also comparing screens
with the expected UI result. with the expected UI result.
See artifacts for a comprehensive report of UI. See artifacts for a comprehensive report of UI.
See [docs/tests/ui-tests](../docs/tests/ui-tests.md) for more info. See [docs/tests/ui-tests](../docs/tests/ui-tests.md) for more info.
### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L87) ### [core device ui2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L93)
### [core device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L127)
### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L104) ### [core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L144)
Device tests excluding altcoins, only for BTC. Device tests excluding altcoins, only for BTC.
### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L124) ### [core btconly device asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L164)
### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L144) ### [core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L184)
Monero tests. Monero tests.
### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L163) ### [core monero asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L203)
### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L185) ### [core u2f test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L225)
Tests for U2F and HID. Tests for U2F and HID.
### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L204) ### [core u2f asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L244)
### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L222) ### [core fido2 test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L262)
FIDO2 device tests. FIDO2 device tests.
### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L245) ### [core fido2 asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L285)
### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L265) ### [core click test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L305)
Click tests. Click tests.
See [docs/tests/click-tests](../docs/tests/click-tests.md) for more info. See [docs/tests/click-tests](../docs/tests/click-tests.md) for more info.
### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L282) ### [core click asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L322)
### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L303) ### [core upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L343)
Upgrade tests. Upgrade tests.
See [docs/tests/upgrade-tests](../docs/tests/upgrade-tests.md) for more info. See [docs/tests/upgrade-tests](../docs/tests/upgrade-tests.md) for more info.
### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L322) ### [core upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L362)
### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L344) ### [core persistence test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L384)
Persistence tests. Persistence tests.
### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L360) ### [core persistence asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L400)
### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L378) ### [core hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L418)
### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L396) ### [crypto test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L436)
### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L427) ### [legacy device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L467)
### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L454) ### [legacy asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L494)
### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L466) ### [legacy btconly test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L506)
### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L486) ### [legacy btconly asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L526)
### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L501) ### [legacy upgrade test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L541)
### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L520) ### [legacy upgrade asan test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L560)
### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L541) ### [legacy hwi test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L581)
### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L560) ### [python test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L600)
### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L589) ### [storage test](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L629)
### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L613) ### [core unix memory profiler](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L653)
### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L637) ### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L677)
--- ---
## TEST-HW stage - [test-hw.yml](../../ci/test-hw.yml) ## TEST-HW stage - [test-hw.yml](../../ci/test-hw.yml)

19
tests/ui_tests/reporting/report_master_diff.py
Unescape View File

@ -1,8 +1,9 @@
import shutil import shutil
import sys
import tempfile import tempfile
from contextlib import contextmanager from contextlib import contextmanager
from pathlib import Path from pathlib import Path
from typing import Dict, Tuple from typing import Dict, Sequence, Tuple
import dominate import dominate
from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr
@ -15,10 +16,18 @@ REPORTS_PATH = Path(__file__).resolve().parent / "reports" / "master_diff"
RECORDED_SCREENS_PATH = Path(__file__).resolve().parent.parent / "screens" RECORDED_SCREENS_PATH = Path(__file__).resolve().parent.parent / "screens"
def get_diff() -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]: def get_diff(
test_prefixes: Sequence[str],
) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
master = download.fetch_fixtures_master() master = download.fetch_fixtures_master()
current = download.fetch_fixtures_current() current = download.fetch_fixtures_current()
def matches_prefix(name: str) -> bool:
any(name.startswith(prefix) for prefix in test_prefixes)
master = {name: value for name, value in master.items() if matches_prefix(name)}
current = {name: value for name, value in current.items() if matches_prefix(name)}
# removed items # removed items
removed = {test: master[test] for test in (master.keys() - current.keys())} removed = {test: master[test] for test in (master.keys() - current.keys())}
# added items # added items
@ -154,8 +163,8 @@ def create_dirs() -> None:
(REPORTS_PATH / "diff").mkdir() (REPORTS_PATH / "diff").mkdir()
def create_reports() -> None: def create_reports(test_prefixes: Sequence[str]) -> None:
removed_tests, added_tests, diff_tests = get_diff() removed_tests, added_tests, diff_tests = get_diff(test_prefixes)
@contextmanager @contextmanager
def tmpdir(): def tmpdir():
@ -193,5 +202,5 @@ def create_reports() -> None:
if __name__ == "__main__": if __name__ == "__main__":
create_dirs() create_dirs()
create_reports() create_reports(sys.argv[1:] or [""])
index() index()

Write Preview
Loading…
Cancel
Save