trezor-firmware/ci/test.yml

# All the tests run test cases on the freshly built emulators from the previous `BUILD` stage.

image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix

# Caching
.gitlab_caching: &gitlab_caching
  cache:
    key: "$CI_COMMIT_REF_SLUG"
    paths:
      - .venv/

# Core

# Python and rust unit tests, checking TT functionality.
core unit test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix regular build
  script:
    - nix-shell --run "poetry run make -C core test | ts -s"

core unit ui2 test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen ui2 debug build
  script:
    - nix-shell --run "poetry run make -C core clippy | ts -s"
    - nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s"

core unit asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix regular asan build
  only:
    - schedules  # nightly build
  variables:
    RUSTC_BOOTSTRAP: "1"
    RUSTFLAGS: "-Z sanitizer=address"
    ADDRESS_SANITIZER: "1"
    LSAN_OPTIONS: "suppressions=../../asan_suppressions.txt"
  script:
    - nix-shell --run "poetry run make -C core test | ts -s"
    - nix-shell --run "poetry run make -C core clean build_unix UI2=1 | ts -s"
    - nix-shell --run "poetry run make -C core test_rust UI2=1 | ts -s"

core unit t1 test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen btconly debug t1 build
  variables:
    BITCOIN_ONLY: "1"
    TREZOR_MODEL: "1"
  script:
    - nix-shell --run "poetry run make -C core test_rust | ts -s"

# Device tests for Core. Running device tests and also comparing screens
# with the expected UI result.
# See artifacts for a comprehensive report of UI.
# See [docs/tests/ui-tests](../tests/ui-tests.md) for more info.
core device test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug build
  variables:
    TREZOR_PROFILING: 1
  script:
    - nix-shell --run "poetry run make -C core test_emu_ui | ts -s"
    - mv core/src/.coverage core/.coverage.test_emu
  after_script:
    - mv tests/ui_tests/reporting/reports/test/ test_ui_report
    - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py TT | ts -s"
    - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - ci/ui_test_records/
      - test_ui_report
      - tests/ui_tests/screens/
      - tests/ui_tests/fixtures.suggestion.json
      - tests/junit.xml
      - tests/trezor.log
      - core/.coverage.*
    when: always
    expire_in: 1 week
    reports:
      junit: tests/junit.xml

core device ui2 test:
  stage: test
  <<: *gitlab_caching
  when: manual
  needs:
    - core unix frozen ui2 debug build
  variables:
    TREZOR_PROFILING: "1"
    UI2: "1"
  script:
    - nix-shell --run "poetry run make -C core test_emu_ui | ts -s"
    # - mv core/src/.coverage core/.coverage.test_emu
  after_script:
    - mv tests/ui_tests/reporting/reports/test/ test_ui_report
    - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py TTui2 | ts -s"
    - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json
    - nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TTui2_"
    - mv tests/ui_tests/reporting/reports/master_diff/ .
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - ci/ui_test_records/
      - test_ui_report
      - tests/ui_tests/screens/
      - tests/ui_tests/fixtures.suggestion.json
      - tests/junit.xml
      - tests/trezor.log
      # - core/.coverage.*
      - master_diff
    when: always
    expire_in: 1 week
    reports:
      junit: tests/junit.xml

core device R test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen R debug build
  allow_failure: true
  variables:
    TREZOR_PROFILING: "1"
    TREZOR_MODEL: "R"
  script:
    - nix-shell --run "poetry run make -C core test_emu_ui | ts -s"
  after_script:
    - mv tests/ui_tests/reporting/reports/test/ test_ui_report
    - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py TR | ts -s"
    - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json
    - nix-shell --run "cd tests/ui_tests ; poetry run python reporting/report_master_diff.py TR_"
    - mv tests/ui_tests/reporting/reports/master_diff/ .
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - ci/ui_test_records/
      - test_ui_report
      - tests/ui_tests/screens/
      - tests/ui_tests/fixtures.suggestion.json
      - tests/junit.xml
      - tests/trezor.log
      - master_diff
    when: always
    expire_in: 4 weeks
    reports:
      junit: tests/junit.xml

core device asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug asan build
  only:
    - schedules  # nightly build
  variables:
    PYTEST_TIMEOUT: "600"
  script:
    - nix-shell --run "poetry run make -C core test_emu | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
    expire_in: 1 week
    when: always

# Device tests excluding altcoins, only for BTC.
core btconly device test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen btconly debug build
  variables:
    MICROPYTHON: "build/unix/trezor-emu-core-bitcoinonly"
    TREZOR_PYTEST_SKIP_ALTCOINS: 1
  script:
    - nix-shell --run "poetry run make -C core test_emu | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - tests/junit.xml
    expire_in: 1 week
    when: always
    reports:
      junit: tests/junit.xml

core btconly device asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen btconly debug asan build
  only:
    - schedules  # nightly build
  variables:
    MICROPYTHON: "build/unix/trezor-emu-core-bitcoinonly"
    TREZOR_PYTEST_SKIP_ALTCOINS: 1
    PYTEST_TIMEOUT: "600"
  script:
    - nix-shell --run "poetry run make -C core test_emu | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
    expire_in: 1 week
    when: always

# Monero tests.
core monero test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug build
  variables:
    TREZOR_PROFILING: 1
  script:
    - nix-shell --arg fullDeps true --run "poetry run make -C core test_emu_monero | ts -s"
    - mv core/src/.coverage core/.coverage.test_emu_monero
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - core/tests/trezor_monero_tests.log
      - core/.coverage.*
    expire_in: 1 week
    when: always

core monero asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug asan build
  only:
    - schedules  # nightly build
  variables:
    TREZOR_PROFILING: 1
  script:
    - nix-shell --arg fullDeps true --run "poetry run make -C core test_emu_monero | ts -s"
    - mv core/src/.coverage core/.coverage.test_emu_monero
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - core/tests/trezor_monero_tests.log
      - core/.coverage.*
    expire_in: 1 week
    when: always

# Tests for U2F and HID.
core u2f test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug build
  variables:
    TREZOR_PROFILING: 1
  script:
    - nix-shell --run "poetry run make -C tests/fido_tests/u2f-tests-hid | ts -s"
    - nix-shell --run "poetry run make -C core test_emu_u2f | ts -s"
    - mv core/src/.coverage core/.coverage.test_emu_u2f
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - core/.coverage.*
    expire_in: 1 week
    when: always

core u2f asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug asan build
  only:
    - schedules  # nightly build
  script:
    - nix-shell --run "poetry run make -C tests/fido_tests/u2f-tests-hid | ts -s"
    - nix-shell --run "poetry run make -C core test_emu_u2f | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
    expire_in: 1 week
    when: always

# FIDO2 device tests.
core fido2 test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug build
  variables:
    TREZOR_PROFILING: 1
  script:
    - pgrep trezor-emu-core || true
    - nix-shell --run "poetry run make -C core test_emu_fido2 | ts -s"
    - pgrep trezor-emu-core || true
    - mv core/src/.coverage core/.coverage.test_emu_fido2
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - tests/junit.xml
      - core/.coverage.*
    expire_in: 1 week
    reports:
      junit: tests/junit.xml
    when: always

core fido2 asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug asan build
  only:
    - schedules  # nightly build
  script:
    - pgrep trezor-emu-core || true
    - nix-shell --run "poetry run make -C core test_emu_fido2 | ts -s"
    - pgrep trezor-emu-core || true
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
    expire_in: 1 week
    when: always

# Click tests.
# See [docs/tests/click-tests](../tests/click-tests.md) for more info.
core click test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug build
  script:
    - nix-shell --run "poetry run make -C core test_emu_click | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

core click asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug asan build
  only:
    - schedules  # nightly build
  script:
    - nix-shell --run "poetry run make -C core test_emu_click | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

# Upgrade tests.
# See [docs/tests/upgrade-tests](../tests/upgrade-tests.md) for more info.
core upgrade test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug build
  variables:
    TREZOR_UPGRADE_TEST: "core"
  script:
    - nix-shell --run "tests/download_emulators.sh"
    - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

core upgrade asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug asan build
  only:
    - schedules  # nightly build
  variables:
    TREZOR_UPGRADE_TEST: "core"
  script:
    - nix-shell --run "tests/download_emulators.sh"
    - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

# Persistence tests.
core persistence test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug build
  script:
    - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/persistence_tests | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

core persistence asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug asan build
  only:
    - schedules  # nightly build
  script:
    - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/persistence_tests | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

core hwi test:
  stage: test
  <<: *gitlab_caching
  needs:
    - core unix frozen debug build
  allow_failure: true
  script:
    - nix-shell --run "git clone https://github.com/bitcoin-core/HWI.git"
    - nix-shell --arg fullDeps true --run "cd HWI && poetry install && poetry run ./test/test_trezor.py --model_t ../core/build/unix/trezor-emu-core bitcoind"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - HWI/trezor-t-emulator.stdout
    expire_in: 1 week
    when: always

# Crypto

crypto test:
  stage: test
  only:
    changes:
      - .gitlab-ci.yml
      - ci/**
      - crypto/**
  <<: *gitlab_caching
  needs:
    - crypto build
  variables:
    ASAN_OPTIONS: "verify_asan_link_order=0"
    CK_TIMEOUT_MULTIPLIER: 5
  script:
    - ./crypto/tests/aestst
    - ./crypto/tests/test_check
    - ./crypto/tests/test_openssl 1000
    - nix-shell --run "cd crypto && ITERS=10 poetry run pytest --junitxml=tests/junit.xml tests | ts -s"
    - nix-shell --run "CK_TIMEOUT_MULTIPLIER=20 valgrind -q --error-exitcode=1 ./crypto/tests/test_check_noasan | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - crypto/tests/junit.xml
    reports:
      junit: crypto/tests/junit.xml
    expire_in: 1 week
    when: always


# Legacy

legacy device test:
  stage: test
  <<: *gitlab_caching
  needs:
    - legacy emu regular debug build
  variables:
    EMULATOR: "1"
  script:
    - nix-shell --run "poetry run make -C legacy test_emu_ui | ts -s"
  after_script:
    - mv tests/ui_tests/reporting/reports/test/ test_ui_report
    - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py T1 | ts -s"
    - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - ci/ui_test_records/
      - test_ui_report
      - tests/ui_tests/screens/
      - tests/ui_tests/fixtures.suggestion.json
      - tests/junit.xml
      - tests/trezor.log
    when: always
    expire_in: 1 week
    reports:
      junit: tests/junit.xml

legacy asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - legacy emu regular debug asan build
  only:
    - schedules  # nightly build
  variables:
    EMULATOR: "1"
  script:
    - nix-shell --run "poetry run make -C legacy test_emu | ts -s"

legacy btconly test:
  stage: test
  <<: *gitlab_caching
  needs:
    - legacy emu btconly debug build
  variables:
    EMULATOR: "1"
    EMULATOR_BINARY: "firmware/trezor-bitcoinonly.elf"
    TREZOR_PYTEST_SKIP_ALTCOINS: 1
  script:
    - nix-shell --run "poetry run make -C legacy test_emu | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

legacy btconly asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - legacy emu btconly debug asan build
  only:
    - schedules  # nightly build
  variables:
    EMULATOR: "1"
    EMULATOR_BINARY: "firmware/trezor-bitcoinonly.elf"
    TREZOR_PYTEST_SKIP_ALTCOINS: 1
  script:
    - nix-shell --run "poetry run make -C legacy test_emu | ts -s"


legacy upgrade test:
  stage: test
  <<: *gitlab_caching
  needs:
    - legacy emu regular debug build
  variables:
    TREZOR_UPGRADE_TEST: "legacy"
  script:
    - nix-shell --run "tests/download_emulators.sh"
    - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

legacy upgrade asan test:
  stage: test
  <<: *gitlab_caching
  needs:
    - legacy emu regular debug asan build
  only:
    - schedules  # nightly build
  variables:
    TREZOR_UPGRADE_TEST: "legacy"
  script:
    - nix-shell --run "tests/download_emulators.sh"
    - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

legacy hwi test:
  stage: test
  <<: *gitlab_caching
  needs:
    - legacy emu regular debug build
  variables:
    EMULATOR: "1"
  allow_failure: true
  script:
    - nix-shell --run "git clone https://github.com/bitcoin-core/HWI.git"
    - nix-shell --arg fullDeps true --run "cd HWI && poetry install && poetry run ./test/test_trezor.py --model_1 ../legacy/firmware/trezor.elf bitcoind"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - HWI/trezor-1-emulator.stdout
    expire_in: 1 week
    when: always
# Python

python test:
  stage: test
  <<: *gitlab_caching
  needs: []
  variables:
    LC_ALL: "C.UTF-8"
    LANG: "C.UTF-8"
  script:
    # Workaround for nixpkgs+tox integration failure which results in:
    #     ModuleNotFoundError: No module named '_sysconfigdata__linux_x86_64-linux-gnu'
    # The value of _PYTHON_SYSCONFIGDATA_NAME has changed between python 3.7 and 3.8 and with
    # multiple versions in your environment the older pythons don't seem to work under tox.
    # When the variable is unset the interpreter seems to do the right thing. Can be removed in
    # july 2023 when python 3.7 is EOLed.
    # See also:
    # https://github.com/NixOS/nixpkgs/blob/b00c7c2d1d905eb63c81a0917f1a94b763a7843b/pkgs/development/interpreters/python/cpython/default.nix#L103
    # https://github.com/NixOS/nixpkgs/pull/98915
    - nix-shell --arg fullDeps true --run "unset _PYTHON_SYSCONFIGDATA_NAME && cd python && poetry run tox | ts -s"

python support test:
  stage: test
  <<: *gitlab_caching
  needs: []
  script:
    - nix-shell --run "poetry run make python_support_check | ts -s"


# Storage

storage test:
  stage: test
  only:
    changes:
      - .gitlab-ci.yml
      - ci/**
      - storage/**
  <<: *gitlab_caching
  needs: []
  script:
    - unset PYTEST_TIMEOUT
    - nix-shell --run "poetry run make -C storage/tests build | ts -s"
    - nix-shell --run "poetry run make -C storage/tests tests_all | ts -s"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

# Memory profiling

core unix memory profiler:
  stage: test
  when: manual
  <<: *gitlab_caching
  needs: []
  variables:
    PYOPT: "0"
    TREZOR_MEMPERF: "1"
    PYTEST_TIMEOUT: "900"
  script:
    - nix-shell --run "poetry run make -C core build_unix_frozen | ts -s"
    - nix-shell --run "poetry run make -C core test_emu | ts -s"
    - nix-shell --run "mkdir core/prof/memperf-html"
    - nix-shell --run "poetry run core/tools/alloc.py --alloc-data=core/src/alloc_data.txt html core/prof/memperf-html"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - core/prof/memperf-html
    expire_in: 1 week
    when: always

# Connect

connect test core:
  image: ghcr.io/trezor/trezor-user-env
  stage: test
  tags:
    - runner-internal
  needs:
    - core unix frozen debug build
  variables:
    SDL_VIDEODRIVER: "dummy"
  before_script:
    - cp /builds/satoshilabs/trezor/trezor-firmware/core/build/unix/trezor-emu-core /trezor-user-env/src/binaries/firmware/bin/trezor-emu-core-v2.99.99
    - chmod +x /trezor-user-env/src/binaries/firmware/bin/trezor-emu-core-v2.99.99
    - nix-shell --run "autoPatchelf /trezor-user-env/src/binaries/firmware/bin/trezor-emu-core-v2.99.99"
  script:
    - /trezor-user-env/run-nix.sh &
    - nix-shell --run "tests/connect_tests/connect_tests.sh 2.99.99"
  after_script:
    - cp /trezor-user-env/logs/debugging.log trezor-user-env-debugging.log
    - cp /trezor-user-env/logs/emulator_bridge.log tenv-emulator-bridge-debugging.log
  artifacts:
    paths:
      - trezor-user-env-debugging.log
      - tenv-emulator-bridge-debugging.log
    expire_in: 1 week
    when: always