trezor-firmware/ci/test.yml

image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix

# Core

core unit test:
  stage: test
  needs:
    - core unix regular build
  script:
    - nix-shell --run "poetry run make -C core test"

core device ui test:
  stage: test
  needs:
    - core unix frozen debug build
  script:
    - nix-shell --run "poetry run make -C core test_emu_ui"
  after_script:
    - mv tests/ui_tests/reporting/reports/test/ test_ui_report
    - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py"
    - diff tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - ci/ui_test_records/
      - test_ui_report
      - tests/ui_tests/screens/
      - tests/ui_tests/fixtures.suggestion.json
      - tests/junit.xml
      - tests/trezor.log
    when: always
    expire_in: 1 week
    reports:
      junit: tests/junit.xml

core device test:
  stage: test
  needs:
    - core unix frozen debug build
  variables:
    TREZOR_PROFILING: 1
  script:
    - nix-shell --run "poetry run make -C core test_emu"
    - mv core/src/.coverage core/.coverage.test_emu
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - tests/junit.xml
      - core/.coverage.*
    expire_in: 1 week
    when: always
    reports:
      junit: tests/junit.xml

core btconly device test:
  stage: test
  needs:
    - core unix frozen btconly debug build
  variables:
    MICROPYTHON: "build/unix/trezor-emu-core-bitcoinonly"
    TREZOR_PYTEST_SKIP_ALTCOINS: 1
  script:
    - nix-shell --run "poetry run make -C core test_emu"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - tests/junit.xml
    expire_in: 1 week
    when: always
    reports:
      junit: tests/junit.xml

core monero test:
  stage: test
  needs:
    - core unix frozen debug build
  variables:
    TREZOR_PROFILING: 1
  script:
    - nix-shell --arg fullDeps true --run "poetry run make -C core test_emu_monero"
    - mv core/src/.coverage core/.coverage.test_emu_monero
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - core/.coverage.*
    expire_in: 1 week
    when: always

core u2f test:
  stage: test
  needs:
    - core unix frozen debug build
  variables:
    TREZOR_PROFILING: 1
  script:
    - nix-shell --run "poetry run make -C tests/fido_tests/u2f-tests-hid"
    - nix-shell --run "poetry run make -C core test_emu_u2f"
    - mv core/src/.coverage core/.coverage.test_emu_u2f
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - core/.coverage.*
    expire_in: 1 week
    when: always

core fido2 test:
  stage: test
  needs:
    - core unix frozen debug build
  variables:
    TREZOR_PROFILING: 1
  script:
    - pgrep trezor-emu-core || true
    - nix-shell --run "poetry run make -C core test_emu_fido2"
    - pgrep trezor-emu-core || true
    - mv core/src/.coverage core/.coverage.test_emu_fido2
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - tests/junit.xml
      - core/.coverage.*
    expire_in: 1 week
    reports:
      junit: tests/junit.xml
    when: always

core click test:
  stage: test
  needs:
    - core unix frozen debug build
  script:
    - nix-shell --run "poetry run make -C core test_emu_click"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

core upgrade test:
  stage: test
  needs:
    - core unix frozen debug build
  variables:
    TREZOR_UPGRADE_TEST: "core"
  script:
    - nix-shell --run "tests/download_emulators.sh"
    - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

core persistence test:
  stage: test
  needs:
    - core unix frozen debug build
  script:
    - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/persistence_tests"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always


# Crypto

crypto test:
  stage: test
  only:
    changes:
      - .gitlab-ci.yml
      - ci/**/*
      - crypto/**/*
  needs:
    - crypto build
  script:
    - ./crypto/tests/aestst
    - ./crypto/tests/test_check
    - ./crypto/tests/test_openssl 1000
    - nix-shell --run "cd crypto && ITERS=10 poetry run pytest --junitxml=tests/junit.xml tests"
    - nix-shell --run "CK_TIMEOUT_MULTIPLIER=20 valgrind -q --error-exitcode=1 ./crypto/tests/test_check"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always


# Legacy

legacy test:
  stage: test
  needs:
    - legacy emu regular debug build
  variables:
    EMULATOR: "1"
  script:
    - nix-shell --run "poetry run legacy/script/test"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

legacy btconly test:
  stage: test
  needs:
    - legacy emu btconly debug build
  variables:
    EMULATOR: "1"
    EMULATOR_BINARY: "firmware/trezor-bitcoinonly.elf"
    TREZOR_PYTEST_SKIP_ALTCOINS: 1
  script:
    - nix-shell --run "poetry run legacy/script/test"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always


legacy upgrade test:
  stage: test
  needs:
    - legacy emu regular debug build
  variables:
    TREZOR_UPGRADE_TEST: "legacy"
  script:
    - nix-shell --run "tests/download_emulators.sh"
    - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always


# Python

python test:
  stage: test
  needs: []
  variables:
    LC_ALL: "C.UTF-8"
    LANG: "C.UTF-8"
  only:
    changes:
      - poetry.lock
      - .gitlab-ci.yml
      - ci/**/*
      - common/**/*
      - python/**/*
  script:
    - nix-shell --arg fullDeps true --run "cd python && poetry run tox"


# Storage

storage test:
  stage: test
  only:
    changes:
      - .gitlab-ci.yml
      - ci/**/*
      - storage/**/*
  needs: []
  script:
    - nix-shell --run "poetry run make -C storage/tests build"
    - nix-shell --run "poetry run make -C storage/tests tests_all"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/junit.xml
    reports:
      junit: tests/junit.xml
    expire_in: 1 week
    when: always

# Hardware
hardware legacy regular device test:
  stage: test
  only:
    - schedules  # nightly build
    - /^legacy\//
    - /^release\//
    - /^secfix\//
  tags:
    - tpmb
  needs:
    - legacy fw regular debug build
  script:
    - cd ci/hardware_tests
    - nix-shell --run "./record_video.sh ${CI_COMMIT_SHORT_SHA} start"
    - nix-shell --run "cd ../.. && poetry install"
    - nix-shell --run "poetry run python bootstrap.py"
    - nix-shell --run "poetry run python bootstrap.py ../../trezor-*.bin"
    - nix-shell --run "poetry run pytest ../../tests/device_tests"
    - nix-shell --run "./record_video.sh ${CI_COMMIT_SHORT_SHA} stop"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - ci/hardware_tests/video*.mp4
    expire_in: 2 days
    when: always

hardware legacy btconly device test:
  stage: test
  variables:
    TREZOR_PYTEST_SKIP_ALTCOINS: 1
  only:
    - schedules  # nightly build
    - /^legacy\//
    - /^release\//
    - /^secfix\//
  tags:
    - tpmb
  needs:
    - legacy fw btconly debug build
  script:
    - cd ci/hardware_tests
    - nix-shell --run "./record_video.sh ${CI_COMMIT_SHORT_SHA} start"
    - nix-shell --run "cd ../.. && poetry install"
    - nix-shell --run "poetry run python bootstrap.py"
    - nix-shell --run "poetry run python bootstrap.py ../../trezor-*.bin"
    - nix-shell --run "poetry run pytest ../../tests/device_tests"
    - nix-shell --run "./record_video.sh ${CI_COMMIT_SHORT_SHA} stop"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - ci/hardware_tests/video*.mp4
    expire_in: 2 days
    when: always

core unix memory profiler:
  stage: test
  when: manual
  needs: []
  variables:
    PYOPT: "0"
    TREZOR_MEMPERF: "1"
    PYTEST_TIMEOUT: "900"
  script:
    - nix-shell --run "poetry run make -C core build_unix_frozen"
    - nix-shell --run "poetry run make -C core test_emu"
    - nix-shell --run "mkdir core/prof/memperf-html"
    - nix-shell --run "poetry run core/tools/alloc.py --alloc-data=core/src/alloc_data.txt html core/prof/memperf-html"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
      - tests/trezor.log
      - core/prof/memperf-html
    expire_in: 1 week
    when: always