From 1e4b42792dd7f86c0bf10518f76f258a049010db Mon Sep 17 00:00:00 2001 From: Martin Milata Date: Tue, 31 Jan 2023 15:01:51 +0100 Subject: [PATCH] ci(nix): use cached-nix-shell for speed [no changelog] --- .gitlab-ci.yml | 6 +-- ci/Dockerfile | 4 +- ci/build.yml | 98 ++++++++++++++++++++++---------------------- ci/deploy.yml | 8 ++-- ci/posttest.yml | 4 +- ci/prebuild.yml | 14 +++---- ci/shell.nix | 1 + ci/test-hw.yml | 22 +++++----- ci/test.yml | 106 ++++++++++++++++++++++++------------------------ 9 files changed, 133 insertions(+), 130 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5dbd7e0c..1c83f57ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,9 +29,9 @@ stages: before_script: - . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh || true - - nix-shell --run "poetry install --remove-untracked" - - export LEGACY_VERSION=$(nix-shell --run "./tools/version.sh legacy/firmware/version.h") - - export CORE_VERSION=$(nix-shell --run "./tools/version.sh core/embed/firmware/version.h") + - cached-nix-shell --run "poetry install --remove-untracked" + - export LEGACY_VERSION=$(cached-nix-shell --run "./tools/version.sh legacy/firmware/version.h") + - export CORE_VERSION=$(cached-nix-shell --run "./tools/version.sh core/embed/firmware/version.h") include: - ci/environment.yml diff --git a/ci/Dockerfile b/ci/Dockerfile index 6df7d784d..904df07a9 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -50,5 +50,7 @@ ARG FULLDEPS_TESTING=0 ENV FULLDEPS_TESTING=${FULLDEPS_TESTING} RUN nix-shell --arg fullDeps "$([ ${FULLDEPS_TESTING} = 1 ] && echo true || echo false)" --run "echo deps pre-installed" +RUN nix-env -iA nixpkgs.cached-nix-shell +RUN cached-nix-shell --arg fullDeps "$([ ${FULLDEPS_TESTING} = 1 ] && echo true || echo false)" --run "echo cached-nix-shell initialized" -CMD [ "nix-shell" ] +CMD [ "cached-nix-shell" ] diff --git a/ci/build.yml b/ci/build.yml index d4bc6e10b..afc55a4a9 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -22,12 +22,12 @@ core fw regular build: <<: *gitlab_caching needs: [] script: - - nix-shell --run "poetry run make -C core build_boardloader" - - nix-shell --run "poetry run make -C core build_bootloader" - - nix-shell --run "poetry run make -C core build_bootloader_ci" - - nix-shell --run "poetry run make -C core build_prodtest" - - nix-shell --run "poetry run make -C core build_firmware" - - nix-shell --run "poetry run make -C core sizecheck" + - cached-nix-shell --run "poetry run make -C core build_boardloader" + - cached-nix-shell --run "poetry run make -C core build_bootloader" + - cached-nix-shell --run "poetry run make -C core build_bootloader_ci" + - cached-nix-shell --run "poetry run make -C core build_prodtest" + - cached-nix-shell --run "poetry run make -C core build_firmware" + - cached-nix-shell --run "poetry run make -C core sizecheck" - cp core/build/firmware/firmware.bin firmware-T2T1-$CORE_VERSION-$CI_COMMIT_SHORT_SHA.bin artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -43,7 +43,7 @@ core fw regular debug build: <<: *gitlab_caching needs: [] script: - - nix-shell --run "PYOPT=0 poetry run make -C core build_firmware" + - cached-nix-shell --run "PYOPT=0 poetry run make -C core build_firmware" - cp core/build/firmware/firmware.bin firmware-T2T1-debug-$CORE_VERSION-$CI_COMMIT_SHORT_SHA.bin artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -60,12 +60,12 @@ core fw regular production build: variables: PRODUCTION: "1" script: - - nix-shell --run "poetry run make -C core build_boardloader" - - nix-shell --run "poetry run make -C core build_bootloader" - - nix-shell --run "poetry run make -C core build_bootloader_ci" - - nix-shell --run "poetry run make -C core build_prodtest" - - nix-shell --run "poetry run make -C core build_firmware" - - nix-shell --run "poetry run make -C core sizecheck" + - cached-nix-shell --run "poetry run make -C core build_boardloader" + - cached-nix-shell --run "poetry run make -C core build_bootloader" + - cached-nix-shell --run "poetry run make -C core build_bootloader_ci" + - cached-nix-shell --run "poetry run make -C core build_prodtest" + - cached-nix-shell --run "poetry run make -C core build_firmware" + - cached-nix-shell --run "poetry run make -C core sizecheck" - cp core/build/firmware/firmware.bin firmware-T2T1-production-$CORE_VERSION-$CI_COMMIT_SHORT_SHA.bin artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -81,9 +81,9 @@ core fw btconly build: variables: BITCOIN_ONLY: "1" script: - - nix-shell --run "poetry run make -C core build_firmware" + - cached-nix-shell --run "poetry run make -C core build_firmware" - mv core/build/firmware/firmware.bin core/build/firmware/firmware-bitcoinonly.bin - - nix-shell --run "poetry run ./tools/check-bitcoin-only core/build/firmware/firmware-bitcoinonly.bin" + - cached-nix-shell --run "poetry run ./tools/check-bitcoin-only core/build/firmware/firmware-bitcoinonly.bin" - cp core/build/firmware/firmware-bitcoinonly.bin firmware-T2T1-btconly-$CORE_VERSION-$CI_COMMIT_SHORT_SHA.bin artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -99,7 +99,7 @@ core fw btconly debug build: BITCOIN_ONLY: "1" PYOPT: "0" script: - - nix-shell --run "poetry run make -C core build_firmware" + - cached-nix-shell --run "poetry run make -C core build_firmware" - cp core/build/firmware/firmware.bin firmware-T2T1-btconly-debug-$CORE_VERSION-$CI_COMMIT_SHORT_SHA.bin only: # currently used only for HW tests and deploys - schedules # nightly build @@ -124,8 +124,8 @@ core fw btconly production build: PRODUCTION: "1" BITCOIN_ONLY: "1" script: - - nix-shell --run "poetry run make -C core build_firmware" - - nix-shell --run "poetry run ./tools/check-bitcoin-only core/build/firmware/firmware.bin" + - cached-nix-shell --run "poetry run make -C core build_firmware" + - cached-nix-shell --run "poetry run ./tools/check-bitcoin-only core/build/firmware/firmware.bin" - cp core/build/firmware/firmware.bin firmware-T2T1-btconly-production-$CORE_VERSION-$CI_COMMIT_SHORT_SHA.bin artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -140,7 +140,7 @@ core unix regular build: <<: *gitlab_caching needs: [] script: - - nix-shell --run "poetry run make -C core build_unix" + - cached-nix-shell --run "poetry run make -C core build_unix" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -156,7 +156,7 @@ core unix regular asan build: variables: ADDRESS_SANITIZER: "1" script: - - nix-shell --run "poetry run make -C core build_unix" + - cached-nix-shell --run "poetry run make -C core build_unix" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -171,7 +171,7 @@ core unix frozen regular build: <<: *gitlab_caching needs: [] script: - - nix-shell --run "poetry run make -C core build_unix_frozen" + - cached-nix-shell --run "poetry run make -C core build_unix_frozen" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -191,7 +191,7 @@ core unix frozen btconly debug build: PYOPT: "0" BITCOIN_ONLY: "1" script: - - nix-shell --run "poetry run make -C core build_unix_frozen" + - cached-nix-shell --run "poetry run make -C core build_unix_frozen" - mv core/build/unix/trezor-emu-core core/build/unix/trezor-emu-core-bitcoinonly artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -210,7 +210,7 @@ core unix frozen btconly debug asan build: BITCOIN_ONLY: "1" ADDRESS_SANITIZER: "1" script: - - nix-shell --run "poetry run make -C core build_unix_frozen" + - cached-nix-shell --run "poetry run make -C core build_unix_frozen" - mv core/build/unix/trezor-emu-core core/build/unix/trezor-emu-core-bitcoinonly artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -229,7 +229,7 @@ core unix frozen debug build: variables: PYOPT: "0" script: - - nix-shell --run "poetry run make -C core build_unix_frozen" + - cached-nix-shell --run "poetry run make -C core build_unix_frozen" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" untracked: true @@ -245,7 +245,7 @@ core unix frozen debug asan build: PYOPT: "0" ADDRESS_SANITIZER: "1" script: - - nix-shell --run "poetry run make -C core build_unix_frozen" + - cached-nix-shell --run "poetry run make -C core build_unix_frozen" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" untracked: true @@ -264,7 +264,7 @@ core unix frozen debug build arm: variables: PYOPT: "0" script: - - nix-shell --run "poetry run make -C core build_unix_frozen" + - cached-nix-shell --run "poetry run make -C core build_unix_frozen" - mv core/build/unix/trezor-emu-core core/build/unix/trezor-emu-core-arm artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -281,7 +281,7 @@ core macos frozen regular build: tags: - darwin_arm script: - - nix-shell --option system x86_64-darwin --run "poetry run make -C core build_unix_frozen" + - cached-nix-shell --option system x86_64-darwin --run "poetry run make -C core build_unix_frozen" - export NAME="trezor-emu-core.darwin" - cp -v core/build/unix/trezor-emu-core ./$NAME - chmod +x $NAME @@ -311,8 +311,8 @@ crypto build: - crypto/** script: - cp -r crypto crypto_noasan - - nix-shell --run "poetry run make -C crypto" - - nix-shell --run "export ADDRESS_SANITIZER=0; poetry run make -C crypto_noasan" + - cached-nix-shell --run "poetry run make -C crypto" + - cached-nix-shell --run "export ADDRESS_SANITIZER=0; poetry run make -C crypto_noasan" - mv crypto_noasan/tests/test_check crypto/tests/test_check_noasan artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -332,10 +332,10 @@ legacy fw regular build: <<: *gitlab_caching needs: [] script: - - nix-shell --run "export PRODUCTION=1 && poetry run legacy/script/cibuild" - - nix-shell --run "poetry run legacy/script/setup" - - nix-shell --run "export PRODUCTION=0 && poetry run legacy/script/cibuild" - - nix-shell --run "poetry run make -C legacy/demo" + - cached-nix-shell --run "export PRODUCTION=1 && poetry run legacy/script/cibuild" + - cached-nix-shell --run "poetry run legacy/script/setup" + - cached-nix-shell --run "export PRODUCTION=0 && poetry run legacy/script/cibuild" + - cached-nix-shell --run "poetry run make -C legacy/demo" - mv legacy/firmware/trezor.bin firmware-T1B1-$LEGACY_VERSION-$CI_COMMIT_SHORT_SHA.bin artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -350,9 +350,9 @@ legacy fw regular debug build: variables: DEBUG_LINK: "1" script: - - nix-shell --run "export PRODUCTION=1 && poetry run legacy/script/cibuild" - - nix-shell --run "poetry run legacy/script/setup" - - nix-shell --run "export PRODUCTION=0 && poetry run legacy/script/cibuild" + - cached-nix-shell --run "export PRODUCTION=1 && poetry run legacy/script/cibuild" + - cached-nix-shell --run "poetry run legacy/script/setup" + - cached-nix-shell --run "export PRODUCTION=0 && poetry run legacy/script/cibuild" - mv legacy/firmware/trezor.bin firmware-T1B1-debug-$LEGACY_VERSION-$CI_COMMIT_SHORT_SHA.bin artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -367,11 +367,11 @@ legacy fw btconly build: variables: BITCOIN_ONLY: "1" script: - - nix-shell --run "export PRODUCTION=1 && poetry run legacy/script/cibuild" - - nix-shell --run "poetry run legacy/script/setup" - - nix-shell --run "export PRODUCTION=0 && poetry run legacy/script/cibuild" + - cached-nix-shell --run "export PRODUCTION=1 && poetry run legacy/script/cibuild" + - cached-nix-shell --run "poetry run legacy/script/setup" + - cached-nix-shell --run "export PRODUCTION=0 && poetry run legacy/script/cibuild" - mv legacy/firmware/trezor.bin legacy/firmware/trezor-bitcoinonly.bin - - nix-shell --run "poetry run ./tools/check-bitcoin-only legacy/firmware/trezor-bitcoinonly.bin" + - cached-nix-shell --run "poetry run ./tools/check-bitcoin-only legacy/firmware/trezor-bitcoinonly.bin" - mv legacy/firmware/trezor-bitcoinonly.bin firmware-T1B1-btconly-$LEGACY_VERSION-$CI_COMMIT_SHORT_SHA.bin artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -387,10 +387,10 @@ legacy fw btconly debug build: BITCOIN_ONLY: "1" DEBUG_LINK: "1" script: - - nix-shell --run "export PRODUCTION=1 && poetry run legacy/script/cibuild" - - nix-shell --run "poetry run legacy/script/setup" - - nix-shell --run "export PRODUCTION=0 && poetry run legacy/script/cibuild" - - nix-shell --run "poetry run ./tools/check-bitcoin-only legacy/firmware/trezor.bin" + - cached-nix-shell --run "export PRODUCTION=1 && poetry run legacy/script/cibuild" + - cached-nix-shell --run "poetry run legacy/script/setup" + - cached-nix-shell --run "export PRODUCTION=0 && poetry run legacy/script/cibuild" + - cached-nix-shell --run "poetry run ./tools/check-bitcoin-only legacy/firmware/trezor.bin" - mv legacy/firmware/trezor.bin firmware-T1B1-btconly-debug-$LEGACY_VERSION-$CI_COMMIT_SHORT_SHA.bin artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -408,7 +408,7 @@ legacy emu regular debug build: DEBUG_LINK: "1" EMULATOR: "1" script: - - nix-shell --run "poetry run legacy/script/cibuild" + - cached-nix-shell --run "poetry run legacy/script/cibuild" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -426,7 +426,7 @@ legacy emu regular debug asan build: EMULATOR: "1" ADDRESS_SANITIZER: "1" script: - - nix-shell --run "poetry run legacy/script/cibuild" + - cached-nix-shell --run "poetry run legacy/script/cibuild" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -447,7 +447,7 @@ legacy emu regular debug build arm: DEBUG_LINK: "1" EMULATOR: "1" script: - - nix-shell --run "poetry run legacy/script/cibuild" + - cached-nix-shell --run "poetry run legacy/script/cibuild" - mv legacy/firmware/trezor.elf legacy/firmware/trezor-arm.elf artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -468,7 +468,7 @@ legacy emu btconly debug build: DEBUG_LINK: "1" EMULATOR: "1" script: - - nix-shell --run "poetry run legacy/script/cibuild" + - cached-nix-shell --run "poetry run legacy/script/cibuild" - mv legacy/firmware/trezor.elf legacy/firmware/trezor-bitcoinonly.elf artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -488,7 +488,7 @@ legacy emu btconly debug asan build: EMULATOR: "1" ADDRESS_SANITIZER: "1" script: - - nix-shell --run "poetry run legacy/script/cibuild" + - cached-nix-shell --run "poetry run legacy/script/cibuild" - mv legacy/firmware/trezor.elf legacy/firmware/trezor-bitcoinonly.elf artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" diff --git a/ci/deploy.yml b/ci/deploy.yml index 669f4c820..205c43bbb 100644 --- a/ci/deploy.yml +++ b/ci/deploy.yml @@ -188,8 +188,8 @@ release core unix debug deploy: - DEST="$DEPLOY_PATH/trezor-emu-core-v$VERSION" - DEST_ARM="$DEPLOY_PATH/arm/trezor-emu-core-v$VERSION-arm" - echo "Deploying to $DEST and $DEST_ARM" - - nix-shell -p patchelf --run "patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 core/build/unix/trezor-emu-core" - - nix-shell -p patchelf --run "patchelf --set-interpreter /lib/ld-linux-aarch64.so.1 core/build/unix/trezor-emu-core-arm" + - cached-nix-shell -p patchelf --run "patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 core/build/unix/trezor-emu-core" + - cached-nix-shell -p patchelf --run "patchelf --set-interpreter /lib/ld-linux-aarch64.so.1 core/build/unix/trezor-emu-core-arm" - rsync --delete -va core/build/unix/trezor-emu-core "$DEST" - rsync --delete -va core/build/unix/trezor-emu-core-arm "$DEST_ARM" only: @@ -213,8 +213,8 @@ release legacy unix debug deploy: - DEST="$DEPLOY_PATH/trezor-emu-legacy-v$VERSION" - DEST_ARM="$DEPLOY_PATH/arm/trezor-emu-legacy-v$VERSION-arm" - echo "Deploying to $DEST and $DEST_ARM" - - nix-shell -p patchelf --run "patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 legacy/firmware/trezor.elf" - - nix-shell -p patchelf --run "patchelf --set-interpreter /lib/ld-linux-aarch64.so.1 legacy/firmware/trezor-arm.elf" + - cached-nix-shell -p patchelf --run "patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 legacy/firmware/trezor.elf" + - cached-nix-shell -p patchelf --run "patchelf --set-interpreter /lib/ld-linux-aarch64.so.1 legacy/firmware/trezor-arm.elf" - rsync --delete -va legacy/firmware/trezor.elf "$DEST" - rsync --delete -va legacy/firmware/trezor-arm.elf "$DEST_ARM" only: diff --git a/ci/posttest.yml b/ci/posttest.yml index ed1ea9c2f..fee09bf2e 100644 --- a/ci/posttest.yml +++ b/ci/posttest.yml @@ -19,7 +19,7 @@ core unix coverage posttest: - core u2f test - core fido2 test script: - - nix-shell --run "poetry run make -C core coverage" + - cached-nix-shell --run "poetry run make -C core coverage" coverage: '/COVERAGE: \d+%/' artifacts: name: core-unix-coverage-posttest @@ -39,7 +39,7 @@ unix ui changes: - core device test - legacy device test script: - - nix-shell --run "poetry run python -m tests.ui_tests.reporting master-diff" + - cached-nix-shell --run "poetry run python -m tests.ui_tests.reporting master-diff" - mv tests/ui_tests/reports/master_diff . artifacts: name: core-unix-ui-changes diff --git a/ci/prebuild.yml b/ci/prebuild.yml index e97215402..8a1e3085f 100644 --- a/ci/prebuild.yml +++ b/ci/prebuild.yml @@ -19,35 +19,35 @@ style prebuild: GIT_SUBMODULE_STRATEGY: "none" <<: *gitlab_caching script: - - nix-shell --run "poetry run make -j1 style_check" + - cached-nix-shell --run "poetry run make -j1 style_check" # Check validity of coin definitions and protobuf files. common prebuild: stage: prebuild <<: *gitlab_caching script: - - nix-shell --run "poetry run make defs_check" + - cached-nix-shell --run "poetry run make defs_check" # Check validity of auto-generated files. gen prebuild: stage: prebuild <<: *gitlab_caching script: - - nix-shell --run "poetry run make -j1 gen_check" + - cached-nix-shell --run "poetry run make -j1 gen_check" # Checking format of .editorconfig files. editor prebuild: stage: prebuild <<: *gitlab_caching script: - - nix-shell --run "make editor_check" + - cached-nix-shell --run "make editor_check" # All .yml/.yaml files are checked for syntax validity and other correctness. yaml prebuild: stage: prebuild <<: *gitlab_caching script: - - nix-shell --run "poetry run make yaml_check" + - cached-nix-shell --run "poetry run make yaml_check" # Checking the format of release commit messages. release commit messages prebuild: @@ -63,7 +63,7 @@ release commit messages prebuild: # We want this to run on gitlab.com/satoshilabs/trezor/trezor-firmware only. - $CI_PROJECT_PATH_SLUG == 'satoshilabs-trezor-trezor-firmware' script: - - nix-shell --run "ci/check_release_commit_messages.sh" + - cached-nix-shell --run "ci/check_release_commit_messages.sh" # Verifying that all commits changing some functionality have a changelog entry # or contain `[no changelog]` in the commit message. @@ -76,4 +76,4 @@ changelog prebuild: GIT_SUBMODULE_STRATEGY: "none" GIT_STRATEGY: clone script: - - nix-shell --run "ci/check_changelog.sh" + - cached-nix-shell --run "ci/check_changelog.sh" diff --git a/ci/shell.nix b/ci/shell.nix index 6f5a3f1d2..d8d3ba691 100644 --- a/ci/shell.nix +++ b/ci/shell.nix @@ -71,6 +71,7 @@ stdenvNoCC.mkDerivation ({ SDL2 SDL2_image bash + cached-nix-shell check curl # for connect tests editorconfig-checker diff --git a/ci/test-hw.yml b/ci/test-hw.yml index c805bb426..c03de27ce 100644 --- a/ci/test-hw.yml +++ b/ci/test-hw.yml @@ -41,9 +41,9 @@ hardware core regular device test: - set -a - source hardware.cfg - set +a - - nix-shell --run "cd ../.. && poetry install" - - nix-shell --run "poetry run python bootstrap.py tt ../../firmware-T2*.bin | ts -s" - - nix-shell --run "poetry run pytest ../../tests/device_tests | ts -s" + - cached-nix-shell --run "cd ../.. && poetry install" + - cached-nix-shell --run "poetry run python bootstrap.py tt ../../firmware-T2*.bin | ts -s" + - cached-nix-shell --run "poetry run pytest ../../tests/device_tests | ts -s" timeout: 6h artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -71,9 +71,9 @@ hardware core btconly device test: - set -a - source hardware.cfg - set +a - - nix-shell --run "cd ../.. && poetry install" - - nix-shell --run "poetry run python bootstrap.py tt ../../firmware-T2*.bin | ts -s" - - nix-shell --run "poetry run pytest ../../tests/device_tests | ts -s" + - cached-nix-shell --run "cd ../.. && poetry install" + - cached-nix-shell --run "poetry run python bootstrap.py tt ../../firmware-T2*.bin | ts -s" + - cached-nix-shell --run "poetry run pytest ../../tests/device_tests | ts -s" timeout: 4h artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -98,9 +98,9 @@ hardware core monero test: - set -a - source hardware.cfg - set +a - - nix-shell --run "cd ../.. && poetry install" - - nix-shell --run "poetry run python bootstrap.py tt ../../firmware-T2*.bin | ts -s" - - nix-shell --arg fullDeps true --run "cd ../../core/tests && ./run_tests_device_emu_monero.sh $TESTOPTS | ts -s" + - cached-nix-shell --run "cd ../.. && poetry install" + - cached-nix-shell --run "poetry run python bootstrap.py tt ../../firmware-T2*.bin | ts -s" + - cached-nix-shell --arg fullDeps true --run "cd ../../core/tests && ./run_tests_device_emu_monero.sh $TESTOPTS | ts -s" timeout: 1h artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -124,7 +124,7 @@ hardware legacy regular device test: - legacy fw regular debug build script: - cd ci/hardware_tests - - nix-shell --run "./t1_hw_test.sh | ts -s" + - cached-nix-shell --run "./t1_hw_test.sh | ts -s" timeout: 1h10m artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -150,7 +150,7 @@ hardware legacy btconly device test: - legacy fw btconly debug build script: - cd ci/hardware_tests - - nix-shell --run "./t1_hw_test.sh | ts -s" + - cached-nix-shell --run "./t1_hw_test.sh | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: diff --git a/ci/test.yml b/ci/test.yml index aa0327467..1bea024c3 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -18,7 +18,7 @@ core unit python test: needs: - core unix regular build script: - - nix-shell --run "poetry run make -C core test | ts -s" + - cached-nix-shell --run "poetry run make -C core test | ts -s" # Rust unit tests. core unit rust test: @@ -27,8 +27,8 @@ core unit rust test: needs: - core unix frozen debug build script: - - nix-shell --run "poetry run make -C core clippy | ts -s" - - nix-shell --run "poetry run make -C core test_rust | ts -s" + - cached-nix-shell --run "poetry run make -C core clippy | ts -s" + - cached-nix-shell --run "poetry run make -C core test_rust | ts -s" core unit asan test: stage: test @@ -43,9 +43,9 @@ core unit asan test: ADDRESS_SANITIZER: "1" LSAN_OPTIONS: "suppressions=../../asan_suppressions.txt" script: - - nix-shell --run "poetry run make -C core test | ts -s" - - nix-shell --run "poetry run make -C core clean build_unix | ts -s" - - nix-shell --run "poetry run make -C core test_rust | ts -s" + - cached-nix-shell --run "poetry run make -C core test | ts -s" + - cached-nix-shell --run "poetry run make -C core clean build_unix | ts -s" + - cached-nix-shell --run "poetry run make -C core test_rust | ts -s" # Device tests for Core. Running device tests and also comparing screens # with the expected UI result. @@ -59,11 +59,11 @@ core device test: variables: TREZOR_PROFILING: 1 script: - - nix-shell --run "poetry run make -C core test_emu_ui | ts -s" + - cached-nix-shell --run "poetry run make -C core test_emu_ui | ts -s" after_script: - mv core/src/.coverage core/.coverage.test_emu - mv tests/ui_tests/reports/test/ test_ui_report - - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py | ts -s" + - cached-nix-shell --run "poetry run python ci/prepare_ui_artifacts.py | ts -s" - diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -90,7 +90,7 @@ core device asan test: variables: PYTEST_TIMEOUT: "600" script: - - nix-shell --run "poetry run make -C core test_emu | ts -s" + - cached-nix-shell --run "poetry run make -C core test_emu | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -108,7 +108,7 @@ core btconly device test: MICROPYTHON: "build/unix/trezor-emu-core-bitcoinonly" TREZOR_PYTEST_SKIP_ALTCOINS: 1 script: - - nix-shell --run "poetry run make -C core test_emu | ts -s" + - cached-nix-shell --run "poetry run make -C core test_emu | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -131,7 +131,7 @@ core btconly device asan test: TREZOR_PYTEST_SKIP_ALTCOINS: 1 PYTEST_TIMEOUT: "600" script: - - nix-shell --run "poetry run make -C core test_emu | ts -s" + - cached-nix-shell --run "poetry run make -C core test_emu | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -148,7 +148,7 @@ core monero test: variables: TREZOR_PROFILING: 1 script: - - nix-shell --arg fullDeps true --run "poetry run make -C core test_emu_monero | ts -s" + - cached-nix-shell --arg fullDeps true --run "poetry run make -C core test_emu_monero | ts -s" - mv core/src/.coverage core/.coverage.test_emu_monero artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -169,7 +169,7 @@ core monero asan test: variables: TREZOR_PROFILING: 1 script: - - nix-shell --arg fullDeps true --run "poetry run make -C core test_emu_monero | ts -s" + - cached-nix-shell --arg fullDeps true --run "poetry run make -C core test_emu_monero | ts -s" - mv core/src/.coverage core/.coverage.test_emu_monero artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -189,8 +189,8 @@ core u2f test: variables: TREZOR_PROFILING: 1 script: - - nix-shell --run "poetry run make -C tests/fido_tests/u2f-tests-hid | ts -s" - - nix-shell --run "poetry run make -C core test_emu_u2f | ts -s" + - cached-nix-shell --run "poetry run make -C tests/fido_tests/u2f-tests-hid | ts -s" + - cached-nix-shell --run "poetry run make -C core test_emu_u2f | ts -s" - mv core/src/.coverage core/.coverage.test_emu_u2f artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -208,8 +208,8 @@ core u2f asan test: only: - schedules # nightly build script: - - nix-shell --run "poetry run make -C tests/fido_tests/u2f-tests-hid | ts -s" - - nix-shell --run "poetry run make -C core test_emu_u2f | ts -s" + - cached-nix-shell --run "poetry run make -C tests/fido_tests/u2f-tests-hid | ts -s" + - cached-nix-shell --run "poetry run make -C core test_emu_u2f | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -227,7 +227,7 @@ core fido2 test: TREZOR_PROFILING: 1 script: - pgrep trezor-emu-core || true - - nix-shell --run "poetry run make -C core test_emu_fido2 | ts -s" + - cached-nix-shell --run "poetry run make -C core test_emu_fido2 | ts -s" - pgrep trezor-emu-core || true - mv core/src/.coverage core/.coverage.test_emu_fido2 artifacts: @@ -250,7 +250,7 @@ core fido2 asan test: - schedules # nightly build script: - pgrep trezor-emu-core || true - - nix-shell --run "poetry run make -C core test_emu_fido2 | ts -s" + - cached-nix-shell --run "poetry run make -C core test_emu_fido2 | ts -s" - pgrep trezor-emu-core || true artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -269,11 +269,11 @@ core click test: variables: TREZOR_PROFILING: 1 script: - - nix-shell --run "poetry run make -C core test_emu_click_ui | ts -s" + - cached-nix-shell --run "poetry run make -C core test_emu_click_ui | ts -s" after_script: - mv core/src/.coverage core/.coverage.test_click - mv tests/ui_tests/reports/test/ test_ui_report - - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py | ts -s" + - cached-nix-shell --run "poetry run python ci/prepare_ui_artifacts.py | ts -s" - diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -297,7 +297,7 @@ core click asan test: only: - schedules # nightly build script: - - nix-shell --run "poetry run make -C core test_emu_click | ts -s" + - cached-nix-shell --run "poetry run make -C core test_emu_click | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -318,8 +318,8 @@ core upgrade test: variables: TREZOR_UPGRADE_TEST: "core" script: - - nix-shell --run "tests/download_emulators.sh" - - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s" + - cached-nix-shell --run "tests/download_emulators.sh" + - cached-nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -339,8 +339,8 @@ core upgrade asan test: variables: TREZOR_UPGRADE_TEST: "core" script: - - nix-shell --run "tests/download_emulators.sh" - - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s" + - cached-nix-shell --run "tests/download_emulators.sh" + - cached-nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -357,7 +357,7 @@ core persistence test: needs: - core unix frozen debug build script: - - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/persistence_tests | ts -s" + - cached-nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/persistence_tests | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -375,7 +375,7 @@ core persistence asan test: only: - schedules # nightly build script: - - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/persistence_tests | ts -s" + - cached-nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/persistence_tests | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -392,8 +392,8 @@ core hwi test: - core unix frozen debug build allow_failure: true script: - - nix-shell --run "git clone https://github.com/bitcoin-core/HWI.git" - - nix-shell --arg fullDeps true --run "cd HWI && poetry install && poetry run ./test/test_trezor.py --model_t ../core/build/unix/trezor-emu-core bitcoind" + - cached-nix-shell --run "git clone https://github.com/bitcoin-core/HWI.git" + - cached-nix-shell --arg fullDeps true --run "cd HWI && poetry install && poetry run ./test/test_trezor.py --model_t ../core/build/unix/trezor-emu-core bitcoind" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -420,8 +420,8 @@ crypto test: - ./crypto/tests/aestst - ./crypto/tests/test_check - ./crypto/tests/test_openssl 1000 - - nix-shell --run "cd crypto && ITERS=10 poetry run pytest --junitxml=tests/junit.xml tests | ts -s" - - nix-shell --run "CK_TIMEOUT_MULTIPLIER=20 valgrind -q --error-exitcode=1 ./crypto/tests/test_check_noasan | ts -s" + - cached-nix-shell --run "cd crypto && ITERS=10 poetry run pytest --junitxml=tests/junit.xml tests | ts -s" + - cached-nix-shell --run "CK_TIMEOUT_MULTIPLIER=20 valgrind -q --error-exitcode=1 ./crypto/tests/test_check_noasan | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -442,10 +442,10 @@ legacy device test: variables: EMULATOR: "1" script: - - nix-shell --run "poetry run make -C legacy test_emu_ui | ts -s" + - cached-nix-shell --run "poetry run make -C legacy test_emu_ui | ts -s" after_script: - mv tests/ui_tests/reports/test/ test_ui_report - - nix-shell --run "poetry run python ci/prepare_ui_artifacts.py | ts -s" + - cached-nix-shell --run "poetry run python ci/prepare_ui_artifacts.py | ts -s" - diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" @@ -471,7 +471,7 @@ legacy asan test: variables: EMULATOR: "1" script: - - nix-shell --run "poetry run make -C legacy test_emu | ts -s" + - cached-nix-shell --run "poetry run make -C legacy test_emu | ts -s" legacy btconly test: stage: test @@ -483,7 +483,7 @@ legacy btconly test: EMULATOR_BINARY: "firmware/trezor-bitcoinonly.elf" TREZOR_PYTEST_SKIP_ALTCOINS: 1 script: - - nix-shell --run "poetry run make -C legacy test_emu | ts -s" + - cached-nix-shell --run "poetry run make -C legacy test_emu | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -505,7 +505,7 @@ legacy btconly asan test: EMULATOR_BINARY: "firmware/trezor-bitcoinonly.elf" TREZOR_PYTEST_SKIP_ALTCOINS: 1 script: - - nix-shell --run "poetry run make -C legacy test_emu | ts -s" + - cached-nix-shell --run "poetry run make -C legacy test_emu | ts -s" legacy upgrade test: @@ -516,8 +516,8 @@ legacy upgrade test: variables: TREZOR_UPGRADE_TEST: "legacy" script: - - nix-shell --run "tests/download_emulators.sh" - - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s" + - cached-nix-shell --run "tests/download_emulators.sh" + - cached-nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -537,8 +537,8 @@ legacy upgrade asan test: variables: TREZOR_UPGRADE_TEST: "legacy" script: - - nix-shell --run "tests/download_emulators.sh" - - nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s" + - cached-nix-shell --run "tests/download_emulators.sh" + - cached-nix-shell --run "poetry run pytest --junitxml=tests/junit.xml tests/upgrade_tests | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -557,8 +557,8 @@ legacy hwi test: EMULATOR: "1" allow_failure: true script: - - nix-shell --run "git clone https://github.com/bitcoin-core/HWI.git" - - nix-shell --arg fullDeps true --run "cd HWI && poetry install && poetry run ./test/test_trezor.py --model_1 ../legacy/firmware/trezor.elf bitcoind" + - cached-nix-shell --run "git clone https://github.com/bitcoin-core/HWI.git" + - cached-nix-shell --arg fullDeps true --run "cd HWI && poetry install && poetry run ./test/test_trezor.py --model_1 ../legacy/firmware/trezor.elf bitcoind" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -584,14 +584,14 @@ python test: # See also: # https://github.com/NixOS/nixpkgs/blob/b00c7c2d1d905eb63c81a0917f1a94b763a7843b/pkgs/development/interpreters/python/cpython/default.nix#L103 # https://github.com/NixOS/nixpkgs/pull/98915 - - nix-shell --arg fullDeps true --run "unset _PYTHON_SYSCONFIGDATA_NAME && cd python && poetry run tox | ts -s" + - cached-nix-shell --arg fullDeps true --run "unset _PYTHON_SYSCONFIGDATA_NAME && cd python && poetry run tox | ts -s" python support test: stage: test <<: *gitlab_caching needs: [] script: - - nix-shell --run "poetry run make python_support_check | ts -s" + - cached-nix-shell --run "poetry run make python_support_check | ts -s" # Storage @@ -607,8 +607,8 @@ storage test: needs: [] script: - unset PYTEST_TIMEOUT - - nix-shell --run "poetry run make -C storage/tests build | ts -s" - - nix-shell --run "poetry run make -C storage/tests tests_all | ts -s" + - cached-nix-shell --run "poetry run make -C storage/tests build | ts -s" + - cached-nix-shell --run "poetry run make -C storage/tests tests_all | ts -s" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -630,10 +630,10 @@ core unix memory profiler: TREZOR_MEMPERF: "1" PYTEST_TIMEOUT: "900" script: - - nix-shell --run "poetry run make -C core build_unix_frozen | ts -s" - - nix-shell --run "poetry run make -C core test_emu | ts -s" - - nix-shell --run "mkdir core/prof/memperf-html" - - nix-shell --run "poetry run core/tools/alloc.py --alloc-data=core/src/alloc_data.txt html core/prof/memperf-html" + - cached-nix-shell --run "poetry run make -C core build_unix_frozen | ts -s" + - cached-nix-shell --run "poetry run make -C core test_emu | ts -s" + - cached-nix-shell --run "mkdir core/prof/memperf-html" + - cached-nix-shell --run "poetry run core/tools/alloc.py --alloc-data=core/src/alloc_data.txt html core/prof/memperf-html" artifacts: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: @@ -657,10 +657,10 @@ connect test core: before_script: - cp /builds/satoshilabs/trezor/trezor-firmware/core/build/unix/trezor-emu-core /trezor-user-env/src/binaries/firmware/bin/trezor-emu-core-v2.99.99 - chmod +x /trezor-user-env/src/binaries/firmware/bin/trezor-emu-core-v2.99.99 - - nix-shell --run "autoPatchelf /trezor-user-env/src/binaries/firmware/bin/trezor-emu-core-v2.99.99" + - cached-nix-shell --run "autoPatchelf /trezor-user-env/src/binaries/firmware/bin/trezor-emu-core-v2.99.99" script: - /trezor-user-env/run-nix.sh & - - nix-shell --run "tests/connect_tests/connect_tests.sh 2.99.99" + - cached-nix-shell --run "tests/connect_tests/connect_tests.sh 2.99.99" after_script: - cp /trezor-user-env/logs/debugging.log trezor-user-env-debugging.log - cp /trezor-user-env/logs/emulator_bridge.log tenv-emulator-bridge-debugging.log