ci: use nix docker image

.gitlab-ci.yml

@@ -33,7 +33,7 @@ stages:
   - deploy
 
 before_script:
-  - command -v pipenv >/dev/null && pipenv sync --python=/usr/bin/python3
+  - nix-shell --run "pipenv sync"
 
 include:
   - ci/environment.yml

build-docker.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-CONTAINER_NAME=trezor-firmware-build.nix
+CONTAINER_NAME=trezor-firmware-env.nix
 
 TAG=${1:-master}
 REPOSITORY=${2:-local}

ci/build.yml

@@ -1,4 +1,4 @@
-image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix
 
 variables:
   SDL_VIDEODRIVER: "dummy"
@@ -9,14 +9,12 @@ variables:
 core fw regular build:
   stage: build
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core build_cross"
-    - pipenv run make build_cross
+    - nix-shell --run "pipenv run make -C core build_boardloader"
-    - pipenv run make build_boardloader
+    - nix-shell --run "pipenv run make -C core build_bootloader"
-    - pipenv run make build_bootloader
+    - nix-shell --run "pipenv run make -C core build_prodtest"
-    - pipenv run make build_prodtest
+    - nix-shell --run "pipenv run make -C core build_firmware"
-    - pipenv run make build_firmware
+    - nix-shell --run "pipenv run make -C core sizecheck"
-    - pipenv run make sizecheck
-    - cd ..
     - export VERSION=$(./tools/version.sh core/embed/firmware/version.h)
     - cp core/build/firmware/firmware.bin trezor-fw-regular-$VERSION-$CI_COMMIT_SHORT_SHA.bin
   artifacts:
@@ -28,9 +26,7 @@ core fw regular build:
 core fw regular debug build:
   stage: build
   script:
-    - cd core
+    - nix-shell --run "PYOPT=0 pipenv run make -C core build_firmware"
-    - PYOPT=0 pipenv run make build_firmware
-    - cd ..
     - export VERSION=$(./tools/version.sh core/embed/firmware/version.h)
     - cp core/build/firmware/firmware.bin trezor-fw-regular-debug-$VERSION-$CI_COMMIT_SHORT_SHA.bin
   artifacts:
@@ -44,11 +40,9 @@ core fw btconly build:
   variables:
     BITCOIN_ONLY: "1"
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core build_firmware"
-    - pipenv run make build_firmware
+    - mv core/build/firmware/firmware.bin core/build/firmware/firmware-bitcoinonly.bin
-    - mv build/firmware/firmware.bin build/firmware/firmware-bitcoinonly.bin
+    - nix-shell --run "pipenv run ./tools/check-bitcoin-only core/build/firmware/firmware-bitcoinonly.bin"
-    - cd ..
-    - pipenv run ./tools/check-bitcoin-only core/build/firmware/firmware-bitcoinonly.bin
     - export VERSION=$(./tools/version.sh core/embed/firmware/version.h)
     - cp core/build/firmware/firmware-bitcoinonly.bin trezor-fw-btconly-$VERSION-$CI_COMMIT_SHORT_SHA.bin
   artifacts:
@@ -60,8 +54,7 @@ core fw btconly build:
 core unix regular build:
   stage: build
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core build_unix"
-    - pipenv run make build_unix
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -72,8 +65,7 @@ core unix regular build:
 core unix frozen regular build:
   stage: build
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core build_unix_frozen"
-    - pipenv run make build_unix_frozen
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -86,9 +78,8 @@ core unix frozen btconly debug build:
     PYOPT: "0"
     BITCOIN_ONLY: "1"
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core build_unix_frozen"
-    - pipenv run make build_unix_frozen
+    - mv core/build/unix/micropython core/build/unix/micropython-bitcoinonly
-    - mv build/unix/micropython build/unix/micropython-bitcoinonly
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -100,8 +91,7 @@ core unix frozen debug build:
   variables:
     PYOPT: "0"
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core build_unix_frozen"
-    - pipenv run make build_unix_frozen
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     untracked: true
@@ -114,7 +104,7 @@ core unix frozen regular darwin:
   when: manual
   script:
     - . $HOME/.nix-profile/etc/profile.d/nix.sh
-    - nix-shell --run "pipenv sync && cd core && pipenv run make build_unix_frozen"
+    - nix-shell --run "pipenv run make -C core build_unix_frozen"
     - mkdir -p TrezorEmu-$CI_COMMIT_SHORT_SHA.app/Contents/{MacOS,libs}
     - cp -v core/build/unix/micropython TrezorEmu-$CI_COMMIT_SHORT_SHA.app/Contents/MacOS/TrezorEmu-$CI_COMMIT_SHORT_SHA
     - dylibbundler -of -b -i /usr/lib/system -d TrezorEmu-$CI_COMMIT_SHORT_SHA.app/Contents/libs -x TrezorEmu-$CI_COMMIT_SHORT_SHA.app/Contents/MacOS/TrezorEmu-$CI_COMMIT_SHORT_SHA
@@ -135,8 +125,7 @@ crypto build:
       - ci/**/*
       - crypto/**/*
   script:
-    - cd crypto
+    - nix-shell --run "pipenv run make -C crypto"
-    - pipenv run make
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -154,10 +143,8 @@ legacy fw regular build:
   variables:
     MEMORY_PROTECT: "0"
   script:
-    - cd legacy
+    - nix-shell --run "pipenv run legacy/script/cibuild"
-    - pipenv run script/cibuild
+    - nix-shell --run "pipenv run make -C legacy/demo"
-    - pipenv run make -C demo
-    - cd ..
     - export VERSION=$(./tools/version.sh legacy/firmware/version.h)
     - mv legacy/firmware/trezor.bin trezor-fw-regular-$VERSION-$CI_COMMIT_SHORT_SHA.bin
   artifacts:
@@ -172,9 +159,7 @@ legacy fw debug build:
     DEBUG_LINK: "1"
     MEMORY_PROTECT: "0"
   script:
-    - cd legacy
+    - nix-shell --run "pipenv run legacy/script/cibuild"
-    - pipenv run script/cibuild
-    - cd ..
     - export VERSION=$(./tools/version.sh legacy/firmware/version.h)
     - mv legacy/firmware/trezor.bin trezor-fw-regular-debug-$VERSION-$CI_COMMIT_SHORT_SHA.bin
   artifacts:
@@ -189,11 +174,9 @@ legacy fw btconly build:
     BITCOIN_ONLY: "1"
     MEMORY_PROTECT: "0"
   script:
-    - cd legacy
+    - nix-shell --run "pipenv run legacy/script/cibuild"
-    - pipenv run script/cibuild
+    - mv legacy/firmware/trezor.bin legacy/firmware/trezor-bitcoinonly.bin
-    - mv firmware/trezor.bin firmware/trezor-bitcoinonly.bin
+    - nix-shell --run "pipenv run ./tools/check-bitcoin-only legacy/firmware/trezor-bitcoinonly.bin"
-    - cd ..
-    - pipenv run ./tools/check-bitcoin-only legacy/firmware/trezor-bitcoinonly.bin
     - export VERSION=$(./tools/version.sh legacy/firmware/version.h)
     - mv legacy/firmware/trezor-bitcoinonly.bin trezor-fw-btconly-$VERSION-$CI_COMMIT_SHORT_SHA.bin
   artifacts:
@@ -208,8 +191,7 @@ legacy emu regular build:
     DEBUG_LINK: "1"
     EMULATOR: "1"
   script:
-    - cd legacy
+    - nix-shell --run "pipenv run legacy/script/cibuild"
-    - pipenv run script/cibuild
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -223,9 +205,8 @@ legacy emu btconly build:
     DEBUG_LINK: "1"
     EMULATOR: "1"
   script:
-    - cd legacy
+    - nix-shell --run "pipenv run legacy/script/cibuild"
-    - pipenv run script/cibuild
+    - mv legacy/firmware/trezor.elf legacy/firmware/trezor-bitcoinonly.elf
-    - mv firmware/trezor.elf firmware/trezor-bitcoinonly.elf
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:

ci/check_release_commit_messages.sh

@@ -11,7 +11,7 @@ do
     # The commit message must contain either
     # 1. "cherry-picked from [some commit in master]"
     if [[ $message =~ "(cherry picked from commit" ]]; then
-    	# remove last ")" and extract commit hash
+      # remove last ")" and extract commit hash
       master_commit=$(echo ${message:0:-1} | tr ' ' '\n' | tail -1)
       # check if master really contains this commit hash
       if [[ $(git branch -a --contains $master_commit | grep --only-matching master) == "master" ]]; then

ci/deploy.yml

@@ -1,4 +1,4 @@
-image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix
 
 # Releases
 

ci/environment.yml

@@ -4,7 +4,7 @@ environment:
   when: manual
   variables:
     GIT_SUBMODULE_STRATEGY: none  # no need to fetch submodules
-    CONTAINER_NAME: "$CI_REGISTRY/satoshilabs/trezor/trezor-firmware/trezor-firmware-build.nix"
+    CONTAINER_NAME: "$CI_REGISTRY/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix"
   services:
     - docker:dind
   before_script:

ci/posttest.yml

@@ -1,4 +1,4 @@
-image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix
 
 core unix coverage posttest:
   stage: posttest
@@ -8,9 +8,8 @@ core unix coverage posttest:
     - core u2f test
     - core fido2 test
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core res"  # we need to regenerate resources.py
-    - pipenv run make res  # we need to regenerate resources.py
+    - nix-shell --run "pipenv run make -C core coverage"
-    - pipenv run make coverage
   coverage: '/>\d+%</'
   artifacts:
     name: core-unix-coverage-posttest
@@ -26,9 +25,8 @@ core unix ui changes:
   dependencies:
     - core device ui test
   script:
-    - cd tests/ui_tests
+    - nix-shell --run "cd tests/ui_tests ; pipenv run python reporting/report_master_diff.py"
-    - pipenv run python reporting/report_master_diff.py
+    - mv tests/ui_tests/reporting/reports/master_diff/ .
-    - mv reporting/reports/master_diff/ ../..
   artifacts:
     name: core-unix-ui-changes
     paths:

ci/prebuild.yml

@@ -1,4 +1,4 @@
-image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix
 
 style prebuild:
   stage: prebuild
@@ -9,7 +9,7 @@ style prebuild:
       - "**/*.h"
       - "**/*.py"
   script:
-    - pipenv run make style_check
+    - nix-shell --run "pipenv run make style_check"
 
 common prebuild:
   stage: prebuild
@@ -19,7 +19,7 @@ common prebuild:
       - "**/*.json"
       - "**/*.proto"
   script:
-    - pipenv run make defs_check
+    - nix-shell --run "pipenv run make defs_check"
 
 gen prebuild:
   stage: prebuild
@@ -33,11 +33,11 @@ gen prebuild:
       - "**/*.py"
       - "**/*.pyi"
   script:
-    - pipenv run make gen_check
+    - nix-shell --run "pipenv run make gen_check"
 
 release commit messages prebuild:
   stage: prebuild
   only:
     - /^release\//
   script:
-    - ci/check_release_commit_messages.sh
+    - nix-shell --run "ci/check_release_commit_messages.sh"

ci/test.yml

@@ -1,4 +1,4 @@
-image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix
 
 # Core
 
@@ -7,20 +7,17 @@ core unit test:
   dependencies:
     - core unix regular build
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core test"
-    - pipenv run make test
 
 core device ui test:
   stage: test
   dependencies:
     - core unix frozen debug build
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core test_emu_ui"
-    - pipenv run make test_emu_ui
-    - cd ..
   after_script:
     - mv tests/ui_tests/reporting/reports/test/ test_ui_report
-    - pipenv run python ci/prepare_ui_artifacts.py
+    - nix-shell --run "pipenv run python ci/prepare_ui_artifacts.py"
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -41,9 +38,8 @@ core device test:
   variables:
     TREZOR_PROFILING: 1
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core test_emu"
-    - pipenv run make test_emu
+    - mv ./core/src/.coverage .coverage.test_emu
-    - mv ./src/.coverage .coverage.test_emu
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -63,8 +59,7 @@ core btconly device test:
     MICROPYTHON: "build/unix/micropython-bitcoinonly"
     TREZOR_PYTEST_SKIP_ALTCOINS: 1
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core test_emu"
-    - pipenv run make test_emu
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -82,9 +77,8 @@ core monero test:
   variables:
     TREZOR_PROFILING: 1
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core test_emu_monero"
-    - pipenv run make test_emu_monero
+    - mv ./core/src/.coverage .coverage.test_emu_monero
-    - mv ./src/.coverage .coverage.test_emu_monero
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -100,10 +94,9 @@ core u2f test:
   variables:
     TREZOR_PROFILING: 1
   script:
-    - make -C tests/fido_tests/u2f-tests-hid
+    - nix-shell --run "pipenv run make -C tests/fido_tests/u2f-tests-hid"
-    - cd core
+    - nix-shell --run "pipenv run make -C core test_emu_u2f"
-    - pipenv run make test_emu_u2f
+    - mv ./core/src/.coverage .coverage.test_emu_u2f
-    - mv ./src/.coverage .coverage.test_emu_u2f
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -119,12 +112,10 @@ core fido2 test:
   variables:
     TREZOR_PROFILING: 1
   script:
-    - cd core
-    - apt-get -y install procps
     - pgrep micropython || true
-    - pipenv run make test_emu_fido2
+    - nix-shell --run "pipenv run make -C core test_emu_fido2"
     - pgrep micropython || true
-    - mv ./src/.coverage .coverage.test_emu_fido2
+    - mv ./core/src/.coverage .coverage.test_emu_fido2
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -141,8 +132,7 @@ core click test:
   dependencies:
     - core unix frozen debug build
   script:
-    - cd core
+    - nix-shell --run "pipenv run make -C core test_emu_click"
-    - pipenv run make test_emu_click
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -160,8 +150,8 @@ core upgrade test:
   variables:
     TREZOR_UPGRADE_TEST: "core"
   script:
-    - tests/download_emulators.sh
+    - nix-shell --run "tests/download_emulators.sh"
-    - pipenv run pytest --junitxml=tests/junit.xml tests/upgrade_tests
+    - nix-shell --run "pipenv run pytest --junitxml=tests/junit.xml tests/upgrade_tests"
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -176,7 +166,7 @@ core persistence test:
   dependencies:
     - core unix frozen debug build
   script:
-    - pipenv run pytest --junitxml=tests/junit.xml tests/persistence_tests
+    - nix-shell --run "pipenv run pytest --junitxml=tests/junit.xml tests/persistence_tests"
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -198,12 +188,11 @@ crypto test:
   dependencies:
     - crypto build
   script:
-    - cd crypto
+    - ./crypto/tests/aestst
-    - ./tests/aestst
+    - ./crypto/tests/test_check
-    - ./tests/test_check
+    - ./crypto/tests/test_openssl 1000
-    - ./tests/test_openssl 1000
+    - nix-shell --run "cd crypto && ITERS=10 pipenv run pytest --junitxml=tests/junit.xml tests"
-    - ITERS=10 pipenv run pytest --junitxml=tests/junit.xml tests
+    - nix-shell --run "CK_TIMEOUT_MULTIPLIER=20 valgrind -q --error-exitcode=1 ./crypto/tests/test_check"
-    - CK_TIMEOUT_MULTIPLIER=20 valgrind -q --error-exitcode=1 ./tests/test_check
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -223,8 +212,7 @@ legacy test:
   variables:
     EMULATOR: "1"
   script:
-    - cd legacy
+    - nix-shell --run "pipenv run legacy/script/test"
-    - pipenv run script/test
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -243,8 +231,7 @@ legacy btconly test:
     EMULATOR_BINARY: "firmware/trezor-bitcoinonly.elf"
     TREZOR_PYTEST_SKIP_ALTCOINS: 1
   script:
-    - cd legacy
+    - nix-shell --run "pipenv run legacy/script/test"
-    - pipenv run script/test
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -262,8 +249,8 @@ legacy upgrade test:
   variables:
     TREZOR_UPGRADE_TEST: "legacy"
   script:
-    - tests/download_emulators.sh
+    - nix-shell --run "tests/download_emulators.sh"
-    - pipenv run pytest --junitxml=tests/junit.xml tests/upgrade_tests
+    - nix-shell --run "pipenv run pytest --junitxml=tests/junit.xml tests/upgrade_tests"
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths:
@@ -285,8 +272,7 @@ python test:
       - common/**/*
       - python/**/*
   script:
-    - cd python
+    - nix-shell --run "cd python && pipenv run tox"
-    - pipenv run tox
 
 
 # Storage
@@ -299,9 +285,8 @@ storage test:
       - storage/**/*
   dependencies: []
   script:
-    - cd storage/tests
+    - nix-shell --run "pipenv run make -C storage/tests build"
-    - pipenv run make build
+    - nix-shell --run "pipenv run make -C storage/tests tests_all"
-    - pipenv run make tests_all
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
     paths: