ci: use nix docker image

4 years ago · 2b4de8e270
parent 8f6a6c098e
commit 2b4de8e270
9 changed files with 72 additions and 108 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -33,7 +33,7 @@ stages:
  - deploy

 before_script:
-  - command -v pipenv >/dev/null && pipenv sync --python=/usr/bin/python3
+  - nix-shell --run "pipenv sync"

 include:
  - ci/environment.yml
--- a/build-docker.sh
+++ b/build-docker.sh
@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e

-CONTAINER_NAME=trezor-firmware-build.nix
+CONTAINER_NAME=trezor-firmware-env.nix

 TAG=${1:-master}
 REPOSITORY=${2:-local}
--- a/ci/build.yml
+++ b/ci/build.yml
@ -1,4 +1,4 @@
-image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix

 variables:
  SDL_VIDEODRIVER: "dummy"
@ -9,14 +9,12 @@ variables:
 core fw regular build:
  stage: build
  script:
-    - cd core
-    - pipenv run make build_cross
-    - pipenv run make build_boardloader
-    - pipenv run make build_bootloader
-    - pipenv run make build_prodtest
-    - pipenv run make build_firmware
-    - pipenv run make sizecheck
-    - cd ..
+    - nix-shell --run "pipenv run make -C core build_cross"
+    - nix-shell --run "pipenv run make -C core build_boardloader"
+    - nix-shell --run "pipenv run make -C core build_bootloader"
+    - nix-shell --run "pipenv run make -C core build_prodtest"
+    - nix-shell --run "pipenv run make -C core build_firmware"
+    - nix-shell --run "pipenv run make -C core sizecheck"
    - export VERSION=$(./tools/version.sh core/embed/firmware/version.h)
    - cp core/build/firmware/firmware.bin trezor-fw-regular-$VERSION-$CI_COMMIT_SHORT_SHA.bin
  artifacts:
@ -28,9 +26,7 @@ core fw regular build:
 core fw regular debug build:
  stage: build
  script:
-    - cd core
-    - PYOPT=0 pipenv run make build_firmware
-    - cd ..
+    - nix-shell --run "PYOPT=0 pipenv run make -C core build_firmware"
    - export VERSION=$(./tools/version.sh core/embed/firmware/version.h)
    - cp core/build/firmware/firmware.bin trezor-fw-regular-debug-$VERSION-$CI_COMMIT_SHORT_SHA.bin
  artifacts:
@ -44,11 +40,9 @@ core fw btconly build:
  variables:
    BITCOIN_ONLY: "1"
  script:
-    - cd core
-    - pipenv run make build_firmware
-    - mv build/firmware/firmware.bin build/firmware/firmware-bitcoinonly.bin
-    - cd ..
-    - pipenv run ./tools/check-bitcoin-only core/build/firmware/firmware-bitcoinonly.bin
+    - nix-shell --run "pipenv run make -C core build_firmware"
+    - mv core/build/firmware/firmware.bin core/build/firmware/firmware-bitcoinonly.bin
+    - nix-shell --run "pipenv run ./tools/check-bitcoin-only core/build/firmware/firmware-bitcoinonly.bin"
    - export VERSION=$(./tools/version.sh core/embed/firmware/version.h)
    - cp core/build/firmware/firmware-bitcoinonly.bin trezor-fw-btconly-$VERSION-$CI_COMMIT_SHORT_SHA.bin
  artifacts:
@ -60,8 +54,7 @@ core fw btconly build:
 core unix regular build:
  stage: build
  script:
-    - cd core
-    - pipenv run make build_unix
+    - nix-shell --run "pipenv run make -C core build_unix"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -72,8 +65,7 @@ core unix regular build:
 core unix frozen regular build:
  stage: build
  script:
-    - cd core
-    - pipenv run make build_unix_frozen
+    - nix-shell --run "pipenv run make -C core build_unix_frozen"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -86,9 +78,8 @@ core unix frozen btconly debug build:
    PYOPT: "0"
    BITCOIN_ONLY: "1"
  script:
-    - cd core
-    - pipenv run make build_unix_frozen
-    - mv build/unix/micropython build/unix/micropython-bitcoinonly
+    - nix-shell --run "pipenv run make -C core build_unix_frozen"
+    - mv core/build/unix/micropython core/build/unix/micropython-bitcoinonly
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -100,8 +91,7 @@ core unix frozen debug build:
  variables:
    PYOPT: "0"
  script:
-    - cd core
-    - pipenv run make build_unix_frozen
+    - nix-shell --run "pipenv run make -C core build_unix_frozen"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    untracked: true
@ -114,7 +104,7 @@ core unix frozen regular darwin:
  when: manual
  script:
    - . $HOME/.nix-profile/etc/profile.d/nix.sh
-    - nix-shell --run "pipenv sync && cd core && pipenv run make build_unix_frozen"
+    - nix-shell --run "pipenv run make -C core build_unix_frozen"
    - mkdir -p TrezorEmu-$CI_COMMIT_SHORT_SHA.app/Contents/{MacOS,libs}
    - cp -v core/build/unix/micropython TrezorEmu-$CI_COMMIT_SHORT_SHA.app/Contents/MacOS/TrezorEmu-$CI_COMMIT_SHORT_SHA
    - dylibbundler -of -b -i /usr/lib/system -d TrezorEmu-$CI_COMMIT_SHORT_SHA.app/Contents/libs -x TrezorEmu-$CI_COMMIT_SHORT_SHA.app/Contents/MacOS/TrezorEmu-$CI_COMMIT_SHORT_SHA
@ -135,8 +125,7 @@ crypto build:
      - ci/**/*
      - crypto/**/*
  script:
-    - cd crypto
-    - pipenv run make
+    - nix-shell --run "pipenv run make -C crypto"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -154,10 +143,8 @@ legacy fw regular build:
  variables:
    MEMORY_PROTECT: "0"
  script:
-    - cd legacy
-    - pipenv run script/cibuild
-    - pipenv run make -C demo
-    - cd ..
+    - nix-shell --run "pipenv run legacy/script/cibuild"
+    - nix-shell --run "pipenv run make -C legacy/demo"
    - export VERSION=$(./tools/version.sh legacy/firmware/version.h)
    - mv legacy/firmware/trezor.bin trezor-fw-regular-$VERSION-$CI_COMMIT_SHORT_SHA.bin
  artifacts:
@ -172,9 +159,7 @@ legacy fw debug build:
    DEBUG_LINK: "1"
    MEMORY_PROTECT: "0"
  script:
-    - cd legacy
-    - pipenv run script/cibuild
-    - cd ..
+    - nix-shell --run "pipenv run legacy/script/cibuild"
    - export VERSION=$(./tools/version.sh legacy/firmware/version.h)
    - mv legacy/firmware/trezor.bin trezor-fw-regular-debug-$VERSION-$CI_COMMIT_SHORT_SHA.bin
  artifacts:
@ -189,11 +174,9 @@ legacy fw btconly build:
    BITCOIN_ONLY: "1"
    MEMORY_PROTECT: "0"
  script:
-    - cd legacy
-    - pipenv run script/cibuild
-    - mv firmware/trezor.bin firmware/trezor-bitcoinonly.bin
-    - cd ..
-    - pipenv run ./tools/check-bitcoin-only legacy/firmware/trezor-bitcoinonly.bin
+    - nix-shell --run "pipenv run legacy/script/cibuild"
+    - mv legacy/firmware/trezor.bin legacy/firmware/trezor-bitcoinonly.bin
+    - nix-shell --run "pipenv run ./tools/check-bitcoin-only legacy/firmware/trezor-bitcoinonly.bin"
    - export VERSION=$(./tools/version.sh legacy/firmware/version.h)
    - mv legacy/firmware/trezor-bitcoinonly.bin trezor-fw-btconly-$VERSION-$CI_COMMIT_SHORT_SHA.bin
  artifacts:
@ -208,8 +191,7 @@ legacy emu regular build:
    DEBUG_LINK: "1"
    EMULATOR: "1"
  script:
-    - cd legacy
-    - pipenv run script/cibuild
+    - nix-shell --run "pipenv run legacy/script/cibuild"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -223,9 +205,8 @@ legacy emu btconly build:
    DEBUG_LINK: "1"
    EMULATOR: "1"
  script:
-    - cd legacy
-    - pipenv run script/cibuild
-    - mv firmware/trezor.elf firmware/trezor-bitcoinonly.elf
+    - nix-shell --run "pipenv run legacy/script/cibuild"
+    - mv legacy/firmware/trezor.elf legacy/firmware/trezor-bitcoinonly.elf
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
--- a/ci/check_release_commit_messages.sh
+++ b/ci/check_release_commit_messages.sh
@ -11,7 +11,7 @@ do
    # The commit message must contain either
    # 1. "cherry-picked from [some commit in master]"
    if [[ $message =~ "(cherry picked from commit" ]]; then
-    	# remove last ")" and extract commit hash
+      # remove last ")" and extract commit hash
      master_commit=$(echo ${message:0:-1} | tr ' ' '\n' | tail -1)
      # check if master really contains this commit hash
      if [[ $(git branch -a --contains $master_commit | grep --only-matching master) == "master" ]]; then
--- a/ci/deploy.yml
+++ b/ci/deploy.yml
@ -1,4 +1,4 @@
-image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix

 # Releases

--- a/ci/environment.yml
+++ b/ci/environment.yml
@ -4,7 +4,7 @@ environment:
  when: manual
  variables:
    GIT_SUBMODULE_STRATEGY: none  # no need to fetch submodules
-    CONTAINER_NAME: "$CI_REGISTRY/satoshilabs/trezor/trezor-firmware/trezor-firmware-build.nix"
+    CONTAINER_NAME: "$CI_REGISTRY/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix"
  services:
    - docker:dind
  before_script:
--- a/ci/posttest.yml
+++ b/ci/posttest.yml
@ -1,4 +1,4 @@
-image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix

 core unix coverage posttest:
  stage: posttest
@ -8,9 +8,8 @@ core unix coverage posttest:
    - core u2f test
    - core fido2 test
  script:
-    - cd core
-    - pipenv run make res  # we need to regenerate resources.py
-    - pipenv run make coverage
+    - nix-shell --run "pipenv run make -C core res"  # we need to regenerate resources.py
+    - nix-shell --run "pipenv run make -C core coverage"
  coverage: '/>\d+%</'
  artifacts:
    name: core-unix-coverage-posttest
@ -26,9 +25,8 @@ core unix ui changes:
  dependencies:
    - core device ui test
  script:
-    - cd tests/ui_tests
-    - pipenv run python reporting/report_master_diff.py
-    - mv reporting/reports/master_diff/ ../..
+    - nix-shell --run "cd tests/ui_tests ; pipenv run python reporting/report_master_diff.py"
+    - mv tests/ui_tests/reporting/reports/master_diff/ .
  artifacts:
    name: core-unix-ui-changes
    paths:
--- a/ci/prebuild.yml
+++ b/ci/prebuild.yml
@ -1,4 +1,4 @@
-image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix

 style prebuild:
  stage: prebuild
@ -9,7 +9,7 @@ style prebuild:
      - "**/*.h"
      - "**/*.py"
  script:
-    - pipenv run make style_check
+    - nix-shell --run "pipenv run make style_check"

 common prebuild:
  stage: prebuild
@ -19,7 +19,7 @@ common prebuild:
      - "**/*.json"
      - "**/*.proto"
  script:
-    - pipenv run make defs_check
+    - nix-shell --run "pipenv run make defs_check"

 gen prebuild:
  stage: prebuild
@ -33,11 +33,11 @@ gen prebuild:
      - "**/*.py"
      - "**/*.pyi"
  script:
-    - pipenv run make gen_check
+    - nix-shell --run "pipenv run make gen_check"

 release commit messages prebuild:
  stage: prebuild
  only:
    - /^release\//
  script:
-    - ci/check_release_commit_messages.sh
+    - nix-shell --run "ci/check_release_commit_messages.sh"
--- a/ci/test.yml
+++ b/ci/test.yml
@ -1,4 +1,4 @@
-image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix

 # Core

@ -7,20 +7,17 @@ core unit test:
  dependencies:
    - core unix regular build
  script:
-    - cd core
-    - pipenv run make test
+    - nix-shell --run "pipenv run make -C core test"

 core device ui test:
  stage: test
  dependencies:
    - core unix frozen debug build
  script:
-    - cd core
-    - pipenv run make test_emu_ui
-    - cd ..
+    - nix-shell --run "pipenv run make -C core test_emu_ui"
  after_script:
    - mv tests/ui_tests/reporting/reports/test/ test_ui_report
-    - pipenv run python ci/prepare_ui_artifacts.py
+    - nix-shell --run "pipenv run python ci/prepare_ui_artifacts.py"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -41,9 +38,8 @@ core device test:
  variables:
    TREZOR_PROFILING: 1
  script:
-    - cd core
-    - pipenv run make test_emu
-    - mv ./src/.coverage .coverage.test_emu
+    - nix-shell --run "pipenv run make -C core test_emu"
+    - mv ./core/src/.coverage .coverage.test_emu
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -63,8 +59,7 @@ core btconly device test:
    MICROPYTHON: "build/unix/micropython-bitcoinonly"
    TREZOR_PYTEST_SKIP_ALTCOINS: 1
  script:
-    - cd core
-    - pipenv run make test_emu
+    - nix-shell --run "pipenv run make -C core test_emu"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -82,9 +77,8 @@ core monero test:
  variables:
    TREZOR_PROFILING: 1
  script:
-    - cd core
-    - pipenv run make test_emu_monero
-    - mv ./src/.coverage .coverage.test_emu_monero
+    - nix-shell --run "pipenv run make -C core test_emu_monero"
+    - mv ./core/src/.coverage .coverage.test_emu_monero
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -100,10 +94,9 @@ core u2f test:
  variables:
    TREZOR_PROFILING: 1
  script:
-    - make -C tests/fido_tests/u2f-tests-hid
-    - cd core
-    - pipenv run make test_emu_u2f
-    - mv ./src/.coverage .coverage.test_emu_u2f
+    - nix-shell --run "pipenv run make -C tests/fido_tests/u2f-tests-hid"
+    - nix-shell --run "pipenv run make -C core test_emu_u2f"
+    - mv ./core/src/.coverage .coverage.test_emu_u2f
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -119,12 +112,10 @@ core fido2 test:
  variables:
    TREZOR_PROFILING: 1
  script:
-    - cd core
-    - apt-get -y install procps
    - pgrep micropython || true
-    - pipenv run make test_emu_fido2
+    - nix-shell --run "pipenv run make -C core test_emu_fido2"
    - pgrep micropython || true
-    - mv ./src/.coverage .coverage.test_emu_fido2
+    - mv ./core/src/.coverage .coverage.test_emu_fido2
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -141,8 +132,7 @@ core click test:
  dependencies:
    - core unix frozen debug build
  script:
-    - cd core
-    - pipenv run make test_emu_click
+    - nix-shell --run "pipenv run make -C core test_emu_click"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -160,8 +150,8 @@ core upgrade test:
  variables:
    TREZOR_UPGRADE_TEST: "core"
  script:
-    - tests/download_emulators.sh
-    - pipenv run pytest --junitxml=tests/junit.xml tests/upgrade_tests
+    - nix-shell --run "tests/download_emulators.sh"
+    - nix-shell --run "pipenv run pytest --junitxml=tests/junit.xml tests/upgrade_tests"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -176,7 +166,7 @@ core persistence test:
  dependencies:
    - core unix frozen debug build
  script:
-    - pipenv run pytest --junitxml=tests/junit.xml tests/persistence_tests
+    - nix-shell --run "pipenv run pytest --junitxml=tests/junit.xml tests/persistence_tests"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -198,12 +188,11 @@ crypto test:
  dependencies:
    - crypto build
  script:
-    - cd crypto
-    - ./tests/aestst
-    - ./tests/test_check
-    - ./tests/test_openssl 1000
-    - ITERS=10 pipenv run pytest --junitxml=tests/junit.xml tests
-    - CK_TIMEOUT_MULTIPLIER=20 valgrind -q --error-exitcode=1 ./tests/test_check
+    - ./crypto/tests/aestst
+    - ./crypto/tests/test_check
+    - ./crypto/tests/test_openssl 1000
+    - nix-shell --run "cd crypto && ITERS=10 pipenv run pytest --junitxml=tests/junit.xml tests"
+    - nix-shell --run "CK_TIMEOUT_MULTIPLIER=20 valgrind -q --error-exitcode=1 ./crypto/tests/test_check"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -223,8 +212,7 @@ legacy test:
  variables:
    EMULATOR: "1"
  script:
-    - cd legacy
-    - pipenv run script/test
+    - nix-shell --run "pipenv run legacy/script/test"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -243,8 +231,7 @@ legacy btconly test:
    EMULATOR_BINARY: "firmware/trezor-bitcoinonly.elf"
    TREZOR_PYTEST_SKIP_ALTCOINS: 1
  script:
-    - cd legacy
-    - pipenv run script/test
+    - nix-shell --run "pipenv run legacy/script/test"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -262,8 +249,8 @@ legacy upgrade test:
  variables:
    TREZOR_UPGRADE_TEST: "legacy"
  script:
-    - tests/download_emulators.sh
-    - pipenv run pytest --junitxml=tests/junit.xml tests/upgrade_tests
+    - nix-shell --run "tests/download_emulators.sh"
+    - nix-shell --run "pipenv run pytest --junitxml=tests/junit.xml tests/upgrade_tests"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths:
@ -285,8 +272,7 @@ python test:
      - common/**/*
      - python/**/*
  script:
-    - cd python
-    - pipenv run tox
+    - nix-shell --run "cd python && pipenv run tox"


 # Storage
@ -299,9 +285,8 @@ storage test:
      - storage/**/*
  dependencies: []
  script:
-    - cd storage/tests
-    - pipenv run make build
-    - pipenv run make tests_all
+    - nix-shell --run "pipenv run make -C storage/tests build"
+    - nix-shell --run "pipenv run make -C storage/tests tests_all"
  artifacts:
    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
    paths: