ci: handle full dependencies in shell.nix

Handling of full dependencies (multiple python versions, Monero tests)
was moved from Dockerfile to shell.nix.

The Python packages are installed from the pinned nixpkgs revision and
do not depend on channel state at the time of docker build anymore.

The Monero test binary is now downloaded using fetchurl which fails the
build if checksum doesn't match.

ci/Dockerfile

@@ -37,36 +37,11 @@ ENV \
 
 COPY shell.nix shell.nix
 
-RUN nix-shell --run "echo deps pre-installed"
-
-CMD [ "nix-shell" ]
-
-# the rest of the file only applies when docker build is called
+# to make multiple python versions and monero test suite available, run docker build
 # with the following argument: "--build-arg FULLDEPS_TESTING=1"
-
 ARG FULLDEPS_TESTING=0
 ENV FULLDEPS_TESTING=${FULLDEPS_TESTING}
 
-# install other python versions for tox testing
-# 3.8 is already included in the default install
+RUN nix-shell --arg fullDeps "$([ ${FULLDEPS_TESTING} = 1 ] && echo true || echo false)" --run "echo deps pre-installed"
 
-RUN if [ "${FULLDEPS_TESTING}" = "1" ]; then \
-        nix-env --preserve-installed -iA nixpkgs.python36 ; \
-        nix-env --set-flag priority 8 $(nix-env -q python3 | grep 'python3-3\.6\.') ; \
-        nix-env --preserve-installed -iA nixpkgs.python37 ; \
-        nix-env --set-flag priority 7 $(nix-env -q python3 | grep 'python3-3\.7\.') ; \
-        nix-env --preserve-installed -iA nixpkgs.python39 ; \
-        nix-env --set-flag priority 6 $(nix-env -q python3 | grep 'python3-3\.9\.') ; \
-    fi
-
-# download monero tests binary
-
-ENV TREZOR_MONERO_TESTS_PATH="/opt/trezor_monero_tests"
-RUN if [ "${FULLDEPS_TESTING}" = "1" ]; then \
-        TREZOR_MONERO_TESTS_SHA256SUM=1e5dfdb07de4ea46088f4a5bdb0d51f040fe479019efae30f76427eee6edb3f7 ; \
-        TREZOR_MONERO_TESTS_URL="https://github.com/ph4r05/monero/releases/download/v0.15.0.0-tests-u18.04-03/trezor_tests" ; \
-        wget --no-verbose "${TREZOR_MONERO_TESTS_URL}" -O "${TREZOR_MONERO_TESTS_PATH}" ; \
-        chmod +x "${TREZOR_MONERO_TESTS_PATH}" ; \
-        echo "${TREZOR_MONERO_TESTS_SHA256SUM} ${TREZOR_MONERO_TESTS_PATH}" | sha256sum -c ; \
-        nix-shell -p patchelf --run 'patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "${TREZOR_MONERO_TESTS_PATH}"' ; \
-    fi
+CMD [ "nix-shell" ]

ci/shell.nix

@@ -1,3 +1,5 @@
+{ fullDeps ? false }:
+
 # the last successful build of nixos-20.09 (stable) as of 2020-10-11
 with import
   (builtins.fetchTarball {
@@ -6,9 +8,29 @@ with import
   })
 { };
 
-stdenv.mkDerivation {
+let
+  moneroTests = fetchurl {
+    url = "https://github.com/ph4r05/monero/releases/download/v0.15.0.0-tests-u18.04-03/trezor_tests";
+    sha256 = "1e5dfdb07de4ea46088f4a5bdb0d51f040fe479019efae30f76427eee6edb3f7";
+  };
+  moneroTestsPatched = runCommandCC "monero_trezor_tests" {} ''
+    cp ${moneroTests} $out
+    chmod +wx $out
+    ${patchelf}/bin/patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$out"
+    chmod -w $out
+  '';
+in
+stdenv.mkDerivation ({
   name = "trezor-firmware-env";
-  buildInputs = [
+  buildInputs = stdenv.lib.optionals fullDeps [
+    # install other python versions for tox testing
+    # NOTE: running e.g. "python3" in the shell runs the first version in the following list,
+    #       and poetry uses the default version (currently 3.8)
+    python38
+    python39
+    python37
+    python36
+  ] ++ [
     SDL2
     SDL2_image
     autoflake
@@ -51,4 +73,7 @@ stdenv.mkDerivation {
 
   # Fix bdist-wheel problem by setting source date epoch to a more recent date
   SOURCE_DATE_EPOCH = 1600000000;
-}
+
+} // (stdenv.lib.optionalAttrs fullDeps) {
+  TREZOR_MONERO_TESTS_PATH = moneroTestsPatched;
+})

ci/test.yml

@@ -79,7 +79,7 @@ core monero test:
   variables:
     TREZOR_PROFILING: 1
   script:
-    - nix-shell --run "poetry run make -C core test_emu_monero"
+    - nix-shell --arg fullDeps true --run "poetry run make -C core test_emu_monero"
     - mv core/src/.coverage core/.coverage.test_emu_monero
   artifacts:
     name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
@@ -280,7 +280,7 @@ python test:
       - common/**/*
       - python/**/*
   script:
-    - nix-shell --run "cd python && poetry run tox"
+    - nix-shell --arg fullDeps true --run "cd python && poetry run tox"
 
 
 # Storage

core/tests/run_tests_device_emu_monero.sh

@@ -34,6 +34,7 @@ if [[ "$OSTYPE" != "linux-gnu" && "$OSTYPE" != "darwin"* ]]; then
   exit 0
 fi
 
+# When updating URL and sha256sum also update the URL in ci/shell.nix.
 error=1
 : "${TREZOR_MONERO_TESTS_URL:=https://github.com/ph4r05/monero/releases/download/v0.15.0.0-tests-u18.04-03/trezor_tests}"
 : "${TREZOR_MONERO_TESTS_SHA256SUM:=1e5dfdb07de4ea46088f4a5bdb0d51f040fe479019efae30f76427eee6edb3f7}"