From 1d68c9b38646de2054c77439b96a03a47d3addfa Mon Sep 17 00:00:00 2001 From: Pavol Rusnak Date: Sun, 3 May 2020 19:00:57 +0200 Subject: [PATCH] ci: use NixOS in Docker --- build-docker.sh | 15 ++--- ci/Dockerfile | 143 +++++++----------------------------------------- ci/shell.nix | 29 ++++++++++ shell.nix | 6 +- 4 files changed, 55 insertions(+), 138 deletions(-) create mode 100644 ci/shell.nix diff --git a/build-docker.sh b/build-docker.sh index 544a77206..e81df8a2b 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -1,14 +1,7 @@ #!/usr/bin/env bash set -e -if [ "$1" = "--gcc_source" ]; then - TOOLCHAIN_FLAVOR=src - shift -else - TOOLCHAIN_FLAVOR=x86_64-linux -fi - -IMAGE=trezor-firmware-build.$TOOLCHAIN_FLAVOR +IMAGE=trezor-firmware-build.nixos TAG=${1:-master} REPOSITORY=${2:-local} @@ -21,7 +14,7 @@ else REPOSITORY=https://github.com/$REPOSITORY/trezor-firmware.git fi -docker build -t "$IMAGE" --build-arg TOOLCHAIN_FLAVOR=$TOOLCHAIN_FLAVOR ci/ +docker build -t "$IMAGE" ci/ USER=$(ls -lnd . | awk '{ print $3 }') GROUP=$(ls -lnd . | awk '{ print $4 }') @@ -43,7 +36,7 @@ for BITCOIN_ONLY in 0 1; do --env PRODUCTION="$PRODUCTION" \ --user="$USER:$GROUP" \ "$IMAGE" \ - /bin/sh -c "\ + /nix/var/nix/profiles/default/bin/nix-shell --run "\ cd /tmp && \ git clone $REPOSITORY trezor-firmware && \ cd trezor-firmware/core && \ @@ -69,7 +62,7 @@ for BITCOIN_ONLY in 0 1; do --env MEMORY_PROTECT="$MEMORY_PROTECT" \ --user="$USER:$GROUP" \ "$IMAGE" \ - /bin/sh -c "\ + /nix/var/nix/profiles/default/bin/nix-shell --run "\ cd /tmp && \ git clone $REPOSITORY trezor-firmware && \ cd trezor-firmware/legacy && \ diff --git a/ci/Dockerfile b/ci/Dockerfile index 454e99c94..1119f616d 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -1,142 +1,37 @@ -# initialize from the image +FROM nixos/nix:2.3.4 -FROM debian:10 +COPY shell.nix shell.nix -ARG TOOLCHAIN_FLAVOR=x86_64-linux -ENV TOOLCHAIN_FLAVOR=${TOOLCHAIN_FLAVOR} +RUN nix-env -i -f shell.nix -A buildInputs -ARG FULLDEPS_TESTING=0 -ENV FULLDEPS_TESTING=${FULLDEPS_TESTING} - -# install build tools and dependencies - -RUN apt-get update && apt-get install -y \ - build-essential \ - check \ - clang-format \ - git \ - graphviz \ - libjpeg-dev \ - libsdl2-dev \ - libsdl2-image-dev \ - libsodium-dev \ - libssl-dev \ - libudev-dev \ - libusb-1.0-0-dev \ - valgrind \ - wget \ - zlib1g-dev - -# install python 3.7.3 + pip from the image +CMD [ "nix-shell" ] -RUN apt-get install -y \ - python3-dev \ - python3-pip +# the rest of the file only applies when docker build is called +# with the following argument: "--build-arg FULLDEPS_TESTING=1" -# install other python versions from their sources +ENV TREZOR_MONERO_TESTS_PATH="/opt/trezor_monero_tests" -RUN if [ "${FULLDEPS_TESTING}" = "1" ]; then \ - export PYTHON35VER="3.5.7" ; \ - wget --no-verbose https://www.python.org/ftp/python/${PYTHON35VER}/Python-${PYTHON35VER}.tgz ; \ - tar zxf Python-${PYTHON35VER}.tgz ; \ - cd Python-${PYTHON35VER}/ && ./configure && make && make install ; \ - fi +ARG FULLDEPS_TESTING=0 +ENV FULLDEPS_TESTING=${FULLDEPS_TESTING} -RUN if [ "${FULLDEPS_TESTING}" = "1" ]; then \ - export PYTHON36VER="3.6.9" ; \ - wget --no-verbose https://www.python.org/ftp/python/${PYTHON36VER}/Python-${PYTHON36VER}.tgz ; \ - tar zxf Python-${PYTHON36VER}.tgz ; \ - cd Python-${PYTHON36VER}/ && ./configure && make && make install ; \ - fi +# install other python versions for tox testing RUN if [ "${FULLDEPS_TESTING}" = "1" ]; then \ - export PYTHON38VER="3.8.0" ; \ - export PYTHONSUBVER="b3"; \ - wget --no-verbose https://www.python.org/ftp/python/${PYTHON38VER}/Python-${PYTHON38VER}${PYTHONSUBVER}.tgz ; \ - tar zxf Python-${PYTHON38VER}${PYTHONSUBVER}.tgz ; \ - cd Python-${PYTHON38VER}${PYTHONSUBVER}/ && ./configure && make && make install ; \ - fi - -# remove symlinks to newly installed pythons -RUN cd /usr/local/bin; \ - rm -f 2to3; \ - rm -f python3; \ - rm -f python3-config; \ - rm -f pydoc3; \ - rm -f pip3; \ - rm -f pyvenv; - -# install dependencies from toolchain source build - -RUN if [ "${TOOLCHAIN_FLAVOR}" = "src" ]; then \ - apt-get install -y autoconf autogen bison dejagnu \ - flex flip gawk git gperf gzip nsis \ - openssh-client p7zip-full perl python-dev \ - libisl-dev tcl tofrodos zip \ - texinfo texlive texlive-extra-utils; \ - fi - -# download toolchain - -ENV TOOLCHAIN_LONGVER=gcc-arm-none-eabi-9-2019-q4-major -ENV TOOLCHAIN_SUBDIR="9-2019q4/RC2.1" -ENV TOOLCHAIN_URL=https://developer.arm.com/-/media/Files/downloads/gnu-rm/${TOOLCHAIN_SUBDIR}/${TOOLCHAIN_LONGVER}-${TOOLCHAIN_FLAVOR}.tar.bz2 -ENV TOOLCHAIN_HASH_linux=bcd840f839d5bf49279638e9f67890b2ef3a7c9c7a9b25271e83ec4ff41d177a -ENV TOOLCHAIN_HASH_src=f162a655f222319f75862d7aba9ff8a4a86f752392e4f3c5d9ef2ee8bc13be58 - -# extract toolchain - -RUN cd /opt && wget --no-verbose ${TOOLCHAIN_URL} -RUN cd /opt && echo "${TOOLCHAIN_HASH_linux} ${TOOLCHAIN_LONGVER}-x86_64-linux.tar.bz2\n${TOOLCHAIN_HASH_src} ${TOOLCHAIN_LONGVER}-src.tar.bz2" | sha256sum -c --ignore-missing -RUN cd /opt && tar xfj ${TOOLCHAIN_LONGVER}-${TOOLCHAIN_FLAVOR}.tar.bz2 - -# build toolchain (if required) - -RUN if [ "${TOOLCHAIN_FLAVOR}" = "src" ]; then \ - pushd /opt/${TOOLCHAIN_LONGVER} ; \ - ./install-sources.sh --skip_steps=mingw32 ; \ - ./build-prerequisites.sh --skip_steps=mingw32 ; \ - ./build-toolchain.sh --skip_steps=mingw32,manual ; \ - popd ; \ + nix-env -iP python3-3.8.2 ; \ + nix-env --set-flag priority 6 python3-3.8.2 ; \ + nix-env -iP python3-3.6.10 ; \ + nix-env --set-flag priority 7 python3-3.6.10 ; \ + nix-env -iP python3-3.5.9 ; \ + nix-env --set-flag priority 8 python3-3.5.9 ; \ fi -# download protobuf - -ENV PROTOBUF_VERSION=3.6.1 -ENV PROTOBUF_HASH=6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807 -RUN wget --no-verbose "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protoc-${PROTOBUF_VERSION}-linux-x86_64.zip" -RUN echo "${PROTOBUF_HASH} protoc-${PROTOBUF_VERSION}-linux-x86_64.zip" | sha256sum -c - -# setup toolchain - -ENV PATH=/opt/${TOOLCHAIN_LONGVER}/bin:${PATH} - -ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 - -ENV PYTHON=python3 - -# use zipfile module to extract files world-readable -RUN ${PYTHON} -m zipfile -e "protoc-${PROTOBUF_VERSION}-linux-x86_64.zip" /usr/local && chmod 755 /usr/local/bin/protoc - # download monero tests binary -ENV TREZOR_MONERO_TESTS_SHA256SUM=5b35342c79eb91265f5f427224016a52994fff32c8ea078de5d502b37d3022d6 -ENV TREZOR_MONERO_TESTS_URL="https://github.com/ph4r05/monero/releases/download/v0.15.0.0-tests-u18.04-03/trezor_tests" -ENV TREZOR_MONERO_TESTS_PATH="/opt/trezor_monero_tests" - RUN if [ "${FULLDEPS_TESTING}" = "1" ]; then \ + TREZOR_MONERO_TESTS_SHA256SUM=1e5dfdb07de4ea46088f4a5bdb0d51f040fe479019efae30f76427eee6edb3f7 ; \ + TREZOR_MONERO_TESTS_URL="https://github.com/ph4r05/monero/releases/download/v0.15.0.0-tests-u18.04-03/trezor_tests" ; \ wget --no-verbose "${TREZOR_MONERO_TESTS_URL}" -O "${TREZOR_MONERO_TESTS_PATH}" ; \ chmod +x "${TREZOR_MONERO_TESTS_PATH}" ; \ echo "${TREZOR_MONERO_TESTS_SHA256SUM} ${TREZOR_MONERO_TESTS_PATH}" | sha256sum -c ; \ + nix-shell -p patchelf --run 'patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "${TREZOR_MONERO_TESTS_PATH}"' ; \ fi - -# install python dependencies - -ENV WORKON_HOME=/tmp/.venvs -ENV PIPENV_CACHE_DIR=/tmp/.pipenv-cache - -RUN ${PYTHON} -m pip install pipenv - -RUN ${PYTHON} --version -RUN ${PYTHON} -m pip --version -RUN pipenv --version diff --git a/ci/shell.nix b/ci/shell.nix new file mode 100644 index 000000000..67a66b7e1 --- /dev/null +++ b/ci/shell.nix @@ -0,0 +1,29 @@ +# nixos-unstable from 2020-06-02 +with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/467ce5a9f45aaf96110b41eb863a56866e1c2c3c.tar.gz) {}; + +stdenv.mkDerivation { + name = "trezor-firmware-docker"; + buildInputs = [ + SDL2 + SDL2_image + autoflake + check + clang-tools + gcc + gcc-arm-embedded + git + gnumake + graphviz + libffi + libjpeg + libressl + libusb1 + pipenv + pkgconfig + protobuf3_6 + valgrind + zlib + ]; + LD_LIBRARY_PATH = "${libffi}/lib:${libjpeg.out}/lib:${libusb1}/lib:${libressl.out}/lib"; + NIX_ENFORCE_PURITY = 0; +} diff --git a/shell.nix b/shell.nix index 3bd7da5df..3a62563ba 100644 --- a/shell.nix +++ b/shell.nix @@ -1,4 +1,5 @@ -with import {}; +# nixos-unstable from 2020-06-02 +with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/467ce5a9f45aaf96110b41eb863a56866e1c2c3c.tar.gz) {}; stdenv.mkDerivation { name = "trezor-firmware-dev"; @@ -9,6 +10,7 @@ stdenv.mkDerivation { check clang-tools gcc + gcc-arm-embedded git gnumake graphviz @@ -21,8 +23,6 @@ stdenv.mkDerivation { protobuf3_6 valgrind zlib - ] ++ stdenv.lib.optionals (!stdenv.isDarwin) [ - gcc-arm-embedded ] ++ stdenv.lib.optionals (stdenv.isDarwin) [ darwin.apple_sdk.frameworks.CoreAudio darwin.apple_sdk.frameworks.AudioToolbox