From 806ded05ae2713799ae50301ae138ec080be1f99 Mon Sep 17 00:00:00 2001 From: matejcik Date: Wed, 5 Apr 2023 09:08:37 +0200 Subject: [PATCH] build: faster docker builds by reusing snapshots --- build-docker.sh | 169 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 132 insertions(+), 37 deletions(-) diff --git a/build-docker.sh b/build-docker.sh index a123dca574..2b98dfbff4 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -3,6 +3,8 @@ set -e -o pipefail cd "$(dirname "${BASH_SOURCE[0]}")" +############## Select the right Alpine architecture ############## + if [ -z "$ALPINE_ARCH" ]; then arch="$(uname -m)" case "$arch" in @@ -41,14 +43,17 @@ ALPINE_TARBALL=${ALPINE_FILE:-alpine-minirootfs-$ALPINE_VERSION-$ALPINE_ARCH.tar NIX_VERSION=${NIX_VERSION:-2.4} CONTAINER_FS_URL=${CONTAINER_FS_URL:-"$ALPINE_CDN/v$ALPINE_RELEASE/releases/$ALPINE_ARCH/$ALPINE_TARBALL"} +############## Options parsing ############## + function help_and_die() { echo "Usage: $0 [options] tag" echo "Options:" - echo " --skip-bitcoinonly" - echo " --skip-normal" - echo " --skip-core" - echo " --skip-legacy" - echo " --repository path/to/repo" + echo " --skip-bitcoinonly - do not build bitcoin-only firmwares" + echo " --skip-normal - do not build regular firmwares" + echo " --skip-core - do not build core" + echo " --skip-legacy - do not build legacy" + echo " --repository path/to/repo - checkout the repository from the given path/url" + echo " --no-init - do not recreate docker environments" echo " --help" echo echo "Set PRODUCTION=0 to run non-production builds." @@ -59,6 +64,7 @@ OPT_BUILD_CORE=1 OPT_BUILD_LEGACY=1 OPT_BUILD_NORMAL=1 OPT_BUILD_BITCOINONLY=1 +INIT=1 REPOSITORY="/local" @@ -87,6 +93,10 @@ while true; do REPOSITORY="$2" shift 2 ;; + --no-init) + INIT=0 + shift + ;; *) break ;; @@ -97,6 +107,8 @@ if [ -z "$1" ]; then help_and_die fi +################## Variant selection ################## + variants=() if [ "$OPT_BUILD_NORMAL" -eq 1 ]; then variants+=(0) @@ -118,7 +130,6 @@ fi TAG="$1" PRODUCTION=${PRODUCTION:-1} - if which wget > /dev/null ; then wget --no-config -nc -P ci/ "$CONTAINER_FS_URL" else @@ -134,25 +145,103 @@ else echo "${ALPINE_CHECKSUM} ci/${ALPINE_TARBALL}" | shasum -a 256 -c fi +tag_clean="${TAG//[^a-zA-Z0-9]/_}" +SNAPSHOT_NAME="${CONTAINER_NAME}__${tag_clean}" + +mkdir -p build/core build/legacy +mkdir -p build/core-bitcoinonly build/legacy-bitcoinonly + +# if not initializing, does the image exist? +if [ $INIT -eq 0 ] && ! $DOCKER image inspect $SNAPSHOT_NAME > /dev/null; then + echo "Image $SNAPSHOT_NAME does not exist." + exit 1 +fi + +GIT_CLEAN_REPO="git clean -dfx -e .venv" +SCRIPT_NAME="._setup_script" + +if [ $INIT -eq 1 ]; then + + SELECTED_CONTAINER="$CONTAINER_NAME" + + echo + echo ">>> DOCKER BUILD ALPINE_VERSION=$ALPINE_VERSION ALPINE_ARCH=$ALPINE_ARCH NIX_VERSION=$NIX_VERSION -t $CONTAINER_NAME" + echo + + $DOCKER build \ + --network=host \ + --build-arg ALPINE_VERSION="$ALPINE_VERSION" \ + --build-arg ALPINE_ARCH="$ALPINE_ARCH" \ + --build-arg NIX_VERSION="$NIX_VERSION" \ + -t "$CONTAINER_NAME" \ + ci/ + + cat < "$SCRIPT_NAME" + #!/bin/bash + set -e -o pipefail + + mkdir -p /reproducible-build + cd /reproducible-build + git clone "$REPOSITORY" trezor-firmware + cd trezor-firmware +EOF + +else # init == 0 + + SELECTED_CONTAINER="$SNAPSHOT_NAME" + + cat < "$SCRIPT_NAME" + #!/bin/bash + set -e -o pipefail + + cd /reproducible-build/trezor-firmware +EOF + +fi # init + +# append common part to script +cat <> "$SCRIPT_NAME" + $GIT_CLEAN_REPO + git fetch origin "$TAG" + git checkout -B "$TAG" "origin/$TAG" + git submodule update --init --recursive + poetry install + cd core/embed/rust + cargo fetch + + git rev-parse HEAD > /build/git-rev + echo + echo ">>> AT COMMIT \$(git rev-parse HEAD)" + echo +EOF + echo -echo ">>> DOCKER BUILD ALPINE_VERSION=$ALPINE_VERSION ALPINE_ARCH=$ALPINE_ARCH NIX_VERSION=$NIX_VERSION -t $CONTAINER_NAME" +echo ">>> DOCKER REFRESH $SNAPSHOT_NAME" echo -$DOCKER build \ +$DOCKER run \ --network=host \ - --build-arg ALPINE_VERSION="$ALPINE_VERSION" \ - --build-arg ALPINE_ARCH="$ALPINE_ARCH" \ - --build-arg NIX_VERSION="$NIX_VERSION" \ - -t "$CONTAINER_NAME" \ - ci/ + -t \ + -v "$PWD:/local" \ + -v "$PWD/build:/build" \ + --name "$SNAPSHOT_NAME" \ + "$SELECTED_CONTAINER" \ + /nix/var/nix/profiles/default/bin/nix-shell --run "bash /local/$SCRIPT_NAME" \ + || ($DOCKER rm "$SNAPSHOT_NAME"; exit 1) + +rm $SCRIPT_NAME + +echo +echo ">>> DOCKER COMMIT $SNAPSHOT_NAME" +echo + +$DOCKER commit "$SNAPSHOT_NAME" "$SNAPSHOT_NAME" +$DOCKER rm "$SNAPSHOT_NAME" # stat under macOS has slightly different cli interface USER=$(stat -c "%u" . 2>/dev/null || stat -f "%u" .) GROUP=$(stat -c "%g" . 2>/dev/null || stat -f "%g" .) -mkdir -p build/core build/legacy -mkdir -p build/core-bitcoinonly build/legacy-bitcoinonly - DIR=$(pwd) # build core @@ -168,17 +257,15 @@ for BITCOIN_ONLY in ${VARIANTS_core[@]}; do # this file was generated by ${BASH_SOURCE[0]} # variant: core build BITCOIN_ONLY=$BITCOIN_ONLY set -e -o pipefail - cd /tmp - git clone "$REPOSITORY" trezor-firmware - cd trezor-firmware/core - git checkout "$TAG" - git submodule update --init --recursive - poetry install + cd /reproducible-build/trezor-firmware/core + $GIT_CLEAN_REPO poetry run make clean vendor build_bootloader build_firmware - poetry run ../python/tools/firmware-fingerprint.py \ - -o build/firmware/firmware.bin.fingerprint \ - build/firmware/firmware.bin - rm -r /build/* + for item in bootloader firmware; do + poetry run ../python/tools/firmware-fingerprint.py \ + -o build/\$item/\$item.bin.fingerprint \ + build/\$item/\$item.bin + done + rm -rf /build/* cp -r build/* /build chown -R $USER:$GROUP /build EOF @@ -196,7 +283,7 @@ EOF --env BITCOIN_ONLY="$BITCOIN_ONLY" \ --env PRODUCTION="$PRODUCTION" \ --init \ - "$CONTAINER_NAME" \ + "$SNAPSHOT_NAME" \ /nix/var/nix/profiles/default/bin/nix-shell --run "bash /local/build/$SCRIPT_NAME" done @@ -213,13 +300,9 @@ for BITCOIN_ONLY in ${VARIANTS_legacy[@]}; do # this file was generated by ${BASH_SOURCE[0]} # variant: legacy build BITCOIN_ONLY=$BITCOIN_ONLY set -e -o pipefail - cd /tmp - git clone "$REPOSITORY" trezor-firmware - cd trezor-firmware/legacy + cd /reproducible-build/trezor-firmware/legacy + $GIT_CLEAN_REPO ln -s /build build - git checkout "$TAG" - git submodule update --init --recursive - poetry install poetry run script/cibuild mkdir -p build/bootloader build/firmware build/intermediate_fw cp bootloader/bootloader.bin build/bootloader/bootloader.bin @@ -246,12 +329,20 @@ EOF --env BITCOIN_ONLY="$BITCOIN_ONLY" \ --env PRODUCTION="$PRODUCTION" \ --init \ - "$CONTAINER_NAME" \ + "$SNAPSHOT_NAME" \ /nix/var/nix/profiles/default/bin/nix-shell --run "bash /local/build/$SCRIPT_NAME" done +echo +echo "Docker image retained as $SNAPSHOT_NAME" +echo "To remove it, run:" +echo " docker rmi $SNAPSHOT_NAME" + # all built, show fingerprints +echo +echo "Built from commit $(cat build/git-rev)" +echo echo "Fingerprints:" for VARIANT in core legacy; do @@ -262,8 +353,12 @@ for VARIANT in core legacy; do DIRSUFFIX=${BITCOIN_ONLY/1/-bitcoinonly} DIRSUFFIX=${DIRSUFFIX/0/} - FWPATH=build/${VARIANT}${DIRSUFFIX}/firmware/firmware.bin - FINGERPRINT=$(tr -d '\n' < $FWPATH.fingerprint) - echo "$FINGERPRINT $FWPATH" + for item in bootloader firmware; do + FWPATH=build/${VARIANT}${DIRSUFFIX}/${item}/${item}.bin + if [ -f $FWPATH -a -f "$FWPATH.fingerprint" ]; then + FINGERPRINT=$(tr -d '\n' < $FWPATH.fingerprint) + echo "$FINGERPRINT $FWPATH" + fi + done done done