mirror of
https://github.com/trezor/trezor-firmware.git
synced 2024-12-18 20:38:10 +00:00
build: faster docker builds by reusing snapshots
This commit is contained in:
matejcik
parent
799c420dda
commit
806ded05ae
169
build-docker.sh
169
build-docker.sh
@ -3,6 +3,8 @@ set -e -o pipefail
|
||||
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")"
|
||||
|
||||
############## Select the right Alpine architecture ##############
|
||||
|
||||
if [ -z "$ALPINE_ARCH" ]; then
|
||||
arch="$(uname -m)"
|
||||
case "$arch" in
|
||||
@ -41,14 +43,17 @@ ALPINE_TARBALL=${ALPINE_FILE:-alpine-minirootfs-$ALPINE_VERSION-$ALPINE_ARCH.tar
|
||||
NIX_VERSION=${NIX_VERSION:-2.4}
|
||||
CONTAINER_FS_URL=${CONTAINER_FS_URL:-"$ALPINE_CDN/v$ALPINE_RELEASE/releases/$ALPINE_ARCH/$ALPINE_TARBALL"}
|
||||
|
||||
############## Options parsing ##############
|
||||
|
||||
function help_and_die() {
|
||||
echo "Usage: $0 [options] tag"
|
||||
echo "Options:"
|
||||
echo " --skip-bitcoinonly"
|
||||
echo " --skip-normal"
|
||||
echo " --skip-core"
|
||||
echo " --skip-legacy"
|
||||
echo " --repository path/to/repo"
|
||||
echo " --skip-bitcoinonly - do not build bitcoin-only firmwares"
|
||||
echo " --skip-normal - do not build regular firmwares"
|
||||
echo " --skip-core - do not build core"
|
||||
echo " --skip-legacy - do not build legacy"
|
||||
echo " --repository path/to/repo - checkout the repository from the given path/url"
|
||||
echo " --no-init - do not recreate docker environments"
|
||||
echo " --help"
|
||||
echo
|
||||
echo "Set PRODUCTION=0 to run non-production builds."
|
||||
@ -59,6 +64,7 @@ OPT_BUILD_CORE=1
|
||||
OPT_BUILD_LEGACY=1
|
||||
OPT_BUILD_NORMAL=1
|
||||
OPT_BUILD_BITCOINONLY=1
|
||||
INIT=1
|
||||
|
||||
REPOSITORY="/local"
|
||||
|
||||
@ -87,6 +93,10 @@ while true; do
|
||||
REPOSITORY="$2"
|
||||
shift 2
|
||||
;;
|
||||
--no-init)
|
||||
INIT=0
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
@ -97,6 +107,8 @@ if [ -z "$1" ]; then
|
||||
help_and_die
|
||||
fi
|
||||
|
||||
################## Variant selection ##################
|
||||
|
||||
variants=()
|
||||
if [ "$OPT_BUILD_NORMAL" -eq 1 ]; then
|
||||
variants+=(0)
|
||||
@ -118,7 +130,6 @@ fi
|
||||
TAG="$1"
|
||||
PRODUCTION=${PRODUCTION:-1}
|
||||
|
||||
|
||||
if which wget > /dev/null ; then
|
||||
wget --no-config -nc -P ci/ "$CONTAINER_FS_URL"
|
||||
else
|
||||
@ -134,25 +145,103 @@ else
|
||||
echo "${ALPINE_CHECKSUM} ci/${ALPINE_TARBALL}" | shasum -a 256 -c
|
||||
fi
|
||||
|
||||
tag_clean="${TAG//[^a-zA-Z0-9]/_}"
|
||||
SNAPSHOT_NAME="${CONTAINER_NAME}__${tag_clean}"
|
||||
|
||||
mkdir -p build/core build/legacy
|
||||
mkdir -p build/core-bitcoinonly build/legacy-bitcoinonly
|
||||
|
||||
# if not initializing, does the image exist?
|
||||
if [ $INIT -eq 0 ] && ! $DOCKER image inspect $SNAPSHOT_NAME > /dev/null; then
|
||||
echo "Image $SNAPSHOT_NAME does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
GIT_CLEAN_REPO="git clean -dfx -e .venv"
|
||||
SCRIPT_NAME="._setup_script"
|
||||
|
||||
if [ $INIT -eq 1 ]; then
|
||||
|
||||
SELECTED_CONTAINER="$CONTAINER_NAME"
|
||||
|
||||
echo
|
||||
echo ">>> DOCKER BUILD ALPINE_VERSION=$ALPINE_VERSION ALPINE_ARCH=$ALPINE_ARCH NIX_VERSION=$NIX_VERSION -t $CONTAINER_NAME"
|
||||
echo
|
||||
|
||||
$DOCKER build \
|
||||
--network=host \
|
||||
--build-arg ALPINE_VERSION="$ALPINE_VERSION" \
|
||||
--build-arg ALPINE_ARCH="$ALPINE_ARCH" \
|
||||
--build-arg NIX_VERSION="$NIX_VERSION" \
|
||||
-t "$CONTAINER_NAME" \
|
||||
ci/
|
||||
|
||||
cat <<EOF > "$SCRIPT_NAME"
|
||||
#!/bin/bash
|
||||
set -e -o pipefail
|
||||
|
||||
mkdir -p /reproducible-build
|
||||
cd /reproducible-build
|
||||
git clone "$REPOSITORY" trezor-firmware
|
||||
cd trezor-firmware
|
||||
EOF
|
||||
|
||||
else # init == 0
|
||||
|
||||
SELECTED_CONTAINER="$SNAPSHOT_NAME"
|
||||
|
||||
cat <<EOF > "$SCRIPT_NAME"
|
||||
#!/bin/bash
|
||||
set -e -o pipefail
|
||||
|
||||
cd /reproducible-build/trezor-firmware
|
||||
EOF
|
||||
|
||||
fi # init
|
||||
|
||||
# append common part to script
|
||||
cat <<EOF >> "$SCRIPT_NAME"
|
||||
$GIT_CLEAN_REPO
|
||||
git fetch origin "$TAG"
|
||||
git checkout -B "$TAG" "origin/$TAG"
|
||||
git submodule update --init --recursive
|
||||
poetry install
|
||||
cd core/embed/rust
|
||||
cargo fetch
|
||||
|
||||
git rev-parse HEAD > /build/git-rev
|
||||
echo
|
||||
echo ">>> AT COMMIT \$(git rev-parse HEAD)"
|
||||
echo
|
||||
EOF
|
||||
|
||||
echo
|
||||
echo ">>> DOCKER BUILD ALPINE_VERSION=$ALPINE_VERSION ALPINE_ARCH=$ALPINE_ARCH NIX_VERSION=$NIX_VERSION -t $CONTAINER_NAME"
|
||||
echo ">>> DOCKER REFRESH $SNAPSHOT_NAME"
|
||||
echo
|
||||
|
||||
$DOCKER build \
|
||||
$DOCKER run \
|
||||
--network=host \
|
||||
--build-arg ALPINE_VERSION="$ALPINE_VERSION" \
|
||||
--build-arg ALPINE_ARCH="$ALPINE_ARCH" \
|
||||
--build-arg NIX_VERSION="$NIX_VERSION" \
|
||||
-t "$CONTAINER_NAME" \
|
||||
ci/
|
||||
-t \
|
||||
-v "$PWD:/local" \
|
||||
-v "$PWD/build:/build" \
|
||||
--name "$SNAPSHOT_NAME" \
|
||||
"$SELECTED_CONTAINER" \
|
||||
/nix/var/nix/profiles/default/bin/nix-shell --run "bash /local/$SCRIPT_NAME" \
|
||||
|| ($DOCKER rm "$SNAPSHOT_NAME"; exit 1)
|
||||
|
||||
rm $SCRIPT_NAME
|
||||
|
||||
echo
|
||||
echo ">>> DOCKER COMMIT $SNAPSHOT_NAME"
|
||||
echo
|
||||
|
||||
$DOCKER commit "$SNAPSHOT_NAME" "$SNAPSHOT_NAME"
|
||||
$DOCKER rm "$SNAPSHOT_NAME"
|
||||
|
||||
# stat under macOS has slightly different cli interface
|
||||
USER=$(stat -c "%u" . 2>/dev/null || stat -f "%u" .)
|
||||
GROUP=$(stat -c "%g" . 2>/dev/null || stat -f "%g" .)
|
||||
|
||||
mkdir -p build/core build/legacy
|
||||
mkdir -p build/core-bitcoinonly build/legacy-bitcoinonly
|
||||
|
||||
DIR=$(pwd)
|
||||
|
||||
# build core
|
||||
@ -168,17 +257,15 @@ for BITCOIN_ONLY in ${VARIANTS_core[@]}; do
|
||||
# this file was generated by ${BASH_SOURCE[0]}
|
||||
# variant: core build BITCOIN_ONLY=$BITCOIN_ONLY
|
||||
set -e -o pipefail
|
||||
cd /tmp
|
||||
git clone "$REPOSITORY" trezor-firmware
|
||||
cd trezor-firmware/core
|
||||
git checkout "$TAG"
|
||||
git submodule update --init --recursive
|
||||
poetry install
|
||||
cd /reproducible-build/trezor-firmware/core
|
||||
$GIT_CLEAN_REPO
|
||||
poetry run make clean vendor build_bootloader build_firmware
|
||||
poetry run ../python/tools/firmware-fingerprint.py \
|
||||
-o build/firmware/firmware.bin.fingerprint \
|
||||
build/firmware/firmware.bin
|
||||
rm -r /build/*
|
||||
for item in bootloader firmware; do
|
||||
poetry run ../python/tools/firmware-fingerprint.py \
|
||||
-o build/\$item/\$item.bin.fingerprint \
|
||||
build/\$item/\$item.bin
|
||||
done
|
||||
rm -rf /build/*
|
||||
cp -r build/* /build
|
||||
chown -R $USER:$GROUP /build
|
||||
EOF
|
||||
@ -196,7 +283,7 @@ EOF
|
||||
--env BITCOIN_ONLY="$BITCOIN_ONLY" \
|
||||
--env PRODUCTION="$PRODUCTION" \
|
||||
--init \
|
||||
"$CONTAINER_NAME" \
|
||||
"$SNAPSHOT_NAME" \
|
||||
/nix/var/nix/profiles/default/bin/nix-shell --run "bash /local/build/$SCRIPT_NAME"
|
||||
done
|
||||
|
||||
@ -213,13 +300,9 @@ for BITCOIN_ONLY in ${VARIANTS_legacy[@]}; do
|
||||
# this file was generated by ${BASH_SOURCE[0]}
|
||||
# variant: legacy build BITCOIN_ONLY=$BITCOIN_ONLY
|
||||
set -e -o pipefail
|
||||
cd /tmp
|
||||
git clone "$REPOSITORY" trezor-firmware
|
||||
cd trezor-firmware/legacy
|
||||
cd /reproducible-build/trezor-firmware/legacy
|
||||
$GIT_CLEAN_REPO
|
||||
ln -s /build build
|
||||
git checkout "$TAG"
|
||||
git submodule update --init --recursive
|
||||
poetry install
|
||||
poetry run script/cibuild
|
||||
mkdir -p build/bootloader build/firmware build/intermediate_fw
|
||||
cp bootloader/bootloader.bin build/bootloader/bootloader.bin
|
||||
@ -246,12 +329,20 @@ EOF
|
||||
--env BITCOIN_ONLY="$BITCOIN_ONLY" \
|
||||
--env PRODUCTION="$PRODUCTION" \
|
||||
--init \
|
||||
"$CONTAINER_NAME" \
|
||||
"$SNAPSHOT_NAME" \
|
||||
/nix/var/nix/profiles/default/bin/nix-shell --run "bash /local/build/$SCRIPT_NAME"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "Docker image retained as $SNAPSHOT_NAME"
|
||||
echo "To remove it, run:"
|
||||
echo " docker rmi $SNAPSHOT_NAME"
|
||||
|
||||
# all built, show fingerprints
|
||||
|
||||
echo
|
||||
echo "Built from commit $(cat build/git-rev)"
|
||||
echo
|
||||
echo "Fingerprints:"
|
||||
for VARIANT in core legacy; do
|
||||
|
||||
@ -262,8 +353,12 @@ for VARIANT in core legacy; do
|
||||
DIRSUFFIX=${BITCOIN_ONLY/1/-bitcoinonly}
|
||||
DIRSUFFIX=${DIRSUFFIX/0/}
|
||||
|
||||
FWPATH=build/${VARIANT}${DIRSUFFIX}/firmware/firmware.bin
|
||||
FINGERPRINT=$(tr -d '\n' < $FWPATH.fingerprint)
|
||||
echo "$FINGERPRINT $FWPATH"
|
||||
for item in bootloader firmware; do
|
||||
FWPATH=build/${VARIANT}${DIRSUFFIX}/${item}/${item}.bin
|
||||
if [ -f $FWPATH -a -f "$FWPATH.fingerprint" ]; then
|
||||
FINGERPRINT=$(tr -d '\n' < $FWPATH.fingerprint)
|
||||
echo "$FINGERPRINT $FWPATH"
|
||||
fi
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
Loading…
Reference in New Issue
Block a user