arno/trezor-firmware
1
0
Fork 0
mirror of https://github.com/trezor/trezor-firmware.git synced 2025-03-19 01:26:06 +00:00
Code Issues Releases Wiki Activity

ci: move hardware tests over to github actions

Browse Source 
[skip_ci]
This commit is contained in:
Martin Milata 2024-02-08 18:27:40 +01:00
parent cd8a0ea690
commit 1ff789101b
15 changed files with 199 additions and 254 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/actions/ui-report/action.yml vendored
View File

@ -34,7 +34,7 @@ runs:
shell: sh
- uses: actions/upload-artifact@v3
with:
name: ui-records
name: ui-records-${{ github.job }}
path: |
# used by core_ui_master
screens_${{ inputs.model }}-${{ github.job }}.tar

138
.github/workflows/core-hw.yml vendored Normal file
View File

@ -0,0 +1,138 @@
name: Hardware tests
on:
schedule:
- cron: '15 23 * * *' # every day @ 23:15
workflow_dispatch:
# [Device tests](../tests/device-tests.md) that run against an actual physical Trezors.
# The device needs to have special bootloader, found in `core/embed/bootloader_ci`, that
# makes it possible to flash firmware without confirmation on the touchscreen.
#
# All hardware tests are run nightly on the `main` branch,
# and also can be started manually.
#
# Currently it's not possible to run all regular TT tests without getting into
# a state where the micropython heap is too fragmented and allocations fail
# (often manifesting as a stuck test case). For that reason some tests are
# skipped.
# See also: https://github.com/trezor/trezor-firmware/issues/1371
jobs:
core_device_test:
name: Device tests
runs-on:
- self-hosted
- ${{ matrix.model == 'T2B1' && 'runner3' || 'hw-t2t1' }}
# runner4 does not work at the moment
# - ${{ matrix.model == 'T2B1' && 'hw-t2b1' || 'hw-t2t1' }}
strategy:
fail-fast: false
matrix:
model: [T2T1, T2B1]
coins: [universal, btconly]
env:
TREZOR_MODEL: ${{ matrix.model == 'T2T1' && 'T' || 'R' }}
TREZOR_PYTEST_SKIP_ALTCOINS: ${{ matrix.coins == 'btconly' && '1' || '0' }}
PYTEST_TIMEOUT: 1200
PYOPT: 0
DISABLE_OPTIGA: 1
BOOTLOADER_DEVEL: ${{ matrix.model == 'T2B1' && '1' || '0' }}
TESTOPTS: "-k 'not authenticate and not recovery and not lots'"
TT_UHUB_PORT: 1
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- uses: ./.github/actions/environment
- run: nix-shell --arg hardwareTest true --run uhubctl
- run: nix-shell --run "poetry run make -C core build_firmware"
- run: nix-shell --arg hardwareTest true --run "poetry run python ci/hardware_tests/bootstrap.py ${{ matrix.model }} core/build/firmware/firmware.bin"
- run: nix-shell --run "poetry run trezorctl list"
- run: nix-shell --run "poetry run trezorctl get-features"
- run: |
# log serial console to file; sleep is used because tio needs stdin that is not /dev/null
nix-shell --arg hardwareTest true --run "sleep 8h | tio --no-autoconnect /dev/ttyTREZOR &> trezor.log" &
nix-shell --run "poetry run pytest -v tests/device_tests"
- run: tail -n50 trezor.log || true
if: failure()
- uses: actions/upload-artifact@v3
with:
name: core-hardware-${{ matrix.model }}-${{ matrix.coins }}
path: trezor.log
retention-days: 7
if: always()
core_monero_test:
name: Monero tests
runs-on:
- self-hosted
- ${{ matrix.model == 'T2B1' && 'runner3' || 'hw-t2t1' }}
# runner4 does not work at the moment
# - ${{ matrix.model == 'T2B1' && 'hw-t2b1' || 'hw-t2t1' }}
strategy:
fail-fast: false
matrix:
model: [T2T1, T2B1]
env:
TREZOR_MODEL: ${{ matrix.model == 'T2T1' && 'T' || 'R' }}
PYTEST_TIMEOUT: 1200
PYOPT: 0
DISABLE_OPTIGA: 1
BOOTLOADER_DEVEL: ${{ matrix.model == 'T2B1' && '1' || '0' }}
TT_UHUB_PORT: 1
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- uses: ./.github/actions/environment
- run: nix-shell --arg hardwareTest true --run uhubctl
- run: nix-shell --run "poetry run make -C core build_firmware"
- run: nix-shell --arg hardwareTest true --run "poetry run python ci/hardware_tests/bootstrap.py ${{ matrix.model }} core/build/firmware/firmware.bin"
- run: nix-shell --run "poetry run trezorctl list"
- run: nix-shell --run "poetry run trezorctl get-features"
- run: |
# log serial console to file; sleep is used because tio needs stdin that is not /dev/null
nix-shell --arg hardwareTest true --run "sleep 8h | tio --no-autoconnect /dev/ttyTREZOR &> trezor.log" &
nix-shell --arg fullDeps true --run "./core/tests/run_tests_device_emu_monero.sh $TESTOPTS"
- run: tail -n50 trezor.log || true
if: failure()
- uses: actions/upload-artifact@v3
with:
name: core-hardware-monero-${{ matrix.model }}
path: trezor.log
retention-days: 7
if: always()
legacy_device_test:
name: Device tests T1B1
runs-on:
- self-hosted
- hw-t1b1
strategy:
fail-fast: false
matrix:
coins: [universal, btconly]
env:
TREZOR_PYTEST_SKIP_ALTCOINS: ${{ matrix.coins == 'btconly' && '1' || '0' }}
PYTEST_TIMEOUT: 1200
T1_UHUB_LOCATION: 3-1
T1_UHUB_PORT: 2
T1_CAMERA: /dev/video0 # camera device
T1_ARDUINO_SERIAL: /dev/ttyTPMB # arduino that pushes T1 buttons
BITCOIN_ONLY: ${{ matrix.coins == 'universal' && '0' || '1' }}
DEBUG_LINK: 1
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- uses: ./.github/actions/environment
- run: nix-shell --arg hardwareTest true --run uhubctl
- run: nix-shell --run "poetry run legacy/script/setup"
- run: nix-shell --run "export PRODUCTION=0 && poetry run legacy/script/cibuild"
- run: nix-shell --arg hardwareTest true --run "ci/hardware_tests/t1_hw_test.sh"
- uses: actions/upload-artifact@v4.3.1
with:
name: legacy-hardware-${{ matrix.coins }}
path: ci/hardware_tests/*.mp4
retention-days: 7
if: always()

22
.github/workflows/core.yml vendored
View File

@ -199,7 +199,7 @@ jobs:
if: failure()
- uses: actions/upload-artifact@v3
with:
name: core-device-${{ matrix.model }}-${{ matrix.asan }}
name: core-device-${{ matrix.model }}-${{ matrix.coins }}-${{ matrix.asan }}
path: tests/trezor.log
retention-days: 7
if: always()
@ -211,7 +211,7 @@ jobs:
if: ${{ failure() && env.ACTIONS_DO_UI_TEST == 'true' }}
- uses: actions/upload-artifact@v3
with:
name: core-coverage-${{ matrix.model }}
name: core-coverage-${{ matrix.model }}-${{ github.job }}
# there will be more coverage files (one per core)
path: core/src/.coverage.*
retention-days: 7
@ -257,7 +257,7 @@ jobs:
- run: mv core/src/.coverage core/.coverage.test_click || true
- uses: actions/upload-artifact@v3
with:
name: core-coverage-${{ matrix.model }}
name: core-coverage-${{ matrix.model }}-${{ github.job }}
path: core/.coverage.*
retention-days: 7
@ -321,7 +321,7 @@ jobs:
- run: mv core/src/.coverage core/.coverage.test_persistence || true
- uses: actions/upload-artifact@v3
with:
name: core-coverage-${{ matrix.model }}
name: core-coverage-${{ matrix.model }}-${{ github.job }}
path: core/.coverage.*
retention-days: 7
@ -468,7 +468,7 @@ jobs:
- run: mv core/src/.coverage core/.coverage.test_emu_monero || true
- uses: actions/upload-artifact@v3
with:
name: core-coverage-${{ matrix.model }}
name: core-coverage-${{ matrix.model }}-${{ github.job }}
path: core/.coverage.*
retention-days: 7
@ -505,7 +505,7 @@ jobs:
- run: mv core/src/.coverage core/.coverage.test_emu_u2f || true
- uses: actions/upload-artifact@v3
with:
name: core-coverage-${{ matrix.model }}
name: core-coverage-${{ matrix.model }}-${{ github.job }}
path: core/.coverage.*
retention-days: 7
@ -542,7 +542,7 @@ jobs:
- run: mv core/src/.coverage core/.coverage.test_emu_fido2 || true
- uses: actions/upload-artifact@v3
with:
name: core-coverage-${{ matrix.model }}
name: core-coverage-${{ matrix.model }}-${{ github.job }}
path: core/.coverage.*
retention-days: 7
@ -571,13 +571,14 @@ jobs:
submodules: recursive
- uses: actions/download-artifact@v3
with:
name: core-coverage-${{ matrix.model }}
pattern: core-coverage-${{ matrix.model }}-*
path: core
merge-multiple: true
- uses: ./.github/actions/environment
- run: nix-shell --run "poetry run make -C core coverage"
- uses: actions/upload-artifact@v3
with:
name: core-coverage-${{ matrix.model }}
name: core-coverage-report-${{ matrix.model }}
path: core/htmlcov
retention-days: 7
@ -592,7 +593,8 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v3
with:
name: ui-records
pattern: ui-records-*
merge-multiple: true
- uses: ./.github/actions/environment
- name: Configure aws credentials
uses: aws-actions/configure-aws-credentials@v4

3
.github/workflows/legacy.yml vendored
View File

@ -152,7 +152,8 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v3
with:
name: ui-records
pattern: ui-records-*
merge-multiple: true
- uses: ./.github/actions/environment
- name: Configure aws credentials
uses: aws-actions/configure-aws-credentials@v4

1
.gitlab-ci.yml
View File

@ -45,7 +45,6 @@ include:
- ci/prebuild.yml
- ci/build.yml
- ci/test.yml
- ci/test-hw.yml
- ci/test-nonenglish.yml
- ci/posttest.yml
- ci/deploy.yml

28
ci/hardware_tests/bootstrap.py
View File

@ -1,24 +1,30 @@
import os
import sys
from device.t1 import TrezorOne
from device.tt import TrezorT
from device.core import TrezorCore
from device.legacy import TrezorOne
# https://www.uugear.com/product/mega4-4-port-usb-3-ppps-hub-for-raspberry-pi-4b/
# as long as every runner has this hub we don't have to configure a per-runner hub location
HUB_VENDOR = "2109:2817"
def main(model: str, file: str = None):
t1 = TrezorOne(
os.environ["T1_UHUB_LOCATION"],
os.environ["T1_ARDUINO_SERIAL"],
os.environ["T1_UHUB_PORT"],
os.getenv("T1_UHUB_LOCATION"),
os.getenv("T1_ARDUINO_SERIAL"),
os.getenv("T1_UHUB_PORT"),
)
tt = TrezorT(os.environ["TT_UHUB_LOCATION"], os.environ["TT_UHUB_PORT"])
tt = TrezorCore(hub_vendor=HUB_VENDOR, device_port=os.getenv("TT_UHUB_PORT"))
if model == "t1":
tt.power_off()
if model == "T1B1":
# tt.power_off()
path = t1.update_firmware(file)
elif model == "tt":
t1.power_off()
path = tt.update_firmware(file)
elif model == "T2T1":
# t1.power_off()
path = tt.update_firmware(file, "Trezor T")
elif model == "T2B1":
path = tt.update_firmware(file, "Safe 3")
else:
raise ValueError("Unknown Trezor model.")

11
ci/hardware_tests/device/tt.py → ci/hardware_tests/device/core.py
View File

@ -1,8 +1,8 @@
from .device import Device
class TrezorT(Device):
def update_firmware(self, file=None):
class TrezorCore(Device):
def update_firmware(self, file=None, model_name="Trezor T"):
if not file:
raise ValueError(
"Uploading production firmware will replace the bootloader, it is not allowed!"
@ -10,10 +10,11 @@ class TrezorT(Device):
# reset to enter bootloader again
self.power_off()
self.wait(5)
self.power_on()
self.wait(5)
self.check_model("Trezor T bootloader")
self.wait(10)
self.check_model("bootloader")
self.run_trezorctl("device wipe --bootloader || true")
self.wait(5)
@ -26,4 +27,4 @@ class TrezorT(Device):
# after firmware-update finishes wait for reboot
self.wait(15)
return self.check_model("Trezor T")
return self.check_model(model_name)

13
ci/hardware_tests/device/device.py
View File

@ -5,8 +5,9 @@ from subprocess import run
class Device:
def __init__(self, uhub_location, device_port):
def __init__(self, *, uhub_location=None, hub_vendor=None, device_port=None):
self.uhub_location = uhub_location
self.hub_vendor = hub_vendor
self.device_port = device_port
@staticmethod
@ -35,11 +36,17 @@ class Device:
self.power_off()
self.power_on()
def _hub(self):
if self.hub_vendor:
return f"--vendor {self.hub_vendor}"
else:
return f"-l {self.uhub_location}"
def power_on(self):
self.now()
self.log("[hardware/usb] Turning power on...")
run(
f"uhubctl -l {self.uhub_location} -p {self.device_port} -a on",
f"uhubctl {self._hub()} -p {self.device_port} -a on",
shell=True,
check=True,
)
@ -49,7 +56,7 @@ class Device:
self.now()
self.log("[hardware/usb] Turning power off...")
run(
f"uhubctl -l {self.uhub_location} -p {self.device_port} -r 100 -a off",
f"uhubctl {self._hub()} -p {self.device_port} -r 5 -a off",
shell=True,
check=True,
)

4
ci/hardware_tests/device/t1.py → ci/hardware_tests/device/legacy.py
View File

@ -5,7 +5,7 @@ from .device import Device
class TrezorOne(Device):
def __init__(self, uhub_location, arduino_serial, device_port):
super().__init__(uhub_location, device_port)
super().__init__(uhub_location=uhub_location, device_port=device_port)
self.serial = serial.Serial(arduino_serial, 9600)
def touch(self, location, action):
@ -41,7 +41,7 @@ class TrezorOne(Device):
self.touch("right", "click")
self.wait(5)
self.touch("right", "click")
self.wait(10)
self.wait(15)
return self.check_model("Trezor 1")
def _enter_bootloader(self):

16
ci/hardware_tests/hardware.cfg
View File

@ -1,16 +0,0 @@
# location of the uhub, can be found out by running `uhubctl`
T1_UHUB_LOCATION="3-2"
# to which port of the uhub the device is connected
T1_UHUB_PORT="2"
# camera device
T1_CAMERA="/dev/video0"
# arduino that pushes T1 buttons
T1_ARDUINO_SERIAL="/dev/serial/by-id/usb-Arduino__www.arduino.cc__0043_8573332373935181A0C0-if00"
# location of the uhub, can be found out by running `uhubctl`
TT_UHUB_LOCATION="3-2"
# to which port of the uhub the device is connected
TT_UHUB_PORT="3"

2
ci/hardware_tests/record_video.sh
View File

@ -11,7 +11,7 @@ OUTPUTFILE=video_${COMMIT}_$(date +%s).mp4
if [ "$ACTION" == "start" ]; then
echo "[software/video] Starting record to $OUTPUTFILE"
ffmpeg -loglevel warning -f oss -f video4linux2 -i $INPUTDEVICE \
ffmpeg -loglevel warning -f oss -f v4l2 -i $INPUTDEVICE \
-flush_packets 1 \
-vf "drawtext=font=Dejavu Sans: \
text='$COMMIT | %{localtime} | %{pts}': x=(w-tw)/2: y=h-(2*lh): fontcolor=white: box=1: boxcolor=0x00000000@1: fontsize=15" $OUTPUTFILE &

18
ci/hardware_tests/t1_hw_test.sh
View File

@ -1,21 +1,19 @@
#!/usr/bin/env bash
HERE=`dirname "$0"`
SHA=${GITHUB_SHA:-unknown}
cd $HERE
function finish {
./record_video.sh ${T1_CAMERA} ${CI_COMMIT_SHORT_SHA} stop
ls -l *.mp4
./record_video.sh ${T1_CAMERA} ${SHA} stop
}
trap finish EXIT
set -e # exit on nonzero exitcode
set -x # trace commands
# export variables defined in the file
set -a
source hardware.cfg
set +a
./record_video.sh ${T1_CAMERA} ${CI_COMMIT_SHORT_SHA} start
./record_video.sh ${T1_CAMERA} ${SHA} start
(cd ../.. && poetry install)
poetry run python bootstrap.py t1
poetry run python bootstrap.py t1 ../../firmware-T1*.bin
#poetry run python bootstrap.py T1B1 # install official firmware first
poetry run python bootstrap.py T1B1 ../../firmware-T1*.bin
poetry run pytest ../../tests/device_tests

3
ci/shell.nix
View File

@ -137,7 +137,8 @@ stdenvNoCC.mkDerivation ({
libiconv
] ++ lib.optionals hardwareTest [
uhubctl
ffmpeg
tio
ffmpeg_5-full
dejavu_fonts
] ++ lib.optionals devTools [
shellcheck

159
ci/test-hw.yml
View File

@ -1,159 +0,0 @@
image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-env.nix
# Caching
.gitlab_caching: &gitlab_caching
cache:
key: "$CI_COMMIT_REF_SLUG"
paths:
- .venv/
# Hardware
# [Device tests](../tests/device-tests.md) that run against an actual physical Trezor T.
# The device needs to have special bootloader, found in `core/embed/bootloader_ci`, that
# makes it possible to flash firmware without confirmation on the touchscreen.
#
# All hardware tests are run nightly on the `master` branch, as well as on push to branches
# with whitelisted prefix. If you want hardware tests ran on your branch, make sure its
# name starts with `hw/`.
#
# Currently it's not possible to run all regular TT tests without getting into
# a state where the micropython heap is too fragmented and allocations fail
# (often manifesting as a stuck test case). For that reason some tests are
# skipped.
# See also: https://github.com/trezor/trezor-firmware/issues/1371
hardware core regular device test:
stage: test
only:
- schedules # nightly build
- /^legacy\//
- /^release\//
- /^secfix\//
- /^hw\//
tags:
- tpmb
needs:
- core fw regular debug build
variables:
PYTEST_TIMEOUT: "1200"
script:
- cd ci/hardware_tests
- set -a
- source hardware.cfg
- set +a
- $NIX_SHELL --run "cd ../.. && poetry install"
- $NIX_SHELL --run "poetry run python bootstrap.py tt ../../firmware-T2*.bin | ts -s"
- $NIX_SHELL --run "poetry run pytest ../../tests/device_tests | ts -s"
timeout: 6h
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
expire_in: 2 days
when: always
# Also device tests on physical Trezor T but with Bitcoin-only firmware.
hardware core btconly device test:
stage: test
only:
- schedules # nightly build
- /^legacy\//
- /^release\//
- /^secfix\//
- /^hw\//
tags:
- tpmb
needs:
- core fw btconly debug build
variables:
TREZOR_PYTEST_SKIP_ALTCOINS: 1
PYTEST_TIMEOUT: "1200"
script:
- cd ci/hardware_tests
- set -a
- source hardware.cfg
- set +a
- $NIX_SHELL --run "cd ../.. && poetry install"
- $NIX_SHELL --run "poetry run python bootstrap.py tt ../../firmware-T2*.bin | ts -s"
- $NIX_SHELL --run "poetry run pytest ../../tests/device_tests | ts -s"
timeout: 4h
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
expire_in: 2 days
when: always
hardware core monero test:
stage: test
only:
- schedules # nightly build
- /^release\//
- /^secfix\//
- /^hw\//
tags:
- tpmb
needs:
- core fw regular debug build
variables:
TESTOPTS: --trezor-path webusb
script:
- cd ci/hardware_tests
- set -a
- source hardware.cfg
- set +a
- $NIX_SHELL --run "cd ../.. && poetry install"
- $NIX_SHELL --run "poetry run python bootstrap.py tt ../../firmware-T2*.bin | ts -s"
- $NIX_SHELL --arg fullDeps true --run "cd ../../core/tests && ./run_tests_device_emu_monero.sh $TESTOPTS | ts -s"
timeout: 1h
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
expire_in: 2 days
when: always
# [Device tests](../tests/device-tests.md) executed on physical Trezor 1.
# This works thanks to [tpmb](https://github.com/mmahut/tpmb), which is a small arduino
# device capable of pushing an actual buttons on the device.
hardware legacy regular device test:
stage: test
only:
- schedules # nightly build
- /^legacy\//
- /^release\//
- /^secfix\//
- /^hw\//
tags:
- tpmb
needs:
- legacy fw regular debug build
script:
- cd ci/hardware_tests
- $NIX_SHELL --run "./t1_hw_test.sh | ts -s"
timeout: 1h20m
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
paths:
- ci/hardware_tests/*.mp4
expire_in: 2 days
when: always
# Also device tests on physical Trezor 1 but with Bitcoin-only firmware.
hardware legacy btconly device test:
stage: test
variables:
TREZOR_PYTEST_SKIP_ALTCOINS: 1
only:
- schedules # nightly build
- /^legacy\//
- /^release\//
- /^secfix\//
- /^hw\//
tags:
- tpmb
needs:
- legacy fw btconly debug build
script:
- cd ci/hardware_tests
- $NIX_SHELL --run "./t1_hw_test.sh | ts -s"
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
paths:
- ci/hardware_tests/*.mp4
expire_in: 2 days
when: always

33
docs/ci/jobs.md
View File

@ -245,39 +245,6 @@ Also generates a report with the current situation
### [connect test core](https://github.com/trezor/trezor-firmware/blob/master/ci/test.yml#L773)
---
## TEST-HW stage - [test-hw.yml](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml)
Consists of **5 jobs** below:
### [hardware core regular device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml#L25)
[Device tests](../tests/device-tests.md) that run against an actual physical Trezor T.
The device needs to have special bootloader, found in `core/embed/bootloader_ci`, that
makes it possible to flash firmware without confirmation on the touchscreen.
All hardware tests are run nightly on the `master` branch, as well as on push to branches
with whitelisted prefix. If you want hardware tests ran on your branch, make sure its
name starts with `hw/`.
Currently it's not possible to run all regular TT tests without getting into
a state where the micropython heap is too fragmented and allocations fail
(often manifesting as a stuck test case). For that reason some tests are
skipped.
See also: https://github.com/trezor/trezor-firmware/issues/1371
### [hardware core btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml#L54)
Also device tests on physical Trezor T but with Bitcoin-only firmware.
### [hardware core monero test](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml#L83)
### [hardware legacy regular device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml#L113)
[Device tests](../tests/device-tests.md) executed on physical Trezor 1.
This works thanks to [tpmb](https://github.com/mmahut/tpmb), which is a small arduino
device capable of pushing an actual buttons on the device.
### [hardware legacy btconly device test](https://github.com/trezor/trezor-firmware/blob/master/ci/test-hw.yml#L137)
Also device tests on physical Trezor 1 but with Bitcoin-only firmware.
---
## TEST-NONENGLISH stage - [test-nonenglish.yml](https://github.com/trezor/trezor-firmware/blob/master/ci/test-nonenglish.yml)
Tests for non-english languages, that run only nightly