arno/trezor-firmware
1
0
Fork 0
mirror of https://github.com/trezor/trezor-firmware.git synced 2024-12-18 04:18:10 +00:00
Code Issues Releases Wiki Activity

ci: HTML UI reports for github actions

Browse Source 
Co-authored-by: vdovhanych <dovhanych@me.com>
This commit is contained in:
Martin Milata 2023-12-08 16:50:56 +01:00
parent cd7899dbcc
commit e9c18d69ec
4 changed files with 191 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
.github/actions/ui-comment/action.yml vendored Normal file
View File

@ -0,0 +1,22 @@
name: 'UI comment links'
description: 'Post URLs of HTML test reports to pull request discussion'
runs:
using: composite
steps:
- name: Find Comment
uses: peter-evans/find-comment@v2
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: 'github-actions[bot]'
body-includes: ui-comment-${{ github.workflow }}
- name: Create or update comment
uses: peter-evans/create-or-update-comment@v3
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
<!-- ui-comment-${{ github.workflow }} -->
${{ env.PULL_COMMENT }}
edit-mode: replace

48
.github/actions/ui-report/action.yml vendored
View File

@ -1,33 +1,41 @@
name: 'UI report'
description: 'Prepare and upload HTML report of UI test results'
inputs:
artifact-name:
description: 'Name of the uploaded artifact'
model:
description: 'Internal model name'
required: true
default: ui-report
runs:
using: composite
steps:
- run: mv tests/ui_tests/reports/test/ test_ui_report || true
- name: Set AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data
aws-region: eu-west-1
- run: |
MODELJOB=${{ inputs.model }}-${{ github.job }}
OUTDIR=${{ github.run_id }}/$MODELJOB
mkdir -p $OUTDIR
nix-shell --run "poetry run python ci/prepare_ui_artifacts.py || true"
mv tests/ui_tests/reports/test/* $OUTDIR || true
mv tests/ui_tests/fixtures.*.json $OUTDIR || true
mv tests/trezor.log $OUTDIR || true
diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json || true
tar -cf screens_$MODELJOB.tar tests/ui_tests/screens || true
shell: sh
- run: nix-shell --run "poetry run python ci/prepare_ui_artifacts.py || true"
- name: Upload report
run: |
aws s3 sync ${{ github.run_id }} s3://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}
shell: sh
- run: diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json || true
shell: sh
- run: tar -cf test_ui_report.tar test_ui_report/ || true
shell: sh
- run: tar -cf tests/ui_tests/screens.tar tests/ui_tests/screens/ || true
- name: Upload test screen recording
run: |
aws s3 sync ci/ui_test_records s3://data.trezor.io/dev/firmware/ui_tests
# TODO: generate directory listing / autoindex
shell: sh
- uses: actions/upload-artifact@v3
with:
name: ${{ inputs.artifact-name }}
name: ui-records
path: |
ci/ui_test_records/
# test_ui_report/ # can't have :: on ntfs
# tests/ui_tests/screens/ # can't have :: on ntfs
test_ui_report.tar
tests/ui_tests/screens.tar
tests/ui_tests/fixtures.suggestion.json
tests/ui_tests/fixtures.results.json
tests/trezor.log
retention-days: 7
# used by core_ui_master
screens_${{ inputs.model }}-${{ github.job }}.tar
retention-days: 1 # not useful after workflow finishes

98
.github/workflows/core.yml vendored
View File

@ -2,6 +2,18 @@ name: Core
on: [pull_request]
permissions:
id-token: write # for fetching the OIDC token
contents: read # for actions/checkout
pull-requests: write # For dflook comments on PR
env:
PULL_COMMENT: |
|core UI changes|[master diff](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/)| | |
|---------------|------------------------------------------------------------------------------------|----|----|
|Model T |[device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_device_test/)|[click test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_click_test/)|[persistence test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_persistence_test/)|
|Model Safe 3|[device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_device_test/)|[click test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_click_test/)|[persistence test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_persistence_test/)|
jobs:
core_firmware:
name: Build firmware
@ -171,6 +183,7 @@ jobs:
TREZOR_PYTEST_SKIP_ALTCOINS: ${{ matrix.coins == 'btconly' && '1' || '0' }}
ADDRESS_SANITIZER: ${{ matrix.asan == 'asan' && '1' || '0' }}
PYTEST_TIMEOUT: ${{ matrix.asan == 'asan' && 600 || 400 }}
ACTIONS_DO_UI_TEST: ${{ matrix.coins == 'universal' && matrix.asan == 'noasan' }}
steps:
- uses: actions/checkout@v4
with:
@ -181,21 +194,26 @@ jobs:
path: core/build
- run: chmod +x core/build/unix/trezor-emu-core*
- uses: ./.github/actions/environment
- run: nix-shell --run "poetry run make -C core test_emu_ui_multicore" # TODO: can-fail or whatisit
if: ${{ matrix.asan == 'noasan' && matrix.coins == 'universal' }}
- run: nix-shell --run "poetry run make -C core test_emu"
if: ${{ matrix.asan != 'noasan' || matrix.coins != 'universal' }}
- run: tail -n20 tests/trezor.log || true
if: ${{ failure() }}
- run: nix-shell --run "poetry run make -C core ${{ env.ACTIONS_DO_UI_TEST == 'true' && 'test_emu_ui_multicore' || 'test_emu' }}"
- run: tail -n50 tests/trezor.log || true
if: failure()
- uses: actions/upload-artifact@v3
with:
name: core-device-${{ matrix.model }}-${{ matrix.asan }}
path: tests/trezor.log
retention-days: 7
if: always()
- uses: ./.github/actions/ui-report
with:
artifact-name: core-test-device-${{ matrix.model }}-${{ matrix.coins }}-${{ matrix.asan }}
if: ${{ always() }}
- run: mv core/src/.coverage.* core || true # there will be more coverage files (one per core)
model: ${{ matrix.model }}
if: ${{ always() && env.ACTIONS_DO_UI_TEST == 'true' }}
- uses: ./.github/actions/ui-comment
if: ${{ failure() && env.ACTIONS_DO_UI_TEST == 'true' }}
- uses: actions/upload-artifact@v3
with:
name: core-coverage-${{ matrix.model }}
path: core/.coverage.*
# there will be more coverage files (one per core)
path: core/src/.coverage.*
retention-days: 7
# Click tests - UI.
@ -226,9 +244,16 @@ jobs:
if: ${{ matrix.asan == 'noasan' }}
- run: nix-shell --run "poetry run make -C core test_emu_click"
if: ${{ matrix.asan == 'asan' }}
- uses: actions/upload-artifact@v3
with:
name: core-click-${{ matrix.model }}-${{ matrix.asan }}
path: tests/trezor.log
retention-days: 7
if: always()
- uses: ./.github/actions/ui-report
with:
artifact-name: core-test-click-${{ matrix.model }}-${{ matrix.asan }}
model: ${{ matrix.model }}
if: always()
- run: mv core/src/.coverage core/.coverage.test_click || true
- uses: actions/upload-artifact@v3
with:
@ -264,7 +289,7 @@ jobs:
# Persistence tests - UI.
core_persitence_test:
core_persistence_test:
name: Persistence tests
runs-on: ubuntu-latest
needs: core_emu
@ -291,7 +316,8 @@ jobs:
if: ${{ matrix.asan == 'asan' }}
- uses: ./.github/actions/ui-report
with:
artifact-name: core-test-persistence-${{ matrix.model }}-${{ matrix.asan }}
model: ${{ matrix.model }}
if: always()
- run: mv core/src/.coverage core/.coverage.test_persistence || true
- uses: actions/upload-artifact@v3
with:
@ -352,6 +378,7 @@ jobs:
tests/trezor.log
core/prof/memperf-html
retention-days: 7
if: always()
# Flash size profiling
@ -437,6 +464,7 @@ jobs:
tests/trezor.log
core/tests/trezor_monero_tests.log
retention-days: 7
if: always()
- run: mv core/src/.coverage core/.coverage.test_emu_monero || true
- uses: actions/upload-artifact@v3
with:
@ -473,6 +501,7 @@ jobs:
name: core-test-u2f-${{ matrix.model }}-${{ matrix.asan }}
path: tests/trezor.log
retention-days: 7
if: always()
- run: mv core/src/.coverage core/.coverage.test_emu_u2f || true
- uses: actions/upload-artifact@v3
with:
@ -509,6 +538,7 @@ jobs:
path: |
tests/trezor.log
retention-days: 7
if: always()
- run: mv core/src/.coverage core/.coverage.test_emu_fido2 || true
- uses: actions/upload-artifact@v3
with:
@ -521,7 +551,7 @@ jobs:
runs-on: ubuntu-latest
needs:
- core_click_test
- core_persitence_test
- core_persistence_test
- core_device_test
- core_monero_test
- core_u2f_test
@ -545,13 +575,51 @@ jobs:
path: core
- uses: ./.github/actions/environment
- run: nix-shell --run "poetry run make -C core coverage"
# TODO fail if too little
- uses: actions/upload-artifact@v3
with:
name: core-coverage-${{ matrix.model }}
path: core/htmlcov
retention-days: 7
core_ui_master:
name: UI diff from master
runs-on: ubuntu-latest
needs:
- core_click_test
- core_persistence_test
- core_device_test
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v3
with:
name: ui-records
- uses: ./.github/actions/environment
- name: Configure aws credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data
aws-region: eu-west-1
- run: "for F in screens_*.tar; do tar xvf $F; done"
- run: nix-shell --run "poetry run python -m tests.ui_tests.reporting master-diff TT TR"
- run: mv tests/ui_tests/reports/master_diff .
- name: Upload master diff
run: |
aws s3 sync master_diff s3://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}
core_ui_comment:
name: Post comment with UI diff URLs
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- run: |
git fetch origin main
git diff --quiet origin/main...HEAD -- tests/ui_tests/fixtures.json || echo "FIXTURES_CHANGED=$?" >> $GITHUB_OUTPUT
id: check-fixtures-changed
- uses: ./.github/actions/ui-comment
if: ${{ steps.check-fixtures-changed.outputs.FIXTURES_CHANGED == '1' }}
# Connect
# TODO: core_connect_test

59
.github/workflows/legacy.yml vendored
View File

@ -2,6 +2,16 @@ name: Legacy
on: [pull_request]
permissions:
id-token: write # for fetching the OIDC token
contents: read # for actions/checkout
pull-requests: write # For dflook comments on PR
env:
PULL_COMMENT: |
|legacy UI changes|[master diff](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/)|[device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-legacy_device_test/)|
|-----------------|------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|
jobs:
legacy_firmware:
name: Firmware
@ -78,8 +88,16 @@ jobs:
path: legacy/firmware
- run: chmod +x legacy/firmware/*.elf
- uses: ./.github/actions/environment
- run: nix-shell --run "poetry run make -C legacy test_emu_ui"
if: matrix.coins == 'universal'
- run: nix-shell --run "poetry run make -C legacy test_emu"
# if: matrix.coins == 'universal' && matrix.type == 'normal'
if: matrix.coins != 'universal'
- uses: ./.github/actions/ui-report
with:
model: T1B1
if: ${{ always() && matrix.coins == 'universal' }}
- uses: ./.github/actions/ui-comment
if: ${{ failure() && matrix.coins == 'universal' }}
legacy_upgrade_test:
name: Upgrade test
@ -124,3 +142,42 @@ jobs:
- run: nix-shell --run "git clone --depth=1 https://github.com/bitcoin-core/HWI.git"
# see python_test for explanation of _PYTHON_SYSCONFIGDATA_NAME
- run: nix-shell --arg fullDeps true --run "unset _PYTHON_SYSCONFIGDATA_NAME && cd HWI && poetry install && poetry run ./test/test_trezor.py --model_1 ../legacy/firmware/trezor.elf bitcoind"
legacy_ui_master:
name: UI diff from master
runs-on: ubuntu-latest
needs:
- legacy_device_test
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v3
with:
name: ui-records
- uses: ./.github/actions/environment
- name: Configure aws credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data
aws-region: eu-west-1
- run: "for F in screens_*.tar; do tar xvf $F; done"
- run: nix-shell --run "poetry run python -m tests.ui_tests.reporting master-diff T1"
- run: mv tests/ui_tests/reports/master_diff .
- name: Upload master diff
run: |
aws s3 sync master_diff s3://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}
legacy_ui_comment:
name: Post comment with UI diff URLs
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- run: |
git fetch origin main
git diff --quiet origin/main...HEAD -- tests/ui_tests/fixtures.json
echo "FIXTURES_CHANGED=$?" >> $GITHUB_OUTPUT
id: check-fixtures-changed
- uses: ./.github/actions/ui-comment
if: ${{ steps.check-fixtures-changed.outputs.FIXTURES_CHANGED == '1' }}