diff --git a/.github/actions/ui-report/action.yml b/.github/actions/ui-report/action.yml index 500439eae8..ff78a05125 100644 --- a/.github/actions/ui-report/action.yml +++ b/.github/actions/ui-report/action.yml @@ -24,27 +24,36 @@ runs: shell: sh - run: | MODELJOB=${{ inputs.model }}-${{ inputs.lang }}-${{ github.job }} - OUTDIR=${{ github.run_id }}/$MODELJOB + OUTDIR=${{ github.run_id }} mkdir -p $OUTDIR nix-shell --run "poetry run python ci/prepare_ui_artifacts.py || true" - mv tests/ui_tests/reports/test/* $OUTDIR || true - mv tests/ui_tests/fixtures.*.json $OUTDIR || true - mv tests/trezor*.log $OUTDIR || true - diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json || true + mv -v tests/ui_tests/reports/test/* $OUTDIR + + # rename all model/job-speficific report files, so they won't be overwritten during upload + cd $OUTDIR + for F in *; do + if [ -f "$F" ]; then + mv -v $F $MODELJOB-$F + fi + done + cd .. if [ "${{ inputs.status }}" = "success" ]; then - cp .github/actions/ui-report/success.png $OUTDIR/status.png + cp -v .github/actions/ui-report/success.png $OUTDIR/$MODELJOB-status.png else - cp .github/actions/ui-report/failure.png $OUTDIR/status.png + cp -v .github/actions/ui-report/failure.png $OUTDIR/$MODELJOB-status.png fi shell: sh - name: Upload test results run: | # Upload report + du -sh ${{ github.run_id }} + ls -l ${{ github.run_id }} aws s3 cp --recursive --only-show-errors ${{ github.run_id }} s3://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }} & PID1=$! # Upload test screen recording + du -sh ci/ui_test_records aws s3 sync --only-show-errors ci/ui_test_records s3://data.trezor.io/dev/firmware/ui_tests & PID2=$! # TODO: generate directory listing / autoindex diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 18a4177fee..48b972f1da 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -28,10 +28,11 @@ env: PULL_COMMENT: | |core UI changes|device test|click test|persistence test| |---------------|-----------|----------|----------------| - |T2T1 Model T | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test/master_diff.html))|| - |T3B1 Safe 3 | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_device_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_device_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_device_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_device_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_click_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_click_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_click_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_click_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_persistence_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_persistence_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_persistence_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_persistence_test/master_diff.html))|| - |T3T1 Safe 5 | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test/master_diff.html))|| - |T3W1 | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_device_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_device_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_device_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_device_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_click_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_click_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_click_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_click_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_persistence_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_persistence_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_persistence_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_persistence_test/master_diff.html))|| + |T2T1 Model T | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}-T2T1-en-core_device_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test-master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test-master_diff.html))|| + |T3B1 Safe 3 | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_device_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_device_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_device_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}-T3B1-en-core_device_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_click_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_click_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_click_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_click_test-master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_persistence_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_persistence_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_persistence_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3B1-en-core_persistence_test-master_diff.html))|| + |T3T1 Safe 5 | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}-T3T1-en-core_device_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test-master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test-master_diff.html))|| + |T3W1 | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_device_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_device_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_device_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}-T3W1-en-core_device_test/master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_click_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_click_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_click_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_click_test-master_diff.html)) | [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_persistence_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_persistence_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_persistence_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3W1-en-core_persistence_test-master_diff.html))|| + |All |[main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/all_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/all_diff.html))| TREZOR_PYTEST_LOGS_DIR: ${{ github.workspace }}/tests/ jobs: @@ -706,6 +707,18 @@ jobs: - run: sleep 1m # try avoiding github api rate limit - uses: ./.github/actions/ui-comment + - name: Configure aws credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data + aws-region: eu-west-1 + - name: Upload aggregate diff from main branch + run: | + for F in all_index.html all_diff.html + do + aws s3 cp tests/ui_tests/reporting/$F s3://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/$F + done + core_upload_emu: name: Upload emulator binaries if: github.event_name == 'schedule' diff --git a/.github/workflows/legacy.yml b/.github/workflows/legacy.yml index a960519b3b..5688331867 100644 --- a/.github/workflows/legacy.yml +++ b/.github/workflows/legacy.yml @@ -21,8 +21,7 @@ permissions: env: PULL_COMMENT: | - |legacy UI changes| [device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-en-legacy_device_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-en-legacy_device_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/)([screens](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/master_diff.html))| - |-----------------|-----| + |legacy UI changes| [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-en-legacy_device_test-index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-en-legacy_device_test-differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-en-legacy_device_test-master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-en-legacy_device_test-master_diff.html)) | jobs: legacy_firmware: diff --git a/tests/ui_tests/reporting/all_diff.html b/tests/ui_tests/reporting/all_diff.html new file mode 100644 index 0000000000..bca586a782 --- /dev/null +++ b/tests/ui_tests/reporting/all_diff.html @@ -0,0 +1,46 @@ + + + + + + Master diff + + + +

T2T1

+ + + + +

T3B1

+ + + + +

T3T1

+ + + + +

T3W1

+ + + + + + + + diff --git a/tests/ui_tests/reporting/all_index.html b/tests/ui_tests/reporting/all_index.html new file mode 100644 index 0000000000..696b6ae772 --- /dev/null +++ b/tests/ui_tests/reporting/all_index.html @@ -0,0 +1,46 @@ + + + + + + Master index + + + +

T2T1

+ + + + +

T3B1

+ + + + +

T3T1

+ + + + +

T3W1

+ + + + + + + + diff --git a/tests/ui_tests/reporting/testreport.py b/tests/ui_tests/reporting/testreport.py index 8ffe636682..8e91184461 100644 --- a/tests/ui_tests/reporting/testreport.py +++ b/tests/ui_tests/reporting/testreport.py @@ -275,12 +275,9 @@ def master_index() -> Path: doc = document(title=title) with doc: - h1(title) - hr() - - h2("Differs:", style="color: grey;") - i("UI fixtures that have been modified:") - html.report_links(diff, TESTREPORT_PATH) + if diff: + i("UI fixtures that have been modified:") + html.report_links(diff, TESTREPORT_PATH) return html.write(TESTREPORT_PATH, doc, "master_index.html")