From e9c18d69ec50af7b35e9ad7acb0d9cd2051bd236 Mon Sep 17 00:00:00 2001 From: Martin Milata Date: Fri, 8 Dec 2023 16:50:56 +0100 Subject: [PATCH] ci: HTML UI reports for github actions Co-authored-by: vdovhanych --- .github/actions/ui-comment/action.yml | 22 ++++++ .github/actions/ui-report/action.yml | 48 +++++++------ .github/workflows/core.yml | 98 +++++++++++++++++++++++---- .github/workflows/legacy.yml | 59 +++++++++++++++- 4 files changed, 191 insertions(+), 36 deletions(-) create mode 100644 .github/actions/ui-comment/action.yml diff --git a/.github/actions/ui-comment/action.yml b/.github/actions/ui-comment/action.yml new file mode 100644 index 0000000000..cb8b5946f4 --- /dev/null +++ b/.github/actions/ui-comment/action.yml @@ -0,0 +1,22 @@ +name: 'UI comment links' +description: 'Post URLs of HTML test reports to pull request discussion' +runs: + using: composite + steps: + - name: Find Comment + uses: peter-evans/find-comment@v2 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: ui-comment-${{ github.workflow }} + + - name: Create or update comment + uses: peter-evans/create-or-update-comment@v3 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: | + + ${{ env.PULL_COMMENT }} + edit-mode: replace diff --git a/.github/actions/ui-report/action.yml b/.github/actions/ui-report/action.yml index 272001e652..cea9f23cd6 100644 --- a/.github/actions/ui-report/action.yml +++ b/.github/actions/ui-report/action.yml @@ -1,33 +1,41 @@ name: 'UI report' description: 'Prepare and upload HTML report of UI test results' inputs: - artifact-name: - description: 'Name of the uploaded artifact' + model: + description: 'Internal model name' required: true - default: ui-report runs: using: composite steps: - - run: mv tests/ui_tests/reports/test/ test_ui_report || true + - name: Set AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data + aws-region: eu-west-1 + - run: | + MODELJOB=${{ inputs.model }}-${{ github.job }} + OUTDIR=${{ github.run_id }}/$MODELJOB + mkdir -p $OUTDIR + nix-shell --run "poetry run python ci/prepare_ui_artifacts.py || true" + mv tests/ui_tests/reports/test/* $OUTDIR || true + mv tests/ui_tests/fixtures.*.json $OUTDIR || true + mv tests/trezor.log $OUTDIR || true + diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json || true + tar -cf screens_$MODELJOB.tar tests/ui_tests/screens || true shell: sh - - run: nix-shell --run "poetry run python ci/prepare_ui_artifacts.py || true" + - name: Upload report + run: | + aws s3 sync ${{ github.run_id }} s3://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }} shell: sh - - run: diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json || true - shell: sh - - run: tar -cf test_ui_report.tar test_ui_report/ || true - shell: sh - - run: tar -cf tests/ui_tests/screens.tar tests/ui_tests/screens/ || true + - name: Upload test screen recording + run: | + aws s3 sync ci/ui_test_records s3://data.trezor.io/dev/firmware/ui_tests + # TODO: generate directory listing / autoindex shell: sh - uses: actions/upload-artifact@v3 with: - name: ${{ inputs.artifact-name }} + name: ui-records path: | - ci/ui_test_records/ - # test_ui_report/ # can't have :: on ntfs - # tests/ui_tests/screens/ # can't have :: on ntfs - test_ui_report.tar - tests/ui_tests/screens.tar - tests/ui_tests/fixtures.suggestion.json - tests/ui_tests/fixtures.results.json - tests/trezor.log - retention-days: 7 + # used by core_ui_master + screens_${{ inputs.model }}-${{ github.job }}.tar + retention-days: 1 # not useful after workflow finishes diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 03e0987681..6656e09cfb 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -2,6 +2,18 @@ name: Core on: [pull_request] +permissions: + id-token: write # for fetching the OIDC token + contents: read # for actions/checkout + pull-requests: write # For dflook comments on PR + +env: + PULL_COMMENT: | + |core UI changes|[master diff](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/)| | | + |---------------|------------------------------------------------------------------------------------|----|----| + |Model T |[device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_device_test/)|[click test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_click_test/)|[persistence test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_persistence_test/)| + |Model Safe 3|[device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_device_test/)|[click test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_click_test/)|[persistence test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_persistence_test/)| + jobs: core_firmware: name: Build firmware @@ -171,6 +183,7 @@ jobs: TREZOR_PYTEST_SKIP_ALTCOINS: ${{ matrix.coins == 'btconly' && '1' || '0' }} ADDRESS_SANITIZER: ${{ matrix.asan == 'asan' && '1' || '0' }} PYTEST_TIMEOUT: ${{ matrix.asan == 'asan' && 600 || 400 }} + ACTIONS_DO_UI_TEST: ${{ matrix.coins == 'universal' && matrix.asan == 'noasan' }} steps: - uses: actions/checkout@v4 with: @@ -181,21 +194,26 @@ jobs: path: core/build - run: chmod +x core/build/unix/trezor-emu-core* - uses: ./.github/actions/environment - - run: nix-shell --run "poetry run make -C core test_emu_ui_multicore" # TODO: can-fail or whatisit - if: ${{ matrix.asan == 'noasan' && matrix.coins == 'universal' }} - - run: nix-shell --run "poetry run make -C core test_emu" - if: ${{ matrix.asan != 'noasan' || matrix.coins != 'universal' }} - - run: tail -n20 tests/trezor.log || true - if: ${{ failure() }} + - run: nix-shell --run "poetry run make -C core ${{ env.ACTIONS_DO_UI_TEST == 'true' && 'test_emu_ui_multicore' || 'test_emu' }}" + - run: tail -n50 tests/trezor.log || true + if: failure() + - uses: actions/upload-artifact@v3 + with: + name: core-device-${{ matrix.model }}-${{ matrix.asan }} + path: tests/trezor.log + retention-days: 7 + if: always() - uses: ./.github/actions/ui-report with: - artifact-name: core-test-device-${{ matrix.model }}-${{ matrix.coins }}-${{ matrix.asan }} - if: ${{ always() }} - - run: mv core/src/.coverage.* core || true # there will be more coverage files (one per core) + model: ${{ matrix.model }} + if: ${{ always() && env.ACTIONS_DO_UI_TEST == 'true' }} + - uses: ./.github/actions/ui-comment + if: ${{ failure() && env.ACTIONS_DO_UI_TEST == 'true' }} - uses: actions/upload-artifact@v3 with: name: core-coverage-${{ matrix.model }} - path: core/.coverage.* + # there will be more coverage files (one per core) + path: core/src/.coverage.* retention-days: 7 # Click tests - UI. @@ -226,9 +244,16 @@ jobs: if: ${{ matrix.asan == 'noasan' }} - run: nix-shell --run "poetry run make -C core test_emu_click" if: ${{ matrix.asan == 'asan' }} + - uses: actions/upload-artifact@v3 + with: + name: core-click-${{ matrix.model }}-${{ matrix.asan }} + path: tests/trezor.log + retention-days: 7 + if: always() - uses: ./.github/actions/ui-report with: - artifact-name: core-test-click-${{ matrix.model }}-${{ matrix.asan }} + model: ${{ matrix.model }} + if: always() - run: mv core/src/.coverage core/.coverage.test_click || true - uses: actions/upload-artifact@v3 with: @@ -264,7 +289,7 @@ jobs: # Persistence tests - UI. - core_persitence_test: + core_persistence_test: name: Persistence tests runs-on: ubuntu-latest needs: core_emu @@ -291,7 +316,8 @@ jobs: if: ${{ matrix.asan == 'asan' }} - uses: ./.github/actions/ui-report with: - artifact-name: core-test-persistence-${{ matrix.model }}-${{ matrix.asan }} + model: ${{ matrix.model }} + if: always() - run: mv core/src/.coverage core/.coverage.test_persistence || true - uses: actions/upload-artifact@v3 with: @@ -352,6 +378,7 @@ jobs: tests/trezor.log core/prof/memperf-html retention-days: 7 + if: always() # Flash size profiling @@ -437,6 +464,7 @@ jobs: tests/trezor.log core/tests/trezor_monero_tests.log retention-days: 7 + if: always() - run: mv core/src/.coverage core/.coverage.test_emu_monero || true - uses: actions/upload-artifact@v3 with: @@ -473,6 +501,7 @@ jobs: name: core-test-u2f-${{ matrix.model }}-${{ matrix.asan }} path: tests/trezor.log retention-days: 7 + if: always() - run: mv core/src/.coverage core/.coverage.test_emu_u2f || true - uses: actions/upload-artifact@v3 with: @@ -509,6 +538,7 @@ jobs: path: | tests/trezor.log retention-days: 7 + if: always() - run: mv core/src/.coverage core/.coverage.test_emu_fido2 || true - uses: actions/upload-artifact@v3 with: @@ -521,7 +551,7 @@ jobs: runs-on: ubuntu-latest needs: - core_click_test - - core_persitence_test + - core_persistence_test - core_device_test - core_monero_test - core_u2f_test @@ -545,13 +575,51 @@ jobs: path: core - uses: ./.github/actions/environment - run: nix-shell --run "poetry run make -C core coverage" - # TODO fail if too little - uses: actions/upload-artifact@v3 with: name: core-coverage-${{ matrix.model }} path: core/htmlcov retention-days: 7 + core_ui_master: + name: UI diff from master + runs-on: ubuntu-latest + needs: + - core_click_test + - core_persistence_test + - core_device_test + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v3 + with: + name: ui-records + - uses: ./.github/actions/environment + - name: Configure aws credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data + aws-region: eu-west-1 + - run: "for F in screens_*.tar; do tar xvf $F; done" + - run: nix-shell --run "poetry run python -m tests.ui_tests.reporting master-diff TT TR" + - run: mv tests/ui_tests/reports/master_diff . + - name: Upload master diff + run: | + aws s3 sync master_diff s3://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }} + + core_ui_comment: + name: Post comment with UI diff URLs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - run: | + git fetch origin main + git diff --quiet origin/main...HEAD -- tests/ui_tests/fixtures.json || echo "FIXTURES_CHANGED=$?" >> $GITHUB_OUTPUT + id: check-fixtures-changed + - uses: ./.github/actions/ui-comment + if: ${{ steps.check-fixtures-changed.outputs.FIXTURES_CHANGED == '1' }} # Connect # TODO: core_connect_test diff --git a/.github/workflows/legacy.yml b/.github/workflows/legacy.yml index 8e8ba0371b..c482d2aa80 100644 --- a/.github/workflows/legacy.yml +++ b/.github/workflows/legacy.yml @@ -2,6 +2,16 @@ name: Legacy on: [pull_request] +permissions: + id-token: write # for fetching the OIDC token + contents: read # for actions/checkout + pull-requests: write # For dflook comments on PR + +env: + PULL_COMMENT: | + |legacy UI changes|[master diff](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/)|[device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-legacy_device_test/)| + |-----------------|------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------| + jobs: legacy_firmware: name: Firmware @@ -78,8 +88,16 @@ jobs: path: legacy/firmware - run: chmod +x legacy/firmware/*.elf - uses: ./.github/actions/environment + - run: nix-shell --run "poetry run make -C legacy test_emu_ui" + if: matrix.coins == 'universal' - run: nix-shell --run "poetry run make -C legacy test_emu" - # if: matrix.coins == 'universal' && matrix.type == 'normal' + if: matrix.coins != 'universal' + - uses: ./.github/actions/ui-report + with: + model: T1B1 + if: ${{ always() && matrix.coins == 'universal' }} + - uses: ./.github/actions/ui-comment + if: ${{ failure() && matrix.coins == 'universal' }} legacy_upgrade_test: name: Upgrade test @@ -124,3 +142,42 @@ jobs: - run: nix-shell --run "git clone --depth=1 https://github.com/bitcoin-core/HWI.git" # see python_test for explanation of _PYTHON_SYSCONFIGDATA_NAME - run: nix-shell --arg fullDeps true --run "unset _PYTHON_SYSCONFIGDATA_NAME && cd HWI && poetry install && poetry run ./test/test_trezor.py --model_1 ../legacy/firmware/trezor.elf bitcoind" + + legacy_ui_master: + name: UI diff from master + runs-on: ubuntu-latest + needs: + - legacy_device_test + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v3 + with: + name: ui-records + - uses: ./.github/actions/environment + - name: Configure aws credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data + aws-region: eu-west-1 + - run: "for F in screens_*.tar; do tar xvf $F; done" + - run: nix-shell --run "poetry run python -m tests.ui_tests.reporting master-diff T1" + - run: mv tests/ui_tests/reports/master_diff . + - name: Upload master diff + run: | + aws s3 sync master_diff s3://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }} + + legacy_ui_comment: + name: Post comment with UI diff URLs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - run: | + git fetch origin main + git diff --quiet origin/main...HEAD -- tests/ui_tests/fixtures.json + echo "FIXTURES_CHANGED=$?" >> $GITHUB_OUTPUT + id: check-fixtures-changed + - uses: ./.github/actions/ui-comment + if: ${{ steps.check-fixtures-changed.outputs.FIXTURES_CHANGED == '1' }}