ci: HTML UI reports for github actions

Co-authored-by: vdovhanych <dovhanych@me.com>

.github/actions/ui-comment/action.yml

@@ -0,0 +1,22 @@
+name: 'UI comment links'
+description: 'Post URLs of HTML test reports to pull request discussion'
+runs:
+  using: composite
+  steps:
+    - name: Find Comment
+      uses: peter-evans/find-comment@v2
+      id: fc
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        comment-author: 'github-actions[bot]'
+        body-includes: ui-comment-${{ github.workflow }}
+
+    - name: Create or update comment
+      uses: peter-evans/create-or-update-comment@v3
+      with:
+        comment-id: ${{ steps.fc.outputs.comment-id }}
+        issue-number: ${{ github.event.pull_request.number }}
+        body: |
+          <!-- ui-comment-${{ github.workflow }} -->
+          ${{ env.PULL_COMMENT }}
+        edit-mode: replace

.github/actions/ui-report/action.yml

@@ -1,33 +1,41 @@
 name: 'UI report'
 description: 'Prepare and upload HTML report of UI test results'
 inputs:
-  artifact-name:
+  model:
-    description: 'Name of the uploaded artifact'
+    description: 'Internal model name'
     required: true
-    default: ui-report
 runs:
   using: composite
   steps:
-    - run: mv tests/ui_tests/reports/test/ test_ui_report || true
+    - name: Set AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data
+        aws-region: eu-west-1
+    - run: |
+        MODELJOB=${{ inputs.model }}-${{ github.job }}
+        OUTDIR=${{ github.run_id }}/$MODELJOB
+        mkdir -p $OUTDIR
+        nix-shell --run "poetry run python ci/prepare_ui_artifacts.py || true"
+        mv tests/ui_tests/reports/test/* $OUTDIR || true
+        mv tests/ui_tests/fixtures.*.json $OUTDIR || true
+        mv tests/trezor.log $OUTDIR || true
+        diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json || true
+        tar -cf screens_$MODELJOB.tar tests/ui_tests/screens || true
       shell: sh
-    - run: nix-shell --run "poetry run python ci/prepare_ui_artifacts.py || true"
+    - name: Upload report
+      run: |
+        aws s3 sync ${{ github.run_id }} s3://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}
       shell: sh
-    - run: diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json || true
+    - name: Upload test screen recording
-      shell: sh
+      run: |
-    - run: tar -cf test_ui_report.tar test_ui_report/ || true
+        aws s3 sync ci/ui_test_records s3://data.trezor.io/dev/firmware/ui_tests
-      shell: sh
+        # TODO: generate directory listing / autoindex
-    - run: tar -cf tests/ui_tests/screens.tar tests/ui_tests/screens/ || true
       shell: sh
     - uses: actions/upload-artifact@v3
       with:
-        name: ${{ inputs.artifact-name }}
+        name: ui-records
         path: |
-          ci/ui_test_records/
+          # used by core_ui_master
-          # test_ui_report/  # can't have :: on ntfs
+          screens_${{ inputs.model }}-${{ github.job }}.tar
-          # tests/ui_tests/screens/  # can't have :: on ntfs
+        retention-days: 1  # not useful after workflow finishes
-          test_ui_report.tar
-          tests/ui_tests/screens.tar
-          tests/ui_tests/fixtures.suggestion.json
-          tests/ui_tests/fixtures.results.json
-          tests/trezor.log
-        retention-days: 7

.github/workflows/core.yml

@@ -2,6 +2,18 @@ name: Core
 
 on: [pull_request]
 
+permissions:
+  id-token: write      # for fetching the OIDC token
+  contents: read       # for actions/checkout
+  pull-requests: write # For dflook comments on PR
+
+env:
+  PULL_COMMENT: |
+    |core UI changes|[master diff](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/)|    |    |
+    |---------------|------------------------------------------------------------------------------------|----|----|
+    |Model T     |[device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_device_test/)|[click test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_click_test/)|[persistence test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_persistence_test/)|
+    |Model Safe 3|[device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_device_test/)|[click test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_click_test/)|[persistence test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_persistence_test/)|
+
 jobs:
   core_firmware:
     name: Build firmware
@@ -171,6 +183,7 @@ jobs:
       TREZOR_PYTEST_SKIP_ALTCOINS: ${{ matrix.coins == 'btconly' && '1' || '0' }}
       ADDRESS_SANITIZER: ${{ matrix.asan == 'asan' && '1' || '0' }}
       PYTEST_TIMEOUT: ${{ matrix.asan == 'asan' && 600 || 400 }}
+      ACTIONS_DO_UI_TEST: ${{ matrix.coins == 'universal' && matrix.asan == 'noasan' }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -181,21 +194,26 @@ jobs:
           path: core/build
       - run: chmod +x core/build/unix/trezor-emu-core*
       - uses: ./.github/actions/environment
-      - run: nix-shell --run "poetry run make -C core test_emu_ui_multicore"  # TODO: can-fail or whatisit
+      - run: nix-shell --run "poetry run make -C core ${{ env.ACTIONS_DO_UI_TEST == 'true' && 'test_emu_ui_multicore' || 'test_emu' }}"
-        if: ${{ matrix.asan == 'noasan' && matrix.coins == 'universal' }}
+      - run: tail -n50 tests/trezor.log || true
-      - run: nix-shell --run "poetry run make -C core test_emu"
+        if: failure()
-        if: ${{ matrix.asan != 'noasan' || matrix.coins != 'universal' }}
+      - uses: actions/upload-artifact@v3
-      - run: tail -n20 tests/trezor.log || true
+        with:
-        if: ${{ failure() }}
+          name: core-device-${{ matrix.model }}-${{ matrix.asan }}
+          path: tests/trezor.log
+          retention-days: 7
+        if: always()
       - uses: ./.github/actions/ui-report
         with:
-          artifact-name: core-test-device-${{ matrix.model }}-${{ matrix.coins }}-${{ matrix.asan }}
+          model: ${{ matrix.model }}
-        if: ${{ always() }}
+        if: ${{ always() && env.ACTIONS_DO_UI_TEST == 'true' }}
-      - run: mv core/src/.coverage.* core || true  # there will be more coverage files (one per core)
+      - uses: ./.github/actions/ui-comment
+        if: ${{ failure() && env.ACTIONS_DO_UI_TEST == 'true' }}
       - uses: actions/upload-artifact@v3
         with:
           name: core-coverage-${{ matrix.model }}
-          path: core/.coverage.*
+          # there will be more coverage files (one per core)
+          path: core/src/.coverage.*
           retention-days: 7
 
   # Click tests - UI.
@@ -226,9 +244,16 @@ jobs:
         if: ${{ matrix.asan == 'noasan' }}
       - run: nix-shell --run "poetry run make -C core test_emu_click"
         if: ${{ matrix.asan == 'asan' }}
+      - uses: actions/upload-artifact@v3
+        with:
+          name: core-click-${{ matrix.model }}-${{ matrix.asan }}
+          path: tests/trezor.log
+          retention-days: 7
+        if: always()
       - uses: ./.github/actions/ui-report
         with:
-          artifact-name: core-test-click-${{ matrix.model }}-${{ matrix.asan }}
+          model: ${{ matrix.model }}
+        if: always()
       - run: mv core/src/.coverage core/.coverage.test_click || true
       - uses: actions/upload-artifact@v3
         with:
@@ -264,7 +289,7 @@ jobs:
 
 
   # Persistence tests - UI.
-  core_persitence_test:
+  core_persistence_test:
     name: Persistence tests
     runs-on: ubuntu-latest
     needs: core_emu
@@ -291,7 +316,8 @@ jobs:
         if: ${{ matrix.asan == 'asan' }}
       - uses: ./.github/actions/ui-report
         with:
-          artifact-name: core-test-persistence-${{ matrix.model }}-${{ matrix.asan }}
+          model: ${{ matrix.model }}
+        if: always()
       - run: mv core/src/.coverage core/.coverage.test_persistence || true
       - uses: actions/upload-artifact@v3
         with:
@@ -352,6 +378,7 @@ jobs:
             tests/trezor.log
             core/prof/memperf-html
           retention-days: 7
+        if: always()
 
   # Flash size profiling
 
@@ -437,6 +464,7 @@ jobs:
             tests/trezor.log
             core/tests/trezor_monero_tests.log
           retention-days: 7
+        if: always()
       - run: mv core/src/.coverage core/.coverage.test_emu_monero || true
       - uses: actions/upload-artifact@v3
         with:
@@ -473,6 +501,7 @@ jobs:
           name: core-test-u2f-${{ matrix.model }}-${{ matrix.asan }}
           path: tests/trezor.log
           retention-days: 7
+        if: always()
       - run: mv core/src/.coverage core/.coverage.test_emu_u2f || true
       - uses: actions/upload-artifact@v3
         with:
@@ -509,6 +538,7 @@ jobs:
           path: |
             tests/trezor.log
           retention-days: 7
+        if: always()
       - run: mv core/src/.coverage core/.coverage.test_emu_fido2 || true
       - uses: actions/upload-artifact@v3
         with:
@@ -521,7 +551,7 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - core_click_test
-      - core_persitence_test
+      - core_persistence_test
       - core_device_test
       - core_monero_test
       - core_u2f_test
@@ -545,13 +575,51 @@ jobs:
           path: core
       - uses: ./.github/actions/environment
       - run: nix-shell --run "poetry run make -C core coverage"
-      # TODO fail if too little
       - uses: actions/upload-artifact@v3
         with:
           name: core-coverage-${{ matrix.model }}
           path: core/htmlcov
           retention-days: 7
 
+  core_ui_master:
+    name: UI diff from master
+    runs-on: ubuntu-latest
+    needs:
+      - core_click_test
+      - core_persistence_test
+      - core_device_test
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v3
+        with:
+          name: ui-records
+      - uses: ./.github/actions/environment
+      - name: Configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data
+          aws-region: eu-west-1
+      - run: "for F in screens_*.tar; do tar xvf $F; done"
+      - run: nix-shell --run "poetry run python -m tests.ui_tests.reporting master-diff TT TR"
+      - run: mv tests/ui_tests/reports/master_diff .
+      - name: Upload master diff
+        run: |
+          aws s3 sync master_diff s3://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}
+
+  core_ui_comment:
+    name: Post comment with UI diff URLs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+      - run: |
+          git fetch origin main
+          git diff --quiet origin/main...HEAD -- tests/ui_tests/fixtures.json || echo "FIXTURES_CHANGED=$?" >> $GITHUB_OUTPUT
+        id: check-fixtures-changed
+      - uses: ./.github/actions/ui-comment
+        if: ${{ steps.check-fixtures-changed.outputs.FIXTURES_CHANGED == '1' }}
 
   # Connect
   # TODO: core_connect_test

.github/workflows/legacy.yml

@@ -2,6 +2,16 @@ name: Legacy
 
 on: [pull_request]
 
+permissions:
+  id-token: write      # for fetching the OIDC token
+  contents: read       # for actions/checkout
+  pull-requests: write # For dflook comments on PR
+
+env:
+  PULL_COMMENT: |
+    |legacy UI changes|[master diff](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/)|[device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-legacy_device_test/)|
+    |-----------------|------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------|
+
 jobs:
   legacy_firmware:
     name: Firmware
@@ -78,8 +88,16 @@ jobs:
           path: legacy/firmware
       - run: chmod +x legacy/firmware/*.elf
       - uses: ./.github/actions/environment
+      - run: nix-shell --run "poetry run make -C legacy test_emu_ui"
+        if: matrix.coins == 'universal'
       - run: nix-shell --run "poetry run make -C legacy test_emu"
-        # if: matrix.coins == 'universal' && matrix.type == 'normal'
+        if: matrix.coins != 'universal'
+      - uses: ./.github/actions/ui-report
+        with:
+          model: T1B1
+        if: ${{ always() && matrix.coins == 'universal' }}
+      - uses: ./.github/actions/ui-comment
+        if: ${{ failure() && matrix.coins == 'universal' }}
 
   legacy_upgrade_test:
     name: Upgrade test
@@ -124,3 +142,42 @@ jobs:
       - run: nix-shell --run "git clone --depth=1 https://github.com/bitcoin-core/HWI.git"
       # see python_test for explanation of _PYTHON_SYSCONFIGDATA_NAME
       - run: nix-shell --arg fullDeps true --run "unset _PYTHON_SYSCONFIGDATA_NAME && cd HWI && poetry install && poetry run ./test/test_trezor.py --model_1 ../legacy/firmware/trezor.elf bitcoind"
+
+  legacy_ui_master:
+    name: UI diff from master
+    runs-on: ubuntu-latest
+    needs:
+      - legacy_device_test
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v3
+        with:
+          name: ui-records
+      - uses: ./.github/actions/environment
+      - name: Configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data
+          aws-region: eu-west-1
+      - run: "for F in screens_*.tar; do tar xvf $F; done"
+      - run: nix-shell --run "poetry run python -m tests.ui_tests.reporting master-diff T1"
+      - run: mv tests/ui_tests/reports/master_diff .
+      - name: Upload master diff
+        run: |
+          aws s3 sync master_diff s3://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}
+
+  legacy_ui_comment:
+    name: Post comment with UI diff URLs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+      - run: |
+          git fetch origin main
+          git diff --quiet origin/main...HEAD -- tests/ui_tests/fixtures.json
+          echo "FIXTURES_CHANGED=$?" >> $GITHUB_OUTPUT
+        id: check-fixtures-changed
+      - uses: ./.github/actions/ui-comment
+        if: ${{ steps.check-fixtures-changed.outputs.FIXTURES_CHANGED == '1' }}