core/ci: add coverage target, introduce posttest stage

.gitlab-ci.yml

@@ -26,6 +26,7 @@ stages:
   - prebuild
   - build
   - test
+  - posttest
   - deploy
 
 before_script:
@@ -36,4 +37,5 @@ include:
   - ci/prebuild.yml
   - ci/build.yml
   - ci/test.yml
+  - ci/posttest.yml
   - ci/deploy.yml

ci/posttest.yml

@@ -0,0 +1,21 @@
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+
+core unix coverage posttest:
+  stage: posttest
+  dependencies:
+    - core unix device test
+    - core unix monero test
+    - core unix u2f test
+    - core unix fido2 test
+  script:
+    - cd core
+    - pipenv run make res  # we need to regenerate resources.py
+    - pipenv run make coverage
+  coverage: '/>\d+%</'
+  artifacts:
+    name: core-unix-coverage-posttest
+    paths:
+    - core/.coverage.*
+    - core/htmlcov
+    expire_in: 1 week
+

ci/test.yml

@@ -43,15 +43,21 @@ core unix device test:
   <<: *only_changes_core
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - cd core
     - pipenv run make test_emu
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu
   artifacts:
     name: core-unix-device-test
     paths:
       - trezor.log
       - tests/junit.xml
+      - core/.coverage.*
     expire_in: 1 week
     when: always
     reports:
@@ -84,14 +90,20 @@ core unix monero test:
   <<: *only_changes_core
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - cd core
     - pipenv run make test_emu_monero
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu_monero
   artifacts:
     name: core-unix-monero-test
     paths:
       - trezor.log
+      - core/.coverage.*
     expire_in: 1 week
     when: always
 
@@ -101,15 +113,21 @@ core unix u2f test:
   retry: 2  # see https://github.com/trezor/trezor-firmware/issues/596
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - make -C tests/fido_tests/u2f-tests-hid
     - cd core
     - pipenv run make test_emu_u2f
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu_u2f
   artifacts:
     name: core-unix-u2f-test
     paths:
       - trezor.log
+      - core/.coverage.*
     expire_in: 1 week
     when: always
 
@@ -118,15 +136,21 @@ core unix fido2 test:
   <<: *only_changes_core
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - cd core
     - pipenv run make test_emu_fido2
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu_fido2
   artifacts:
     name: core-unix-fido2-test
     paths:
       - trezor.log
       - tests/junit.xml
+      - core/.coverage.*
     expire_in: 1 week
     reports:
       junit: tests/junit.xml

core/Makefile

@@ -234,3 +234,10 @@ upload: ## upload firmware using trezorctl
 
 upload_prodtest: ## upload prodtest using trezorctl
 	trezorctl firmware_update -f $(PRODTEST_BUILD_DIR)/prodtest.bin
+
+coverage:  # generate coverage report
+	coverage run --source=./src /dev/null 2>/dev/null && \
+	mv .coverage .coverage.empty && \
+	coverage combine .coverage.* && \
+	coverage html
+	grep pc_cov htmlcov/index.html

core/SConscript.unix

@@ -265,6 +265,7 @@ SOURCE_MICROPYTHON = [
     'vendor/micropython/py/parsenum.c',
     'vendor/micropython/py/parsenumbase.c',
     'vendor/micropython/py/persistentcode.c',
+    'vendor/micropython/py/profile.c',
     'vendor/micropython/py/pystack.c',
     'vendor/micropython/py/qstr.c',
     'vendor/micropython/py/reader.c',
@@ -295,6 +296,7 @@ SOURCE_UNIX = [
     'embed/unix/touch.c',
     'embed/unix/usb.c',
     'vendor/micropython/ports/unix/alloc.c',
+    'vendor/micropython/ports/unix/file.c',
     'vendor/micropython/ports/unix/gccollect.c',
     'vendor/micropython/ports/unix/input.c',
     'vendor/micropython/ports/unix/unix_mphal.c',

docs/tests/device-tests.md

@@ -91,3 +91,19 @@ If you wan't to catch some memory errors use this.
 ```sh
 time ASAN_OPTIONS=verbosity=1:detect_invalid_pointer_pairs=1:strict_init_order=true:strict_string_checks=true TREZOR_PROFILE="" pipenv run make test_emu
 ```
+
+### Coverage (Emulator only)
+
+Get the Python code coverage report.
+
+If you want to get HTML/console summary output you need to install the __coverage.py__ tool.
+
+```sh
+pip3 install coverage
+```
+
+Run the tests with coverage output.
+
+```sh
+make build_unix && make coverage
+```