From 33b7e378c2d0af90c967238e6c6f2c5855ec6e00 Mon Sep 17 00:00:00 2001
From: Milan Rossa <rossa.milan@gmail.com>
Date: Mon, 27 May 2019 14:17:43 +0200
Subject: [PATCH] core/ci: add coverage target, introduce posttest stage

---
 .gitlab-ci.yml             |  2 ++
 ci/posttest.yml            | 21 +++++++++++++++++++++
 ci/test.yml                | 24 ++++++++++++++++++++++++
 core/Makefile              |  7 +++++++
 core/SConscript.unix       |  2 ++
 docs/tests/device-tests.md | 16 ++++++++++++++++
 6 files changed, 72 insertions(+)
 create mode 100644 ci/posttest.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5619cfae4..df1b8ef4f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,6 +26,7 @@ stages:
   - prebuild
   - build
   - test
+  - posttest
   - deploy
 
 before_script:
@@ -36,4 +37,5 @@ include:
   - ci/prebuild.yml
   - ci/build.yml
   - ci/test.yml
+  - ci/posttest.yml
   - ci/deploy.yml
diff --git a/ci/posttest.yml b/ci/posttest.yml
new file mode 100644
index 000000000..57990c11c
--- /dev/null
+++ b/ci/posttest.yml
@@ -0,0 +1,21 @@
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+
+core unix coverage posttest:
+  stage: posttest
+  dependencies:
+    - core unix device test
+    - core unix monero test
+    - core unix u2f test
+    - core unix fido2 test
+  script:
+    - cd core
+    - pipenv run make res  # we need to regenerate resources.py
+    - pipenv run make coverage
+  coverage: '/>\d+%</'
+  artifacts:
+    name: core-unix-coverage-posttest
+    paths:
+    - core/.coverage.*
+    - core/htmlcov
+    expire_in: 1 week
+
diff --git a/ci/test.yml b/ci/test.yml
index 926b575a3..6289f341a 100644
--- a/ci/test.yml
+++ b/ci/test.yml
@@ -43,15 +43,21 @@ core unix device test:
   <<: *only_changes_core
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - cd core
     - pipenv run make test_emu
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu
   artifacts:
     name: core-unix-device-test
     paths:
       - trezor.log
       - tests/junit.xml
+      - core/.coverage.*
     expire_in: 1 week
     when: always
     reports:
@@ -84,14 +90,20 @@ core unix monero test:
   <<: *only_changes_core
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - cd core
     - pipenv run make test_emu_monero
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu_monero
   artifacts:
     name: core-unix-monero-test
     paths:
       - trezor.log
+      - core/.coverage.*
     expire_in: 1 week
     when: always
 
@@ -101,15 +113,21 @@ core unix u2f test:
   retry: 2  # see https://github.com/trezor/trezor-firmware/issues/596
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - make -C tests/fido_tests/u2f-tests-hid
     - cd core
     - pipenv run make test_emu_u2f
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu_u2f
   artifacts:
     name: core-unix-u2f-test
     paths:
       - trezor.log
+      - core/.coverage.*
     expire_in: 1 week
     when: always
 
@@ -118,15 +136,21 @@ core unix fido2 test:
   <<: *only_changes_core
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - cd core
     - pipenv run make test_emu_fido2
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu_fido2
   artifacts:
     name: core-unix-fido2-test
     paths:
       - trezor.log
       - tests/junit.xml
+      - core/.coverage.*
     expire_in: 1 week
     reports:
       junit: tests/junit.xml
diff --git a/core/Makefile b/core/Makefile
index 52dba4601..f8b721451 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -234,3 +234,10 @@ upload: ## upload firmware using trezorctl
 
 upload_prodtest: ## upload prodtest using trezorctl
 	trezorctl firmware_update -f $(PRODTEST_BUILD_DIR)/prodtest.bin
+
+coverage:  # generate coverage report
+	coverage run --source=./src /dev/null 2>/dev/null && \
+	mv .coverage .coverage.empty && \
+	coverage combine .coverage.* && \
+	coverage html
+	grep pc_cov htmlcov/index.html
diff --git a/core/SConscript.unix b/core/SConscript.unix
index 13cd861d8..d2ed5ff20 100644
--- a/core/SConscript.unix
+++ b/core/SConscript.unix
@@ -265,6 +265,7 @@ SOURCE_MICROPYTHON = [
     'vendor/micropython/py/parsenum.c',
     'vendor/micropython/py/parsenumbase.c',
     'vendor/micropython/py/persistentcode.c',
+    'vendor/micropython/py/profile.c',
     'vendor/micropython/py/pystack.c',
     'vendor/micropython/py/qstr.c',
     'vendor/micropython/py/reader.c',
@@ -295,6 +296,7 @@ SOURCE_UNIX = [
     'embed/unix/touch.c',
     'embed/unix/usb.c',
     'vendor/micropython/ports/unix/alloc.c',
+    'vendor/micropython/ports/unix/file.c',
     'vendor/micropython/ports/unix/gccollect.c',
     'vendor/micropython/ports/unix/input.c',
     'vendor/micropython/ports/unix/unix_mphal.c',
diff --git a/docs/tests/device-tests.md b/docs/tests/device-tests.md
index 42928b189..f47e98d3b 100644
--- a/docs/tests/device-tests.md
+++ b/docs/tests/device-tests.md
@@ -91,3 +91,19 @@ If you wan't to catch some memory errors use this.
 ```sh
 time ASAN_OPTIONS=verbosity=1:detect_invalid_pointer_pairs=1:strict_init_order=true:strict_string_checks=true TREZOR_PROFILE="" pipenv run make test_emu
 ```
+
+### Coverage (Emulator only)
+
+Get the Python code coverage report.
+
+If you want to get HTML/console summary output you need to install the __coverage.py__ tool.
+
+```sh
+pip3 install coverage
+```
+
+Run the tests with coverage output.
+
+```sh
+make build_unix && make coverage
+```