Merge branch 'coverage'

.gitlab-ci.yml

@@ -26,6 +26,7 @@ stages:
   - prebuild
   - build
   - test
+  - posttest
   - deploy
 
 before_script:
@@ -36,4 +37,5 @@ include:
   - ci/prebuild.yml
   - ci/build.yml
   - ci/test.yml
+  - ci/posttest.yml
   - ci/deploy.yml

ci/posttest.yml

@@ -0,0 +1,21 @@
+image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/environment
+
+core unix coverage posttest:
+  stage: posttest
+  dependencies:
+    - core unix device test
+    - core unix monero test
+    - core unix u2f test
+    - core unix fido2 test
+  script:
+    - cd core
+    - pipenv run make res  # we need to regenerate resources.py
+    - pipenv run make coverage
+  coverage: '/>\d+%</'
+  artifacts:
+    name: core-unix-coverage-posttest
+    paths:
+    - core/.coverage.*
+    - core/htmlcov
+    expire_in: 1 week
+

ci/test.yml

@@ -43,15 +43,21 @@ core unix device test:
   <<: *only_changes_core
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - cd core
     - pipenv run make test_emu
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu
   artifacts:
     name: core-unix-device-test
     paths:
       - trezor.log
       - tests/junit.xml
+      - core/.coverage.*
     expire_in: 1 week
     when: always
     reports:
@@ -84,14 +90,20 @@ core unix monero test:
   <<: *only_changes_core
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - cd core
     - pipenv run make test_emu_monero
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu_monero
   artifacts:
     name: core-unix-monero-test
     paths:
       - trezor.log
+      - core/.coverage.*
     expire_in: 1 week
     when: always
 
@@ -101,15 +113,21 @@ core unix u2f test:
   retry: 2  # see https://github.com/trezor/trezor-firmware/issues/596
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - make -C tests/fido_tests/u2f-tests-hid
     - cd core
     - pipenv run make test_emu_u2f
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu_u2f
   artifacts:
     name: core-unix-u2f-test
     paths:
       - trezor.log
+      - core/.coverage.*
     expire_in: 1 week
     when: always
 
@@ -118,15 +136,21 @@ core unix fido2 test:
   <<: *only_changes_core
   dependencies:
     - core unix frozen regular build
+  variables:
+    TREZOR_PROFILING: 1
   script:
     - cd core
     - pipenv run make test_emu_fido2
     - cp /var/tmp/trezor.log ${CI_PROJECT_DIR}
+    - sync
+    - sleep 1
+    - mv ./src/.coverage .coverage.test_emu_fido2
   artifacts:
     name: core-unix-fido2-test
     paths:
       - trezor.log
       - tests/junit.xml
+      - core/.coverage.*
     expire_in: 1 week
     reports:
       junit: tests/junit.xml

core/Makefile

@@ -234,3 +234,10 @@ upload: ## upload firmware using trezorctl
 
 upload_prodtest: ## upload prodtest using trezorctl
 	trezorctl firmware_update -f $(PRODTEST_BUILD_DIR)/prodtest.bin
+
+coverage:  # generate coverage report
+	coverage run --source=./src /dev/null 2>/dev/null && \
+	mv .coverage .coverage.empty && \
+	coverage combine .coverage.* && \
+	coverage html
+	grep pc_cov htmlcov/index.html

core/SConscript.unix

@@ -265,6 +265,7 @@ SOURCE_MICROPYTHON = [
     'vendor/micropython/py/parsenum.c',
     'vendor/micropython/py/parsenumbase.c',
     'vendor/micropython/py/persistentcode.c',
+    'vendor/micropython/py/profile.c',
     'vendor/micropython/py/pystack.c',
     'vendor/micropython/py/qstr.c',
     'vendor/micropython/py/reader.c',
@@ -295,6 +296,7 @@ SOURCE_UNIX = [
     'embed/unix/touch.c',
     'embed/unix/usb.c',
     'vendor/micropython/ports/unix/alloc.c',
+    'vendor/micropython/ports/unix/file.c',
     'vendor/micropython/ports/unix/gccollect.c',
     'vendor/micropython/ports/unix/input.c',
     'vendor/micropython/ports/unix/unix_mphal.c',

core/embed/unix/main.c

@@ -461,7 +461,7 @@ MP_NOINLINE int main_(int argc, char **argv) {
   signal(SIGPIPE, SIG_IGN);
 #endif
 
-  mp_stack_set_limit(60000 * (BYTES_PER_WORD / 4));
+  mp_stack_set_limit(600000 * (BYTES_PER_WORD / 4));
 
   pre_process_options(argc, argv);
 

core/prof/prof.py

@@ -0,0 +1,70 @@
+import sys
+
+import uos
+from uio import open
+
+sys.path.insert(0, uos.getenv("TREZOR_SRC"))
+del uos
+
+
+class Coverage:
+    def __init__(self):
+        self.__files = {}
+
+    def line_tick(self, filename, lineno):
+        if not filename in self.__files:
+            self.__files[filename] = set()
+        self.__files[filename].add(lineno)
+
+    def lines_execution(self):
+        lines_execution = {"lines": {}}
+        lines = lines_execution["lines"]
+        this_file = globals()["__file__"]
+        for filename in self.__files:
+            if not filename == this_file:
+                lines[filename] = list(self.__files[filename])
+
+        return lines_execution
+
+
+class _Prof:
+    trace_count = 0
+    display_flags = 0
+    __coverage = Coverage()
+
+    def trace_tick(self, frame, event, arg):
+        self.trace_count += 1
+
+        # if frame.f_code.co_filename.endswith('/loop.py'):
+        #     print(event, frame.f_code.co_filename, frame.f_lineno)
+
+        if event == "line":
+            self.__coverage.line_tick(frame.f_code.co_filename, frame.f_lineno)
+
+    def coverage_data(self):
+        return self.__coverage.lines_execution()
+
+
+def trace_handler(frame, event, arg):
+    __prof__.trace_tick(frame, event, arg)
+    return trace_handler
+
+
+def atexit():
+    print("\n------------------ script exited ------------------")
+    print("Total traces executed: ", __prof__.trace_count)
+    with open(".coverage", "w") as f:
+        # wtf so private much beautiful wow
+        f.write("!coverage.py: This is a private format, don't read it directly!")
+        # poormans json
+        f.write(str(__prof__.coverage_data()).replace("'", '"'))
+
+
+sys.atexit(atexit)
+
+global __prof__
+if not "__prof__" in globals():
+    __prof__ = _Prof()
+
+sys.settrace(trace_handler)
+import main

docs/tests/device-tests.md

@@ -91,3 +91,19 @@ If you wan't to catch some memory errors use this.
 ```sh
 time ASAN_OPTIONS=verbosity=1:detect_invalid_pointer_pairs=1:strict_init_order=true:strict_string_checks=true TREZOR_PROFILE="" pipenv run make test_emu
 ```
+
+### Coverage (Emulator only)
+
+Get the Python code coverage report.
+
+If you want to get HTML/console summary output you need to install the __coverage.py__ tool.
+
+```sh
+pip3 install coverage
+```
+
+Run the tests with coverage output.
+
+```sh
+make build_unix && make coverage
+```