ci: introduce upgrade tests

closes #126

ci/build.yml

@@ -71,6 +71,18 @@ build core unix frozen bitcoinonly:
     - core/src/trezor/res/resources.py
     expire_in: 1 day
 
+build core unix frozen debug:
+  stage: build
+  variables:
+    PYOPT: "0"
+  script:
+    - cd core
+    - pipenv run make build_unix_frozen
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA"
+    untracked: true
+    expire_in: 1 day
+
 
 # Crypto
 

ci/test.yml

@@ -96,3 +96,15 @@ test storage:
     - cd storage/tests
     - pipenv run make build
     - pipenv run make tests_all
+
+
+# Other
+
+test upgrade:
+  stage: test
+  dependencies:
+    - build core unix frozen debug
+    - build legacy emu
+  script:
+    - tests/upgrade_tests/download_emulators.sh
+    - pipenv run pytest tests/upgrade_tests

core/tests/run_tests_device_emu.sh

@@ -22,7 +22,7 @@ export TREZOR_PATH=udp:127.0.0.1:21324
 
 # run tests
 error=0
-if ! pytest ../../tests "$@"; then
+if ! pytest ../../tests/device_tests "$@"; then
   error=1
 fi
 kill $upy_pid

legacy/script/test

@@ -16,4 +16,4 @@ if [ "$EMULATOR" = 1 ]; then
     "${PYTHON:-python}" script/wait_for_emulator.py
 fi
 
-"${PYTHON:-python}" -m pytest ../tests "$@"
+"${PYTHON:-python}" -m pytest ../tests/device_tests "$@"

tests/upgrade_tests/download_emulators.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+SITE="https://firmware.corp.sldev.cz/upgrade_tests/"
+cd "$(dirname "$0")"
+
+# download all emulators without index files, without directories and only if not present
+wget --no-verbose --no-clobber --no-parent --cut-dirs=2 --no-host-directories --recursive --reject "index.html*" -P emulators/ $SITE
+
+# TODO: is this a good idea?
+chmod u+x emulators/trezor-emu-*

tests/upgrade_tests/emulator_wrapper.py

@@ -0,0 +1,63 @@
+import os
+import subprocess
+import tempfile
+import time
+
+from trezorlib.debuglink import TrezorClientDebugLink
+from trezorlib.transport import TransportException, get_transport
+
+BINDIR = os.path.dirname(os.path.abspath(__file__)) + "/emulators"
+ENV = {"SDL_VIDEODRIVER": "dummy"}
+
+
+class EmulatorWrapper:
+    def __init__(self, gen, tag, storage=None):
+        self.gen = gen
+        self.tag = tag
+        self.workdir = tempfile.TemporaryDirectory()
+        if storage:
+            open(self._storage_file(), "wb").write(storage)
+
+    def __enter__(self):
+        if self.tag.startswith("/"):  # full path+filename provided
+            args = [self.tag]
+        else:  # only gen+tag provided
+            args = ["%s/trezor-emu-%s-%s" % (BINDIR, self.gen, self.tag)]
+        env = ENV
+        if self.gen == "core":
+            args += ["-m", "main"]
+            env["TREZOR_PROFILE_DIR"] = self.workdir.name
+        self.process = subprocess.Popen(
+            args, cwd=self.workdir.name, env=ENV, stdout=open(os.devnull, "w")
+        )
+        # wait until emulator is started
+        while True:
+            try:
+                self.transport = get_transport("udp:127.0.0.1:21324")
+            except TransportException:
+                time.sleep(0.1)
+                continue
+            break
+        self.client = TrezorClientDebugLink(self.transport)
+        self.client.open()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.client.close()
+        self.process.terminate()
+        try:
+            self.process.wait(1)
+        except subprocess.TimeoutExpired:
+            self.process.kill()
+        self.workdir.cleanup()
+
+    def _storage_file(self):
+        if self.gen == "legacy":
+            return self.workdir.name + "/emulator.img"
+        elif self.gen == "core":
+            return self.workdir.name + "/trezor.flash"
+        else:
+            raise ValueError("Unknown gen")
+
+    def storage(self):
+        return open(self._storage_file(), "rb").read()

tests/upgrade_tests/emulators/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

tests/upgrade_tests/test_firmware_upgrades.py

@@ -0,0 +1,223 @@
+import os
+from collections import defaultdict
+
+import pytest
+
+from trezorlib import MINIMUM_FIRMWARE_VERSION, btc, debuglink, device
+from trezorlib.tools import H_
+
+MINIMUM_FIRMWARE_VERSION["1"] = (1, 0, 0)
+MINIMUM_FIRMWARE_VERSION["T"] = (2, 0, 0)
+
+try:
+    from .emulator_wrapper import EmulatorWrapper
+except ImportError:
+    pass
+
+# **** COMMON DEFINITIONS ****
+
+MNEMONIC = " ".join(["all"] * 12)
+PATH = [H_(44), H_(0), H_(0), 0, 0]
+ADDRESS = "1JAd7XCBzGudGpJQSDSfpmJhiygtLQWaGL"
+LABEL = "test"
+LANGUAGE = "english"
+STRENGTH = 128
+
+ROOT = os.path.dirname(os.path.abspath(__file__)) + "/../../"
+LOCAL_BUILDS = {
+    "core": ROOT + "core/build/unix/micropython",
+    "legacy": ROOT + "legacy/firmware/trezor.elf",
+}
+BIN_DIR = os.path.dirname(os.path.abspath(__file__)) + "/emulators"
+
+
+def check_version(tag, ver_emu):
+    if tag.startswith("v") and len(tag.split(".")) == 3:
+        assert tag == "v" + ".".join(["%d" % i for i in ver_emu])
+
+
+def check_file(gen, tag):
+    if tag.startswith("/"):
+        filename = tag
+    else:
+        filename = "%s/trezor-emu-%s-%s" % (BIN_DIR, gen, tag)
+    if not os.path.exists(filename):
+        raise ValueError(filename + " not found. Do not forget to build firmware.")
+
+
+def get_tags():
+    files = os.listdir(BIN_DIR)
+    if not files:
+        raise ValueError(
+            "No files found. Use download_emulators.sh to download emulators."
+        )
+
+    result = defaultdict(list)
+    for f in sorted(files):
+        try:
+            _, _, gen, tag = f.split("-", maxsplit=3)
+            result[gen].append(tag)
+        except ValueError:
+            pass
+    return result
+
+
+ALL_TAGS = get_tags()
+
+
+def for_all(*args, minimum_version=(1, 0, 0)):
+    if not args:
+        args = ("core", "legacy")
+
+    all_params = []
+    for gen in args:
+        try:
+            to_tag = LOCAL_BUILDS[gen]
+            from_tags = ALL_TAGS[gen] + [to_tag]
+            for from_tag in from_tags:
+                if from_tag.startswith("v"):
+                    tag_version = tuple(int(n) for n in from_tag[1:].split("."))
+                    if tag_version < minimum_version:
+                        continue
+                check_file(gen, from_tag)
+                all_params.append((gen, from_tag, to_tag))
+        except KeyError:
+            pass
+
+    return pytest.mark.parametrize("gen, from_tag, to_tag", all_params)
+
+
+@for_all()
+def test_upgrade_load(gen, from_tag, to_tag):
+    def asserts(tag, client):
+        check_version(tag, emu.client.version)
+        assert not client.features.pin_protection
+        assert not client.features.passphrase_protection
+        assert client.features.initialized
+        assert client.features.label == LABEL
+        assert client.features.language == LANGUAGE
+        assert btc.get_address(client, "Bitcoin", PATH) == ADDRESS
+
+    with EmulatorWrapper(gen, from_tag) as emu:
+        debuglink.load_device_by_mnemonic(
+            emu.client,
+            mnemonic=MNEMONIC,
+            pin="",
+            passphrase_protection=False,
+            label=LABEL,
+            language=LANGUAGE,
+        )
+        device_id = emu.client.features.device_id
+        asserts(from_tag, emu.client)
+        storage = emu.storage()
+
+    with EmulatorWrapper(gen, to_tag, storage=storage) as emu:
+        assert device_id == emu.client.features.device_id
+        asserts(to_tag, emu.client)
+
+
+@for_all("legacy")
+def test_upgrade_reset(gen, from_tag, to_tag):
+    def asserts(tag, client):
+        check_version(tag, emu.client.version)
+        assert not client.features.pin_protection
+        assert not client.features.passphrase_protection
+        assert client.features.initialized
+        assert client.features.label == LABEL
+        assert client.features.language == LANGUAGE
+        assert not client.features.needs_backup
+        assert not client.features.unfinished_backup
+        assert not client.features.no_backup
+
+    with EmulatorWrapper(gen, from_tag) as emu:
+        device.reset(
+            emu.client,
+            display_random=False,
+            strength=STRENGTH,
+            passphrase_protection=False,
+            pin_protection=False,
+            label=LABEL,
+            language=LANGUAGE,
+        )
+        device_id = emu.client.features.device_id
+        asserts(from_tag, emu.client)
+        storage = emu.storage()
+
+    with EmulatorWrapper(gen, to_tag, storage=storage) as emu:
+        assert device_id == emu.client.features.device_id
+        asserts(to_tag, emu.client)
+
+
+@for_all()
+def test_upgrade_reset_skip_backup(gen, from_tag, to_tag):
+    def asserts(tag, client):
+        check_version(tag, emu.client.version)
+        assert not client.features.pin_protection
+        assert not client.features.passphrase_protection
+        assert client.features.initialized
+        assert client.features.label == LABEL
+        assert client.features.language == LANGUAGE
+        assert client.features.needs_backup
+        assert not client.features.unfinished_backup
+        assert not client.features.no_backup
+
+    with EmulatorWrapper(gen, from_tag) as emu:
+        device.reset(
+            emu.client,
+            display_random=False,
+            strength=STRENGTH,
+            passphrase_protection=False,
+            pin_protection=False,
+            label=LABEL,
+            language=LANGUAGE,
+            skip_backup=True,
+        )
+        device_id = emu.client.features.device_id
+        asserts(from_tag, emu.client)
+        storage = emu.storage()
+
+    with EmulatorWrapper(gen, to_tag, storage=storage) as emu:
+        assert device_id == emu.client.features.device_id
+        asserts(to_tag, emu.client)
+
+
+@for_all(minimum_version=(1, 7, 2))
+def test_upgrade_reset_no_backup(gen, from_tag, to_tag):
+    def asserts(tag, client):
+        check_version(tag, emu.client.version)
+        assert not client.features.pin_protection
+        assert not client.features.passphrase_protection
+        assert client.features.initialized
+        assert client.features.label == LABEL
+        assert client.features.language == LANGUAGE
+        assert not client.features.needs_backup
+        assert not client.features.unfinished_backup
+        assert client.features.no_backup
+
+    with EmulatorWrapper(gen, from_tag) as emu:
+        device.reset(
+            emu.client,
+            display_random=False,
+            strength=STRENGTH,
+            passphrase_protection=False,
+            pin_protection=False,
+            label=LABEL,
+            language=LANGUAGE,
+            no_backup=True,
+        )
+        device_id = emu.client.features.device_id
+        asserts(from_tag, emu.client)
+        storage = emu.storage()
+
+    with EmulatorWrapper(gen, to_tag, storage=storage) as emu:
+        assert device_id == emu.client.features.device_id
+        asserts(to_tag, emu.client)
+
+
+if __name__ == "__main__":
+    if not ALL_TAGS:
+        print("No versions found. Remember to run download_emulators.sh")
+    for k, v in ALL_TAGS.items():
+        print("Found versions for {}:".format(k), v)
+    print()
+    print("Use `pytest {}` to run tests".format(__file__))