From 077735f57b53ed9798d6a4c3739f169c7ab2fc82 Mon Sep 17 00:00:00 2001 From: Tomas Susanka Date: Fri, 16 Aug 2019 15:29:21 +0200 Subject: [PATCH] ci: introduce upgrade tests closes #126 --- ci/build.yml | 12 + ci/test.yml | 12 + core/tests/run_tests_device_emu.sh | 2 +- legacy/script/test | 2 +- tests/upgrade_tests/__init__.py | 0 tests/upgrade_tests/download_emulators.sh | 9 + tests/upgrade_tests/emulator_wrapper.py | 63 +++++ tests/upgrade_tests/emulators/.gitignore | 2 + tests/upgrade_tests/test_firmware_upgrades.py | 223 ++++++++++++++++++ 9 files changed, 323 insertions(+), 2 deletions(-) create mode 100644 tests/upgrade_tests/__init__.py create mode 100755 tests/upgrade_tests/download_emulators.sh create mode 100644 tests/upgrade_tests/emulator_wrapper.py create mode 100644 tests/upgrade_tests/emulators/.gitignore create mode 100644 tests/upgrade_tests/test_firmware_upgrades.py diff --git a/ci/build.yml b/ci/build.yml index 25e2c9656..0997f5571 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -71,6 +71,18 @@ build core unix frozen bitcoinonly: - core/src/trezor/res/resources.py expire_in: 1 day +build core unix frozen debug: + stage: build + variables: + PYOPT: "0" + script: + - cd core + - pipenv run make build_unix_frozen + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" + untracked: true + expire_in: 1 day + # Crypto diff --git a/ci/test.yml b/ci/test.yml index 48e8d6598..a4e3c8141 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -96,3 +96,15 @@ test storage: - cd storage/tests - pipenv run make build - pipenv run make tests_all + + +# Other + +test upgrade: + stage: test + dependencies: + - build core unix frozen debug + - build legacy emu + script: + - tests/upgrade_tests/download_emulators.sh + - pipenv run pytest tests/upgrade_tests diff --git a/core/tests/run_tests_device_emu.sh b/core/tests/run_tests_device_emu.sh index 4ce02e27e..c5f32449e 100755 --- a/core/tests/run_tests_device_emu.sh +++ b/core/tests/run_tests_device_emu.sh @@ -22,7 +22,7 @@ export TREZOR_PATH=udp:127.0.0.1:21324 # run tests error=0 -if ! pytest ../../tests "$@"; then +if ! pytest ../../tests/device_tests "$@"; then error=1 fi kill $upy_pid diff --git a/legacy/script/test b/legacy/script/test index e4f1758d9..fbd51180e 100755 --- a/legacy/script/test +++ b/legacy/script/test @@ -16,4 +16,4 @@ if [ "$EMULATOR" = 1 ]; then "${PYTHON:-python}" script/wait_for_emulator.py fi -"${PYTHON:-python}" -m pytest ../tests "$@" +"${PYTHON:-python}" -m pytest ../tests/device_tests "$@" diff --git a/tests/upgrade_tests/__init__.py b/tests/upgrade_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/upgrade_tests/download_emulators.sh b/tests/upgrade_tests/download_emulators.sh new file mode 100755 index 000000000..90aaa4bc1 --- /dev/null +++ b/tests/upgrade_tests/download_emulators.sh @@ -0,0 +1,9 @@ +#!/bin/bash +SITE="https://firmware.corp.sldev.cz/upgrade_tests/" +cd "$(dirname "$0")" + +# download all emulators without index files, without directories and only if not present +wget --no-verbose --no-clobber --no-parent --cut-dirs=2 --no-host-directories --recursive --reject "index.html*" -P emulators/ $SITE + +# TODO: is this a good idea? +chmod u+x emulators/trezor-emu-* diff --git a/tests/upgrade_tests/emulator_wrapper.py b/tests/upgrade_tests/emulator_wrapper.py new file mode 100644 index 000000000..adbb22f30 --- /dev/null +++ b/tests/upgrade_tests/emulator_wrapper.py @@ -0,0 +1,63 @@ +import os +import subprocess +import tempfile +import time + +from trezorlib.debuglink import TrezorClientDebugLink +from trezorlib.transport import TransportException, get_transport + +BINDIR = os.path.dirname(os.path.abspath(__file__)) + "/emulators" +ENV = {"SDL_VIDEODRIVER": "dummy"} + + +class EmulatorWrapper: + def __init__(self, gen, tag, storage=None): + self.gen = gen + self.tag = tag + self.workdir = tempfile.TemporaryDirectory() + if storage: + open(self._storage_file(), "wb").write(storage) + + def __enter__(self): + if self.tag.startswith("/"): # full path+filename provided + args = [self.tag] + else: # only gen+tag provided + args = ["%s/trezor-emu-%s-%s" % (BINDIR, self.gen, self.tag)] + env = ENV + if self.gen == "core": + args += ["-m", "main"] + env["TREZOR_PROFILE_DIR"] = self.workdir.name + self.process = subprocess.Popen( + args, cwd=self.workdir.name, env=ENV, stdout=open(os.devnull, "w") + ) + # wait until emulator is started + while True: + try: + self.transport = get_transport("udp:127.0.0.1:21324") + except TransportException: + time.sleep(0.1) + continue + break + self.client = TrezorClientDebugLink(self.transport) + self.client.open() + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.client.close() + self.process.terminate() + try: + self.process.wait(1) + except subprocess.TimeoutExpired: + self.process.kill() + self.workdir.cleanup() + + def _storage_file(self): + if self.gen == "legacy": + return self.workdir.name + "/emulator.img" + elif self.gen == "core": + return self.workdir.name + "/trezor.flash" + else: + raise ValueError("Unknown gen") + + def storage(self): + return open(self._storage_file(), "rb").read() diff --git a/tests/upgrade_tests/emulators/.gitignore b/tests/upgrade_tests/emulators/.gitignore new file mode 100644 index 000000000..d6b7ef32c --- /dev/null +++ b/tests/upgrade_tests/emulators/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/upgrade_tests/test_firmware_upgrades.py b/tests/upgrade_tests/test_firmware_upgrades.py new file mode 100644 index 000000000..e33bca1eb --- /dev/null +++ b/tests/upgrade_tests/test_firmware_upgrades.py @@ -0,0 +1,223 @@ +import os +from collections import defaultdict + +import pytest + +from trezorlib import MINIMUM_FIRMWARE_VERSION, btc, debuglink, device +from trezorlib.tools import H_ + +MINIMUM_FIRMWARE_VERSION["1"] = (1, 0, 0) +MINIMUM_FIRMWARE_VERSION["T"] = (2, 0, 0) + +try: + from .emulator_wrapper import EmulatorWrapper +except ImportError: + pass + +# **** COMMON DEFINITIONS **** + +MNEMONIC = " ".join(["all"] * 12) +PATH = [H_(44), H_(0), H_(0), 0, 0] +ADDRESS = "1JAd7XCBzGudGpJQSDSfpmJhiygtLQWaGL" +LABEL = "test" +LANGUAGE = "english" +STRENGTH = 128 + +ROOT = os.path.dirname(os.path.abspath(__file__)) + "/../../" +LOCAL_BUILDS = { + "core": ROOT + "core/build/unix/micropython", + "legacy": ROOT + "legacy/firmware/trezor.elf", +} +BIN_DIR = os.path.dirname(os.path.abspath(__file__)) + "/emulators" + + +def check_version(tag, ver_emu): + if tag.startswith("v") and len(tag.split(".")) == 3: + assert tag == "v" + ".".join(["%d" % i for i in ver_emu]) + + +def check_file(gen, tag): + if tag.startswith("/"): + filename = tag + else: + filename = "%s/trezor-emu-%s-%s" % (BIN_DIR, gen, tag) + if not os.path.exists(filename): + raise ValueError(filename + " not found. Do not forget to build firmware.") + + +def get_tags(): + files = os.listdir(BIN_DIR) + if not files: + raise ValueError( + "No files found. Use download_emulators.sh to download emulators." + ) + + result = defaultdict(list) + for f in sorted(files): + try: + _, _, gen, tag = f.split("-", maxsplit=3) + result[gen].append(tag) + except ValueError: + pass + return result + + +ALL_TAGS = get_tags() + + +def for_all(*args, minimum_version=(1, 0, 0)): + if not args: + args = ("core", "legacy") + + all_params = [] + for gen in args: + try: + to_tag = LOCAL_BUILDS[gen] + from_tags = ALL_TAGS[gen] + [to_tag] + for from_tag in from_tags: + if from_tag.startswith("v"): + tag_version = tuple(int(n) for n in from_tag[1:].split(".")) + if tag_version < minimum_version: + continue + check_file(gen, from_tag) + all_params.append((gen, from_tag, to_tag)) + except KeyError: + pass + + return pytest.mark.parametrize("gen, from_tag, to_tag", all_params) + + +@for_all() +def test_upgrade_load(gen, from_tag, to_tag): + def asserts(tag, client): + check_version(tag, emu.client.version) + assert not client.features.pin_protection + assert not client.features.passphrase_protection + assert client.features.initialized + assert client.features.label == LABEL + assert client.features.language == LANGUAGE + assert btc.get_address(client, "Bitcoin", PATH) == ADDRESS + + with EmulatorWrapper(gen, from_tag) as emu: + debuglink.load_device_by_mnemonic( + emu.client, + mnemonic=MNEMONIC, + pin="", + passphrase_protection=False, + label=LABEL, + language=LANGUAGE, + ) + device_id = emu.client.features.device_id + asserts(from_tag, emu.client) + storage = emu.storage() + + with EmulatorWrapper(gen, to_tag, storage=storage) as emu: + assert device_id == emu.client.features.device_id + asserts(to_tag, emu.client) + + +@for_all("legacy") +def test_upgrade_reset(gen, from_tag, to_tag): + def asserts(tag, client): + check_version(tag, emu.client.version) + assert not client.features.pin_protection + assert not client.features.passphrase_protection + assert client.features.initialized + assert client.features.label == LABEL + assert client.features.language == LANGUAGE + assert not client.features.needs_backup + assert not client.features.unfinished_backup + assert not client.features.no_backup + + with EmulatorWrapper(gen, from_tag) as emu: + device.reset( + emu.client, + display_random=False, + strength=STRENGTH, + passphrase_protection=False, + pin_protection=False, + label=LABEL, + language=LANGUAGE, + ) + device_id = emu.client.features.device_id + asserts(from_tag, emu.client) + storage = emu.storage() + + with EmulatorWrapper(gen, to_tag, storage=storage) as emu: + assert device_id == emu.client.features.device_id + asserts(to_tag, emu.client) + + +@for_all() +def test_upgrade_reset_skip_backup(gen, from_tag, to_tag): + def asserts(tag, client): + check_version(tag, emu.client.version) + assert not client.features.pin_protection + assert not client.features.passphrase_protection + assert client.features.initialized + assert client.features.label == LABEL + assert client.features.language == LANGUAGE + assert client.features.needs_backup + assert not client.features.unfinished_backup + assert not client.features.no_backup + + with EmulatorWrapper(gen, from_tag) as emu: + device.reset( + emu.client, + display_random=False, + strength=STRENGTH, + passphrase_protection=False, + pin_protection=False, + label=LABEL, + language=LANGUAGE, + skip_backup=True, + ) + device_id = emu.client.features.device_id + asserts(from_tag, emu.client) + storage = emu.storage() + + with EmulatorWrapper(gen, to_tag, storage=storage) as emu: + assert device_id == emu.client.features.device_id + asserts(to_tag, emu.client) + + +@for_all(minimum_version=(1, 7, 2)) +def test_upgrade_reset_no_backup(gen, from_tag, to_tag): + def asserts(tag, client): + check_version(tag, emu.client.version) + assert not client.features.pin_protection + assert not client.features.passphrase_protection + assert client.features.initialized + assert client.features.label == LABEL + assert client.features.language == LANGUAGE + assert not client.features.needs_backup + assert not client.features.unfinished_backup + assert client.features.no_backup + + with EmulatorWrapper(gen, from_tag) as emu: + device.reset( + emu.client, + display_random=False, + strength=STRENGTH, + passphrase_protection=False, + pin_protection=False, + label=LABEL, + language=LANGUAGE, + no_backup=True, + ) + device_id = emu.client.features.device_id + asserts(from_tag, emu.client) + storage = emu.storage() + + with EmulatorWrapper(gen, to_tag, storage=storage) as emu: + assert device_id == emu.client.features.device_id + asserts(to_tag, emu.client) + + +if __name__ == "__main__": + if not ALL_TAGS: + print("No versions found. Remember to run download_emulators.sh") + for k, v in ALL_TAGS.items(): + print("Found versions for {}:".format(k), v) + print() + print("Use `pytest {}` to run tests".format(__file__))