diff --git a/ci/build.yml b/ci/build.yml index d6ff06f30..52fca1292 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -93,6 +93,12 @@ core unix regular build: script: - cd core - pipenv run make build_unix + artifacts: + name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" + paths: + - core/build/unix/micropython + - core/src/trezor/res/resources.py + expire_in: 1 week core unix frozen regular build: stage: build @@ -104,13 +110,13 @@ core unix frozen regular build: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: - core/build/unix/micropython - - core/src/trezor/res/resources.py expire_in: 1 week -core unix frozen btconly build: +core unix frozen btconly debug build: stage: build <<: *only_changes_core variables: + PYOPT: "0" BITCOIN_ONLY: "1" script: - cd core @@ -120,7 +126,6 @@ core unix frozen btconly build: name: "$CI_JOB_NAME-$CI_COMMIT_SHORT_SHA" paths: - core/build/unix/micropython-bitcoinonly - - core/src/trezor/res/resources.py expire_in: 1 week core unix frozen debug build: diff --git a/ci/test.yml b/ci/test.yml index 107c80c42..64e7ce0b7 100644 --- a/ci/test.yml +++ b/ci/test.yml @@ -33,7 +33,7 @@ core unix unit test: stage: test <<: *only_changes_core dependencies: - - core unix frozen regular build + - core unix regular build script: - cd core - pipenv run make test @@ -42,20 +42,19 @@ core unix device ui test: stage: test <<: *only_changes_core dependencies: - - core unix frozen regular build + - core unix frozen debug build script: - cd core - pipenv run make test_emu_ui - - cp /var/tmp/trezor.log ${CI_PROJECT_DIR} - cd ../ci - pipenv run python prepare_ui_artifacts.py artifacts: name: core-unix-device-ui-test paths: - - trezor.log - ci/ui_test_records/ - tests/ui_tests/reports/ - tests/junit.xml + - tests/trezor.log when: always expire_in: 1 week reports: @@ -65,20 +64,19 @@ core unix device test: stage: test <<: *only_changes_core dependencies: - - core unix frozen regular build + - core unix frozen debug build variables: TREZOR_PROFILING: 1 script: - cd core - pipenv run make test_emu - - cp /var/tmp/trezor.log ${CI_PROJECT_DIR} - sync - sleep 1 - mv ./src/.coverage .coverage.test_emu artifacts: name: core-unix-device-test paths: - - trezor.log + - tests/trezor.log - tests/junit.xml - core/.coverage.* expire_in: 1 week @@ -90,18 +88,17 @@ core unix btconly device test: stage: test <<: *only_changes_core dependencies: - - core unix frozen btconly build + - core unix frozen btconly debug build variables: - MICROPYTHON: "../build/unix/micropython-bitcoinonly" + MICROPYTHON: "build/unix/micropython-bitcoinonly" TREZOR_PYTEST_SKIP_ALTCOINS: 1 script: - cd core - pipenv run make test_emu - - cp /var/tmp/trezor.log ${CI_PROJECT_DIR} artifacts: name: core-unix-btconly-device-test paths: - - trezor.log + - tests/trezor.log - tests/junit.xml expire_in: 1 week when: always @@ -112,20 +109,19 @@ core unix monero test: stage: test <<: *only_changes_core dependencies: - - core unix frozen regular build + - core unix frozen debug build variables: TREZOR_PROFILING: 1 script: - cd core - pipenv run make test_emu_monero - - cp /var/tmp/trezor.log ${CI_PROJECT_DIR} - sync - sleep 1 - mv ./src/.coverage .coverage.test_emu_monero artifacts: name: core-unix-monero-test paths: - - trezor.log + - tests/trezor.log - core/.coverage.* expire_in: 1 week when: always @@ -134,21 +130,20 @@ core unix u2f test: stage: test <<: *only_changes_core dependencies: - - core unix frozen regular build + - core unix frozen debug build variables: TREZOR_PROFILING: 1 script: - make -C tests/fido_tests/u2f-tests-hid - cd core - pipenv run make test_emu_u2f - - cp /var/tmp/trezor.log ${CI_PROJECT_DIR} - sync - sleep 1 - mv ./src/.coverage .coverage.test_emu_u2f artifacts: name: core-unix-u2f-test paths: - - trezor.log + - tests/trezor.log - core/.coverage.* expire_in: 1 week when: always @@ -157,20 +152,19 @@ core unix fido2 test: stage: test <<: *only_changes_core dependencies: - - core unix frozen regular build + - core unix frozen debug build variables: TREZOR_PROFILING: 1 script: - cd core - pipenv run make test_emu_fido2 - - cp /var/tmp/trezor.log ${CI_PROJECT_DIR} - sync - sleep 1 - mv ./src/.coverage .coverage.test_emu_fido2 artifacts: name: core-unix-fido2-test paths: - - trezor.log + - tests/trezor.log - tests/junit.xml - core/.coverage.* expire_in: 1 week @@ -186,11 +180,10 @@ core unix click test: script: - cd core - pipenv run make test_emu_click - - cp /var/tmp/trezor.log ${CI_PROJECT_DIR} artifacts: name: core-unix-click-test paths: - - trezor.log + - tests/trezor.log - tests/junit.xml reports: junit: tests/junit.xml diff --git a/core/Makefile b/core/Makefile index 7c1ca0e8c..84b9cf7d6 100644 --- a/core/Makefile +++ b/core/Makefile @@ -38,6 +38,17 @@ FIRMWARE_MAXSIZE = 1703936 GITREV=$(shell git describe --always --dirty | tr '-' '_') CFLAGS += -DGITREV=$(GITREV) +TESTPATH = $(CURDIR)/../tests + +EMU = $(CURDIR)/emu.py +EMU_LOG_FILE ?= $(TESTPATH)/trezor.log +EMU_TEST_ARGS = --disable-animation --headless --output=$(EMU_LOG_FILE) --temporary-profile +EMU_TEST = $(EMU) $(EMU_TEST_ARGS) -c + +JUNIT_XML ?= $(TESTPATH)/junit.xml +PYTEST = pytest --junitxml=$(JUNIT_XML) +TREZOR_FIDO2_UDP_PORT = 21326 + ## help commands: help: ## show this help @@ -57,7 +68,7 @@ run: ## run unix port cd src ; ../$(UNIX_BUILD_DIR)/micropython emu: ## run emulator - ./emu.sh + $(EMU) ## test commands: @@ -65,25 +76,27 @@ test: ## run unit tests cd tests ; ./run_tests.sh $(TESTOPTS) test_emu: ## run selected device tests from python-trezor - cd tests ; ./run_tests_device_emu.sh $(TESTOPTS) + $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) test_emu_monero: ## run selected monero device tests from monero-agent cd tests ; ./run_tests_device_emu_monero.sh $(TESTOPTS) test_emu_u2f: ## run selected u2f device tests from u2f-tests-hid - cd tests ; ./run_tests_device_emu_u2f.sh $(TESTOPTS) + $(EMU_TEST) --slip0014 $(TESTPATH)/fido_tests/u2f-tests-hid/HIDTest $(TREZOR_FIDO2_UDP_PORT) $(TESTOPTS) + $(EMU_TEST) --slip0014 $(TESTPATH)/fido_tests/u2f-tests-hid/U2FTest $(TREZOR_FIDO2_UDP_PORT) $(TESTOPTS) test_emu_fido2: ## run fido2 device tests - cd tests ; ./run_tests_device_emu_fido2.sh $(TESTOPTS) + cd $(TESTPATH)/fido_tests/fido2 ; \ + $(EMU_TEST) $(PYTEST) --sim tests/standard/ --vendor trezor $(TESTOPTS) test_emu_click: ## run click tests - cd tests ; ./run_tests_click_emu.sh $(TESTOPTS) + $(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests $(TESTOPTS) test_emu_ui: ## run ui integration tests - cd tests ; ./run_tests_device_emu.sh --ui=test -m "not skip_ui" $(TESTOPTS) + $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests --ui=test -m "not skip_ui" $(TESTOPTS) test_emu_ui_record: ## record and hash screens for ui integration tests - cd tests ; ./run_tests_device_emu.sh --ui=record -m "not skip_ui" $(TESTOPTS) + $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests --ui=record -m "not skip_ui" $(TESTOPTS) pylint: ## run pylint on application sources and tests pylint -E $(shell find src tests -name *.py) diff --git a/core/embed/unix/main.c b/core/embed/unix/main.c index 3de795cdf..1ceef024d 100644 --- a/core/embed/unix/main.c +++ b/core/embed/unix/main.c @@ -692,6 +692,9 @@ MP_NOINLINE int main_(int argc, char **argv) { return ret & 0xff; } +#ifdef TREZOR_EMULATOR_FROZEN +uint mp_import_stat(const char *path) { return MP_IMPORT_STAT_NO_EXIST; } +#else uint mp_import_stat(const char *path) { struct stat st; if (stat(path, &st) == 0) { @@ -703,6 +706,7 @@ uint mp_import_stat(const char *path) { } return MP_IMPORT_STAT_NO_EXIST; } +#endif void nlr_jump_fail(void *val) { printf("FATAL: uncaught NLR %p\n", val); diff --git a/core/emu.py b/core/emu.py new file mode 100755 index 000000000..95432b8be --- /dev/null +++ b/core/emu.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +import gzip +import os +import platform +import signal +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import click + +import trezorlib.debuglink +import trezorlib.device +from trezorlib._internal.emulator import CoreEmulator + +try: + import inotify.adapters +except ImportError: + inotify = None + + +HERE = Path(__file__).parent.resolve() +MICROPYTHON = HERE / "build" / "unix" / "micropython" +SRC_DIR = HERE / "src" +SD_CARD_GZ = HERE / "trezor.sdcard.gz" + +PROFILING_WRAPPER = HERE / "prof" / "prof.py" + +PROFILE_BASE = Path.home() / ".trezoremu" + + +def run_command_with_emulator(emulator, command): + with emulator: + # first start the subprocess + process = subprocess.Popen(command) + # After the subprocess is started, ignore SIGINT in parent + # (so that we don't need to handle KeyboardInterrupts) + signal.signal(signal.SIGINT, signal.SIG_IGN) + # SIGINTs will be sent to all children by the OS, so we should be able to safely + # wait for their exit. + return process.wait() + + +def run_emulator(emulator): + with emulator: + signal.signal(signal.SIGINT, signal.SIG_IGN) + return emulator.wait() + + +def watch_emulator(emulator): + watch = inotify.adapters.InotifyTree(str(SRC_DIR)) + try: + for _, type_names, _, _ in watch.event_gen(yield_nones=False): + if "IN_CLOSE_WRITE" in type_names: + click.echo("Restarting...") + emulator.restart() + except KeyboardInterrupt: + emulator.stop() + return 0 + + +def run_debugger(emulator): + os.chdir(emulator.workdir) + env = emulator.make_env() + if platform.system() == "Darwin": + env["PATH"] = "/usr/bin" + os.execvpe( + "lldb", + ["lldb", "-f", emulator.executable, "--"] + emulator.make_args(), + env, + ) + else: + os.execvpe( + "gdb", ["gdb", "--args", emulator.executable] + emulator.make_args(), env + ) + + +@click.command(context_settings=dict(ignore_unknown_options=True)) +# fmt: off +@click.option("-a", "--disable-animation", is_flag=True, default=os.environ.get("TREZOR_DISABLE_ANIMATION") == "1", help="Disable animation") +@click.option("-c", "--command", "run_command", is_flag=True, help="Run command while emulator is running") +@click.option("-d", "--production", is_flag=True, default=os.environ.get("PYOPT") == "1", help="Production mode (debuglink disabled)") +@click.option("-D", "--debugger", is_flag=True, help="Run emulator in debugger (gdb/lldb)") +@click.option("--executable", type=click.Path(exists=True, dir_okay=False), default=os.environ.get("MICROPYTHON"), help="Alternate emulator executable") +@click.option("-g", "--profiling", is_flag=True, default=os.environ.get("TREZOR_PROFILING"), help="Run with profiler wrapper") +@click.option("-h", "--headless", is_flag=True, help="Headless mode (no display)") +@click.option("--heap-size", metavar="SIZE", default="20M", help="Configure heap size") +@click.option("--main", help="Path to python main file") +@click.option("--mnemonic", "mnemonics", multiple=True, help="Initialize device with given mnemonic. Specify multiple times for Shamir shares.") +@click.option("--log-memory", is_flag=True, default=os.environ.get("TREZOR_LOG_MEMORY") == "1", help="Print memory usage after workflows") +@click.option("-o", "--output", type=click.File("w"), default="-", help="Redirect emulator output to file") +@click.option("-p", "--profile", metavar="NAME", help="Profile name or path") +@click.option("-P", "--port", metavar="PORT", type=int, default=int(os.environ.get("TREZOR_UDP_PORT", 0)) or None, help="UDP port number") +@click.option("-q", "--quiet", is_flag=True, help="Silence emulator output") +@click.option("-s", "--slip0014", is_flag=True, help="Initialize device with SLIP-14 seed (all all all...)") +@click.option("-t", "--temporary-profile", is_flag=True, help="Create an empty temporary profile") +@click.option("-w", "--watch", is_flag=True, help="Restart emulator if sources change") +@click.option("-X", "--extra-arg", "extra_args", multiple=True, help="Extra argument to pass to micropython") +# fmt: on +@click.argument("command", nargs=-1, type=click.UNPROCESSED) +def cli( + disable_animation, + run_command, + production, + debugger, + executable, + profiling, + headless, + heap_size, + main, + mnemonics, + log_memory, + profile, + port, + output, + quiet, + slip0014, + temporary_profile, + watch, + extra_args, + command, +): + """Run the trezor-core emulator. + + If -c is specified, extra arguments are treated as a command that is executed with + the running emulator. This command can access the following environment variables: + + \b + TREZOR_PROFILE_DIR - path to storage directory + TREZOR_PATH - trezorlib connection string + TREZOR_UDP_PORT - UDP port on which the emulator listens + TREZOR_FIDO2_UDP_PORT - UDP port for FIDO2 + + By default, emulator output goes to stdout. If silenced with -q, it is redirected + to $TREZOR_PROFILE_DIR/trezor.log. You can also specify a custom path with -o. + """ + if executable: + executable = Path(executable) + else: + executable = MICROPYTHON + + if command and not run_command: + raise click.ClickException("Extra arguments found. Did you mean to use -c?") + + if watch and (command or debugger or frozen): + raise click.ClickException("Cannot use -w together with -c or -D or -F") + + if watch and inotify is None: + raise click.ClickException("inotify module is missing, install with pip") + + if main and profiling: + raise click.ClickException("Cannot use --main and -g together") + + if slip0014 and mnemonics: + raise click.ClickException("Cannot use -s and --mnemonic together") + + if slip0014: + mnemonics = [" ".join(["all"] * 12)] + + if mnemonics and debugger: + raise click.ClickException("Cannot load mnemonics when running in debugger") + + if mnemonics and production: + raise click.ClickException("Cannot load mnemonics in production mode") + + if profiling: + main_args = [str(PROFILING_WRAPPER)] + elif main: + main_args = [main] + else: + main_args = ["-m", "main"] + + if profile and temporary_profile: + raise click.ClickException("Cannot use -p and -t together") + + tempdir = None + if profile: + if "/" in profile: + profile_dir = Path(profile) + else: + profile_dir = PROFILE_BASE / profile + + elif temporary_profile: + tempdir = tempfile.TemporaryDirectory(prefix="trezor-emulator-") + profile_dir = Path(tempdir.name) + # unpack empty SD card for faster start-up + with gzip.open(SD_CARD_GZ, "rb") as gz: + (profile_dir / "trezor.sdcard").write_bytes(gz.read()) + + elif "TREZOR_PROFILE_DIR" in os.environ: + profile_dir = Path(os.environ["TREZOR_PROFILE_DIR"]) + + else: + profile_dir = Path("/var/tmp") + + if quiet: + output = None + + emulator = CoreEmulator( + executable, + profile_dir, + logfile=output, + port=port, + headless=headless, + debug=not production, + extra_args=extra_args, + main_args=main_args, + heap_size=heap_size, + disable_animation=disable_animation, + workdir=SRC_DIR, + ) + + emulator_env = dict( + TREZOR_PATH=f"udp:127.0.0.1:{emulator.port}", + TREZOR_PROFILE_DIR=str(profile_dir.resolve()), + TREZOR_UDP_PORT=str(emulator.port), + TREZOR_FIDO2_UDP_PORT=str(emulator.port + 2), + TREZOR_SRC=str(SRC_DIR), + ) + os.environ.update(emulator_env) + for k, v in emulator_env.items(): + click.echo(f"{k}={v}") + + if log_memory: + os.environ["TREZOR_LOG_MEMORY"] = "1" + + if debugger: + run_debugger(emulator) + raise RuntimeError("run_debugger should not return") + + click.echo("Waiting for emulator to come up... ", err=True) + start = time.monotonic() + emulator.start() + end = time.monotonic() + click.echo(f"Emulator ready after {end - start:.3f} seconds", err=True) + + if mnemonics: + if slip0014: + label = "SLIP-0014" + elif profile: + label = profile_dir.name + else: + label = "Emulator" + + trezorlib.device.wipe(emulator.client) + trezorlib.debuglink.load_device( + emulator.client, + mnemonics, + pin=None, + passphrase_protection=False, + label=label, + ) + + if run_command: + ret = run_command_with_emulator(emulator, command) + elif watch: + ret = watch_emulator(emulator) + else: + ret = run_emulator(emulator) + + if tempdir is not None: + tempdir.cleanup() + sys.exit(ret) + + +if __name__ == "__main__": + cli() diff --git a/core/emu.sh b/core/emu.sh index 4481de249..970d689a1 100755 --- a/core/emu.sh +++ b/core/emu.sh @@ -1,35 +1,10 @@ -#!/usr/bin/env bash +#!/bin/sh +PYOPT="${PYOPT:-1}" -MICROPYTHON="${MICROPYTHON:-${PWD}/build/unix/micropython}" -TREZOR_SRC=$(cd "${PWD}/src/"; pwd) -BROWSER="${BROWSER:-chromium}" +if [ -n "$1" ]; then + echo "This is just a compatibility wrapper. Use emu.py if you want features." + exit 1 +fi -source ./trezor_cmd.sh - -cd "${TREZOR_SRC}" - -case "$1" in - "-d") - shift - OPERATING_SYSTEM=$(uname) - if [ "$OPERATING_SYSTEM" = "Darwin" ]; then - PATH=/usr/bin /usr/bin/lldb -f $MICROPYTHON -- $ARGS $* $MAIN - else - gdb --args $MICROPYTHON $ARGS $* $MAIN - fi - ;; - "-r") - shift - while true; do - $MICROPYTHON $ARGS $* $MAIN & - UPY_PID=$! - find -name '*.py' | inotifywait -q -e close_write --fromfile - - echo Restarting ... - kill $UPY_PID - done - ;; - *) - echo "Starting emulator: $MICROPYTHON $ARGS $* $MAIN" - $MICROPYTHON $ARGS $* $MAIN 2>&1 | tee "${TREZOR_LOGFILE}" - exit ${PIPESTATUS[0]} -esac +cd src +../build/unix/micropython -O$PYOPT -X heapsize=20M -m main diff --git a/core/prof/prof.py b/core/prof/prof.py index fcb13017c..75703ed05 100644 --- a/core/prof/prof.py +++ b/core/prof/prof.py @@ -1,10 +1,13 @@ import sys -import uos from uio import open +from uos import getenv -sys.path.insert(0, uos.getenv("TREZOR_SRC")) -del uos +# We need to insert "" to sys.path so that the frozen build can import main from the +# frozen modules, and regular build can import it from current directory. +sys.path.insert(0, "") + +PATH_PREFIX = (getenv("TREZOR_SRC") or ".") + "/" class Coverage: @@ -22,7 +25,7 @@ class Coverage: this_file = globals()["__file__"] for filename in self.__files: if not filename == this_file: - lines[filename] = list(self.__files[filename]) + lines[PATH_PREFIX + filename] = list(self.__files[filename]) return lines_execution diff --git a/core/src/apps/debug/__init__.py b/core/src/apps/debug/__init__.py index 6366f7742..4f1004f98 100644 --- a/core/src/apps/debug/__init__.py +++ b/core/src/apps/debug/__init__.py @@ -38,7 +38,7 @@ if __debug__: current_content = None # type: Optional[List[str]] def screenshot() -> bool: - if utils.SAVE_SCREEN or save_screen: + if save_screen: ui.display.save(save_screen_directory + "/refresh-") return True return False diff --git a/core/src/trezor/utils.py b/core/src/trezor/utils.py index 9aef7a541..009ec4141 100644 --- a/core/src/trezor/utils.py +++ b/core/src/trezor/utils.py @@ -19,13 +19,9 @@ if __debug__: if EMULATOR: import uos - TEST = int(uos.getenv("TREZOR_TEST") or "0") DISABLE_ANIMATION = int(uos.getenv("TREZOR_DISABLE_ANIMATION") or "0") - SAVE_SCREEN = int(uos.getenv("TREZOR_SAVE_SCREEN") or "0") LOG_MEMORY = int(uos.getenv("TREZOR_LOG_MEMORY") or "0") else: - TEST = 0 - SAVE_SCREEN = 0 LOG_MEMORY = 0 if False: diff --git a/core/tests/run_tests_device_emu.sh b/core/tests/run_tests_device_emu.sh deleted file mode 100755 index b823bbc43..000000000 --- a/core/tests/run_tests_device_emu.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -: "${RUN_TEST_EMU:=1}" - -CORE_DIR="$(SHELL_SESSION_FILE='' && cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )" -MICROPYTHON="${MICROPYTHON:-$CORE_DIR/build/unix/micropython}" -TREZOR_SRC="${CORE_DIR}/src" - -PYOPT="${PYOPT:-0}" -upy_pid="" - -# run emulator if RUN_TEST_EMU -if [[ $RUN_TEST_EMU > 0 ]]; then - source ../trezor_cmd.sh - - # remove flash and sdcard files before run to prevent inconsistent states - mv "${TREZOR_PROFILE_DIR}/trezor.flash" "${TREZOR_PROFILE_DIR}/trezor.flash.bkp" 2>/dev/null - mv "${TREZOR_PROFILE_DIR}/trezor.sdcard" "${TREZOR_PROFILE_DIR}/trezor.sdcard.bkp" 2>/dev/null - - cd "${TREZOR_SRC}" - echo "Starting emulator: $MICROPYTHON $ARGS ${MAIN}" - - TREZOR_TEST=1 \ - TREZOR_DISABLE_ANIMATION=1 \ - $MICROPYTHON $ARGS "${MAIN}" &> "${TREZOR_LOGFILE}" & - upy_pid=$! - cd - - sleep 30 -fi - -# run tests -error=0 -if ! pytest --junitxml=../../tests/junit.xml ../../tests/device_tests "$@"; then - error=1 -fi -kill $upy_pid -exit $error diff --git a/core/tests/run_tests_device_emu_fido2.sh b/core/tests/run_tests_device_emu_fido2.sh deleted file mode 100755 index bcbedbdd9..000000000 --- a/core/tests/run_tests_device_emu_fido2.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -: "${RUN_TEST_EMU:=1}" - -CORE_DIR="$(SHELL_SESSION_FILE='' && cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )" -MICROPYTHON="${MICROPYTHON:-$CORE_DIR/build/unix/micropython}" -TREZOR_SRC="${CORE_DIR}/src" - -DISABLE_ANIMATION=1 -PYOPT="${PYOPT:-0}" -upy_pid="" - -# run emulator if RUN_TEST_EMU -if [[ $RUN_TEST_EMU > 0 ]]; then - source ../trezor_cmd.sh - - # remove flash and sdcard files before run to prevent inconsistent states - mv "${TREZOR_PROFILE_DIR}/trezor.flash" "${TREZOR_PROFILE_DIR}/trezor.flash.bkp" 2>/dev/null - mv "${TREZOR_PROFILE_DIR}/trezor.sdcard" "${TREZOR_PROFILE_DIR}/trezor.sdcard.bkp" 2>/dev/null - - cd "${TREZOR_SRC}" - echo "Starting emulator: $MICROPYTHON $ARGS ${MAIN}" - - TREZOR_TEST=1 \ - TREZOR_DISABLE_ANIMATION=$DISABLE_ANIMATION \ - $MICROPYTHON $ARGS "${MAIN}" &> "${TREZOR_LOGFILE}" & - upy_pid=$! - cd - - sleep 30 -fi - -cd ../../tests/fido_tests/fido2 -# run tests -error=0 -export TREZOR_FIDO2_UDP_PORT=21326 -if ! pytest --junitxml=../../tests/junit.xml --sim tests/standard/ --vendor trezor "$@"; then - error=1 -fi -kill $upy_pid -exit $error diff --git a/core/tests/run_tests_device_emu_monero.sh b/core/tests/run_tests_device_emu_monero.sh index ea2140e43..2071fe1eb 100755 --- a/core/tests/run_tests_device_emu_monero.sh +++ b/core/tests/run_tests_device_emu_monero.sh @@ -5,30 +5,22 @@ : "${RUN_TEST_EMU:=1}" CORE_DIR="$(SHELL_SESSION_FILE='' && cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )" -MICROPYTHON="${MICROPYTHON:-$CORE_DIR/build/unix/micropython}" -TREZOR_SRC="${CORE_DIR}/src" -DISABLE_ANIMATION=1 -PYOPT="${PYOPT:-0}" upy_pid="" # run emulator if RUN_TEST_EMU if [[ $RUN_TEST_EMU > 0 ]]; then - source ../trezor_cmd.sh - - # remove flash and sdcard files before run to prevent inconsistent states - mv "${TREZOR_PROFILE_DIR}/trezor.flash" "${TREZOR_PROFILE_DIR}/trezor.flash.bkp" 2>/dev/null - mv "${TREZOR_PROFILE_DIR}/trezor.sdcard" "${TREZOR_PROFILE_DIR}/trezor.sdcard.bkp" 2>/dev/null - - cd "${TREZOR_SRC}" - echo "Starting emulator: $MICROPYTHON $ARGS ${MAIN}" - - TREZOR_TEST=1 \ - TREZOR_DISABLE_ANIMATION=$DISABLE_ANIMATION \ - $MICROPYTHON $ARGS "${MAIN}" &> "${TREZOR_LOGFILE}" & - upy_pid=$! - cd - - sleep 30 + t=$(mktemp) + ../emu.py \ + --disable-animation \ + --temporary-profile \ + --headless \ + --output=../../tests/trezor.log \ + > $t & + trezorctl wait-for-emulator + source $t + upy_pid=$(cat $TREZOR_PROFILE_DIR/trezor.pid) + rm $t fi DOCKER_ID="" diff --git a/core/tests/run_tests_device_emu_u2f.sh b/core/tests/run_tests_device_emu_u2f.sh deleted file mode 100755 index 93c58fadb..000000000 --- a/core/tests/run_tests_device_emu_u2f.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -: "${RUN_TEST_EMU:=1}" - -CORE_DIR="$(SHELL_SESSION_FILE='' && cd "$( dirname "${BASH_SOURCE[0]}" )/.." >/dev/null 2>&1 && pwd )" -MICROPYTHON="${MICROPYTHON:-$CORE_DIR/build/unix/micropython}" -TREZOR_SRC="${CORE_DIR}/src" - -DISABLE_ANIMATION=1 -PYOPT="${PYOPT:-0}" -upy_pid="" - -# run emulator if RUN_TEST_EMU -if [[ $RUN_TEST_EMU > 0 ]]; then - source ../trezor_cmd.sh - - # remove flash and sdcard files before run to prevent inconsistent states - mv "${TREZOR_PROFILE_DIR}/trezor.flash" "${TREZOR_PROFILE_DIR}/trezor.flash.bkp" 2>/dev/null - mv "${TREZOR_PROFILE_DIR}/trezor.sdcard" "${TREZOR_PROFILE_DIR}/trezor.sdcard.bkp" 2>/dev/null - - cd "${TREZOR_SRC}" - echo "Starting emulator: $MICROPYTHON $ARGS ${MAIN}" - - TREZOR_TEST=1 \ - TREZOR_DISABLE_ANIMATION=$DISABLE_ANIMATION \ - $MICROPYTHON $ARGS "${MAIN}" &> "${TREZOR_LOGFILE}" & - upy_pid=$! - cd - - sleep 30 -fi - -# run tests -error=0 -TREZOR_FIDO2_UDP_PORT=21326 -# missuse loaddevice test to initialize the device -if ! pytest ../../tests/device_tests -k "test_msg_loaddevice" "$@"; then - error=1 -fi -if ! ../../tests/fido_tests/u2f-tests-hid/HIDTest "${TREZOR_FIDO2_UDP_PORT}" "$@"; then - error=1 -fi -if ! ../../tests/fido_tests/u2f-tests-hid/U2FTest "${TREZOR_FIDO2_UDP_PORT}" "$@"; then - error=1 -fi -kill $upy_pid -exit $error diff --git a/core/trezor.sdcard.gz b/core/trezor.sdcard.gz new file mode 100644 index 000000000..e9c6efc52 Binary files /dev/null and b/core/trezor.sdcard.gz differ diff --git a/core/trezor_cmd.sh b/core/trezor_cmd.sh deleted file mode 100644 index cb96072f1..000000000 --- a/core/trezor_cmd.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env bash - -# expected inputs: -# TREZOR_SRC -- directory containing python code for uMP - -if [[ ! "${TREZOR_SRC}" ]]; then echo "expecting TREZOR_SRC"; exit 0; fi - -# optional inputs: -# TREZOR_PROFILE -- profile name (directory) in ~/.trezoremu or full path -# TREZOR_PROFILING -- wrap the uMP/python in the profiler script - -# outputs: -## uMP -# PYOPT -# HEAPSIZE -# ARGS -- uMP arguments -# MAIN -- uMP file to execute -## Trezor core -# TREZOR_PROFILE_DIR -# TREZOR_PROFILE_NAME -# TREZOR_UDP_PORT -## this script -# TREZOR_SRC -# TREZOR_LOGFILE -## python-trezor -# TREZOR_PATH -- connect string - - -# defaults -PYOPT="${PYOPT:-1}" -HEAPSIZE="${HEAPSIZE:-20M}" - -TREZOR_PROFILE="${TREZOR_PROFILE:-/var/tmp}" -TREZOR_PROFILE_DIR="${TREZOR_PROFILE}" -TREZOR_PROFILE_NAME="${TREZOR_PROFILE}" - -# for profile names create profile directory if not existent -if ! [[ "$TREZOR_PROFILE" == "/"* ]]; then - TREZOR_PROFILE_DIR="${HOME}/.trezoremu/${TREZOR_PROFILE}" - if ! [[ -d "${TREZOR_PROFILE_DIR}" ]]; then - mkdir -p "${TREZOR_PROFILE_DIR}" - PORT=$(( ( RANDOM % 1000 ) + 1 + 21324 )) - echo "# autogenerated config" > "${TREZOR_PROFILE_DIR}/emu.config" - echo "TREZOR_UDP_PORT=\"\${TREZOR_UDP_PORT:-${PORT}}\"" >> "${TREZOR_PROFILE_DIR}/emu.config" - fi -fi - -# load profile config -if [[ -f "${TREZOR_PROFILE_DIR}/emu.config" ]]; then - source "${TREZOR_PROFILE_DIR}/emu.config" -fi - -# for profiling wrap -if [[ "$TREZOR_PROFILING" -gt 0 ]]; then - MAIN="${TREZOR_SRC}/../prof/prof.py" -else - MAIN="${TREZOR_SRC}/main.py" -fi - -TREZOR_LOGFILE="${TREZOR_PROFILE_DIR}/trezor.log" -TREZOR_UDP_PORT="${TREZOR_UDP_PORT:-21324}" -TREZOR_PATH="${TREZOR_PATH:-udp:127.0.0.1:${TREZOR_UDP_PORT}}" - -echo "Trezor^emu profile name: ${TREZOR_PROFILE_NAME}" -echo "Trezor^emu profile directory: ${TREZOR_PROFILE_DIR}" -echo "Trezor^emu log file: ${TREZOR_LOGFILE}" -echo "Trezor^emu UDP port: ${TREZOR_UDP_PORT}" -echo "Trezor^emu path: ${TREZOR_PATH}" -echo "Trezor^emu src: ${TREZOR_SRC}" - -export TREZOR_PROFILE_NAME="${TREZOR_PROFILE_NAME}" -export TREZOR_PROFILE_DIR="${TREZOR_PROFILE_DIR}" -export TREZOR_LOGFILE="${TREZOR_LOGFILE}" -export TREZOR_UDP_PORT="${TREZOR_UDP_PORT}" -export TREZOR_PATH="${TREZOR_PATH}" -export TREZOR_SRC="${TREZOR_SRC}" - -ARGS="-O${PYOPT} -X heapsize=${HEAPSIZE}" diff --git a/docs/core/build/emulator.md b/docs/core/build/emulator.md index 3ff68a0cc..1488b986b 100644 --- a/docs/core/build/emulator.md +++ b/docs/core/build/emulator.md @@ -59,7 +59,7 @@ pipenv run make build_unix Now you can start the emulator: ```sh -./emu.sh +./emu.py ``` The emulator has a number of interesting features all documented in the [Emulator](../emulator/index.md) section. diff --git a/docs/core/emulator/index.md b/docs/core/emulator/index.md index 1eae65af3..236fc9f8a 100644 --- a/docs/core/emulator/index.md +++ b/docs/core/emulator/index.md @@ -11,81 +11,117 @@ Emulator significantly speeds up development and has several features to help yo ## How to run 1. [build](../build/emulator.md) the emulator -2. run `emu.sh` +2. run `emu.py` inside the pipenv environment: + - either enter `pipenv shell` first, and then use `./emu.py` + - or always use `pipenv run ./emu.py` 3. to use [bridge](https://github.com/trezor/trezord-go) with the emulator support, start it with `trezord -e 21324` Now you can use the emulator the same way as you use the device, for example you can visit our Wallet (https://wallet.trezor.io), use our Python CLI tool (`trezorctl`) etc. Simply click to emulate screen touches. ## Features -### Debug mode +Run `./emu.py --help` to see all supported command line options and shortcuts. The +sections below only list long option names and most notable features. -To allow debug link (to run tests), see exceptions and log output, run emulator with `PYOPT=0 ./emu.sh`. To properly distinguish the debug mode from production there is a tiny red square in the top right corner. The debug mode is obviously disabled on production firmwares. +### Debug and production mode + +By default the emulator runs in debug mode. Debuglink is available (on port 21325 by +default), exceptions and log output goes to console. To indicate debug mode, there is a +red square in the upper right corner of Trezor screen. ![emulator](emulator-debug.png) +To enable production mode, run `./emu.py --production`, or set environment variable `PYOPT=1`. + ### Initialize with mnemonic words -If the debug mode is enabled, you can load the device with any recovery seed directly from the console. This feature is otherwise disabled. To enter seed use `trezorctl`: +In debug mode, the emulator can be pre-configured with a mnemonic phrase. + +To use a specific mnemonic phrase: ```sh -trezorctl -m "your mnemonic words" +./emu.py --mnemonic "such deposit very security much theme..." ``` -or to use the "all all all" seed defined in [SLIP-14](https://github.com/satoshilabs/slips/blob/master/slip-0014.md): +When using Shamir shares, repeat the `--mnemonic` option: ```sh -trezorctl -s +./emu.py --mnemonic "your first share" --mnemonic "your second share" ... ``` -Shamir Backup is also supported: +To use the "all all all" seed defined in [SLIP-14](https://github.com/satoshilabs/slips/blob/master/slip-0014.md): ```sh -trezorctl -m "share 1 words" -m "share 2 words" +./emu.py --slip0014 ``` ### Storage and Profiles -Internal Trezor's storage is emulated and stored in the `/var/tmp/trezor.flash` file on default. Deleting this file is similar to calling _wipe device_. You can also find `/var/tmp/trezor.sdcard` for SD card and `/var/tmp/trezor.log`, which contains the communication log, the same as is in the emulator's stdout. +Internal Trezor's storage is emulated and stored in the `/var/tmp/trezor.flash` file on +default. Deleting this file is similar to calling _wipe device_. You can also find +`/var/tmp/trezor.sdcard` for SD card. -To run emulator with different files set the environment variable **TREZOR_PROFILE** like so: +You can specify a different location for the storage and log files via the `-p` / +`--profile` option: ```sh -TREZOR_PROFILE=foobar ./emu.sh +./emu.py -p foobar ``` -This will create a profile directory in your home ``` ~/.trezoremu/foobar``` containing emulator run files. Alternatively you can set a full path like so: +This will create a profile directory in your home `~/.trezoremu/foobar` containing +emulator run files. Alternatively you can set a full path like so: ```sh -TREZOR_PROFILE=/var/tmp/foobar ./emu.sh +./emu.py -p /var/tmp/foobar ``` -### Run in gdb +You can also set a full profile path to `TREZOR_PROFILE_DIR` environment variable. -Running `emu.sh` with `-d` runs emulator inside gdb/lldb. +Specifying `-t` / `--temporary-profile` will start the emulator in a clean temporary +profile that will be erased when the emulator stops. This is useful, e.g., for tests. -### Watch for file changes +### Logging -Running `emu.sh` with `-r` watches for file changes and reloads the emulator if any occur. Note that this does not do rebuild, i.e. this works for MicroPython code (which is interpreted) but if you make C changes, you need to rebuild your self. +By default, emulator output goes to stdout. When silenced with `--quiet`, it is +redirected to `${TREZOR_PROFILE_DIR}/trezor.log`. You can specify an alternate output +file with `--output`. -### Print screen +### Running subcommands with the emulator -Press `p` on your keyboard to capture emulator's screen. You will find a png screenshot in the `src` directory. +In scripts, it is often necessary to start the emulator, run a commmand while it is +available, and then stop it. The following command runs the device test suite using the +emulator: -### Environment Variables +```sh +./emu.py --command pytest ../tests/device_tests +``` -#### Auto print screen +### Profiling support -If ``` TREZOR_SAVE_SCREEN=1 ``` is set, the emulator makes print screen on every screen change. +Run `./emu.py --profiling`, or set environment variable `TREZOR_PROFILING=1`, to run the +emulator with a profiling wrapper that generates statistics of executed lines. -#### Memory statistics +### Memory statistics -If ```TREZOR_LOG_MEMORY=1``` is set, the emulator prints memory usage information after each workflow task is finished. +Run `./emu.py --log-memory`, or set environment variable `TREZOR_LOG_MEMORY=1`, to dump +memory usage information after each workflow task is finished. -#### Disable animations +### Run in gdb + +Running `./emu.py --debugger` runs emulator inside gdb/lldb. + +### Watch for file changes + +Running `./emu.py --watch` watches for file changes and reloads the emulator if any +occur. Note that this does not do rebuild, i.e. this works for MicroPython code (which +is interpreted) but if you make C changes, you need to rebuild yourself. + +### Print screen -```TREZOR_DISABLE_ANIMATION=1``` disables fading and other animations, which speeds up the UI workflows significantly (useful for tests). This is also requirement for UI integration tests. +Press `p` on your keyboard to capture emulator's screen. You will find a png screenshot +in the `src` directory. -#### Tests +### Disable animation -```TREZOR_TEST``` informs whether device tests are to be run. Currently unused. +Run `./emu.py --disable-animation`, or set environment variable +`TREZOR_DISABLE_ANIMATION=1` to disable all animations. diff --git a/docs/tests/device-tests.md b/docs/tests/device-tests.md index f47e98d3b..c996d6169 100644 --- a/docs/tests/device-tests.md +++ b/docs/tests/device-tests.md @@ -25,10 +25,9 @@ environment: pipenv shell ``` -If you want to test against the emulator, run it in a separate terminal from the `core` -subdirectory: +If you want to test against the emulator, run it in a separate terminal: ```sh -PYOPT=0 ./emu.sh +./core/emu.py ``` Now you can run the test suite with `pytest` from the root directory: diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index 0110aeb85..4ba30f8a0 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). _At the moment, the project does **not** adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). That is expected to change with version 1.0._ +## [0.11.7] - Unreleased + +### Added + +- built-in functionality of UdpTransport to wait until an emulator comes up, and the + related command `trezorctl wait-for-emulator` + ## [0.11.6] - 2019-12-30 [0.11.6]: https://github.com/trezor/trezor-firmware/compare/python/v0.11.5...python/v0.11.6 diff --git a/python/src/trezorlib/_internal/emulator.py b/python/src/trezorlib/_internal/emulator.py new file mode 100644 index 000000000..fc2898f53 --- /dev/null +++ b/python/src/trezorlib/_internal/emulator.py @@ -0,0 +1,237 @@ +# This file is part of the Trezor project. +# +# Copyright (C) 2012-2019 SatoshiLabs and contributors +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the License along with this library. +# If not, see . + +import os +import subprocess +import time +from pathlib import Path + +from trezorlib.debuglink import TrezorClientDebugLink +from trezorlib.transport.udp import UdpTransport + + +def _rm_f(path): + try: + path.unlink() + except FileNotFoundError: + pass + + +class Emulator: + STORAGE_FILENAME = None + + def __init__( + self, + executable, + profile_dir, + *, + logfile=None, + storage=None, + headless=False, + debug=True, + extra_args=() + ): + self.executable = Path(executable).resolve() + if not executable.exists(): + raise ValueError( + "emulator executable not found: {}".format(self.executable) + ) + + self.profile_dir = Path(profile_dir).resolve() + if not self.profile_dir.exists(): + self.profile_dir.mkdir(parents=True) + elif not self.profile_dir.is_dir(): + raise ValueError("profile_dir is not a directory") + + self.workdir = self.profile_dir + + self.storage = self.profile_dir / self.STORAGE_FILENAME + if storage: + self.storage.write_bytes(storage) + + if logfile: + self.logfile = logfile + else: + self.logfile = self.profile_dir / "trezor.log" + + self.client = None + self.process = None + + self.port = 21324 + self.headless = headless + self.debug = debug + self.extra_args = list(extra_args) + + def make_args(self): + return [] + + def make_env(self): + return os.environ.copy() + + def _get_transport(self): + return UdpTransport("127.0.0.1:{}".format(self.port)) + + def wait_until_ready(self, timeout=30): + transport = self._get_transport() + transport.open() + start = time.monotonic() + try: + while True: + if transport._ping(): + break + if self.process.poll() is not None: + raise RuntimeError("Emulator proces died") + + elapsed = time.monotonic() - start + if elapsed >= timeout: + raise RuntimeError("Can't connect to emulator") + + time.sleep(0.1) + finally: + transport.close() + + def wait(self, timeout=None): + ret = self.process.wait(timeout=None) + self.stop() + return ret + + def launch_process(self): + args = self.make_args() + env = self.make_env() + + if hasattr(self.logfile, "write"): + output = self.logfile + else: + output = open(self.logfile, "w") + + return subprocess.Popen( + [self.executable] + args + self.extra_args, + cwd=self.workdir, + stdout=output, + stderr=subprocess.STDOUT, + env=env, + ) + + def start(self): + if self.process: + if self.process.poll() is not None: + # process has died, stop and start again + self.stop() + else: + # process is running, no need to start again + return + + self.process = self.launch_process() + self.wait_until_ready() + + (self.profile_dir / "trezor.pid").write_text(str(self.process.pid) + "\n") + (self.profile_dir / "trezor.port").write_text(str(self.port) + "\n") + + transport = self._get_transport() + self.client = TrezorClientDebugLink(transport, auto_interact=self.debug) + + self.client.open() + + def stop(self): + if self.client: + self.client.close() + self.client = None + + if self.process: + self.process.terminate() + try: + self.process.wait(1) + except subprocess.TimeoutExpired: + self.process.kill() + + _rm_f(self.profile_dir / "trezor.pid") + _rm_f(self.profile_dir / "trezor.port") + self.process = None + + def restart(self): + self.stop() + self.start() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.stop() + + def get_storage(self): + return self.storage.read_bytes() + + +class CoreEmulator(Emulator): + STORAGE_FILENAME = "trezor.flash" + + def __init__( + self, + *args, + port=None, + main_args=("-m", "main"), + workdir=None, + sdcard=None, + disable_animation=True, + heap_size="20M", + **kwargs + ): + super().__init__(*args, **kwargs) + if workdir is not None: + self.workdir = Path(workdir).resolve() + + self.sdcard = self.profile_dir / "trezor.sdcard" + if sdcard is not None: + self.sdcard.write_bytes(sdcard) + + if port: + self.port = port + self.disable_animation = disable_animation + self.main_args = list(main_args) + self.heap_size = heap_size + + def make_env(self): + env = super().make_env() + env.update( + TREZOR_PROFILE_DIR=str(self.profile_dir), + TREZOR_PROFILE=str(self.profile_dir), + TREZOR_UDP_PORT=str(self.port), + ) + if self.headless: + env["SDL_VIDEODRIVER"] = "dummy" + if self.disable_animation: + env["TREZOR_DISABLE_FADE"] = "1" + env["TREZOR_DISABLE_ANIMATION"] = "1" + + return env + + def make_args(self): + pyopt = "-O0" if self.debug else "-O1" + return ( + [pyopt, "-X", "heapsize={}".format(self.heap_size)] + + self.main_args + + self.extra_args + ) + + +class LegacyEmulator(Emulator): + STORAGE_FILENAME = "emulator.img" + + def make_env(self): + env = super().make_env() + if self.headless: + env["SDL_VIDEODRIVER"] = "dummy" + return env diff --git a/python/src/trezorlib/cli/trezorctl.py b/python/src/trezorlib/cli/trezorctl.py index 2db84a42c..95878fd9d 100755 --- a/python/src/trezorlib/cli/trezorctl.py +++ b/python/src/trezorlib/cli/trezorctl.py @@ -19,12 +19,14 @@ import json import os import sys +import time import click from .. import coins, log, messages, protobuf, ui from ..client import TrezorClient from ..transport import enumerate_devices, get_transport +from ..transport.udp import UdpTransport from . import ( binance, btc, @@ -180,7 +182,7 @@ def print_result(res, path, verbose, is_json): click.echo("%s: %s" % (k, v)) elif isinstance(res, protobuf.MessageType): click.echo(protobuf.format_message(res)) - else: + elif res is not None: click.echo(res) @@ -250,6 +252,28 @@ def usb_reset(): WebUsbTransport.enumerate(usb_reset=True) +@cli.command() +@click.option("-t", "--timeout", type=float, default=10, help="Timeout in seconds") +@click.pass_context +def wait_for_emulator(ctx, timeout): + """Wait until Trezor Emulator comes up. + + Tries to connect to emulator and returns when it succeeds. + """ + path = ctx.parent.params.get("path") + if path: + if not path.startswith("udp:"): + raise click.ClickException("You must use UDP path, not {}".format(path)) + path = path.replace("udp:", "") + + start = time.monotonic() + UdpTransport(path).wait_until_ready(timeout) + end = time.monotonic() + + if ctx.parent.params.get("verbose"): + click.echo("Waited for {:.3f} seconds".format(end - start)) + + # # Basic coin functions # diff --git a/python/src/trezorlib/transport/udp.py b/python/src/trezorlib/transport/udp.py index 6a04c2421..7a79079ed 100644 --- a/python/src/trezorlib/transport/udp.py +++ b/python/src/trezorlib/transport/udp.py @@ -15,6 +15,7 @@ # If not, see . import socket +import time from typing import Iterable, Optional, cast from . import TransportException @@ -60,7 +61,7 @@ class UdpTransport(ProtocolBasedTransport): return d else: raise TransportException( - "No Trezor device found at address {}".format(path) + "No Trezor device found at address {}".format(d.get_path()) ) finally: d.close() @@ -84,6 +85,22 @@ class UdpTransport(ProtocolBasedTransport): path = path.replace("{}:".format(cls.PATH_PREFIX), "") return cls._try_path(path) + def wait_until_ready(self, timeout: float = 10) -> None: + try: + self.open() + self.socket.settimeout(0) + start = time.monotonic() + while True: + if self._ping(): + break + elapsed = time.monotonic() - start + if elapsed >= timeout: + raise TransportException("Timed out waiting for connection.") + + time.sleep(0.05) + finally: + self.close() + def open(self) -> None: self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket.connect(self.device) diff --git a/tests/emulators.py b/tests/emulators.py index 71ee6cf73..c1df8cb12 100644 --- a/tests/emulators.py +++ b/tests/emulators.py @@ -15,23 +15,22 @@ # If not, see . import gzip -import os -import subprocess import tempfile -import time from collections import defaultdict +from pathlib import Path -from trezorlib.debuglink import TrezorClientDebugLink -from trezorlib.transport.udp import UdpTransport +from trezorlib._internal.emulator import CoreEmulator, LegacyEmulator + +ROOT = Path(__file__).parent.parent.resolve() +BINDIR = ROOT / "tests" / "emulators" -ROOT = os.path.abspath(os.path.dirname(__file__) + "/..") -BINDIR = ROOT + "/tests/emulators" LOCAL_BUILD_PATHS = { - "core": ROOT + "/core/build/unix/micropython", - "legacy": ROOT + "/legacy/firmware/trezor.elf", + "core": ROOT / "core" / "build" / "unix" / "micropython", + "legacy": ROOT / "legacy" / "firmware" / "trezor.elf", } -SD_CARD_GZ = ROOT + "/tests/trezor.sdcard.gz" +CORE_SRC_DIR = ROOT / "core" / "src" +SD_CARD_GZ = ROOT / "core" / "trezor.sdcard.gz" ENV = {"SDL_VIDEODRIVER": "dummy"} @@ -44,11 +43,11 @@ def check_version(tag, version_tuple): def filename_from_tag(gen, tag): - return f"{BINDIR}/trezor-emu-{gen}-{tag}" + return BINDIR / f"trezor-emu-{gen}-{tag}" def get_tags(): - files = os.listdir(BINDIR) + files = list(BINDIR.iterdir()) if not files: raise ValueError( "No files found. Use download_emulators.sh to download emulators." @@ -58,7 +57,7 @@ def get_tags(): for f in sorted(files): try: # example: "trezor-emu-core-v2.1.1" - _, _, gen, tag = f.split("-", maxsplit=3) + _, _, gen, tag = f.name.split("-", maxsplit=3) result[gen].append(tag) except ValueError: pass @@ -69,116 +68,40 @@ ALL_TAGS = get_tags() class EmulatorWrapper: - def __init__(self, gen, tag=None, executable=None, storage=None): - self.gen = gen - self.tag = tag - - if executable is not None: - self.executable = executable - elif tag is not None: - self.executable = filename_from_tag(gen, tag) - else: - self.executable = LOCAL_BUILD_PATHS[gen] - - if not os.path.exists(self.executable): - raise ValueError(f"emulator executable not found: {self.executable}") - - self.workdir = tempfile.TemporaryDirectory() - if storage: - open(self._storage_file(), "wb").write(storage) - - with gzip.open(SD_CARD_GZ, "rb") as gz: - with open(self.workdir.name + "/trezor.sdcard", "wb") as sd: - sd.write(gz.read()) - - self.client = None - - def _get_params_core(self): - env = ENV.copy() - args = [self.executable, "-m", "main"] - # for firmware 2.1.2 and newer - env["TREZOR_PROFILE_DIR"] = self.workdir.name - # for firmware 2.1.1 and older - env["TREZOR_PROFILE"] = self.workdir.name - - if self.executable == LOCAL_BUILD_PATHS["core"]: - cwd = ROOT + "/core/src" + def __init__(self, gen, tag=None, storage=None): + if tag is not None: + executable = filename_from_tag(gen, tag) else: - cwd = self.workdir.name + executable = LOCAL_BUILD_PATHS[gen] - return env, args, cwd + if not executable.exists(): + raise ValueError(f"emulator executable not found: {executable}") - def _get_params_legacy(self): - env = ENV.copy() - args = [self.executable] - cwd = self.workdir.name - return env, args, cwd - - def _get_params(self): - if self.gen == "core": - return self._get_params_core() - elif self.gen == "legacy": - return self._get_params_legacy() + self.profile_dir = tempfile.TemporaryDirectory() + if executable == LOCAL_BUILD_PATHS["core"]: + workdir = CORE_SRC_DIR else: - raise ValueError("Unknown gen") - - def start(self): - env, args, cwd = self._get_params() - self.process = subprocess.Popen( - args, cwd=cwd, env=env, stdout=open(os.devnull, "w") - ) - - # wait until emulator is listening - transport = UdpTransport("127.0.0.1:21324") - transport.open() - for _ in range(300): - if transport._ping(): - break - if self.process.poll() is not None: - self._cleanup() - raise RuntimeError("Emulator proces died") - time.sleep(0.1) - else: - # could not connect after 300 attempts * 0.1s = 30s of waiting - self._cleanup() - raise RuntimeError("Can't connect to emulator") - transport.close() - - self.client = TrezorClientDebugLink(transport) - self.client.open() - check_version(self.tag, self.client.version) - - def stop(self): - if self.client: - self.client.close() - self.process.terminate() - try: - self.process.wait(1) - except subprocess.TimeoutExpired: - self.process.kill() - - def restart(self): - self.stop() - self.start() + workdir = None + + if gen == "legacy": + self.emulator = LegacyEmulator( + executable, self.profile_dir.name, storage=storage, headless=True, + ) + elif gen == "core": + with gzip.open(SD_CARD_GZ, "rb") as gz: + self.emulator = CoreEmulator( + executable, + self.profile_dir.name, + storage=storage, + workdir=workdir, + sdcard=gz.read(), + headless=True, + ) def __enter__(self): - self.start() - return self + self.emulator.start() + return self.emulator def __exit__(self, exc_type, exc_value, traceback): - self._cleanup() - - def _cleanup(self): - self.stop() - self.workdir.cleanup() - - def _storage_file(self): - if self.gen == "legacy": - return self.workdir.name + "/emulator.img" - elif self.gen == "core": - return self.workdir.name + "/trezor.flash" - else: - raise ValueError("Unknown gen") - - def storage(self): - return open(self._storage_file(), "rb").read() + self.emulator.stop() + self.profile_dir.cleanup() diff --git a/tests/persistence_tests/test_shamir_persistence.py b/tests/persistence_tests/test_shamir_persistence.py index 33a6f8e4d..7118ebc7c 100644 --- a/tests/persistence_tests/test_shamir_persistence.py +++ b/tests/persistence_tests/test_shamir_persistence.py @@ -28,12 +28,11 @@ from ..upgrade_tests import core_only @pytest.fixture def emulator(): - emu = EmulatorWrapper("core") - with emu: + with EmulatorWrapper("core") as emu: yield emu -def _restart(device_handler: BackgroundDeviceHandler, emulator: EmulatorWrapper): +def _restart(device_handler, emulator): device_handler.restart(emulator) return device_handler.debuglink() diff --git a/tests/upgrade_tests/__init__.py b/tests/upgrade_tests/__init__.py index efeb28409..781b5720a 100644 --- a/tests/upgrade_tests/__init__.py +++ b/tests/upgrade_tests/__init__.py @@ -2,7 +2,7 @@ import os import pytest -from ..emulators import EmulatorWrapper +from ..emulators import LOCAL_BUILD_PATHS SELECTED_GENS = [ gen.strip() for gen in os.environ.get("TREZOR_UPGRADE_TEST", "").split(",") if gen @@ -15,17 +15,8 @@ if SELECTED_GENS: else: # if no selection was provided, select those for which we have emulators - try: - EmulatorWrapper("legacy") - LEGACY_ENABLED = True - except Exception: - LEGACY_ENABLED = False - - try: - EmulatorWrapper("core") - CORE_ENABLED = True - except Exception: - CORE_ENABLED = False + LEGACY_ENABLED = LOCAL_BUILD_PATHS["legacy"].exists() + CORE_ENABLED = LOCAL_BUILD_PATHS["core"].exists() legacy_only = pytest.mark.skipif( diff --git a/tests/upgrade_tests/test_firmware_upgrades.py b/tests/upgrade_tests/test_firmware_upgrades.py index ff7ede598..bfd1f9457 100644 --- a/tests/upgrade_tests/test_firmware_upgrades.py +++ b/tests/upgrade_tests/test_firmware_upgrades.py @@ -96,7 +96,7 @@ def test_upgrade_load(gen, from_tag, to_tag): ) device_id = emu.client.features.device_id asserts(from_tag, emu.client) - storage = emu.storage() + storage = emu.get_storage() with EmulatorWrapper(gen, to_tag, storage=storage) as emu: assert device_id == emu.client.features.device_id @@ -128,7 +128,7 @@ def test_upgrade_reset(gen, from_tag, to_tag): device_id = emu.client.features.device_id asserts(from_tag, emu.client) address = btc.get_address(emu.client, "Bitcoin", PATH) - storage = emu.storage() + storage = emu.get_storage() with EmulatorWrapper(gen, to_tag, storage=storage) as emu: assert device_id == emu.client.features.device_id @@ -162,7 +162,7 @@ def test_upgrade_reset_skip_backup(gen, from_tag, to_tag): device_id = emu.client.features.device_id asserts(from_tag, emu.client) address = btc.get_address(emu.client, "Bitcoin", PATH) - storage = emu.storage() + storage = emu.get_storage() with EmulatorWrapper(gen, to_tag, storage=storage) as emu: assert device_id == emu.client.features.device_id @@ -196,7 +196,7 @@ def test_upgrade_reset_no_backup(gen, from_tag, to_tag): device_id = emu.client.features.device_id asserts(from_tag, emu.client) address = btc.get_address(emu.client, "Bitcoin", PATH) - storage = emu.storage() + storage = emu.get_storage() with EmulatorWrapper(gen, to_tag, storage=storage) as emu: assert device_id == emu.client.features.device_id @@ -222,7 +222,7 @@ def test_upgrade_shamir_recovery(gen, from_tag, to_tag): assert "2 more shares" in layout.text device_id = emu.client.features.device_id - storage = emu.storage() + storage = emu.get_storage() device_handler.check_finalize() with EmulatorWrapper(gen, to_tag, storage=storage) as emu, BackgroundDeviceHandler( @@ -258,7 +258,7 @@ def test_upgrade_u2f(gen, from_tag, to_tag): counter = fido.get_next_counter(emu.client) assert counter == 11 - storage = emu.storage() + storage = emu.get_storage() with EmulatorWrapper(gen, to_tag, storage=storage) as emu: counter = fido.get_next_counter(emu.client)