diff --git a/core/Makefile b/core/Makefile index 915aeebb6..0c5c149f1 100644 --- a/core/Makefile +++ b/core/Makefile @@ -88,6 +88,9 @@ test_rust: ## run rs unit tests test_emu: ## run selected device tests from python-trezor $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) +test_emu_multicore: ## run device tests using multiple cores + $(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM) + test_emu_monero: ## run selected monero device tests from monero-agent cd tests ; $(EMU_TEST) ./run_tests_device_emu_monero.sh $(TESTOPTS) @@ -105,9 +108,15 @@ test_emu_click: ## run click tests test_emu_ui: ## run ui integration tests UI2="$(UI2)" $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests --ui=test --ui-check-missing $(TESTOPTS) +test_emu_ui_multicore: ## run ui integration tests using multiple cores + UI2="$(UI2)" $(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) --ui=test --ui-check-missing --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM) + test_emu_ui_record: ## record and hash screens for ui integration tests UI2="$(UI2)" $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests --ui=record --ui-check-missing $(TESTOPTS) +test_emu_ui_record_multicore: ## record and hash screens for ui integration tests using multiple cores + UI2="$(UI2)" $(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) --ui=record --ui-check-missing --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM) + pylint: ## run pylint on application sources and tests pylint -E $(shell find src tests -name *.py) diff --git a/poetry.lock b/poetry.lock index 6bf2e1a09..bd73c8b4a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -222,6 +222,17 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "execnet" +version = "1.9.0" +description = "execnet: rapid multi-Python deployment" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +testing = ["pre-commit"] + [[package]] name = "fido2" version = "0.8.1" @@ -689,6 +700,18 @@ toml = "*" [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] +[[package]] +name = "pytest-forked" +version = "1.4.0" +description = "run tests in isolated forked subprocesses" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +py = "*" +pytest = ">=3.10" + [[package]] name = "pytest-ordering" version = "0.6" @@ -722,6 +745,24 @@ python-versions = ">=3.6" [package.dependencies] pytest = ">=5.0.0" +[[package]] +name = "pytest-xdist" +version = "2.5.0" +description = "pytest xdist plugin for distributed testing and loop-on-failing modes" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +execnet = ">=1.1" +pytest = ">=6.2.0" +pytest-forked = "*" + +[package.extras] +psutil = ["psutil (>=3.0)"] +setproctitle = ["setproctitle"] +testing = ["filelock"] + [[package]] name = "python-bitcoinlib" version = "0.11.0" @@ -888,7 +929,7 @@ testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pytest (>=4.0.0)", "pytes [[package]] name = "trezor" -version = "0.13.1" +version = "0.13.4" description = "Python library for communicating with Trezor Hardware Wallet" category = "main" optional = false @@ -896,18 +937,19 @@ python-versions = ">=3.6" develop = true [package.dependencies] -click = ">=7,<9" -construct = ">=2.9" +click = ">=7,<8.2" +construct = ">=2.9,<2.10.55 || >2.10.55" ecdsa = ">=0.9" libusb1 = ">=1.6.4" mnemonic = ">=0.20" requests = ">=2.4.0" +simple-rlp = {version = ">=0.1.2", markers = "python_version >= \"3.7\""} typing_extensions = ">=3.10" [package.extras] -ethereum = ["rlp (>=1.1.0)", "web3 (>=4.8)"] +ethereum = ["web3 (>=4.8)", "rlp (>=1.1.0)"] extra = ["pillow"] -full = ["hidapi (>=0.7.99.post20)", "rlp (>=1.1.0)", "web3 (>=4.8)", "pyqt5", "pillow", "stellar-sdk (>=4.0.0,<6.0.0)"] +full = ["hidapi (>=0.7.99.post20)", "web3 (>=4.8)", "pyqt5", "pillow", "stellar-sdk (>=4.0.0,<6.0.0)", "rlp (>=1.1.0)"] hidapi = ["hidapi (>=0.7.99.post20)"] qt-widgets = ["pyqt5"] stellar = ["stellar-sdk (>=4.0.0,<6.0.0)"] @@ -1007,7 +1049,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "0104cc6698a15ec443594e47494b450c33ef6f4d3a1f36d6f919a783f932a3fb" +content-hash = "4a28460ada737fff859a9c4aacd94ed4ee76b551c099954ebea353f5b212dd2e" [metadata.files] astroid = [ @@ -1025,31 +1067,7 @@ attrs = [ autoflake = [ {file = "autoflake-1.4.tar.gz", hash = "sha256:61a353012cff6ab94ca062823d1fb2f692c4acda51c76ff83a8d77915fba51ea"}, ] -black = [ - {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, - {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, - {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, - {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, - {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, - {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, - {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, - {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, - {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, - {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, - {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, - {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, - {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, - {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, - {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, - {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, - {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, - {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, - {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, - {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, - {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, - {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, - {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, -] +black = [] certifi = [ {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"}, {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"}, @@ -1201,6 +1219,7 @@ ecdsa = [ ed25519 = [ {file = "ed25519-1.5.tar.gz", hash = "sha256:02053ee019ceef0df97294be2d4d5a8fc120fc86e81e08bec1245fc0f9403358"}, ] +execnet = [] fido2 = [ {file = "fido2-0.8.1.tar.gz", hash = "sha256:449068f6876f397c8bb96ebc6a75c81c2692f045126d3f13ece21d409acdf7c3"}, ] @@ -1543,6 +1562,7 @@ pytest = [ {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, ] +pytest-forked = [] pytest-ordering = [ {file = "pytest-ordering-0.6.tar.gz", hash = "sha256:561ad653626bb171da78e682f6d39ac33bb13b3e272d406cd555adb6b006bda6"}, {file = "pytest_ordering-0.6-py2-none-any.whl", hash = "sha256:27fba3fc265f5d0f8597e7557885662c1bdc1969497cd58aff6ed21c3b617de2"}, @@ -1556,6 +1576,7 @@ pytest-timeout = [ {file = "pytest-timeout-2.1.0.tar.gz", hash = "sha256:c07ca07404c612f8abbe22294b23c368e2e5104b521c1790195561f37e1ac3d9"}, {file = "pytest_timeout-2.1.0-py3-none-any.whl", hash = "sha256:f6f50101443ce70ad325ceb4473c4255e9d74e3c7cd0ef827309dfa4c0d975c6"}, ] +pytest-xdist = [] python-bitcoinlib = [ {file = "python-bitcoinlib-0.11.0.tar.gz", hash = "sha256:3daafd63cb755f6e2067b7c9c514053856034c9f9363c80c37007744d54a2e06"}, {file = "python_bitcoinlib-0.11.0-py3-none-any.whl", hash = "sha256:6e7982734637135599e2136d3c88d622f147e3b29201636665f799365784cd9e"}, diff --git a/pyproject.toml b/pyproject.toml index e4e931b69..34eb3788a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ flaky = "^3.6.1" # https://github.com/box/flaky/issues/156 pytest-ordering = "*" pytest-random-order = "*" pytest-timeout = "*" +pytest-xdist = "*" tox = "*" dominate = "*" diff --git a/python/src/trezorlib/_internal/emulator.py b/python/src/trezorlib/_internal/emulator.py index e35c64f59..39704803c 100644 --- a/python/src/trezorlib/_internal/emulator.py +++ b/python/src/trezorlib/_internal/emulator.py @@ -48,6 +48,7 @@ class Emulator: storage: Optional[bytes] = None, headless: bool = False, debug: bool = True, + auto_interact: bool = True, extra_args: Iterable[str] = (), ) -> None: self.executable = Path(executable).resolve() @@ -77,6 +78,7 @@ class Emulator: self.port = 21324 self.headless = headless self.debug = debug + self.auto_interact = auto_interact self.extra_args = list(extra_args) def make_args(self) -> List[str]: @@ -160,7 +162,7 @@ class Emulator: (self.profile_dir / "trezor.port").write_text(str(self.port) + "\n") transport = self._get_transport() - self.client = TrezorClientDebugLink(transport, auto_interact=self.debug) + self.client = TrezorClientDebugLink(transport, auto_interact=self.auto_interact) self.client.open() diff --git a/tests/conftest.py b/tests/conftest.py index 60ee7b2f6..8885b84da 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,8 @@ # You should have received a copy of the License along with this library. # If not, see . +from __future__ import annotations + import os from typing import TYPE_CHECKING, Generator @@ -26,9 +28,11 @@ from trezorlib.transport import enumerate_devices, get_transport from . import ui_tests from .device_handler import BackgroundDeviceHandler +from .emulators import EmulatorWrapper from .ui_tests.reporting import testreport if TYPE_CHECKING: + from trezorlib._internal.emulator import Emulator from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.terminal import TerminalReporter @@ -38,27 +42,91 @@ pytest.register_assert_rewrite("tests.common") @pytest.fixture(scope="session") -def _raw_client(request: pytest.FixtureRequest) -> Client: - path = os.environ.get("TREZOR_PATH") - interact = int(os.environ.get("INTERACT", 0)) - if path: - try: - transport = get_transport(path) - return Client(transport, auto_interact=not interact) - except Exception as e: - request.session.shouldstop = "Failed to communicate with Trezor" - raise RuntimeError(f"Failed to open debuglink for {path}") from e +def emulator(request: pytest.FixtureRequest) -> Generator["Emulator", None, None]: + """Fixture for getting emulator connection in case tests should operate it on their own. + + Is responsible for starting it at the start of the session and stopping + it at the end of the session - using `with EmulatorWrapper...`. + + Makes sure that each process will run the emulator on a different + port and with different profile directory, which is cleaned afterwards. + + Used so that we can run the device tests in parallel using `pytest-xdist` plugin. + Docs: https://pypi.org/project/pytest-xdist/ + + NOTE for parallel tests: + So that all worker processes will explore the tests in the exact same order, + we cannot use the "built-in" random order, we need to specify our own, + so that all the processes share the same order. + Done by appending `--random-order-seed=$RANDOM` as a `pytest` argument, + using system RNG. + """ + + model = str(request.session.config.getoption("model")) + interact = os.environ.get("INTERACT") == "1" + + assert model in ("core", "legacy") + if model == "legacy": + raise RuntimeError( + "Legacy emulator is not supported until it can be run on arbitrary ports." + ) + + def _get_port() -> int: + """Get a unique port for this worker process on which it can run. + + Guarantees to be unique because each worker has a different name. + gw0=>20000, gw1=>20003, gw2=>20006, etc. + """ + worker_id = os.getenv("PYTEST_XDIST_WORKER") + assert worker_id is not None + assert worker_id.startswith("gw") + # One emulator instance occupies 3 consecutive ports: + # 1. normal link, 2. debug link and 3. webauthn fake interface + return 20000 + int(worker_id[2:]) * 3 + + with EmulatorWrapper( + model, port=_get_port(), headless=True, auto_interact=not interact + ) as emu: + yield emu + +@pytest.fixture(scope="session") +def _raw_client(request: pytest.FixtureRequest) -> Client: + # In case tests run in parallel, each process has its own emulator/client. + # Requesting the emulator fixture only if relevant. + if request.session.config.getoption("control_emulators"): + emu_fixture = request.getfixturevalue("emulator") + return emu_fixture.client else: - devices = enumerate_devices() - for device in devices: - try: - return Client(device, auto_interact=not interact) - except Exception: - pass + interact = os.environ.get("INTERACT") == "1" + path = os.environ.get("TREZOR_PATH") + if path: + return _client_from_path(request, path, interact) + else: + return _find_client(request, interact) + +def _client_from_path( + request: pytest.FixtureRequest, path: str, interact: bool +) -> Client: + try: + transport = get_transport(path) + return Client(transport, auto_interact=not interact) + except Exception as e: request.session.shouldstop = "Failed to communicate with Trezor" - raise RuntimeError("No debuggable device found") + raise RuntimeError(f"Failed to open debuglink for {path}") from e + + +def _find_client(request: pytest.FixtureRequest, interact: bool) -> Client: + devices = enumerate_devices() + for device in devices: + try: + return Client(device, auto_interact=not interact) + except Exception: + pass + + request.session.shouldstop = "Failed to communicate with Trezor" + raise RuntimeError("No debuggable device found") @pytest.fixture(scope="function") @@ -188,7 +256,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) - session.exitstatus = pytest.ExitCode.TESTS_FAILED ui_tests.write_fixtures_suggestion(missing) testreport.index() - if test_ui == "record": + elif test_ui == "record": if exitstatus == pytest.ExitCode.OK: ui_tests.write_fixtures(missing) else: @@ -240,7 +308,7 @@ def pytest_addoption(parser: "Parser") -> None: "--ui", action="store", choices=["test", "record"], - help="Enable UI intergration tests: 'record' or 'test'", + help="Enable UI integration tests: 'record' or 'test'", ) parser.addoption( "--ui-check-missing", @@ -249,6 +317,20 @@ def pytest_addoption(parser: "Parser") -> None: help="Check UI fixtures are containing the appropriate test cases (fails on `test`," "deletes old ones on `record`).", ) + parser.addoption( + "--control-emulators", + action="store_true", + default=False, + help="Pytest will be responsible for starting and stopping the emulators. " + "Useful when running tests in parallel.", + ) + parser.addoption( + "--model", + action="store", + choices=["core", "legacy"], + help="Which emulator to use: 'core' or 'legacy'. " + "Only valid in connection with `--control-emulators`", + ) def pytest_configure(config: "Config") -> None: diff --git a/tests/emulators.py b/tests/emulators.py index 85e899da2..9fe1048f8 100644 --- a/tests/emulators.py +++ b/tests/emulators.py @@ -17,7 +17,7 @@ import tempfile from collections import defaultdict from pathlib import Path -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Tuple from trezorlib._internal.emulator import CoreEmulator, Emulator, LegacyEmulator @@ -67,7 +67,15 @@ ALL_TAGS = get_tags() class EmulatorWrapper: - def __init__(self, gen: str, tag: str = None, storage: bytes = None) -> None: + def __init__( + self, + gen: str, + tag: Optional[str] = None, + storage: Optional[bytes] = None, + port: Optional[int] = None, + headless: bool = True, + auto_interact: bool = True, + ) -> None: if tag is not None: executable = filename_from_tag(gen, tag) else: @@ -87,7 +95,8 @@ class EmulatorWrapper: executable, self.profile_dir.name, storage=storage, - headless=True, + headless=headless, + auto_interact=auto_interact, ) elif gen == "core": self.emulator = CoreEmulator( @@ -95,7 +104,9 @@ class EmulatorWrapper: self.profile_dir.name, storage=storage, workdir=workdir, - headless=True, + port=port, + headless=headless, + auto_interact=auto_interact, ) else: raise ValueError(