feat(tests): add possibility to run device/UI tests in parallel

2024-12-26 00:08:10 +00:00 · 2022-08-18 17:21:27 +02:00 · 2022-08-18 17:21:27 +02:00 · d5b0650cc2
commit d5b0650cc2
parent 837988f61e
6 changed files with 181 additions and 55 deletions
--- a/core/Makefile
+++ b/core/Makefile
@ -88,6 +88,9 @@ test_rust: ## run rs unit tests
 test_emu: ## run selected device tests from python-trezor
 	$(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS)

+test_emu_multicore: ## run device tests using multiple cores
+	$(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM)
+
 test_emu_monero: ## run selected monero device tests from monero-agent
 	cd tests ; $(EMU_TEST) ./run_tests_device_emu_monero.sh $(TESTOPTS)

@ -105,9 +108,15 @@ test_emu_click: ## run click tests
 test_emu_ui: ## run ui integration tests
 	UI2="$(UI2)" $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests --ui=test --ui-check-missing $(TESTOPTS)

+test_emu_ui_multicore: ## run ui integration tests using multiple cores
+	UI2="$(UI2)" $(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) --ui=test --ui-check-missing --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM)
+
 test_emu_ui_record: ## record and hash screens for ui integration tests
 	UI2="$(UI2)" $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests --ui=record --ui-check-missing $(TESTOPTS)

+test_emu_ui_record_multicore: ## record and hash screens for ui integration tests using multiple cores
+	UI2="$(UI2)" $(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) --ui=record --ui-check-missing --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM)
+
 pylint: ## run pylint on application sources and tests
 	pylint -E $(shell find src tests -name *.py)

--- a/poetry.lock
+++ b/poetry.lock
@ -222,6 +222,17 @@ category = "main"
 optional = false
 python-versions = "*"

+[[package]]
+name = "execnet"
+version = "1.9.0"
+description = "execnet: rapid multi-Python deployment"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
+[package.extras]
+testing = ["pre-commit"]
+
 [[package]]
 name = "fido2"
 version = "0.8.1"
@ -689,6 +700,18 @@ toml = "*"
 [package.extras]
 testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]

+[[package]]
+name = "pytest-forked"
+version = "1.4.0"
+description = "run tests in isolated forked subprocesses"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+py = "*"
+pytest = ">=3.10"
+
 [[package]]
 name = "pytest-ordering"
 version = "0.6"
@ -722,6 +745,24 @@ python-versions = ">=3.6"
 [package.dependencies]
 pytest = ">=5.0.0"

+[[package]]
+name = "pytest-xdist"
+version = "2.5.0"
+description = "pytest xdist plugin for distributed testing and loop-on-failing modes"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+execnet = ">=1.1"
+pytest = ">=6.2.0"
+pytest-forked = "*"
+
+[package.extras]
+psutil = ["psutil (>=3.0)"]
+setproctitle = ["setproctitle"]
+testing = ["filelock"]
+
 [[package]]
 name = "python-bitcoinlib"
 version = "0.11.0"
@ -888,7 +929,7 @@ testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pytest (>=4.0.0)", "pytes

 [[package]]
 name = "trezor"
-version = "0.13.1"
+version = "0.13.4"
 description = "Python library for communicating with Trezor Hardware Wallet"
 category = "main"
 optional = false
@ -896,18 +937,19 @@ python-versions = ">=3.6"
 develop = true

 [package.dependencies]
-click = ">=7,<9"
-construct = ">=2.9"
+click = ">=7,<8.2"
+construct = ">=2.9,<2.10.55 || >2.10.55"
 ecdsa = ">=0.9"
 libusb1 = ">=1.6.4"
 mnemonic = ">=0.20"
 requests = ">=2.4.0"
+simple-rlp = {version = ">=0.1.2", markers = "python_version >= \"3.7\""}
 typing_extensions = ">=3.10"

 [package.extras]
-ethereum = ["rlp (>=1.1.0)", "web3 (>=4.8)"]
+ethereum = ["web3 (>=4.8)", "rlp (>=1.1.0)"]
 extra = ["pillow"]
-full = ["hidapi (>=0.7.99.post20)", "rlp (>=1.1.0)", "web3 (>=4.8)", "pyqt5", "pillow", "stellar-sdk (>=4.0.0,<6.0.0)"]
+full = ["hidapi (>=0.7.99.post20)", "web3 (>=4.8)", "pyqt5", "pillow", "stellar-sdk (>=4.0.0,<6.0.0)", "rlp (>=1.1.0)"]
 hidapi = ["hidapi (>=0.7.99.post20)"]
 qt-widgets = ["pyqt5"]
 stellar = ["stellar-sdk (>=4.0.0,<6.0.0)"]
@ -1007,7 +1049,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "0104cc6698a15ec443594e47494b450c33ef6f4d3a1f36d6f919a783f932a3fb"
+content-hash = "4a28460ada737fff859a9c4aacd94ed4ee76b551c099954ebea353f5b212dd2e"

 [metadata.files]
 astroid = [
@ -1025,31 +1067,7 @@ attrs = [
 autoflake = [
    {file = "autoflake-1.4.tar.gz", hash = "sha256:61a353012cff6ab94ca062823d1fb2f692c4acda51c76ff83a8d77915fba51ea"},
 ]
-black = [
-    {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"},
-    {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"},
-    {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"},
-    {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"},
-    {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"},
-    {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"},
-    {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"},
-    {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"},
-    {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"},
-    {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"},
-    {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"},
-    {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"},
-    {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"},
-    {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"},
-    {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"},
-    {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"},
-    {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"},
-    {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"},
-    {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"},
-    {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"},
-    {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"},
-    {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"},
-    {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"},
-]
+black = []
 certifi = [
    {file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"},
    {file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"},
@ -1201,6 +1219,7 @@ ecdsa = [
 ed25519 = [
    {file = "ed25519-1.5.tar.gz", hash = "sha256:02053ee019ceef0df97294be2d4d5a8fc120fc86e81e08bec1245fc0f9403358"},
 ]
+execnet = []
 fido2 = [
    {file = "fido2-0.8.1.tar.gz", hash = "sha256:449068f6876f397c8bb96ebc6a75c81c2692f045126d3f13ece21d409acdf7c3"},
 ]
@ -1543,6 +1562,7 @@ pytest = [
    {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"},
    {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"},
 ]
+pytest-forked = []
 pytest-ordering = [
    {file = "pytest-ordering-0.6.tar.gz", hash = "sha256:561ad653626bb171da78e682f6d39ac33bb13b3e272d406cd555adb6b006bda6"},
    {file = "pytest_ordering-0.6-py2-none-any.whl", hash = "sha256:27fba3fc265f5d0f8597e7557885662c1bdc1969497cd58aff6ed21c3b617de2"},
@ -1556,6 +1576,7 @@ pytest-timeout = [
    {file = "pytest-timeout-2.1.0.tar.gz", hash = "sha256:c07ca07404c612f8abbe22294b23c368e2e5104b521c1790195561f37e1ac3d9"},
    {file = "pytest_timeout-2.1.0-py3-none-any.whl", hash = "sha256:f6f50101443ce70ad325ceb4473c4255e9d74e3c7cd0ef827309dfa4c0d975c6"},
 ]
+pytest-xdist = []
 python-bitcoinlib = [
    {file = "python-bitcoinlib-0.11.0.tar.gz", hash = "sha256:3daafd63cb755f6e2067b7c9c514053856034c9f9363c80c37007744d54a2e06"},
    {file = "python_bitcoinlib-0.11.0-py3-none-any.whl", hash = "sha256:6e7982734637135599e2136d3c88d622f147e3b29201636665f799365784cd9e"},
--- a/pyproject.toml
+++ b/pyproject.toml
@ -21,6 +21,7 @@ flaky = "^3.6.1"  # https://github.com/box/flaky/issues/156
 pytest-ordering = "*"
 pytest-random-order = "*"
 pytest-timeout = "*"
+pytest-xdist = "*"
 tox = "*"
 dominate = "*"

--- a/python/src/trezorlib/_internal/emulator.py
+++ b/python/src/trezorlib/_internal/emulator.py
@ -48,6 +48,7 @@ class Emulator:
        storage: Optional[bytes] = None,
        headless: bool = False,
        debug: bool = True,
+        auto_interact: bool = True,
        extra_args: Iterable[str] = (),
    ) -> None:
        self.executable = Path(executable).resolve()
@ -77,6 +78,7 @@ class Emulator:
        self.port = 21324
        self.headless = headless
        self.debug = debug
+        self.auto_interact = auto_interact
        self.extra_args = list(extra_args)

    def make_args(self) -> List[str]:
@ -160,7 +162,7 @@ class Emulator:
        (self.profile_dir / "trezor.port").write_text(str(self.port) + "\n")

        transport = self._get_transport()
-        self.client = TrezorClientDebugLink(transport, auto_interact=self.debug)
+        self.client = TrezorClientDebugLink(transport, auto_interact=self.auto_interact)

        self.client.open()

--- a/tests/conftest.py
+++ b/tests/conftest.py
@ -14,6 +14,8 @@
 # You should have received a copy of the License along with this library.
 # If not, see <https://www.gnu.org/licenses/lgpl-3.0.html>.

+from __future__ import annotations
+
 import os
 from typing import TYPE_CHECKING, Generator

@ -26,9 +28,11 @@ from trezorlib.transport import enumerate_devices, get_transport

 from . import ui_tests
 from .device_handler import BackgroundDeviceHandler
+from .emulators import EmulatorWrapper
 from .ui_tests.reporting import testreport

 if TYPE_CHECKING:
+    from trezorlib._internal.emulator import Emulator
    from _pytest.config import Config
    from _pytest.config.argparsing import Parser
    from _pytest.terminal import TerminalReporter
@ -37,28 +41,92 @@ if TYPE_CHECKING:
 pytest.register_assert_rewrite("tests.common")


+@pytest.fixture(scope="session")
+def emulator(request: pytest.FixtureRequest) -> Generator["Emulator", None, None]:
+    """Fixture for getting emulator connection in case tests should operate it on their own.
+
+    Is responsible for starting it at the start of the session and stopping
+    it at the end of the session - using `with EmulatorWrapper...`.
+
+    Makes sure that each process will run the emulator on a different
+    port and with different profile directory, which is cleaned afterwards.
+
+    Used so that we can run the device tests in parallel using `pytest-xdist` plugin.
+    Docs: https://pypi.org/project/pytest-xdist/
+
+    NOTE for parallel tests:
+    So that all worker processes will explore the tests in the exact same order,
+    we cannot use the "built-in" random order, we need to specify our own,
+    so that all the processes share the same order.
+    Done by appending `--random-order-seed=$RANDOM` as a `pytest` argument,
+    using system RNG.
+    """
+
+    model = str(request.session.config.getoption("model"))
+    interact = os.environ.get("INTERACT") == "1"
+
+    assert model in ("core", "legacy")
+    if model == "legacy":
+        raise RuntimeError(
+            "Legacy emulator is not supported until it can be run on arbitrary ports."
+        )
+
+    def _get_port() -> int:
+        """Get a unique port for this worker process on which it can run.
+
+        Guarantees to be unique because each worker has a different name.
+        gw0=>20000, gw1=>20003, gw2=>20006, etc.
+        """
+        worker_id = os.getenv("PYTEST_XDIST_WORKER")
+        assert worker_id is not None
+        assert worker_id.startswith("gw")
+        # One emulator instance occupies 3 consecutive ports:
+        # 1. normal link, 2. debug link and 3. webauthn fake interface
+        return 20000 + int(worker_id[2:]) * 3
+
+    with EmulatorWrapper(
+        model, port=_get_port(), headless=True, auto_interact=not interact
+    ) as emu:
+        yield emu
+
+
@pytest.fixture(scope="session")
 def _raw_client(request: pytest.FixtureRequest) -> Client:
-    path = os.environ.get("TREZOR_PATH")
-    interact = int(os.environ.get("INTERACT", 0))
-    if path:
-        try:
-            transport = get_transport(path)
-            return Client(transport, auto_interact=not interact)
-        except Exception as e:
-            request.session.shouldstop = "Failed to communicate with Trezor"
-            raise RuntimeError(f"Failed to open debuglink for {path}") from e
-
+    # In case tests run in parallel, each process has its own emulator/client.
+    # Requesting the emulator fixture only if relevant.
+    if request.session.config.getoption("control_emulators"):
+        emu_fixture = request.getfixturevalue("emulator")
+        return emu_fixture.client
    else:
-        devices = enumerate_devices()
-        for device in devices:
-            try:
-                return Client(device, auto_interact=not interact)
-            except Exception:
-                pass
+        interact = os.environ.get("INTERACT") == "1"
+        path = os.environ.get("TREZOR_PATH")
+        if path:
+            return _client_from_path(request, path, interact)
+        else:
+            return _find_client(request, interact)

+
+def _client_from_path(
+    request: pytest.FixtureRequest, path: str, interact: bool
+) -> Client:
+    try:
+        transport = get_transport(path)
+        return Client(transport, auto_interact=not interact)
+    except Exception as e:
        request.session.shouldstop = "Failed to communicate with Trezor"
-        raise RuntimeError("No debuggable device found")
+        raise RuntimeError(f"Failed to open debuglink for {path}") from e
+
+
+def _find_client(request: pytest.FixtureRequest, interact: bool) -> Client:
+    devices = enumerate_devices()
+    for device in devices:
+        try:
+            return Client(device, auto_interact=not interact)
+        except Exception:
+            pass
+
+    request.session.shouldstop = "Failed to communicate with Trezor"
+    raise RuntimeError("No debuggable device found")


@pytest.fixture(scope="function")
@ -188,7 +256,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) -
            session.exitstatus = pytest.ExitCode.TESTS_FAILED
        ui_tests.write_fixtures_suggestion(missing)
        testreport.index()
-    if test_ui == "record":
+    elif test_ui == "record":
        if exitstatus == pytest.ExitCode.OK:
            ui_tests.write_fixtures(missing)
        else:
@ -240,7 +308,7 @@ def pytest_addoption(parser: "Parser") -> None:
        "--ui",
        action="store",
        choices=["test", "record"],
-        help="Enable UI intergration tests: 'record' or 'test'",
+        help="Enable UI integration tests: 'record' or 'test'",
    )
    parser.addoption(
        "--ui-check-missing",
@ -249,6 +317,20 @@ def pytest_addoption(parser: "Parser") -> None:
        help="Check UI fixtures are containing the appropriate test cases (fails on `test`,"
        "deletes old ones on `record`).",
    )
+    parser.addoption(
+        "--control-emulators",
+        action="store_true",
+        default=False,
+        help="Pytest will be responsible for starting and stopping the emulators. "
+        "Useful when running tests in parallel.",
+    )
+    parser.addoption(
+        "--model",
+        action="store",
+        choices=["core", "legacy"],
+        help="Which emulator to use: 'core' or 'legacy'. "
+        "Only valid in connection with `--control-emulators`",
+    )


 def pytest_configure(config: "Config") -> None:
--- a/tests/emulators.py
+++ b/tests/emulators.py
@ -17,7 +17,7 @@
 import tempfile
 from collections import defaultdict
 from pathlib import Path
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple

 from trezorlib._internal.emulator import CoreEmulator, Emulator, LegacyEmulator

@ -67,7 +67,15 @@ ALL_TAGS = get_tags()


 class EmulatorWrapper:
-    def __init__(self, gen: str, tag: str = None, storage: bytes = None) -> None:
+    def __init__(
+        self,
+        gen: str,
+        tag: Optional[str] = None,
+        storage: Optional[bytes] = None,
+        port: Optional[int] = None,
+        headless: bool = True,
+        auto_interact: bool = True,
+    ) -> None:
        if tag is not None:
            executable = filename_from_tag(gen, tag)
        else:
@ -87,7 +95,8 @@ class EmulatorWrapper:
                executable,
                self.profile_dir.name,
                storage=storage,
-                headless=True,
+                headless=headless,
+                auto_interact=auto_interact,
            )
        elif gen == "core":
            self.emulator = CoreEmulator(
@ -95,7 +104,9 @@ class EmulatorWrapper:
                self.profile_dir.name,
                storage=storage,
                workdir=workdir,
-                headless=True,
+                port=port,
+                headless=headless,
+                auto_interact=auto_interact,
            )
        else:
            raise ValueError(