arno
/
trezor-firmware
mirror of https://github.com/trezor/trezor-firmware.git
1
0
Fork 0
Code Issues Releases Wiki Activity

feat(tests): add possibility to run device/UI tests in parallel

Browse Source
pull/2473/head
grdddj 2 years ago committed by Jiří Musil
parent 837988f61e
commit d5b0650cc2
6 changed files with 181 additions and 55 deletions
Show Stats Download Patch File Download Diff File

9
core/Makefile
Unescape View File

@ -88,6 +88,9 @@ test_rust: ## run rs unit tests
test_emu: ## run selected device tests from python-trezor
$(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS)
test_emu_multicore: ## run device tests using multiple cores
$(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM)
test_emu_monero: ## run selected monero device tests from monero-agent
cd tests ; $(EMU_TEST) ./run_tests_device_emu_monero.sh $(TESTOPTS)
@ -105,9 +108,15 @@ test_emu_click: ## run click tests
test_emu_ui: ## run ui integration tests
UI2="$(UI2)" $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests --ui=test --ui-check-missing $(TESTOPTS)
test_emu_ui_multicore: ## run ui integration tests using multiple cores
UI2="$(UI2)" $(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) --ui=test --ui-check-missing --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM)
test_emu_ui_record: ## record and hash screens for ui integration tests
UI2="$(UI2)" $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests --ui=record --ui-check-missing $(TESTOPTS)
test_emu_ui_record_multicore: ## record and hash screens for ui integration tests using multiple cores
UI2="$(UI2)" $(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) --ui=record --ui-check-missing --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM)
pylint: ## run pylint on application sources and tests
pylint -E $(shell find src tests -name *.py)

83
poetry.lock generated
Unescape View File

@ -222,6 +222,17 @@ category = "main"
optional = false
python-versions = "*"
[[package]]
name = "execnet"
version = "1.9.0"
description = "execnet: rapid multi-Python deployment"
category = "main"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[package.extras]
testing = ["pre-commit"]
[[package]]
name = "fido2"
version = "0.8.1"
@ -689,6 +700,18 @@ toml = "*"
[package.extras]
testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
[[package]]
name = "pytest-forked"
version = "1.4.0"
description = "run tests in isolated forked subprocesses"
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
py = "*"
pytest = ">=3.10"
[[package]]
name = "pytest-ordering"
version = "0.6"
@ -722,6 +745,24 @@ python-versions = ">=3.6"
[package.dependencies]
pytest = ">=5.0.0"
[[package]]
name = "pytest-xdist"
version = "2.5.0"
description = "pytest xdist plugin for distributed testing and loop-on-failing modes"
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
execnet = ">=1.1"
pytest = ">=6.2.0"
pytest-forked = "*"
[package.extras]
psutil = ["psutil (>=3.0)"]
setproctitle = ["setproctitle"]
testing = ["filelock"]
[[package]]
name = "python-bitcoinlib"
version = "0.11.0"
@ -888,7 +929,7 @@ testing = ["flaky (>=3.4.0)", "freezegun (>=0.3.11)", "pytest (>=4.0.0)", "pytes
[[package]]
name = "trezor"
version = "0.13.1"
version = "0.13.4"
description = "Python library for communicating with Trezor Hardware Wallet"
category = "main"
optional = false
@ -896,18 +937,19 @@ python-versions = ">=3.6"
develop = true
[package.dependencies]
click = ">=7,<9"
construct = ">=2.9"
click = ">=7,<8.2"
construct = ">=2.9,<2.10.55 || >2.10.55"
ecdsa = ">=0.9"
libusb1 = ">=1.6.4"
mnemonic = ">=0.20"
requests = ">=2.4.0"
simple-rlp = {version = ">=0.1.2", markers = "python_version >= \"3.7\""}
typing_extensions = ">=3.10"
[package.extras]
ethereum = ["rlp (>=1.1.0)", "web3 (>=4.8)"]
ethereum = ["web3 (>=4.8)", "rlp (>=1.1.0)"]
extra = ["pillow"]
full = ["hidapi (>=0.7.99.post20)", "rlp (>=1.1.0)", "web3 (>=4.8)", "pyqt5", "pillow", "stellar-sdk (>=4.0.0,<6.0.0)"]
full = ["hidapi (>=0.7.99.post20)", "web3 (>=4.8)", "pyqt5", "pillow", "stellar-sdk (>=4.0.0,<6.0.0)", "rlp (>=1.1.0)"]
hidapi = ["hidapi (>=0.7.99.post20)"]
qt-widgets = ["pyqt5"]
stellar = ["stellar-sdk (>=4.0.0,<6.0.0)"]
@ -1007,7 +1049,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
[metadata]
lock-version = "1.1"
python-versions = "^3.7"
content-hash = "0104cc6698a15ec443594e47494b450c33ef6f4d3a1f36d6f919a783f932a3fb"
content-hash = "4a28460ada737fff859a9c4aacd94ed4ee76b551c099954ebea353f5b212dd2e"
[metadata.files]
astroid = [
@ -1025,31 +1067,7 @@ attrs = [
autoflake = [
{file = "autoflake-1.4.tar.gz", hash = "sha256:61a353012cff6ab94ca062823d1fb2f692c4acda51c76ff83a8d77915fba51ea"},
]
black = [
{file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"},
{file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"},
{file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"},
{file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"},
{file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"},
{file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"},
{file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"},
{file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"},
{file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"},
{file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"},
{file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"},
{file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"},
{file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"},
{file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"},
{file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"},
{file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"},
{file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"},
{file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"},
{file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"},
{file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"},
{file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"},
{file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"},
{file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"},
]
black = []
certifi = [
{file = "certifi-2021.10.8-py2.py3-none-any.whl", hash = "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"},
{file = "certifi-2021.10.8.tar.gz", hash = "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872"},
@ -1201,6 +1219,7 @@ ecdsa = [
ed25519 = [
{file = "ed25519-1.5.tar.gz", hash = "sha256:02053ee019ceef0df97294be2d4d5a8fc120fc86e81e08bec1245fc0f9403358"},
]
execnet = []
fido2 = [
{file = "fido2-0.8.1.tar.gz", hash = "sha256:449068f6876f397c8bb96ebc6a75c81c2692f045126d3f13ece21d409acdf7c3"},
]
@ -1543,6 +1562,7 @@ pytest = [
{file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"},
{file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"},
]
pytest-forked = []
pytest-ordering = [
{file = "pytest-ordering-0.6.tar.gz", hash = "sha256:561ad653626bb171da78e682f6d39ac33bb13b3e272d406cd555adb6b006bda6"},
{file = "pytest_ordering-0.6-py2-none-any.whl", hash = "sha256:27fba3fc265f5d0f8597e7557885662c1bdc1969497cd58aff6ed21c3b617de2"},
@ -1556,6 +1576,7 @@ pytest-timeout = [
{file = "pytest-timeout-2.1.0.tar.gz", hash = "sha256:c07ca07404c612f8abbe22294b23c368e2e5104b521c1790195561f37e1ac3d9"},
{file = "pytest_timeout-2.1.0-py3-none-any.whl", hash = "sha256:f6f50101443ce70ad325ceb4473c4255e9d74e3c7cd0ef827309dfa4c0d975c6"},
]
pytest-xdist = []
python-bitcoinlib = [
{file = "python-bitcoinlib-0.11.0.tar.gz", hash = "sha256:3daafd63cb755f6e2067b7c9c514053856034c9f9363c80c37007744d54a2e06"},
{file = "python_bitcoinlib-0.11.0-py3-none-any.whl", hash = "sha256:6e7982734637135599e2136d3c88d622f147e3b29201636665f799365784cd9e"},

1
pyproject.toml
Unescape View File

@ -21,6 +21,7 @@ flaky = "^3.6.1" # https://github.com/box/flaky/issues/156
pytest-ordering = "*"
pytest-random-order = "*"
pytest-timeout = "*"
pytest-xdist = "*"
tox = "*"
dominate = "*"

4
python/src/trezorlib/_internal/emulator.py
Unescape View File

@ -48,6 +48,7 @@ class Emulator:
storage: Optional[bytes] = None,
headless: bool = False,
debug: bool = True,
auto_interact: bool = True,
extra_args: Iterable[str] = (),
) -> None:
self.executable = Path(executable).resolve()
@ -77,6 +78,7 @@ class Emulator:
self.port = 21324
self.headless = headless
self.debug = debug
self.auto_interact = auto_interact
self.extra_args = list(extra_args)
def make_args(self) -> List[str]:
@ -160,7 +162,7 @@ class Emulator:
(self.profile_dir / "trezor.port").write_text(str(self.port) + "\n")
transport = self._get_transport()
self.client = TrezorClientDebugLink(transport, auto_interact=self.debug)
self.client = TrezorClientDebugLink(transport, auto_interact=self.auto_interact)
self.client.open()

120
tests/conftest.py
Unescape View File

@ -14,6 +14,8 @@
# You should have received a copy of the License along with this library.
# If not, see <https://www.gnu.org/licenses/lgpl-3.0.html>.
from __future__ import annotations
import os
from typing import TYPE_CHECKING, Generator
@ -26,9 +28,11 @@ from trezorlib.transport import enumerate_devices, get_transport
from . import ui_tests
from .device_handler import BackgroundDeviceHandler
from .emulators import EmulatorWrapper
from .ui_tests.reporting import testreport
if TYPE_CHECKING:
from trezorlib._internal.emulator import Emulator
from _pytest.config import Config
from _pytest.config.argparsing import Parser
from _pytest.terminal import TerminalReporter
@ -38,27 +42,91 @@ pytest.register_assert_rewrite("tests.common")
@pytest.fixture(scope="session")
def _raw_client(request: pytest.FixtureRequest) -> Client:
path = os.environ.get("TREZOR_PATH")
interact = int(os.environ.get("INTERACT", 0))
if path:
try:
transport = get_transport(path)
return Client(transport, auto_interact=not interact)
except Exception as e:
request.session.shouldstop = "Failed to communicate with Trezor"
raise RuntimeError(f"Failed to open debuglink for {path}") from e
def emulator(request: pytest.FixtureRequest) -> Generator["Emulator", None, None]:
"""Fixture for getting emulator connection in case tests should operate it on their own.
Is responsible for starting it at the start of the session and stopping
it at the end of the session - using `with EmulatorWrapper...`.
Makes sure that each process will run the emulator on a different
port and with different profile directory, which is cleaned afterwards.
Used so that we can run the device tests in parallel using `pytest-xdist` plugin.
Docs: https://pypi.org/project/pytest-xdist/
NOTE for parallel tests:
So that all worker processes will explore the tests in the exact same order,
we cannot use the "built-in" random order, we need to specify our own,
so that all the processes share the same order.
Done by appending `--random-order-seed=$RANDOM` as a `pytest` argument,
using system RNG.
"""
model = str(request.session.config.getoption("model"))
interact = os.environ.get("INTERACT") == "1"
assert model in ("core", "legacy")
if model == "legacy":
raise RuntimeError(
"Legacy emulator is not supported until it can be run on arbitrary ports."
)
def _get_port() -> int:
"""Get a unique port for this worker process on which it can run.
Guarantees to be unique because each worker has a different name.
gw0=>20000, gw1=>20003, gw2=>20006, etc.
"""
worker_id = os.getenv("PYTEST_XDIST_WORKER")
assert worker_id is not None
assert worker_id.startswith("gw")
# One emulator instance occupies 3 consecutive ports:
# 1. normal link, 2. debug link and 3. webauthn fake interface
return 20000 + int(worker_id[2:]) * 3
with EmulatorWrapper(
model, port=_get_port(), headless=True, auto_interact=not interact
) as emu:
yield emu
@pytest.fixture(scope="session")
def _raw_client(request: pytest.FixtureRequest) -> Client:
# In case tests run in parallel, each process has its own emulator/client.
# Requesting the emulator fixture only if relevant.
if request.session.config.getoption("control_emulators"):
emu_fixture = request.getfixturevalue("emulator")
return emu_fixture.client
else:
devices = enumerate_devices()
for device in devices:
try:
return Client(device, auto_interact=not interact)
except Exception:
pass
interact = os.environ.get("INTERACT") == "1"
path = os.environ.get("TREZOR_PATH")
if path:
return _client_from_path(request, path, interact)
else:
return _find_client(request, interact)
def _client_from_path(
request: pytest.FixtureRequest, path: str, interact: bool
) -> Client:
try:
transport = get_transport(path)
return Client(transport, auto_interact=not interact)
except Exception as e:
request.session.shouldstop = "Failed to communicate with Trezor"
raise RuntimeError("No debuggable device found")
raise RuntimeError(f"Failed to open debuglink for {path}") from e
def _find_client(request: pytest.FixtureRequest, interact: bool) -> Client:
devices = enumerate_devices()
for device in devices:
try:
return Client(device, auto_interact=not interact)
except Exception:
pass
request.session.shouldstop = "Failed to communicate with Trezor"
raise RuntimeError("No debuggable device found")
@pytest.fixture(scope="function")
@ -188,7 +256,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) -
session.exitstatus = pytest.ExitCode.TESTS_FAILED
ui_tests.write_fixtures_suggestion(missing)
testreport.index()
if test_ui == "record":
elif test_ui == "record":
if exitstatus == pytest.ExitCode.OK:
ui_tests.write_fixtures(missing)
else:
@ -240,7 +308,7 @@ def pytest_addoption(parser: "Parser") -> None:
"--ui",
action="store",
choices=["test", "record"],
help="Enable UI intergration tests: 'record' or 'test'",
help="Enable UI integration tests: 'record' or 'test'",
)
parser.addoption(
"--ui-check-missing",
@ -249,6 +317,20 @@ def pytest_addoption(parser: "Parser") -> None:
help="Check UI fixtures are containing the appropriate test cases (fails on `test`,"
"deletes old ones on `record`).",
)
parser.addoption(
"--control-emulators",
action="store_true",
default=False,
help="Pytest will be responsible for starting and stopping the emulators. "
"Useful when running tests in parallel.",
)
parser.addoption(
"--model",
action="store",
choices=["core", "legacy"],
help="Which emulator to use: 'core' or 'legacy'. "
"Only valid in connection with `--control-emulators`",
)
def pytest_configure(config: "Config") -> None:

19
tests/emulators.py
Unescape View File

@ -17,7 +17,7 @@
import tempfile
from collections import defaultdict
from pathlib import Path
from typing import Dict, List, Tuple
from typing import Dict, List, Optional, Tuple
from trezorlib._internal.emulator import CoreEmulator, Emulator, LegacyEmulator
@ -67,7 +67,15 @@ ALL_TAGS = get_tags()
class EmulatorWrapper:
def __init__(self, gen: str, tag: str = None, storage: bytes = None) -> None:
def __init__(
self,
gen: str,
tag: Optional[str] = None,
storage: Optional[bytes] = None,
port: Optional[int] = None,
headless: bool = True,
auto_interact: bool = True,
) -> None:
if tag is not None:
executable = filename_from_tag(gen, tag)
else:
@ -87,7 +95,8 @@ class EmulatorWrapper:
executable,
self.profile_dir.name,
storage=storage,
headless=True,
headless=headless,
auto_interact=auto_interact,
)
elif gen == "core":
self.emulator = CoreEmulator(
@ -95,7 +104,9 @@ class EmulatorWrapper:
self.profile_dir.name,
storage=storage,
workdir=workdir,
headless=True,
port=port,
headless=headless,
auto_interact=auto_interact,
)
else:
raise ValueError(

Write Preview
Loading…
Cancel
Save