diff --git a/core/Makefile b/core/Makefile index fe11623d40..6e21e071b5 100644 --- a/core/Makefile +++ b/core/Makefile @@ -80,12 +80,9 @@ test_emu_click: ## run click tests cd tests ; ./run_tests_click_emu.sh $(TESTOPTS) test_emu_ui: ## run ui integration tests - cd tests ; ./run_tests_device_emu.sh --test_screen=test-hash -m "not skip_ui" $(TESTOPTS) + cd tests ; ./run_tests_device_emu.sh --test_screen=test -m "not skip_ui" $(TESTOPTS) -test_emu_ui_hash: ## create hashes of ui integration test fixtures - cd tests ; ./run_tests_device_emu.sh --test_screen=hash -m "not skip_ui" $(TESTOPTS) - -test_emu_ui_record: ## record a full set of new ui fixtures +test_emu_ui_record: ## record and hash screens for ui integration tests cd tests ; ./run_tests_device_emu.sh --test_screen=record -m "not skip_ui" $(TESTOPTS) pylint: ## run pylint on application sources and tests diff --git a/tests/conftest.py b/tests/conftest.py index 794f3aa53d..dfbf383dd1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,11 +14,10 @@ # You should have received a copy of the License along with this library. # If not, see . -import filecmp import hashlib -import itertools import os import re +import shutil from contextlib import contextmanager from pathlib import Path @@ -65,36 +64,29 @@ def _get_test_dirname(node): return "{}_{}".format(node_module_name, node_name) -def _check_screen_fixtures_dir(fixture_dir): - if fixture_dir.exists(): - # remove old fixtures - for fixture in fixture_dir.iterdir(): - fixture.unlink() - else: - # create the fixture dir, if not present +def _check_fixture_directory(fixture_dir, screen_path): + # create the fixture dir if it does not exist + if not fixture_dir.exists(): fixture_dir.mkdir() - -def _record_screen_fixtures(fixture_dir, test_dir): - _check_screen_fixtures_dir(fixture_dir) - - # move recorded screenshots into fixture directory - records = sorted(test_dir.iterdir()) - for index, record in enumerate(sorted(records)): - fixture = fixture_dir / "{:08}.png".format(index) - record.replace(fixture) + # delete old files + shutil.rmtree(screen_path, ignore_errors=True) + screen_path.mkdir() -def _hash_screen_fixtures(fixture_dir, test_dir): - _check_screen_fixtures_dir(fixture_dir) +def _process_recorded(screen_path): + records = sorted(screen_path.iterdir()) - # hash recorded screenshots - records = sorted(test_dir.iterdir()) + # create hash digest = _hash_files(records) - - with open(fixture_dir / "hash.txt", "w") as f: + with open(screen_path / "../hash.txt", "w") as f: f.write(digest) + # rename screenshots + for index, record in enumerate(sorted(records)): + filename = screen_path / "{:08}.png".format(index) + record.replace(filename) + def _hash_files(files): hasher = hashlib.sha256() @@ -106,25 +98,8 @@ def _hash_files(files): return hasher.digest().hex() -def _assert_screen_recording(fixture_dir, test_dir): - fixtures = sorted(fixture_dir.iterdir()) - records = sorted(test_dir.iterdir()) - - if not fixtures: - return - - for fixture, image in itertools.zip_longest(fixtures, records): - if fixture is None: - pytest.fail("Missing fixture for image {}".format(image)) - if image is None: - pytest.fail("Missing image for fixture {}".format(fixture)) - if not filecmp.cmp(fixture, image): - pytest.fail("Image {} and fixture {} differ".format(image, fixture)) - - -def _assert_screen_hashes(fixture_dir, test_dir): - records = sorted(test_dir.iterdir()) - hash_file = fixture_dir / "hash.txt" +def _process_tested(screen_path, test_name): + hash_file = screen_path / "../hash.txt" if not hash_file.exists(): raise ValueError("File hash.txt not found.") @@ -132,46 +107,56 @@ def _assert_screen_hashes(fixture_dir, test_dir): with open(hash_file, "r") as f: expected_hash = f.read() + records = sorted(screen_path.iterdir()) actual_hash = _hash_files(records) if actual_hash != expected_hash: pytest.fail( "Hash of {} differs.\nExpected: {}\nActual: {}".format( - fixture_dir.name, expected_hash, actual_hash + test_name, expected_hash, actual_hash ) ) @contextmanager -def _screen_recording(client, request, tmp_path): +def _screen_recording(client, request): if not request.node.get_closest_marker("skip_ui"): test_screen = request.config.getoption("test_screen") else: test_screen = "" + + if not test_screen: + yield + return + fixture_root = Path(__file__) / "../ui_tests" + test_name = _get_test_dirname(request.node) + fixture_test_path = fixture_root.resolve() / test_name + + if test_screen == "record": + screen_path = fixture_test_path / "recorded" + elif test_screen == "test": + screen_path = fixture_test_path / "actual" + else: + raise ValueError("Invalid test_screen option.") + + _check_fixture_directory(fixture_test_path, screen_path) try: - if test_screen: - client.debug.start_recording(str(tmp_path)) + client.debug.start_recording(str(screen_path)) yield finally: - if test_screen: - client.debug.stop_recording() - fixture_path = fixture_root.resolve() / _get_test_dirname(request.node) - if test_screen == "record": - _record_screen_fixtures(fixture_path, tmp_path) - elif test_screen == "hash": - _hash_screen_fixtures(fixture_path, tmp_path) - elif test_screen == "test-hash": - _assert_screen_hashes(fixture_path, tmp_path) - elif test_screen == "test-record": - _assert_screen_recording(fixture_path, tmp_path) - else: - raise ValueError("Invalid test_screen option.") + client.debug.stop_recording() + if test_screen == "record": + _process_recorded(screen_path) + elif test_screen == "test": + _process_tested(screen_path, test_name) + else: + raise ValueError("Invalid test_screen option.") @pytest.fixture(scope="function") -def client(request, tmp_path): +def client(request): """Client fixture. Every test function that requires a client instance will get it from here. @@ -255,7 +240,7 @@ def client(request, tmp_path): if setup_params["random_seed"] is not None: client.debug.reseed(setup_params["random_seed"]) - with _screen_recording(client, request, tmp_path): + with _screen_recording(client, request): yield client client.close() @@ -266,7 +251,7 @@ def pytest_addoption(parser): "--test_screen", action="store", default="", - help="Enable UI intergration tests: 'record', 'hash' or 'test-hash' and 'test-record'", + help="Enable UI intergration tests: 'record' or 'test'", ) diff --git a/tests/ui_tests/.gitignore b/tests/ui_tests/.gitignore new file mode 100644 index 0000000000..e33609d251 --- /dev/null +++ b/tests/ui_tests/.gitignore @@ -0,0 +1 @@ +*.png