tests: save screens directly to ui_tests; introduce recorded/actual dir

core/Makefile

@@ -80,12 +80,9 @@ test_emu_click: ## run click tests
 	cd tests ; ./run_tests_click_emu.sh $(TESTOPTS)
 
 test_emu_ui: ## run ui integration tests
-	cd tests ; ./run_tests_device_emu.sh --test_screen=test-hash -m "not skip_ui" $(TESTOPTS)
+	cd tests ; ./run_tests_device_emu.sh --test_screen=test -m "not skip_ui" $(TESTOPTS)
 
-test_emu_ui_hash: ## create hashes of ui integration test fixtures
-	cd tests ; ./run_tests_device_emu.sh --test_screen=hash -m "not skip_ui" $(TESTOPTS)
-
-test_emu_ui_record: ## record a full set of new ui fixtures
+test_emu_ui_record: ## record and hash screens for ui integration tests
 	cd tests ; ./run_tests_device_emu.sh --test_screen=record -m "not skip_ui" $(TESTOPTS)
 
 pylint: ## run pylint on application sources and tests

tests/conftest.py

@@ -14,11 +14,10 @@
 # You should have received a copy of the License along with this library.
 # If not, see <https://www.gnu.org/licenses/lgpl-3.0.html>.
 
-import filecmp
 import hashlib
-import itertools
 import os
 import re
+import shutil
 from contextlib import contextmanager
 from pathlib import Path
 
@@ -65,36 +64,29 @@ def _get_test_dirname(node):
     return "{}_{}".format(node_module_name, node_name)
 
 
-def _check_screen_fixtures_dir(fixture_dir):
-    if fixture_dir.exists():
-        # remove old fixtures
-        for fixture in fixture_dir.iterdir():
-            fixture.unlink()
-    else:
-        # create the fixture dir, if not present
+def _check_fixture_directory(fixture_dir, screen_path):
+    # create the fixture dir if it does not exist
+    if not fixture_dir.exists():
         fixture_dir.mkdir()
 
-
-def _record_screen_fixtures(fixture_dir, test_dir):
-    _check_screen_fixtures_dir(fixture_dir)
-
-    # move recorded screenshots into fixture directory
-    records = sorted(test_dir.iterdir())
-    for index, record in enumerate(sorted(records)):
-        fixture = fixture_dir / "{:08}.png".format(index)
-        record.replace(fixture)
+    # delete old files
+    shutil.rmtree(screen_path, ignore_errors=True)
+    screen_path.mkdir()
 
 
-def _hash_screen_fixtures(fixture_dir, test_dir):
-    _check_screen_fixtures_dir(fixture_dir)
+def _process_recorded(screen_path):
+    records = sorted(screen_path.iterdir())
 
-    # hash recorded screenshots
-    records = sorted(test_dir.iterdir())
+    # create hash
     digest = _hash_files(records)
-
-    with open(fixture_dir / "hash.txt", "w") as f:
+    with open(screen_path / "../hash.txt", "w") as f:
         f.write(digest)
 
+    # rename screenshots
+    for index, record in enumerate(sorted(records)):
+        filename = screen_path / "{:08}.png".format(index)
+        record.replace(filename)
+
 
 def _hash_files(files):
     hasher = hashlib.sha256()
@@ -106,25 +98,8 @@ def _hash_files(files):
     return hasher.digest().hex()
 
 
-def _assert_screen_recording(fixture_dir, test_dir):
-    fixtures = sorted(fixture_dir.iterdir())
-    records = sorted(test_dir.iterdir())
-
-    if not fixtures:
-        return
-
-    for fixture, image in itertools.zip_longest(fixtures, records):
-        if fixture is None:
-            pytest.fail("Missing fixture for image {}".format(image))
-        if image is None:
-            pytest.fail("Missing image for fixture {}".format(fixture))
-        if not filecmp.cmp(fixture, image):
-            pytest.fail("Image {} and fixture {} differ".format(image, fixture))
-
-
-def _assert_screen_hashes(fixture_dir, test_dir):
-    records = sorted(test_dir.iterdir())
-    hash_file = fixture_dir / "hash.txt"
+def _process_tested(screen_path, test_name):
+    hash_file = screen_path / "../hash.txt"
 
     if not hash_file.exists():
         raise ValueError("File hash.txt not found.")
@@ -132,46 +107,56 @@ def _assert_screen_hashes(fixture_dir, test_dir):
     with open(hash_file, "r") as f:
         expected_hash = f.read()
 
+    records = sorted(screen_path.iterdir())
     actual_hash = _hash_files(records)
 
     if actual_hash != expected_hash:
         pytest.fail(
             "Hash of {} differs.\nExpected: {}\nActual:   {}".format(
-                fixture_dir.name, expected_hash, actual_hash
+                test_name, expected_hash, actual_hash
             )
         )
 
 
 @contextmanager
-def _screen_recording(client, request, tmp_path):
+def _screen_recording(client, request):
     if not request.node.get_closest_marker("skip_ui"):
         test_screen = request.config.getoption("test_screen")
     else:
         test_screen = ""
+
+    if not test_screen:
+        yield
+        return
+
     fixture_root = Path(__file__) / "../ui_tests"
+    test_name = _get_test_dirname(request.node)
+    fixture_test_path = fixture_root.resolve() / test_name
+
+    if test_screen == "record":
+        screen_path = fixture_test_path / "recorded"
+    elif test_screen == "test":
+        screen_path = fixture_test_path / "actual"
+    else:
+        raise ValueError("Invalid test_screen option.")
+
+    _check_fixture_directory(fixture_test_path, screen_path)
 
     try:
-        if test_screen:
-            client.debug.start_recording(str(tmp_path))
+        client.debug.start_recording(str(screen_path))
         yield
     finally:
-        if test_screen:
-            client.debug.stop_recording()
-            fixture_path = fixture_root.resolve() / _get_test_dirname(request.node)
-            if test_screen == "record":
-                _record_screen_fixtures(fixture_path, tmp_path)
-            elif test_screen == "hash":
-                _hash_screen_fixtures(fixture_path, tmp_path)
-            elif test_screen == "test-hash":
-                _assert_screen_hashes(fixture_path, tmp_path)
-            elif test_screen == "test-record":
-                _assert_screen_recording(fixture_path, tmp_path)
-            else:
-                raise ValueError("Invalid test_screen option.")
+        client.debug.stop_recording()
+        if test_screen == "record":
+            _process_recorded(screen_path)
+        elif test_screen == "test":
+            _process_tested(screen_path, test_name)
+        else:
+            raise ValueError("Invalid test_screen option.")
 
 
 @pytest.fixture(scope="function")
-def client(request, tmp_path):
+def client(request):
     """Client fixture.
 
     Every test function that requires a client instance will get it from here.
@@ -255,7 +240,7 @@ def client(request, tmp_path):
     if setup_params["random_seed"] is not None:
         client.debug.reseed(setup_params["random_seed"])
 
-    with _screen_recording(client, request, tmp_path):
+    with _screen_recording(client, request):
         yield client
 
     client.close()
@@ -266,7 +251,7 @@ def pytest_addoption(parser):
         "--test_screen",
         action="store",
         default="",
-        help="Enable UI intergration tests: 'record', 'hash' or 'test-hash' and 'test-record'",
+        help="Enable UI intergration tests: 'record' or 'test'",
     )
 
 

tests/ui_tests/.gitignore

@@ -0,0 +1 @@
+*.png