diff --git a/core/Makefile b/core/Makefile index 78ed915e3d..629dcc2d60 100644 --- a/core/Makefile +++ b/core/Makefile @@ -114,8 +114,12 @@ test_emu_ui_multicore: ## run ui integration tests using multiple cores test_emu_ui_record: ## record and hash screens for ui integration tests $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests --ui=record --ui-check-missing $(TESTOPTS) -test_emu_ui_record_multicore: ## record and hash screens for ui integration tests using multiple cores - $(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) --ui=record --ui-check-missing --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM) +test_emu_ui_record_multicore: ## quickly record all screens + make test_emu_ui_multicore || echo "All errors are recorded in fixtures.json" + make test_emu_accept_fixtures + +test_emu_accept_fixtures: # accept UI fixtures from the last run of UI tests + ../tests/update_fixtures.py pylint: ## run pylint on application sources and tests pylint -E $(shell find src tests -name *.py) diff --git a/tests/conftest.py b/tests/conftest.py index 2f4648e5ee..001e328837 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -249,6 +249,8 @@ def pytest_sessionstart(session: pytest.Session) -> None: ui_tests.read_fixtures() if session.config.getoption("ui") and _is_main_runner(session): testreport.clear_dir() + # Preparing a new empty file for UI diff + ui_tests.FIXTURES_DIFF.write_bytes(b"") def _should_write_ui_report(exitstatus: pytest.ExitCode) -> bool: @@ -316,6 +318,10 @@ def pytest_terminal_summary( println("Run ./tests/show_results.py to open test summary") println("") + println("-------- Accepting all recent UI changes: --------") + println("Run ./tests/update_fixtures.py to apply all changes") + println("") + def pytest_addoption(parser: "Parser") -> None: parser.addoption( diff --git a/tests/ui_tests/.gitignore b/tests/ui_tests/.gitignore index 047e341006..3f2756cebe 100644 --- a/tests/ui_tests/.gitignore +++ b/tests/ui_tests/.gitignore @@ -2,3 +2,4 @@ *.html *.zip fixtures.suggestion.json +fixtures.json.diff diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py index d0633d9648..eed6927291 100644 --- a/tests/ui_tests/__init__.py +++ b/tests/ui_tests/__init__.py @@ -18,6 +18,7 @@ UI_TESTS_DIR = Path(__file__).resolve().parent SCREENS_DIR = UI_TESTS_DIR / "screens" HASH_FILE = UI_TESTS_DIR / "fixtures.json" SUGGESTION_FILE = UI_TESTS_DIR / "fixtures.suggestion.json" +FIXTURES_DIFF = UI_TESTS_DIR / "fixtures.json.diff" FILE_HASHES: Dict[str, str] = {} ACTUAL_HASHES: Dict[str, str] = {} PROCESSED: Set[str] = set() @@ -92,6 +93,16 @@ def _process_tested(fixture_test_path: Path, test_name: str) -> None: fixture_test_path, test_name, actual_hash, expected_hash ) + # Writing the diff to a file, so that we can process it later + # Appending a new JSON object, not having to regenerate the + # whole file (which could cause issues with multiple processes/threads) + with open(FIXTURES_DIFF, "a") as f: + diff = { + "test_name": test_name, + "actual_hash": actual_hash, + } + f.write(json.dumps(diff) + "\n") + pytest.fail( f"Hash of {test_name} differs.\n" f"Expected: {expected_hash}\n" @@ -186,6 +197,31 @@ def write_fixtures_suggestion( ) +def update_fixtures_with_diff() -> int: + """Update the fixtures.json file with the actual hashes from the diff file. + + Use-case is that the UI test run will generate the differing hashes, + and with this function we can simply update the fixtures.json file + without having to call the UI tests again in recording mode. + """ + if not FIXTURES_DIFF.exists(): + raise ValueError(f"File {FIXTURES_DIFF} not found.") + + read_fixtures() + + changes_amount = 0 + with open(FIXTURES_DIFF) as f: + for line in f: + changes_amount += 1 + diff = json.loads(line) + FILE_HASHES[diff["test_name"]] = diff["actual_hash"] + + write_fixtures(remove_missing=False) + + # Returning the amount of updated hashes + return changes_amount + + def _get_fixtures_content( fixtures: Dict[str, str], remove_missing: bool, only_passed_tests: bool = False ) -> str: diff --git a/tests/update_fixtures.py b/tests/update_fixtures.py new file mode 100755 index 0000000000..3f4848e1fd --- /dev/null +++ b/tests/update_fixtures.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from ui_tests import update_fixtures_with_diff + +changes_amount = update_fixtures_with_diff() + +print(f"{changes_amount} hashes updated in fixtures.json file.")