From a69e43d1a6d7d66b39fc39046b39b02c4657a5a9 Mon Sep 17 00:00:00 2001 From: Martin Novak Date: Wed, 25 May 2022 14:54:03 +0200 Subject: [PATCH] chore(tests): tests recording also generates "results web page" --- tests/conftest.py | 32 +++++++++++----- tests/ui_tests/__init__.py | 52 +++++++++++++++++++------- tests/ui_tests/reporting/html.py | 21 ++++++++--- tests/ui_tests/reporting/testreport.py | 20 ++++++++-- 4 files changed, 93 insertions(+), 32 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index c60713517..60ee7b2f6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -165,7 +165,7 @@ def client( def pytest_sessionstart(session: pytest.Session) -> None: ui_tests.read_fixtures() - if session.config.getoption("ui") == "test": + if session.config.getoption("ui"): testreport.clear_dir() @@ -181,13 +181,19 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) - return missing = session.config.getoption("ui_check_missing") - if session.config.getoption("ui") == "test": + test_ui = session.config.getoption("ui") + + if test_ui == "test": if missing and ui_tests.list_missing(): session.exitstatus = pytest.ExitCode.TESTS_FAILED ui_tests.write_fixtures_suggestion(missing) testreport.index() - if session.config.getoption("ui") == "record": - ui_tests.write_fixtures(missing) + if test_ui == "record": + if exitstatus == pytest.ExitCode.OK: + ui_tests.write_fixtures(missing) + else: + ui_tests.write_fixtures_suggestion(missing, only_passed_tests=True) + testreport.index() def pytest_terminal_summary( @@ -216,6 +222,13 @@ def pytest_terminal_summary( print("See", ui_tests.SUGGESTION_FILE) println("") + if ui_option == "record" and exitstatus != pytest.ExitCode.OK: + println( + f"\n-------- WARNING! Recording to {ui_tests.HASH_FILE.name} was disabled due to failed tests. --------" + ) + print("See", ui_tests.SUGGESTION_FILE, "for suggestions for ONLY PASSED tests.") + println("") + if _should_write_ui_report(exitstatus): println("-------- UI tests summary: --------") println("Run ./tests/show_results.py to open test summary") @@ -278,7 +291,7 @@ def pytest_runtest_teardown(item: pytest.Item) -> None: Dumps the current UI test report HTML. """ - if item.session.config.getoption("ui") == "test": + if item.session.config.getoption("ui"): testreport.index() @@ -298,12 +311,13 @@ def device_handler(client: Client, request: pytest.FixtureRequest) -> None: device_handler = BackgroundDeviceHandler(client) yield device_handler - # if test did not finish, e.g. interrupted by Ctrl+C, the pytest_runtest_makereport - # did not create the attribute we need - if not hasattr(request.node, "rep_call"): + # get call test result + test_res = ui_tests.get_last_call_test_result(request) + + if test_res is None: return # if test finished, make sure all background tasks are done finalized_ok = device_handler.check_finalize() - if request.node.rep_call.passed and not finalized_ok: # type: ignore [rep_call must exist] + if test_res and not finalized_ok: # type: ignore [rep_call must exist] raise RuntimeError("Test did not check result of background task") diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py index 5b12a392d..c4ad71f49 100644 --- a/tests/ui_tests/__init__.py +++ b/tests/ui_tests/__init__.py @@ -5,7 +5,7 @@ import re import shutil from contextlib import contextmanager from pathlib import Path -from typing import Dict, Generator, Set +from typing import Dict, Generator, Optional, Set import pytest from _pytest.outcomes import Failed @@ -22,6 +22,7 @@ SUGGESTION_FILE = UI_TESTS_DIR / "fixtures.suggestion.json" FILE_HASHES: Dict[str, str] = {} ACTUAL_HASHES: Dict[str, str] = {} PROCESSED: Set[str] = set() +FAILED_TESTS: Set[str] = set() # T1/TT, to be set in screen_recording(), as we do not know it beforehand # TODO: it is not the cleanest, we could create a class out of this file @@ -44,9 +45,11 @@ def get_test_name(node_id: str) -> str: def _process_recorded(screen_path: Path, test_name: str) -> None: # calculate hash - FILE_HASHES[test_name] = _hash_files(screen_path) + actual_hash = _hash_files(screen_path) + FILE_HASHES[test_name] = actual_hash + ACTUAL_HASHES[test_name] = actual_hash _rename_records(screen_path) - PROCESSED.add(test_name) + testreport.recorded(screen_path, test_name, actual_hash) def _rename_records(screen_path: Path) -> None: @@ -74,8 +77,6 @@ def _get_bytes_from_png(png_file: str) -> bytes: def _process_tested(fixture_test_path: Path, test_name: str) -> None: - PROCESSED.add(test_name) - actual_path = fixture_test_path / "actual" actual_hash = _hash_files(actual_path) ACTUAL_HASHES[test_name] = actual_hash @@ -102,6 +103,15 @@ def _process_tested(fixture_test_path: Path, test_name: str) -> None: testreport.passed(fixture_test_path, test_name, actual_hash) +def get_last_call_test_result(request: pytest.FixtureRequest) -> Optional[bool]: + # if test did not finish, e.g. interrupted by Ctrl+C, the pytest_runtest_makereport + # did not create the attribute we need + if not hasattr(request.node, "rep_call"): + return None + + return request.node.rep_call.passed + + @contextmanager def screen_recording( client: Client, request: pytest.FixtureRequest @@ -141,10 +151,15 @@ def screen_recording( client.init_device() client.debug.stop_recording() - if test_ui == "record": - _process_recorded(screen_path, test_name) - else: - _process_tested(screens_test_path, test_name) + if test_ui: + PROCESSED.add(test_name) + if get_last_call_test_result(request) is False: + FAILED_TESTS.add(test_name) + + if test_ui == "record": + _process_recorded(screen_path, test_name) + else: + _process_tested(screens_test_path, test_name) def list_missing() -> Set[str]: @@ -166,17 +181,28 @@ def write_fixtures(remove_missing: bool) -> None: HASH_FILE.write_text(_get_fixtures_content(FILE_HASHES, remove_missing)) -def write_fixtures_suggestion(remove_missing: bool) -> None: - SUGGESTION_FILE.write_text(_get_fixtures_content(ACTUAL_HASHES, remove_missing)) +def write_fixtures_suggestion( + remove_missing: bool, only_passed_tests: bool = False +) -> None: + SUGGESTION_FILE.write_text( + _get_fixtures_content(ACTUAL_HASHES, remove_missing, only_passed_tests) + ) -def _get_fixtures_content(fixtures: Dict[str, str], remove_missing: bool) -> str: +def _get_fixtures_content( + fixtures: Dict[str, str], remove_missing: bool, only_passed_tests: bool = False +) -> str: if remove_missing: # Not removing the ones for different model nonrelevant_cases = { f: h for f, h in FILE_HASHES.items() if not f.startswith(f"{MODEL}_") } - processed_fixtures = {i: fixtures[i] for i in PROCESSED} + + filtered_processed_tests = PROCESSED + if only_passed_tests: + filtered_processed_tests = PROCESSED - FAILED_TESTS + + processed_fixtures = {i: fixtures[i] for i in filtered_processed_tests} fixtures = {**nonrelevant_cases, **processed_fixtures} else: fixtures = fixtures diff --git a/tests/ui_tests/reporting/html.py b/tests/ui_tests/reporting/html.py index 611f362dd..ee814c1b3 100644 --- a/tests/ui_tests/reporting/html.py +++ b/tests/ui_tests/reporting/html.py @@ -2,7 +2,7 @@ import base64 import filecmp from itertools import zip_longest from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional from dominate.tags import a, i, img, table, td, th, tr @@ -30,7 +30,7 @@ def write(fixture_test_path: Path, doc, filename: str) -> Path: return fixture_test_path / filename -def image(src: Path) -> None: +def image(src: Path, image_width: Optional[int] = None) -> None: with td(): if src: # open image file @@ -40,17 +40,26 @@ def image(src: Path) -> None: # convert output to str image = image.decode() # img(src=src.relative_to(fixture_test_path)) - img(src="data:image/png;base64, " + image) + img( + src="data:image/png;base64, " + image, + style=f"width: {image_width}px; image-rendering: pixelated;" + if image_width + else "", + ) else: i("missing") -def diff_table(left_screens: List[Path], right_screens: List[Path]) -> None: +def diff_table( + left_screens: List[Path], + right_screens: List[Path], + image_width: Optional[int] = None, +) -> None: for left, right in zip_longest(left_screens, right_screens): if left and right and filecmp.cmp(right, left): background = "white" else: background = "red" with tr(bgcolor=background): - image(left) - image(right) + image(left, image_width) + image(right, image_width) diff --git a/tests/ui_tests/reporting/testreport.py b/tests/ui_tests/reporting/testreport.py index 03e054a14..16f7a4c32 100644 --- a/tests/ui_tests/reporting/testreport.py +++ b/tests/ui_tests/reporting/testreport.py @@ -16,6 +16,11 @@ REPORTS_PATH = HERE / "reports" / "test" STYLE = (HERE / "testreport.css").read_text() SCRIPT = (HERE / "testreport.js").read_text() +SCREENSHOTS_WIDTH_PX_TO_DISPLAY = { + "T1": 128 * 2, # original is 128px + "TT": 240, # original is 240px + "TR": 128 * 2, # original is 128px +} ACTUAL_HASHES: Dict[str, str] = {} @@ -147,7 +152,11 @@ def failed( th("Expected") th("Actual") - html.diff_table(recorded_screens, actual_screens) + html.diff_table( + recorded_screens, + actual_screens, + SCREENSHOTS_WIDTH_PX_TO_DISPLAY[test_name[:2]], + ) return html.write(REPORTS_PATH / "failed", doc, test_name + ".html") @@ -155,9 +164,12 @@ def failed( def passed(fixture_test_path: Path, test_name: str, actual_hash: str) -> Path: copy_tree(str(fixture_test_path / "actual"), str(fixture_test_path / "recorded")) + return recorded(fixture_test_path / "actual", test_name, actual_hash) + + +def recorded(fixture_test_path: Path, test_name: str, actual_hash: str) -> Path: doc = document(title=test_name) - actual_path = fixture_test_path / "actual" - actual_screens = sorted(actual_path.iterdir()) + actual_screens = sorted(fixture_test_path.iterdir()) with doc: _header(test_name, actual_hash, actual_hash) @@ -168,6 +180,6 @@ def passed(fixture_test_path: Path, test_name: str, actual_hash: str) -> Path: for screen in actual_screens: with tr(): - html.image(screen) + html.image(screen, SCREENSHOTS_WIDTH_PX_TO_DISPLAY[test_name[:2]]) return html.write(REPORTS_PATH / "passed", doc, test_name + ".html")