From 56e0c91a719c9cfa4c6f678278a106ee00dd06a4 Mon Sep 17 00:00:00 2001 From: grdddj Date: Mon, 15 Aug 2022 18:30:45 +0200 Subject: [PATCH] feat(tests): add dashboards with indexed unique screens and all screens --- tests/conftest.py | 7 +- tests/ui_tests/__init__.py | 2 +- tests/ui_tests/reporting/html.py | 41 +++-- .../ui_tests/reporting/report_master_diff.py | 4 +- tests/ui_tests/reporting/testreport.py | 146 +++++++++++++++++- 5 files changed, 173 insertions(+), 27 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 48003aa86..2f4648e5e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -269,13 +269,13 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) - if missing and ui_tests.list_missing(): session.exitstatus = pytest.ExitCode.TESTS_FAILED ui_tests.write_fixtures_suggestion(missing) - testreport.index() + testreport.generate_reports() elif test_ui == "record": if exitstatus == pytest.ExitCode.OK: ui_tests.write_fixtures(missing) else: ui_tests.write_fixtures_suggestion(missing, only_passed_tests=True) - testreport.index() + testreport.generate_reports() def pytest_terminal_summary( @@ -390,6 +390,9 @@ def pytest_runtest_teardown(item: pytest.Item) -> None: Dumps the current UI test report HTML. """ + # Not calling `testreport.generate_reports()` not to generate + # the `all_screens` report, as would take a lot of time. + # That will be generated in `pytest_sessionfinish`. if item.session.config.getoption("ui"): testreport.index() diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py index 77f02b7cf..d0633d964 100644 --- a/tests/ui_tests/__init__.py +++ b/tests/ui_tests/__init__.py @@ -219,4 +219,4 @@ def main() -> None: except Failed: print("FAILED:", record.name) - testreport.index() + testreport.generate_reports() diff --git a/tests/ui_tests/reporting/html.py b/tests/ui_tests/reporting/html.py index ee814c1b3..7f21d9e3d 100644 --- a/tests/ui_tests/reporting/html.py +++ b/tests/ui_tests/reporting/html.py @@ -4,6 +4,7 @@ from itertools import zip_longest from pathlib import Path from typing import Dict, List, Optional +from dominate import document from dominate.tags import a, i, img, table, td, th, tr @@ -25,31 +26,37 @@ def report_links( td(a(test.name, href=path)) -def write(fixture_test_path: Path, doc, filename: str) -> Path: +def write(fixture_test_path: Path, doc: document, filename: str) -> Path: (fixture_test_path / filename).write_text(doc.render()) return fixture_test_path / filename -def image(src: Path, image_width: Optional[int] = None) -> None: +def image_column(src: Path, image_width: Optional[int] = None) -> None: + """Put image into table as one cell.""" with td(): if src: - # open image file - image = src.read_bytes() - # encode image as base64 - image = base64.b64encode(image) - # convert output to str - image = image.decode() - # img(src=src.relative_to(fixture_test_path)) - img( - src="data:image/png;base64, " + image, - style=f"width: {image_width}px; image-rendering: pixelated;" - if image_width - else "", - ) + image_raw(src, image_width) else: i("missing") +def image_raw(src: Path, image_width: Optional[int] = None) -> None: + """Display image on the screen""" + # open image file + image = src.read_bytes() + # encode image as base64 + image = base64.b64encode(image) + # convert output to str + image = image.decode() + # img(src=src.relative_to(fixture_test_path)) + img( + src="data:image/png;base64, " + image, + style=f"width: {image_width}px; image-rendering: pixelated;" + if image_width + else "", + ) + + def diff_table( left_screens: List[Path], right_screens: List[Path], @@ -61,5 +68,5 @@ def diff_table( else: background = "red" with tr(bgcolor=background): - image(left, image_width) - image(right, image_width) + image_column(left, image_width) + image_column(right, image_width) diff --git a/tests/ui_tests/reporting/report_master_diff.py b/tests/ui_tests/reporting/report_master_diff.py index a20ff7dd8..314a8d287 100644 --- a/tests/ui_tests/reporting/report_master_diff.py +++ b/tests/ui_tests/reporting/report_master_diff.py @@ -62,7 +62,7 @@ def removed(screens_path: Path, test_name: str) -> Path: for screen in screens: with tr(): - html.image(screen) + html.image_column(screen) return html.write(REPORTS_PATH / "removed", doc, test_name + ".html") @@ -85,7 +85,7 @@ def added(screens_path: Path, test_name: str) -> Path: for screen in screens: with tr(): - html.image(screen) + html.image_column(screen) return html.write(REPORTS_PATH / "added", doc, test_name + ".html") diff --git a/tests/ui_tests/reporting/testreport.py b/tests/ui_tests/reporting/testreport.py index 16f7a4c32..bf359d292 100644 --- a/tests/ui_tests/reporting/testreport.py +++ b/tests/ui_tests/reporting/testreport.py @@ -1,18 +1,21 @@ +import hashlib import shutil +from collections import defaultdict from datetime import datetime from distutils.dir_util import copy_tree from pathlib import Path -from typing import Dict +from typing import Dict, List, Set import dominate import dominate.tags as t -from dominate.tags import div, h1, h2, hr, p, strong, table, th, tr +from dominate.tags import a, div, h1, h2, hr, p, span, strong, table, th, tr from dominate.util import text from . import download, html HERE = Path(__file__).resolve().parent REPORTS_PATH = HERE / "reports" / "test" +RECORDED_SCREENS_PATH = Path(__file__).resolve().parent.parent / "screens" STYLE = (HERE / "testreport.css").read_text() SCRIPT = (HERE / "testreport.js").read_text() @@ -22,9 +25,21 @@ SCREENSHOTS_WIDTH_PX_TO_DISPLAY = { "TR": 128 * 2, # original is 128px } +# These two html files are referencing each other +ALL_SCREENS = "all_screens.html" +ALL_UNIQUE_SCREENS = "all_unique_screens.html" + ACTUAL_HASHES: Dict[str, str] = {} +def _image_width(test_name: str) -> int: + """Return the width of the image to display for the given test name. + + Is model-specific. Model is at the beginning of each test-case. + """ + return SCREENSHOTS_WIDTH_PX_TO_DISPLAY[test_name[:2]] + + def document( title: str, actual_hash: str = None, index: bool = False ) -> dominate.document: @@ -63,7 +78,7 @@ def _header(test_name: str, expected_hash: str, actual_hash: str) -> None: def clear_dir() -> None: - # delete and create the reports dir to clear previous entries + """Delete and create the reports dir to clear previous entries.""" shutil.rmtree(REPORTS_PATH, ignore_errors=True) REPORTS_PATH.mkdir() (REPORTS_PATH / "failed").mkdir() @@ -71,6 +86,7 @@ def clear_dir() -> None: def index() -> Path: + """Generate index.html with all the test results - lists of failed and passed tests.""" passed_tests = list((REPORTS_PATH / "passed").iterdir()) failed_tests = list((REPORTS_PATH / "failed").iterdir()) @@ -112,9 +128,124 @@ def index() -> Path: return html.write(REPORTS_PATH, doc, "index.html") +def all_screens(test_case_dirs: List[Path]) -> Path: + """Generate an HTML file for all the screens from the current test run. + + Shows all test-cases at one place. + """ + title = "All test cases" + doc = dominate.document(title=title) + + with doc: + h1("All test cases") + hr() + + count = 0 + for test_case_dir in test_case_dirs: + test_case_name = test_case_dir.name + h2(test_case_name, id=test_case_name) + actual_dir = test_case_dir / "actual" + for png in sorted(actual_dir.rglob("*.png")): + # Including link to each image to see where else it occurs. + png_hash = _img_hash(png) + with a(href=f"{ALL_UNIQUE_SCREENS}#{png_hash}"): + html.image_raw(png, _image_width(test_case_name)) + count += 1 + + h2(f"{count} screens from {len(test_case_dirs)} testcases.") + + return html.write(REPORTS_PATH, doc, ALL_SCREENS) + + +def all_unique_screens(test_case_dirs: List[Path]) -> Path: + """Generate an HTML file with all the unique screens from the current test run.""" + title = "All unique screens" + doc = dominate.document(title=title) + + with doc: + h1("All unique screens") + hr() + + screen_hashes: Dict[str, List[Path]] = defaultdict(list) + hash_images: Dict[str, Path] = {} + + # Adding all unique images onto the page + for test_case_dir in test_case_dirs: + actual_dir = test_case_dir / "actual" + for png in sorted(actual_dir.rglob("*.png")): + png_hash = _img_hash(png) + if png_hash not in screen_hashes: + # Adding link to the appropriate hash, where other testcases + # with the same hash (screen) are listed. + with a(href=f"#{png_hash}"): + with span(id=png_hash[:8]): + html.image_raw(png, _image_width(test_case_dir.name)) + + screen_hashes[png_hash].append(test_case_dir) + hash_images[png_hash] = png + + # Adding all screen hashes together with links to testcases having these screens. + for png_hash, test_cases in screen_hashes.items(): + h2(png_hash) + with div(id=png_hash): + # Showing the exact image as well (not magnifying it) + with a(href=f"#{png_hash[:8]}"): + html.image_raw(hash_images[png_hash]) + for case in test_cases: + # Adding link to each test-case + with a(href=f"{ALL_SCREENS}#{case.name}"): + p(case.name.split("/")[-1]) + + h2(f"{len(screen_hashes)} unique screens from {len(test_case_dirs)} testcases.") + + return html.write(REPORTS_PATH, doc, ALL_UNIQUE_SCREENS) + + +def generate_reports() -> None: + """Generate HTML reports for the test.""" + index() + + # To only get screens from the last running test-cases, + # we need to get the list of all directories with screenshots. + current_testcases = _get_testcases_dirs() + all_screens(current_testcases) + all_unique_screens(current_testcases) + + +def _img_hash(img: Path) -> str: + """Return the hash of the image.""" + content = img.read_bytes() + return hashlib.md5(content).hexdigest() + + +def _get_testcases_dirs() -> List[Path]: + """Get the list of test-cases dirs that the current test was running.""" + current_testcases = _get_all_current_testcases() + all_test_cases_dirs = [ + case + for case in (RECORDED_SCREENS_PATH).iterdir() + if case.name in current_testcases + ] + return sorted(all_test_cases_dirs) + + +def _get_all_current_testcases() -> Set[str]: + """Get names of all current test-cases. + + Equals to the names of HTML files in the reports dir. + """ + passed_tests = list((REPORTS_PATH / "passed").glob("*.html")) + failed_tests = list((REPORTS_PATH / "failed").glob("*.html")) + return {test.stem for test in (passed_tests + failed_tests)} + + def failed( fixture_test_path: Path, test_name: str, actual_hash: str, expected_hash: str ) -> Path: + """Generate an HTML file for a failed test-case. + + Compares the actual screenshots to the expected ones. + """ ACTUAL_HASHES[test_name] = actual_hash doc = document(title=test_name, actual_hash=actual_hash) @@ -155,19 +286,24 @@ def failed( html.diff_table( recorded_screens, actual_screens, - SCREENSHOTS_WIDTH_PX_TO_DISPLAY[test_name[:2]], + _image_width(test_name), ) return html.write(REPORTS_PATH / "failed", doc, test_name + ".html") def passed(fixture_test_path: Path, test_name: str, actual_hash: str) -> Path: + """Generate an HTML file for a passed test-case.""" copy_tree(str(fixture_test_path / "actual"), str(fixture_test_path / "recorded")) return recorded(fixture_test_path / "actual", test_name, actual_hash) def recorded(fixture_test_path: Path, test_name: str, actual_hash: str) -> Path: + """Generate an HTML file for a passed test-case. + + Shows all the screens from it in exact order. + """ doc = document(title=test_name) actual_screens = sorted(fixture_test_path.iterdir()) @@ -180,6 +316,6 @@ def recorded(fixture_test_path: Path, test_name: str, actual_hash: str) -> Path: for screen in actual_screens: with tr(): - html.image(screen, SCREENSHOTS_WIDTH_PX_TO_DISPLAY[test_name[:2]]) + html.image_column(screen, _image_width(test_name)) return html.write(REPORTS_PATH / "passed", doc, test_name + ".html")