mirror of
https://github.com/trezor/trezor-firmware.git
synced 2024-12-23 06:48:16 +00:00
322 lines
10 KiB
Python
322 lines
10 KiB
Python
import hashlib
|
|
import shutil
|
|
from collections import defaultdict
|
|
from datetime import datetime
|
|
from distutils.dir_util import copy_tree
|
|
from pathlib import Path
|
|
from typing import Dict, List, Set
|
|
|
|
import dominate
|
|
import dominate.tags as t
|
|
from dominate.tags import a, div, h1, h2, hr, p, span, strong, table, th, tr
|
|
from dominate.util import text
|
|
|
|
from . import download, html
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
REPORTS_PATH = HERE / "reports" / "test"
|
|
RECORDED_SCREENS_PATH = Path(__file__).resolve().parent.parent / "screens"
|
|
|
|
STYLE = (HERE / "testreport.css").read_text()
|
|
SCRIPT = (HERE / "testreport.js").read_text()
|
|
SCREENSHOTS_WIDTH_PX_TO_DISPLAY = {
|
|
"T1": 128 * 2, # original is 128px
|
|
"TT": 240, # original is 240px
|
|
"TR": 128 * 2, # original is 128px
|
|
}
|
|
|
|
# These two html files are referencing each other
|
|
ALL_SCREENS = "all_screens.html"
|
|
ALL_UNIQUE_SCREENS = "all_unique_screens.html"
|
|
|
|
ACTUAL_HASHES: Dict[str, str] = {}
|
|
|
|
|
|
def _image_width(test_name: str) -> int:
|
|
"""Return the width of the image to display for the given test name.
|
|
|
|
Is model-specific. Model is at the beginning of each test-case.
|
|
"""
|
|
return SCREENSHOTS_WIDTH_PX_TO_DISPLAY[test_name[:2]]
|
|
|
|
|
|
def document(
|
|
title: str, actual_hash: str = None, index: bool = False
|
|
) -> dominate.document:
|
|
doc = dominate.document(title=title)
|
|
style = t.style()
|
|
style.add_raw_string(STYLE)
|
|
script = t.script()
|
|
script.add_raw_string(SCRIPT)
|
|
doc.head.add(style, script)
|
|
|
|
if actual_hash is not None:
|
|
doc.body["data-actual-hash"] = actual_hash
|
|
|
|
if index:
|
|
doc.body["data-index"] = True
|
|
|
|
return doc
|
|
|
|
|
|
def _header(test_name: str, expected_hash: str, actual_hash: str) -> None:
|
|
h1(test_name)
|
|
with div():
|
|
if actual_hash == expected_hash:
|
|
p(
|
|
"This test succeeded on UI comparison.",
|
|
style="color: green; font-weight: bold;",
|
|
)
|
|
else:
|
|
p(
|
|
"This test failed on UI comparison.",
|
|
style="color: red; font-weight: bold;",
|
|
)
|
|
p("Expected: ", expected_hash)
|
|
p("Actual: ", actual_hash)
|
|
hr()
|
|
|
|
|
|
def clear_dir() -> None:
|
|
"""Delete and create the reports dir to clear previous entries."""
|
|
shutil.rmtree(REPORTS_PATH, ignore_errors=True)
|
|
REPORTS_PATH.mkdir()
|
|
(REPORTS_PATH / "failed").mkdir()
|
|
(REPORTS_PATH / "passed").mkdir()
|
|
|
|
|
|
def index() -> Path:
|
|
"""Generate index.html with all the test results - lists of failed and passed tests."""
|
|
passed_tests = list((REPORTS_PATH / "passed").iterdir())
|
|
failed_tests = list((REPORTS_PATH / "failed").iterdir())
|
|
|
|
title = "UI Test report " + datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
doc = document(title=title, index=True)
|
|
|
|
with doc:
|
|
h1("UI Test report")
|
|
if not failed_tests:
|
|
p("All tests succeeded!", style="color: green; font-weight: bold;")
|
|
else:
|
|
p("Some tests failed!", style="color: red; font-weight: bold;")
|
|
hr()
|
|
|
|
h2("Failed", style="color: red;")
|
|
with p(id="file-hint"):
|
|
strong("Tip:")
|
|
text(" use ")
|
|
t.span("./tests/show_results.sh", style="font-family: monospace")
|
|
text(" to enable smart features.")
|
|
|
|
with div("Test colors", _class="script-hidden"):
|
|
with t.ul():
|
|
with t.li():
|
|
t.span("new", style="color: blue")
|
|
t.button("clear all", onclick="resetState('all')")
|
|
with t.li():
|
|
t.span("marked OK", style="color: grey")
|
|
t.button("clear", onclick="resetState('ok')")
|
|
with t.li():
|
|
t.span("marked BAD", style="color: darkred")
|
|
t.button("clear", onclick="resetState('bad')")
|
|
|
|
html.report_links(failed_tests, REPORTS_PATH, ACTUAL_HASHES)
|
|
|
|
h2("Passed", style="color: green;")
|
|
html.report_links(passed_tests, REPORTS_PATH)
|
|
|
|
return html.write(REPORTS_PATH, doc, "index.html")
|
|
|
|
|
|
def all_screens(test_case_dirs: List[Path]) -> Path:
|
|
"""Generate an HTML file for all the screens from the current test run.
|
|
|
|
Shows all test-cases at one place.
|
|
"""
|
|
title = "All test cases"
|
|
doc = dominate.document(title=title)
|
|
|
|
with doc:
|
|
h1("All test cases")
|
|
hr()
|
|
|
|
count = 0
|
|
for test_case_dir in test_case_dirs:
|
|
test_case_name = test_case_dir.name
|
|
h2(test_case_name, id=test_case_name)
|
|
actual_dir = test_case_dir / "actual"
|
|
for png in sorted(actual_dir.rglob("*.png")):
|
|
# Including link to each image to see where else it occurs.
|
|
png_hash = _img_hash(png)
|
|
with a(href=f"{ALL_UNIQUE_SCREENS}#{png_hash}"):
|
|
html.image_raw(png, _image_width(test_case_name))
|
|
count += 1
|
|
|
|
h2(f"{count} screens from {len(test_case_dirs)} testcases.")
|
|
|
|
return html.write(REPORTS_PATH, doc, ALL_SCREENS)
|
|
|
|
|
|
def all_unique_screens(test_case_dirs: List[Path]) -> Path:
|
|
"""Generate an HTML file with all the unique screens from the current test run."""
|
|
title = "All unique screens"
|
|
doc = dominate.document(title=title)
|
|
|
|
with doc:
|
|
h1("All unique screens")
|
|
hr()
|
|
|
|
screen_hashes: Dict[str, List[Path]] = defaultdict(list)
|
|
hash_images: Dict[str, Path] = {}
|
|
|
|
# Adding all unique images onto the page
|
|
for test_case_dir in test_case_dirs:
|
|
actual_dir = test_case_dir / "actual"
|
|
for png in sorted(actual_dir.rglob("*.png")):
|
|
png_hash = _img_hash(png)
|
|
if png_hash not in screen_hashes:
|
|
# Adding link to the appropriate hash, where other testcases
|
|
# with the same hash (screen) are listed.
|
|
with a(href=f"#{png_hash}"):
|
|
with span(id=png_hash[:8]):
|
|
html.image_raw(png, _image_width(test_case_dir.name))
|
|
|
|
screen_hashes[png_hash].append(test_case_dir)
|
|
hash_images[png_hash] = png
|
|
|
|
# Adding all screen hashes together with links to testcases having these screens.
|
|
for png_hash, test_cases in screen_hashes.items():
|
|
h2(png_hash)
|
|
with div(id=png_hash):
|
|
# Showing the exact image as well (not magnifying it)
|
|
with a(href=f"#{png_hash[:8]}"):
|
|
html.image_raw(hash_images[png_hash])
|
|
for case in test_cases:
|
|
# Adding link to each test-case
|
|
with a(href=f"{ALL_SCREENS}#{case.name}"):
|
|
p(case.name.split("/")[-1])
|
|
|
|
h2(f"{len(screen_hashes)} unique screens from {len(test_case_dirs)} testcases.")
|
|
|
|
return html.write(REPORTS_PATH, doc, ALL_UNIQUE_SCREENS)
|
|
|
|
|
|
def generate_reports() -> None:
|
|
"""Generate HTML reports for the test."""
|
|
index()
|
|
|
|
# To only get screens from the last running test-cases,
|
|
# we need to get the list of all directories with screenshots.
|
|
current_testcases = _get_testcases_dirs()
|
|
all_screens(current_testcases)
|
|
all_unique_screens(current_testcases)
|
|
|
|
|
|
def _img_hash(img: Path) -> str:
|
|
"""Return the hash of the image."""
|
|
content = img.read_bytes()
|
|
return hashlib.md5(content).hexdigest()
|
|
|
|
|
|
def _get_testcases_dirs() -> List[Path]:
|
|
"""Get the list of test-cases dirs that the current test was running."""
|
|
current_testcases = _get_all_current_testcases()
|
|
all_test_cases_dirs = [
|
|
case
|
|
for case in (RECORDED_SCREENS_PATH).iterdir()
|
|
if case.name in current_testcases
|
|
]
|
|
return sorted(all_test_cases_dirs)
|
|
|
|
|
|
def _get_all_current_testcases() -> Set[str]:
|
|
"""Get names of all current test-cases.
|
|
|
|
Equals to the names of HTML files in the reports dir.
|
|
"""
|
|
passed_tests = list((REPORTS_PATH / "passed").glob("*.html"))
|
|
failed_tests = list((REPORTS_PATH / "failed").glob("*.html"))
|
|
return {test.stem for test in (passed_tests + failed_tests)}
|
|
|
|
|
|
def failed(
|
|
fixture_test_path: Path, test_name: str, actual_hash: str, expected_hash: str
|
|
) -> Path:
|
|
"""Generate an HTML file for a failed test-case.
|
|
|
|
Compares the actual screenshots to the expected ones.
|
|
"""
|
|
ACTUAL_HASHES[test_name] = actual_hash
|
|
|
|
doc = document(title=test_name, actual_hash=actual_hash)
|
|
recorded_path = fixture_test_path / "recorded"
|
|
actual_path = fixture_test_path / "actual"
|
|
|
|
download_failed = False
|
|
|
|
if not recorded_path.exists():
|
|
recorded_path.mkdir()
|
|
try:
|
|
download.fetch_recorded(expected_hash, recorded_path)
|
|
except Exception:
|
|
download_failed = True
|
|
|
|
recorded_screens = sorted(recorded_path.iterdir())
|
|
actual_screens = sorted(actual_path.iterdir())
|
|
|
|
with doc:
|
|
_header(test_name, expected_hash, actual_hash)
|
|
|
|
with div(id="markbox", _class="script-hidden"):
|
|
p("Click a button to mark the test result as:")
|
|
with div(id="buttons"):
|
|
t.button("OK", id="mark-ok", onclick="markState('ok')")
|
|
t.button("BAD", id="mark-bad", onclick="markState('bad')")
|
|
|
|
if download_failed:
|
|
with p():
|
|
strong("WARNING:")
|
|
text(" failed to download recorded fixtures. Is this a new test case?")
|
|
|
|
with table(border=1, width=600):
|
|
with tr():
|
|
th("Expected")
|
|
th("Actual")
|
|
|
|
html.diff_table(
|
|
recorded_screens,
|
|
actual_screens,
|
|
_image_width(test_name),
|
|
)
|
|
|
|
return html.write(REPORTS_PATH / "failed", doc, test_name + ".html")
|
|
|
|
|
|
def passed(fixture_test_path: Path, test_name: str, actual_hash: str) -> Path:
|
|
"""Generate an HTML file for a passed test-case."""
|
|
copy_tree(str(fixture_test_path / "actual"), str(fixture_test_path / "recorded"))
|
|
|
|
return recorded(fixture_test_path / "actual", test_name, actual_hash)
|
|
|
|
|
|
def recorded(fixture_test_path: Path, test_name: str, actual_hash: str) -> Path:
|
|
"""Generate an HTML file for a passed test-case.
|
|
|
|
Shows all the screens from it in exact order.
|
|
"""
|
|
doc = document(title=test_name)
|
|
actual_screens = sorted(fixture_test_path.iterdir())
|
|
|
|
with doc:
|
|
_header(test_name, actual_hash, actual_hash)
|
|
|
|
with table(border=1):
|
|
with tr():
|
|
th("Recorded")
|
|
|
|
for screen in actual_screens:
|
|
with tr():
|
|
html.image_column(screen, _image_width(test_name))
|
|
|
|
return html.write(REPORTS_PATH / "passed", doc, test_name + ".html")
|