parent
f62ce2f793
commit
e15662e12d
@ -0,0 +1,308 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import typing as t
|
||||||
|
import warnings
|
||||||
|
from copy import deepcopy
|
||||||
|
from dataclasses import asdict, dataclass, field
|
||||||
|
from difflib import SequenceMatcher
|
||||||
|
from functools import cached_property
|
||||||
|
from itertools import zip_longest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from PIL import Image
|
||||||
|
from typing_extensions import Self
|
||||||
|
|
||||||
|
from trezorlib.debuglink import TrezorClientDebugLink as Client
|
||||||
|
|
||||||
|
UI_TESTS_DIR = Path(__file__).resolve().parent
|
||||||
|
SCREENS_DIR = UI_TESTS_DIR / "screens"
|
||||||
|
IMAGES_DIR = SCREENS_DIR / "all_images"
|
||||||
|
FIXTURES_FILE = UI_TESTS_DIR / "fixtures.json"
|
||||||
|
|
||||||
|
# fixtures.json are structured as follows:
|
||||||
|
# {
|
||||||
|
# "model": {
|
||||||
|
# "group": {
|
||||||
|
# "test_name": "hash",
|
||||||
|
# ...
|
||||||
|
# }}}...
|
||||||
|
# IOW, FixturesType = dict[<model>, dict[<group>, dict[<test_name>, <hash>]]]
|
||||||
|
FixturesType = t.NewType("FixturesType", "dict[str, dict[str, dict[str, str]]]")
|
||||||
|
|
||||||
|
FIXTURES: FixturesType = FixturesType({})
|
||||||
|
|
||||||
|
|
||||||
|
def get_fixtures() -> FixturesType:
|
||||||
|
global FIXTURES
|
||||||
|
if not FIXTURES and FIXTURES_FILE.exists():
|
||||||
|
FIXTURES = FixturesType(json.loads(FIXTURES_FILE.read_text()))
|
||||||
|
|
||||||
|
return FIXTURES
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_fixtures(
|
||||||
|
results: t.Iterable[TestResult],
|
||||||
|
remove_missing: bool = False,
|
||||||
|
) -> tuple[FixturesType, set[TestCase]]:
|
||||||
|
"""Prepare contents of fixtures.json"""
|
||||||
|
# set up brand new contents
|
||||||
|
grouped_tests: dict[tuple[str, str], dict[str, str]] = {}
|
||||||
|
for result in results:
|
||||||
|
idx = result.test.model, result.test.group
|
||||||
|
group = grouped_tests.setdefault(idx, {})
|
||||||
|
group[result.test.fixtures_name] = result.actual_hash
|
||||||
|
|
||||||
|
missing_tests = set()
|
||||||
|
|
||||||
|
# merge with previous fixtures
|
||||||
|
fixtures = deepcopy(get_fixtures())
|
||||||
|
for (model, group), new_content in grouped_tests.items():
|
||||||
|
# for every model/group, update the data with the new content
|
||||||
|
current_content = fixtures.setdefault(model, {}).setdefault(group, {})
|
||||||
|
if remove_missing:
|
||||||
|
new_tests = set(new_content.keys())
|
||||||
|
old_tests = set(current_content.keys())
|
||||||
|
missing_tests |= {
|
||||||
|
TestCase(model, group, test) for test in old_tests - new_tests
|
||||||
|
}
|
||||||
|
current_content.clear()
|
||||||
|
|
||||||
|
current_content.update(new_content)
|
||||||
|
|
||||||
|
return fixtures, missing_tests
|
||||||
|
|
||||||
|
|
||||||
|
def write_fixtures(
|
||||||
|
results: t.Iterable[TestResult],
|
||||||
|
remove_missing: bool = False,
|
||||||
|
dest: Path = FIXTURES_FILE,
|
||||||
|
) -> None:
|
||||||
|
global FIXTURES
|
||||||
|
content, _ = prepare_fixtures(results, remove_missing)
|
||||||
|
FIXTURES = FixturesType(content)
|
||||||
|
dest.write_text(json.dumps(content, indent=0, sort_keys=True) + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
def _rename_records(screen_path: Path) -> None:
|
||||||
|
IMAGES_DIR.mkdir(exist_ok=True)
|
||||||
|
# rename screenshots
|
||||||
|
for index, record in enumerate(sorted(screen_path.iterdir())):
|
||||||
|
record.replace(screen_path / f"{index:08}.png")
|
||||||
|
|
||||||
|
|
||||||
|
def screens_and_hashes(screen_path: Path) -> tuple[list[Path], list[str]]:
|
||||||
|
if not screen_path.exists():
|
||||||
|
return [], []
|
||||||
|
|
||||||
|
hashes = []
|
||||||
|
paths = []
|
||||||
|
for file in sorted(screen_path.iterdir()):
|
||||||
|
paths.append(file)
|
||||||
|
if len(file.stem) == 32:
|
||||||
|
try:
|
||||||
|
hashes.append(bytes.fromhex(file.stem))
|
||||||
|
continue
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
hashes.append(_get_image_hash(file))
|
||||||
|
return paths, hashes
|
||||||
|
|
||||||
|
|
||||||
|
def _get_image_hash(png_file: Path) -> str:
|
||||||
|
return hashlib.sha256(_get_bytes_from_png(png_file)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _get_bytes_from_png(png_file: Path) -> bytes:
|
||||||
|
"""Decode a PNG file into bytes representing all the pixels.
|
||||||
|
|
||||||
|
Is necessary because Linux and Mac are using different PNG encoding libraries,
|
||||||
|
and we need the file hashes to be the same on both platforms.
|
||||||
|
"""
|
||||||
|
return Image.open(str(png_file)).tobytes()
|
||||||
|
|
||||||
|
|
||||||
|
def _hash_files(path: Path) -> str:
|
||||||
|
files = path.iterdir()
|
||||||
|
hasher = hashlib.sha256()
|
||||||
|
for file in sorted(files):
|
||||||
|
hasher.update(_get_bytes_from_png(file))
|
||||||
|
|
||||||
|
return hasher.digest().hex()
|
||||||
|
|
||||||
|
|
||||||
|
def get_last_call_test_result(request: pytest.FixtureRequest) -> bool | None:
|
||||||
|
# if test did not finish, e.g. interrupted by Ctrl+C, the pytest_runtest_makereport
|
||||||
|
# did not create the attribute we need
|
||||||
|
if not hasattr(request.node, "rep_call"):
|
||||||
|
return None
|
||||||
|
|
||||||
|
return request.node.rep_call.passed # type: ignore
|
||||||
|
|
||||||
|
|
||||||
|
def _get_test_name_and_group(node_id: str) -> tuple[str, str]:
|
||||||
|
test_path, func_id = node_id.split("::", maxsplit=1)
|
||||||
|
assert test_path.endswith(".py")
|
||||||
|
|
||||||
|
# tests / device_tests / bitcoin / test_signtx.py
|
||||||
|
_tests, group_name, *path_in_group = test_path.split("/")
|
||||||
|
|
||||||
|
# remove ::TestClass:: if present because it is usually the same as the test file name
|
||||||
|
func_id = re.sub(r"::.*?::", "-", func_id)
|
||||||
|
|
||||||
|
test_path_prefix = "-".join(path_in_group)
|
||||||
|
new_name = f"{test_path_prefix}::{func_id}"
|
||||||
|
new_name = new_name.replace("/", "-")
|
||||||
|
# Test item name is usually function name, but when parametrization is used,
|
||||||
|
# parameters are also part of the name. Some functions have very long parameter
|
||||||
|
# names (tx hashes etc) that run out of maximum allowable filename length, so
|
||||||
|
# we limit the name to first 100 chars. This is not a problem with txhashes.
|
||||||
|
if len(new_name) <= 100:
|
||||||
|
return new_name, group_name
|
||||||
|
|
||||||
|
differentiator = hashlib.sha256(new_name.encode()).hexdigest()
|
||||||
|
shortened_name = new_name[:91] + "-" + differentiator[:8]
|
||||||
|
return shortened_name, group_name
|
||||||
|
|
||||||
|
|
||||||
|
def screens_diff(
|
||||||
|
expected_hashes: list[str], actual_hashes: list[str]
|
||||||
|
) -> t.Iterator[tuple[str | None, str | None]]:
|
||||||
|
diff = SequenceMatcher(
|
||||||
|
None, expected_hashes, actual_hashes, autojunk=False
|
||||||
|
).get_opcodes()
|
||||||
|
for _tag, i1, i2, j1, j2 in diff:
|
||||||
|
# tag is one of "replace", "delete", "equal", "insert"
|
||||||
|
# i1, i2 and j1, j2 are slice indexes for expected/actual respectively
|
||||||
|
# The output of get_opcodes is an ordered sequence of instructions
|
||||||
|
# for converting expected to actual. By taking the subslices and zipping
|
||||||
|
# together, we get the equal subsequences aligned and Nones at deletion
|
||||||
|
# or insertion points.
|
||||||
|
expected_slice = expected_hashes[i1:i2]
|
||||||
|
actual_slice = actual_hashes[j1:j2]
|
||||||
|
yield from zip_longest(expected_slice, actual_slice, fillvalue=None)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class TestCase:
|
||||||
|
model: str
|
||||||
|
group: str
|
||||||
|
name: str
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def build(cls, client: Client, request: pytest.FixtureRequest) -> Self:
|
||||||
|
name, group = _get_test_name_and_group(request.node.nodeid)
|
||||||
|
return cls(
|
||||||
|
model=f"T{client.features.model}",
|
||||||
|
name=name,
|
||||||
|
group=group,
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def id(self) -> str:
|
||||||
|
return f"{self.model}-{self.group}-{self.name}"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def fixtures_name(self) -> str:
|
||||||
|
return f"{self.model}_{self.name}"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def dir(self) -> Path:
|
||||||
|
return SCREENS_DIR / self.id
|
||||||
|
|
||||||
|
@property
|
||||||
|
def actual_dir(self) -> Path:
|
||||||
|
return self.dir / "actual"
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def actual_screens(self) -> tuple[list[Path], list[str]]:
|
||||||
|
_rename_records(self.actual_dir)
|
||||||
|
return screens_and_hashes(self.actual_dir)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def recorded_dir(self) -> Path:
|
||||||
|
return self.dir / "recorded"
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def recorded_screens(self) -> tuple[list[Path], list[str]]:
|
||||||
|
return screens_and_hashes(self.recorded_dir)
|
||||||
|
|
||||||
|
def build_result(self, request: pytest.FixtureRequest) -> TestResult:
|
||||||
|
_rename_records(self.actual_dir)
|
||||||
|
result = TestResult(
|
||||||
|
test=self,
|
||||||
|
passed=get_last_call_test_result(request),
|
||||||
|
actual_hash=_hash_files(self.actual_dir),
|
||||||
|
images=self.actual_screens[1],
|
||||||
|
)
|
||||||
|
result.save_metadata()
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class TestResult:
|
||||||
|
test: TestCase
|
||||||
|
passed: bool | None
|
||||||
|
actual_hash: str
|
||||||
|
images: list[str]
|
||||||
|
expected_hash: str | None = field(default=None)
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
if self.expected_hash is None:
|
||||||
|
self.expected_hash = (
|
||||||
|
get_fixtures()
|
||||||
|
.get(self.test.model, {})
|
||||||
|
.get(self.test.group, {})
|
||||||
|
.get(self.test.fixtures_name)
|
||||||
|
)
|
||||||
|
|
||||||
|
def save_metadata(self) -> None:
|
||||||
|
metadata = asdict(self)
|
||||||
|
(self.test.dir / "metadata.json").write_text(
|
||||||
|
json.dumps(metadata, indent=2, sort_keys=True) + "\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def load(cls, testdir: Path) -> Self:
|
||||||
|
metadata = json.loads((testdir / "metadata.json").read_text())
|
||||||
|
test = TestCase(
|
||||||
|
model=metadata["test"]["model"],
|
||||||
|
group=metadata["test"]["group"],
|
||||||
|
name=metadata["test"]["name"],
|
||||||
|
)
|
||||||
|
return cls(
|
||||||
|
test=test,
|
||||||
|
passed=metadata["passed"],
|
||||||
|
actual_hash=metadata["actual_hash"],
|
||||||
|
expected_hash=metadata["expected_hash"],
|
||||||
|
images=metadata["images"],
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def recent_tests(cls) -> t.Iterator[Self]:
|
||||||
|
for testdir in sorted(SCREENS_DIR.iterdir()):
|
||||||
|
meta = testdir / "metadata.json"
|
||||||
|
if not meta.exists():
|
||||||
|
continue
|
||||||
|
yield cls.load(testdir)
|
||||||
|
|
||||||
|
def store_recorded(self) -> None:
|
||||||
|
self.expected_hash = self.actual_hash
|
||||||
|
shutil.rmtree(self.test.recorded_dir, ignore_errors=True)
|
||||||
|
shutil.copytree(
|
||||||
|
self.test.actual_dir,
|
||||||
|
self.test.recorded_dir,
|
||||||
|
symlinks=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def diff_lines(self) -> t.Iterable[tuple[str | None, str | None]]:
|
||||||
|
_, expected_hashes = self.test.recorded_screens
|
||||||
|
if not expected_hashes:
|
||||||
|
warnings.warn("No recorded screens found, is this a new test?")
|
||||||
|
_, actual_hashes = self.test.actual_screens
|
||||||
|
|
||||||
|
return screens_diff(expected_hashes, actual_hashes)
|
@ -0,0 +1,17 @@
|
|||||||
|
import click
|
||||||
|
|
||||||
|
from . import master_diff
|
||||||
|
|
||||||
|
|
||||||
|
@click.group()
|
||||||
|
def cli():
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@cli.command(name="master-diff")
|
||||||
|
def do_master_diff():
|
||||||
|
master_diff.main()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
cli()
|
@ -0,0 +1,261 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import tempfile
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import dominate
|
||||||
|
from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr
|
||||||
|
|
||||||
|
from ..common import (
|
||||||
|
SCREENS_DIR,
|
||||||
|
FixturesType,
|
||||||
|
get_fixtures,
|
||||||
|
screens_and_hashes,
|
||||||
|
screens_diff,
|
||||||
|
)
|
||||||
|
from . import download, html
|
||||||
|
from .testreport import REPORTS_PATH
|
||||||
|
|
||||||
|
MASTERDIFF_PATH = REPORTS_PATH / "master_diff"
|
||||||
|
IMAGES_PATH = MASTERDIFF_PATH / "images"
|
||||||
|
|
||||||
|
|
||||||
|
def _preprocess_master_compat(master_fixtures: dict[str, Any]) -> FixturesType:
|
||||||
|
if all(isinstance(v, str) for v in master_fixtures.values()):
|
||||||
|
# old format, convert to new format
|
||||||
|
new_fixtures = {}
|
||||||
|
for key, val in master_fixtures.items():
|
||||||
|
model, _test = key.split("_", maxsplit=1)
|
||||||
|
groups_by_model = new_fixtures.setdefault(model, {})
|
||||||
|
default_group = groups_by_model.setdefault("device_tests", {})
|
||||||
|
default_group[key] = val
|
||||||
|
return FixturesType(new_fixtures)
|
||||||
|
else:
|
||||||
|
return FixturesType(master_fixtures)
|
||||||
|
|
||||||
|
|
||||||
|
def get_diff() -> tuple[dict[str, str], dict[str, str], dict[str, tuple[str, str]]]:
|
||||||
|
master = _preprocess_master_compat(download.fetch_fixtures_master())
|
||||||
|
current = get_fixtures()
|
||||||
|
|
||||||
|
removed = {}
|
||||||
|
added = {}
|
||||||
|
diff = {}
|
||||||
|
|
||||||
|
for model in master.keys() | current.keys():
|
||||||
|
master_groups = master.get(model, {})
|
||||||
|
current_groups = current.get(model, {})
|
||||||
|
for group in master_groups.keys() | current_groups.keys():
|
||||||
|
master_tests = master_groups.get(group, {})
|
||||||
|
current_tests = current_groups.get(group, {})
|
||||||
|
|
||||||
|
print(f"checking model {model}, group {group}...")
|
||||||
|
|
||||||
|
def testname(test: str) -> str:
|
||||||
|
assert test.startswith(model + "_")
|
||||||
|
test = test[len(model) + 1 :]
|
||||||
|
return f"{model}-{group}-{test}"
|
||||||
|
|
||||||
|
# removed items
|
||||||
|
removed_here = {
|
||||||
|
testname(test): master_tests[test]
|
||||||
|
for test in (master_tests.keys() - current_tests.keys())
|
||||||
|
}
|
||||||
|
# added items
|
||||||
|
added_here = {
|
||||||
|
testname(test): current_tests[test]
|
||||||
|
for test in (current_tests.keys() - master_tests.keys())
|
||||||
|
}
|
||||||
|
# items in both branches
|
||||||
|
same = master_tests.items() - removed_here.items() - added_here.items()
|
||||||
|
# create the diff
|
||||||
|
diff_here = {}
|
||||||
|
for master_test, master_hash in same:
|
||||||
|
if current_tests.get(master_test) == master_hash:
|
||||||
|
continue
|
||||||
|
diff_here[testname(master_test)] = (
|
||||||
|
master_tests[master_test],
|
||||||
|
current_tests[master_test],
|
||||||
|
)
|
||||||
|
|
||||||
|
removed.update(removed_here)
|
||||||
|
added.update(added_here)
|
||||||
|
diff.update(diff_here)
|
||||||
|
print(f" removed: {len(removed_here)}")
|
||||||
|
print(f" added: {len(added_here)}")
|
||||||
|
print(f" diff: {len(diff_here)}")
|
||||||
|
|
||||||
|
return removed, added, diff
|
||||||
|
|
||||||
|
|
||||||
|
def removed(screens_path: Path, test_name: str) -> Path:
|
||||||
|
doc = dominate.document(title=test_name)
|
||||||
|
screens, hashes = screens_and_hashes(screens_path)
|
||||||
|
html.store_images(screens, hashes)
|
||||||
|
|
||||||
|
with doc:
|
||||||
|
h1(test_name)
|
||||||
|
p(
|
||||||
|
"This UI test has been removed from fixtures.json.",
|
||||||
|
style="color: red; font-weight: bold;",
|
||||||
|
)
|
||||||
|
hr()
|
||||||
|
|
||||||
|
with table(border=1):
|
||||||
|
with tr():
|
||||||
|
th("Removed files")
|
||||||
|
|
||||||
|
for hash in hashes:
|
||||||
|
with tr():
|
||||||
|
html.image_column(hash, MASTERDIFF_PATH / "removed")
|
||||||
|
|
||||||
|
return html.write(MASTERDIFF_PATH / "removed", doc, test_name + ".html")
|
||||||
|
|
||||||
|
|
||||||
|
def added(screens_path: Path, test_name: str) -> Path:
|
||||||
|
doc = dominate.document(title=test_name)
|
||||||
|
screens, hashes = screens_and_hashes(screens_path)
|
||||||
|
html.store_images(screens, hashes)
|
||||||
|
|
||||||
|
with doc:
|
||||||
|
h1(test_name)
|
||||||
|
p(
|
||||||
|
"This UI test has been added to fixtures.json.",
|
||||||
|
style="color: green; font-weight: bold;",
|
||||||
|
)
|
||||||
|
hr()
|
||||||
|
|
||||||
|
with table(border=1):
|
||||||
|
with tr():
|
||||||
|
th("Added files")
|
||||||
|
|
||||||
|
for hash in hashes:
|
||||||
|
with tr():
|
||||||
|
html.image_column(hash, MASTERDIFF_PATH / "added")
|
||||||
|
|
||||||
|
return html.write(MASTERDIFF_PATH / "added", doc, test_name + ".html")
|
||||||
|
|
||||||
|
|
||||||
|
def diff(
|
||||||
|
master_screens_path: Path,
|
||||||
|
current_screens_path: Path,
|
||||||
|
test_name: str,
|
||||||
|
master_hash: str,
|
||||||
|
current_hash: str,
|
||||||
|
) -> Path:
|
||||||
|
doc = dominate.document(title=test_name)
|
||||||
|
master_screens, master_hashes = screens_and_hashes(master_screens_path)
|
||||||
|
current_screens, current_hashes = screens_and_hashes(current_screens_path)
|
||||||
|
html.store_images(master_screens, master_hashes)
|
||||||
|
html.store_images(current_screens, current_hashes)
|
||||||
|
|
||||||
|
with doc:
|
||||||
|
h1(test_name)
|
||||||
|
p("This UI test differs from master.", style="color: grey; font-weight: bold;")
|
||||||
|
with table():
|
||||||
|
with tr():
|
||||||
|
td("Master:")
|
||||||
|
td(master_hash, style="color: red;")
|
||||||
|
with tr():
|
||||||
|
td("Current:")
|
||||||
|
td(current_hash, style="color: green;")
|
||||||
|
hr()
|
||||||
|
|
||||||
|
with table(border=1, width=600):
|
||||||
|
with tr():
|
||||||
|
th("Master")
|
||||||
|
th("Current branch")
|
||||||
|
|
||||||
|
html.diff_table(
|
||||||
|
screens_diff(master_hashes, current_hashes), MASTERDIFF_PATH / "diff"
|
||||||
|
)
|
||||||
|
|
||||||
|
return html.write(MASTERDIFF_PATH / "diff", doc, test_name + ".html")
|
||||||
|
|
||||||
|
|
||||||
|
def index() -> Path:
|
||||||
|
removed = list((MASTERDIFF_PATH / "removed").iterdir())
|
||||||
|
added = list((MASTERDIFF_PATH / "added").iterdir())
|
||||||
|
diff = list((MASTERDIFF_PATH / "diff").iterdir())
|
||||||
|
|
||||||
|
title = "UI changes from master"
|
||||||
|
doc = dominate.document(title=title)
|
||||||
|
|
||||||
|
with doc:
|
||||||
|
h1("UI changes from master")
|
||||||
|
hr()
|
||||||
|
|
||||||
|
h2("Removed:", style="color: red;")
|
||||||
|
i("UI fixtures that have been removed:")
|
||||||
|
html.report_links(removed, MASTERDIFF_PATH)
|
||||||
|
br()
|
||||||
|
hr()
|
||||||
|
|
||||||
|
h2("Added:", style="color: green;")
|
||||||
|
i("UI fixtures that have been added:")
|
||||||
|
html.report_links(added, MASTERDIFF_PATH)
|
||||||
|
br()
|
||||||
|
hr()
|
||||||
|
|
||||||
|
h2("Differs:", style="color: grey;")
|
||||||
|
i("UI fixtures that have been modified:")
|
||||||
|
html.report_links(diff, MASTERDIFF_PATH)
|
||||||
|
|
||||||
|
return html.write(MASTERDIFF_PATH, doc, "index.html")
|
||||||
|
|
||||||
|
|
||||||
|
def create_dirs() -> None:
|
||||||
|
# delete the reports dir to clear previous entries and create folders
|
||||||
|
shutil.rmtree(MASTERDIFF_PATH, ignore_errors=True)
|
||||||
|
MASTERDIFF_PATH.mkdir(parents=True)
|
||||||
|
(MASTERDIFF_PATH / "removed").mkdir()
|
||||||
|
(MASTERDIFF_PATH / "added").mkdir()
|
||||||
|
(MASTERDIFF_PATH / "diff").mkdir()
|
||||||
|
IMAGES_PATH.mkdir(exist_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def create_reports() -> None:
|
||||||
|
removed_tests, added_tests, diff_tests = get_diff()
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def tmpdir():
|
||||||
|
with tempfile.TemporaryDirectory(prefix="trezor-records-") as temp_dir:
|
||||||
|
yield Path(temp_dir)
|
||||||
|
|
||||||
|
for test_name, test_hash in removed_tests.items():
|
||||||
|
with tmpdir() as temp_dir:
|
||||||
|
download.fetch_recorded(test_hash, temp_dir)
|
||||||
|
removed(temp_dir, test_name)
|
||||||
|
|
||||||
|
for test_name, test_hash in added_tests.items():
|
||||||
|
path = SCREENS_DIR / test_name / "actual"
|
||||||
|
if not path.exists():
|
||||||
|
raise RuntimeError("Folder does not exist, has it been recorded?", path)
|
||||||
|
added(path, test_name)
|
||||||
|
|
||||||
|
for test_name, (master_hash, current_hash) in diff_tests.items():
|
||||||
|
with tmpdir() as master_screens:
|
||||||
|
download.fetch_recorded(master_hash, master_screens)
|
||||||
|
|
||||||
|
current_screens = SCREENS_DIR / test_name / "actual"
|
||||||
|
if not current_screens.exists():
|
||||||
|
raise RuntimeError(
|
||||||
|
"Folder does not exist, did the test run?", current_screens
|
||||||
|
)
|
||||||
|
diff(
|
||||||
|
master_screens,
|
||||||
|
current_screens,
|
||||||
|
test_name,
|
||||||
|
master_hash,
|
||||||
|
current_hash,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
create_dirs()
|
||||||
|
html.set_image_dir(IMAGES_PATH)
|
||||||
|
create_reports()
|
||||||
|
index()
|
@ -1,206 +0,0 @@
|
|||||||
import shutil
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
from contextlib import contextmanager
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Dict, Sequence, Tuple
|
|
||||||
|
|
||||||
import dominate
|
|
||||||
from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr
|
|
||||||
|
|
||||||
# These are imported directly because this script is run directly, isort gets confused by that.
|
|
||||||
import download # isort:skip
|
|
||||||
import html # isort:skip
|
|
||||||
|
|
||||||
REPORTS_PATH = Path(__file__).resolve().parent / "reports" / "master_diff"
|
|
||||||
RECORDED_SCREENS_PATH = Path(__file__).resolve().parent.parent / "screens"
|
|
||||||
|
|
||||||
|
|
||||||
def get_diff(
|
|
||||||
test_prefixes: Sequence[str],
|
|
||||||
) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
|
|
||||||
master = download.fetch_fixtures_master()
|
|
||||||
current = download.fetch_fixtures_current()
|
|
||||||
|
|
||||||
def matches_prefix(name: str) -> bool:
|
|
||||||
return any(name.startswith(prefix) for prefix in test_prefixes)
|
|
||||||
|
|
||||||
master = {name: value for name, value in master.items() if matches_prefix(name)}
|
|
||||||
current = {name: value for name, value in current.items() if matches_prefix(name)}
|
|
||||||
|
|
||||||
# removed items
|
|
||||||
removed = {test: master[test] for test in (master.keys() - current.keys())}
|
|
||||||
# added items
|
|
||||||
added = {test: current[test] for test in (current.keys() - master.keys())}
|
|
||||||
# items in both branches
|
|
||||||
same = master.items() - removed.items() - added.items()
|
|
||||||
# create the diff
|
|
||||||
diff = dict()
|
|
||||||
for master_test, master_hash in same:
|
|
||||||
if current.get(master_test) == master_hash:
|
|
||||||
continue
|
|
||||||
diff[master_test] = master[master_test], current[master_test]
|
|
||||||
|
|
||||||
return removed, added, diff
|
|
||||||
|
|
||||||
|
|
||||||
def removed(screens_path: Path, test_name: str) -> Path:
|
|
||||||
doc = dominate.document(title=test_name)
|
|
||||||
screens = sorted(screens_path.iterdir())
|
|
||||||
|
|
||||||
with doc:
|
|
||||||
h1(test_name)
|
|
||||||
p(
|
|
||||||
"This UI test has been removed from fixtures.json.",
|
|
||||||
style="color: red; font-weight: bold;",
|
|
||||||
)
|
|
||||||
hr()
|
|
||||||
|
|
||||||
with table(border=1):
|
|
||||||
with tr():
|
|
||||||
th("Removed files")
|
|
||||||
|
|
||||||
for screen in screens:
|
|
||||||
with tr():
|
|
||||||
html.image_column(screen)
|
|
||||||
|
|
||||||
return html.write(REPORTS_PATH / "removed", doc, test_name + ".html")
|
|
||||||
|
|
||||||
|
|
||||||
def added(screens_path: Path, test_name: str) -> Path:
|
|
||||||
doc = dominate.document(title=test_name)
|
|
||||||
screens = sorted(screens_path.iterdir())
|
|
||||||
|
|
||||||
with doc:
|
|
||||||
h1(test_name)
|
|
||||||
p(
|
|
||||||
"This UI test has been added to fixtures.json.",
|
|
||||||
style="color: green; font-weight: bold;",
|
|
||||||
)
|
|
||||||
hr()
|
|
||||||
|
|
||||||
with table(border=1):
|
|
||||||
with tr():
|
|
||||||
th("Added files")
|
|
||||||
|
|
||||||
for screen in screens:
|
|
||||||
with tr():
|
|
||||||
html.image_column(screen)
|
|
||||||
|
|
||||||
return html.write(REPORTS_PATH / "added", doc, test_name + ".html")
|
|
||||||
|
|
||||||
|
|
||||||
def diff(
|
|
||||||
master_screens_path: Path,
|
|
||||||
current_screens_path: Path,
|
|
||||||
test_name: str,
|
|
||||||
master_hash: str,
|
|
||||||
current_hash: str,
|
|
||||||
) -> Path:
|
|
||||||
doc = dominate.document(title=test_name)
|
|
||||||
master_screens = sorted(master_screens_path.iterdir())
|
|
||||||
current_screens = sorted(current_screens_path.iterdir())
|
|
||||||
|
|
||||||
with doc:
|
|
||||||
h1(test_name)
|
|
||||||
p("This UI test differs from master.", style="color: grey; font-weight: bold;")
|
|
||||||
with table():
|
|
||||||
with tr():
|
|
||||||
td("Master:")
|
|
||||||
td(master_hash, style="color: red;")
|
|
||||||
with tr():
|
|
||||||
td("Current:")
|
|
||||||
td(current_hash, style="color: green;")
|
|
||||||
hr()
|
|
||||||
|
|
||||||
with table(border=1, width=600):
|
|
||||||
with tr():
|
|
||||||
th("Master")
|
|
||||||
th("Current branch")
|
|
||||||
|
|
||||||
html.diff_table(master_screens, current_screens)
|
|
||||||
|
|
||||||
return html.write(REPORTS_PATH / "diff", doc, test_name + ".html")
|
|
||||||
|
|
||||||
|
|
||||||
def index() -> Path:
|
|
||||||
removed = list((REPORTS_PATH / "removed").iterdir())
|
|
||||||
added = list((REPORTS_PATH / "added").iterdir())
|
|
||||||
diff = list((REPORTS_PATH / "diff").iterdir())
|
|
||||||
|
|
||||||
title = "UI changes from master"
|
|
||||||
doc = dominate.document(title=title)
|
|
||||||
|
|
||||||
with doc:
|
|
||||||
h1("UI changes from master")
|
|
||||||
hr()
|
|
||||||
|
|
||||||
h2("Removed:", style="color: red;")
|
|
||||||
i("UI fixtures that have been removed:")
|
|
||||||
html.report_links(removed, REPORTS_PATH)
|
|
||||||
br()
|
|
||||||
hr()
|
|
||||||
|
|
||||||
h2("Added:", style="color: green;")
|
|
||||||
i("UI fixtures that have been added:")
|
|
||||||
html.report_links(added, REPORTS_PATH)
|
|
||||||
br()
|
|
||||||
hr()
|
|
||||||
|
|
||||||
h2("Differs:", style="color: grey;")
|
|
||||||
i("UI fixtures that have been modified:")
|
|
||||||
html.report_links(diff, REPORTS_PATH)
|
|
||||||
|
|
||||||
return html.write(REPORTS_PATH, doc, "index.html")
|
|
||||||
|
|
||||||
|
|
||||||
def create_dirs() -> None:
|
|
||||||
# delete the reports dir to clear previous entries and create folders
|
|
||||||
shutil.rmtree(REPORTS_PATH, ignore_errors=True)
|
|
||||||
REPORTS_PATH.mkdir()
|
|
||||||
(REPORTS_PATH / "removed").mkdir()
|
|
||||||
(REPORTS_PATH / "added").mkdir()
|
|
||||||
(REPORTS_PATH / "diff").mkdir()
|
|
||||||
|
|
||||||
|
|
||||||
def create_reports(test_prefixes: Sequence[str]) -> None:
|
|
||||||
removed_tests, added_tests, diff_tests = get_diff(test_prefixes)
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def tmpdir():
|
|
||||||
with tempfile.TemporaryDirectory(prefix="trezor-records-") as temp_dir:
|
|
||||||
yield Path(temp_dir)
|
|
||||||
|
|
||||||
for test_name, test_hash in removed_tests.items():
|
|
||||||
with tmpdir() as temp_dir:
|
|
||||||
download.fetch_recorded(test_hash, temp_dir)
|
|
||||||
removed(temp_dir, test_name)
|
|
||||||
|
|
||||||
for test_name, test_hash in added_tests.items():
|
|
||||||
path = RECORDED_SCREENS_PATH / test_name / "actual"
|
|
||||||
if not path.exists():
|
|
||||||
raise RuntimeError("Folder does not exist, has it been recorded?", path)
|
|
||||||
added(path, test_name)
|
|
||||||
|
|
||||||
for test_name, (master_hash, current_hash) in diff_tests.items():
|
|
||||||
with tmpdir() as master_screens:
|
|
||||||
download.fetch_recorded(master_hash, master_screens)
|
|
||||||
|
|
||||||
current_screens = RECORDED_SCREENS_PATH / test_name / "actual"
|
|
||||||
if not current_screens.exists():
|
|
||||||
raise RuntimeError(
|
|
||||||
"Folder does not exist, has it been recorded?", current_screens
|
|
||||||
)
|
|
||||||
diff(
|
|
||||||
master_screens,
|
|
||||||
current_screens,
|
|
||||||
test_name,
|
|
||||||
master_hash,
|
|
||||||
current_hash,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
create_dirs()
|
|
||||||
create_reports(sys.argv[1:] or [""])
|
|
||||||
index()
|
|
@ -1,7 +1,16 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
import click
|
||||||
|
|
||||||
from ui_tests import update_fixtures_with_diff
|
from ui_tests import update_fixtures
|
||||||
|
|
||||||
changes_amount = update_fixtures_with_diff()
|
|
||||||
|
|
||||||
print(f"{changes_amount} hashes updated in fixtures.json file.")
|
@click.command()
|
||||||
|
@click.option("-r", "--remove-missing", is_flag=True, help="Remove missing tests")
|
||||||
|
def main(remove_missing: bool) -> None:
|
||||||
|
"""Update fixtures file with results from latest test run."""
|
||||||
|
changes_amount = update_fixtures(remove_missing)
|
||||||
|
print(f"Updated fixtures.json with data from {changes_amount} tests.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
Loading…
Reference in new issue