matejcik
1 year ago
committed by
matejcik
parent
f62ce2f793
commit
e15662e12d
@ -0,0 +1,308 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import typing as t
|
||||
import warnings
|
||||
from copy import deepcopy
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from difflib import SequenceMatcher
|
||||
from functools import cached_property
|
||||
from itertools import zip_longest
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from typing_extensions import Self
|
||||
|
||||
from trezorlib.debuglink import TrezorClientDebugLink as Client
|
||||
|
||||
UI_TESTS_DIR = Path(__file__).resolve().parent
|
||||
SCREENS_DIR = UI_TESTS_DIR / "screens"
|
||||
IMAGES_DIR = SCREENS_DIR / "all_images"
|
||||
FIXTURES_FILE = UI_TESTS_DIR / "fixtures.json"
|
||||
|
||||
# fixtures.json are structured as follows:
|
||||
# {
|
||||
# "model": {
|
||||
# "group": {
|
||||
# "test_name": "hash",
|
||||
# ...
|
||||
# }}}...
|
||||
# IOW, FixturesType = dict[<model>, dict[<group>, dict[<test_name>, <hash>]]]
|
||||
FixturesType = t.NewType("FixturesType", "dict[str, dict[str, dict[str, str]]]")
|
||||
|
||||
FIXTURES: FixturesType = FixturesType({})
|
||||
|
||||
|
||||
def get_fixtures() -> FixturesType:
|
||||
global FIXTURES
|
||||
if not FIXTURES and FIXTURES_FILE.exists():
|
||||
FIXTURES = FixturesType(json.loads(FIXTURES_FILE.read_text()))
|
||||
|
||||
return FIXTURES
|
||||
|
||||
|
||||
def prepare_fixtures(
|
||||
results: t.Iterable[TestResult],
|
||||
remove_missing: bool = False,
|
||||
) -> tuple[FixturesType, set[TestCase]]:
|
||||
"""Prepare contents of fixtures.json"""
|
||||
# set up brand new contents
|
||||
grouped_tests: dict[tuple[str, str], dict[str, str]] = {}
|
||||
for result in results:
|
||||
idx = result.test.model, result.test.group
|
||||
group = grouped_tests.setdefault(idx, {})
|
||||
group[result.test.fixtures_name] = result.actual_hash
|
||||
|
||||
missing_tests = set()
|
||||
|
||||
# merge with previous fixtures
|
||||
fixtures = deepcopy(get_fixtures())
|
||||
for (model, group), new_content in grouped_tests.items():
|
||||
# for every model/group, update the data with the new content
|
||||
current_content = fixtures.setdefault(model, {}).setdefault(group, {})
|
||||
if remove_missing:
|
||||
new_tests = set(new_content.keys())
|
||||
old_tests = set(current_content.keys())
|
||||
missing_tests |= {
|
||||
TestCase(model, group, test) for test in old_tests - new_tests
|
||||
}
|
||||
current_content.clear()
|
||||
|
||||
current_content.update(new_content)
|
||||
|
||||
return fixtures, missing_tests
|
||||
|
||||
|
||||
def write_fixtures(
|
||||
results: t.Iterable[TestResult],
|
||||
remove_missing: bool = False,
|
||||
dest: Path = FIXTURES_FILE,
|
||||
) -> None:
|
||||
global FIXTURES
|
||||
content, _ = prepare_fixtures(results, remove_missing)
|
||||
FIXTURES = FixturesType(content)
|
||||
dest.write_text(json.dumps(content, indent=0, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def _rename_records(screen_path: Path) -> None:
|
||||
IMAGES_DIR.mkdir(exist_ok=True)
|
||||
# rename screenshots
|
||||
for index, record in enumerate(sorted(screen_path.iterdir())):
|
||||
record.replace(screen_path / f"{index:08}.png")
|
||||
|
||||
|
||||
def screens_and_hashes(screen_path: Path) -> tuple[list[Path], list[str]]:
|
||||
if not screen_path.exists():
|
||||
return [], []
|
||||
|
||||
hashes = []
|
||||
paths = []
|
||||
for file in sorted(screen_path.iterdir()):
|
||||
paths.append(file)
|
||||
if len(file.stem) == 32:
|
||||
try:
|
||||
hashes.append(bytes.fromhex(file.stem))
|
||||
continue
|
||||
except ValueError:
|
||||
pass
|
||||
hashes.append(_get_image_hash(file))
|
||||
return paths, hashes
|
||||
|
||||
|
||||
def _get_image_hash(png_file: Path) -> str:
|
||||
return hashlib.sha256(_get_bytes_from_png(png_file)).hexdigest()
|
||||
|
||||
|
||||
def _get_bytes_from_png(png_file: Path) -> bytes:
|
||||
"""Decode a PNG file into bytes representing all the pixels.
|
||||
|
||||
Is necessary because Linux and Mac are using different PNG encoding libraries,
|
||||
and we need the file hashes to be the same on both platforms.
|
||||
"""
|
||||
return Image.open(str(png_file)).tobytes()
|
||||
|
||||
|
||||
def _hash_files(path: Path) -> str:
|
||||
files = path.iterdir()
|
||||
hasher = hashlib.sha256()
|
||||
for file in sorted(files):
|
||||
hasher.update(_get_bytes_from_png(file))
|
||||
|
||||
return hasher.digest().hex()
|
||||
|
||||
|
||||
def get_last_call_test_result(request: pytest.FixtureRequest) -> bool | None:
|
||||
# if test did not finish, e.g. interrupted by Ctrl+C, the pytest_runtest_makereport
|
||||
# did not create the attribute we need
|
||||
if not hasattr(request.node, "rep_call"):
|
||||
return None
|
||||
|
||||
return request.node.rep_call.passed # type: ignore
|
||||
|
||||
|
||||
def _get_test_name_and_group(node_id: str) -> tuple[str, str]:
|
||||
test_path, func_id = node_id.split("::", maxsplit=1)
|
||||
assert test_path.endswith(".py")
|
||||
|
||||
# tests / device_tests / bitcoin / test_signtx.py
|
||||
_tests, group_name, *path_in_group = test_path.split("/")
|
||||
|
||||
# remove ::TestClass:: if present because it is usually the same as the test file name
|
||||
func_id = re.sub(r"::.*?::", "-", func_id)
|
||||
|
||||
test_path_prefix = "-".join(path_in_group)
|
||||
new_name = f"{test_path_prefix}::{func_id}"
|
||||
new_name = new_name.replace("/", "-")
|
||||
# Test item name is usually function name, but when parametrization is used,
|
||||
# parameters are also part of the name. Some functions have very long parameter
|
||||
# names (tx hashes etc) that run out of maximum allowable filename length, so
|
||||
# we limit the name to first 100 chars. This is not a problem with txhashes.
|
||||
if len(new_name) <= 100:
|
||||
return new_name, group_name
|
||||
|
||||
differentiator = hashlib.sha256(new_name.encode()).hexdigest()
|
||||
shortened_name = new_name[:91] + "-" + differentiator[:8]
|
||||
return shortened_name, group_name
|
||||
|
||||
|
||||
def screens_diff(
|
||||
expected_hashes: list[str], actual_hashes: list[str]
|
||||
) -> t.Iterator[tuple[str | None, str | None]]:
|
||||
diff = SequenceMatcher(
|
||||
None, expected_hashes, actual_hashes, autojunk=False
|
||||
).get_opcodes()
|
||||
for _tag, i1, i2, j1, j2 in diff:
|
||||
# tag is one of "replace", "delete", "equal", "insert"
|
||||
# i1, i2 and j1, j2 are slice indexes for expected/actual respectively
|
||||
# The output of get_opcodes is an ordered sequence of instructions
|
||||
# for converting expected to actual. By taking the subslices and zipping
|
||||
# together, we get the equal subsequences aligned and Nones at deletion
|
||||
# or insertion points.
|
||||
expected_slice = expected_hashes[i1:i2]
|
||||
actual_slice = actual_hashes[j1:j2]
|
||||
yield from zip_longest(expected_slice, actual_slice, fillvalue=None)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TestCase:
|
||||
model: str
|
||||
group: str
|
||||
name: str
|
||||
|
||||
@classmethod
|
||||
def build(cls, client: Client, request: pytest.FixtureRequest) -> Self:
|
||||
name, group = _get_test_name_and_group(request.node.nodeid)
|
||||
return cls(
|
||||
model=f"T{client.features.model}",
|
||||
name=name,
|
||||
group=group,
|
||||
)
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return f"{self.model}-{self.group}-{self.name}"
|
||||
|
||||
@property
|
||||
def fixtures_name(self) -> str:
|
||||
return f"{self.model}_{self.name}"
|
||||
|
||||
@property
|
||||
def dir(self) -> Path:
|
||||
return SCREENS_DIR / self.id
|
||||
|
||||
@property
|
||||
def actual_dir(self) -> Path:
|
||||
return self.dir / "actual"
|
||||
|
||||
@cached_property
|
||||
def actual_screens(self) -> tuple[list[Path], list[str]]:
|
||||
_rename_records(self.actual_dir)
|
||||
return screens_and_hashes(self.actual_dir)
|
||||
|
||||
@property
|
||||
def recorded_dir(self) -> Path:
|
||||
return self.dir / "recorded"
|
||||
|
||||
@cached_property
|
||||
def recorded_screens(self) -> tuple[list[Path], list[str]]:
|
||||
return screens_and_hashes(self.recorded_dir)
|
||||
|
||||
def build_result(self, request: pytest.FixtureRequest) -> TestResult:
|
||||
_rename_records(self.actual_dir)
|
||||
result = TestResult(
|
||||
test=self,
|
||||
passed=get_last_call_test_result(request),
|
||||
actual_hash=_hash_files(self.actual_dir),
|
||||
images=self.actual_screens[1],
|
||||
)
|
||||
result.save_metadata()
|
||||
return result
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestResult:
|
||||
test: TestCase
|
||||
passed: bool | None
|
||||
actual_hash: str
|
||||
images: list[str]
|
||||
expected_hash: str | None = field(default=None)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.expected_hash is None:
|
||||
self.expected_hash = (
|
||||
get_fixtures()
|
||||
.get(self.test.model, {})
|
||||
.get(self.test.group, {})
|
||||
.get(self.test.fixtures_name)
|
||||
)
|
||||
|
||||
def save_metadata(self) -> None:
|
||||
metadata = asdict(self)
|
||||
(self.test.dir / "metadata.json").write_text(
|
||||
json.dumps(metadata, indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def load(cls, testdir: Path) -> Self:
|
||||
metadata = json.loads((testdir / "metadata.json").read_text())
|
||||
test = TestCase(
|
||||
model=metadata["test"]["model"],
|
||||
group=metadata["test"]["group"],
|
||||
name=metadata["test"]["name"],
|
||||
)
|
||||
return cls(
|
||||
test=test,
|
||||
passed=metadata["passed"],
|
||||
actual_hash=metadata["actual_hash"],
|
||||
expected_hash=metadata["expected_hash"],
|
||||
images=metadata["images"],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def recent_tests(cls) -> t.Iterator[Self]:
|
||||
for testdir in sorted(SCREENS_DIR.iterdir()):
|
||||
meta = testdir / "metadata.json"
|
||||
if not meta.exists():
|
||||
continue
|
||||
yield cls.load(testdir)
|
||||
|
||||
def store_recorded(self) -> None:
|
||||
self.expected_hash = self.actual_hash
|
||||
shutil.rmtree(self.test.recorded_dir, ignore_errors=True)
|
||||
shutil.copytree(
|
||||
self.test.actual_dir,
|
||||
self.test.recorded_dir,
|
||||
symlinks=True,
|
||||
)
|
||||
|
||||
def diff_lines(self) -> t.Iterable[tuple[str | None, str | None]]:
|
||||
_, expected_hashes = self.test.recorded_screens
|
||||
if not expected_hashes:
|
||||
warnings.warn("No recorded screens found, is this a new test?")
|
||||
_, actual_hashes = self.test.actual_screens
|
||||
|
||||
return screens_diff(expected_hashes, actual_hashes)
|
||||
@ -0,0 +1,17 @@
|
||||
import click
|
||||
|
||||
from . import master_diff
|
||||
|
||||
|
||||
@click.group()
|
||||
def cli():
|
||||
pass
|
||||
|
||||
|
||||
@cli.command(name="master-diff")
|
||||
def do_master_diff():
|
||||
master_diff.main()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
@ -0,0 +1,261 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import dominate
|
||||
from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr
|
||||
|
||||
from ..common import (
|
||||
SCREENS_DIR,
|
||||
FixturesType,
|
||||
get_fixtures,
|
||||
screens_and_hashes,
|
||||
screens_diff,
|
||||
)
|
||||
from . import download, html
|
||||
from .testreport import REPORTS_PATH
|
||||
|
||||
MASTERDIFF_PATH = REPORTS_PATH / "master_diff"
|
||||
IMAGES_PATH = MASTERDIFF_PATH / "images"
|
||||
|
||||
|
||||
def _preprocess_master_compat(master_fixtures: dict[str, Any]) -> FixturesType:
|
||||
if all(isinstance(v, str) for v in master_fixtures.values()):
|
||||
# old format, convert to new format
|
||||
new_fixtures = {}
|
||||
for key, val in master_fixtures.items():
|
||||
model, _test = key.split("_", maxsplit=1)
|
||||
groups_by_model = new_fixtures.setdefault(model, {})
|
||||
default_group = groups_by_model.setdefault("device_tests", {})
|
||||
default_group[key] = val
|
||||
return FixturesType(new_fixtures)
|
||||
else:
|
||||
return FixturesType(master_fixtures)
|
||||
|
||||
|
||||
def get_diff() -> tuple[dict[str, str], dict[str, str], dict[str, tuple[str, str]]]:
|
||||
master = _preprocess_master_compat(download.fetch_fixtures_master())
|
||||
current = get_fixtures()
|
||||
|
||||
removed = {}
|
||||
added = {}
|
||||
diff = {}
|
||||
|
||||
for model in master.keys() | current.keys():
|
||||
master_groups = master.get(model, {})
|
||||
current_groups = current.get(model, {})
|
||||
for group in master_groups.keys() | current_groups.keys():
|
||||
master_tests = master_groups.get(group, {})
|
||||
current_tests = current_groups.get(group, {})
|
||||
|
||||
print(f"checking model {model}, group {group}...")
|
||||
|
||||
def testname(test: str) -> str:
|
||||
assert test.startswith(model + "_")
|
||||
test = test[len(model) + 1 :]
|
||||
return f"{model}-{group}-{test}"
|
||||
|
||||
# removed items
|
||||
removed_here = {
|
||||
testname(test): master_tests[test]
|
||||
for test in (master_tests.keys() - current_tests.keys())
|
||||
}
|
||||
# added items
|
||||
added_here = {
|
||||
testname(test): current_tests[test]
|
||||
for test in (current_tests.keys() - master_tests.keys())
|
||||
}
|
||||
# items in both branches
|
||||
same = master_tests.items() - removed_here.items() - added_here.items()
|
||||
# create the diff
|
||||
diff_here = {}
|
||||
for master_test, master_hash in same:
|
||||
if current_tests.get(master_test) == master_hash:
|
||||
continue
|
||||
diff_here[testname(master_test)] = (
|
||||
master_tests[master_test],
|
||||
current_tests[master_test],
|
||||
)
|
||||
|
||||
removed.update(removed_here)
|
||||
added.update(added_here)
|
||||
diff.update(diff_here)
|
||||
print(f" removed: {len(removed_here)}")
|
||||
print(f" added: {len(added_here)}")
|
||||
print(f" diff: {len(diff_here)}")
|
||||
|
||||
return removed, added, diff
|
||||
|
||||
|
||||
def removed(screens_path: Path, test_name: str) -> Path:
|
||||
doc = dominate.document(title=test_name)
|
||||
screens, hashes = screens_and_hashes(screens_path)
|
||||
html.store_images(screens, hashes)
|
||||
|
||||
with doc:
|
||||
h1(test_name)
|
||||
p(
|
||||
"This UI test has been removed from fixtures.json.",
|
||||
style="color: red; font-weight: bold;",
|
||||
)
|
||||
hr()
|
||||
|
||||
with table(border=1):
|
||||
with tr():
|
||||
th("Removed files")
|
||||
|
||||
for hash in hashes:
|
||||
with tr():
|
||||
html.image_column(hash, MASTERDIFF_PATH / "removed")
|
||||
|
||||
return html.write(MASTERDIFF_PATH / "removed", doc, test_name + ".html")
|
||||
|
||||
|
||||
def added(screens_path: Path, test_name: str) -> Path:
|
||||
doc = dominate.document(title=test_name)
|
||||
screens, hashes = screens_and_hashes(screens_path)
|
||||
html.store_images(screens, hashes)
|
||||
|
||||
with doc:
|
||||
h1(test_name)
|
||||
p(
|
||||
"This UI test has been added to fixtures.json.",
|
||||
style="color: green; font-weight: bold;",
|
||||
)
|
||||
hr()
|
||||
|
||||
with table(border=1):
|
||||
with tr():
|
||||
th("Added files")
|
||||
|
||||
for hash in hashes:
|
||||
with tr():
|
||||
html.image_column(hash, MASTERDIFF_PATH / "added")
|
||||
|
||||
return html.write(MASTERDIFF_PATH / "added", doc, test_name + ".html")
|
||||
|
||||
|
||||
def diff(
|
||||
master_screens_path: Path,
|
||||
current_screens_path: Path,
|
||||
test_name: str,
|
||||
master_hash: str,
|
||||
current_hash: str,
|
||||
) -> Path:
|
||||
doc = dominate.document(title=test_name)
|
||||
master_screens, master_hashes = screens_and_hashes(master_screens_path)
|
||||
current_screens, current_hashes = screens_and_hashes(current_screens_path)
|
||||
html.store_images(master_screens, master_hashes)
|
||||
html.store_images(current_screens, current_hashes)
|
||||
|
||||
with doc:
|
||||
h1(test_name)
|
||||
p("This UI test differs from master.", style="color: grey; font-weight: bold;")
|
||||
with table():
|
||||
with tr():
|
||||
td("Master:")
|
||||
td(master_hash, style="color: red;")
|
||||
with tr():
|
||||
td("Current:")
|
||||
td(current_hash, style="color: green;")
|
||||
hr()
|
||||
|
||||
with table(border=1, width=600):
|
||||
with tr():
|
||||
th("Master")
|
||||
th("Current branch")
|
||||
|
||||
html.diff_table(
|
||||
screens_diff(master_hashes, current_hashes), MASTERDIFF_PATH / "diff"
|
||||
)
|
||||
|
||||
return html.write(MASTERDIFF_PATH / "diff", doc, test_name + ".html")
|
||||
|
||||
|
||||
def index() -> Path:
|
||||
removed = list((MASTERDIFF_PATH / "removed").iterdir())
|
||||
added = list((MASTERDIFF_PATH / "added").iterdir())
|
||||
diff = list((MASTERDIFF_PATH / "diff").iterdir())
|
||||
|
||||
title = "UI changes from master"
|
||||
doc = dominate.document(title=title)
|
||||
|
||||
with doc:
|
||||
h1("UI changes from master")
|
||||
hr()
|
||||
|
||||
h2("Removed:", style="color: red;")
|
||||
i("UI fixtures that have been removed:")
|
||||
html.report_links(removed, MASTERDIFF_PATH)
|
||||
br()
|
||||
hr()
|
||||
|
||||
h2("Added:", style="color: green;")
|
||||
i("UI fixtures that have been added:")
|
||||
html.report_links(added, MASTERDIFF_PATH)
|
||||
br()
|
||||
hr()
|
||||
|
||||
h2("Differs:", style="color: grey;")
|
||||
i("UI fixtures that have been modified:")
|
||||
html.report_links(diff, MASTERDIFF_PATH)
|
||||
|
||||
return html.write(MASTERDIFF_PATH, doc, "index.html")
|
||||
|
||||
|
||||
def create_dirs() -> None:
|
||||
# delete the reports dir to clear previous entries and create folders
|
||||
shutil.rmtree(MASTERDIFF_PATH, ignore_errors=True)
|
||||
MASTERDIFF_PATH.mkdir(parents=True)
|
||||
(MASTERDIFF_PATH / "removed").mkdir()
|
||||
(MASTERDIFF_PATH / "added").mkdir()
|
||||
(MASTERDIFF_PATH / "diff").mkdir()
|
||||
IMAGES_PATH.mkdir(exist_ok=True)
|
||||
|
||||
|
||||
def create_reports() -> None:
|
||||
removed_tests, added_tests, diff_tests = get_diff()
|
||||
|
||||
@contextmanager
|
||||
def tmpdir():
|
||||
with tempfile.TemporaryDirectory(prefix="trezor-records-") as temp_dir:
|
||||
yield Path(temp_dir)
|
||||
|
||||
for test_name, test_hash in removed_tests.items():
|
||||
with tmpdir() as temp_dir:
|
||||
download.fetch_recorded(test_hash, temp_dir)
|
||||
removed(temp_dir, test_name)
|
||||
|
||||
for test_name, test_hash in added_tests.items():
|
||||
path = SCREENS_DIR / test_name / "actual"
|
||||
if not path.exists():
|
||||
raise RuntimeError("Folder does not exist, has it been recorded?", path)
|
||||
added(path, test_name)
|
||||
|
||||
for test_name, (master_hash, current_hash) in diff_tests.items():
|
||||
with tmpdir() as master_screens:
|
||||
download.fetch_recorded(master_hash, master_screens)
|
||||
|
||||
current_screens = SCREENS_DIR / test_name / "actual"
|
||||
if not current_screens.exists():
|
||||
raise RuntimeError(
|
||||
"Folder does not exist, did the test run?", current_screens
|
||||
)
|
||||
diff(
|
||||
master_screens,
|
||||
current_screens,
|
||||
test_name,
|
||||
master_hash,
|
||||
current_hash,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
create_dirs()
|
||||
html.set_image_dir(IMAGES_PATH)
|
||||
create_reports()
|
||||
index()
|
||||
@ -1,206 +0,0 @@
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Dict, Sequence, Tuple
|
||||
|
||||
import dominate
|
||||
from dominate.tags import br, h1, h2, hr, i, p, table, td, th, tr
|
||||
|
||||
# These are imported directly because this script is run directly, isort gets confused by that.
|
||||
import download # isort:skip
|
||||
import html # isort:skip
|
||||
|
||||
REPORTS_PATH = Path(__file__).resolve().parent / "reports" / "master_diff"
|
||||
RECORDED_SCREENS_PATH = Path(__file__).resolve().parent.parent / "screens"
|
||||
|
||||
|
||||
def get_diff(
|
||||
test_prefixes: Sequence[str],
|
||||
) -> Tuple[Dict[str, str], Dict[str, str], Dict[str, str]]:
|
||||
master = download.fetch_fixtures_master()
|
||||
current = download.fetch_fixtures_current()
|
||||
|
||||
def matches_prefix(name: str) -> bool:
|
||||
return any(name.startswith(prefix) for prefix in test_prefixes)
|
||||
|
||||
master = {name: value for name, value in master.items() if matches_prefix(name)}
|
||||
current = {name: value for name, value in current.items() if matches_prefix(name)}
|
||||
|
||||
# removed items
|
||||
removed = {test: master[test] for test in (master.keys() - current.keys())}
|
||||
# added items
|
||||
added = {test: current[test] for test in (current.keys() - master.keys())}
|
||||
# items in both branches
|
||||
same = master.items() - removed.items() - added.items()
|
||||
# create the diff
|
||||
diff = dict()
|
||||
for master_test, master_hash in same:
|
||||
if current.get(master_test) == master_hash:
|
||||
continue
|
||||
diff[master_test] = master[master_test], current[master_test]
|
||||
|
||||
return removed, added, diff
|
||||
|
||||
|
||||
def removed(screens_path: Path, test_name: str) -> Path:
|
||||
doc = dominate.document(title=test_name)
|
||||
screens = sorted(screens_path.iterdir())
|
||||
|
||||
with doc:
|
||||
h1(test_name)
|
||||
p(
|
||||
"This UI test has been removed from fixtures.json.",
|
||||
style="color: red; font-weight: bold;",
|
||||
)
|
||||
hr()
|
||||
|
||||
with table(border=1):
|
||||
with tr():
|
||||
th("Removed files")
|
||||
|
||||
for screen in screens:
|
||||
with tr():
|
||||
html.image_column(screen)
|
||||
|
||||
return html.write(REPORTS_PATH / "removed", doc, test_name + ".html")
|
||||
|
||||
|
||||
def added(screens_path: Path, test_name: str) -> Path:
|
||||
doc = dominate.document(title=test_name)
|
||||
screens = sorted(screens_path.iterdir())
|
||||
|
||||
with doc:
|
||||
h1(test_name)
|
||||
p(
|
||||
"This UI test has been added to fixtures.json.",
|
||||
style="color: green; font-weight: bold;",
|
||||
)
|
||||
hr()
|
||||
|
||||
with table(border=1):
|
||||
with tr():
|
||||
th("Added files")
|
||||
|
||||
for screen in screens:
|
||||
with tr():
|
||||
html.image_column(screen)
|
||||
|
||||
return html.write(REPORTS_PATH / "added", doc, test_name + ".html")
|
||||
|
||||
|
||||
def diff(
|
||||
master_screens_path: Path,
|
||||
current_screens_path: Path,
|
||||
test_name: str,
|
||||
master_hash: str,
|
||||
current_hash: str,
|
||||
) -> Path:
|
||||
doc = dominate.document(title=test_name)
|
||||
master_screens = sorted(master_screens_path.iterdir())
|
||||
current_screens = sorted(current_screens_path.iterdir())
|
||||
|
||||
with doc:
|
||||
h1(test_name)
|
||||
p("This UI test differs from master.", style="color: grey; font-weight: bold;")
|
||||
with table():
|
||||
with tr():
|
||||
td("Master:")
|
||||
td(master_hash, style="color: red;")
|
||||
with tr():
|
||||
td("Current:")
|
||||
td(current_hash, style="color: green;")
|
||||
hr()
|
||||
|
||||
with table(border=1, width=600):
|
||||
with tr():
|
||||
th("Master")
|
||||
th("Current branch")
|
||||
|
||||
html.diff_table(master_screens, current_screens)
|
||||
|
||||
return html.write(REPORTS_PATH / "diff", doc, test_name + ".html")
|
||||
|
||||
|
||||
def index() -> Path:
|
||||
removed = list((REPORTS_PATH / "removed").iterdir())
|
||||
added = list((REPORTS_PATH / "added").iterdir())
|
||||
diff = list((REPORTS_PATH / "diff").iterdir())
|
||||
|
||||
title = "UI changes from master"
|
||||
doc = dominate.document(title=title)
|
||||
|
||||
with doc:
|
||||
h1("UI changes from master")
|
||||
hr()
|
||||
|
||||
h2("Removed:", style="color: red;")
|
||||
i("UI fixtures that have been removed:")
|
||||
html.report_links(removed, REPORTS_PATH)
|
||||
br()
|
||||
hr()
|
||||
|
||||
h2("Added:", style="color: green;")
|
||||
i("UI fixtures that have been added:")
|
||||
html.report_links(added, REPORTS_PATH)
|
||||
br()
|
||||
hr()
|
||||
|
||||
h2("Differs:", style="color: grey;")
|
||||
i("UI fixtures that have been modified:")
|
||||
html.report_links(diff, REPORTS_PATH)
|
||||
|
||||
return html.write(REPORTS_PATH, doc, "index.html")
|
||||
|
||||
|
||||
def create_dirs() -> None:
|
||||
# delete the reports dir to clear previous entries and create folders
|
||||
shutil.rmtree(REPORTS_PATH, ignore_errors=True)
|
||||
REPORTS_PATH.mkdir()
|
||||
(REPORTS_PATH / "removed").mkdir()
|
||||
(REPORTS_PATH / "added").mkdir()
|
||||
(REPORTS_PATH / "diff").mkdir()
|
||||
|
||||
|
||||
def create_reports(test_prefixes: Sequence[str]) -> None:
|
||||
removed_tests, added_tests, diff_tests = get_diff(test_prefixes)
|
||||
|
||||
@contextmanager
|
||||
def tmpdir():
|
||||
with tempfile.TemporaryDirectory(prefix="trezor-records-") as temp_dir:
|
||||
yield Path(temp_dir)
|
||||
|
||||
for test_name, test_hash in removed_tests.items():
|
||||
with tmpdir() as temp_dir:
|
||||
download.fetch_recorded(test_hash, temp_dir)
|
||||
removed(temp_dir, test_name)
|
||||
|
||||
for test_name, test_hash in added_tests.items():
|
||||
path = RECORDED_SCREENS_PATH / test_name / "actual"
|
||||
if not path.exists():
|
||||
raise RuntimeError("Folder does not exist, has it been recorded?", path)
|
||||
added(path, test_name)
|
||||
|
||||
for test_name, (master_hash, current_hash) in diff_tests.items():
|
||||
with tmpdir() as master_screens:
|
||||
download.fetch_recorded(master_hash, master_screens)
|
||||
|
||||
current_screens = RECORDED_SCREENS_PATH / test_name / "actual"
|
||||
if not current_screens.exists():
|
||||
raise RuntimeError(
|
||||
"Folder does not exist, has it been recorded?", current_screens
|
||||
)
|
||||
diff(
|
||||
master_screens,
|
||||
current_screens,
|
||||
test_name,
|
||||
master_hash,
|
||||
current_hash,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
create_dirs()
|
||||
create_reports(sys.argv[1:] or [""])
|
||||
index()
|
||||
@ -1,7 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
import click
|
||||
|
||||
from ui_tests import update_fixtures_with_diff
|
||||
from ui_tests import update_fixtures
|
||||
|
||||
changes_amount = update_fixtures_with_diff()
|
||||
|
||||
print(f"{changes_amount} hashes updated in fixtures.json file.")
|
||||
@click.command()
|
||||
@click.option("-r", "--remove-missing", is_flag=True, help="Remove missing tests")
|
||||
def main(remove_missing: bool) -> None:
|
||||
"""Update fixtures file with results from latest test run."""
|
||||
changes_amount = update_fixtures(remove_missing)
|
||||
print(f"Updated fixtures.json with data from {changes_amount} tests.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Loading…
Reference in new issue