arno/trezor-firmware
1
0
Fork 0
mirror of https://github.com/trezor/trezor-firmware.git synced 2024-11-21 15:08:12 +00:00
Code Issues Releases Wiki Activity

feat(tests): running master-diff report after each UI test

Browse Source 
[no changelog]
This commit is contained in:
grdddj 2023-06-27 14:36:11 +02:00 committed by Jiří Musil
parent eca1fc381b
commit 20c9d81018
9 changed files with 336 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ci/prepare_ui_artifacts.py
View File

@ -5,10 +5,11 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
# Needed for setup purposes, filling the FILE_HASHES dict
from tests.ui_tests.common import TestResult, _hash_files, get_fixtures # isort:skip
from tests.ui_tests.common import TestResult, _hash_files # isort:skip
from tests.ui_tests.common import get_current_fixtures # isort:skip
FIXTURES = get_fixtures()
FIXTURES = get_current_fixtures()
for result in TestResult.recent_results():
if not result.passed or result.expected_hash != result.actual_hash:

12
core/Makefile
View File

@ -115,26 +115,28 @@ test_emu_click: ## run click tests
$(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests $(TESTOPTS)
test_emu_click_ui: ## run click tests with UI testing
$(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests --ui=test --ui-check-missing $(TESTOPTS)
$(EMU_TEST) $(PYTEST) $(TESTPATH)/click_tests $(TESTOPTS) \
--ui=test --ui-check-missing --do-master-diff
test_emu_persistence: ## run persistence tests
$(PYTEST) $(TESTPATH)/persistence_tests $(TESTOPTS)
test_emu_persistence_ui: ## run persistence tests with UI testing
$(PYTEST) $(TESTPATH)/persistence_tests --ui=test --ui-check-missing $(TESTOPTS)
$(PYTEST) $(TESTPATH)/persistence_tests $(TESTOPTS) \
--ui=test --ui-check-missing --do-master-diff
test_emu_ui: ## run ui integration tests
$(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \
--ui=test --ui-check-missing --record-text-layout
--ui=test --ui-check-missing --record-text-layout --do-master-diff
test_emu_ui_multicore: ## run ui integration tests using multiple cores
$(PYTEST) -n $(MULTICORE) $(TESTPATH)/device_tests $(TESTOPTS) --timeout $(PYTEST_TIMEOUT) \
--ui=test --ui-check-missing --record-text-layout \
--ui=test --ui-check-missing --record-text-layout --do-master-diff \
--control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM)
test_emu_ui_record: ## record and hash screens for ui integration tests
$(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \
--ui=record --ui-check-missing
--ui=record --ui-check-missing --do-master-diff
test_emu_ui_record_multicore: ## quickly record all screens
make test_emu_ui_multicore || echo "All errors are recorded in fixtures.json"

10
tests/conftest.py
View File

@ -290,6 +290,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) -
test_ui, # type: ignore
bool(session.config.getoption("ui_check_missing")),
bool(session.config.getoption("record_text_layout")),
bool(session.config.getoption("do_master_diff")),
)
@ -342,7 +343,14 @@ def pytest_addoption(parser: "Parser") -> None:
action="store_true",
default=False,
help="Saving debugging traces for each screen change. "
"Will generate a report with text from all test-cases. ",
"Will generate a report with text from all test-cases.",
)
parser.addoption(
"--do-master-diff",
action="store_true",
default=False,
help="Generating a master-diff report. "
"This shows all unique differing screens compared to master.",
)

1
tests/ui_tests/.gitignore vendored
View File

@ -3,3 +3,4 @@
*.zip
*.txt
fixtures.suggestion.json
reporting/master_cache

3
tests/ui_tests/__init__.py
View File

@ -168,11 +168,12 @@ def sessionfinish(
test_ui: str,
check_missing: bool,
record_text_layout: bool,
do_master_diff: bool,
) -> pytest.ExitCode:
if not _should_write_ui_report(exitstatus):
return exitstatus
testreport.generate_reports(record_text_layout)
testreport.generate_reports(record_text_layout, do_master_diff)
if test_ui == "test" and check_missing and list_missing():
common.write_fixtures(
TestResult.recent_results(),

20
tests/ui_tests/common.py
View File

@ -37,7 +37,7 @@ FixturesType = t.NewType("FixturesType", "dict[str, dict[str, dict[str, str]]]")
FIXTURES: FixturesType = FixturesType({})
def get_fixtures() -> FixturesType:
def get_current_fixtures() -> FixturesType:
global FIXTURES
if not FIXTURES and FIXTURES_FILE.exists():
FIXTURES = FixturesType(json.loads(FIXTURES_FILE.read_text()))
@ -60,7 +60,7 @@ def prepare_fixtures(
missing_tests: set[TestCase] = set()
# merge with previous fixtures
fixtures = deepcopy(get_fixtures())
fixtures = deepcopy(get_current_fixtures())
for (model, group), new_content in grouped_tests.items():
# for every model/group, update the data with the new content
current_content = fixtures.setdefault(model, {}).setdefault(group, {})
@ -163,6 +163,20 @@ def _get_test_name_and_group(node_id: str) -> tuple[str, str]:
return shortened_name, group_name
def get_screen_path(test_name: str) -> Path | None:
path = SCREENS_DIR / test_name / "actual"
if path.exists():
return path
path = SCREENS_DIR / test_name / "recorded"
if path.exists():
print(
f"WARNING: no actual screens for {test_name}, recording may be outdated: {path}"
)
return path
print(f"WARNING: missing screens for {test_name}. Did the test run?")
return None
def screens_diff(
expected_hashes: list[str], actual_hashes: list[str]
) -> t.Iterator[tuple[str | None, str | None]]:
@ -258,7 +272,7 @@ class TestResult:
def __post_init__(self) -> None:
if self.expected_hash is None:
self.expected_hash = (
get_fixtures()
get_current_fixtures()
.get(self.test.model, {})
.get(self.test.group, {})
.get(self.test.fixtures_name)

239
tests/ui_tests/reporting/common.py Normal file
View File

@ -0,0 +1,239 @@
from pathlib import Path
from typing import Any
import dominate
import dominate.tags as t
from dominate.tags import a, h1, hr, i, p, table, td, th, tr
from ..common import (
UI_TESTS_DIR,
FixturesType,
get_screen_path,
screens_and_hashes,
screens_diff,
)
from . import download, html
HERE = Path(__file__).resolve().parent
STYLE = (HERE / "testreport.css").read_text()
SCRIPT = (HERE / "testreport.js").read_text()
GIF_SCRIPT = (HERE / "create-gif.js").read_text()
REPORTS_PATH = UI_TESTS_DIR / "reports"
# Saving the master screens on disk not to fetch them all the time
MASTER_CACHE_DIR = HERE / "master_cache"
if not MASTER_CACHE_DIR.exists():
MASTER_CACHE_DIR.mkdir()
def generate_master_diff_report(
diff_tests: dict[str, tuple[str, str]], base_dir: Path
) -> None:
unique_differing_screens = _get_unique_differing_screens(diff_tests, base_dir)
_differing_screens_report(unique_differing_screens, base_dir)
def get_diff(
current: FixturesType, print_to_console: bool = False
) -> tuple[dict[str, str], dict[str, str], dict[str, tuple[str, str]]]:
master = _get_preprocessed_master_fixtures()
removed = {}
added = {}
diff = {}
for model in master.keys() | current.keys():
master_groups = master.get(model, {})
current_groups = current.get(model, {})
for group in master_groups.keys() | current_groups.keys():
master_tests = master_groups.get(group, {})
current_tests = current_groups.get(group, {})
def testname(test: str) -> str:
assert test.startswith(model + "_")
test = test[len(model) + 1 :]
return f"{model}-{group}-{test}"
# removed items
removed_here = {
testname(test): master_tests[test]
for test in (master_tests.keys() - current_tests.keys())
}
# added items
added_here = {
testname(test): current_tests[test]
for test in (current_tests.keys() - master_tests.keys())
}
# create the diff from items in both branches
diff_here = {}
for master_test, master_hash in master_tests.items():
full_test_name = testname(master_test)
if full_test_name in removed_here:
continue
if current_tests.get(master_test) == master_hash:
continue
diff_here[full_test_name] = (
master_tests[master_test],
current_tests[master_test],
)
removed.update(removed_here)
added.update(added_here)
diff.update(diff_here)
if print_to_console:
print(f"{model} {group}")
print(f" removed: {len(removed_here)}")
print(f" added: {len(added_here)}")
print(f" diff: {len(diff_here)}")
return removed, added, diff
def document(
title: str,
actual_hash: str | None = None,
index: bool = False,
model: str | None = None,
) -> dominate.document:
doc = dominate.document(title=title)
style = t.style()
style.add_raw_string(STYLE)
script = t.script()
script.add_raw_string(GIF_SCRIPT)
script.add_raw_string(SCRIPT)
doc.head.add(style, script)
if actual_hash is not None:
doc.body["data-actual-hash"] = actual_hash
if index:
doc.body["data-index"] = True
if model:
doc.body["class"] = f"model-{model}"
return doc
def _preprocess_master_compat(master_fixtures: dict[str, Any]) -> FixturesType:
if all(isinstance(v, str) for v in master_fixtures.values()):
# old format, convert to new format
new_fixtures = {}
for key, val in master_fixtures.items():
model, _test = key.split("_", maxsplit=1)
groups_by_model = new_fixtures.setdefault(model, {})
default_group = groups_by_model.setdefault("device_tests", {})
default_group[key] = val
return FixturesType(new_fixtures)
else:
return FixturesType(master_fixtures)
def _get_preprocessed_master_fixtures() -> FixturesType:
return _preprocess_master_compat(download.fetch_fixtures_master())
def _create_testcase_html_diff_file(
zipped_screens: list[tuple[str | None, str | None]],
test_name: str,
master_hash: str,
current_hash: str,
base_dir: Path,
) -> Path:
doc = document(title=test_name, model=test_name[:2])
with doc:
h1(test_name)
p("This UI test differs from master.", style="color: grey; font-weight: bold;")
with table():
with tr():
td("Master:")
td(master_hash, style="color: red;")
with tr():
td("Current:")
td(current_hash, style="color: green;")
hr()
with table(border=1, width=600):
with tr():
th("Master")
th("Current branch")
html.diff_table(zipped_screens, base_dir / "diff")
return html.write(base_dir / "diff", doc, test_name + ".html")
def _differing_screens_report(
unique_differing_screens: dict[tuple[str | None, str | None], str], base_dir: Path
) -> None:
try:
model = next(iter(unique_differing_screens.values()))[:2]
except StopIteration:
model = ""
doc = document(title="Master differing screens", model=model)
with doc:
with table(border=1, width=600):
with tr():
th("Expected")
th("Actual")
th("Testcase (link)")
for (master, current), testcase in unique_differing_screens.items():
with tr(bgcolor="red"):
html.image_column(master, base_dir)
html.image_column(current, base_dir)
with td():
with a(href=f"diff/{testcase}.html"):
i(testcase)
html.write(base_dir, doc, "master_diff.html")
def _get_unique_differing_screens(
diff_tests: dict[str, tuple[str, str]], base_dir: Path
) -> dict[tuple[str | None, str | None], str]:
# Holding unique screen differences, connected with a certain testcase
# Used for diff report
unique_differing_screens: dict[tuple[str | None, str | None], str] = {}
for test_name, (master_hash, current_hash) in diff_tests.items():
# Downloading master recordings only if we do not have them already
master_screens_path = MASTER_CACHE_DIR / master_hash
if not master_screens_path.exists():
master_screens_path.mkdir()
try:
download.fetch_recorded(master_hash, master_screens_path)
except RuntimeError as e:
print("WARNING:", e)
current_screens_path = get_screen_path(test_name)
if not current_screens_path:
current_screens_path = MASTER_CACHE_DIR / "empty_current_screens"
current_screens_path.mkdir()
# Saving all the images to a common directory
# They will be referenced from the HTML files
master_screens, master_hashes = screens_and_hashes(master_screens_path)
current_screens, current_hashes = screens_and_hashes(current_screens_path)
html.store_images(master_screens, master_hashes)
html.store_images(current_screens, current_hashes)
# List of tuples of master and current screens
# Useful for both testcase HTML report and the differing screen report
zipped_screens = list(screens_diff(master_hashes, current_hashes))
# Create testcase HTML report
_create_testcase_html_diff_file(
zipped_screens, test_name, master_hash, current_hash, base_dir
)
# Save differing screens for differing screens report
for master, current in zipped_screens:
if master != current:
unique_differing_screens[(master, current)] = test_name
return unique_differing_screens

197
tests/ui_tests/reporting/master_diff.py
View File

@ -4,92 +4,17 @@ import shutil
import tempfile
from contextlib import contextmanager
from pathlib import Path
from typing import Any
from dominate.tags import a, br, h1, h2, hr, i, p, table, td, th, tr
from dominate.tags import br, h1, h2, hr, i, p, table, th, tr
from ..common import (
SCREENS_DIR,
FixturesType,
get_fixtures,
screens_and_hashes,
screens_diff,
)
from ..common import get_current_fixtures, get_screen_path, screens_and_hashes
from . import download, html
from .testreport import REPORTS_PATH, document
from .common import REPORTS_PATH, document, generate_master_diff_report, get_diff
MASTERDIFF_PATH = REPORTS_PATH / "master_diff"
IMAGES_PATH = MASTERDIFF_PATH / "images"
def _preprocess_master_compat(master_fixtures: dict[str, Any]) -> FixturesType:
if all(isinstance(v, str) for v in master_fixtures.values()):
# old format, convert to new format
new_fixtures = {}
for key, val in master_fixtures.items():
model, _test = key.split("_", maxsplit=1)
groups_by_model = new_fixtures.setdefault(model, {})
default_group = groups_by_model.setdefault("device_tests", {})
default_group[key] = val
return FixturesType(new_fixtures)
else:
return FixturesType(master_fixtures)
def get_diff() -> tuple[dict[str, str], dict[str, str], dict[str, tuple[str, str]]]:
master = _preprocess_master_compat(download.fetch_fixtures_master())
current = get_fixtures()
removed = {}
added = {}
diff = {}
for model in master.keys() | current.keys():
master_groups = master.get(model, {})
current_groups = current.get(model, {})
for group in master_groups.keys() | current_groups.keys():
master_tests = master_groups.get(group, {})
current_tests = current_groups.get(group, {})
def testname(test: str) -> str:
assert test.startswith(model + "_")
test = test[len(model) + 1 :]
return f"{model}-{group}-{test}"
# removed items
removed_here = {
testname(test): master_tests[test]
for test in (master_tests.keys() - current_tests.keys())
}
# added items
added_here = {
testname(test): current_tests[test]
for test in (current_tests.keys() - master_tests.keys())
}
# create the diff from items in both branches
diff_here = {}
for master_test, master_hash in master_tests.items():
full_test_name = testname(master_test)
if full_test_name in removed_here:
continue
if current_tests.get(master_test) == master_hash:
continue
diff_here[full_test_name] = (
master_tests[master_test],
current_tests[master_test],
)
removed.update(removed_here)
added.update(added_here)
diff.update(diff_here)
print(f"{model} {group}")
print(f" removed: {len(removed_here)}")
print(f" added: {len(added_here)}")
print(f" diff: {len(diff_here)}")
return removed, added, diff
def removed(screens_path: Path, test_name: str) -> Path:
doc = document(title=test_name, model=test_name[:2])
screens, hashes = screens_and_hashes(screens_path)
@ -138,35 +63,6 @@ def added(screens_path: Path, test_name: str) -> Path:
return html.write(MASTERDIFF_PATH / "added", doc, test_name + ".html")
def create_testcase_html_diff_file(
zipped_screens: list[tuple[str | None, str | None]],
test_name: str,
master_hash: str,
current_hash: str,
) -> Path:
doc = document(title=test_name, model=test_name[:2])
with doc:
h1(test_name)
p("This UI test differs from master.", style="color: grey; font-weight: bold;")
with table():
with tr():
td("Master:")
td(master_hash, style="color: red;")
with tr():
td("Current:")
td(current_hash, style="color: green;")
hr()
with table(border=1, width=600):
with tr():
th("Master")
th("Current branch")
html.diff_table(zipped_screens, MASTERDIFF_PATH / "diff")
return html.write(MASTERDIFF_PATH / "diff", doc, test_name + ".html")
def index() -> Path:
removed = list((MASTERDIFF_PATH / "removed").iterdir())
added = list((MASTERDIFF_PATH / "added").iterdir())
@ -176,7 +72,7 @@ def index() -> Path:
doc = document(title=title)
with doc:
h1("UI changes from master")
h1(title)
hr()
h2("Removed:", style="color: red;")
@ -208,22 +104,9 @@ def create_dirs() -> None:
IMAGES_PATH.mkdir(exist_ok=True)
def _get_screen_path(test_name: str) -> Path | None:
path = SCREENS_DIR / test_name / "actual"
if path.exists():
return path
path = SCREENS_DIR / test_name / "recorded"
if path.exists():
print(
f"WARNING: no actual screens for {test_name}, recording may be outdated: {path}"
)
return path
print(f"WARNING: missing screens for {test_name}. Did the test run?")
return None
def create_reports() -> None:
removed_tests, added_tests, diff_tests = get_diff()
current = get_current_fixtures()
removed_tests, added_tests, diff_tests = get_diff(current, print_to_console=True)
@contextmanager
def tmpdir():
@ -240,76 +123,12 @@ def create_reports() -> None:
removed(temp_dir, test_name)
for test_name, test_hash in added_tests.items():
screen_path = _get_screen_path(test_name)
screen_path = get_screen_path(test_name)
if not screen_path:
continue
added(screen_path, test_name)
# Holding unique screen differences, connected with a certain testcase
# Used for diff report
unique_differing_screens: dict[tuple[str | None, str | None], str] = {}
for test_name, (master_hash, current_hash) in diff_tests.items():
with tmpdir() as master_root:
master_screens_path = master_root / "downloaded"
master_screens_path.mkdir()
try:
download.fetch_recorded(master_hash, master_screens_path)
except RuntimeError as e:
print("WARNING:", e)
current_screens_path = _get_screen_path(test_name)
if not current_screens_path:
current_screens_path = master_root / "empty_current_screens"
current_screens_path.mkdir()
# Saving all the images to a common directory
# They will be referenced from the HTML files
master_screens, master_hashes = screens_and_hashes(master_screens_path)
current_screens, current_hashes = screens_and_hashes(current_screens_path)
html.store_images(master_screens, master_hashes)
html.store_images(current_screens, current_hashes)
# List of tuples of master and current screens
# Useful for both testcase HTML report and the differing screen report
zipped_screens = list(screens_diff(master_hashes, current_hashes))
# Create testcase HTML report
create_testcase_html_diff_file(
zipped_screens,
test_name,
master_hash,
current_hash,
)
# Save differing screens for differing screens report
for master, current in zipped_screens:
if master != current:
unique_differing_screens[(master, current)] = test_name
differing_screens_report(unique_differing_screens)
def differing_screens_report(
unique_differing_screens: dict[tuple[str | None, str | None], str]
) -> None:
doc = document(title="Master differing screens")
with doc:
with table(border=1, width=600):
with tr():
th("Expected")
th("Actual")
th("Testcase (link)")
for (master, current), testcase in unique_differing_screens.items():
with tr(bgcolor="red"):
html.image_column(master, MASTERDIFF_PATH)
html.image_column(current, MASTERDIFF_PATH)
with td():
with a(href=f"diff/{testcase}.html"):
i(testcase)
html.write(MASTERDIFF_PATH, doc, "differing_screens.html")
generate_master_diff_report(diff_tests, MASTERDIFF_PATH)
def main() -> None:

85
tests/ui_tests/reporting/testreport.py
View File

@ -10,50 +10,19 @@ import dominate.tags as t
from dominate.tags import a, div, h1, h2, hr, i, p, span, strong, table, td, th, tr
from dominate.util import text
from ..common import UI_TESTS_DIR, TestCase, TestResult
from ..common import FixturesType, TestCase, TestResult
from . import download, html
from .common import REPORTS_PATH, document, generate_master_diff_report, get_diff
HERE = Path(__file__).resolve().parent
REPORTS_PATH = UI_TESTS_DIR / "reports"
TESTREPORT_PATH = REPORTS_PATH / "test"
IMAGES_PATH = TESTREPORT_PATH / "images"
SCREEN_TEXT_FILE = TESTREPORT_PATH / "screen_text.txt"
STYLE = (HERE / "testreport.css").read_text()
SCRIPT = (HERE / "testreport.js").read_text()
GIF_SCRIPT = (HERE / "create-gif.js").read_text()
# These two html files are referencing each other
ALL_SCREENS = "all_screens.html"
ALL_UNIQUE_SCREENS = "all_unique_screens.html"
def document(
title: str,
actual_hash: str | None = None,
index: bool = False,
model: str | None = None,
) -> dominate.document:
doc = dominate.document(title=title)
style = t.style()
style.add_raw_string(STYLE)
script = t.script()
script.add_raw_string(GIF_SCRIPT)
script.add_raw_string(SCRIPT)
doc.head.add(style, script)
if actual_hash is not None:
doc.body["data-actual-hash"] = actual_hash
if index:
doc.body["data-index"] = True
if model:
doc.body["class"] = f"model-{model}"
return doc
def _header(test_name: str, expected_hash: str | None, actual_hash: str) -> None:
h1(test_name)
with div():
@ -85,6 +54,7 @@ def setup(main_runner: bool) -> None:
(TESTREPORT_PATH / "failed").mkdir()
(TESTREPORT_PATH / "passed").mkdir()
(TESTREPORT_PATH / "new").mkdir()
(TESTREPORT_PATH / "diff").mkdir()
IMAGES_PATH.mkdir(parents=True)
html.set_image_dir(IMAGES_PATH)
@ -282,15 +252,60 @@ def differing_screens() -> None:
html.write(TESTREPORT_PATH, doc, "differing_screens.html")
def generate_reports(do_screen_text: bool = False) -> None:
def _get_current_results() -> FixturesType:
current: FixturesType = {} # type: ignore
for res in TestResult.recent_results():
model = res.test.model
group = res.test.group
fixtures_name = res.test.fixtures_name
actual_hash = res.actual_hash
if model not in current:
current[model] = {}
if group not in current[model]:
current[model][group] = {}
current[model][group][fixtures_name] = actual_hash
return current
def master_diff() -> None:
"""Creating an HTML page showing all screens differing from master."""
current = _get_current_results()
_removed_tests, _added_tests, diff_tests = get_diff(current)
generate_master_diff_report(diff_tests, TESTREPORT_PATH)
def master_index() -> Path:
"""Shows all the differing tests from master."""
diff = list((TESTREPORT_PATH / "diff").iterdir())
title = "UI changes from master"
doc = document(title=title)
with doc:
h1(title)
hr()
h2("Differs:", style="color: grey;")
i("UI fixtures that have been modified:")
html.report_links(diff, TESTREPORT_PATH)
return html.write(TESTREPORT_PATH, doc, "master_index.html")
def generate_reports(
do_screen_text: bool = False, do_master_diff: bool = False
) -> None:
"""Generate HTML reports for the test."""
html.set_image_dir(IMAGES_PATH)
index()
all_screens()
all_unique_screens()
differing_screens()
if do_screen_text:
screen_text_report()
differing_screens()
if do_master_diff:
master_diff()
master_index()
def _copy_deduplicated(test: TestCase) -> None: