From b9dfd3a8d0770193c0b90b6190e58fb06b7b7619 Mon Sep 17 00:00:00 2001 From: grdddj Date: Wed, 18 Jan 2023 11:57:32 +0100 Subject: [PATCH] feat(tests): save text representation of all screens during UI tests --- core/Makefile | 2 +- python/src/trezorlib/debuglink.py | 32 ++++++++++++++++++++++++++ tests/conftest.py | 13 +++++++++-- tests/ui_tests/.gitignore | 1 + tests/ui_tests/__init__.py | 10 ++++++++ tests/ui_tests/reporting/testreport.py | 17 +++++++++++++- 6 files changed, 71 insertions(+), 4 deletions(-) diff --git a/core/Makefile b/core/Makefile index 17d4e4eb9c..20da39ebc0 100644 --- a/core/Makefile +++ b/core/Makefile @@ -111,7 +111,7 @@ test_emu_click: ## run click tests test_emu_ui: ## run ui integration tests $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \ - --ui=test --ui-check-missing --not-generate-report-after-each-test + --ui=test --ui-check-missing --not-generate-report-after-each-test --record-text-layout test_emu_ui_multicore: ## run ui integration tests using multiple cores PYTEST_TIMEOUT=200 $(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) \ diff --git a/python/src/trezorlib/debuglink.py b/python/src/trezorlib/debuglink.py index 470dc6257a..e50d2fdb56 100644 --- a/python/src/trezorlib/debuglink.py +++ b/python/src/trezorlib/debuglink.py @@ -197,6 +197,15 @@ class DebugLink: self.t1_screenshot_directory: Optional[Path] = None self.t1_screenshot_counter = 0 + # Optional file for saving text representation of the screen + self.screen_text_file: Optional[Path] = None + self.last_screen_content = "" + + def set_screen_text_file(self, file_path: Optional[Path]) -> None: + if file_path is not None: + Path(file_path).write_bytes(b"") + self.screen_text_file = file_path + def open(self) -> None: self.transport.begin_session() @@ -302,12 +311,35 @@ class DebugLink: wait=wait, hold_ms=hold_ms, ) + + # Optionally saving the textual screen output + if self.screen_text_file is not None: + layout = self.read_layout() + self.save_debug_screen(layout.lines) + ret = self._call(decision, nowait=not wait) if ret is not None: return LayoutContent(ret.lines) return None + def save_debug_screen(self, lines: List[str]) -> None: + if self.screen_text_file is not None: + if not self.screen_text_file.exists(): + self.screen_text_file.write_bytes(b"") + + content = "\n".join(lines) + + # Not writing the same screen twice + if content == self.last_screen_content: + return + + self.last_screen_content = content + + with open(self.screen_text_file, "a") as f: + f.write(content) + f.write("\n" + 80 * "/" + "\n") + # Type overloads make sure that when we supply `wait=True` into `click()`, # it will always return `LayoutContent` and we do not need to assert `is not None`. diff --git a/tests/conftest.py b/tests/conftest.py index 400cf2359d..737175aca1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -268,18 +268,19 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) - missing = session.config.getoption("ui_check_missing") test_ui = session.config.getoption("ui") + record_text_layout = bool(session.config.getoption("record_text_layout")) if test_ui == "test": if missing and ui_tests.list_missing(): session.exitstatus = pytest.ExitCode.TESTS_FAILED ui_tests.write_fixtures_suggestion(missing) - testreport.generate_reports() + testreport.generate_reports(record_text_layout) elif test_ui == "record": if exitstatus == pytest.ExitCode.OK: ui_tests.write_fixtures(missing) else: ui_tests.write_fixtures_suggestion(missing, only_passed_tests=True) - testreport.generate_reports() + testreport.generate_reports(record_text_layout) def pytest_terminal_summary( @@ -360,6 +361,14 @@ def pytest_addoption(parser: "Parser") -> None: help="Not generating HTML reports after each test case. " "Useful for CI tests to speed them up.", ) + parser.addoption( + "--record-text-layout", + action="store_true", + default=False, + help="Saving debugging traces for each screen change. " + "Will generate a report with text from all test-cases. " + "WARNING: does not work well with multicore (causes freezing).", + ) def pytest_configure(config: "Config") -> None: diff --git a/tests/ui_tests/.gitignore b/tests/ui_tests/.gitignore index 3f2756cebe..c50d2d2aa9 100644 --- a/tests/ui_tests/.gitignore +++ b/tests/ui_tests/.gitignore @@ -1,5 +1,6 @@ *.png *.html *.zip +*.txt fixtures.suggestion.json fixtures.json.diff diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py index eed6927291..ee1fad614f 100644 --- a/tests/ui_tests/__init__.py +++ b/tests/ui_tests/__init__.py @@ -127,6 +127,7 @@ def screen_recording( client: Client, request: pytest.FixtureRequest ) -> Generator[None, None, None]: test_ui = request.config.getoption("ui") + record_text_layout = request.config.getoption("record_text_layout") test_name = get_test_name(request.node.nodeid) # Differentiating test names between T1 and TT @@ -138,11 +139,18 @@ def screen_recording( screens_test_path = SCREENS_DIR / test_name + # In which directory to save the screenshots if test_ui == "record": screen_path = screens_test_path / "recorded" else: screen_path = screens_test_path / "actual" + # Whether and where to save the text layout + if record_text_layout: + screen_text_file = screens_test_path / "screens.txt" + else: + screen_text_file = None + if not screens_test_path.exists(): screens_test_path.mkdir() # remove previous files @@ -151,6 +159,7 @@ def screen_recording( try: client.debug.start_recording(str(screen_path)) + client.debug.set_screen_text_file(screen_text_file) yield finally: # Wait for response to Initialize, which gives the emulator time to catch up @@ -158,6 +167,7 @@ def screen_recording( # and stopping recording. client.init_device() client.debug.stop_recording() + client.debug.set_screen_text_file(None) if test_ui: PROCESSED.add(test_name) diff --git a/tests/ui_tests/reporting/testreport.py b/tests/ui_tests/reporting/testreport.py index 4c496a011f..88b5a5736e 100644 --- a/tests/ui_tests/reporting/testreport.py +++ b/tests/ui_tests/reporting/testreport.py @@ -16,6 +16,7 @@ from . import download, html HERE = Path(__file__).resolve().parent REPORTS_PATH = HERE / "reports" / "test" RECORDED_SCREENS_PATH = Path(__file__).resolve().parent.parent / "screens" +SCREEN_TEXT_FILE = REPORTS_PATH / "screen_text.txt" STYLE = (HERE / "testreport.css").read_text() SCRIPT = (HERE / "testreport.js").read_text() @@ -201,7 +202,19 @@ def all_unique_screens(test_case_dirs: List[Path]) -> Path: return html.write(REPORTS_PATH, doc, ALL_UNIQUE_SCREENS) -def generate_reports() -> None: +def screen_text_report(test_case_dirs: List[Path]) -> None: + with open(SCREEN_TEXT_FILE, "w") as f2: + for test_case_dir in test_case_dirs: + screen_file = test_case_dir / "screens.txt" + if not screen_file.exists(): + continue + f2.write(f"\n{test_case_dir.name}\n") + with open(screen_file, "r") as f: + for line in f.readlines(): + f2.write(f"\t{line}") + + +def generate_reports(do_screen_text: bool = False) -> None: """Generate HTML reports for the test.""" index() @@ -210,6 +223,8 @@ def generate_reports() -> None: current_testcases = _get_testcases_dirs() all_screens(current_testcases) all_unique_screens(current_testcases) + if do_screen_text: + screen_text_report(current_testcases) def _img_hash(img: Path) -> str: