diff --git a/core/Makefile b/core/Makefile index f3d9fbbbb..3045d5141 100644 --- a/core/Makefile +++ b/core/Makefile @@ -111,11 +111,11 @@ test_emu_click_ui: ## run click tests with UI testing test_emu_ui: ## run ui integration tests $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \ - --ui=test --ui-check-missing + --ui=test --ui-check-missing --record-text-layout test_emu_ui_multicore: ## run ui integration tests using multiple cores $(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) \ - --ui=test --ui-check-missing \ + --ui=test --ui-check-missing --record-text-layout \ --control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM) test_emu_ui_record: ## record and hash screens for ui integration tests diff --git a/python/src/trezorlib/debuglink.py b/python/src/trezorlib/debuglink.py index c9ef48c5b..5ad51775a 100644 --- a/python/src/trezorlib/debuglink.py +++ b/python/src/trezorlib/debuglink.py @@ -197,6 +197,15 @@ class DebugLink: self.t1_screenshot_directory: Optional[Path] = None self.t1_screenshot_counter = 0 + # Optional file for saving text representation of the screen + self.screen_text_file: Optional[Path] = None + self.last_screen_content = "" + + def set_screen_text_file(self, file_path: Optional[Path]) -> None: + if file_path is not None: + file_path.write_bytes(b"") + self.screen_text_file = file_path + def open(self) -> None: self.transport.begin_session() @@ -292,12 +301,43 @@ class DebugLink: decision = messages.DebugLinkDecision( button=button, swipe=swipe, input=word, x=x, y=y, wait=wait, hold_ms=hold_ms ) + ret = self._call(decision, nowait=not wait) if ret is not None: return LayoutContent(ret.lines) + # Getting the current screen after the (nowait) decision + self.save_current_screen_if_relevant(wait=False) + return None + def save_current_screen_if_relevant(self, wait: bool = True) -> None: + """Optionally saving the textual screen output.""" + if self.screen_text_file is None: + return + + if wait: + layout = self.wait_layout() + else: + layout = self.read_layout() + self.save_debug_screen(layout.lines) + + def save_debug_screen(self, lines: List[str]) -> None: + if self.screen_text_file is None: + return + + content = "\n".join(lines) + + # Not writing the same screen twice + if content == self.last_screen_content: + return + + self.last_screen_content = content + + with open(self.screen_text_file, "a") as f: + f.write(content) + f.write("\n" + 80 * "/" + "\n") + # Type overloads make sure that when we supply `wait=True` into `click()`, # it will always return `LayoutContent` and we do not need to assert `is not None`. @@ -449,6 +489,12 @@ class DebugUI: self.debuglink.take_t1_screenshot_if_relevant() if self.input_flow is None: + # Only calling screen-saver when not in input-flow + # as it collides with wait-layout of input flows. + # All input flows call debuglink.input(), so + # recording their screens that way (as well as + # possible swipes below). + self.debuglink.save_current_screen_if_relevant(wait=True) if br.code == messages.ButtonRequestType.PinEntry: self.debuglink.input(self.get_pin()) else: @@ -720,7 +766,6 @@ class TrezorClientDebugLink(TrezorClient): def __exit__(self, exc_type: Any, value: Any, traceback: Any) -> None: __tracebackhide__ = True # for pytest # pylint: disable=W0612 - self.watch_layout(False) # copy expected/actual responses before clearing them expected_responses = self.expected_responses actual_responses = self.actual_responses diff --git a/tests/conftest.py b/tests/conftest.py index 66d704b43..abd364b97 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -253,6 +253,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) - exitstatus, test_ui, # type: ignore bool(session.config.getoption("ui_check_missing")), + bool(session.config.getoption("record_text_layout")), ) @@ -300,6 +301,13 @@ def pytest_addoption(parser: "Parser") -> None: help="Which emulator to use: 'core' or 'legacy'. " "Only valid in connection with `--control-emulators`", ) + parser.addoption( + "--record-text-layout", + action="store_true", + default=False, + help="Saving debugging traces for each screen change. " + "Will generate a report with text from all test-cases. ", + ) def pytest_configure(config: "Config") -> None: diff --git a/tests/ui_tests/.gitignore b/tests/ui_tests/.gitignore index 047e34100..5e3fedad2 100644 --- a/tests/ui_tests/.gitignore +++ b/tests/ui_tests/.gitignore @@ -1,4 +1,5 @@ *.png *.html *.zip +*.txt fixtures.suggestion.json diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py index 7a80a12d2..5a3762bb6 100644 --- a/tests/ui_tests/__init__.py +++ b/tests/ui_tests/__init__.py @@ -52,6 +52,8 @@ def screen_recording( yield return + record_text_layout = request.config.getoption("record_text_layout") + testcase = TestCase.build(client, request) testcase.dir.mkdir(exist_ok=True, parents=True) @@ -61,12 +63,18 @@ def screen_recording( try: client.debug.start_recording(str(testcase.actual_dir)) + if record_text_layout: + client.debug.set_screen_text_file(testcase.screen_text_file) + client.debug.watch_layout(True) yield finally: client.ensure_open() # Wait for response to Initialize, which gives the emulator time to catch up # and redraw the homescreen. Otherwise there's a race condition between that # and stopping recording. + if record_text_layout: + client.debug.set_screen_text_file(None) + client.debug.watch_layout(False) client.init_device() client.debug.stop_recording() @@ -156,12 +164,15 @@ def terminal_summary( def sessionfinish( - exitstatus: pytest.ExitCode, test_ui: str, check_missing: bool + exitstatus: pytest.ExitCode, + test_ui: str, + check_missing: bool, + record_text_layout: bool, ) -> pytest.ExitCode: if not _should_write_ui_report(exitstatus): return exitstatus - testreport.generate_reports() + testreport.generate_reports(record_text_layout) if test_ui == "test" and check_missing and list_missing(): common.write_fixtures( TestResult.recent_results(), diff --git a/tests/ui_tests/common.py b/tests/ui_tests/common.py index cef677e21..396c3e3d6 100644 --- a/tests/ui_tests/common.py +++ b/tests/ui_tests/common.py @@ -214,6 +214,10 @@ class TestCase: def dir(self) -> Path: return SCREENS_DIR / self.id + @property + def screen_text_file(self) -> Path: + return self.dir / "screens.txt" + @property def actual_dir(self) -> Path: return self.dir / "actual" diff --git a/tests/ui_tests/reporting/testreport.py b/tests/ui_tests/reporting/testreport.py index f6d8ff3e3..f22744d72 100644 --- a/tests/ui_tests/reporting/testreport.py +++ b/tests/ui_tests/reporting/testreport.py @@ -17,6 +17,7 @@ HERE = Path(__file__).resolve().parent REPORTS_PATH = UI_TESTS_DIR / "reports" TESTREPORT_PATH = REPORTS_PATH / "test" IMAGES_PATH = TESTREPORT_PATH / "images" +SCREEN_TEXT_FILE = TESTREPORT_PATH / "screen_text.txt" STYLE = (HERE / "testreport.css").read_text() SCRIPT = (HERE / "testreport.js").read_text() @@ -213,12 +214,43 @@ def all_unique_screens() -> Path: return html.write(TESTREPORT_PATH, doc, ALL_UNIQUE_SCREENS) -def generate_reports() -> None: +def screen_text_report() -> None: + """Generate a report with text representation of all screens.""" + recent_results = list(TestResult.recent_results()) + + # Creating both a text file (suitable for offline usage) + # and an HTML file (suitable for online usage). + + with open(SCREEN_TEXT_FILE, "w") as f2: + for result in recent_results: + if not result.test.screen_text_file.exists(): + continue + f2.write(f"\n{result.test.id}\n") + with open(result.test.screen_text_file, "r") as f: + for line in f.readlines(): + f2.write(f"\t{line}") + + doc = dominate.document(title="Screen text report") + with doc: + for result in recent_results: + if not result.test.screen_text_file.exists(): + continue + with a(href=f"{ALL_SCREENS}#{result.test.id}"): + h2(result.test.id) + with open(result.test.screen_text_file, "r") as f: + for line in f.readlines(): + p(line) + html.write(TESTREPORT_PATH, doc, "screen_text.html") + + +def generate_reports(do_screen_text: bool = False) -> None: """Generate HTML reports for the test.""" html.set_image_dir(IMAGES_PATH) index() all_screens() all_unique_screens() + if do_screen_text: + screen_text_report() def _copy_deduplicated(test: TestCase) -> None: