diff --git a/core/Makefile b/core/Makefile index 17d4e4eb9c..20da39ebc0 100644 --- a/core/Makefile +++ b/core/Makefile @@ -111,7 +111,7 @@ test_emu_click: ## run click tests test_emu_ui: ## run ui integration tests $(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \ - --ui=test --ui-check-missing --not-generate-report-after-each-test + --ui=test --ui-check-missing --not-generate-report-after-each-test --record-text-layout test_emu_ui_multicore: ## run ui integration tests using multiple cores PYTEST_TIMEOUT=200 $(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) \ diff --git a/python/src/trezorlib/debuglink.py b/python/src/trezorlib/debuglink.py index 218f2c1cb8..e50d2fdb56 100644 --- a/python/src/trezorlib/debuglink.py +++ b/python/src/trezorlib/debuglink.py @@ -197,13 +197,14 @@ class DebugLink: self.t1_screenshot_directory: Optional[Path] = None self.t1_screenshot_counter = 0 - # Optional debug screen saver - self.debug_screen_file: Optional[Path] = None + # Optional file for saving text representation of the screen + self.screen_text_file: Optional[Path] = None self.last_screen_content = "" - def set_debug_screen_file(self, file_path: Path) -> None: - Path(file_path).write_bytes(b"") - self.debug_screen_file = file_path + def set_screen_text_file(self, file_path: Optional[Path]) -> None: + if file_path is not None: + Path(file_path).write_bytes(b"") + self.screen_text_file = file_path def open(self) -> None: self.transport.begin_session() @@ -310,22 +311,22 @@ class DebugLink: wait=wait, hold_ms=hold_ms, ) - ret = self._call(decision, nowait=not wait) - if ret is not None: - if self.debug_screen_file is not None: - self.save_debug_screen(ret.lines) - return LayoutContent(ret.lines) - if self.debug_screen_file is not None: + # Optionally saving the textual screen output + if self.screen_text_file is not None: layout = self.read_layout() self.save_debug_screen(layout.lines) + ret = self._call(decision, nowait=not wait) + if ret is not None: + return LayoutContent(ret.lines) + return None def save_debug_screen(self, lines: List[str]) -> None: - if self.debug_screen_file is not None: - if not self.debug_screen_file.exists(): - self.debug_screen_file.write_bytes(b"") + if self.screen_text_file is not None: + if not self.screen_text_file.exists(): + self.screen_text_file.write_bytes(b"") content = "\n".join(lines) @@ -335,7 +336,7 @@ class DebugLink: self.last_screen_content = content - with open(self.debug_screen_file, "a") as f: + with open(self.screen_text_file, "a") as f: f.write(content) f.write("\n" + 80 * "/" + "\n") diff --git a/tests/conftest.py b/tests/conftest.py index 400cf2359d..737175aca1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -268,18 +268,19 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) - missing = session.config.getoption("ui_check_missing") test_ui = session.config.getoption("ui") + record_text_layout = bool(session.config.getoption("record_text_layout")) if test_ui == "test": if missing and ui_tests.list_missing(): session.exitstatus = pytest.ExitCode.TESTS_FAILED ui_tests.write_fixtures_suggestion(missing) - testreport.generate_reports() + testreport.generate_reports(record_text_layout) elif test_ui == "record": if exitstatus == pytest.ExitCode.OK: ui_tests.write_fixtures(missing) else: ui_tests.write_fixtures_suggestion(missing, only_passed_tests=True) - testreport.generate_reports() + testreport.generate_reports(record_text_layout) def pytest_terminal_summary( @@ -360,6 +361,14 @@ def pytest_addoption(parser: "Parser") -> None: help="Not generating HTML reports after each test case. " "Useful for CI tests to speed them up.", ) + parser.addoption( + "--record-text-layout", + action="store_true", + default=False, + help="Saving debugging traces for each screen change. " + "Will generate a report with text from all test-cases. " + "WARNING: does not work well with multicore (causes freezing).", + ) def pytest_configure(config: "Config") -> None: diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py index d8a026f617..ee1fad614f 100644 --- a/tests/ui_tests/__init__.py +++ b/tests/ui_tests/__init__.py @@ -127,6 +127,7 @@ def screen_recording( client: Client, request: pytest.FixtureRequest ) -> Generator[None, None, None]: test_ui = request.config.getoption("ui") + record_text_layout = request.config.getoption("record_text_layout") test_name = get_test_name(request.node.nodeid) # Differentiating test names between T1 and TT @@ -138,23 +139,27 @@ def screen_recording( screens_test_path = SCREENS_DIR / test_name + # In which directory to save the screenshots if test_ui == "record": screen_path = screens_test_path / "recorded" else: screen_path = screens_test_path / "actual" + # Whether and where to save the text layout + if record_text_layout: + screen_text_file = screens_test_path / "screens.txt" + else: + screen_text_file = None + if not screens_test_path.exists(): screens_test_path.mkdir() # remove previous files shutil.rmtree(screen_path, ignore_errors=True) screen_path.mkdir() - # Start saving debugging trace into a file - debug_screen_file = screens_test_path / "screens.txt" - client.debug.set_debug_screen_file(debug_screen_file) - try: client.debug.start_recording(str(screen_path)) + client.debug.set_screen_text_file(screen_text_file) yield finally: # Wait for response to Initialize, which gives the emulator time to catch up @@ -162,6 +167,7 @@ def screen_recording( # and stopping recording. client.init_device() client.debug.stop_recording() + client.debug.set_screen_text_file(None) if test_ui: PROCESSED.add(test_name) diff --git a/tests/ui_tests/reporting/testreport.py b/tests/ui_tests/reporting/testreport.py index a6d8fa5633..88b5a5736e 100644 --- a/tests/ui_tests/reporting/testreport.py +++ b/tests/ui_tests/reporting/testreport.py @@ -214,7 +214,7 @@ def screen_text_report(test_case_dirs: List[Path]) -> None: f2.write(f"\t{line}") -def generate_reports() -> None: +def generate_reports(do_screen_text: bool = False) -> None: """Generate HTML reports for the test.""" index() @@ -223,7 +223,8 @@ def generate_reports() -> None: current_testcases = _get_testcases_dirs() all_screens(current_testcases) all_unique_screens(current_testcases) - screen_text_report(current_testcases) + if do_screen_text: + screen_text_report(current_testcases) def _img_hash(img: Path) -> str: