feat(tests): allow for saving text layout of all screens during UI tests

[no changelog]

core/Makefile

@@ -111,11 +111,11 @@ test_emu_click_ui: ## run click tests with UI testing
 
 test_emu_ui: ## run ui integration tests
 	$(EMU_TEST) $(PYTEST) $(TESTPATH)/device_tests $(TESTOPTS) \
-		--ui=test --ui-check-missing
+		--ui=test --ui-check-missing --record-text-layout
 
 test_emu_ui_multicore: ## run ui integration tests using multiple cores
 	$(PYTEST) -n auto $(TESTPATH)/device_tests $(TESTOPTS) \
-		--ui=test --ui-check-missing \
+		--ui=test --ui-check-missing --record-text-layout \
 		--control-emulators --model=core --random-order-seed=$(shell echo $$RANDOM)
 
 test_emu_ui_record: ## record and hash screens for ui integration tests

python/src/trezorlib/debuglink.py

@@ -197,6 +197,15 @@ class DebugLink:
         self.t1_screenshot_directory: Optional[Path] = None
         self.t1_screenshot_counter = 0
 
+        # Optional file for saving text representation of the screen
+        self.screen_text_file: Optional[Path] = None
+        self.last_screen_content = ""
+
+    def set_screen_text_file(self, file_path: Optional[Path]) -> None:
+        if file_path is not None:
+            file_path.write_bytes(b"")
+        self.screen_text_file = file_path
+
     def open(self) -> None:
         self.transport.begin_session()
 
@@ -292,12 +301,43 @@ class DebugLink:
         decision = messages.DebugLinkDecision(
             button=button, swipe=swipe, input=word, x=x, y=y, wait=wait, hold_ms=hold_ms
         )
+
         ret = self._call(decision, nowait=not wait)
         if ret is not None:
             return LayoutContent(ret.lines)
 
+        # Getting the current screen after the (nowait) decision
+        self.save_current_screen_if_relevant(wait=False)
+
         return None
 
+    def save_current_screen_if_relevant(self, wait: bool = True) -> None:
+        """Optionally saving the textual screen output."""
+        if self.screen_text_file is None:
+            return
+
+        if wait:
+            layout = self.wait_layout()
+        else:
+            layout = self.read_layout()
+        self.save_debug_screen(layout.lines)
+
+    def save_debug_screen(self, lines: List[str]) -> None:
+        if self.screen_text_file is None:
+            return
+
+        content = "\n".join(lines)
+
+        # Not writing the same screen twice
+        if content == self.last_screen_content:
+            return
+
+        self.last_screen_content = content
+
+        with open(self.screen_text_file, "a") as f:
+            f.write(content)
+            f.write("\n" + 80 * "/" + "\n")
+
     # Type overloads make sure that when we supply `wait=True` into `click()`,
     # it will always return `LayoutContent` and we do not need to assert `is not None`.
 
@@ -449,6 +489,12 @@ class DebugUI:
         self.debuglink.take_t1_screenshot_if_relevant()
 
         if self.input_flow is None:
+            # Only calling screen-saver when not in input-flow
+            # as it collides with wait-layout of input flows.
+            # All input flows call debuglink.input(), so
+            # recording their screens that way (as well as
+            # possible swipes below).
+            self.debuglink.save_current_screen_if_relevant(wait=True)
             if br.code == messages.ButtonRequestType.PinEntry:
                 self.debuglink.input(self.get_pin())
             else:
@@ -720,7 +766,6 @@ class TrezorClientDebugLink(TrezorClient):
     def __exit__(self, exc_type: Any, value: Any, traceback: Any) -> None:
         __tracebackhide__ = True  # for pytest # pylint: disable=W0612
 
-        self.watch_layout(False)
         # copy expected/actual responses before clearing them
         expected_responses = self.expected_responses
         actual_responses = self.actual_responses

tests/conftest.py

@@ -253,6 +253,7 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: pytest.ExitCode) -
             exitstatus,
             test_ui,  # type: ignore
             bool(session.config.getoption("ui_check_missing")),
+            bool(session.config.getoption("record_text_layout")),
         )
 
 
@@ -300,6 +301,13 @@ def pytest_addoption(parser: "Parser") -> None:
         help="Which emulator to use: 'core' or 'legacy'. "
         "Only valid in connection with `--control-emulators`",
     )
+    parser.addoption(
+        "--record-text-layout",
+        action="store_true",
+        default=False,
+        help="Saving debugging traces for each screen change. "
+        "Will generate a report with text from all test-cases. ",
+    )
 
 
 def pytest_configure(config: "Config") -> None:

tests/ui_tests/.gitignore

@@ -1,4 +1,5 @@
 *.png
 *.html
 *.zip
+*.txt
 fixtures.suggestion.json

tests/ui_tests/__init__.py

@@ -52,6 +52,8 @@ def screen_recording(
         yield
         return
 
+    record_text_layout = request.config.getoption("record_text_layout")
+
     testcase = TestCase.build(client, request)
     testcase.dir.mkdir(exist_ok=True, parents=True)
 
@@ -61,12 +63,18 @@ def screen_recording(
 
     try:
         client.debug.start_recording(str(testcase.actual_dir))
+        if record_text_layout:
+            client.debug.set_screen_text_file(testcase.screen_text_file)
+            client.debug.watch_layout(True)
         yield
     finally:
         client.ensure_open()
         # Wait for response to Initialize, which gives the emulator time to catch up
         # and redraw the homescreen. Otherwise there's a race condition between that
         # and stopping recording.
+        if record_text_layout:
+            client.debug.set_screen_text_file(None)
+            client.debug.watch_layout(False)
         client.init_device()
         client.debug.stop_recording()
 
@@ -156,12 +164,15 @@ def terminal_summary(
 
 
 def sessionfinish(
-    exitstatus: pytest.ExitCode, test_ui: str, check_missing: bool
+    exitstatus: pytest.ExitCode,
+    test_ui: str,
+    check_missing: bool,
+    record_text_layout: bool,
 ) -> pytest.ExitCode:
     if not _should_write_ui_report(exitstatus):
         return exitstatus
 
-    testreport.generate_reports()
+    testreport.generate_reports(record_text_layout)
     if test_ui == "test" and check_missing and list_missing():
         common.write_fixtures(
             TestResult.recent_results(),

tests/ui_tests/common.py

@@ -214,6 +214,10 @@ class TestCase:
     def dir(self) -> Path:
         return SCREENS_DIR / self.id
 
+    @property
+    def screen_text_file(self) -> Path:
+        return self.dir / "screens.txt"
+
     @property
     def actual_dir(self) -> Path:
         return self.dir / "actual"

tests/ui_tests/reporting/testreport.py

@@ -17,6 +17,7 @@ HERE = Path(__file__).resolve().parent
 REPORTS_PATH = UI_TESTS_DIR / "reports"
 TESTREPORT_PATH = REPORTS_PATH / "test"
 IMAGES_PATH = TESTREPORT_PATH / "images"
+SCREEN_TEXT_FILE = TESTREPORT_PATH / "screen_text.txt"
 
 STYLE = (HERE / "testreport.css").read_text()
 SCRIPT = (HERE / "testreport.js").read_text()
@@ -213,12 +214,43 @@ def all_unique_screens() -> Path:
     return html.write(TESTREPORT_PATH, doc, ALL_UNIQUE_SCREENS)
 
 
-def generate_reports() -> None:
+def screen_text_report() -> None:
+    """Generate a report with text representation of all screens."""
+    recent_results = list(TestResult.recent_results())
+
+    # Creating both a text file (suitable for offline usage)
+    # and an HTML file (suitable for online usage).
+
+    with open(SCREEN_TEXT_FILE, "w") as f2:
+        for result in recent_results:
+            if not result.test.screen_text_file.exists():
+                continue
+            f2.write(f"\n{result.test.id}\n")
+            with open(result.test.screen_text_file, "r") as f:
+                for line in f.readlines():
+                    f2.write(f"\t{line}")
+
+    doc = dominate.document(title="Screen text report")
+    with doc:
+        for result in recent_results:
+            if not result.test.screen_text_file.exists():
+                continue
+            with a(href=f"{ALL_SCREENS}#{result.test.id}"):
+                h2(result.test.id)
+            with open(result.test.screen_text_file, "r") as f:
+                for line in f.readlines():
+                    p(line)
+    html.write(TESTREPORT_PATH, doc, "screen_text.html")
+
+
+def generate_reports(do_screen_text: bool = False) -> None:
     """Generate HTML reports for the test."""
     html.set_image_dir(IMAGES_PATH)
     index()
     all_screens()
     all_unique_screens()
+    if do_screen_text:
+        screen_text_report()
 
 
 def _copy_deduplicated(test: TestCase) -> None: