diff --git a/ci/make_pull_comment.py b/ci/make_pull_comment.py index 88913fbb47..28f28217d9 100644 --- a/ci/make_pull_comment.py +++ b/ci/make_pull_comment.py @@ -30,7 +30,7 @@ def main(): def print_table(lang): main = f"[main]({REPORT_URL}/{lang}_index.html)" - screens = f"[screens]({REPORT_URL}/{lang}_diff.html)" + screens = f"[all]({REPORT_URL}/{lang}_diff.html)" print(f"\n# `{lang}` {main}({screens})\n") # Currently, persistence_test is not running with translations @@ -45,15 +45,17 @@ def print_table(lang): for test_type in test_types: test_prefix = f"{REPORT_URL}/{model}-{lang}-core_{test_type}" - img = f'' + job_img = f'' test_diff = f"[test]({test_prefix}-index.html)" - test_screens = f"[screens]({test_prefix}-differing_screens.html)" + test_screens = f"[all]({test_prefix}-differing_screens.html)" + test_img = f'' main_diff = f"[main]({test_prefix}-master_index.html)" - main_screens = f"[screens]({test_prefix}-master_diff.html)" + main_screens = f"[all]({test_prefix}-master_diff.html)" + main_img = f'' - cell = f"{img} {test_diff}({test_screens}) {main_diff}({main_screens})" + cell = f"{job_img} {test_diff}({test_screens}) {test_img} {main_diff}({main_screens}) {main_img}" row.append(cell) print("|".join(row)) diff --git a/tests/ui_tests/reporting/common.py b/tests/ui_tests/reporting/common.py index a7f652abae..121c39f5d5 100644 --- a/tests/ui_tests/reporting/common.py +++ b/tests/ui_tests/reporting/common.py @@ -1,3 +1,4 @@ +import shutil from pathlib import Path from typing import Any @@ -36,6 +37,15 @@ LEGACY_MODEL_NAMES = { } +def get_status_icon(diff: bool) -> Path: + status_icon = "success-diff.png" + if diff: + status_icon = "failure-diff.png" + result = HERE / status_icon + assert result.exists() + return result + + def generate_master_diff_report( diff_tests: dict[TestCase, tuple[str, str]], base_dir: Path ) -> None: @@ -212,6 +222,8 @@ def _differing_screens_report( i(testcase.id) html.write(base_dir, doc, "master_diff.html") + status_icon = get_status_icon(unique_differing_screens) + shutil.copy(status_icon, base_dir / "main_diff.png") def _get_unique_differing_screens( diff --git a/tests/ui_tests/reporting/failure-diff.png b/tests/ui_tests/reporting/failure-diff.png new file mode 100644 index 0000000000..07f01219d7 Binary files /dev/null and b/tests/ui_tests/reporting/failure-diff.png differ diff --git a/tests/ui_tests/reporting/success-diff.png b/tests/ui_tests/reporting/success-diff.png new file mode 100644 index 0000000000..7339d74c38 Binary files /dev/null and b/tests/ui_tests/reporting/success-diff.png differ diff --git a/tests/ui_tests/reporting/testreport.py b/tests/ui_tests/reporting/testreport.py index 8e91184461..0f4e521a15 100644 --- a/tests/ui_tests/reporting/testreport.py +++ b/tests/ui_tests/reporting/testreport.py @@ -26,7 +26,13 @@ from dominate.util import text from ..common import FixturesType, TestCase, TestResult from . import download, html -from .common import REPORTS_PATH, document, generate_master_diff_report, get_diff +from .common import ( + REPORTS_PATH, + document, + generate_master_diff_report, + get_diff, + get_status_icon, +) TESTREPORT_PATH = REPORTS_PATH / "test" IMAGES_PATH = TESTREPORT_PATH / "images" @@ -240,6 +246,8 @@ def differing_screens() -> None: i(ui_failure.test.id) html.write(TESTREPORT_PATH, doc, "differing_screens.html") + status_icon = get_status_icon(recent_ui_failures) + shutil.copy(status_icon, TESTREPORT_PATH / "test_diff.png") def _get_current_results() -> FixturesType: