From 6cc2d4e7d2fea557088a9caa1ad1bd671adb5e97 Mon Sep 17 00:00:00 2001 From: Roman Zeyde Date: Wed, 16 Jul 2025 13:30:59 +0300 Subject: [PATCH] ci: add UI-specific diff markers for PR comment This way, we can quickly see if there are any UI diffs found by the tests (the first round icon) or there are any UI diffs from `main` (the second one). [no changelog] --- ci/make_pull_comment.py | 12 +++++++----- tests/ui_tests/reporting/common.py | 12 ++++++++++++ tests/ui_tests/reporting/failure-diff.png | Bin 0 -> 638 bytes tests/ui_tests/reporting/success-diff.png | Bin 0 -> 634 bytes tests/ui_tests/reporting/testreport.py | 10 +++++++++- 5 files changed, 28 insertions(+), 6 deletions(-) create mode 100644 tests/ui_tests/reporting/failure-diff.png create mode 100644 tests/ui_tests/reporting/success-diff.png diff --git a/ci/make_pull_comment.py b/ci/make_pull_comment.py index 88913fbb47..28f28217d9 100644 --- a/ci/make_pull_comment.py +++ b/ci/make_pull_comment.py @@ -30,7 +30,7 @@ def main(): def print_table(lang): main = f"[main]({REPORT_URL}/{lang}_index.html)" - screens = f"[screens]({REPORT_URL}/{lang}_diff.html)" + screens = f"[all]({REPORT_URL}/{lang}_diff.html)" print(f"\n# `{lang}` {main}({screens})\n") # Currently, persistence_test is not running with translations @@ -45,15 +45,17 @@ def print_table(lang): for test_type in test_types: test_prefix = f"{REPORT_URL}/{model}-{lang}-core_{test_type}" - img = f'' + job_img = f'' test_diff = f"[test]({test_prefix}-index.html)" - test_screens = f"[screens]({test_prefix}-differing_screens.html)" + test_screens = f"[all]({test_prefix}-differing_screens.html)" + test_img = f'' main_diff = f"[main]({test_prefix}-master_index.html)" - main_screens = f"[screens]({test_prefix}-master_diff.html)" + main_screens = f"[all]({test_prefix}-master_diff.html)" + main_img = f'' - cell = f"{img} {test_diff}({test_screens}) {main_diff}({main_screens})" + cell = f"{job_img} {test_diff}({test_screens}) {test_img} {main_diff}({main_screens}) {main_img}" row.append(cell) print("|".join(row)) diff --git a/tests/ui_tests/reporting/common.py b/tests/ui_tests/reporting/common.py index a7f652abae..121c39f5d5 100644 --- a/tests/ui_tests/reporting/common.py +++ b/tests/ui_tests/reporting/common.py @@ -1,3 +1,4 @@ +import shutil from pathlib import Path from typing import Any @@ -36,6 +37,15 @@ LEGACY_MODEL_NAMES = { } +def get_status_icon(diff: bool) -> Path: + status_icon = "success-diff.png" + if diff: + status_icon = "failure-diff.png" + result = HERE / status_icon + assert result.exists() + return result + + def generate_master_diff_report( diff_tests: dict[TestCase, tuple[str, str]], base_dir: Path ) -> None: @@ -212,6 +222,8 @@ def _differing_screens_report( i(testcase.id) html.write(base_dir, doc, "master_diff.html") + status_icon = get_status_icon(unique_differing_screens) + shutil.copy(status_icon, base_dir / "main_diff.png") def _get_unique_differing_screens( diff --git a/tests/ui_tests/reporting/failure-diff.png b/tests/ui_tests/reporting/failure-diff.png new file mode 100644 index 0000000000000000000000000000000000000000..07f01219d701a32bf4d494c416d66cf2ff6cc650 GIT binary patch literal 638 zcmeAS@N?(olHy`uVBq!ia0vp^A|TAc3?z4jzqJQa3dtTpz6=aiY77hwEes65fIPxkm;KfNz2{yMUY6?L-Toh9MGu~ljm}-Z=f=yo?{2Qh7ukPK{^R$! z+Mj=V>iF1#j-D1z+VJqnwv3S1%?p4=GA4PuyLfzHy7CIh;VkfoEM{QfI}E~%$MaXD z00r4gJbhi+U$P4d7+e2~Q3eUE@^o;ne*r!}+-W?hBv2S$`>63T)+fZD!H#O|bcR>SfPd%ktev zC*9Rsf6Z_E&ALO*waW7W1T@n0%--g_rhX%qj=`d)uXu%X(fa~42X OGkCiCxvXanMpdfpRr>`sfOLjp4W2??+;ZH!JwVp1HArhB!FH{RVIf%4B zEH+ydqof(Aso0sfYy18G#jS?DW!3*{($dep;b&m*bQjn?>CS6yEyoqkzVZ@V&dEeA zj_S0X_Huitv94(B`6X-9&dD&AcK%a|=HQ-hq#GK+sw2gxxe_dMpH^V}8x28?gQ)GY+mn?COC<)F_D=AMbN@eiPOfJeV2J#gA)6z1NQ|sTF R76KJ9c)I$ztaD0e0s#9C!;}C3 literal 0 HcmV?d00001 diff --git a/tests/ui_tests/reporting/testreport.py b/tests/ui_tests/reporting/testreport.py index 8e91184461..0f4e521a15 100644 --- a/tests/ui_tests/reporting/testreport.py +++ b/tests/ui_tests/reporting/testreport.py @@ -26,7 +26,13 @@ from dominate.util import text from ..common import FixturesType, TestCase, TestResult from . import download, html -from .common import REPORTS_PATH, document, generate_master_diff_report, get_diff +from .common import ( + REPORTS_PATH, + document, + generate_master_diff_report, + get_diff, + get_status_icon, +) TESTREPORT_PATH = REPORTS_PATH / "test" IMAGES_PATH = TESTREPORT_PATH / "images" @@ -240,6 +246,8 @@ def differing_screens() -> None: i(ui_failure.test.id) html.write(TESTREPORT_PATH, doc, "differing_screens.html") + status_icon = get_status_icon(recent_ui_failures) + shutil.copy(status_icon, TESTREPORT_PATH / "test_diff.png") def _get_current_results() -> FixturesType: