chore(tests): improve UI reports

tests/ui_tests/reporting/create-gif.js

@@ -0,0 +1,209 @@
+
+
+function createGif() {
+    // Finds all the screenshots on the screen, creates a new img
+    // element at the top and switches the src attribute every 200ms
+    // to create a notion of GIF.
+    // Adds some controlling possibilities - buttons, input fields
+    // and sliders to enable pausing, stepping back and forth, changing
+    // the delay, etc.
+
+    const allImages = document.body.querySelectorAll('img:not(#gif)');
+
+    // When no images there, do nothing
+    if (allImages.length === 0) {
+        return;
+    }
+
+    // Globals that will be changed by individual functions
+    let globCurrentIndex = 0;
+    let globTimerId = null;
+
+    // Global constants
+    const pauseText = 'Pause (Space)';
+    const continueText = 'Continue (Space)';
+    const prevText = 'Prev (<)';
+    const nextText = 'Next (>)';
+
+    const delayText = 'Delay (ms):';
+    const sliderText = 'Progress:';
+
+    const defaultDelay = 200;
+
+    const keyboardShortcutPrev = 'ArrowLeft';
+    const keyboardShortcutNext = 'ArrowRight';
+    const keyboardShortcutPauseContinue = 'Space';
+
+    const pauseColor = '#ffa500'; // Orange
+    const continueColor = '#4CAF50'; // Green
+
+    const btnClass = 'gifBtn';
+
+    // Gif itself
+    const gif = document.createElement('img');
+    gif.id = 'gif';
+
+    // Update the image source and the slider value according to the current index
+    // Lazy-loading all the lazy-loaded images
+    function updateGifSourceAndSlider() {
+        const currentImage = allImages[globCurrentIndex];
+        // When the currentImage is not loaded (because of `loading=lazy` attribute), load it
+        if (!currentImage.complete) {
+            const tempImg = new Image();
+            tempImg.src = currentImage.src;
+            tempImg.onload = function () {
+                currentImage.src = tempImg.src;
+            };
+        }
+        gif.src = currentImage.src;
+        slider.value = globCurrentIndex;
+    }
+
+    // Switching between running and paused state
+    function toggleGif() {
+        if (globTimerId) {
+            clearInterval(globTimerId);
+            globTimerId = null;
+            pauseContinueButton.textContent = continueText;
+            pauseContinueButton.style.backgroundColor = continueColor;
+            delayInput.disabled = false;
+            prevButton.disabled = false;
+            nextButton.disabled = false;
+        } else {
+            pauseContinueButton.textContent = pauseText;
+            pauseContinueButton.style.backgroundColor = pauseColor;
+            delayInput.disabled = true;
+            prevButton.disabled = true;
+            nextButton.disabled = true;
+            globTimerId = runGif();
+        }
+    }
+
+    // Start the gif, return the timer id
+    function runGif() {
+        const delay = parseInt(delayInput.value) || defaultDelay;
+        return setInterval(() => {
+            changeGifFrame(1);
+        }, delay);
+    }
+
+    // Go to the previous or next frame (when supplied with -1 or 1, respectively)
+    function changeGifFrame(delta) {
+        globCurrentIndex = (globCurrentIndex + delta + allImages.length) % allImages.length;
+        updateGifSourceAndSlider();
+    }
+
+    // Pause/continue button
+    const pauseContinueButton = document.createElement('button');
+    pauseContinueButton.id = 'pauseContinueButton';
+    pauseContinueButton.classList.add(btnClass);
+    pauseContinueButton.textContent = pauseText;
+    pauseContinueButton.style.backgroundColor = pauseColor;
+    pauseContinueButton.addEventListener('click', toggleGif);
+
+    // Prev button
+    const prevButton = document.createElement('button');
+    prevButton.id = 'prevButton';
+    prevButton.textContent = prevText;
+    prevButton.classList.add(btnClass);
+    prevButton.disabled = true; // Disabled until the gif is paused
+    prevButton.addEventListener('click', () => changeGifFrame(-1));
+
+    // Next button
+    const nextButton = document.createElement('button');
+    nextButton.id = 'nextButton';
+    nextButton.textContent = nextText;
+    nextButton.classList.add(btnClass);
+    nextButton.disabled = true; // Disabled until the gif is paused
+    nextButton.addEventListener('click', () => changeGifFrame(1));
+
+    // Delay label
+    const delayLabel = document.createElement('label');
+    delayLabel.id = 'delayLabel';
+    delayLabel.textContent = delayText;
+    delayLabel.htmlFor = 'delayInput';
+
+    // Delay input
+    const delayInput = document.createElement('input');
+    delayInput.id = 'delayInput';
+    delayInput.type = 'number';
+    delayInput.value = defaultDelay;
+    delayInput.size = '5';
+    delayInput.disabled = true; // Disabled until the gif is paused
+
+    // Slider label
+    const sliderLabel = document.createElement('label');
+    sliderLabel.id = 'sliderLabel';
+    sliderLabel.textContent = sliderText;
+    sliderLabel.htmlFor = 'slider';
+
+    // Slider
+    const slider = document.createElement('input');
+    slider.id = 'slider';
+    slider.type = 'range';
+    slider.min = '0';
+    slider.max = allImages.length - 1;
+    slider.value = globCurrentIndex;
+    slider.addEventListener('input', () => {
+        globCurrentIndex = parseInt(slider.value);
+        updateGifSourceAndSlider();
+    });
+
+    // Div for buttons
+    const buttonContainer = document.createElement('div');
+    buttonContainer.id = 'buttonContainer';
+    buttonContainer.appendChild(prevButton);
+    buttonContainer.appendChild(pauseContinueButton);
+    buttonContainer.appendChild(nextButton);
+
+    // Div for input
+    const inputContainer = document.createElement('div');
+    inputContainer.id = 'inputContainer';
+    inputContainer.appendChild(delayLabel);
+    inputContainer.appendChild(delayInput);
+
+    // Div for slider
+    const sliderContainer = document.createElement('div');
+    sliderContainer.id = 'sliderContainer';
+    sliderContainer.appendChild(sliderLabel);
+    sliderContainer.appendChild(slider);
+
+    // Insert everything above the <hr> or at the top of the page when missing
+    const hr = document.querySelector('hr');
+    if (hr) {
+        hr.parentNode.insertBefore(gif, hr);
+        hr.parentNode.insertBefore(buttonContainer, hr);
+        hr.parentNode.insertBefore(inputContainer, hr);
+        hr.parentNode.insertBefore(sliderContainer, hr);
+    } else {
+        document.body.insertBefore(gif, document.body.firstChild);
+        document.body.insertBefore(buttonContainer, document.body.firstChild);
+        document.body.insertBefore(inputContainer, document.body.firstChild);
+        document.body.insertBefore(sliderContainer, document.body.firstChild);
+    }
+
+    // Add keyboard shortcuts and disable default shortcuts behavior
+    document.addEventListener('keydown', (event) => {
+        switch (event.code) {
+            case keyboardShortcutPauseContinue:
+                event.preventDefault();
+                toggleGif();
+                break;
+            case keyboardShortcutPrev:
+                if (!prevButton.disabled) {
+                    event.preventDefault();
+                    changeGifFrame(-1);
+                }
+                break;
+            case keyboardShortcutNext:
+                if (!nextButton.disabled) {
+                    event.preventDefault();
+                    changeGifFrame(1);
+                }
+                break;
+        }
+    });
+
+    // Start the gif
+    globTimerId = runGif();
+}

tests/ui_tests/reporting/master_diff.py

@@ -232,7 +232,11 @@ def create_reports() -> None:
 
     for test_name, test_hash in removed_tests.items():
         with tmpdir() as temp_dir:
-            download.fetch_recorded(test_hash, temp_dir)
+            try:
+                download.fetch_recorded(test_hash, temp_dir)
+            except RuntimeError:
+                print("Could not download recorded files for", test_name)
+                continue
             removed(temp_dir, test_name)
 
     for test_name, test_hash in added_tests.items():

tests/ui_tests/reporting/testreport.css

@@ -2,11 +2,13 @@
     color: blue;
 }
 
-tr.ok a, tr.ok a:visited {
+tr.ok a,
+tr.ok a:visited {
     color: grey;
 }
 
-tr.bad a, tr.bad a:visited {
+tr.bad a,
+tr.bad a:visited {
     color: darkred;
 }
 
@@ -61,7 +63,103 @@ tr.bad a, tr.bad a:visited {
     display: none;
 }
 
-.model-T1 img, .model-TR img {
+.model-T1 img,
+.model-TR img {
     image-rendering: pixelated;
     width: 256px;
 }
+
+/* GIF styling */
+
+/* Style the input field */
+#delayInput {
+    padding: 8px;
+    border: 1px solid #ccc;
+    border-radius: 4px;
+    font-size: 16px;
+    margin-top: 10px;
+    margin-left: 10px;
+}
+
+/* Style the buttons */
+button.gifBtn {
+    padding: 8px;
+    border: none;
+    border-radius: 4px;
+    font-size: 16px;
+    background-color: #4CAF50;
+    color: white;
+    cursor: pointer;
+    transition: background-color 0.3s ease;
+    /* Add a 10 px space to the right of the button */
+    margin-right: 10px;
+}
+
+/* Remove the margin-right from the last button */
+button.gifBtn:last-child {
+    margin-right: 0;
+}
+
+/* Apply a gray background to disabled buttons */
+button.gifBtn:disabled {
+    background-color: #ccc;
+    cursor: not-allowed;
+}
+
+button.gifBtn:hover {
+    background-color: #3e8e41;
+}
+
+/* Style the slider */
+#slider {
+    width: 30%;
+    margin-top: 10px;
+}
+
+/* Style the progress bar */
+#slider::-webkit-slider-runnable-track {
+    height: 8px;
+    background-color: #ddd;
+    border-radius: 4px;
+}
+
+#slider::-webkit-slider-thumb {
+    height: 20px;
+    width: 20px;
+    border-radius: 50%;
+    background-color: #4CAF50;
+    cursor: pointer;
+    -webkit-appearance: none;
+    margin-top: -7px;
+}
+
+#slider::-moz-range-track {
+    height: 8px;
+    background-color: #ddd;
+    border-radius: 4px;
+}
+
+#slider::-moz-range-thumb {
+    height: 20px;
+    width: 20px;
+    border-radius: 50%;
+    background-color: #4CAF50;
+    cursor: pointer;
+}
+
+#slider::-ms-track {
+    height: 8px;
+    background-color: #ddd;
+    border-radius: 4px;
+    border: none;
+    color: transparent;
+}
+
+#slider::-ms-thumb {
+    height: 20px;
+    width: 20px;
+    border-radius: 50%;
+    background-color: #4CAF50;
+    cursor: pointer;
+    margin-top: 0;
+}

tests/ui_tests/reporting/testreport.js

@@ -38,7 +38,7 @@ async function markState(state) {
                 "test": stem,
                 "hash": document.body.dataset.actualHash
             })
-         })
+        })
         window.localStorage.setItem(itemKeyFromOneTest(), 'ok')
     } else {
         window.localStorage.setItem(itemKeyFromOneTest(), state)
@@ -139,7 +139,6 @@ function onLoadTestCase() {
     }
 }
 
-
 function onLoad() {
     if (window.location.protocol === "file") return
 
@@ -147,9 +146,18 @@ function onLoad() {
         elem.classList.remove("script-hidden")
     }
 
+    // Comes from create-gif.js, which is loaded in the final HTML
+    // Do it only in case of individual tests (which have "UI comparison" written on page),
+    // not on the main `index.html` page nor on `differing_screens.html` or other screen pages.
+    if (document.body.textContent.includes("UI comparison")) {
+        createGif()
+    }
+
     if (document.body.dataset.index) {
         onLoadIndex()
     } else {
+        // TODO: this is triggering some exception in console:
+        // Uncaught DOMException: Permission denied to access property "document" on cross-origin object
         onLoadTestCase()
     }
 }

tests/ui_tests/reporting/testreport.py

@@ -21,6 +21,7 @@ SCREEN_TEXT_FILE = TESTREPORT_PATH / "screen_text.txt"
 
 STYLE = (HERE / "testreport.css").read_text()
 SCRIPT = (HERE / "testreport.js").read_text()
+GIF_SCRIPT = (HERE / "create-gif.js").read_text()
 
 # These two html files are referencing each other
 ALL_SCREENS = "all_screens.html"
@@ -37,6 +38,7 @@ def document(
     style = t.style()
     style.add_raw_string(STYLE)
     script = t.script()
+    script.add_raw_string(GIF_SCRIPT)
     script.add_raw_string(SCRIPT)
     doc.head.add(style, script)