fixup! WIP - dasbhoard with all recent PRs branches

tools/ui_reports_generator/app.py

@@ -3,7 +3,6 @@
 from __future__ import annotations
 
 import time
-from datetime import datetime
 from pathlib import Path
 
 from fastapi import FastAPI, HTTPException, Request
@@ -12,7 +11,7 @@ from starlette.responses import RedirectResponse
 
 from cli import do_update_pulls
 from common_all import get_logger
-from github import load_cache_file
+from github import load_branches_cache, load_metadata_cache
 from gitlab import get_latest_infos_for_branch
 
 HERE = Path(__file__).parent
@@ -42,22 +41,22 @@ async def get_branch_info(branch_name: str):
 async def get_dashboard(request: Request):
     try:
         logger.info("get_dashboard")
-        cached_info = load_cache_file()
+        branches_info = load_branches_cache()
-        last_update = cached_info["metadata"]["last_update"]
+        metadata = load_metadata_cache()
-        branches_dict = cached_info["branches"]
+        last_update = metadata["last_update"]
         branches_list = sorted(
-            branches_dict.values(),
+            branches_info.values(),
-            key=lambda branch_info: branch_info["pull_request_number"],
+            key=lambda branch_info: branch_info.last_commit_timestamp,
             reverse=True,
         )
-        branches_list = [branch for branch in branches_list if branch["job_infos"]]
+        branches_with_ui = [branch for branch in branches_list if branch.job_infos]
-        for branch in branches_list:
-            branch[
-                "pr_link"
-            ] = f"https://github.com/trezor/trezor-firmware/pull/{branch['pull_request_number']}"
         return templates.TemplateResponse(  # type: ignore
             "dashboard.html",
-            {"request": request, "branches": branches_list, "last_update": last_update},
+            {
+                "request": request,
+                "branches": branches_with_ui,
+                "last_update": last_update,
+            },
         )
     except Exception as e:
         logger.exception(f"Error: {e}")
@@ -67,10 +66,14 @@ async def get_dashboard(request: Request):
 @app.get("/update")
 async def update_dashboard():
     logger.info("update_dashboard")
-    global LAST_UPDATE_TS
+    try:
-    if time.time() - LAST_UPDATE_TS > UPDATE_ALLOWED_EVERY_S:
+        global LAST_UPDATE_TS
-        do_update_pulls()
+        if time.time() - LAST_UPDATE_TS > UPDATE_ALLOWED_EVERY_S:
-        LAST_UPDATE_TS = time.time()
+            do_update_pulls()
-    else:
+            LAST_UPDATE_TS = time.time()  # type: ignore
-        time.sleep(5)
+        else:
-    return RedirectResponse(url="/dashboard")
+            time.sleep(5)
+        return RedirectResponse(url="/dashboard")
+    except Exception as e:
+        logger.exception(f"Error: {e}")
+        raise HTTPException(status_code=500, detail="Internal server error")

tools/ui_reports_generator/common_all.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import logging
-from dataclasses import dataclass
+from dataclasses import dataclass, asdict
 from pathlib import Path
 from typing import Any
 
@@ -11,13 +11,29 @@ AnyDict = dict[Any, Any]
 @dataclass
 class BranchInfo:
     name: str
+    branch_link: str
     pull_request_number: int
     pull_request_name: str
+    pull_request_link: str
     last_commit_sha: str
     last_commit_timestamp: int
     last_commit_datetime: str
     job_infos: dict[str, JobInfo]
 
+    @classmethod
+    def from_dict(cls, data: AnyDict) -> BranchInfo:
+        self = BranchInfo(**data)
+        # Need to transform job_info dict to JobInfo objects,
+        # as that was not done automatically by dataclass
+        self.job_infos = {
+            job_name: JobInfo.from_dict(job_info_dict)  # type: ignore
+            for job_name, job_info_dict in self.job_infos.items()
+        }
+        return self
+
+    def to_dict(self) -> AnyDict:
+        return asdict(self)
+
 
 @dataclass
 class JobInfo:
@@ -26,6 +42,13 @@ class JobInfo:
     status: str | None = None
     diff_screens: int | None = None
 
+    @classmethod
+    def from_dict(cls, data: AnyDict) -> JobInfo:
+        return JobInfo(**data)
+
+    def to_dict(self) -> AnyDict:
+        return asdict(self)
+
 
 def get_logger(name: str, log_file_path: str | Path) -> logging.Logger:
     logger = logging.getLogger(name)

tools/ui_reports_generator/github.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import json
 import os
-from dataclasses import asdict
 from datetime import datetime
 from pathlib import Path
 from typing import Iterator
@@ -24,12 +23,16 @@ def load_cache_file() -> AnyDict:
 
 def load_branches_cache() -> dict[str, BranchInfo]:
     cache_dict = load_cache_file()["branches"]
-    return {key: BranchInfo(**value) for key, value in cache_dict.items()}
+    return {key: BranchInfo.from_dict(value) for key, value in cache_dict.items()}
+
+
+def load_metadata_cache() -> AnyDict:
+    return load_cache_file()["metadata"]
 
 
 def update_cache(cache_dict: dict[str, BranchInfo]) -> None:
     CACHE.update(cache_dict)
-    json_writable_cache_dict = {key: asdict(value) for key, value in CACHE.items()}
+    json_writable_cache_dict = {key: value.to_dict() for key, value in CACHE.items()}
     content = {
         "branches": json_writable_cache_dict,
         "metadata": {
@@ -71,12 +74,17 @@ def yield_recently_updated_gh_pr_branches() -> Iterator[BranchInfo]:
         branch_name = pr["head"]["ref"]
         print(f"Getting branch {branch_name}")
 
-        # Skip when we already have this commit in cache
+        # Skip when we already have this commit in cache (and pipeline is finished)
         if branch_name in CACHE:
             cache_info = CACHE[branch_name]
             if cache_info.last_commit_sha == last_commit_sha:
-                print(f"Skipping, commit did not change - {last_commit_sha}")
+                still_running = False
-                continue
+                for job_info in cache_info.job_infos.values():
+                    if job_info.status == "Running...":
+                        still_running = True
+                if not still_running:
+                    print(f"Skipping, commit did not change - {last_commit_sha}")
+                    continue
 
         # It can come from a fork - we do not have UI tests for it
         if branch_name == "master":
@@ -84,12 +92,21 @@ def yield_recently_updated_gh_pr_branches() -> Iterator[BranchInfo]:
             continue
 
         last_commit_timestamp = get_commit_ts(last_commit_sha)
-        last_commit_datetime = datetime.fromtimestamp(last_commit_timestamp).isoformat()
+        last_commit_datetime = datetime.fromtimestamp(last_commit_timestamp).strftime(
+            "%Y-%m-%d %H:%M"
+        )
+        pull_request_number = pr["number"]
+        pull_request_link = (
+            f"https://github.com/trezor/trezor-firmware/pull/{pull_request_number}"
+        )
+        branch_link = f"https://github.com/trezor/trezor-firmware/tree/{branch_name}"
 
         yield BranchInfo(
             name=branch_name,
-            pull_request_number=pr["number"],
+            branch_link=branch_link,
+            pull_request_number=pull_request_number,
             pull_request_name=pr["title"],
+            pull_request_link=pull_request_link,
             last_commit_sha=last_commit_sha,
             last_commit_timestamp=last_commit_timestamp,
             last_commit_datetime=last_commit_datetime,

tools/ui_reports_generator/gitlab.py

@@ -26,16 +26,25 @@ def update_branch_cache(link: str, amount: int) -> None:
 
 
 @lru_cache(maxsize=32)
-def get_gitlab_branches(page: int) -> list[AnyDict]:
+def get_gitlab_branches_cached(page: int) -> list[AnyDict]:
     return requests.get(BRANCHES_API_TEMPLATE.format(page)).json()["pipelines"]
 
 
+def get_newest_gitlab_branches() -> list[AnyDict]:
+    return requests.get(BRANCHES_API_TEMPLATE.format(1)).json()["pipelines"]
+
+
 def get_branch_obj(branch_name: str) -> AnyDict:
     # Trying first 10 pages of branches
     for page in range(1, 11):
-        if page > 1:
+        if page == 1:
+            # First page should be always updated,
+            # rest can be cached
+            branches = get_newest_gitlab_branches()
+        else:
+            branches = get_gitlab_branches_cached(page)
             print(f"Checking page {page} / 10")
-        for branch_obj in get_gitlab_branches(page):
+        for branch_obj in branches:
             if branch_obj["ref"]["name"] == branch_name:
                 return branch_obj
     raise ValueError(f"Branch {branch_name} not found")
@@ -81,22 +90,26 @@ def yield_pipeline_jobs(pipeline_iid: int) -> Iterator[AnyDict]:
                 yield job
 
 
+def get_diff_screens_from_text(html_text: str) -> int:
+    row_identifier = 'bgcolor="red"'
+    return html_text.count(row_identifier)
+
+
 def get_status_from_link(job: AnyDict, link: str) -> tuple[str, int]:
     if job["status"]["label"] == "skipped":
-        return "SKIPPED", 0
+        return "Skipped", 0
 
     if link in BRANCH_CACHE:
-        return "OK", BRANCH_CACHE[link]
+        return "Finished", BRANCH_CACHE[link]
 
     res = requests.get(link)
     status = res.status_code
     if status == 200:
-        row_identifier = 'bgcolor="red"'
+        diff_screens = get_diff_screens_from_text(res.text)
-        diff_screens = res.text.count(row_identifier)
         update_branch_cache(link, diff_screens)
-        return "OK", diff_screens
+        return "Finished", diff_screens
     else:
-        return "NOT YET AVAILABLE", 0
+        return "Running...", 0
 
 
 def get_job_info(job: AnyDict, link: str, find_status: bool = True) -> JobInfo:

tools/ui_reports_generator/templates/dashboard.html

@@ -28,18 +28,32 @@
     </script>
     <hr>
     {% for branch in branches %}
-    <p><b>PR:</b> <a href="{{ branch['pr_link'] }}" target="_blank">{{ branch["pull_request_name"] }}</a></p>
+    <p><b>PR:</b> <a href="{{ branch.pull_request_link }}" target="_blank">{{ branch.pull_request_name }}</a></p>
-    <p><b>Branch:</b> {{ branch["name"] }}</p>
+    <p><b>Branch:</b> <a href="{{ branch.branch_link }}" target="_blank">{{ branch.name }}</a></p>
-    <p><b>Last commit:</b> {{ branch["last_commit_datetime"] }}</p>
+    <p><b>Last commit:</b> {{ branch.last_commit_datetime }}</p>
     <table>
         <tr>
             <th>Test</th>
+            <th>Status</th>
             <th>Diff screens</th>
         </tr>
-        {% for job in branch["job_infos"].values() %}
+        {% for job in branch.job_infos.values() %}
-        <tr style="{% if job.diff_screens > 0 %}background-color: red;{% endif %}">
+        <tr>
-            <td><a href="{{ job['link'] }}" target="_blank">{{ job["name"] }}</a></td>
+            <td><a href="{{ job.link }}" target="_blank">{{ job.name }}</a></td>
-            <td>{{ job["diff_screens"] }}</td>
+            <td style="
+                {% if job.status == 'Running...' %}
+                background-color: orange;
+                {% elif job.status == 'Skipped' %}
+                    background-color: red;
+                {% endif %}">
+                {{ job.status }}
+            </td>
+            <td style="
+                {% if job.diff_screens > 0 %}
+                    background-color: red;
+                {% endif %}">
+                {{ job.diff_screens }}
+            </td>
         </tr>
         {% endfor %}
     </table>