From 36a97c9b65d93f6a49dcc092a1e4ccf389e288b8 Mon Sep 17 00:00:00 2001 From: grdddj Date: Thu, 22 Jun 2023 10:41:35 +0200 Subject: [PATCH] fixup! WIP - dasbhoard with all recent PRs branches --- tools/ui_reports_generator/app.py | 43 ++++++++++--------- tools/ui_reports_generator/common_all.py | 25 ++++++++++- tools/ui_reports_generator/github.py | 33 ++++++++++---- tools/ui_reports_generator/gitlab.py | 31 +++++++++---- .../templates/dashboard.html | 28 +++++++++--- 5 files changed, 115 insertions(+), 45 deletions(-) diff --git a/tools/ui_reports_generator/app.py b/tools/ui_reports_generator/app.py index 20849398d..76ba8030f 100644 --- a/tools/ui_reports_generator/app.py +++ b/tools/ui_reports_generator/app.py @@ -3,7 +3,6 @@ from __future__ import annotations import time -from datetime import datetime from pathlib import Path from fastapi import FastAPI, HTTPException, Request @@ -12,7 +11,7 @@ from starlette.responses import RedirectResponse from cli import do_update_pulls from common_all import get_logger -from github import load_cache_file +from github import load_branches_cache, load_metadata_cache from gitlab import get_latest_infos_for_branch HERE = Path(__file__).parent @@ -42,22 +41,22 @@ async def get_branch_info(branch_name: str): async def get_dashboard(request: Request): try: logger.info("get_dashboard") - cached_info = load_cache_file() - last_update = cached_info["metadata"]["last_update"] - branches_dict = cached_info["branches"] + branches_info = load_branches_cache() + metadata = load_metadata_cache() + last_update = metadata["last_update"] branches_list = sorted( - branches_dict.values(), - key=lambda branch_info: branch_info["pull_request_number"], + branches_info.values(), + key=lambda branch_info: branch_info.last_commit_timestamp, reverse=True, ) - branches_list = [branch for branch in branches_list if branch["job_infos"]] - for branch in branches_list: - branch[ - "pr_link" - ] = f"https://github.com/trezor/trezor-firmware/pull/{branch['pull_request_number']}" + branches_with_ui = [branch for branch in branches_list if branch.job_infos] return templates.TemplateResponse( # type: ignore "dashboard.html", - {"request": request, "branches": branches_list, "last_update": last_update}, + { + "request": request, + "branches": branches_with_ui, + "last_update": last_update, + }, ) except Exception as e: logger.exception(f"Error: {e}") @@ -67,10 +66,14 @@ async def get_dashboard(request: Request): @app.get("/update") async def update_dashboard(): logger.info("update_dashboard") - global LAST_UPDATE_TS - if time.time() - LAST_UPDATE_TS > UPDATE_ALLOWED_EVERY_S: - do_update_pulls() - LAST_UPDATE_TS = time.time() - else: - time.sleep(5) - return RedirectResponse(url="/dashboard") + try: + global LAST_UPDATE_TS + if time.time() - LAST_UPDATE_TS > UPDATE_ALLOWED_EVERY_S: + do_update_pulls() + LAST_UPDATE_TS = time.time() # type: ignore + else: + time.sleep(5) + return RedirectResponse(url="/dashboard") + except Exception as e: + logger.exception(f"Error: {e}") + raise HTTPException(status_code=500, detail="Internal server error") diff --git a/tools/ui_reports_generator/common_all.py b/tools/ui_reports_generator/common_all.py index 93c05bc06..7030e67af 100644 --- a/tools/ui_reports_generator/common_all.py +++ b/tools/ui_reports_generator/common_all.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from dataclasses import dataclass +from dataclasses import dataclass, asdict from pathlib import Path from typing import Any @@ -11,13 +11,29 @@ AnyDict = dict[Any, Any] @dataclass class BranchInfo: name: str + branch_link: str pull_request_number: int pull_request_name: str + pull_request_link: str last_commit_sha: str last_commit_timestamp: int last_commit_datetime: str job_infos: dict[str, JobInfo] + @classmethod + def from_dict(cls, data: AnyDict) -> BranchInfo: + self = BranchInfo(**data) + # Need to transform job_info dict to JobInfo objects, + # as that was not done automatically by dataclass + self.job_infos = { + job_name: JobInfo.from_dict(job_info_dict) # type: ignore + for job_name, job_info_dict in self.job_infos.items() + } + return self + + def to_dict(self) -> AnyDict: + return asdict(self) + @dataclass class JobInfo: @@ -26,6 +42,13 @@ class JobInfo: status: str | None = None diff_screens: int | None = None + @classmethod + def from_dict(cls, data: AnyDict) -> JobInfo: + return JobInfo(**data) + + def to_dict(self) -> AnyDict: + return asdict(self) + def get_logger(name: str, log_file_path: str | Path) -> logging.Logger: logger = logging.getLogger(name) diff --git a/tools/ui_reports_generator/github.py b/tools/ui_reports_generator/github.py index b06fdcafb..5f6c13c39 100644 --- a/tools/ui_reports_generator/github.py +++ b/tools/ui_reports_generator/github.py @@ -2,7 +2,6 @@ from __future__ import annotations import json import os -from dataclasses import asdict from datetime import datetime from pathlib import Path from typing import Iterator @@ -24,12 +23,16 @@ def load_cache_file() -> AnyDict: def load_branches_cache() -> dict[str, BranchInfo]: cache_dict = load_cache_file()["branches"] - return {key: BranchInfo(**value) for key, value in cache_dict.items()} + return {key: BranchInfo.from_dict(value) for key, value in cache_dict.items()} + + +def load_metadata_cache() -> AnyDict: + return load_cache_file()["metadata"] def update_cache(cache_dict: dict[str, BranchInfo]) -> None: CACHE.update(cache_dict) - json_writable_cache_dict = {key: asdict(value) for key, value in CACHE.items()} + json_writable_cache_dict = {key: value.to_dict() for key, value in CACHE.items()} content = { "branches": json_writable_cache_dict, "metadata": { @@ -71,12 +74,17 @@ def yield_recently_updated_gh_pr_branches() -> Iterator[BranchInfo]: branch_name = pr["head"]["ref"] print(f"Getting branch {branch_name}") - # Skip when we already have this commit in cache + # Skip when we already have this commit in cache (and pipeline is finished) if branch_name in CACHE: cache_info = CACHE[branch_name] if cache_info.last_commit_sha == last_commit_sha: - print(f"Skipping, commit did not change - {last_commit_sha}") - continue + still_running = False + for job_info in cache_info.job_infos.values(): + if job_info.status == "Running...": + still_running = True + if not still_running: + print(f"Skipping, commit did not change - {last_commit_sha}") + continue # It can come from a fork - we do not have UI tests for it if branch_name == "master": @@ -84,12 +92,21 @@ def yield_recently_updated_gh_pr_branches() -> Iterator[BranchInfo]: continue last_commit_timestamp = get_commit_ts(last_commit_sha) - last_commit_datetime = datetime.fromtimestamp(last_commit_timestamp).isoformat() + last_commit_datetime = datetime.fromtimestamp(last_commit_timestamp).strftime( + "%Y-%m-%d %H:%M" + ) + pull_request_number = pr["number"] + pull_request_link = ( + f"https://github.com/trezor/trezor-firmware/pull/{pull_request_number}" + ) + branch_link = f"https://github.com/trezor/trezor-firmware/tree/{branch_name}" yield BranchInfo( name=branch_name, - pull_request_number=pr["number"], + branch_link=branch_link, + pull_request_number=pull_request_number, pull_request_name=pr["title"], + pull_request_link=pull_request_link, last_commit_sha=last_commit_sha, last_commit_timestamp=last_commit_timestamp, last_commit_datetime=last_commit_datetime, diff --git a/tools/ui_reports_generator/gitlab.py b/tools/ui_reports_generator/gitlab.py index 5f2667f1b..1ff76a24c 100644 --- a/tools/ui_reports_generator/gitlab.py +++ b/tools/ui_reports_generator/gitlab.py @@ -26,16 +26,25 @@ def update_branch_cache(link: str, amount: int) -> None: @lru_cache(maxsize=32) -def get_gitlab_branches(page: int) -> list[AnyDict]: +def get_gitlab_branches_cached(page: int) -> list[AnyDict]: return requests.get(BRANCHES_API_TEMPLATE.format(page)).json()["pipelines"] +def get_newest_gitlab_branches() -> list[AnyDict]: + return requests.get(BRANCHES_API_TEMPLATE.format(1)).json()["pipelines"] + + def get_branch_obj(branch_name: str) -> AnyDict: # Trying first 10 pages of branches for page in range(1, 11): - if page > 1: + if page == 1: + # First page should be always updated, + # rest can be cached + branches = get_newest_gitlab_branches() + else: + branches = get_gitlab_branches_cached(page) print(f"Checking page {page} / 10") - for branch_obj in get_gitlab_branches(page): + for branch_obj in branches: if branch_obj["ref"]["name"] == branch_name: return branch_obj raise ValueError(f"Branch {branch_name} not found") @@ -81,22 +90,26 @@ def yield_pipeline_jobs(pipeline_iid: int) -> Iterator[AnyDict]: yield job +def get_diff_screens_from_text(html_text: str) -> int: + row_identifier = 'bgcolor="red"' + return html_text.count(row_identifier) + + def get_status_from_link(job: AnyDict, link: str) -> tuple[str, int]: if job["status"]["label"] == "skipped": - return "SKIPPED", 0 + return "Skipped", 0 if link in BRANCH_CACHE: - return "OK", BRANCH_CACHE[link] + return "Finished", BRANCH_CACHE[link] res = requests.get(link) status = res.status_code if status == 200: - row_identifier = 'bgcolor="red"' - diff_screens = res.text.count(row_identifier) + diff_screens = get_diff_screens_from_text(res.text) update_branch_cache(link, diff_screens) - return "OK", diff_screens + return "Finished", diff_screens else: - return "NOT YET AVAILABLE", 0 + return "Running...", 0 def get_job_info(job: AnyDict, link: str, find_status: bool = True) -> JobInfo: diff --git a/tools/ui_reports_generator/templates/dashboard.html b/tools/ui_reports_generator/templates/dashboard.html index 596c7b7c5..8da1a9b31 100644 --- a/tools/ui_reports_generator/templates/dashboard.html +++ b/tools/ui_reports_generator/templates/dashboard.html @@ -28,18 +28,32 @@
{% for branch in branches %} -

PR: {{ branch["pull_request_name"] }}

-

Branch: {{ branch["name"] }}

-

Last commit: {{ branch["last_commit_datetime"] }}

+

PR: {{ branch.pull_request_name }}

+

Branch: {{ branch.name }}

+

Last commit: {{ branch.last_commit_datetime }}

+ - {% for job in branch["job_infos"].values() %} - - - + {% for job in branch.job_infos.values() %} + + + + {% endfor %}
TestStatus Diff screens
{{ job["name"] }}{{ job["diff_screens"] }}
{{ job.name }} + {{ job.status }} + + {{ job.diff_screens }} +