arno/trezor-firmware
1
0
Fork 0
mirror of https://github.com/trezor/trezor-firmware.git synced 2025-08-04 21:05:29 +00:00
Code Issues Releases Wiki Activity

fixup! WIP - dasbhoard with all recent PRs branches

Browse Source
This commit is contained in:
grdddj 2023-06-22 10:41:35 +02:00
parent 5f8bdfb2f4
commit 36a97c9b65
5 changed files with 115 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
tools/ui_reports_generator/app.py
View File

@ -3,7 +3,6 @@
from __future__ import annotations
import time
from datetime import datetime
from pathlib import Path
from fastapi import FastAPI, HTTPException, Request
@ -12,7 +11,7 @@ from starlette.responses import RedirectResponse
from cli import do_update_pulls
from common_all import get_logger
from github import load_cache_file
from github import load_branches_cache, load_metadata_cache
from gitlab import get_latest_infos_for_branch
HERE = Path(__file__).parent
@ -42,22 +41,22 @@ async def get_branch_info(branch_name: str):
async def get_dashboard(request: Request):
try:
logger.info("get_dashboard")
cached_info = load_cache_file()
last_update = cached_info["metadata"]["last_update"]
branches_dict = cached_info["branches"]
branches_info = load_branches_cache()
metadata = load_metadata_cache()
last_update = metadata["last_update"]
branches_list = sorted(
branches_dict.values(),
key=lambda branch_info: branch_info["pull_request_number"],
branches_info.values(),
key=lambda branch_info: branch_info.last_commit_timestamp,
reverse=True,
)
branches_list = [branch for branch in branches_list if branch["job_infos"]]
for branch in branches_list:
branch[
"pr_link"
] = f"https://github.com/trezor/trezor-firmware/pull/{branch['pull_request_number']}"
branches_with_ui = [branch for branch in branches_list if branch.job_infos]
return templates.TemplateResponse( # type: ignore
"dashboard.html",
{"request": request, "branches": branches_list, "last_update": last_update},
{
"request": request,
"branches": branches_with_ui,
"last_update": last_update,
},
)
except Exception as e:
logger.exception(f"Error: {e}")
@ -67,10 +66,14 @@ async def get_dashboard(request: Request):
@app.get("/update")
async def update_dashboard():
logger.info("update_dashboard")
global LAST_UPDATE_TS
if time.time() - LAST_UPDATE_TS > UPDATE_ALLOWED_EVERY_S:
do_update_pulls()
LAST_UPDATE_TS = time.time()
else:
time.sleep(5)
return RedirectResponse(url="/dashboard")
try:
global LAST_UPDATE_TS
if time.time() - LAST_UPDATE_TS > UPDATE_ALLOWED_EVERY_S:
do_update_pulls()
LAST_UPDATE_TS = time.time() # type: ignore
else:
time.sleep(5)
return RedirectResponse(url="/dashboard")
except Exception as e:
logger.exception(f"Error: {e}")
raise HTTPException(status_code=500, detail="Internal server error")

25
tools/ui_reports_generator/common_all.py
View File

@ -1,7 +1,7 @@
from __future__ import annotations
import logging
from dataclasses import dataclass
from dataclasses import dataclass, asdict
from pathlib import Path
from typing import Any
@ -11,13 +11,29 @@ AnyDict = dict[Any, Any]
@dataclass
class BranchInfo:
name: str
branch_link: str
pull_request_number: int
pull_request_name: str
pull_request_link: str
last_commit_sha: str
last_commit_timestamp: int
last_commit_datetime: str
job_infos: dict[str, JobInfo]
@classmethod
def from_dict(cls, data: AnyDict) -> BranchInfo:
self = BranchInfo(**data)
# Need to transform job_info dict to JobInfo objects,
# as that was not done automatically by dataclass
self.job_infos = {
job_name: JobInfo.from_dict(job_info_dict) # type: ignore
for job_name, job_info_dict in self.job_infos.items()
}
return self
def to_dict(self) -> AnyDict:
return asdict(self)
@dataclass
class JobInfo:
@ -26,6 +42,13 @@ class JobInfo:
status: str | None = None
diff_screens: int | None = None
@classmethod
def from_dict(cls, data: AnyDict) -> JobInfo:
return JobInfo(**data)
def to_dict(self) -> AnyDict:
return asdict(self)
def get_logger(name: str, log_file_path: str | Path) -> logging.Logger:
logger = logging.getLogger(name)

33
tools/ui_reports_generator/github.py
View File

@ -2,7 +2,6 @@ from __future__ import annotations
import json
import os
from dataclasses import asdict
from datetime import datetime
from pathlib import Path
from typing import Iterator
@ -24,12 +23,16 @@ def load_cache_file() -> AnyDict:
def load_branches_cache() -> dict[str, BranchInfo]:
cache_dict = load_cache_file()["branches"]
return {key: BranchInfo(**value) for key, value in cache_dict.items()}
return {key: BranchInfo.from_dict(value) for key, value in cache_dict.items()}
def load_metadata_cache() -> AnyDict:
return load_cache_file()["metadata"]
def update_cache(cache_dict: dict[str, BranchInfo]) -> None:
CACHE.update(cache_dict)
json_writable_cache_dict = {key: asdict(value) for key, value in CACHE.items()}
json_writable_cache_dict = {key: value.to_dict() for key, value in CACHE.items()}
content = {
"branches": json_writable_cache_dict,
"metadata": {
@ -71,12 +74,17 @@ def yield_recently_updated_gh_pr_branches() -> Iterator[BranchInfo]:
branch_name = pr["head"]["ref"]
print(f"Getting branch {branch_name}")
# Skip when we already have this commit in cache
# Skip when we already have this commit in cache (and pipeline is finished)
if branch_name in CACHE:
cache_info = CACHE[branch_name]
if cache_info.last_commit_sha == last_commit_sha:
print(f"Skipping, commit did not change - {last_commit_sha}")
continue
still_running = False
for job_info in cache_info.job_infos.values():
if job_info.status == "Running...":
still_running = True
if not still_running:
print(f"Skipping, commit did not change - {last_commit_sha}")
continue
# It can come from a fork - we do not have UI tests for it
if branch_name == "master":
@ -84,12 +92,21 @@ def yield_recently_updated_gh_pr_branches() -> Iterator[BranchInfo]:
continue
last_commit_timestamp = get_commit_ts(last_commit_sha)
last_commit_datetime = datetime.fromtimestamp(last_commit_timestamp).isoformat()
last_commit_datetime = datetime.fromtimestamp(last_commit_timestamp).strftime(
"%Y-%m-%d %H:%M"
)
pull_request_number = pr["number"]
pull_request_link = (
f"https://github.com/trezor/trezor-firmware/pull/{pull_request_number}"
)
branch_link = f"https://github.com/trezor/trezor-firmware/tree/{branch_name}"
yield BranchInfo(
name=branch_name,
pull_request_number=pr["number"],
branch_link=branch_link,
pull_request_number=pull_request_number,
pull_request_name=pr["title"],
pull_request_link=pull_request_link,
last_commit_sha=last_commit_sha,
last_commit_timestamp=last_commit_timestamp,
last_commit_datetime=last_commit_datetime,

31
tools/ui_reports_generator/gitlab.py
View File

@ -26,16 +26,25 @@ def update_branch_cache(link: str, amount: int) -> None:
@lru_cache(maxsize=32)
def get_gitlab_branches(page: int) -> list[AnyDict]:
def get_gitlab_branches_cached(page: int) -> list[AnyDict]:
return requests.get(BRANCHES_API_TEMPLATE.format(page)).json()["pipelines"]
def get_newest_gitlab_branches() -> list[AnyDict]:
return requests.get(BRANCHES_API_TEMPLATE.format(1)).json()["pipelines"]
def get_branch_obj(branch_name: str) -> AnyDict:
# Trying first 10 pages of branches
for page in range(1, 11):
if page > 1:
if page == 1:
# First page should be always updated,
# rest can be cached
branches = get_newest_gitlab_branches()
else:
branches = get_gitlab_branches_cached(page)
print(f"Checking page {page} / 10")
for branch_obj in get_gitlab_branches(page):
for branch_obj in branches:
if branch_obj["ref"]["name"] == branch_name:
return branch_obj
raise ValueError(f"Branch {branch_name} not found")
@ -81,22 +90,26 @@ def yield_pipeline_jobs(pipeline_iid: int) -> Iterator[AnyDict]:
yield job
def get_diff_screens_from_text(html_text: str) -> int:
row_identifier = 'bgcolor="red"'
return html_text.count(row_identifier)
def get_status_from_link(job: AnyDict, link: str) -> tuple[str, int]:
if job["status"]["label"] == "skipped":
return "SKIPPED", 0
return "Skipped", 0
if link in BRANCH_CACHE:
return "OK", BRANCH_CACHE[link]
return "Finished", BRANCH_CACHE[link]
res = requests.get(link)
status = res.status_code
if status == 200:
row_identifier = 'bgcolor="red"'
diff_screens = res.text.count(row_identifier)
diff_screens = get_diff_screens_from_text(res.text)
update_branch_cache(link, diff_screens)
return "OK", diff_screens
return "Finished", diff_screens
else:
return "NOT YET AVAILABLE", 0
return "Running...", 0
def get_job_info(job: AnyDict, link: str, find_status: bool = True) -> JobInfo:

28
tools/ui_reports_generator/templates/dashboard.html
View File

@ -28,18 +28,32 @@
</script>
<hr>
{% for branch in branches %}
<p><b>PR:</b> <a href="{{ branch['pr_link'] }}" target="_blank">{{ branch["pull_request_name"] }}</a></p>
<p><b>Branch:</b> {{ branch["name"] }}</p>
<p><b>Last commit:</b> {{ branch["last_commit_datetime"] }}</p>
<p><b>PR:</b> <a href="{{ branch.pull_request_link }}" target="_blank">{{ branch.pull_request_name }}</a></p>
<p><b>Branch:</b> <a href="{{ branch.branch_link }}" target="_blank">{{ branch.name }}</a></p>
<p><b>Last commit:</b> {{ branch.last_commit_datetime }}</p>
<table>
<tr>
<th>Test</th>
<th>Status</th>
<th>Diff screens</th>
</tr>
{% for job in branch["job_infos"].values() %}
<tr style="{% if job.diff_screens > 0 %}background-color: red;{% endif %}">
<td><a href="{{ job['link'] }}" target="_blank">{{ job["name"] }}</a></td>
<td>{{ job["diff_screens"] }}</td>
{% for job in branch.job_infos.values() %}
<tr>
<td><a href="{{ job.link }}" target="_blank">{{ job.name }}</a></td>
<td style="
{% if job.status == 'Running...' %}
background-color: orange;
{% elif job.status == 'Skipped' %}
background-color: red;
{% endif %}">
{{ job.status }}
</td>
<td style="
{% if job.diff_screens > 0 %}
background-color: red;
{% endif %}">
{{ job.diff_screens }}
</td>
</tr>
{% endfor %}
</table>