From 5f8bdfb2f48ecf62400435bc842e9d3e4bf71a1d Mon Sep 17 00:00:00 2001 From: grdddj Date: Wed, 21 Jun 2023 22:13:13 +0200 Subject: [PATCH] WIP - dasbhoard with all recent PRs branches --- tools/ui_reports_generator/.gitignore | 3 + tools/ui_reports_generator/app.py | 76 +++++++++++++++ tools/ui_reports_generator/cli.py | 50 ++++++++++ tools/ui_reports_generator/common_all.py | 37 +++++++ tools/ui_reports_generator/github.py | 97 +++++++++++++++++++ .../gitlab.py} | 90 +++++++++++------ .../templates/dashboard.html | 51 ++++++++++ 7 files changed, 374 insertions(+), 30 deletions(-) create mode 100644 tools/ui_reports_generator/.gitignore create mode 100644 tools/ui_reports_generator/app.py create mode 100644 tools/ui_reports_generator/cli.py create mode 100644 tools/ui_reports_generator/common_all.py create mode 100644 tools/ui_reports_generator/github.py rename tools/{gitlab_ui_reports.py => ui_reports_generator/gitlab.py} (69%) create mode 100644 tools/ui_reports_generator/templates/dashboard.html diff --git a/tools/ui_reports_generator/.gitignore b/tools/ui_reports_generator/.gitignore new file mode 100644 index 000000000..00ded45a7 --- /dev/null +++ b/tools/ui_reports_generator/.gitignore @@ -0,0 +1,3 @@ +*.json +*.log +*.out diff --git a/tools/ui_reports_generator/app.py b/tools/ui_reports_generator/app.py new file mode 100644 index 000000000..20849398d --- /dev/null +++ b/tools/ui_reports_generator/app.py @@ -0,0 +1,76 @@ +# Deployed by: +# uvicorn app:app --reload --host 0.0.0.0 --port 8002 +from __future__ import annotations + +import time +from datetime import datetime +from pathlib import Path + +from fastapi import FastAPI, HTTPException, Request +from fastapi.templating import Jinja2Templates +from starlette.responses import RedirectResponse + +from cli import do_update_pulls +from common_all import get_logger +from github import load_cache_file +from gitlab import get_latest_infos_for_branch + +HERE = Path(__file__).parent +log_file = HERE / "app.log" +logger = get_logger(__name__, log_file) + +app = FastAPI() + +templates = Jinja2Templates(directory="templates", trim_blocks=True, lstrip_blocks=True) + +LAST_UPDATE_TS = 0 +UPDATE_ALLOWED_EVERY_S = 30 + + +@app.get("/branch/{branch_name:path}") +async def get_branch_info(branch_name: str): + try: + logger.info(f"Branch: {branch_name}") + info = get_latest_infos_for_branch(branch_name, find_status=True) + return {"info": info} + except Exception as e: + logger.exception(f"Error: {e}") + raise HTTPException(status_code=500, detail="Internal server error") + + +@app.get("/dashboard") +async def get_dashboard(request: Request): + try: + logger.info("get_dashboard") + cached_info = load_cache_file() + last_update = cached_info["metadata"]["last_update"] + branches_dict = cached_info["branches"] + branches_list = sorted( + branches_dict.values(), + key=lambda branch_info: branch_info["pull_request_number"], + reverse=True, + ) + branches_list = [branch for branch in branches_list if branch["job_infos"]] + for branch in branches_list: + branch[ + "pr_link" + ] = f"https://github.com/trezor/trezor-firmware/pull/{branch['pull_request_number']}" + return templates.TemplateResponse( # type: ignore + "dashboard.html", + {"request": request, "branches": branches_list, "last_update": last_update}, + ) + except Exception as e: + logger.exception(f"Error: {e}") + raise HTTPException(status_code=500, detail="Internal server error") + + +@app.get("/update") +async def update_dashboard(): + logger.info("update_dashboard") + global LAST_UPDATE_TS + if time.time() - LAST_UPDATE_TS > UPDATE_ALLOWED_EVERY_S: + do_update_pulls() + LAST_UPDATE_TS = time.time() + else: + time.sleep(5) + return RedirectResponse(url="/dashboard") diff --git a/tools/ui_reports_generator/cli.py b/tools/ui_reports_generator/cli.py new file mode 100644 index 000000000..c200227f8 --- /dev/null +++ b/tools/ui_reports_generator/cli.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import click + +from github import update_cache, yield_recently_updated_gh_pr_branches +from gitlab import get_latest_infos_for_branch + + +@click.group() +def cli(): + pass + + +@cli.command(name="branch") +@click.argument("branch", default="master") +@click.option("--no-status", is_flag=True, default=False) +def get_branch(branch: str, no_status: bool): + print(f"Getting links for branch: {branch}") + tests_info = get_latest_infos_for_branch(branch, not no_status) + + for name, info in tests_info.items(): + print( + f"{name}\n - LINK: {info.link}\n - STATUS: {info.status}\n - DIFF SCREENS: {info.diff_screens}" + ) + + +def do_update_pulls(): + new_branch_infos = list(yield_recently_updated_gh_pr_branches()) + print(80 * "*") + print(f"Found {len(new_branch_infos)} new branches") + for branch in new_branch_infos: + print(f"Getting links for branch: {branch}") + try: + tests_info = get_latest_infos_for_branch(branch.name, True) + branch.job_infos = tests_info + except Exception as e: + print(f"Failed to get links for branch: {branch.name}") + print(e) + + branch_dict = {branch.name: branch for branch in new_branch_infos} + update_cache(branch_dict) + + +@cli.command(name="pulls") +def update_pulls(): + do_update_pulls() + + +if __name__ == "__main__": + cli() diff --git a/tools/ui_reports_generator/common_all.py b/tools/ui_reports_generator/common_all.py new file mode 100644 index 000000000..93c05bc06 --- /dev/null +++ b/tools/ui_reports_generator/common_all.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +AnyDict = dict[Any, Any] + + +@dataclass +class BranchInfo: + name: str + pull_request_number: int + pull_request_name: str + last_commit_sha: str + last_commit_timestamp: int + last_commit_datetime: str + job_infos: dict[str, JobInfo] + + +@dataclass +class JobInfo: + name: str + link: str + status: str | None = None + diff_screens: int | None = None + + +def get_logger(name: str, log_file_path: str | Path) -> logging.Logger: + logger = logging.getLogger(name) + logger.setLevel(logging.INFO) + log_handler = logging.FileHandler(log_file_path) + log_formatter = logging.Formatter("%(asctime)s %(message)s") + log_handler.setFormatter(log_formatter) + logger.addHandler(log_handler) + return logger diff --git a/tools/ui_reports_generator/github.py b/tools/ui_reports_generator/github.py new file mode 100644 index 000000000..b06fdcafb --- /dev/null +++ b/tools/ui_reports_generator/github.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import json +import os +from dataclasses import asdict +from datetime import datetime +from pathlib import Path +from typing import Iterator + +import requests + +from common_all import AnyDict, BranchInfo + +HERE = Path(__file__).parent + +GITHUB_PR_API = "https://api.github.com/repos/trezor/trezor-firmware/pulls" +GH_TOKEN = os.getenv("GH_TOKEN") +GH_HEADERS = {"Authorization": f"token {GH_TOKEN}"} if GH_TOKEN else {} + + +def load_cache_file() -> AnyDict: + return json.loads(CACHE_FILE.read_text()) + + +def load_branches_cache() -> dict[str, BranchInfo]: + cache_dict = load_cache_file()["branches"] + return {key: BranchInfo(**value) for key, value in cache_dict.items()} + + +def update_cache(cache_dict: dict[str, BranchInfo]) -> None: + CACHE.update(cache_dict) + json_writable_cache_dict = {key: asdict(value) for key, value in CACHE.items()} + content = { + "branches": json_writable_cache_dict, + "metadata": { + "last_update_timestamp": int(datetime.now().timestamp()), + "last_update": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + }, + } + CACHE_FILE.write_text(json.dumps(content, indent=2)) + + +CACHE_FILE = HERE / "github_cache.json" +if not CACHE_FILE.exists(): + CACHE_FILE.write_text("{}") +CACHE: dict[str, BranchInfo] = load_branches_cache() + + +def get_commit_ts(commit_hash: str) -> int: + res = requests.get( + f"https://api.github.com/repos/trezor/trezor-firmware/commits/{commit_hash}", + headers=GH_HEADERS, + ) + res.raise_for_status() + return int( + datetime.fromisoformat( + res.json()["commit"]["committer"]["date"].replace("Z", "") + ).timestamp() + ) + + +def get_all_gh_pulls() -> list[AnyDict]: + res = requests.get(GITHUB_PR_API, headers=GH_HEADERS) + res.raise_for_status() + return res.json() + + +def yield_recently_updated_gh_pr_branches() -> Iterator[BranchInfo]: + for pr in get_all_gh_pulls(): + last_commit_sha = pr["head"]["sha"] + branch_name = pr["head"]["ref"] + print(f"Getting branch {branch_name}") + + # Skip when we already have this commit in cache + if branch_name in CACHE: + cache_info = CACHE[branch_name] + if cache_info.last_commit_sha == last_commit_sha: + print(f"Skipping, commit did not change - {last_commit_sha}") + continue + + # It can come from a fork - we do not have UI tests for it + if branch_name == "master": + print("Ignoring a fork") + continue + + last_commit_timestamp = get_commit_ts(last_commit_sha) + last_commit_datetime = datetime.fromtimestamp(last_commit_timestamp).isoformat() + + yield BranchInfo( + name=branch_name, + pull_request_number=pr["number"], + pull_request_name=pr["title"], + last_commit_sha=last_commit_sha, + last_commit_timestamp=last_commit_timestamp, + last_commit_datetime=last_commit_datetime, + job_infos={}, + ) diff --git a/tools/gitlab_ui_reports.py b/tools/ui_reports_generator/gitlab.py similarity index 69% rename from tools/gitlab_ui_reports.py rename to tools/ui_reports_generator/gitlab.py index 3bf73c769..5f2667f1b 100644 --- a/tools/gitlab_ui_reports.py +++ b/tools/ui_reports_generator/gitlab.py @@ -1,19 +1,36 @@ from __future__ import annotations -import requests +import json +from functools import lru_cache +from pathlib import Path from typing import Callable, Iterator -import click +import requests + +from common_all import AnyDict, JobInfo + +HERE = Path(__file__).parent BRANCHES_API_TEMPLATE = "https://gitlab.com/satoshilabs/trezor/trezor-firmware/-/pipelines.json?scope=branches&page={}" GRAPHQL_API = "https://gitlab.com/api/graphql" +SCREEN_AMOUNT_CACHE_FILE = HERE / "gitlab_cache.json" +if not SCREEN_AMOUNT_CACHE_FILE.exists(): + SCREEN_AMOUNT_CACHE_FILE.write_text("{}") +BRANCH_CACHE: dict[str, int] = json.loads(SCREEN_AMOUNT_CACHE_FILE.read_text()) -def get_gitlab_branches(page: int) -> list[dict]: + +def update_branch_cache(link: str, amount: int) -> None: + BRANCH_CACHE[link] = amount + SCREEN_AMOUNT_CACHE_FILE.write_text(json.dumps(BRANCH_CACHE, indent=2)) + + +@lru_cache(maxsize=32) +def get_gitlab_branches(page: int) -> list[AnyDict]: return requests.get(BRANCHES_API_TEMPLATE.format(page)).json()["pipelines"] -def get_branch_obj(branch_name: str) -> dict: +def get_branch_obj(branch_name: str) -> AnyDict: # Trying first 10 pages of branches for page in range(1, 11): if page > 1: @@ -24,12 +41,7 @@ def get_branch_obj(branch_name: str) -> dict: raise ValueError(f"Branch {branch_name} not found") -def get_last_pipeline_id(branch_name: str) -> int: - branch_obj = get_branch_obj(branch_name) - return branch_obj["id"] - - -def get_pipeline_jobs_info(pipeline_iid: int) -> dict: +def get_pipeline_jobs_info(pipeline_iid: int) -> AnyDict: query = { "query": "fragment CiNeeds on JobNeedUnion {\n ...CiBuildNeedFields\n ...CiJobNeedFields\n}\n\nfragment CiBuildNeedFields on CiBuildNeed {\n id\n name\n}\n\nfragment CiJobNeedFields on CiJob {\n id\n name\n}\n\nfragment LinkedPipelineData on Pipeline {\n __typename\n id\n iid\n path\n cancelable\n retryable\n userPermissions {\n updatePipeline\n }\n status: detailedStatus {\n __typename\n id\n group\n label\n icon\n }\n sourceJob {\n __typename\n id\n name\n retried\n }\n project {\n __typename\n id\n name\n fullPath\n }\n}\n\nquery getPipelineDetails($projectPath: ID!, $iid: ID!) {\n project(fullPath: $projectPath) {\n __typename\n id\n pipeline(iid: $iid) {\n __typename\n id\n iid\n complete\n usesNeeds\n userPermissions {\n updatePipeline\n }\n downstream {\n __typename\n nodes {\n ...LinkedPipelineData\n }\n }\n upstream {\n ...LinkedPipelineData\n }\n stages {\n __typename\n nodes {\n __typename\n id\n name\n status: detailedStatus {\n __typename\n id\n action {\n __typename\n id\n icon\n path\n title\n }\n }\n groups {\n __typename\n nodes {\n __typename\n id\n status: detailedStatus {\n __typename\n id\n label\n group\n icon\n }\n name\n size\n jobs {\n __typename\n nodes {\n __typename\n id\n name\n kind\n scheduledAt\n needs {\n __typename\n nodes {\n __typename\n id\n name\n }\n }\n previousStageJobsOrNeeds {\n __typename\n nodes {\n ...CiNeeds\n }\n }\n status: detailedStatus {\n __typename\n id\n icon\n tooltip\n hasDetails\n detailsPath\n group\n label\n action {\n __typename\n id\n buttonTitle\n icon\n path\n title\n }\n }\n }\n }\n }\n }\n }\n }\n }\n }\n}\n", "variables": { @@ -58,7 +70,7 @@ def get_jobs_of_interests() -> list[tuple[str, Callable[[str], str]]]: ] -def yield_pipeline_jobs(pipeline_iid: int) -> Iterator[dict]: +def yield_pipeline_jobs(pipeline_iid: int) -> Iterator[AnyDict]: jobs_info = get_pipeline_jobs_info(pipeline_iid) stages = jobs_info["data"]["project"]["pipeline"]["stages"]["nodes"] for stage in stages: @@ -69,29 +81,47 @@ def yield_pipeline_jobs(pipeline_iid: int) -> Iterator[dict]: yield job -def get_latest_links_for_branch(branch_name: str) -> dict[str, str]: - branch_obj = get_branch_obj(branch_name) - pipeline_iid = branch_obj["iid"] +def get_status_from_link(job: AnyDict, link: str) -> tuple[str, int]: + if job["status"]["label"] == "skipped": + return "SKIPPED", 0 - links: dict[str, str] = {} + if link in BRANCH_CACHE: + return "OK", BRANCH_CACHE[link] - for job in yield_pipeline_jobs(pipeline_iid): - for job_of_interest, func in get_jobs_of_interests(): - if job["name"] == job_of_interest: - job_id = job["id"].split("/")[-1] - links[job["name"]] = func(job_id) + res = requests.get(link) + status = res.status_code + if status == 200: + row_identifier = 'bgcolor="red"' + diff_screens = res.text.count(row_identifier) + update_branch_cache(link, diff_screens) + return "OK", diff_screens + else: + return "NOT YET AVAILABLE", 0 - return links +def get_job_info(job: AnyDict, link: str, find_status: bool = True) -> JobInfo: + if find_status: + status, diff_screens = get_status_from_link(job, link) + else: + status, diff_screens = None, None -@click.command() -@click.argument("branch", default="master") -def main(branch: str): - print(f"Getting links for branch: {branch}") - links = get_latest_links_for_branch(branch) - for name, link in links.items(): - print(f"{name}: {link}") + return JobInfo( + name=job["name"], link=link, status=status, diff_screens=diff_screens + ) + + +def get_latest_infos_for_branch( + branch_name: str, find_status: bool +) -> dict[str, JobInfo]: + branch_obj = get_branch_obj(branch_name) + pipeline_iid = branch_obj["iid"] + def yield_key_value() -> Iterator[tuple[str, JobInfo]]: + for job in yield_pipeline_jobs(pipeline_iid): + for job_of_interest, link_func in get_jobs_of_interests(): + if job["name"] == job_of_interest: + job_id = job["id"].split("/")[-1] + link = link_func(job_id) + yield job["name"], get_job_info(job, link, find_status) -if __name__ == "__main__": - main() + return dict(yield_key_value()) diff --git a/tools/ui_reports_generator/templates/dashboard.html b/tools/ui_reports_generator/templates/dashboard.html new file mode 100644 index 000000000..596c7b7c5 --- /dev/null +++ b/tools/ui_reports_generator/templates/dashboard.html @@ -0,0 +1,51 @@ + + + + + Dashboard + + + + +

UI diff dashboard

+

Last Update: {{ last_update }}

+
+ +
+ + +
+ {% for branch in branches %} +

PR: {{ branch["pull_request_name"] }}

+

Branch: {{ branch["name"] }}

+

Last commit: {{ branch["last_commit_datetime"] }}

+ + + + + + {% for job in branch["job_infos"].values() %} + + + + + {% endfor %} +
TestDiff screens
{{ job["name"] }}{{ job["diff_screens"] }}
+
+
+ {% endfor %} + + +