1
0
mirror of https://github.com/trezor/trezor-firmware.git synced 2024-12-05 22:18:16 +00:00
trezor-firmware/tools/generate_ci_docs.py

206 lines
7.1 KiB
Python
Raw Normal View History

#!/usr/bin/env python3
"""
Automatic generator of documentation about CI jobs.
Analyzes all .yml files connected with CI, takes the most important information
and writes it into a README file.
Features:
- reads a job description from a comment above job definition
- includes a link to each file and also to job definition
Usage:
- put comments (starting with "#") directly above the job definition in .yml file
Running the script:
- `python generate_ci_docs.py` to generate the documentation
- `python generate_ci_docs.py --check` to check if documentation is up-to-date
"""
from __future__ import annotations
import argparse
import filecmp
import os
import re
import sys
from collections import OrderedDict
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Any
import yaml
from mako.template import Template
parser = argparse.ArgumentParser()
parser.add_argument(
"--check",
action="store_true",
help="Check if there are no new changes in all CI .yml files",
)
args = parser.parse_args()
class DocsGenerator:
def __init__(self) -> None:
# Going to the root directory, so the relative
# locations of CI files are valid
os.chdir(Path(__file__).resolve().parent.parent)
self.GITLAB_CI_FILE = ".gitlab-ci.yml"
self.DOC_FILE = "docs/ci/jobs.md"
# Some keywords that are not job definitions and we should not care about them
self.NOT_JOBS = [
"variables:",
"image:",
".gitlab_caching:",
]
self.ALL_JOBS: dict[Path, dict[str, Any]] = OrderedDict()
self.FILES = self.get_all_ci_files()
def generate_docs(self) -> None:
"""Whole pipeline of getting and saving the CI information."""
for file in self.FILES:
self.ALL_JOBS[file] = {
"jobs": self.get_jobs_from_file(file),
"overall_description": self.get_overall_description_from_file(file),
}
self.save_docs_into_file()
def verify_docs(self) -> None:
"""Checking if the docs are up-to-date with current CI .yml files.
Creating a new doc file and comparing it against already existing one.
Exit with non-zero exit code when these files do not match.
"""
already_filled_doc_file = self.DOC_FILE
with NamedTemporaryFile() as temp_file:
self.DOC_FILE = temp_file.name
self.generate_docs()
if filecmp.cmp(already_filled_doc_file, self.DOC_FILE):
print("SUCCESS: Documentation is up-to-date!")
sys.exit(0)
else:
print("FAIL: Documentation is not up-to-date with CI .yml files!")
print(" Please run this script or `make gen`")
sys.exit(1)
def get_all_ci_files(self) -> list[Path]:
"""Loading all the CI files which are used in Gitlab."""
if not os.path.exists(self.GITLAB_CI_FILE):
raise RuntimeError(
f"Main Gitlab CI file under {self.GITLAB_CI_FILE} does not exist!"
)
with open(self.GITLAB_CI_FILE, "r") as f:
gitlab_file_content = yaml.safe_load(f)
all_ci_files = [Path(file) for file in gitlab_file_content["include"]]
for file in all_ci_files:
if not file.exists():
raise RuntimeError(f"File {file} does not exist!")
return all_ci_files
@staticmethod
def get_overall_description_from_file(file: Path) -> list[str]:
"""Looking for comments at the very beginning of the file."""
description_lines: list[str] = []
with open(file, "r") as f:
for line in f:
if line.startswith("#"):
# Taking just the text - no hashes, no whitespace
description_lines.append(line.strip("# \n"))
else:
break
return description_lines
def get_jobs_from_file(self, file: Path) -> dict[str, dict[str, Any]]:
"""Extract all jobs and their details from a certain file."""
all_jobs: dict[str, dict[str, Any]] = OrderedDict()
# Taking all the comments above a non-indented non-comment, which is
# always a job definition, unless defined in NOT_JOBS
with open(file, "r") as f:
comment_buffer: list[str] = []
for index, line in enumerate(f):
if line.startswith("#"):
# Taking just the text - no hashes, no whitespace
comment_buffer.append(line.strip("# \n"))
else:
# regex: first character of a line is a word-character or a dot
if re.search(r"\A[\w\.]", line) and not any(
[line.startswith(not_job) for not_job in self.NOT_JOBS]
):
job_name = line.rstrip(":\n")
all_jobs[job_name] = {
"description": comment_buffer,
"line_no": index + 1,
}
comment_buffer = []
return all_jobs
def save_docs_into_file(self) -> None:
"""Dump all the information into a documentation file."""
template_text = """
# CI pipeline
(Generated automatically by `tools/generate_ci_docs.py`. Do not edit by hand.)
It consists of multiple stages below, each having one or more jobs
Latest CI pipeline of master branch can be seen at [${latest_master}](${latest_master})
<%
## Needed because "##" is a comment in Mako templates
header_2 = "##"
header_3 = "###"
%>
% for file, file_info in all_jobs_items:
${header_2} ${file.stem.upper()} stage - [${file.name}](https://github.com/trezor/trezor-firmware/blob/master/${file})
% if file_info["overall_description"]:
% for stage_overall_description_line in file_info["overall_description"]:
${stage_overall_description_line}
% endfor
% endif
<%
job_amount = f"{len(file_info['jobs'])} job{'s' if len(file_info['jobs']) > 1 else ''}"
%>
Consists of **${job_amount}** below:
% for job_name, job_info in file_info["jobs"].items():
<%
github_job_link = f"https://github.com/trezor/trezor-firmware/blob/master/{file}#L{job_info['line_no']}"
%>
${header_3} [${job_name}](${github_job_link})
% if job_info["description"]:
% for job_description_line in job_info["description"]:
${job_description_line}
%endfor
% endif
% endfor
---
% endfor
""".strip()
with open(self.DOC_FILE, "w") as doc_file:
doc_text: str = Template(template_text).render(
latest_master="https://gitlab.com/satoshilabs/trezor/trezor-firmware/-/pipelines/master/latest",
all_jobs_items=self.ALL_JOBS.items(),
)
# Remove trailing whitespace coming from the template and include final newline
doc_file.writelines(line.rstrip() + "\n" for line in doc_text.splitlines())
if __name__ == "__main__":
if args.check:
DocsGenerator().verify_docs()
else:
DocsGenerator().generate_docs()