mirror of
https://github.com/trezor/trezor-firmware.git
synced 2024-12-13 01:48:21 +00:00
206 lines
7.1 KiB
Python
Executable File
206 lines
7.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Automatic generator of documentation about CI jobs.
|
|
Analyzes all .yml files connected with CI, takes the most important information
|
|
and writes it into a README file.
|
|
|
|
Features:
|
|
- reads a job description from a comment above job definition
|
|
- includes a link to each file and also to job definition
|
|
|
|
Usage:
|
|
- put comments (starting with "#") directly above the job definition in .yml file
|
|
|
|
Running the script:
|
|
- `python generate_ci_docs.py` to generate the documentation
|
|
- `python generate_ci_docs.py --check` to check if documentation is up-to-date
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import filecmp
|
|
import os
|
|
import re
|
|
import sys
|
|
from collections import OrderedDict
|
|
from pathlib import Path
|
|
from tempfile import NamedTemporaryFile
|
|
from typing import Any
|
|
|
|
import yaml
|
|
from mako.template import Template
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"--check",
|
|
action="store_true",
|
|
help="Check if there are no new changes in all CI .yml files",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
|
|
class DocsGenerator:
|
|
def __init__(self) -> None:
|
|
# Going to the root directory, so the relative
|
|
# locations of CI files are valid
|
|
os.chdir(Path(__file__).resolve().parent.parent)
|
|
|
|
self.GITLAB_CI_FILE = ".gitlab-ci.yml"
|
|
self.DOC_FILE = "docs/ci/jobs.md"
|
|
|
|
# Some keywords that are not job definitions and we should not care about them
|
|
self.NOT_JOBS = [
|
|
"variables:",
|
|
"image:",
|
|
".gitlab_caching:",
|
|
]
|
|
|
|
self.ALL_JOBS: dict[Path, dict[str, Any]] = OrderedDict()
|
|
|
|
self.FILES = self.get_all_ci_files()
|
|
|
|
def generate_docs(self) -> None:
|
|
"""Whole pipeline of getting and saving the CI information."""
|
|
for file in self.FILES:
|
|
self.ALL_JOBS[file] = {
|
|
"jobs": self.get_jobs_from_file(file),
|
|
"overall_description": self.get_overall_description_from_file(file),
|
|
}
|
|
|
|
self.save_docs_into_file()
|
|
|
|
def verify_docs(self) -> None:
|
|
"""Checking if the docs are up-to-date with current CI .yml files.
|
|
|
|
Creating a new doc file and comparing it against already existing one.
|
|
Exit with non-zero exit code when these files do not match.
|
|
"""
|
|
already_filled_doc_file = self.DOC_FILE
|
|
|
|
with NamedTemporaryFile() as temp_file:
|
|
self.DOC_FILE = temp_file.name
|
|
|
|
self.generate_docs()
|
|
if filecmp.cmp(already_filled_doc_file, self.DOC_FILE):
|
|
print("SUCCESS: Documentation is up-to-date!")
|
|
sys.exit(0)
|
|
else:
|
|
print("FAIL: Documentation is not up-to-date with CI .yml files!")
|
|
print(" Please run this script or `make gen`")
|
|
sys.exit(1)
|
|
|
|
def get_all_ci_files(self) -> list[Path]:
|
|
"""Loading all the CI files which are used in Gitlab."""
|
|
if not os.path.exists(self.GITLAB_CI_FILE):
|
|
raise RuntimeError(
|
|
f"Main Gitlab CI file under {self.GITLAB_CI_FILE} does not exist!"
|
|
)
|
|
|
|
with open(self.GITLAB_CI_FILE, "r") as f:
|
|
gitlab_file_content = yaml.safe_load(f)
|
|
|
|
all_ci_files = [Path(file) for file in gitlab_file_content["include"]]
|
|
|
|
for file in all_ci_files:
|
|
if not file.exists():
|
|
raise RuntimeError(f"File {file} does not exist!")
|
|
|
|
return all_ci_files
|
|
|
|
@staticmethod
|
|
def get_overall_description_from_file(file: Path) -> list[str]:
|
|
"""Looking for comments at the very beginning of the file."""
|
|
description_lines: list[str] = []
|
|
with open(file, "r") as f:
|
|
for line in f:
|
|
if line.startswith("#"):
|
|
# Taking just the text - no hashes, no whitespace
|
|
description_lines.append(line.strip("# \n"))
|
|
else:
|
|
break
|
|
|
|
return description_lines
|
|
|
|
def get_jobs_from_file(self, file: Path) -> dict[str, dict[str, Any]]:
|
|
"""Extract all jobs and their details from a certain file."""
|
|
all_jobs: dict[str, dict[str, Any]] = OrderedDict()
|
|
|
|
# Taking all the comments above a non-indented non-comment, which is
|
|
# always a job definition, unless defined in NOT_JOBS
|
|
with open(file, "r") as f:
|
|
comment_buffer: list[str] = []
|
|
for index, line in enumerate(f):
|
|
if line.startswith("#"):
|
|
# Taking just the text - no hashes, no whitespace
|
|
comment_buffer.append(line.strip("# \n"))
|
|
else:
|
|
# regex: first character of a line is a word-character or a dot
|
|
if re.search(r"\A[\w\.]", line) and not any(
|
|
[line.startswith(not_job) for not_job in self.NOT_JOBS]
|
|
):
|
|
job_name = line.rstrip(":\n")
|
|
all_jobs[job_name] = {
|
|
"description": comment_buffer,
|
|
"line_no": index + 1,
|
|
}
|
|
comment_buffer = []
|
|
|
|
return all_jobs
|
|
|
|
def save_docs_into_file(self) -> None:
|
|
"""Dump all the information into a documentation file."""
|
|
|
|
template_text = """
|
|
# CI pipeline
|
|
(Generated automatically by `tools/generate_ci_docs.py`. Do not edit by hand.)
|
|
|
|
It consists of multiple stages below, each having one or more jobs
|
|
Latest CI pipeline of master branch can be seen at [${latest_master}](${latest_master})
|
|
<%
|
|
## Needed because "##" is a comment in Mako templates
|
|
header_2 = "##"
|
|
header_3 = "###"
|
|
%>
|
|
% for file, file_info in all_jobs_items:
|
|
${header_2} ${file.stem.upper()} stage - [${file.name}](https://github.com/trezor/trezor-firmware/blob/master/${file})
|
|
% if file_info["overall_description"]:
|
|
% for stage_overall_description_line in file_info["overall_description"]:
|
|
${stage_overall_description_line}
|
|
% endfor
|
|
% endif
|
|
<%
|
|
job_amount = f"{len(file_info['jobs'])} job{'s' if len(file_info['jobs']) > 1 else ''}"
|
|
%>
|
|
Consists of **${job_amount}** below:
|
|
% for job_name, job_info in file_info["jobs"].items():
|
|
<%
|
|
github_job_link = f"https://github.com/trezor/trezor-firmware/blob/master/{file}#L{job_info['line_no']}"
|
|
%>
|
|
${header_3} [${job_name}](${github_job_link})
|
|
% if job_info["description"]:
|
|
% for job_description_line in job_info["description"]:
|
|
${job_description_line}
|
|
%endfor
|
|
% endif
|
|
% endfor
|
|
|
|
---
|
|
% endfor
|
|
""".strip()
|
|
|
|
with open(self.DOC_FILE, "w") as doc_file:
|
|
doc_text: str = Template(template_text).render(
|
|
latest_master="https://gitlab.com/satoshilabs/trezor/trezor-firmware/-/pipelines/master/latest",
|
|
all_jobs_items=self.ALL_JOBS.items(),
|
|
)
|
|
# Remove trailing whitespace coming from the template and include final newline
|
|
doc_file.writelines(line.rstrip() + "\n" for line in doc_text.splitlines())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if args.check:
|
|
DocsGenerator().verify_docs()
|
|
else:
|
|
DocsGenerator().generate_docs()
|