grdddj
2 years ago
committed by
matejcik
parent
cdca0bad9e
commit
f0cc7f3327
@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Automatic generator of documentation about CI jobs.
|
||||
Analyzes all .yml files connected with CI, takes the most important information
|
||||
and writes it into a README file.
|
||||
|
||||
Features:
|
||||
- reads a job description from a comment above job definition
|
||||
- includes a link to each file and also to job definition
|
||||
|
||||
Usage:
|
||||
- put comments (starting with "#") directly above the job definition in .yml file
|
||||
|
||||
Running the script:
|
||||
- `python generate_ci_docs.py` to generate the documentation
|
||||
- `python generate_ci_docs.py --check` to check if documentation is up-to-date
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import filecmp
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from mako.template import Template
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="Check if there are no new changes in all CI .yml files",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
class DocsGenerator:
|
||||
def __init__(self) -> None:
|
||||
# Going to the root directory, so the relative
|
||||
# locations of CI files are valid
|
||||
os.chdir(Path(__file__).resolve().parent.parent)
|
||||
|
||||
self.GITLAB_CI_FILE = ".gitlab-ci.yml"
|
||||
self.DOC_FILE = "docs/ci/jobs.md"
|
||||
|
||||
# Some keywords that are not job definitions and we should not care about them
|
||||
self.NOT_JOBS = [
|
||||
"variables:",
|
||||
"image:",
|
||||
".gitlab_caching:",
|
||||
]
|
||||
|
||||
self.ALL_JOBS: dict[Path, dict[str, Any]] = OrderedDict()
|
||||
|
||||
self.FILES = self.get_all_ci_files()
|
||||
|
||||
def generate_docs(self) -> None:
|
||||
"""Whole pipeline of getting and saving the CI information."""
|
||||
for file in self.FILES:
|
||||
self.ALL_JOBS[file] = {
|
||||
"jobs": self.get_jobs_from_file(file),
|
||||
"overall_description": self.get_overall_description_from_file(file),
|
||||
}
|
||||
|
||||
self.save_docs_into_file()
|
||||
|
||||
def verify_docs(self) -> None:
|
||||
"""Checking if the docs are up-to-date with current CI .yml files.
|
||||
|
||||
Creating a new doc file and comparing it against already existing one.
|
||||
Exit with non-zero exit code when these files do not match.
|
||||
"""
|
||||
already_filled_doc_file = self.DOC_FILE
|
||||
|
||||
with NamedTemporaryFile() as temp_file:
|
||||
self.DOC_FILE = temp_file.name
|
||||
|
||||
self.generate_docs()
|
||||
if filecmp.cmp(already_filled_doc_file, self.DOC_FILE):
|
||||
print("SUCCESS: Documentation is up-to-date!")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("FAIL: Documentation is not up-to-date with CI .yml files!")
|
||||
print(" Please run this script or `make gen`")
|
||||
sys.exit(1)
|
||||
|
||||
def get_all_ci_files(self) -> list[Path]:
|
||||
"""Loading all the CI files which are used in Gitlab."""
|
||||
if not os.path.exists(self.GITLAB_CI_FILE):
|
||||
raise RuntimeError(
|
||||
f"Main Gitlab CI file under {self.GITLAB_CI_FILE} does not exist!"
|
||||
)
|
||||
|
||||
with open(self.GITLAB_CI_FILE, "r") as f:
|
||||
gitlab_file_content = yaml.safe_load(f)
|
||||
|
||||
all_ci_files = [Path(file) for file in gitlab_file_content["include"]]
|
||||
|
||||
for file in all_ci_files:
|
||||
if not file.exists():
|
||||
raise RuntimeError(f"File {file} does not exist!")
|
||||
|
||||
return all_ci_files
|
||||
|
||||
@staticmethod
|
||||
def get_overall_description_from_file(file: Path) -> list[str]:
|
||||
"""Looking for comments at the very beginning of the file."""
|
||||
description_lines: list[str] = []
|
||||
with open(file, "r") as f:
|
||||
for line in f:
|
||||
if line.startswith("#"):
|
||||
# Taking just the text - no hashes, no whitespace
|
||||
description_lines.append(line.strip("# \n"))
|
||||
else:
|
||||
break
|
||||
|
||||
return description_lines
|
||||
|
||||
def get_jobs_from_file(self, file: Path) -> dict[str, dict[str, Any]]:
|
||||
"""Extract all jobs and their details from a certain file."""
|
||||
all_jobs: dict[str, dict[str, Any]] = OrderedDict()
|
||||
|
||||
# Taking all the comments above a non-indented non-comment, which is
|
||||
# always a job definition, unless defined in NOT_JOBS
|
||||
with open(file, "r") as f:
|
||||
comment_buffer: list[str] = []
|
||||
for index, line in enumerate(f):
|
||||
if line.startswith("#"):
|
||||
# Taking just the text - no hashes, no whitespace
|
||||
comment_buffer.append(line.strip("# \n"))
|
||||
else:
|
||||
# regex: first character of a line is a word-character or a dot
|
||||
if re.search(r"\A[\w\.]", line) and not any(
|
||||
[line.startswith(not_job) for not_job in self.NOT_JOBS]
|
||||
):
|
||||
job_name = line.rstrip(":\n")
|
||||
all_jobs[job_name] = {
|
||||
"description": comment_buffer,
|
||||
"line_no": index + 1,
|
||||
}
|
||||
comment_buffer = []
|
||||
|
||||
return all_jobs
|
||||
|
||||
def save_docs_into_file(self) -> None:
|
||||
"""Dump all the information into a documentation file."""
|
||||
|
||||
template_text = """
|
||||
# CI pipeline
|
||||
(Generated automatically by `tools/generate_ci_docs.py`. Do not edit by hand.)
|
||||
|
||||
It consists of multiple stages below, each having one or more jobs
|
||||
Latest CI pipeline of master branch can be seen at [${latest_master}](${latest_master})
|
||||
<%
|
||||
## Needed because "##" is a comment in Mako templates
|
||||
header_2 = "##"
|
||||
header_3 = "###"
|
||||
%>
|
||||
% for file, file_info in all_jobs_items:
|
||||
${header_2} ${file.stem.upper()} stage - [${file.name}](../../${file})
|
||||
% if file_info["overall_description"]:
|
||||
% for stage_overall_description_line in file_info["overall_description"]:
|
||||
${stage_overall_description_line}
|
||||
% endfor
|
||||
% endif
|
||||
<%
|
||||
job_amount = f"{len(file_info['jobs'])} job{'s' if len(file_info['jobs']) > 1 else ''}"
|
||||
%>
|
||||
Consists of **${job_amount}** below:
|
||||
% for job_name, job_info in file_info["jobs"].items():
|
||||
<%
|
||||
github_job_link = f"https://github.com/trezor/trezor-firmware/blob/master/{file}#L{job_info['line_no']}"
|
||||
%>
|
||||
${header_3} [${job_name}](${github_job_link})
|
||||
% if job_info["description"]:
|
||||
% for job_description_line in job_info["description"]:
|
||||
${job_description_line}
|
||||
%endfor
|
||||
% endif
|
||||
% endfor
|
||||
|
||||
---
|
||||
% endfor
|
||||
""".strip()
|
||||
|
||||
with open(self.DOC_FILE, "w") as doc_file:
|
||||
doc_text: str = Template(template_text).render(
|
||||
latest_master="https://gitlab.com/satoshilabs/trezor/trezor-firmware/-/pipelines/master/latest",
|
||||
all_jobs_items=self.ALL_JOBS.items(),
|
||||
)
|
||||
# Remove trailing whitespace coming from the template and include final newline
|
||||
doc_file.writelines(line.rstrip() + "\n" for line in doc_text.splitlines())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if args.check:
|
||||
DocsGenerator().verify_docs()
|
||||
else:
|
||||
DocsGenerator().generate_docs()
|
||||
Loading…
Reference in new issue