You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
trezor-firmware/tools/pyright_tool.py

639 lines
24 KiB

#!/usr/bin/env python3
"""
Wrapper around pyright type checking to allow for easy ignore of specific error messages.
Thanks to it the `# type: ignore` does not affect the whole line,
so other problems at the same line cannot be masked by it.
Features:
- ignores specific pyright errors based on substring or regex
- reports empty `# type: ignore`s (without ignore reason in `[]`)
- reports unused `# type: ignore`s (for example after pyright is updated)
- allows for ignoring some errors in the whole file - see `FILE_SPECIFIC_IGNORES` variable
- allows for error aliases - see `ALIASES` variable
Usage:
- there are multiple options how to ignore/silence a pyright error:
1 - "# type: ignore [<error_substring>]"
- put it as a comment to the line we want to ignore
- "# type: ignore [<error1>;;<error2>;;...]" if there are more than one errors on that line
- also regex patterns are valid substrings
2 - "# pyright: off" / "# pyright: on"
- all errors in block of code between these marks will be ignored
3 - FILE_SPECIFIC_IGNORES
- ignore specific rules (defined by pyright) or error substrings in the whole file
4 - ALIASES
- create an alias for a common error and use is with option 1 - "# type: ignore [<error_alias>]"
Running the script:
- see all script argument by calling `python pyright_tool.py --help`
Simplified program flow (as it happens in PyrightTool.run()):
- extract and validate pyright config data from pyrightconfig.json
- collect all the pyright errors by actually running the pyright itself
- extract type-ignore information for all the files pyright was analyzing
- loop through all the pyright errors and try to match them against all the type-ignore rules
- if there are some unmatched errors, report them and exit with nonzero value
- also report unused ignores and other inconsistencies
"""
from __future__ import annotations
import io
import json
import re
import subprocess
import sys
import tempfile
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Any, Final, Iterator, TypedDict
import click
if TYPE_CHECKING:
LineIgnores = list["LineIgnore"]
FileIgnores = dict[str, LineIgnores]
FileSpecificIgnores = dict[str, list["FileSpecificIgnore"]]
PyrightOffIgnores = list["PyrightOffIgnore"]
FilePyrightOffIgnores = dict[str, PyrightOffIgnores]
class RangeDetail(TypedDict):
line: int
character: int
class Range(TypedDict):
start: RangeDetail
end: RangeDetail
class Error(TypedDict):
file: str
severity: str
message: str
range: Range
rule: str
class Summary(TypedDict):
filesAnalyzed: int
errorCount: int
warningCount: int
informationCount: int
timeInSec: float
class PyrightResults(TypedDict):
version: str
time: str
generalDiagnostics: list[Error]
summary: Summary
@dataclass
class IgnoreStatement:
substring: str
already_used: bool = False
@dataclass
class LineIgnore:
line_no: int
ignore_statements: list[IgnoreStatement]
@dataclass
class FileSpecificIgnore:
rule: str = ""
substring: str = ""
already_used: bool = False
def __post_init__(self) -> None:
if self.rule and self.substring:
raise ValueError("Only one of rule|substring should be set")
@dataclass
class PyrightOffIgnore:
start_line: int
end_line: int
already_used: bool = False
# TODO: move into a JSON or other config file
# Files need to have a relative location to the directory being tested
# Example (when checking `python` directory):
# "tools/helloworld.py": [
# FileSpecificIgnore(rule="reportMissingParameterType"),
# FileSpecificIgnore(substring="cannot be assigned to parameter"),
# ],
FILE_SPECIFIC_IGNORES: FileSpecificIgnores = {}
# Allowing for more readable ignore of common problems, with an easy-to-understand alias
ALIASES: dict[str, str] = {
"awaitable-is-generator": 'Return type of generator function must be "Generator" or "Iterable"',
"obscured-by-same-name": "is obscured by a declaration of the same name",
"int-into-enum": 'Expression of type "int.*" cannot be assigned to return type ".*"',
}
class PyrightTool:
ON_PATTERN: Final = "# pyright: on"
OFF_PATTERN: Final = "# pyright: off"
IGNORE_PATTERN: Final = "# type: ignore"
IGNORE_DELIMITER: Final = ";;"
original_pyright_results: PyrightResults
all_files_to_check: set[str]
all_pyright_ignores: FileIgnores
pyright_off_ignores: FilePyrightOffIgnores
real_errors: list[Error]
unused_ignores: list[str]
inconsistencies: list[str] = []
def __init__(
self,
workdir: Path,
pyright_config_file: io.TextIOWrapper,
*,
file_specific_ignores: FileSpecificIgnores | None = None,
aliases: dict[str, str] | None = None,
input_file: io.TextIOWrapper | None = None,
error_file: io.TextIOWrapper | None = None,
verbose: bool = False,
) -> None:
# validate arguments
if not pyright_config_file.readable():
raise RuntimeError("pyright config file is not readable")
if input_file is not None and not input_file.readable():
raise RuntimeError("input file is not readable")
if error_file is not None and not error_file.writable():
raise RuntimeError("error file is not writable")
# save config
self.workdir = workdir.resolve()
self.pyright_config_data = self.load_config(pyright_config_file)
self.file_specific_ignores = file_specific_ignores or {}
self.aliases = aliases or {}
self.input_file = input_file
self.error_file = error_file
self.verbose = verbose
self.count_of_ignored_errors = 0
self.check_input_correctness()
def check_input_correctness(self) -> None:
"""Verify the input data structures are correct."""
# Checking for correct file_specific_ignores structure
for file, ignores in self.file_specific_ignores.items():
for ignore in ignores:
if not isinstance(ignore, FileSpecificIgnore):
raise RuntimeError(
"All items of file_specific_ignores must be FileSpecificIgnore classes. "
f"Got {ignore} - type {type(ignore)}"
)
# Also putting substrings at the beginning of ignore-lists, so they are matched before rules
# (Not to leave them potentially unused when error would be matched by a rule instead)
self.file_specific_ignores[file].sort(
key=lambda x: x.substring, reverse=True
)
# Checking for correct aliases (dict[str, str] type)
for alias, full_substring in self.aliases.items():
if not isinstance(alias, str) or not isinstance(full_substring, str):
raise RuntimeError(
"All alias keys and values must be strings. "
f"Got {alias} (type {type(alias)}), {full_substring} (type {type(full_substring)}"
)
def run(self) -> None:
"""Main function, putting together all logic and evaluating result."""
self.original_pyright_results = self.get_original_pyright_results()
self.all_files_to_check = self.get_all_files_to_check()
self.all_pyright_ignores = self.get_all_pyright_ignores()
self.pyright_off_ignores = self.get_pyright_off_ignores()
self.real_errors = self.get_all_real_errors()
self.unused_ignores = self.get_unused_ignores()
self.evaluate_final_result()
def evaluate_final_result(self) -> None:
"""Reporting results to the user/CI (printing stuff, deciding exit value)."""
print(
f"\nIgnored {self.count_of_ignored_errors} custom-defined errors "
f"from {len(self.all_pyright_ignores)} files."
)
if self.unused_ignores:
print("\nWARNING: there are unused ignores!")
for unused_ignore in self.unused_ignores:
print(unused_ignore)
if self.inconsistencies:
print("\nWARNING: there are inconsistencies!")
for inconsistency in self.inconsistencies:
print(inconsistency)
if not self.real_errors:
print("\nSUCCESS: Everything is fine!")
if self.unused_ignores or self.inconsistencies:
print("But we have unused ignores or inconsistencies!")
sys.exit(2)
else:
sys.exit(0)
else:
print("\nERROR: We have issues!\n")
for error in self.real_errors:
print(self.get_human_readable_error_string(error))
print(f"Found {len(self.real_errors)} issues above")
if self.unused_ignores or self.inconsistencies:
print("And we have unused ignores or inconsistencies!")
sys.exit(1)
def load_config(self, config: io.TextIOWrapper) -> dict[str, Any]:
"""Load pyright config and validate any errors."""
try:
return json.load(config)
except json.decoder.JSONDecodeError as err:
raise RuntimeError(
f"Pyright config does not contain valid JSON! Err: {err}"
) from err
def get_pyright_output(self) -> str:
"""Run pyright and return its output."""
# generate config with enableTypeIgnoreComments: false
config_data = self.pyright_config_data.copy()
config_data["enableTypeIgnoreComments"] = False
with tempfile.NamedTemporaryFile("w", suffix=".json", dir=self.workdir) as tmp:
json.dump(config_data, tmp)
tmp.flush()
cmd = (
"pyright",
"--outputjson",
"--project",
str(Path(tmp.name).resolve()),
)
# run pyright with generated config
result = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
# Checking if there was no non-type-checking error when running the above command
# Exit code 0 = all fine, no type-checking issues in pyright
# Exit code 1 = pyright has found some type-checking issues (expected)
# All other exit codes mean something non-type-related got wrong (or pyright was not found)
# https://github.com/microsoft/pyright/blob/main/docs/command-line.md#pyright-exit-codes
if result.returncode not in (0, 1):
raise RuntimeError(
f"Running '{' '.join(cmd)}' produced a non-expected exit code (see output above)."
)
if not result.stdout:
raise RuntimeError(
f"Running '{' '.join(cmd)}' produced no data (see output above)."
)
return result.stdout
def get_original_pyright_results(self) -> PyrightResults:
"""Extract pyright results data in a structured format.
That means either running `pyright --outputjson`, or loading the provided JSON
file created by an earlier run.
"""
if self.input_file is not None:
pyright_result_str = self.input_file.read()
else:
pyright_result_str = self.get_pyright_output()
if self.error_file is not None:
self.error_file.write(pyright_result_str)
try:
pyright_results: PyrightResults = json.loads(pyright_result_str)
except json.decoder.JSONDecodeError as err:
raise RuntimeError(
f"Input error file does not contain valid JSON! Err: {err}"
) from None
return pyright_results
def get_all_real_errors(self) -> list[Error]:
"""Analyze all pyright errors and discard all that should be ignored.
Ignores can be different:
- as per "# type: ignore [<error_substring>]" comment
- as per "file_specific_ignores"
- as per "# pyright: off" mark
"""
real_errors: list[Error] = []
for error in self.original_pyright_results["generalDiagnostics"]:
# Special handling of cycle import issues, which have different format
if "range" not in error:
error["range"] = {"start": {"line": 0}}
error["rule"] = "cycleImport"
real_errors.append(error)
continue
file_path = error["file"]
error_message = error["message"]
line_no = error["range"]["start"]["line"]
# Checking for "# type: ignore [<error_substring>]" comment
if self.should_ignore_per_inline_substring(
file_path, error_message, line_no
):
self.count_of_ignored_errors += 1
self.log_ignore(error, "error substring matched")
continue
# Checking in file_specific_ignores
if self.should_ignore_file_specific_error(file_path, error):
self.count_of_ignored_errors += 1
self.log_ignore(error, "file specific error")
continue
# Checking for "# pyright: off" mark
if self.is_line_in_pyright_off_block(file_path, line_no):
self.count_of_ignored_errors += 1
self.log_ignore(error, "pyright disabled for this line")
continue
real_errors.append(error)
return real_errors
def get_all_files_to_check(self) -> set[str]:
"""Get all files to be analyzed by pyright, based on its config."""
all_files: set[Path] = set()
def _all_files(entry: str) -> Iterator[Path]:
file_or_dir = Path(self.workdir / entry)
if file_or_dir.is_file():
yield file_or_dir
else:
yield from file_or_dir.glob("**/*.py")
# include all relevant files.
# use either the entries in `include`, or the current directory
for entry in self.pyright_config_data.get("include", ("",)):
all_files.update(_all_files(entry))
# exclude specified files
for entry in self.pyright_config_data.get("exclude", ()):
all_files -= set(_all_files(entry))
return {str(f) for f in all_files}
def get_all_pyright_ignores(self) -> FileIgnores:
"""Get ignore information from all the files to be analyzed."""
file_ignores: FileIgnores = {}
for file in self.all_files_to_check:
ignores = self.get_inline_type_ignores_from_file(file)
if ignores:
file_ignores[file] = ignores
return file_ignores
def get_pyright_off_ignores(self) -> FilePyrightOffIgnores:
"""Get ignore information based on `# pyright: on/off` marks."""
pyright_off_ignores: FilePyrightOffIgnores = {}
for file in self.all_files_to_check:
ignores = self.find_pyright_off_from_file(file)
if ignores:
pyright_off_ignores[file] = ignores
return pyright_off_ignores
def get_unused_ignores(self) -> list[str]:
"""Evaluate if there are no ignores not matched by pyright errors."""
unused_ignores: list[str] = []
# type: ignore
for file, file_ignores in self.all_pyright_ignores.items():
for line_ignore in file_ignores:
for ignore_statement in line_ignore.ignore_statements:
if not ignore_statement.already_used:
unused_ignores.append(
f"File {file}:{line_ignore.line_no + 1} has unused ignore. "
f"Substring: {ignore_statement.substring}"
)
# Pyright: off
for file, file_ignores in self.pyright_off_ignores.items():
for off_ignore in file_ignores:
if not off_ignore.already_used:
unused_ignores.append(
f"File {file} has unused # pyright: off ignore between lines "
f"{off_ignore.start_line + 1} and {off_ignore.end_line + 1}."
)
# File-specific
for file, file_ignores in self.file_specific_ignores.items():
for ignore_object in file_ignores:
if not ignore_object.already_used:
if ignore_object.substring:
unused_ignores.append(
f"File {file} has unused specific ignore substring. "
f"Substring: {ignore_object.substring}"
)
elif ignore_object.rule:
unused_ignores.append(
f"File {file} has unused specific ignore rule. "
f"Rule: {ignore_object.rule}"
)
return unused_ignores
def should_ignore_per_inline_substring(
self, file: str, error_message: str, line_no: int
) -> bool:
"""Check if line should be ignored based on inline substring/regex."""
if file not in self.all_pyright_ignores:
return False
for ignore_index, ignore in enumerate(self.all_pyright_ignores[file]):
if line_no == ignore.line_no:
for substring_index, ignore_statement in enumerate(
ignore.ignore_statements
):
# Supporting both text substrings and regex patterns
if ignore_statement.substring in error_message or re.search(
ignore_statement.substring, error_message
):
# Marking this ignore to be used (so we can identify unused ignores)
self.all_pyright_ignores[file][ignore_index].ignore_statements[
substring_index
].already_used = True
return True
return False
def should_ignore_file_specific_error(self, file: str, error: Error) -> bool:
"""Check if line should be ignored based on file-specific ignores."""
if file not in self.file_specific_ignores:
return False
for ignore_object in self.file_specific_ignores[file]:
if ignore_object.rule:
if error["rule"] == ignore_object.rule:
ignore_object.already_used = True
return True
elif ignore_object.substring:
# Supporting both text substrings and regex patterns
if ignore_object.substring in error["message"] or re.search(
ignore_object.substring, error["message"]
):
ignore_object.already_used = True
return True
return False
def is_line_in_pyright_off_block(self, file: str, line_no: int) -> bool:
"""Check if line should be ignored based on `# pyright: off` mark."""
if file not in self.pyright_off_ignores:
return False
for off_ignore in self.pyright_off_ignores[file]:
if off_ignore.start_line < line_no < off_ignore.end_line:
off_ignore.already_used = True
return True
return False
def find_pyright_off_from_file(self, file: str) -> PyrightOffIgnores:
"""Get sections in file to be ignored based on `# pyright: off`."""
pyright_off_ignores: PyrightOffIgnores = []
with open(file, "r") as f:
pyright_off = False
start_line = 0
index = 0
for index, line in enumerate(f):
if self.OFF_PATTERN in line and not pyright_off:
start_line = index
pyright_off = True
elif self.ON_PATTERN in line and pyright_off:
pyright_off_ignores.append(PyrightOffIgnore(start_line, index))
pyright_off = False
if pyright_off:
pyright_off_ignores.append(PyrightOffIgnore(start_line, index))
return pyright_off_ignores
def get_inline_type_ignores_from_file(self, file: str) -> LineIgnores:
"""Get all type ignore lines and statements from a certain file."""
ignores: LineIgnores = []
with open(file, "r") as f:
for index, line in enumerate(f):
if self.IGNORE_PATTERN in line:
ignore_statements = self.get_ignore_statements(line)
if not ignore_statements:
self.inconsistencies.append(
f"There is an empty `{self.IGNORE_PATTERN}` in {file}:{index+1}"
)
else:
ignores.append(LineIgnore(index, ignore_statements))
return ignores
def get_ignore_statements(self, line: str) -> list[IgnoreStatement]:
"""Extract error substrings to be ignored from a certain line."""
# Extracting content of [error_substring(s)] after the ignore comment
ignore_part = line.split(self.IGNORE_PATTERN, maxsplit=2)[1]
ignore_content = re.search(r"\[(.*)\]", ignore_part)
# We should not be using empty `# type: ignore` without content in []
# Notifying the parent function that we should do something about it
if not ignore_content:
return []
# There might be more than one substring
statement_substrings = ignore_content.group(1).split(self.IGNORE_DELIMITER)
# When finding aliases, replacing them with a real substring
statement_substrings = [self.aliases.get(ss, ss) for ss in statement_substrings]
return [IgnoreStatement(substr) for substr in statement_substrings]
def log_ignore(self, error: Error, reason: str) -> None:
"""Print the action of ignoring certain error into the console."""
if self.verbose:
err = self.get_human_readable_error_string(error)
print(f"\nError ignored. Reason: {reason}.\nErr: {err}")
@staticmethod
def get_human_readable_error_string(error: Error) -> str:
"""Transform error object to a string readable by human."""
file = error["file"]
message = error["message"]
rule = error.get("rule", "No specific rule")
line = error["range"]["start"]["line"]
# Need to add +1 to the line, as it is zero-based index
return f"{file}:{line + 1}: - error: {message} ({rule})\n"
@click.command()
@click.argument(
"workdir", type=click.Path(exists=True, file_okay=False, dir_okay=True), default="."
)
@click.option(
"--config",
type=click.File("r"),
help="Pyright configuration file. Defaults to pyrightconfig.json in the selected (or current) directory.",
)
@click.option(
"-o",
"--output",
"output_file",
type=click.File("w"),
help="Save pyright JSON output to file",
)
@click.option(
"-i",
"--input",
"input_file",
type=click.File("r"),
help="Use input file instead of running pyright",
)
@click.option("-v", "--verbose", is_flag=True, help="Print verbose output")
def main(
config: io.TextIOWrapper | None,
input_file: io.TextIOWrapper | None,
output_file: io.TextIOWrapper | None,
verbose: bool,
workdir: str | Path,
) -> None:
workdir = Path(workdir)
if config is None:
config_path = workdir / "pyrightconfig.json"
try:
config = open(config_path)
except Exception:
raise click.ClickException(f"Failed to load {config_path}")
try:
tool = PyrightTool(
workdir=workdir,
pyright_config_file=config,
file_specific_ignores=FILE_SPECIFIC_IGNORES,
aliases=ALIASES,
input_file=input_file,
error_file=output_file,
verbose=verbose,
)
tool.run()
except Exception as e:
raise click.ClickException(str(e)) from e
if __name__ == "__main__":
main()