diff --git a/api/libs/pyrefly_type_coverage.py b/api/libs/pyrefly_type_coverage.py new file mode 100644 index 0000000000..369b8dff3c --- /dev/null +++ b/api/libs/pyrefly_type_coverage.py @@ -0,0 +1,145 @@ +"""Helpers for generating type-coverage summaries from pyrefly report output.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import TypedDict + + +class CoverageSummary(TypedDict): + n_modules: int + n_typable: int + n_typed: int + n_any: int + n_untyped: int + coverage: float + strict_coverage: float + + +_REQUIRED_KEYS = frozenset(CoverageSummary.__annotations__) + +_EMPTY_SUMMARY: CoverageSummary = { + "n_modules": 0, + "n_typable": 0, + "n_typed": 0, + "n_any": 0, + "n_untyped": 0, + "coverage": 0.0, + "strict_coverage": 0.0, +} + + +def parse_summary(report_json: str) -> CoverageSummary: + """Extract the summary section from ``pyrefly report`` JSON output. + + Returns an empty summary when *report_json* is empty or malformed so that + the CI workflow can degrade gracefully instead of crashing. + """ + if not report_json or not report_json.strip(): + return _EMPTY_SUMMARY.copy() + + try: + data = json.loads(report_json) + except json.JSONDecodeError: + return _EMPTY_SUMMARY.copy() + + summary = data.get("summary") + if not isinstance(summary, dict) or not _REQUIRED_KEYS.issubset(summary): + return _EMPTY_SUMMARY.copy() + + return { + "n_modules": summary["n_modules"], + "n_typable": summary["n_typable"], + "n_typed": summary["n_typed"], + "n_any": summary["n_any"], + "n_untyped": summary["n_untyped"], + "coverage": summary["coverage"], + "strict_coverage": summary["strict_coverage"], + } + + +def format_summary_markdown(summary: CoverageSummary) -> str: + """Format a single coverage summary as a Markdown table.""" + + return ( + "| Metric | Value |\n" + "| --- | ---: |\n" + f"| Modules | {summary['n_modules']} |\n" + f"| Typable symbols | {summary['n_typable']:,} |\n" + f"| Typed symbols | {summary['n_typed']:,} |\n" + f"| Untyped symbols | {summary['n_untyped']:,} |\n" + f"| Any symbols | {summary['n_any']:,} |\n" + f"| **Type coverage** | **{summary['coverage']:.2f}%** |\n" + f"| Strict coverage | {summary['strict_coverage']:.2f}% |" + ) + + +def format_comparison_markdown( + base: CoverageSummary, + pr: CoverageSummary, +) -> str: + """Format a comparison between base and PR coverage as Markdown.""" + + coverage_delta = pr["coverage"] - base["coverage"] + strict_delta = pr["strict_coverage"] - base["strict_coverage"] + typed_delta = pr["n_typed"] - base["n_typed"] + untyped_delta = pr["n_untyped"] - base["n_untyped"] + + def _fmt_delta(value: float, fmt: str = ".2f") -> str: + sign = "+" if value > 0 else "" + return f"{sign}{value:{fmt}}" + + lines = [ + "| Metric | Base | PR | Delta |", + "| --- | ---: | ---: | ---: |", + (f"| **Type coverage** | {base['coverage']:.2f}% | {pr['coverage']:.2f}% | {_fmt_delta(coverage_delta)}% |"), + ( + f"| Strict coverage | {base['strict_coverage']:.2f}% " + f"| {pr['strict_coverage']:.2f}% " + f"| {_fmt_delta(strict_delta)}% |" + ), + (f"| Typed symbols | {base['n_typed']:,} | {pr['n_typed']:,} | {_fmt_delta(typed_delta, ',')} |"), + (f"| Untyped symbols | {base['n_untyped']:,} | {pr['n_untyped']:,} | {_fmt_delta(untyped_delta, ',')} |"), + ( + f"| Modules | {base['n_modules']} " + f"| {pr['n_modules']} " + f"| {_fmt_delta(pr['n_modules'] - base['n_modules'], ',')} |" + ), + ] + return "\n".join(lines) + + +def main() -> int: + """Read pyrefly report JSON from stdin and print a Markdown summary. + + Accepts an optional ``--base `` argument. When provided, the output + includes a base-vs-PR comparison table. + """ + + args = sys.argv[1:] + + base_file: str | None = None + if "--base" in args: + idx = args.index("--base") + if idx + 1 >= len(args): + sys.stderr.write("error: --base requires a file path\n") + return 1 + base_file = args[idx + 1] + + pr_report = sys.stdin.read() + pr_summary = parse_summary(pr_report) + + if base_file is not None: + base_text = Path(base_file).read_text() if Path(base_file).exists() else "" + base_summary = parse_summary(base_text) + sys.stdout.write(format_comparison_markdown(base_summary, pr_summary) + "\n") + else: + sys.stdout.write(format_summary_markdown(pr_summary) + "\n") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/api/tests/unit_tests/libs/test_pyrefly_type_coverage.py b/api/tests/unit_tests/libs/test_pyrefly_type_coverage.py new file mode 100644 index 0000000000..7087490845 --- /dev/null +++ b/api/tests/unit_tests/libs/test_pyrefly_type_coverage.py @@ -0,0 +1,138 @@ +import json + +from libs.pyrefly_type_coverage import ( + CoverageSummary, + format_comparison_markdown, + format_summary_markdown, + parse_summary, +) + + +def _make_report(summary: dict) -> str: + return json.dumps({"module_reports": [], "summary": summary}) + + +_SAMPLE_SUMMARY: dict = { + "n_modules": 100, + "n_typable": 1000, + "n_typed": 400, + "n_any": 50, + "n_untyped": 550, + "coverage": 45.0, + "strict_coverage": 40.0, + "n_functions": 200, + "n_methods": 300, + "n_function_params": 150, + "n_method_params": 250, + "n_classes": 80, + "n_attrs": 40, + "n_properties": 20, + "n_type_ignores": 10, +} + + +def _make_summary( + *, + n_modules: int = 100, + n_typable: int = 1000, + n_typed: int = 400, + n_any: int = 50, + n_untyped: int = 550, + coverage: float = 45.0, + strict_coverage: float = 40.0, +) -> CoverageSummary: + return { + "n_modules": n_modules, + "n_typable": n_typable, + "n_typed": n_typed, + "n_any": n_any, + "n_untyped": n_untyped, + "coverage": coverage, + "strict_coverage": strict_coverage, + } + + +def test_parse_summary_extracts_fields() -> None: + report_json = _make_report(_SAMPLE_SUMMARY) + + result = parse_summary(report_json) + + assert result["n_modules"] == 100 + assert result["n_typable"] == 1000 + assert result["n_typed"] == 400 + assert result["n_any"] == 50 + assert result["n_untyped"] == 550 + assert result["coverage"] == 45.0 + assert result["strict_coverage"] == 40.0 + + +def test_parse_summary_handles_empty_input() -> None: + assert parse_summary("")["n_modules"] == 0 + assert parse_summary(" ")["n_modules"] == 0 + + +def test_parse_summary_handles_invalid_json() -> None: + assert parse_summary("not json")["n_modules"] == 0 + + +def test_parse_summary_handles_missing_summary_key() -> None: + assert parse_summary(json.dumps({"other": 1}))["n_modules"] == 0 + + +def test_parse_summary_handles_incomplete_summary() -> None: + partial = json.dumps({"summary": {"n_modules": 5}}) + assert parse_summary(partial)["n_modules"] == 0 + + +def test_format_summary_markdown_contains_key_metrics() -> None: + summary = _make_summary() + + result = format_summary_markdown(summary) + + assert "**Type coverage**" in result + assert "45.00%" in result + assert "40.00%" in result + assert "| Modules | 100 |" in result + + +def test_format_comparison_markdown_shows_positive_delta() -> None: + base = _make_summary() + pr = _make_summary( + n_modules=101, + n_typable=1010, + n_typed=420, + n_untyped=540, + coverage=46.53, + strict_coverage=41.58, + ) + + result = format_comparison_markdown(base, pr) + + assert "| Base | PR | Delta |" in result + assert "+1.53%" in result + assert "+1.58%" in result + assert "+20" in result + + +def test_format_comparison_markdown_shows_negative_delta() -> None: + base = _make_summary() + pr = _make_summary( + n_typed=390, + n_any=60, + coverage=44.0, + strict_coverage=39.0, + ) + + result = format_comparison_markdown(base, pr) + + assert "-1.00%" in result + assert "-10" in result + + +def test_format_comparison_markdown_shows_zero_delta() -> None: + summary = _make_summary() + + result = format_comparison_markdown(summary, summary) + + assert "0.00%" in result + assert "| 0 |" in result