feat(ci): add pyrefly type coverage reporting to CI (#34754)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

.github/workflows/pyrefly-type-coverage-comment.yml

@@ -0,0 +1,118 @@
+name: Comment with Pyrefly Type Coverage
+
+on:
+  workflow_run:
+    workflows:
+      - Pyrefly Type Coverage
+    types:
+      - completed
+
+permissions: {}
+
+jobs:
+  comment:
+    name: Comment PR with type coverage
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+      pull-requests: write
+    if: ${{ github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.pull_requests[0].head.repo.full_name != github.repository }}
+    steps:
+      - name: Checkout default branch (trusted code)
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+
+      - name: Setup Python & UV
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync --project api --dev
+
+      - name: Download type coverage artifact
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: ${{ github.event.workflow_run.id }},
+            });
+            const match = artifacts.data.artifacts.find((artifact) =>
+              artifact.name === 'pyrefly_type_coverage'
+            );
+            if (!match) {
+              throw new Error('pyrefly_type_coverage artifact not found');
+            }
+            const download = await github.rest.actions.downloadArtifact({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              artifact_id: match.id,
+              archive_format: 'zip',
+            });
+            fs.writeFileSync('pyrefly_type_coverage.zip', Buffer.from(download.data));
+
+      - name: Unzip artifact
+        run: unzip -o pyrefly_type_coverage.zip
+
+      - name: Render coverage markdown from structured data
+        id: render
+        run: |
+          comment_body="$(uv run --directory api python api/libs/pyrefly_type_coverage.py \
+            --base base_report.json \
+            < pr_report.json)"
+
+          {
+            echo "### Pyrefly Type Coverage"
+            echo ""
+            echo "$comment_body"
+          } > /tmp/type_coverage_comment.md
+
+      - name: Post comment
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const body = fs.readFileSync('/tmp/type_coverage_comment.md', { encoding: 'utf8' });
+            let prNumber = null;
+            try {
+              prNumber = parseInt(fs.readFileSync('pr_number.txt', { encoding: 'utf8' }), 10);
+            } catch (err) {
+              const prs = context.payload.workflow_run.pull_requests || [];
+              if (prs.length > 0 && prs[0].number) {
+                prNumber = prs[0].number;
+              }
+            }
+            if (!prNumber) {
+              throw new Error('PR number not found in artifact or workflow_run payload');
+            }
+
+            // Update existing comment if one exists, otherwise create new
+            const { data: comments } = await github.rest.issues.listComments({
+              issue_number: prNumber,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            });
+            const marker = '### Pyrefly Type Coverage';
+            const existing = comments.find(c => c.body.startsWith(marker));
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                comment_id: existing.id,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                issue_number: prNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body,
+              });
+            }

.github/workflows/pyrefly-type-coverage.yml

@@ -0,0 +1,120 @@
+name: Pyrefly Type Coverage
+
+on:
+  pull_request:
+    paths:
+      - 'api/**/*.py'
+
+permissions:
+  contents: read
+
+jobs:
+  pyrefly-type-coverage:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+      pull-requests: write
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python & UV
+        uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync --project api --dev
+
+      - name: Run pyrefly report on PR branch
+        run: |
+          uv run --directory api --dev pyrefly report 2>/dev/null > /tmp/pyrefly_report_pr.tmp && \
+            mv /tmp/pyrefly_report_pr.tmp /tmp/pyrefly_report_pr.json || \
+            echo '{}' > /tmp/pyrefly_report_pr.json
+
+      - name: Save helper script from base branch
+        run: |
+          git show ${{ github.event.pull_request.base.sha }}:api/libs/pyrefly_type_coverage.py > /tmp/pyrefly_type_coverage.py 2>/dev/null \
+            || cp api/libs/pyrefly_type_coverage.py /tmp/pyrefly_type_coverage.py
+
+      - name: Checkout base branch
+        run: git checkout ${{ github.base_ref }}
+
+      - name: Run pyrefly report on base branch
+        run: |
+          uv run --directory api --dev pyrefly report 2>/dev/null > /tmp/pyrefly_report_base.tmp && \
+            mv /tmp/pyrefly_report_base.tmp /tmp/pyrefly_report_base.json || \
+            echo '{}' > /tmp/pyrefly_report_base.json
+
+      - name: Generate coverage comparison
+        id: coverage
+        run: |
+          comment_body="$(uv run --directory api python /tmp/pyrefly_type_coverage.py \
+            --base /tmp/pyrefly_report_base.json \
+            < /tmp/pyrefly_report_pr.json)"
+
+          {
+            echo "### Pyrefly Type Coverage"
+            echo ""
+            echo "$comment_body"
+          } | tee -a "$GITHUB_STEP_SUMMARY" > /tmp/type_coverage_comment.md
+
+          # Save structured data for the fork-PR comment workflow
+          cp /tmp/pyrefly_report_pr.json pr_report.json
+          cp /tmp/pyrefly_report_base.json base_report.json
+
+      - name: Save PR number
+        run: |
+          echo ${{ github.event.pull_request.number }} > pr_number.txt
+
+      - name: Upload type coverage artifact
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: pyrefly_type_coverage
+          path: |
+            pr_report.json
+            base_report.json
+            pr_number.txt
+
+      - name: Comment PR with type coverage
+        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const marker = '### Pyrefly Type Coverage';
+            let body;
+            try {
+              body = fs.readFileSync('/tmp/type_coverage_comment.md', { encoding: 'utf8' });
+            } catch {
+              body = `${marker}\n\n_Coverage report unavailable._`;
+            }
+            const prNumber = context.payload.pull_request.number;
+
+            // Update existing comment if one exists, otherwise create new
+            const { data: comments } = await github.rest.issues.listComments({
+              issue_number: prNumber,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            });
+            const existing = comments.find(c => c.body.startsWith(marker));
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                comment_id: existing.id,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                issue_number: prNumber,
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                body,
+              });
+            }

api/libs/pyrefly_type_coverage.py

@@ -0,0 +1,145 @@
+"""Helpers for generating type-coverage summaries from pyrefly report output."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import TypedDict
+
+
+class CoverageSummary(TypedDict):
+    n_modules: int
+    n_typable: int
+    n_typed: int
+    n_any: int
+    n_untyped: int
+    coverage: float
+    strict_coverage: float
+
+
+_REQUIRED_KEYS = frozenset(CoverageSummary.__annotations__)
+
+_EMPTY_SUMMARY: CoverageSummary = {
+    "n_modules": 0,
+    "n_typable": 0,
+    "n_typed": 0,
+    "n_any": 0,
+    "n_untyped": 0,
+    "coverage": 0.0,
+    "strict_coverage": 0.0,
+}
+
+
+def parse_summary(report_json: str) -> CoverageSummary:
+    """Extract the summary section from ``pyrefly report`` JSON output.
+
+    Returns an empty summary when *report_json* is empty or malformed so that
+    the CI workflow can degrade gracefully instead of crashing.
+    """
+    if not report_json or not report_json.strip():
+        return _EMPTY_SUMMARY.copy()
+
+    try:
+        data = json.loads(report_json)
+    except json.JSONDecodeError:
+        return _EMPTY_SUMMARY.copy()
+
+    summary = data.get("summary")
+    if not isinstance(summary, dict) or not _REQUIRED_KEYS.issubset(summary):
+        return _EMPTY_SUMMARY.copy()
+
+    return {
+        "n_modules": summary["n_modules"],
+        "n_typable": summary["n_typable"],
+        "n_typed": summary["n_typed"],
+        "n_any": summary["n_any"],
+        "n_untyped": summary["n_untyped"],
+        "coverage": summary["coverage"],
+        "strict_coverage": summary["strict_coverage"],
+    }
+
+
+def format_summary_markdown(summary: CoverageSummary) -> str:
+    """Format a single coverage summary as a Markdown table."""
+
+    return (
+        "| Metric | Value |\n"
+        "| --- | ---: |\n"
+        f"| Modules | {summary['n_modules']} |\n"
+        f"| Typable symbols | {summary['n_typable']:,} |\n"
+        f"| Typed symbols | {summary['n_typed']:,} |\n"
+        f"| Untyped symbols | {summary['n_untyped']:,} |\n"
+        f"| Any symbols | {summary['n_any']:,} |\n"
+        f"| **Type coverage** | **{summary['coverage']:.2f}%** |\n"
+        f"| Strict coverage | {summary['strict_coverage']:.2f}% |"
+    )
+
+
+def format_comparison_markdown(
+    base: CoverageSummary,
+    pr: CoverageSummary,
+) -> str:
+    """Format a comparison between base and PR coverage as Markdown."""
+
+    coverage_delta = pr["coverage"] - base["coverage"]
+    strict_delta = pr["strict_coverage"] - base["strict_coverage"]
+    typed_delta = pr["n_typed"] - base["n_typed"]
+    untyped_delta = pr["n_untyped"] - base["n_untyped"]
+
+    def _fmt_delta(value: float, fmt: str = ".2f") -> str:
+        sign = "+" if value > 0 else ""
+        return f"{sign}{value:{fmt}}"
+
+    lines = [
+        "| Metric | Base | PR | Delta |",
+        "| --- | ---: | ---: | ---: |",
+        (f"| **Type coverage** | {base['coverage']:.2f}% | {pr['coverage']:.2f}% | {_fmt_delta(coverage_delta)}% |"),
+        (
+            f"| Strict coverage | {base['strict_coverage']:.2f}% "
+            f"| {pr['strict_coverage']:.2f}% "
+            f"| {_fmt_delta(strict_delta)}% |"
+        ),
+        (f"| Typed symbols | {base['n_typed']:,} | {pr['n_typed']:,} | {_fmt_delta(typed_delta, ',')} |"),
+        (f"| Untyped symbols | {base['n_untyped']:,} | {pr['n_untyped']:,} | {_fmt_delta(untyped_delta, ',')} |"),
+        (
+            f"| Modules | {base['n_modules']} "
+            f"| {pr['n_modules']} "
+            f"| {_fmt_delta(pr['n_modules'] - base['n_modules'], ',')} |"
+        ),
+    ]
+    return "\n".join(lines)
+
+
+def main() -> int:
+    """Read pyrefly report JSON from stdin and print a Markdown summary.
+
+    Accepts an optional ``--base <file>`` argument. When provided, the output
+    includes a base-vs-PR comparison table.
+    """
+
+    args = sys.argv[1:]
+
+    base_file: str | None = None
+    if "--base" in args:
+        idx = args.index("--base")
+        if idx + 1 >= len(args):
+            sys.stderr.write("error: --base requires a file path\n")
+            return 1
+        base_file = args[idx + 1]
+
+    pr_report = sys.stdin.read()
+    pr_summary = parse_summary(pr_report)
+
+    if base_file is not None:
+        base_text = Path(base_file).read_text() if Path(base_file).exists() else ""
+        base_summary = parse_summary(base_text)
+        sys.stdout.write(format_comparison_markdown(base_summary, pr_summary) + "\n")
+    else:
+        sys.stdout.write(format_summary_markdown(pr_summary) + "\n")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

api/tests/unit_tests/libs/test_pyrefly_type_coverage.py

@@ -0,0 +1,138 @@
+import json
+
+from libs.pyrefly_type_coverage import (
+    CoverageSummary,
+    format_comparison_markdown,
+    format_summary_markdown,
+    parse_summary,
+)
+
+
+def _make_report(summary: dict) -> str:
+    return json.dumps({"module_reports": [], "summary": summary})
+
+
+_SAMPLE_SUMMARY: dict = {
+    "n_modules": 100,
+    "n_typable": 1000,
+    "n_typed": 400,
+    "n_any": 50,
+    "n_untyped": 550,
+    "coverage": 45.0,
+    "strict_coverage": 40.0,
+    "n_functions": 200,
+    "n_methods": 300,
+    "n_function_params": 150,
+    "n_method_params": 250,
+    "n_classes": 80,
+    "n_attrs": 40,
+    "n_properties": 20,
+    "n_type_ignores": 10,
+}
+
+
+def _make_summary(
+    *,
+    n_modules: int = 100,
+    n_typable: int = 1000,
+    n_typed: int = 400,
+    n_any: int = 50,
+    n_untyped: int = 550,
+    coverage: float = 45.0,
+    strict_coverage: float = 40.0,
+) -> CoverageSummary:
+    return {
+        "n_modules": n_modules,
+        "n_typable": n_typable,
+        "n_typed": n_typed,
+        "n_any": n_any,
+        "n_untyped": n_untyped,
+        "coverage": coverage,
+        "strict_coverage": strict_coverage,
+    }
+
+
+def test_parse_summary_extracts_fields() -> None:
+    report_json = _make_report(_SAMPLE_SUMMARY)
+
+    result = parse_summary(report_json)
+
+    assert result["n_modules"] == 100
+    assert result["n_typable"] == 1000
+    assert result["n_typed"] == 400
+    assert result["n_any"] == 50
+    assert result["n_untyped"] == 550
+    assert result["coverage"] == 45.0
+    assert result["strict_coverage"] == 40.0
+
+
+def test_parse_summary_handles_empty_input() -> None:
+    assert parse_summary("")["n_modules"] == 0
+    assert parse_summary("   ")["n_modules"] == 0
+
+
+def test_parse_summary_handles_invalid_json() -> None:
+    assert parse_summary("not json")["n_modules"] == 0
+
+
+def test_parse_summary_handles_missing_summary_key() -> None:
+    assert parse_summary(json.dumps({"other": 1}))["n_modules"] == 0
+
+
+def test_parse_summary_handles_incomplete_summary() -> None:
+    partial = json.dumps({"summary": {"n_modules": 5}})
+    assert parse_summary(partial)["n_modules"] == 0
+
+
+def test_format_summary_markdown_contains_key_metrics() -> None:
+    summary = _make_summary()
+
+    result = format_summary_markdown(summary)
+
+    assert "**Type coverage**" in result
+    assert "45.00%" in result
+    assert "40.00%" in result
+    assert "| Modules | 100 |" in result
+
+
+def test_format_comparison_markdown_shows_positive_delta() -> None:
+    base = _make_summary()
+    pr = _make_summary(
+        n_modules=101,
+        n_typable=1010,
+        n_typed=420,
+        n_untyped=540,
+        coverage=46.53,
+        strict_coverage=41.58,
+    )
+
+    result = format_comparison_markdown(base, pr)
+
+    assert "| Base | PR | Delta |" in result
+    assert "+1.53%" in result
+    assert "+1.58%" in result
+    assert "+20" in result
+
+
+def test_format_comparison_markdown_shows_negative_delta() -> None:
+    base = _make_summary()
+    pr = _make_summary(
+        n_typed=390,
+        n_any=60,
+        coverage=44.0,
+        strict_coverage=39.0,
+    )
+
+    result = format_comparison_markdown(base, pr)
+
+    assert "-1.00%" in result
+    assert "-10" in result
+
+
+def test_format_comparison_markdown_shows_zero_delta() -> None:
+    summary = _make_summary()
+
+    result = format_comparison_markdown(summary, summary)
+
+    assert "0.00%" in result
+    assert "| 0 |" in result