eval-gate.yml

name: Eval Gate

# Step 5.3 — offline golden-set eval gate.
#
# Runs the deterministic golden-set harness, compares the fresh metrics against
# the committed baseline (tests/eval/baselines/main.json), enforces the
# thresholds.yaml hard gates (absolute floors + regression deltas), and posts a
# baseline-vs-current diff table as a sticky PR comment.  The job fails when a
# gated metric drops below its floor or regresses beyond tolerance.
#
# Deliberately a standalone workflow (not folded into ci.yml): it is the only
# job that needs `pull-requests: write`, so isolating it keeps the heavy test
# matrix on a least-privilege read-only token.  It is intentionally NOT
# path-filtered — the harness is fast and deterministic, and an always-running
# job is a clean required status check (no "skipped required check" ambiguity).
# Add "Eval gate" as a required check in branch protection to make it blocking.

on:
  push:
    branches: [main]
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  eval-gate:
    name: Eval gate
    runs-on: ubuntu-22.04
    permissions:
      contents: read
      pull-requests: write
    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          version: "latest"
          enable-cache: true
          cache-dependency-glob: "uv.lock"

      - name: Set up Python 3.12
        run: uv python install 3.12

      - name: Install Python dependencies
        run: uv sync --all-packages

      - name: Run eval gate
        id: gate
        # Always exits 0 so the comment is posted even on a regression; the
        # pass/fail verdict is recorded as a step output and enforced below.
        run: |
          if uv run python -m eval.golden_set_v0.gate --comment-out gate-comment.md; then
            echo "passed=true" >> "$GITHUB_OUTPUT"
          else
            echo "passed=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Post eval-gate PR comment
        if: github.event_name == 'pull_request'
        continue-on-error: true # fork PRs get a read-only token — never fail the gate on a comment error
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const body = fs.readFileSync('gate-comment.md', 'utf8');
            const marker = '<!-- agentcontextos-eval-gate -->';
            const { owner, repo } = context.repo;
            const issue_number = context.issue.number;
            const comments = await github.paginate(github.rest.issues.listComments, {
              owner, repo, issue_number, per_page: 100,
            });
            const existing = comments.find((c) => c.body && c.body.includes(marker));
            if (existing) {
              await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
            } else {
              await github.rest.issues.createComment({ owner, repo, issue_number, body });
            }

      - name: Enforce eval gate
        if: steps.gate.outputs.passed == 'false'
        run: |
          echo "::error::Eval gate failed — a gated metric is below its floor or regressed beyond tolerance. See the PR comment / log above."
          exit 1