vercel/.github/workflows/cli-evals.yml at main · CodeBoarding/vercel · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
name: CLI Evals

# Runs weekly and can also be started manually via "Run workflow".
on:
  workflow_dispatch:
  schedule:
    # Run weekly on Tuesday at 9:00 AM UTC.
    - cron: '0 9 * * 2'

jobs:
  evals:
    runs-on: ubuntu-latest
    timeout-minutes: 90
    # TODO: Remove continue-on-error when evals should block PR merge.
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        experiment:
          - cli-claude-opus-4.7
          - cli-gpt-5.4-medium
          - cli-gpt-5.5-pro
          - cli-claude-opus-4.6
        skills_mode:
          - with-skills
          - without-skills
    # Evals use separate secrets (EVAL_*) so you can point to a dedicated evals team.
    # Same pattern as test.yml / test-e2e.yml: VERCEL_TOKEN + team/project IDs; add EVAL_AI_GATEWAY_API_KEY for the agent.
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # Check out the PR commit directly instead of a merge commit.
          # Empty on schedule/workflow_dispatch, which falls back to the default ref.
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20.x

      - name: Install pnpm
        run: npm i -g pnpm@10.29.3

      - name: Install dependencies
        run: pnpm install --no-frozen-lockfile

      - name: Build CLI
        if: false # skip for now; enable when monorepo build works in CI
        run: cd packages/cli && pnpm build

      - name: Run CLI evals
        id: evals
        run: cd packages/cli && pnpm test:evals
        env:
          AI_GATEWAY_API_KEY: ${{ secrets.EVAL_AI_GATEWAY_API_KEY }}
          VERCEL_TOKEN: ${{ secrets.EVAL_TOKEN }}
          VERCEL_TEAM_ID: ${{ secrets.EVAL_TEAM_ID }}
          VERCEL_PROJECT_ID: ${{ secrets.EVAL_PROJECT_ID }}
          CLI_EVAL_TEAM_ID: ${{ secrets.EVAL_TEAM_ID }}
          CLI_EVAL_PROJECT_ID: ${{ secrets.EVAL_PROJECT_ID }}
          # Run one skills mode per matrix job so slower models do not block the full matrix.
          CLI_EVAL_SKILLS_MODES: ${{ matrix.skills_mode }}
          # Run comparable agent/model experiments. Matches the next-evals-oss
          # pattern where each model has its own experiment file.
          CLI_EVAL_EXPERIMENTS: ${{ matrix.experiment }}
          # Limit memory pressure from concurrent Docker + Node (reduce OOM exit 137)
          NODE_OPTIONS: --max-old-space-size=4096

      - name: Upload results folder to centralized evals app
        if: always()
        env:
          BATCH_ID: daily-${{ github.repository }}-${{ github.run_id }}
          EXPERIMENTS: vercel-cli
        run: |
          cd packages/cli && node ./evals/scripts/transform-agent-eval-to-canonical.js \
            --results-dir ./evals/results \
            --product evals \
            --repo ${{ github.repository }} \
            --branch ${{ github.ref_name }} \
            --commit-sha ${{ github.sha }} \
            --run-type "${{ github.event_name == 'schedule' && 'cron' || 'manual' }}" \
            --runner agent-eval \
            --batch-id "$BATCH_ID" \
            --ingest-url "${{ secrets.EVALS_INGEST_URL }}" \
            --token "${{ secrets.EVALS_INGEST_TOKEN }}" \
            --protection-bypass-secret "${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }}" \
            --bypass-via header