Add benchmark harness with ClawBench, Kernel MCP, and site-config

.github/workflows/_run.yml

@@ -0,0 +1,121 @@
+name: Run benchmark
+
+on:
+  workflow_call:
+    inputs:
+      benchmark:
+        description: Name of a file in benchmarks/
+        required: true
+        type: string
+      arms:
+        description: Comma-separated arm names, or empty for every arm in the benchmark
+        required: false
+        default: ""
+        type: string
+      tasks:
+        description: Comma-separated task IDs, or all
+        required: false
+        default: all
+        type: string
+      concurrency:
+        description: Concurrent trials per arm
+        required: false
+        default: "10"
+        type: string
+      toolset_ref:
+        description: Git ref for toolsets built from source
+        required: false
+        default: ""
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    # Repository and organization secrets; a repository secret overrides an
+    # organization secret with the same name. Every benchmark sees all of
+    # them; suites and agents only read the ones they need.
+    env:
+      BENCHMARK_NAME: ${{ inputs.benchmark }}
+      KERNEL_API_KEY: ${{ secrets.KERNEL_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      PURELY_MAIL_API_KEY: ${{ secrets.PURELY_MAIL_API_KEY }}
+      PURELY_MAIL_DOMAIN: ${{ secrets.PURELY_MAIL_DOMAIN }}
+      # ClawBench's judge is part of the benchmark definition, so it is fixed here.
+      CLAWBENCH_JUDGE_BASE_URL: https://openrouter.ai/api/v1
+      CLAWBENCH_JUDGE_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+      CLAWBENCH_JUDGE_MODEL: deepseek/deepseek-v4-pro
+      CLAWBENCH_JUDGE_API_TYPE: openai-completions
+      BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
+      BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }}
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          persist-credentials: false
+
+      - uses: astral-sh/setup-uv@b75a909f75acd358c2196fb9a5f1299a9a8868a4 # v6.7.0
+        with:
+          version: "0.8.17"
+
+      - uses: oven-sh/setup-bun@3d267786b128fe76c2f16a390aa2448b815359f3 # v2.1.2
+        with:
+          bun-version: "1.3.3"
+
+      - name: Install
+        run: |
+          uv sync --frozen
+          bun install --cwd report --frozen-lockfile
+
+      # Leave time for the report when a run hits GitHub's six-hour job limit.
+      - name: Run arms
+        id: run
+        timeout-minutes: 330
+        env:
+          ARMS: ${{ inputs.arms }}
+          TASKS: ${{ inputs.tasks }}
+          CONCURRENCY: ${{ inputs.concurrency }}
+          TOOLSET_REF: ${{ inputs.toolset_ref }}
+        run: |
+          run_id="$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT"
+          jobs_dir="$RUNNER_TEMP/jobs"
+          echo "run_file=$jobs_dir/run-$run_id.json" >>"$GITHUB_OUTPUT"
+          echo "experiment=$BENCHMARK_NAME-$run_id" >>"$GITHUB_OUTPUT"
+          args=(--tasks "$TASKS" --concurrency "$CONCURRENCY" --jobs-dir "$jobs_dir" --run-id "$run_id")
+          [[ -n "$ARMS" ]] && args+=(--arms "$ARMS")
+          [[ -n "$TOOLSET_REF" ]] && args+=(--toolset-ref "$TOOLSET_REF")
+          uv run bench run "$BENCHMARK_NAME" "${args[@]}"
+
+      - name: Report
+        if: always() && steps.run.outputs.run_file != ''
+        env:
+          RUN_FILE: ${{ steps.run.outputs.run_file }}
+          EXPERIMENT: ${{ steps.run.outputs.experiment }}
+        run: |
+          [[ -f "$RUN_FILE" ]] || { echo "No arm ran" >&2; exit 1; }
+          arms=() statuses=()
+          # An arm without a status was still running when the step timed out.
+          while IFS=$'\t' read -r name status job_dir; do
+            statuses+=(--status "$name=$status")
+            [[ -f "$job_dir/result.json" ]] && arms+=(--arm "$name=$job_dir")
+          done < <(jq -r 'to_entries[] | [.key, (.value.status // 124), .value.job_dir] | @tsv' "$RUN_FILE")
+          ((${#arms[@]} > 0)) || { echo "No Harbor job completed" >&2; exit 1; }
+
+          publication=()
+          if [[ -n "$BRAINTRUST_API_KEY" ]]; then
+            if bun report/publish-braintrust.ts --experiment "$EXPERIMENT" \
+              --output "$RUNNER_TEMP/publication.json" "${arms[@]}"; then
+              publication=(--publication "$RUNNER_TEMP/publication.json")
+            else
+              echo "::warning::Braintrust publishing failed"
+            fi
+          fi
+
+          bun report/report.ts \
+            --json "$RUNNER_TEMP/summary.json" --markdown "$RUNNER_TEMP/summary.md" \
+            "${publication[@]}" "${statuses[@]}" "${arms[@]}"
+          cat "$RUNNER_TEMP/summary.md" >>"$GITHUB_STEP_SUMMARY"
+          jq -e 'all(.arms[]; .complete == true)' "$RUNNER_TEMP/summary.json" >/dev/null

.github/workflows/ci.yml

@@ -0,0 +1,43 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  harness:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@b75a909f75acd358c2196fb9a5f1299a9a8868a4 # v6.7.0
+        with:
+          version: "0.8.17"
+      - run: uv sync --frozen
+      - run: uv run ruff check .
+      - run: uv run ruff format --check .
+      - run: uv run pytest -q
+      - name: Prepare every static arm for one task
+        run: uv run bench prepare kernel-mcp-clawbench --arms codex-kernel-mcp,claude-kernel-mcp --tasks v2-1134-chapter-finder-redcross --out "$RUNNER_TEMP/prepared"
+
+  report:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: report
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          persist-credentials: false
+      - uses: oven-sh/setup-bun@3d267786b128fe76c2f16a390aa2448b815359f3 # v2.1.2
+        with:
+          bun-version: "1.3.3"
+      - run: bun install --frozen-lockfile
+      - run: bunx prettier --check .
+      - run: bunx tsc --noEmit
+      - run: bun test

.github/workflows/kernel-mcp-clawbench.yml

@@ -0,0 +1,39 @@
+name: Kernel MCP on ClawBench
+
+on:
+  workflow_dispatch:
+    inputs:
+      arms:
+        description: Comma-separated arm names (empty runs every arm in benchmarks/kernel-mcp-clawbench.toml)
+        required: false
+        default: ""
+        type: string
+      tasks:
+        description: Comma-separated ClawBench task IDs, or all
+        required: false
+        default: all
+        type: string
+      concurrency:
+        description: Concurrent trials per arm
+        required: false
+        default: "10"
+        type: string
+      kernel_mcp_server_ref:
+        description: kernel/kernel-mcp-server git ref to build (empty builds main)
+        required: false
+        default: ""
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  benchmark:
+    uses: ./.github/workflows/_run.yml
+    secrets: inherit
+    with:
+      benchmark: kernel-mcp-clawbench
+      arms: ${{ inputs.arms }}
+      tasks: ${{ inputs.tasks }}
+      concurrency: ${{ inputs.concurrency }}
+      toolset_ref: ${{ inputs.kernel_mcp_server_ref }}

.gitignore

@@ -0,0 +1,8 @@
+.venv/
+__pycache__/
+*.pyc
+.pytest_cache/
+.ruff_cache/
+report/node_modules/
+jobs/
+.env

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Kernel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.