coder_eval/mkdocs.yml at main · UiPath/coder_eval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
site_name: Coder Eval
site_description: >-
  Evaluate and benchmark AI coding agents and their skills — a sandboxed,
  agent-agnostic framework with declarative YAML eval suites for Claude Code,
  Codex, Antigravity (Gemini), OpenCode and Pi, A/B experiments, weighted scoring,
  and CI gates.
site_url: https://coder-eval.com/docs/
repo_url: https://github.com/UiPath/coder_eval
repo_name: UiPath/coder_eval
edit_uri: edit/main/docs/
copyright: © 2026 UiPath — Apache License 2.0

theme:
  name: material
  features:
    - navigation.instant
    - navigation.tracking
    - navigation.sections
    - navigation.top
    - search.suggest
    - search.highlight
    - content.code.copy
    - content.action.edit
  palette:
    - media: "(prefers-color-scheme: light)"
      scheme: default
      primary: deep orange
      accent: deep orange
      toggle:
        icon: material/weather-night
        name: Switch to dark mode
    - media: "(prefers-color-scheme: dark)"
      scheme: slate
      primary: deep orange
      accent: deep orange
      toggle:
        icon: material/weather-sunny
        name: Switch to light mode
  icon:
    repo: fontawesome/brands/github

# The social plugin generates per-page Open Graph / Twitter card images (and
# the matching og:/twitter: meta tags) so shared links unfurl properly. It
# needs Cairo system libraries, so it is enabled only in CI (the Docs workflow
# installs the deps); local builds skip it.
plugins:
  - search
  - social:
      enabled: !ENV [CI, false]

markdown_extensions:
  - admonition
  - attr_list
  - md_in_html
  - tables
  - toc:
      permalink: true
  - pymdownx.highlight:
      anchor_linenums: true
  - pymdownx.superfences
  - pymdownx.inlinehilite
  - pymdownx.snippets
  - pymdownx.tabbed:
      alternate_style: true

extra:
  social:
    - icon: fontawesome/brands/github
      link: https://github.com/UiPath/coder_eval
    - icon: fontawesome/brands/python
      link: https://pypi.org/project/coder-eval/
  # docs_index: one-line blurb per nav page (docs-root-relative path -> blurb),
  # for every page EXCEPT tutorial leaves (tutorials/0*.md). This map plus `nav`
  # are the single source of truth for the flat index surfaces — README.md's
  # Documentation table, docs/index.md's "Where to go next" table, and the
  # ## Docs / ## Tutorials sections of docs/llms.txt. Regenerate them all with
  # `make docs-indexes`; CE028 fails the build if any of the three drifts, if a
  # nav page has no blurb (or vice-versa), or if a docs/ page is missing from nav.
  docs_index:
    index.md: "What Coder Eval is, its features, and a quick start"
    tutorials/README.md: "Step-by-step walkthroughs — start here"
    USER_GUIDE.md: "Full CLI, configuration, output, and environment-variable reference"
    TASK_DEFINITION_GUIDE.md: "The task-file schema — all criterion types, scoring, templates"
    agents/CLAUDE_CODE.md: "Configuring and running the default Claude Code agent"
    agents/CODEX.md: "Running the OpenAI Codex agent"
    agents/ANTIGRAVITY.md: "Running the Google Antigravity / Gemini agent"
    agents/OPENCODE.md: "Running the OpenCode agent on open-weight models"
    agents/PI.md: "Running the Pi agent on open-weight models"
    agents/HARNESS_PARITY.md: "What each run_limits field means on every harness"
    AB_EXPERIMENTS.md: "Compare models / tools / prompts across the same tasks"
    DATASETS.md: "Fan a single task out over a dataset"
    DIALOG_MODE.md: "Evaluate agents in multi-turn conversation via a simulated user"
    DOCKER_ISOLATION.md: "The container sandbox driver, with custom images"
    CI_GATE.md: "Run Coder Eval as a CI gate — the Marketplace Action, JUnit output, run reports"
    PLUGIN.md: "Install the Claude Code plugin — author, run, and analyze suites from inside the agent"
    EXTENDING.md: "Author a custom agent, criterion, or model pricing via the plugin SPI"
    REPORT_SCHEMA.md: "Field-level reference for run.json / variant.json / task.json"
    comparison.md: "vs. SWE-bench, SkillsBench, Harbor, OpenAI Evals, hand-rolled scripts"

nav:
  - Home: index.md
  - Tutorials:
      - Overview: tutorials/README.md
      - 01 · Your first evaluation: tutorials/01-first-evaluation.md
      - 02 · Running Coder Eval in CI: tutorials/02-ci-pipeline.md
      - 03 · Browsing results with evalboard: tutorials/03-evalboard-local.md
      - 04 · Writing a task: tutorials/04-writing-a-task.md
      - 05 · Comparing two models: tutorials/05-comparing-models.md
      - 06 · Docker isolation: tutorials/06-use-docker-isolation.md
      - 07 · Driving Coder Eval from Claude Code: tutorials/07-plugin-in-claude-code.md
  - Guides:
      - User Guide: USER_GUIDE.md
      - Task Definition Guide: TASK_DEFINITION_GUIDE.md
  - Agents:
      - Claude Code: agents/CLAUDE_CODE.md
      - Codex: agents/CODEX.md
      - Antigravity (Gemini): agents/ANTIGRAVITY.md
      - OpenCode: agents/OPENCODE.md
      - Pi: agents/PI.md
      - Run-Limit Parity: agents/HARNESS_PARITY.md
  - Advanced:
      - A/B Experiments: AB_EXPERIMENTS.md
      - Bring Your Own Dataset: DATASETS.md
      - Dialog Mode: DIALOG_MODE.md
      - Docker Isolation: DOCKER_ISOLATION.md
      - CI Gate & GitHub Action: CI_GATE.md
      - Claude Code Plugin: PLUGIN.md
      - Extending Coder Eval: EXTENDING.md
      - Report Schema: REPORT_SCHEMA.md
  - How It Compares: comparison.md