From 874b67313f1e1c57cb583798d15b3843bfc8aefa Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Mon, 13 Jul 2026 18:16:10 -0400 Subject: [PATCH 1/4] Add evaluations setup and first eval definition for definition-of-done skill --- .../.agents/skills/.gitignore | 4 ++ .../dart_skills_lint/.agents/skills/README.md | 47 +++++++++++++++++++ .../definition-of-done/evals/evals.json | 16 +++++++ tool/dart_skills_lint/.gitignore | 6 +++ tool/dart_skills_lint/.npmrc | 1 + tool/dart_skills_lint/skills-lock.json | 6 +++ 6 files changed, 80 insertions(+) create mode 100644 tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json create mode 100644 tool/dart_skills_lint/.npmrc diff --git a/tool/dart_skills_lint/.agents/skills/.gitignore b/tool/dart_skills_lint/.agents/skills/.gitignore index 466a859..1471a06 100644 --- a/tool/dart_skills_lint/.agents/skills/.gitignore +++ b/tool/dart_skills_lint/.agents/skills/.gitignore @@ -6,6 +6,10 @@ !dart-skills-lint-integration/ !definition-of-done/ +# Keep evals inside un-ignored skill folders +!**/evals/ +!**/evals/** + # Keep essential configuration and docs !.gitignore !README.md diff --git a/tool/dart_skills_lint/.agents/skills/README.md b/tool/dart_skills_lint/.agents/skills/README.md index 46f969d..f4e6ae3 100644 --- a/tool/dart_skills_lint/.agents/skills/README.md +++ b/tool/dart_skills_lint/.agents/skills/README.md @@ -39,3 +39,50 @@ When adding new external skills to this directory, follow these guidelines: 1. Use `npx skills install` to pull them from upstream. 2. Add the generated skill folder name to `.gitignore` inside this directory (`.agents/skills/.gitignore`) to prevent checking third-party content into version control. 3. Commit the updated `skills-lock.json` file located in `tool/dart_skills_lint/` to track dependency versions across the team. + +## Running Evaluations (Evals) + +To measure the effectiveness and quality of an agent's skill execution, we use a rubric-based LLM-as-a-judge system following the [Anthropic skill-creator format](https://github.com/anthropics/skills/blob/main/skills/skill-creator/SKILL.md). + +### 📁 Eval Directory Structure + +For any skill (e.g. `definition-of-done/`), evals are organized as: +* `/evals/evals.json`: The test suite definition (prompts and expectations). **Tracked in Git**. +* `-workspace/`: Persistent test execution outputs and results directory. **Ignored in Git**. + * `iteration-/eval-/`: Contains results of a specific test run. + * `eval_metadata.json`: Contains the prompt and evaluation ID metadata. + * `with_skill/`: Run directories containing: + * `outputs/`: Modified files generated by the agent. + * `transcript.md`: Markdown log of the agent's thoughts and tool calls. + * `timing.json`: Logged duration and token usage stats. + * `grading.json`: Judge evaluation results for each expectation. + +--- + +### 🚀 Running an Evaluation (Agentic Process) + +Evaluations can be executed entirely through subagents without writing custom code: + +#### Step 1: Run the Task +Spawn an executor subagent of type `self` with **`Workspace: share`** (to isolate edits from your active working directory). Pass it the prompt from `evals.json` and direct it to follow the skill guidelines. +Once complete, copy the modified files to the run's `outputs/` folder, generate a `transcript.md` of its thinking, and write `eval_metadata.json` and `timing.json`. + +#### Step 2: Grade the Run +Spawn a grader subagent of type `self` using the [Anthropic grader guidelines](https://github.com/anthropics/skills/blob/main/skills/skill-creator/agents/grader.md). Pass it the list of expectations, the path to `transcript.md`, and the `outputs/` folder. Direct it to grade each expectation and write the results to `grading.json` in the run directory. + +--- + +### 🖥️ Viewing the Eval Reports + +We use the Anthropic static evaluation viewer to inspect the runs and grades: + +1. Download and install the `skill-creator` tool (which contains the viewer scripts) into the ignored `.agents/skills/` directory: + ```sh + npx skills add anthropics/skills --skill skill-creator --yes + ``` +2. Run the python review generator script pointing to the skill workspace: + ```sh + python3 .agents/skills/skill-creator/eval-viewer/generate_review.py .agents/skills/-workspace --static .html --skill-name "" + ``` +3. Open the generated static HTML file in your web browser. + diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json new file mode 100644 index 0000000..9499d01 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json @@ -0,0 +1,16 @@ +{ + "skill_name": "definition-of-done", + "evals": [ + { + "id": 1, + "prompt": "Modify lib/src/levenshtein.dart to add a comment explaining how the algorithm calculates distance (e.g. tracking deletion, insertion, substitution cost), and make sure you finish the task completely.", + "expected_output": "The file lib/src/levenshtein.dart has a new explanatory comment, code is formatted, analyzed cleanly, and contains no forbidden temporal terms.", + "expectations": [ + "The levenshtein.dart file contains a new comment explaining how edit distance is calculated.", + "The new comment does not contain any relative temporal words such as 'now', 'currently', 'new', 'old', or 'existing'.", + "The levenshtein.dart file is formatted cleanly according to dart format.", + "The project has no dart analyze errors or warnings." + ] + } + ] +} diff --git a/tool/dart_skills_lint/.gitignore b/tool/dart_skills_lint/.gitignore index 6875411..15504fa 100644 --- a/tool/dart_skills_lint/.gitignore +++ b/tool/dart_skills_lint/.gitignore @@ -22,3 +22,9 @@ build/ # Logs .agents/*.log + +# Dynamic evaluation workspace directories (convention derived from the skill-creator skill: https://github.com/anthropics/skills/blob/main/skills/skill-creator/SKILL.md) +# (e.g. definition-of-done-workspace/ containing outputs, transcripts, timing, and grading JSONs) +.agents/skills/*-workspace/ +skills/*-workspace/ + diff --git a/tool/dart_skills_lint/.npmrc b/tool/dart_skills_lint/.npmrc new file mode 100644 index 0000000..214c29d --- /dev/null +++ b/tool/dart_skills_lint/.npmrc @@ -0,0 +1 @@ +registry=https://registry.npmjs.org/ diff --git a/tool/dart_skills_lint/skills-lock.json b/tool/dart_skills_lint/skills-lock.json index 4279009..a9d2efe 100644 --- a/tool/dart_skills_lint/skills-lock.json +++ b/tool/dart_skills_lint/skills-lock.json @@ -97,6 +97,12 @@ "skillPath": "skills/productivity/grill-me/SKILL.md", "computedHash": "784f0dbb7403b0f00324bce9a112f715342777a0daee7bbb7385f9c6f0a170ea" }, + "skill-creator": { + "source": "anthropics/skills", + "sourceType": "github", + "skillPath": "skills/skill-creator/SKILL.md", + "computedHash": "5ea13a6d9f0d4bb694405d79acd00cadec0d21bb138c4dd10fcf3c500cb835c2" + }, "test-driven-development": { "source": "obra/superpowers", "sourceType": "github", From a48be661572740e41c50d22cb82ca7e1a20d9e60 Mon Sep 17 00:00:00 2001 From: Reid Baker Date: Mon, 13 Jul 2026 19:49:38 -0400 Subject: [PATCH 2/4] Fix skills linter tests by adding ignore for definition-of-done-workspace and restructuring yaml configuration --- tool/dart_skills_lint/.agents/skills/ignore.json | 9 ++++++++- tool/dart_skills_lint/dart_skills_lint.yaml | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tool/dart_skills_lint/.agents/skills/ignore.json b/tool/dart_skills_lint/.agents/skills/ignore.json index 1ab118f..07cac70 100644 --- a/tool/dart_skills_lint/.agents/skills/ignore.json +++ b/tool/dart_skills_lint/.agents/skills/ignore.json @@ -1,3 +1,10 @@ { - "skills": {} + "skills": { + "definition-of-done-workspace": [ + { + "rule_id": "path-does-not-exist", + "file_name": ".agents/skills/definition-of-done-workspace" + } + ] + } } \ No newline at end of file diff --git a/tool/dart_skills_lint/dart_skills_lint.yaml b/tool/dart_skills_lint/dart_skills_lint.yaml index 330bf5b..31191a8 100644 --- a/tool/dart_skills_lint/dart_skills_lint.yaml +++ b/tool/dart_skills_lint/dart_skills_lint.yaml @@ -19,6 +19,7 @@ dart_skills_lint: - path: "example/valid" rules: prevent-skills-sh-publishing: error + individual_skills: - path: ".agents/skills/add-dart-lint-validation-rule" rules: prevent-skills-sh-publishing: error From 7824e6bd04691ed5e6ba91d2a012d0fa03ede7df Mon Sep 17 00:00:00 2001 From: Reid Baker <1063596+reidbaker@users.noreply.github.com> Date: Mon, 13 Jul 2026 19:58:15 -0400 Subject: [PATCH 3/4] Update tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../.agents/skills/definition-of-done/evals/evals.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json index 9499d01..7a12e81 100644 --- a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json @@ -3,8 +3,8 @@ "evals": [ { "id": 1, - "prompt": "Modify lib/src/levenshtein.dart to add a comment explaining how the algorithm calculates distance (e.g. tracking deletion, insertion, substitution cost), and make sure you finish the task completely.", - "expected_output": "The file lib/src/levenshtein.dart has a new explanatory comment, code is formatted, analyzed cleanly, and contains no forbidden temporal terms.", + "prompt": "Modify tool/dart_skills_lint/lib/src/levenshtein.dart to add a comment explaining how the algorithm calculates distance (e.g. tracking deletion, insertion, substitution cost), and make sure you finish the task completely.", + "expected_output": "The file tool/dart_skills_lint/lib/src/levenshtein.dart has a new explanatory comment, code is formatted, analyzed cleanly, and contains no forbidden temporal terms.", "expectations": [ "The levenshtein.dart file contains a new comment explaining how edit distance is calculated.", "The new comment does not contain any relative temporal words such as 'now', 'currently', 'new', 'old', or 'existing'.", From 1304c55635beab4a087efd073bbc8b17b58da055 Mon Sep 17 00:00:00 2001 From: Reid Baker <1063596+reidbaker@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:04:42 -0400 Subject: [PATCH 4/4] Update .gitignore --- tool/dart_skills_lint/.agents/skills/.gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tool/dart_skills_lint/.agents/skills/.gitignore b/tool/dart_skills_lint/.agents/skills/.gitignore index 1471a06..466a859 100644 --- a/tool/dart_skills_lint/.agents/skills/.gitignore +++ b/tool/dart_skills_lint/.agents/skills/.gitignore @@ -6,10 +6,6 @@ !dart-skills-lint-integration/ !definition-of-done/ -# Keep evals inside un-ignored skill folders -!**/evals/ -!**/evals/** - # Keep essential configuration and docs !.gitignore !README.md