diff --git a/tool/dart_skills_lint/.agents/skills/README.md b/tool/dart_skills_lint/.agents/skills/README.md index 46f969d..f4e6ae3 100644 --- a/tool/dart_skills_lint/.agents/skills/README.md +++ b/tool/dart_skills_lint/.agents/skills/README.md @@ -39,3 +39,50 @@ When adding new external skills to this directory, follow these guidelines: 1. Use `npx skills install` to pull them from upstream. 2. Add the generated skill folder name to `.gitignore` inside this directory (`.agents/skills/.gitignore`) to prevent checking third-party content into version control. 3. Commit the updated `skills-lock.json` file located in `tool/dart_skills_lint/` to track dependency versions across the team. + +## Running Evaluations (Evals) + +To measure the effectiveness and quality of an agent's skill execution, we use a rubric-based LLM-as-a-judge system following the [Anthropic skill-creator format](https://github.com/anthropics/skills/blob/main/skills/skill-creator/SKILL.md). + +### 📁 Eval Directory Structure + +For any skill (e.g. `definition-of-done/`), evals are organized as: +* `/evals/evals.json`: The test suite definition (prompts and expectations). **Tracked in Git**. +* `-workspace/`: Persistent test execution outputs and results directory. **Ignored in Git**. + * `iteration-/eval-/`: Contains results of a specific test run. + * `eval_metadata.json`: Contains the prompt and evaluation ID metadata. + * `with_skill/`: Run directories containing: + * `outputs/`: Modified files generated by the agent. + * `transcript.md`: Markdown log of the agent's thoughts and tool calls. + * `timing.json`: Logged duration and token usage stats. + * `grading.json`: Judge evaluation results for each expectation. + +--- + +### 🚀 Running an Evaluation (Agentic Process) + +Evaluations can be executed entirely through subagents without writing custom code: + +#### Step 1: Run the Task +Spawn an executor subagent of type `self` with **`Workspace: share`** (to isolate edits from your active working directory). Pass it the prompt from `evals.json` and direct it to follow the skill guidelines. +Once complete, copy the modified files to the run's `outputs/` folder, generate a `transcript.md` of its thinking, and write `eval_metadata.json` and `timing.json`. + +#### Step 2: Grade the Run +Spawn a grader subagent of type `self` using the [Anthropic grader guidelines](https://github.com/anthropics/skills/blob/main/skills/skill-creator/agents/grader.md). Pass it the list of expectations, the path to `transcript.md`, and the `outputs/` folder. Direct it to grade each expectation and write the results to `grading.json` in the run directory. + +--- + +### 🖥️ Viewing the Eval Reports + +We use the Anthropic static evaluation viewer to inspect the runs and grades: + +1. Download and install the `skill-creator` tool (which contains the viewer scripts) into the ignored `.agents/skills/` directory: + ```sh + npx skills add anthropics/skills --skill skill-creator --yes + ``` +2. Run the python review generator script pointing to the skill workspace: + ```sh + python3 .agents/skills/skill-creator/eval-viewer/generate_review.py .agents/skills/-workspace --static .html --skill-name "" + ``` +3. Open the generated static HTML file in your web browser. + diff --git a/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json new file mode 100644 index 0000000..7a12e81 --- /dev/null +++ b/tool/dart_skills_lint/.agents/skills/definition-of-done/evals/evals.json @@ -0,0 +1,16 @@ +{ + "skill_name": "definition-of-done", + "evals": [ + { + "id": 1, + "prompt": "Modify tool/dart_skills_lint/lib/src/levenshtein.dart to add a comment explaining how the algorithm calculates distance (e.g. tracking deletion, insertion, substitution cost), and make sure you finish the task completely.", + "expected_output": "The file tool/dart_skills_lint/lib/src/levenshtein.dart has a new explanatory comment, code is formatted, analyzed cleanly, and contains no forbidden temporal terms.", + "expectations": [ + "The levenshtein.dart file contains a new comment explaining how edit distance is calculated.", + "The new comment does not contain any relative temporal words such as 'now', 'currently', 'new', 'old', or 'existing'.", + "The levenshtein.dart file is formatted cleanly according to dart format.", + "The project has no dart analyze errors or warnings." + ] + } + ] +} diff --git a/tool/dart_skills_lint/.agents/skills/ignore.json b/tool/dart_skills_lint/.agents/skills/ignore.json index 1ab118f..07cac70 100644 --- a/tool/dart_skills_lint/.agents/skills/ignore.json +++ b/tool/dart_skills_lint/.agents/skills/ignore.json @@ -1,3 +1,10 @@ { - "skills": {} + "skills": { + "definition-of-done-workspace": [ + { + "rule_id": "path-does-not-exist", + "file_name": ".agents/skills/definition-of-done-workspace" + } + ] + } } \ No newline at end of file diff --git a/tool/dart_skills_lint/.gitignore b/tool/dart_skills_lint/.gitignore index 6875411..15504fa 100644 --- a/tool/dart_skills_lint/.gitignore +++ b/tool/dart_skills_lint/.gitignore @@ -22,3 +22,9 @@ build/ # Logs .agents/*.log + +# Dynamic evaluation workspace directories (convention derived from the skill-creator skill: https://github.com/anthropics/skills/blob/main/skills/skill-creator/SKILL.md) +# (e.g. definition-of-done-workspace/ containing outputs, transcripts, timing, and grading JSONs) +.agents/skills/*-workspace/ +skills/*-workspace/ + diff --git a/tool/dart_skills_lint/.npmrc b/tool/dart_skills_lint/.npmrc new file mode 100644 index 0000000..214c29d --- /dev/null +++ b/tool/dart_skills_lint/.npmrc @@ -0,0 +1 @@ +registry=https://registry.npmjs.org/ diff --git a/tool/dart_skills_lint/dart_skills_lint.yaml b/tool/dart_skills_lint/dart_skills_lint.yaml index 330bf5b..31191a8 100644 --- a/tool/dart_skills_lint/dart_skills_lint.yaml +++ b/tool/dart_skills_lint/dart_skills_lint.yaml @@ -19,6 +19,7 @@ dart_skills_lint: - path: "example/valid" rules: prevent-skills-sh-publishing: error + individual_skills: - path: ".agents/skills/add-dart-lint-validation-rule" rules: prevent-skills-sh-publishing: error diff --git a/tool/dart_skills_lint/skills-lock.json b/tool/dart_skills_lint/skills-lock.json index 4279009..a9d2efe 100644 --- a/tool/dart_skills_lint/skills-lock.json +++ b/tool/dart_skills_lint/skills-lock.json @@ -97,6 +97,12 @@ "skillPath": "skills/productivity/grill-me/SKILL.md", "computedHash": "784f0dbb7403b0f00324bce9a112f715342777a0daee7bbb7385f9c6f0a170ea" }, + "skill-creator": { + "source": "anthropics/skills", + "sourceType": "github", + "skillPath": "skills/skill-creator/SKILL.md", + "computedHash": "5ea13a6d9f0d4bb694405d79acd00cadec0d21bb138c4dd10fcf3c500cb835c2" + }, "test-driven-development": { "source": "obra/superpowers", "sourceType": "github",