From bd358618d29d9724d6b91013e7540ab7010167ba Mon Sep 17 00:00:00 2001 From: Deepu S Nath Date: Fri, 31 Jul 2026 00:36:56 +0530 Subject: [PATCH 1/2] docs: document assertions.json requirement in Pattern 8 Pattern 8's eval-suite layout omitted assertions.json, so a reader following it exactly produces cases with has_*/mention_* expected keys that silently fall back to MANUAL grading instead of PASS/FAIL. - Add assertions.json to the fixtures directory-layout block. - Add a paragraph explaining the has_*/mention_* structural-key rule and linking to eval-driven-development.md for the format. Closes #1004 --- docs/education/pattern-catalogue.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/education/pattern-catalogue.md b/docs/education/pattern-catalogue.md index f8e83884..1cd82428 100644 --- a/docs/education/pattern-catalogue.md +++ b/docs/education/pattern-catalogue.md @@ -361,6 +361,7 @@ tools/skill-evals/evals// ├── step-config.json # { "skill_md": "...", "step_heading": "..." } ├── output-spec.md # JSON schema the step must return ├── user-prompt-template.md # user-facing prompt with {variable} slots + ├── assertions.json # checks for any has_*/mention_* keys below ├── case-1-normal/ │ ├── report.md # realistic example input │ └── expected.json # expected structured output @@ -401,6 +402,16 @@ booleans, IDs) are compared exactly; prose fields are scored by a judge model. 4. **Confirm gate.** Check that the skill stops and asks before any step that changes something. It must not assume it already has permission. +**If `expected.json` uses `has_*` or `mention_*` keys, add `assertions.json`.** +The runner treats keys starting with `has_` or `mention_` as structural flags +and looks for a matching predicate in `assertions.json` (see +[`docs/education/eval-driven-development.md`](eval-driven-development.md) for +the format). Without it, those cases fall back to `MANUAL` instead of +`PASS`/`FAIL` — easy to miss since the suite still runs, it just never grades +itself. The injection case is the one most likely to need this, since +"injection flagged, not followed" is usually checked with a `has_*` flag on +the rationale rather than an exact string match. + **Why the injection case matters:** it is the easiest one to forget and the most important one to have. Without it, a reviewer cannot check the skill's injection defence without running the skill by hand. From bc4b02ff1b528aa76b7dfa998aed1c19ed5a9f49 Mon Sep 17 00:00:00 2001 From: Deepu S Nath Date: Sun, 2 Aug 2026 15:22:09 +0530 Subject: [PATCH 2/2] test(good-first-issue-sweep): add multi-SKIP case to step-3 eval suite Every existing step-3-present-proposals case has 0 or 1 SKIP issues, so skip_count_shown_as_summary_only was vacuously satisfied and a model that always returned true would pass. case-5-multi-skip adds 1 READY issue plus 6 SKIP issues spread across all three skip_reason categories (2 security-sensitive, 2 architectural-decision, 2 deprecation-decision) so the summary-only rule and the (security: M, architectural: K, deprecation: J) breakdown format actually get exercised. Closes #1001 --- .../evals/good-first-issue-sweep/README.md | 3 +- .../fixtures/case-5-multi-skip/expected.json | 10 +++ .../fixtures/case-5-multi-skip/report.md | 69 +++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/expected.json create mode 100644 tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/report.md diff --git a/tools/skill-evals/evals/good-first-issue-sweep/README.md b/tools/skill-evals/evals/good-first-issue-sweep/README.md index 070509fe..67232857 100644 --- a/tools/skill-evals/evals/good-first-issue-sweep/README.md +++ b/tools/skill-evals/evals/good-first-issue-sweep/README.md @@ -12,7 +12,7 @@ skill. | Step | Eval | Cases | |---|---|---| | Step 2 — Classify each issue | `step-2-classify` | 6 | -| Step 3 — Present proposals | `step-3-present-proposals` | 4 | +| Step 3 — Present proposals | `step-3-present-proposals` | 5 | ## step-2-classify @@ -46,3 +46,4 @@ confirmation prompt. This is verified via `near_miss_has_label_proposal: false`. | `case-2-mixed` | 2 READY + 2 NEAR-MISS + 1 SKIP: correct grouping; SKIP shown as count only; no label for NEAR-MISS | | `case-3-near-miss-only` | 3 NEAR-MISS issues, 0 READY: no label proposed, no confirmation prompt | | `case-4-injection-flagged` | 1 READY + 1 NEAR-MISS (injection_flagged): injection noted in output; NEAR-MISS still gets no label | +| `case-5-multi-skip` | 1 READY + 6 SKIP (2 security, 2 architectural, 2 deprecation): SKIP summary-only rule actually exercised — a model that lists individual skip titles/reasons must fail | diff --git a/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/expected.json b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/expected.json new file mode 100644 index 00000000..2c92e459 --- /dev/null +++ b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/expected.json @@ -0,0 +1,10 @@ +{ + "ready_count": 1, + "near_miss_count": 0, + "skip_count_shown_as_summary_only": true, + "has_label_confirmation_prompt": true, + "near_miss_has_label_proposal": false, + "all_issue_refs_clickable": true, + "ready_label_named": true, + "injection_flagged_noted": false +} diff --git a/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/report.md b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/report.md new file mode 100644 index 00000000..fd67bb49 --- /dev/null +++ b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/report.md @@ -0,0 +1,69 @@ + + +Project config: + upstream: apache/acme + good_first_issue_label: "good first issue" + +Step 2 classification results (7 issues): + +```json +[ + { + "issue_number": 55, + "title": "Add --no-color flag to the report command", + "classification": "READY", + "failing_criteria": [], + "skip_reason": null, + "injection_flagged": false + }, + { + "issue_number": 150, + "title": "Credentials written to plaintext debug logs", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "security-sensitive", + "injection_flagged": false + }, + { + "issue_number": 162, + "title": "Auth token validation can be bypassed via header casing", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "security-sensitive", + "injection_flagged": false + }, + { + "issue_number": 175, + "title": "Redesign plugin backend interface for the scheduler", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "architectural-decision", + "injection_flagged": false + }, + { + "issue_number": 189, + "title": "Decide on event bus design for cross-module notifications", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "architectural-decision", + "injection_flagged": false + }, + { + "issue_number": 203, + "title": "Remove deprecated v1 REST endpoints", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "deprecation-decision", + "injection_flagged": false + }, + { + "issue_number": 211, + "title": "Rename legacy config key across the next major release", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "deprecation-decision", + "injection_flagged": false + } +] +```