diff --git a/tools/skill-evals/evals/good-first-issue-sweep/README.md b/tools/skill-evals/evals/good-first-issue-sweep/README.md index 070509fe..67232857 100644 --- a/tools/skill-evals/evals/good-first-issue-sweep/README.md +++ b/tools/skill-evals/evals/good-first-issue-sweep/README.md @@ -12,7 +12,7 @@ skill. | Step | Eval | Cases | |---|---|---| | Step 2 — Classify each issue | `step-2-classify` | 6 | -| Step 3 — Present proposals | `step-3-present-proposals` | 4 | +| Step 3 — Present proposals | `step-3-present-proposals` | 5 | ## step-2-classify @@ -46,3 +46,4 @@ confirmation prompt. This is verified via `near_miss_has_label_proposal: false`. | `case-2-mixed` | 2 READY + 2 NEAR-MISS + 1 SKIP: correct grouping; SKIP shown as count only; no label for NEAR-MISS | | `case-3-near-miss-only` | 3 NEAR-MISS issues, 0 READY: no label proposed, no confirmation prompt | | `case-4-injection-flagged` | 1 READY + 1 NEAR-MISS (injection_flagged): injection noted in output; NEAR-MISS still gets no label | +| `case-5-multi-skip` | 1 READY + 6 SKIP (2 security, 2 architectural, 2 deprecation): SKIP summary-only rule actually exercised — a model that lists individual skip titles/reasons must fail | diff --git a/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/expected.json b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/expected.json new file mode 100644 index 00000000..2c92e459 --- /dev/null +++ b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/expected.json @@ -0,0 +1,10 @@ +{ + "ready_count": 1, + "near_miss_count": 0, + "skip_count_shown_as_summary_only": true, + "has_label_confirmation_prompt": true, + "near_miss_has_label_proposal": false, + "all_issue_refs_clickable": true, + "ready_label_named": true, + "injection_flagged_noted": false +} diff --git a/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/report.md b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/report.md new file mode 100644 index 00000000..fd67bb49 --- /dev/null +++ b/tools/skill-evals/evals/good-first-issue-sweep/step-3-present-proposals/fixtures/case-5-multi-skip/report.md @@ -0,0 +1,69 @@ + + +Project config: + upstream: apache/acme + good_first_issue_label: "good first issue" + +Step 2 classification results (7 issues): + +```json +[ + { + "issue_number": 55, + "title": "Add --no-color flag to the report command", + "classification": "READY", + "failing_criteria": [], + "skip_reason": null, + "injection_flagged": false + }, + { + "issue_number": 150, + "title": "Credentials written to plaintext debug logs", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "security-sensitive", + "injection_flagged": false + }, + { + "issue_number": 162, + "title": "Auth token validation can be bypassed via header casing", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "security-sensitive", + "injection_flagged": false + }, + { + "issue_number": 175, + "title": "Redesign plugin backend interface for the scheduler", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "architectural-decision", + "injection_flagged": false + }, + { + "issue_number": 189, + "title": "Decide on event bus design for cross-module notifications", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "architectural-decision", + "injection_flagged": false + }, + { + "issue_number": 203, + "title": "Remove deprecated v1 REST endpoints", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "deprecation-decision", + "injection_flagged": false + }, + { + "issue_number": 211, + "title": "Rename legacy config key across the next major release", + "classification": "SKIP", + "failing_criteria": [], + "skip_reason": "deprecation-decision", + "injection_flagged": false + } +] +```