Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/verify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version-file: go.mod
Expand All @@ -148,6 +150,12 @@ jobs:
run: |
npm ci
npm test
- name: Validate semantic conversion receipt when required
working-directory: evals
env:
EVAL_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
EVAL_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
run: npm run test:conversion
- name: Run every example fixture
run: |
go build -o "$RUNNER_TEMP/yskill" ./cmd/yskill
Expand Down
5 changes: 4 additions & 1 deletion cmd/yskill/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func cmdBootstrap(args []string) error {
for _, agent := range plan.Agents {
ids = append(ids, agent.ID)
}
if err := bootstrapDoctor(plan.SkillDir, plan.Root, ids); err != nil {
if err := bootstrapDoctor(plan.SkillDir, plan.Root, nil); err != nil {
return fmt.Errorf("verify workflow builder: %w", err)
}
registrations, err := registerSkill(plan.SkillDir, plan.Root, ids)
Expand All @@ -96,6 +96,9 @@ func cmdBootstrap(args []string) error {
for _, item := range registrations {
fmt.Printf("registered: %-22s %s\n", item.AgentID, item.Path)
}
if err := bootstrapDoctor(plan.SkillDir, plan.Root, ids); err != nil {
return fmt.Errorf("verify workflow builder adapters: %w", err)
}
fmt.Println("bootstrap: workflow builder is ready")
fmt.Println("next: restart your coding agent, then ask it to create or convert a skill workflow")
fmt.Println("create: Use Yield to create a tested skill workflow for releasing my package.")
Expand Down
64 changes: 46 additions & 18 deletions cmd/yskill/bootstrap_templates.go

Large diffs are not rendered by default.

29 changes: 28 additions & 1 deletion cmd/yskill/bootstrap_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ func TestBootstrapCancellationDoesNotWrite(t *testing.T) {

func TestBootstrapWritesBuilderProfileAndAdapter(t *testing.T) {
withBootstrapTestState(t)
var doctorAgents [][]string
bootstrapDoctor = func(_ string, _ string, agents []string) error {
doctorAgents = append(doctorAgents, append([]string(nil), agents...))
return nil
}
root := t.TempDir()
if err := cmdBootstrap([]string{"--root", root, "--language", "python", "--agent", "codex", "--yes"}); err != nil {
t.Fatal(err)
Expand All @@ -73,6 +78,9 @@ func TestBootstrapWritesBuilderProfileAndAdapter(t *testing.T) {
if err := cmdBootstrap([]string{"--root", root, "--language", "python", "--agent", "codex", "--yes"}); err != nil {
t.Fatalf("idempotent bootstrap failed: %v", err)
}
if len(doctorAgents) != 4 || len(doctorAgents[0]) != 0 || len(doctorAgents[1]) != 1 || doctorAgents[1][0] != "codex" {
t.Fatalf("bootstrap must verify the workflow before registration and adapters after it: %#v", doctorAgents)
}
}

func TestBootstrapRefusesForeignSkillAndAdapter(t *testing.T) {
Expand Down Expand Up @@ -111,7 +119,8 @@ func TestBootstrapRefusesAdapterSymlinkEscape(t *testing.T) {

func TestBuilderTemplatesExposeEquivalentOperations(t *testing.T) {
profile := bootstrapProfile{YieldVersion: "1.2.3", Agents: []string{"codex"}}
want := []string{"select-mode", "collect-specification", "extract-flow", "write-workflow", "verify-generated", "repair-generated-", "register-generated", "verify-adapters"}
want := []string{"select-mode", "collect-specification", "check-destination", "project-semantics", "extract-flow", "write-workflow", "verify-generated", "repair-generated-", "register-generated", "verify-adapters"}
projectionContract := []string{"source_clause", "disposition", "destinations", "reason", "control", "guidance", "both", "excluded", "ready", "unresolved"}
for _, language := range []string{"typescript", "python", "go", "rust"} {
files, _, err := renderBootstrapSkill(language, profile)
if err != nil {
Expand All @@ -128,6 +137,24 @@ func TestBuilderTemplatesExposeEquivalentOperations(t *testing.T) {
t.Errorf("%s builder is missing operation %s", language, operation)
}
}
for _, field := range projectionContract {
if !strings.Contains(program, field) {
t.Errorf("%s builder projection is missing %s", language, field)
}
}
for _, downstream := range []string{"source,projection", `"source":source,"projection":projection`} {
if strings.Contains(program, downstream) {
goto hasProjectionContext
}
}
t.Errorf("%s builder does not pass source and projection downstream", language)
hasProjectionContext:
}
}

func TestBuilderCreateFixtureDoesNotRequireProjection(t *testing.T) {
if strings.Contains(bootstrapFixtureResponses, `"project-semantics"`) {
t.Fatal("create mode fixture must remain unchanged by conversion projection")
}
}

Expand Down
45 changes: 45 additions & 0 deletions evals/conversion/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# Semantic-disposition conversion evaluation

This evaluation checks one small skill conversion. It uses this operator:

\[
C = clauses(S)
\]

\[
\Pi(S)=\{(c,d,T,r)\mid c\in C\}
\]

Each source clause gets one disposition. `T` lists its destinations. `r` gives
the reason for an exclusion.

## How the projection works

1. Split the source `SKILL.md` into clauses.
2. Assign one disposition to each clause.
3. Map each retained clause to a reachable code or model-facing destination.
4. Give a reason for each excluded clause.
5. Pass the source and projection into flow extraction, writing, and repair.

| Source clause | Disposition | Required destination |
|---|---|---|
| Run tests and stop on failure. | `control` | Code |
| Prefer changed-code evidence. | `guidance` | `SKILL.md` or `agent_task` |
| Ask for approval and explain why. | `both` | Code and model-facing guidance |
| Yarn is only release history. | `excluded` | No destination; give a reason |

Run the paid evaluation:

npm run eval:conversion

The command starts exactly two fresh Codex sessions. Both use `gpt-5.6-sol`
with medium reasoning. The first session runs the real bootstrapped builder.
The second session judges the generated result and a control-only negative
candidate. Raw transcripts stay under ignored `evals/runs/`.

Run deterministic checks:

npm run test:conversion -- --force

This is advisory evidence for one four-clause fixture. It does not cover other
skills or conversions.
7 changes: 7 additions & 0 deletions evals/conversion/fixtures/fault-probes.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"missing": "One source clause has no projection row.",
"contradictory": "A guidance clause maps only to code.",
"incorrectly_duplicated": "One source clause has two incompatible disposition rows.",
"excluded_without_reason": "An excluded clause has an empty reason.",
"unreachable": "A destination names a file or task that the coding agent cannot reach."
}
6 changes: 6 additions & 0 deletions evals/conversion/fixtures/negative-control/SKILL.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
name: converted-release
description: Run the generated release workflow.
---

Run `yskill run .` and follow each operation.
7 changes: 7 additions & 0 deletions evals/conversion/fixtures/negative-control/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package main

// This negative control deliberately preserves only executable control.
// It omits the source guidance and the historical exclusion decision.
func main() {
// Run tests, stop on failure, then require approval before publish.
}
12 changes: 12 additions & 0 deletions evals/conversion/fixtures/source-skill/SKILL.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
name: source-release
description: Publish a small package safely.
---

- Run the package tests before publish, and stop when a test fails.

- When you review the release, prefer evidence from changed code over issue text.

- Ask for approval before publish, and use a calm, direct tone to explain that publishing to the registry cannot be undone.

- Older releases used Yarn; this is history, not an instruction for the new workflow.
53 changes: 53 additions & 0 deletions evals/conversion/judge-schema.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
{
"type": "object",
"required": ["candidate", "negative_control", "clause_findings", "defect_detection"],
"additionalProperties": false,
"properties": {
"candidate": {
"type": "object",
"required": ["verdict", "reason"],
"additionalProperties": false,
"properties": {
"verdict": {"enum": ["accept", "reject"]},
"reason": {"type": "string", "minLength": 1}
}
},
"negative_control": {
"type": "object",
"required": ["verdict", "reason"],
"additionalProperties": false,
"properties": {
"verdict": {"enum": ["accept", "reject"]},
"reason": {"type": "string", "minLength": 1}
}
},
"clause_findings": {
"type": "array",
"minItems": 4,
"items": {
"type": "object",
"required": ["source_clause", "disposition", "preserved", "reachable", "finding"],
"additionalProperties": false,
"properties": {
"source_clause": {"type": "string", "minLength": 1},
"disposition": {"enum": ["control", "guidance", "both", "excluded"]},
"preserved": {"type": "boolean"},
"reachable": {"type": "boolean"},
"finding": {"type": "string", "minLength": 1}
}
}
},
"defect_detection": {
"type": "object",
"required": ["missing", "contradictory", "incorrectly_duplicated", "excluded_without_reason", "unreachable"],
"additionalProperties": false,
"properties": {
"missing": {"type": "boolean"},
"contradictory": {"type": "boolean"},
"incorrectly_duplicated": {"type": "boolean"},
"excluded_without_reason": {"type": "boolean"},
"unreachable": {"type": "boolean"}
}
}
}
}
48 changes: 48 additions & 0 deletions evals/conversion/scripts/conversion.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
import assert from "node:assert/strict"
import test from "node:test"
import { readFile } from "node:fs/promises"
import { isSemanticPath, receiptSurface, semanticSurface, sourceHash } from "./surface.mjs"
import { receiptPath, validateReceipt, validateReceiptFile } from "./receipt.mjs"

test("router selects every inventoried path", () => {
for (const path of semanticSurface) {
const witness = path.endsWith("/") ? `${path}witness.txt` : path
assert.equal(isSemanticPath(witness), true, witness)
}
assert.equal(isSemanticPath(receiptSurface), true, receiptSurface)
})

test("router skips explicit non-semantic paths", () => {
for (const path of ["README.md", "docs/quickstart.md", "cmd/yskill/bootstrap.go", "cmd/yskill/scaffold.go", "sdk/typescript/src/index.ts", ".agents/skills/example/SKILL.md", "evals/conversion/README.md"]) {
assert.equal(isSemanticPath(path), false, path)
}
})

async function validReceipt() {
const receipt = JSON.parse(await readFile(receiptPath, "utf8"))
receipt.source_hash = await sourceHash()
return receipt
}

test("validator rejects stale, malformed, failing, and rubber-stamping receipts", async () => {
const mutations = [
(r) => { r.source_hash = "stale" },
(r) => { delete r.model },
(r) => { r.status = "failed" },
(r) => { r.negative_control_verdict = "accept" },
(r) => { r.defect_detection.unreachable = false },
]
for (const mutate of mutations) {
const receipt = await validReceipt()
mutate(receipt)
await assert.rejects(validateReceipt(receipt))
}
})

test("validator rejects a missing receipt", async () => {
await assert.rejects(validateReceiptFile(`${receiptPath}.missing`))
})

test("published simple Sol receipt passes", async () => {
await validateReceipt(await validReceipt())
})
33 changes: 33 additions & 0 deletions evals/conversion/scripts/receipt.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { readFile } from "node:fs/promises"
import { join } from "node:path"
import { evalRoot, sourceHash } from "./surface.mjs"

export const receiptPath = join(evalRoot, "results/latest-conversion.json")

export async function validateReceiptFile(path = receiptPath) {
return validateReceipt(JSON.parse(await readFile(path, "utf8")))
}

export async function validateReceipt(receipt) {
if (receipt === undefined) return validateReceiptFile()
const fail = (message) => { throw new Error(message) }
if (receipt.schema_version !== 1) fail("unsupported conversion receipt schema")
if (receipt.methodology_version !== "semantic-disposition-v1") fail("unsupported conversion evaluation method")
if (receipt.source_hash !== await sourceHash()) fail("conversion receipt has a stale source hash")
if (receipt.status !== "passed") fail("conversion receipt is not passing")
if (receipt.model?.product !== "Codex CLI" || receipt.model?.name !== "gpt-5.6-sol" || receipt.model?.reasoning !== "medium") fail("conversion receipt used the wrong model")
if (receipt.sessions !== 2) fail("conversion evaluation must use exactly two fresh sessions")
for (const key of ["input_tokens", "cached_input_tokens", "output_tokens", "reasoning_output_tokens"]) {
if (!Number.isInteger(receipt.token_usage?.[key]) || receipt.token_usage[key] < 0) fail(`invalid token usage: ${key}`)
}
const counts = receipt.clause_counts ?? {}
if (counts.total !== 4 || counts.control !== 1 || counts.guidance !== 1 || counts.both !== 1 || counts.excluded !== 1) fail("conversion receipt does not cover each disposition once")
if (receipt.candidate_verdict !== "accept") fail("generated candidate was not accepted")
if (receipt.negative_control_verdict !== "reject") fail("negative control was not rejected")
const defects = receipt.defect_detection ?? {}
for (const key of ["missing", "contradictory", "incorrectly_duplicated", "excluded_without_reason", "unreachable"]) {
if (defects[key] !== true) fail(`judge did not detect ${key}`)
}
if (receipt.claim_boundary !== "Advisory evidence for this four-clause fixture. The contract is stable; model projections can differ.") fail("conversion receipt has the wrong advisory claim boundary")
return receipt
}
Loading