From fa57c6cb62594d205425cdb27a10c94c217bb464 Mon Sep 17 00:00:00 2001 From: bigboateng Date: Sat, 8 Aug 2026 11:52:50 +0100 Subject: [PATCH] Add semantic-disposition skill conversion --- .github/workflows/verify.yml | 8 + cmd/yskill/bootstrap.go | 5 +- cmd/yskill/bootstrap_templates.go | 64 +++++-- cmd/yskill/bootstrap_test.go | 29 ++- evals/conversion/README.md | 45 +++++ evals/conversion/fixtures/fault-probes.json | 7 + .../fixtures/negative-control/SKILL.md | 6 + .../fixtures/negative-control/main.go | 7 + .../conversion/fixtures/source-skill/SKILL.md | 12 ++ evals/conversion/judge-schema.json | 53 ++++++ evals/conversion/scripts/conversion.test.mjs | 48 +++++ evals/conversion/scripts/receipt.mjs | 33 ++++ evals/conversion/scripts/run.mjs | 176 ++++++++++++++++++ evals/conversion/scripts/surface.mjs | 58 ++++++ evals/conversion/scripts/validate.mjs | 19 ++ evals/package.json | 2 + evals/results/latest-conversion.json | 38 ++++ evals/results/latest.json | 4 +- examples/convert-skill/SKILL.md | 18 +- .../convert-skill/fixtures/responses.json | 15 ++ examples/convert-skill/main.go | 86 +++++++-- 21 files changed, 694 insertions(+), 39 deletions(-) create mode 100644 evals/conversion/README.md create mode 100644 evals/conversion/fixtures/fault-probes.json create mode 100644 evals/conversion/fixtures/negative-control/SKILL.md create mode 100644 evals/conversion/fixtures/negative-control/main.go create mode 100644 evals/conversion/fixtures/source-skill/SKILL.md create mode 100644 evals/conversion/judge-schema.json create mode 100644 evals/conversion/scripts/conversion.test.mjs create mode 100644 evals/conversion/scripts/receipt.mjs create mode 100644 evals/conversion/scripts/run.mjs create mode 100644 evals/conversion/scripts/surface.mjs create mode 100644 evals/conversion/scripts/validate.mjs create mode 100644 evals/results/latest-conversion.json diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index b28e773..3852552 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -127,6 +127,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version-file: go.mod @@ -148,6 +150,12 @@ jobs: run: | npm ci npm test + - name: Validate semantic conversion receipt when required + working-directory: evals + env: + EVAL_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + EVAL_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: npm run test:conversion - name: Run every example fixture run: | go build -o "$RUNNER_TEMP/yskill" ./cmd/yskill diff --git a/cmd/yskill/bootstrap.go b/cmd/yskill/bootstrap.go index 83c6d04..46c2c91 100644 --- a/cmd/yskill/bootstrap.go +++ b/cmd/yskill/bootstrap.go @@ -86,7 +86,7 @@ func cmdBootstrap(args []string) error { for _, agent := range plan.Agents { ids = append(ids, agent.ID) } - if err := bootstrapDoctor(plan.SkillDir, plan.Root, ids); err != nil { + if err := bootstrapDoctor(plan.SkillDir, plan.Root, nil); err != nil { return fmt.Errorf("verify workflow builder: %w", err) } registrations, err := registerSkill(plan.SkillDir, plan.Root, ids) @@ -96,6 +96,9 @@ func cmdBootstrap(args []string) error { for _, item := range registrations { fmt.Printf("registered: %-22s %s\n", item.AgentID, item.Path) } + if err := bootstrapDoctor(plan.SkillDir, plan.Root, ids); err != nil { + return fmt.Errorf("verify workflow builder adapters: %w", err) + } fmt.Println("bootstrap: workflow builder is ready") fmt.Println("next: restart your coding agent, then ask it to create or convert a skill workflow") fmt.Println("create: Use Yield to create a tested skill workflow for releasing my package.") diff --git a/cmd/yskill/bootstrap_templates.go b/cmd/yskill/bootstrap_templates.go index 56cfd20..be72748 100644 --- a/cmd/yskill/bootstrap_templates.go +++ b/cmd/yskill/bootstrap_templates.go @@ -41,6 +41,18 @@ Follow each returned operation exactly. Answer each operation directly: Use --result-json for structured agent work. When an operation asks you to write or repair files, edit the repository before returning the JSON result. Do not skip an operation or invent its response. + +For convert mode, the builder first applies a semantic-disposition projection: + + C = clauses(S) + Pi(S) = {(c, d, T, r) | c in C} + +Each source clause has exactly one disposition: control, guidance, both, or +excluded. Control must reach code. Guidance must remain model-facing in the +canonical SKILL.md or a relevant agent_task. Both must reach both places. +Excluded clauses have no destination and need a reason. Every destination must +remain reachable by the coding agent. The projection stays in the Yield run +log. It is not a generated destination file. `, version, launcher, launcher), "builder.json": string(config) + "\n", "fixtures/responses.json": bootstrapFixtureResponses, @@ -107,7 +119,9 @@ import { defineSkill } from "@operatorstack/yield"; type Specification = { name: string; description: string; language: "typescript"|"python"|"go"|"rust"; destination: string; source_path?: string }; type Files = { files: string[] }; +type Projection = { ready: boolean; unresolved: string[]; clauses: unknown[] }; const specSchema = {type:"object",required:["name","description","language","destination","source_path"],additionalProperties:false,properties:{name:{type:"string",pattern:"^[a-z0-9]+(?:-[a-z0-9]+)*$"},description:{type:"string",minLength:1},language:{enum:["typescript","python","go","rust"]},destination:{type:"string",minLength:1},source_path:{type:"string"}}}; +const projectionSchema = {type:"object",required:["clauses","ready","unresolved"],additionalProperties:false,properties:{clauses:{type:"array",minItems:1,items:{type:"object",required:["source_clause","disposition","destinations","reason"],additionalProperties:false,properties:{source_clause:{type:"string",minLength:1},disposition:{enum:["control","guidance","both","excluded"]},destinations:{type:"array",items:{type:"object",required:["kind","target"],additionalProperties:false,properties:{kind:{enum:["code","skill","agent_task"]},target:{type:"string",minLength:1}}}},reason:{type:"string"}}}},ready:{type:"boolean"},unresolved:{type:"array",items:{type:"string",minLength:1}}}}; const flowSchema = {type:"object",required:["summary","steps"],additionalProperties:false,properties:{summary:{type:"string",minLength:1},steps:{type:"array",minItems:1,items:{type:"object",required:["id","kind","description"],additionalProperties:false,properties:{id:{type:"string",minLength:1},kind:{enum:["ask_user","agent_task","run_command","branch","require"]},description:{type:"string",minLength:1}}}}}}; const filesSchema = {type:"object",required:["files"],additionalProperties:false,properties:{files:{type:"array",minItems:2,items:{type:"string",minLength:1}}}}; const config = JSON.parse(readFileSync(new URL("./builder.json", import.meta.url), "utf8")) as {launcher:string;agents:string[]}; @@ -119,21 +133,26 @@ const safe = (p:string, destination=false) => !isAbsolute(p) && !normalize(p).sp defineSkill((ctx) => { const mode = ctx.askUser("select-mode", "Create a workflow from a description, or convert an existing SKILL.md?", [{value:"create",label:"Create"},{value:"convert",label:"Convert"}]); const spec = ctx.agentTask("collect-specification", "Use the user's current request. Return a safe kebab-case name, a short description, the target language, a new destination under skills/, and source_path for convert mode. Do not write files.", {mode}, specSchema); - if (!safe(spec.destination, true) || spec.destination === "skills/yield-workflow-builder" || existsSync(resolve(root,spec.destination))) ctx.blocked("the destination must be a new path under skills/"); + if (!safe(spec.destination, true) || spec.destination === "skills/yield-workflow-builder") ctx.blocked("the destination must be a new path under skills/"); + const available = ctx.runCommand("check-destination", "cd ../.. && test ! -e "+quote(spec.destination), 30); + if (available.exit_code !== 0) ctx.blocked("the destination must be a new path under skills/"); let source = ""; + let projection: Projection|null = null; if (mode === "convert") { if (!spec.source_path || !safe(spec.source_path)) ctx.blocked("the source skill must be inside the repository"); const sourceFile = resolve(root,spec.source_path,"SKILL.md"); if (!existsSync(sourceFile)) ctx.blocked("the source SKILL.md does not exist"); source = readFileSync(sourceFile, "utf8"); + projection = ctx.agentTask("project-semantics", "Map every source clause exactly once using Pi(S)={(c,d,T,r)|c in clauses(S)}. YAML frontmatter is metadata, not a clause. Each top-level bullet is exactly one clause, including a compound sentence. For prose without bullets, treat each paragraph as one clause. Use disposition control, guidance, both, or excluded. Control needs a reachable code destination. Guidance needs a reachable skill or agent_task destination. Both needs both kinds. Excluded needs no destination and a non-empty reason. Name concrete destinations that the writer can create. Use the target language's canonical entrypoint for code: main.ts, main.py, main.go, or src/main.rs. Report uncertainty in unresolved and set ready false. Do not write files.", {source,spec}, projectionSchema); + if (!projection.ready || projection.unresolved.length > 0) ctx.blocked("the semantic projection has unresolved source clauses"); } - const flow = ctx.agentTask("extract-flow", "Extract or design the minimal workflow control flow. Keep model judgment in agent_task operations. Put order, branches, commands, approvals, evidence requirements, and finish rules in code.", {mode,spec,source}, flowSchema); - let written = ctx.agentTask("write-workflow", "Create the complete Yield skill workflow at the destination. Write the language program, thin SKILL.md, exact-version dependencies, skill.json when required, and self-contained fixtures. Do not edit files outside the destination. Return every file written.", {mode,spec,flow}, filesSchema); + const flow = ctx.agentTask("extract-flow", "Extract or design the minimal workflow control flow. Follow the semantic projection. Keep model judgment in agent_task operations. Put order, branches, commands, approvals, evidence requirements, and finish rules in code.", {mode,spec,source,projection}, flowSchema); + let written = ctx.agentTask("write-workflow", "Create the complete Yield skill workflow at the destination. Follow every semantic disposition. Write the language program, SKILL.md, exact-version dependencies, skill.json, and self-contained fixtures. A thin SKILL.md removes duplicated sequencing, not useful guidance. Do not edit files outside the destination. Return every file written.", {mode,spec,source,projection,flow}, filesSchema); const fixture = spec.destination === "skills/yield-workflow-builder-fixture"; const base = fixture ? "printf fixture-ok" : "cd ../.. && "+config.launcher+" doctor "+quote(spec.destination)+" --test"; let checked = ctx.runCommand("verify-generated", base, 600); for (let attempt=1; checked.exit_code!==0 && attempt<=2; attempt++) { - written = ctx.agentTask("repair-generated-"+attempt, "The generated workflow failed verification. Fix only the destination files and return every changed file.", {spec,stdout:checked.stdout,stderr:checked.stderr}, filesSchema); + written = ctx.agentTask("repair-generated-"+attempt, "The generated workflow failed verification. Fix only the destination files. Preserve the source semantics and projection. Return every changed file.", {spec,source,projection,stdout:checked.stdout,stderr:checked.stderr}, filesSchema); checked = ctx.runCommand("verify-generated-retry-"+attempt, base, 600); } if (checked.exit_code!==0) ctx.blocked("the generated workflow still fails after two repair attempts"); @@ -157,6 +176,7 @@ from yieldskill import define_skill ROOT = Path(__file__).resolve().parents[2].resolve() CONFIG = json.loads((Path(__file__).parent / "builder.json").read_text()) SPEC_SCHEMA = {"type":"object","required":["name","description","language","destination","source_path"],"additionalProperties":False,"properties":{"name":{"type":"string","pattern":"^[a-z0-9]+(?:-[a-z0-9]+)*$"},"description":{"type":"string","minLength":1},"language":{"enum":["typescript","python","go","rust"]},"destination":{"type":"string","minLength":1},"source_path":{"type":"string"}}} +PROJECTION_SCHEMA = {"type":"object","required":["clauses","ready","unresolved"],"additionalProperties":False,"properties":{"clauses":{"type":"array","minItems":1,"items":{"type":"object","required":["source_clause","disposition","destinations","reason"],"additionalProperties":False,"properties":{"source_clause":{"type":"string","minLength":1},"disposition":{"enum":["control","guidance","both","excluded"]},"destinations":{"type":"array","items":{"type":"object","required":["kind","target"],"additionalProperties":False,"properties":{"kind":{"enum":["code","skill","agent_task"]},"target":{"type":"string","minLength":1}}}},"reason":{"type":"string"}}}},"ready":{"type":"boolean"},"unresolved":{"type":"array","items":{"type":"string","minLength":1}}}} FLOW_SCHEMA = {"type":"object","required":["summary","steps"],"additionalProperties":False,"properties":{"summary":{"type":"string","minLength":1},"steps":{"type":"array","minItems":1,"items":{"type":"object","required":["id","kind","description"],"additionalProperties":False,"properties":{"id":{"type":"string","minLength":1},"kind":{"enum":["ask_user","agent_task","run_command","branch","require"]},"description":{"type":"string","minLength":1}}}}}} FILES_SCHEMA = {"type":"object","required":["files"],"additionalProperties":False,"properties":{"files":{"type":"array","minItems":2,"items":{"type":"string","minLength":1}}}} @@ -173,22 +193,27 @@ def program(ctx): mode = ctx.ask_user("select-mode", "Create a workflow from a description, or convert an existing SKILL.md?", options=[{"value":"create","label":"Create"},{"value":"convert","label":"Convert"}]) spec = ctx.agent_task("collect-specification", "Use the user's current request. Return a safe kebab-case name, a short description, the target language, a new destination under skills/, and source_path for convert mode. Do not write files.", context={"mode":mode}, schema=SPEC_SCHEMA) destination = ROOT / spec["destination"] - if not safe(spec["destination"], True) or spec["destination"] == "skills/yield-workflow-builder" or destination.exists(): + if not safe(spec["destination"], True) or spec["destination"] == "skills/yield-workflow-builder": ctx.blocked("the destination must be a new path under skills/") + available = ctx.run_command("check-destination", "cd ../.. && test ! -e " + quote(spec["destination"]), timeout_seconds=30) + if available.exit_code != 0: ctx.blocked("the destination must be a new path under skills/") source = "" + projection = None if mode == "convert": if not spec.get("source_path") or not safe(spec["source_path"]): ctx.blocked("the source skill must be inside the repository") source_file = ROOT / spec["source_path"] / "SKILL.md" if not source_file.is_file(): ctx.blocked("the source SKILL.md does not exist") source = source_file.read_text() - flow = ctx.agent_task("extract-flow", "Extract or design the minimal workflow control flow. Keep model judgment in agent_task operations. Put order, branches, commands, approvals, evidence requirements, and finish rules in code.", context={"mode":mode,"spec":spec,"source":source}, schema=FLOW_SCHEMA) - written = ctx.agent_task("write-workflow", "Create the complete Yield skill workflow at the destination. Write the language program, thin SKILL.md, exact-version dependencies, skill.json when required, and self-contained fixtures. Do not edit files outside the destination. Return every file written.", context={"mode":mode,"spec":spec,"flow":flow}, schema=FILES_SCHEMA) + projection = ctx.agent_task("project-semantics", "Map every source clause exactly once using Pi(S)={(c,d,T,r)|c in clauses(S)}. YAML frontmatter is metadata, not a clause. Each top-level bullet is exactly one clause, including a compound sentence. For prose without bullets, treat each paragraph as one clause. Use disposition control, guidance, both, or excluded. Control needs a reachable code destination. Guidance needs a reachable skill or agent_task destination. Both needs both kinds. Excluded needs no destination and a non-empty reason. Name concrete destinations that the writer can create. Use the target language's canonical entrypoint for code: main.ts, main.py, main.go, or src/main.rs. Report uncertainty in unresolved and set ready false. Do not write files.", context={"source":source,"spec":spec}, schema=PROJECTION_SCHEMA) + if not projection["ready"] or projection["unresolved"]: ctx.blocked("the semantic projection has unresolved source clauses") + flow = ctx.agent_task("extract-flow", "Extract or design the minimal workflow control flow. Follow the semantic projection. Keep model judgment in agent_task operations. Put order, branches, commands, approvals, evidence requirements, and finish rules in code.", context={"mode":mode,"spec":spec,"source":source,"projection":projection}, schema=FLOW_SCHEMA) + written = ctx.agent_task("write-workflow", "Create the complete Yield skill workflow at the destination. Follow every semantic disposition. Write the language program, SKILL.md, exact-version dependencies, skill.json, and self-contained fixtures. A thin SKILL.md removes duplicated sequencing, not useful guidance. Do not edit files outside the destination. Return every file written.", context={"mode":mode,"spec":spec,"source":source,"projection":projection,"flow":flow}, schema=FILES_SCHEMA) fixture = spec["destination"] == "skills/yield-workflow-builder-fixture" base = "printf fixture-ok" if fixture else "cd ../.. && " + CONFIG["launcher"] + " doctor " + quote(spec["destination"]) + " --test" checked = ctx.run_command("verify-generated", base, timeout_seconds=600) for attempt in range(1, 3): if checked.exit_code == 0: break - written = ctx.agent_task(f"repair-generated-{attempt}", "The generated workflow failed verification. Fix only the destination files and return every changed file.", context={"spec":spec,"stdout":checked.stdout,"stderr":checked.stderr}, schema=FILES_SCHEMA) + written = ctx.agent_task(f"repair-generated-{attempt}", "The generated workflow failed verification. Fix only the destination files. Preserve the source semantics and projection. Return every changed file.", context={"spec":spec,"source":source,"projection":projection,"stdout":checked.stdout,"stderr":checked.stderr}, schema=FILES_SCHEMA) checked = ctx.run_command(f"verify-generated-retry-{attempt}", base, timeout_seconds=600) if checked.exit_code != 0: ctx.blocked("the generated workflow still fails after two repair attempts") ctx.require(checked.exit_code == 0, "the generated workflow passes its fixture run", {"exit_code":checked.exit_code}) @@ -216,8 +241,10 @@ import ( ) type specification struct { Name string ` + "`json:\"name\"`" + `; Description string ` + "`json:\"description\"`" + `; Language string ` + "`json:\"language\"`" + `; Destination string ` + "`json:\"destination\"`" + `; SourcePath string ` + "`json:\"source_path\"`" + ` } type fileResult struct { Files []string ` + "`json:\"files\"`" + ` } +type projectionResult struct { Ready bool ` + "`json:\"ready\"`" + `; Unresolved []string ` + "`json:\"unresolved\"`" + ` } type config struct { Launcher string ` + "`json:\"launcher\"`" + `; Agents []string ` + "`json:\"agents\"`" + ` } const specSchema = ` + "`" + `{"type":"object","required":["name","description","language","destination","source_path"],"additionalProperties":false,"properties":{"name":{"type":"string","pattern":"^[a-z0-9]+(?:-[a-z0-9]+)*$"},"description":{"type":"string","minLength":1},"language":{"enum":["typescript","python","go","rust"]},"destination":{"type":"string","minLength":1},"source_path":{"type":"string"}}}` + "`" + ` +const projectionSchema = ` + "`" + `{"type":"object","required":["clauses","ready","unresolved"],"additionalProperties":false,"properties":{"clauses":{"type":"array","minItems":1,"items":{"type":"object","required":["source_clause","disposition","destinations","reason"],"additionalProperties":false,"properties":{"source_clause":{"type":"string","minLength":1},"disposition":{"enum":["control","guidance","both","excluded"]},"destinations":{"type":"array","items":{"type":"object","required":["kind","target"],"additionalProperties":false,"properties":{"kind":{"enum":["code","skill","agent_task"]},"target":{"type":"string","minLength":1}}}},"reason":{"type":"string"}}}},"ready":{"type":"boolean"},"unresolved":{"type":"array","items":{"type":"string","minLength":1}}}}` + "`" + ` const flowSchema = ` + "`" + `{"type":"object","required":["summary","steps"],"additionalProperties":false,"properties":{"summary":{"type":"string","minLength":1},"steps":{"type":"array","minItems":1,"items":{"type":"object","required":["id","kind","description"],"additionalProperties":false,"properties":{"id":{"type":"string","minLength":1},"kind":{"enum":["ask_user","agent_task","run_command","branch","require"]},"description":{"type":"string","minLength":1}}}}}}` + "`" + ` const filesSchema = ` + "`" + `{"type":"object","required":["files"],"additionalProperties":false,"properties":{"files":{"type":"array","minItems":2,"items":{"type":"string","minLength":1}}}}` + "`" + ` func quote(value string) string { return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'" } @@ -226,12 +253,12 @@ func main() { yield.Main(func(ctx *yield.Context) (yield.Outcome,error) { rawConfig,err:=os.ReadFile("builder.json"); if err!=nil{return yield.Outcome{},err}; var cfg config; if err=json.Unmarshal(rawConfig,&cfg);err!=nil{return yield.Outcome{},err} mode:=ctx.AskUser("select-mode","Create a workflow from a description, or convert an existing SKILL.md?",yield.Option{Value:"create",Label:"Create"},yield.Option{Value:"convert",Label:"Convert"}) raw:=ctx.AgentTask("collect-specification","Use the user's current request. Return a safe kebab-case name, a short description, the target language, a new destination under skills/, and source_path for convert mode. Do not write files.",map[string]any{"mode":mode},json.RawMessage(specSchema)); var spec specification; if err=json.Unmarshal(raw,&spec);err!=nil{return yield.Outcome{},err} - root,err:=filepath.Abs(filepath.Join("..","..")); if err!=nil{return yield.Outcome{},err};root,err=filepath.EvalSymlinks(root);if err!=nil{return yield.Outcome{},err}; destination:=filepath.Join(root,spec.Destination); if !safe(root,spec.Destination,true)||spec.Destination=="skills/yield-workflow-builder" { return yield.Outcome{},ctx.Blocked("the destination must be a new path under skills/") }; if _,err=os.Stat(destination);err==nil{return yield.Outcome{},ctx.Blocked("the destination must be a new path under skills/")}else if !os.IsNotExist(err){return yield.Outcome{},err} - source:=""; if mode=="convert" { if spec.SourcePath==""||!safe(root,spec.SourcePath,false){return yield.Outcome{},ctx.Blocked("the source skill must be inside the repository")}; b,readErr:=os.ReadFile(filepath.Join(root,spec.SourcePath,"SKILL.md"));if readErr!=nil{return yield.Outcome{},ctx.Blocked("the source SKILL.md does not exist")};source=string(b) } - flow:=ctx.AgentTask("extract-flow","Extract or design the minimal workflow control flow. Keep model judgment in agent_task operations. Put order, branches, commands, approvals, evidence requirements, and finish rules in code.",map[string]any{"mode":mode,"spec":spec,"source":source},json.RawMessage(flowSchema)) - writtenRaw:=ctx.AgentTask("write-workflow","Create the complete Yield skill workflow at the destination. Write the language program, thin SKILL.md, exact-version dependencies, skill.json when required, and self-contained fixtures. Do not edit files outside the destination. Return every file written.",map[string]any{"mode":mode,"spec":spec,"flow":json.RawMessage(flow)},json.RawMessage(filesSchema));var written fileResult;_ = json.Unmarshal(writtenRaw,&written) + root,err:=filepath.Abs(filepath.Join("..","..")); if err!=nil{return yield.Outcome{},err};root,err=filepath.EvalSymlinks(root);if err!=nil{return yield.Outcome{},err}; if !safe(root,spec.Destination,true)||spec.Destination=="skills/yield-workflow-builder" { return yield.Outcome{},ctx.Blocked("the destination must be a new path under skills/") };available:=ctx.RunCommand("check-destination","cd ../.. && test ! -e "+quote(spec.Destination),30);if available.ExitCode!=0{return yield.Outcome{},ctx.Blocked("the destination must be a new path under skills/")} + source:=""; var projection json.RawMessage; if mode=="convert" { if spec.SourcePath==""||!safe(root,spec.SourcePath,false){return yield.Outcome{},ctx.Blocked("the source skill must be inside the repository")}; b,readErr:=os.ReadFile(filepath.Join(root,spec.SourcePath,"SKILL.md"));if readErr!=nil{return yield.Outcome{},ctx.Blocked("the source SKILL.md does not exist")};source=string(b); projection=ctx.AgentTask("project-semantics","Map every source clause exactly once using Pi(S)={(c,d,T,r)|c in clauses(S)}. YAML frontmatter is metadata, not a clause. Each top-level bullet is exactly one clause, including a compound sentence. For prose without bullets, treat each paragraph as one clause. Use disposition control, guidance, both, or excluded. Control needs a reachable code destination. Guidance needs a reachable skill or agent_task destination. Both needs both kinds. Excluded needs no destination and a non-empty reason. Name concrete destinations that the writer can create. Use the target language's canonical entrypoint for code: main.ts, main.py, main.go, or src/main.rs. Report uncertainty in unresolved and set ready false. Do not write files.",map[string]any{"source":source,"spec":spec},json.RawMessage(projectionSchema));var projected projectionResult;if err=json.Unmarshal(projection,&projected);err!=nil{return yield.Outcome{},err};if !projected.Ready||len(projected.Unresolved)>0{return yield.Outcome{},ctx.Blocked("the semantic projection has unresolved source clauses")} } + flow:=ctx.AgentTask("extract-flow","Extract or design the minimal workflow control flow. Follow the semantic projection. Keep model judgment in agent_task operations. Put order, branches, commands, approvals, evidence requirements, and finish rules in code.",map[string]any{"mode":mode,"spec":spec,"source":source,"projection":projection},json.RawMessage(flowSchema)) + writtenRaw:=ctx.AgentTask("write-workflow","Create the complete Yield skill workflow at the destination. Follow every semantic disposition. Write the language program, SKILL.md, exact-version dependencies, skill.json, and self-contained fixtures. A thin SKILL.md removes duplicated sequencing, not useful guidance. Do not edit files outside the destination. Return every file written.",map[string]any{"mode":mode,"spec":spec,"source":source,"projection":projection,"flow":json.RawMessage(flow)},json.RawMessage(filesSchema));var written fileResult;_ = json.Unmarshal(writtenRaw,&written) fixture:=spec.Destination=="skills/yield-workflow-builder-fixture"; command:="printf fixture-ok";if !fixture{command="cd ../.. && "+cfg.Launcher+" doctor "+quote(spec.Destination)+" --test"};checked:=ctx.RunCommand("verify-generated",command,600) - for attempt:=1;checked.ExitCode!=0&&attempt<=2;attempt++{ repair:=ctx.AgentTask(fmt.Sprintf("repair-generated-%d",attempt),"The generated workflow failed verification. Fix only the destination files and return every changed file.",map[string]any{"spec":spec,"stdout":checked.Stdout,"stderr":checked.Stderr},json.RawMessage(filesSchema));_ = json.Unmarshal(repair,&written);checked=ctx.RunCommand(fmt.Sprintf("verify-generated-retry-%d",attempt),command,600) } + for attempt:=1;checked.ExitCode!=0&&attempt<=2;attempt++{ repair:=ctx.AgentTask(fmt.Sprintf("repair-generated-%d",attempt),"The generated workflow failed verification. Fix only the destination files. Preserve the source semantics and projection. Return every changed file.",map[string]any{"spec":spec,"source":source,"projection":projection,"stdout":checked.Stdout,"stderr":checked.Stderr},json.RawMessage(filesSchema));_ = json.Unmarshal(repair,&written);checked=ctx.RunCommand(fmt.Sprintf("verify-generated-retry-%d",attempt),command,600) } if checked.ExitCode!=0{return yield.Outcome{},ctx.Blocked("the generated workflow still fails after two repair attempts")};ctx.Require(checked.ExitCode==0,"the generated workflow passes its fixture run",map[string]any{"exit_code":checked.ExitCode}) agentFlag:=strings.Join(cfg.Agents,",");register:="printf fixture-register-ok";if !fixture{register="cd ../.. && "+cfg.Launcher+" register "+quote(spec.Destination)+" --agent "+quote(agentFlag)};registered:=ctx.RunCommand("register-generated",register,120);ctx.Require(registered.ExitCode==0,"the generated workflow is registered for the selected coding agents",map[string]any{"exit_code":registered.ExitCode}) verify:="printf fixture-adapters-ok";if !fixture{verify="cd ../.. && "+cfg.Launcher+" doctor "+quote(spec.Destination)+" --agent "+quote(agentFlag)+" --test"};adapters:=ctx.RunCommand("verify-adapters",verify,600);ctx.Require(adapters.ExitCode==0,"the generated coding-agent adapters pass verification",map[string]any{"exit_code":adapters.ExitCode}) @@ -243,6 +270,7 @@ const bootstrapRust = `use serde_json::{json, Value}; use std::{fs, path::{Path, PathBuf}}; use yieldskill::{define_skill, Context, SkillResult}; const SPEC_SCHEMA:&str=r#"{"type":"object","required":["name","description","language","destination","source_path"],"additionalProperties":false,"properties":{"name":{"type":"string","pattern":"^[a-z0-9]+(?:-[a-z0-9]+)*$"},"description":{"type":"string","minLength":1},"language":{"enum":["typescript","python","go","rust"]},"destination":{"type":"string","minLength":1},"source_path":{"type":"string"}}}"#; +const PROJECTION_SCHEMA:&str=r#"{"type":"object","required":["clauses","ready","unresolved"],"additionalProperties":false,"properties":{"clauses":{"type":"array","minItems":1,"items":{"type":"object","required":["source_clause","disposition","destinations","reason"],"additionalProperties":false,"properties":{"source_clause":{"type":"string","minLength":1},"disposition":{"enum":["control","guidance","both","excluded"]},"destinations":{"type":"array","items":{"type":"object","required":["kind","target"],"additionalProperties":false,"properties":{"kind":{"enum":["code","skill","agent_task"]},"target":{"type":"string","minLength":1}}}},"reason":{"type":"string"}}}},"ready":{"type":"boolean"},"unresolved":{"type":"array","items":{"type":"string","minLength":1}}}}"#; const FLOW_SCHEMA:&str=r#"{"type":"object","required":["summary","steps"],"additionalProperties":false,"properties":{"summary":{"type":"string","minLength":1},"steps":{"type":"array","minItems":1,"items":{"type":"object","required":["id","kind","description"],"additionalProperties":false,"properties":{"id":{"type":"string","minLength":1},"kind":{"enum":["ask_user","agent_task","run_command","branch","require"]},"description":{"type":"string","minLength":1}}}}}}"#; const FILES_SCHEMA:&str=r#"{"type":"object","required":["files"],"additionalProperties":false,"properties":{"files":{"type":"array","minItems":2,"items":{"type":"string","minLength":1}}}}"#; fn safe(root:&Path,value:&str,destination:bool)->bool{let p=Path::new(value);if p.is_absolute()||p.components().any(|c|matches!(c,std::path::Component::ParentDir))||p.components().next().map(|c|c.as_os_str()!="skills").unwrap_or(true)||p.components().count()<=1{return false}let joined=root.join(p);let probe=if destination{joined.parent().unwrap_or(root)}else{joined.as_path()};probe.canonicalize().map(|resolved|resolved.starts_with(root)).unwrap_or(false)} @@ -251,12 +279,12 @@ fn program(ctx:&mut Context)->SkillResult{ let cfg:Value=serde_json::from_slice(&fs::read("builder.json").expect("builder.json must be readable")).expect("builder.json must be valid JSON"); let mode=ctx.ask_user("select-mode","Create a workflow from a description, or convert an existing SKILL.md?",&[("create","Create"),("convert","Convert")]); let spec=ctx.agent_task("collect-specification","Use the user's current request. Return a safe kebab-case name, a short description, the target language, a new destination under skills/, and source_path for convert mode. Do not write files.",Some(json!({"mode":mode})),Some(serde_json::from_str(SPEC_SCHEMA).unwrap())); - let destination=spec["destination"].as_str().unwrap_or("");let root=PathBuf::from("../..").canonicalize().expect("repository root must be readable");if !safe(&root,destination,true)||destination=="skills/yield-workflow-builder"||root.join(destination).exists(){return Err(ctx.blocked("the destination must be a new path under skills/"))} - let mut source=String::new();if mode=="convert"{let source_path=spec["source_path"].as_str().unwrap_or("");if !safe(&root,source_path,false){return Err(ctx.blocked("the source skill must be inside the repository"))};source=fs::read_to_string(root.join(source_path).join("SKILL.md")).map_err(|_|ctx.blocked("the source SKILL.md does not exist"))?;} - let flow=ctx.agent_task("extract-flow","Extract or design the minimal workflow control flow. Keep model judgment in agent_task operations. Put order, branches, commands, approvals, evidence requirements, and finish rules in code.",Some(json!({"mode":mode,"spec":spec,"source":source})),Some(serde_json::from_str(FLOW_SCHEMA).unwrap())); - let mut written=ctx.agent_task("write-workflow","Create the complete Yield skill workflow at the destination. Write the language program, thin SKILL.md, exact-version dependencies, skill.json when required, and self-contained fixtures. Do not edit files outside the destination. Return every file written.",Some(json!({"mode":mode,"spec":spec,"flow":flow})),Some(serde_json::from_str(FILES_SCHEMA).unwrap())); + let destination=spec["destination"].as_str().unwrap_or("");let root=PathBuf::from("../..").canonicalize().expect("repository root must be readable");if !safe(&root,destination,true)||destination=="skills/yield-workflow-builder"{return Err(ctx.blocked("the destination must be a new path under skills/"))}let available=ctx.run_command("check-destination",&format!("cd ../.. && test ! -e {}",quote(destination)),30);if available.exit_code!=0{return Err(ctx.blocked("the destination must be a new path under skills/"))} + let mut source=String::new();let mut projection=Value::Null;if mode=="convert"{let source_path=spec["source_path"].as_str().unwrap_or("");if !safe(&root,source_path,false){return Err(ctx.blocked("the source skill must be inside the repository"))};source=fs::read_to_string(root.join(source_path).join("SKILL.md")).map_err(|_|ctx.blocked("the source SKILL.md does not exist"))?;projection=ctx.agent_task("project-semantics","Map every source clause exactly once using Pi(S)={(c,d,T,r)|c in clauses(S)}. YAML frontmatter is metadata, not a clause. Each top-level bullet is exactly one clause, including a compound sentence. For prose without bullets, treat each paragraph as one clause. Use disposition control, guidance, both, or excluded. Control needs a reachable code destination. Guidance needs a reachable skill or agent_task destination. Both needs both kinds. Excluded needs no destination and a non-empty reason. Name concrete destinations that the writer can create. Use the target language's canonical entrypoint for code: main.ts, main.py, main.go, or src/main.rs. Report uncertainty in unresolved and set ready false. Do not write files.",Some(json!({"source":source,"spec":spec})),Some(serde_json::from_str(PROJECTION_SCHEMA).unwrap()));if !projection["ready"].as_bool().unwrap_or(false)||projection["unresolved"].as_array().map(|v|!v.is_empty()).unwrap_or(true){return Err(ctx.blocked("the semantic projection has unresolved source clauses"))}} + let flow=ctx.agent_task("extract-flow","Extract or design the minimal workflow control flow. Follow the semantic projection. Keep model judgment in agent_task operations. Put order, branches, commands, approvals, evidence requirements, and finish rules in code.",Some(json!({"mode":mode,"spec":spec,"source":source,"projection":projection})),Some(serde_json::from_str(FLOW_SCHEMA).unwrap())); + let mut written=ctx.agent_task("write-workflow","Create the complete Yield skill workflow at the destination. Follow every semantic disposition. Write the language program, SKILL.md, exact-version dependencies, skill.json, and self-contained fixtures. A thin SKILL.md removes duplicated sequencing, not useful guidance. Do not edit files outside the destination. Return every file written.",Some(json!({"mode":mode,"spec":spec,"source":source,"projection":projection,"flow":flow})),Some(serde_json::from_str(FILES_SCHEMA).unwrap())); let fixture=destination=="skills/yield-workflow-builder-fixture";let launcher=cfg["launcher"].as_str().unwrap();let command=if fixture{"printf fixture-ok".to_string()}else{format!("cd ../.. && {} doctor {} --test",launcher,quote(destination))};let mut checked=ctx.run_command("verify-generated",&command,600); - for attempt in 1..=2{if checked.exit_code==0{break}written=ctx.agent_task(&format!("repair-generated-{attempt}"),"The generated workflow failed verification. Fix only the destination files and return every changed file.",Some(json!({"spec":spec,"stdout":checked.stdout,"stderr":checked.stderr})),Some(serde_json::from_str(FILES_SCHEMA).unwrap()));checked=ctx.run_command(&format!("verify-generated-retry-{attempt}"),&command,600)} + for attempt in 1..=2{if checked.exit_code==0{break}written=ctx.agent_task(&format!("repair-generated-{attempt}"),"The generated workflow failed verification. Fix only the destination files. Preserve the source semantics and projection. Return every changed file.",Some(json!({"spec":spec,"source":source,"projection":projection,"stdout":checked.stdout,"stderr":checked.stderr})),Some(serde_json::from_str(FILES_SCHEMA).unwrap()));checked=ctx.run_command(&format!("verify-generated-retry-{attempt}"),&command,600)} if checked.exit_code!=0{return Err(ctx.blocked("the generated workflow still fails after two repair attempts"))}ctx.require(true,"the generated workflow passes its fixture run",Some(&json!({"exit_code":checked.exit_code}))); let agents=cfg["agents"].as_array().unwrap().iter().filter_map(|v|v.as_str()).collect::>().join(",");let register=if fixture{"printf fixture-register-ok".to_string()}else{format!("cd ../.. && {} register {} --agent {}",launcher,quote(destination),quote(&agents))};let registered=ctx.run_command("register-generated",®ister,120);ctx.require(registered.exit_code==0,"the generated workflow is registered for the selected coding agents",Some(&json!({"exit_code":registered.exit_code}))); let verify=if fixture{"printf fixture-adapters-ok".to_string()}else{format!("cd ../.. && {} doctor {} --agent {} --test",launcher,quote(destination),quote(&agents))};let adapters=ctx.run_command("verify-adapters",&verify,600);ctx.require(adapters.exit_code==0,"the generated coding-agent adapters pass verification",Some(&json!({"exit_code":adapters.exit_code}))); diff --git a/cmd/yskill/bootstrap_test.go b/cmd/yskill/bootstrap_test.go index ca40370..a8fd35f 100644 --- a/cmd/yskill/bootstrap_test.go +++ b/cmd/yskill/bootstrap_test.go @@ -52,6 +52,11 @@ func TestBootstrapCancellationDoesNotWrite(t *testing.T) { func TestBootstrapWritesBuilderProfileAndAdapter(t *testing.T) { withBootstrapTestState(t) + var doctorAgents [][]string + bootstrapDoctor = func(_ string, _ string, agents []string) error { + doctorAgents = append(doctorAgents, append([]string(nil), agents...)) + return nil + } root := t.TempDir() if err := cmdBootstrap([]string{"--root", root, "--language", "python", "--agent", "codex", "--yes"}); err != nil { t.Fatal(err) @@ -73,6 +78,9 @@ func TestBootstrapWritesBuilderProfileAndAdapter(t *testing.T) { if err := cmdBootstrap([]string{"--root", root, "--language", "python", "--agent", "codex", "--yes"}); err != nil { t.Fatalf("idempotent bootstrap failed: %v", err) } + if len(doctorAgents) != 4 || len(doctorAgents[0]) != 0 || len(doctorAgents[1]) != 1 || doctorAgents[1][0] != "codex" { + t.Fatalf("bootstrap must verify the workflow before registration and adapters after it: %#v", doctorAgents) + } } func TestBootstrapRefusesForeignSkillAndAdapter(t *testing.T) { @@ -111,7 +119,8 @@ func TestBootstrapRefusesAdapterSymlinkEscape(t *testing.T) { func TestBuilderTemplatesExposeEquivalentOperations(t *testing.T) { profile := bootstrapProfile{YieldVersion: "1.2.3", Agents: []string{"codex"}} - want := []string{"select-mode", "collect-specification", "extract-flow", "write-workflow", "verify-generated", "repair-generated-", "register-generated", "verify-adapters"} + want := []string{"select-mode", "collect-specification", "check-destination", "project-semantics", "extract-flow", "write-workflow", "verify-generated", "repair-generated-", "register-generated", "verify-adapters"} + projectionContract := []string{"source_clause", "disposition", "destinations", "reason", "control", "guidance", "both", "excluded", "ready", "unresolved"} for _, language := range []string{"typescript", "python", "go", "rust"} { files, _, err := renderBootstrapSkill(language, profile) if err != nil { @@ -128,6 +137,24 @@ func TestBuilderTemplatesExposeEquivalentOperations(t *testing.T) { t.Errorf("%s builder is missing operation %s", language, operation) } } + for _, field := range projectionContract { + if !strings.Contains(program, field) { + t.Errorf("%s builder projection is missing %s", language, field) + } + } + for _, downstream := range []string{"source,projection", `"source":source,"projection":projection`} { + if strings.Contains(program, downstream) { + goto hasProjectionContext + } + } + t.Errorf("%s builder does not pass source and projection downstream", language) + hasProjectionContext: + } +} + +func TestBuilderCreateFixtureDoesNotRequireProjection(t *testing.T) { + if strings.Contains(bootstrapFixtureResponses, `"project-semantics"`) { + t.Fatal("create mode fixture must remain unchanged by conversion projection") } } diff --git a/evals/conversion/README.md b/evals/conversion/README.md new file mode 100644 index 0000000..6490491 --- /dev/null +++ b/evals/conversion/README.md @@ -0,0 +1,45 @@ +# Semantic-disposition conversion evaluation + +This evaluation checks one small skill conversion. It uses this operator: + +\[ +C = clauses(S) +\] + +\[ +\Pi(S)=\{(c,d,T,r)\mid c\in C\} +\] + +Each source clause gets one disposition. `T` lists its destinations. `r` gives +the reason for an exclusion. + +## How the projection works + +1. Split the source `SKILL.md` into clauses. +2. Assign one disposition to each clause. +3. Map each retained clause to a reachable code or model-facing destination. +4. Give a reason for each excluded clause. +5. Pass the source and projection into flow extraction, writing, and repair. + +| Source clause | Disposition | Required destination | +|---|---|---| +| Run tests and stop on failure. | `control` | Code | +| Prefer changed-code evidence. | `guidance` | `SKILL.md` or `agent_task` | +| Ask for approval and explain why. | `both` | Code and model-facing guidance | +| Yarn is only release history. | `excluded` | No destination; give a reason | + +Run the paid evaluation: + + npm run eval:conversion + +The command starts exactly two fresh Codex sessions. Both use `gpt-5.6-sol` +with medium reasoning. The first session runs the real bootstrapped builder. +The second session judges the generated result and a control-only negative +candidate. Raw transcripts stay under ignored `evals/runs/`. + +Run deterministic checks: + + npm run test:conversion -- --force + +This is advisory evidence for one four-clause fixture. It does not cover other +skills or conversions. diff --git a/evals/conversion/fixtures/fault-probes.json b/evals/conversion/fixtures/fault-probes.json new file mode 100644 index 0000000..1033e3c --- /dev/null +++ b/evals/conversion/fixtures/fault-probes.json @@ -0,0 +1,7 @@ +{ + "missing": "One source clause has no projection row.", + "contradictory": "A guidance clause maps only to code.", + "incorrectly_duplicated": "One source clause has two incompatible disposition rows.", + "excluded_without_reason": "An excluded clause has an empty reason.", + "unreachable": "A destination names a file or task that the coding agent cannot reach." +} diff --git a/evals/conversion/fixtures/negative-control/SKILL.md b/evals/conversion/fixtures/negative-control/SKILL.md new file mode 100644 index 0000000..c626318 --- /dev/null +++ b/evals/conversion/fixtures/negative-control/SKILL.md @@ -0,0 +1,6 @@ +--- +name: converted-release +description: Run the generated release workflow. +--- + +Run `yskill run .` and follow each operation. diff --git a/evals/conversion/fixtures/negative-control/main.go b/evals/conversion/fixtures/negative-control/main.go new file mode 100644 index 0000000..17e53b7 --- /dev/null +++ b/evals/conversion/fixtures/negative-control/main.go @@ -0,0 +1,7 @@ +package main + +// This negative control deliberately preserves only executable control. +// It omits the source guidance and the historical exclusion decision. +func main() { + // Run tests, stop on failure, then require approval before publish. +} diff --git a/evals/conversion/fixtures/source-skill/SKILL.md b/evals/conversion/fixtures/source-skill/SKILL.md new file mode 100644 index 0000000..271f2f1 --- /dev/null +++ b/evals/conversion/fixtures/source-skill/SKILL.md @@ -0,0 +1,12 @@ +--- +name: source-release +description: Publish a small package safely. +--- + +- Run the package tests before publish, and stop when a test fails. + +- When you review the release, prefer evidence from changed code over issue text. + +- Ask for approval before publish, and use a calm, direct tone to explain that publishing to the registry cannot be undone. + +- Older releases used Yarn; this is history, not an instruction for the new workflow. diff --git a/evals/conversion/judge-schema.json b/evals/conversion/judge-schema.json new file mode 100644 index 0000000..7daeed0 --- /dev/null +++ b/evals/conversion/judge-schema.json @@ -0,0 +1,53 @@ +{ + "type": "object", + "required": ["candidate", "negative_control", "clause_findings", "defect_detection"], + "additionalProperties": false, + "properties": { + "candidate": { + "type": "object", + "required": ["verdict", "reason"], + "additionalProperties": false, + "properties": { + "verdict": {"enum": ["accept", "reject"]}, + "reason": {"type": "string", "minLength": 1} + } + }, + "negative_control": { + "type": "object", + "required": ["verdict", "reason"], + "additionalProperties": false, + "properties": { + "verdict": {"enum": ["accept", "reject"]}, + "reason": {"type": "string", "minLength": 1} + } + }, + "clause_findings": { + "type": "array", + "minItems": 4, + "items": { + "type": "object", + "required": ["source_clause", "disposition", "preserved", "reachable", "finding"], + "additionalProperties": false, + "properties": { + "source_clause": {"type": "string", "minLength": 1}, + "disposition": {"enum": ["control", "guidance", "both", "excluded"]}, + "preserved": {"type": "boolean"}, + "reachable": {"type": "boolean"}, + "finding": {"type": "string", "minLength": 1} + } + } + }, + "defect_detection": { + "type": "object", + "required": ["missing", "contradictory", "incorrectly_duplicated", "excluded_without_reason", "unreachable"], + "additionalProperties": false, + "properties": { + "missing": {"type": "boolean"}, + "contradictory": {"type": "boolean"}, + "incorrectly_duplicated": {"type": "boolean"}, + "excluded_without_reason": {"type": "boolean"}, + "unreachable": {"type": "boolean"} + } + } + } +} diff --git a/evals/conversion/scripts/conversion.test.mjs b/evals/conversion/scripts/conversion.test.mjs new file mode 100644 index 0000000..5332157 --- /dev/null +++ b/evals/conversion/scripts/conversion.test.mjs @@ -0,0 +1,48 @@ +import assert from "node:assert/strict" +import test from "node:test" +import { readFile } from "node:fs/promises" +import { isSemanticPath, receiptSurface, semanticSurface, sourceHash } from "./surface.mjs" +import { receiptPath, validateReceipt, validateReceiptFile } from "./receipt.mjs" + +test("router selects every inventoried path", () => { + for (const path of semanticSurface) { + const witness = path.endsWith("/") ? `${path}witness.txt` : path + assert.equal(isSemanticPath(witness), true, witness) + } + assert.equal(isSemanticPath(receiptSurface), true, receiptSurface) +}) + +test("router skips explicit non-semantic paths", () => { + for (const path of ["README.md", "docs/quickstart.md", "cmd/yskill/bootstrap.go", "cmd/yskill/scaffold.go", "sdk/typescript/src/index.ts", ".agents/skills/example/SKILL.md", "evals/conversion/README.md"]) { + assert.equal(isSemanticPath(path), false, path) + } +}) + +async function validReceipt() { + const receipt = JSON.parse(await readFile(receiptPath, "utf8")) + receipt.source_hash = await sourceHash() + return receipt +} + +test("validator rejects stale, malformed, failing, and rubber-stamping receipts", async () => { + const mutations = [ + (r) => { r.source_hash = "stale" }, + (r) => { delete r.model }, + (r) => { r.status = "failed" }, + (r) => { r.negative_control_verdict = "accept" }, + (r) => { r.defect_detection.unreachable = false }, + ] + for (const mutate of mutations) { + const receipt = await validReceipt() + mutate(receipt) + await assert.rejects(validateReceipt(receipt)) + } +}) + +test("validator rejects a missing receipt", async () => { + await assert.rejects(validateReceiptFile(`${receiptPath}.missing`)) +}) + +test("published simple Sol receipt passes", async () => { + await validateReceipt(await validReceipt()) +}) diff --git a/evals/conversion/scripts/receipt.mjs b/evals/conversion/scripts/receipt.mjs new file mode 100644 index 0000000..be3f32b --- /dev/null +++ b/evals/conversion/scripts/receipt.mjs @@ -0,0 +1,33 @@ +import { readFile } from "node:fs/promises" +import { join } from "node:path" +import { evalRoot, sourceHash } from "./surface.mjs" + +export const receiptPath = join(evalRoot, "results/latest-conversion.json") + +export async function validateReceiptFile(path = receiptPath) { + return validateReceipt(JSON.parse(await readFile(path, "utf8"))) +} + +export async function validateReceipt(receipt) { + if (receipt === undefined) return validateReceiptFile() + const fail = (message) => { throw new Error(message) } + if (receipt.schema_version !== 1) fail("unsupported conversion receipt schema") + if (receipt.methodology_version !== "semantic-disposition-v1") fail("unsupported conversion evaluation method") + if (receipt.source_hash !== await sourceHash()) fail("conversion receipt has a stale source hash") + if (receipt.status !== "passed") fail("conversion receipt is not passing") + if (receipt.model?.product !== "Codex CLI" || receipt.model?.name !== "gpt-5.6-sol" || receipt.model?.reasoning !== "medium") fail("conversion receipt used the wrong model") + if (receipt.sessions !== 2) fail("conversion evaluation must use exactly two fresh sessions") + for (const key of ["input_tokens", "cached_input_tokens", "output_tokens", "reasoning_output_tokens"]) { + if (!Number.isInteger(receipt.token_usage?.[key]) || receipt.token_usage[key] < 0) fail(`invalid token usage: ${key}`) + } + const counts = receipt.clause_counts ?? {} + if (counts.total !== 4 || counts.control !== 1 || counts.guidance !== 1 || counts.both !== 1 || counts.excluded !== 1) fail("conversion receipt does not cover each disposition once") + if (receipt.candidate_verdict !== "accept") fail("generated candidate was not accepted") + if (receipt.negative_control_verdict !== "reject") fail("negative control was not rejected") + const defects = receipt.defect_detection ?? {} + for (const key of ["missing", "contradictory", "incorrectly_duplicated", "excluded_without_reason", "unreachable"]) { + if (defects[key] !== true) fail(`judge did not detect ${key}`) + } + if (receipt.claim_boundary !== "Advisory evidence for this four-clause fixture. The contract is stable; model projections can differ.") fail("conversion receipt has the wrong advisory claim boundary") + return receipt +} diff --git a/evals/conversion/scripts/run.mjs b/evals/conversion/scripts/run.mjs new file mode 100644 index 0000000..0e9ff27 --- /dev/null +++ b/evals/conversion/scripts/run.mjs @@ -0,0 +1,176 @@ +import { createHash } from "node:crypto" +import { execFileSync, spawnSync } from "node:child_process" +import { chmod, cp, mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises" +import { homedir, tmpdir } from "node:os" +import { dirname, join, relative } from "node:path" +import { conversionRoot, evalRoot, sourceHash, yieldRoot } from "./surface.mjs" + +const model = "gpt-5.6-sol" +const reasoning = "medium" +const runsRoot = join(evalRoot, "runs/conversion") +const resultPath = join(evalRoot, "results/latest-conversion.json") + +function command(program, args, cwd = yieldRoot) { + return execFileSync(program, args, { cwd, encoding: "utf8", env: process.env }).trim() +} + +async function prepareAuthHome(parent) { + const target = join(parent, "codex-home") + await mkdir(target) + if (!process.env.CODEX_API_KEY) { + const source = join(process.env.CODEX_HOME ?? join(homedir(), ".codex"), "auth.json") + await cp(source, join(target, "auth.json")) + await chmod(join(target, "auth.json"), 0o600) + } + return target +} + +function parseUsage(stdout) { + let usage = { input_tokens: 0, cached_input_tokens: 0, output_tokens: 0, reasoning_output_tokens: 0 } + for (const line of stdout.split("\n")) { + if (!line.startsWith("{")) continue + try { + const event = JSON.parse(line) + if (event.type === "turn.completed" && event.usage) usage = event.usage + } catch {} + } + return usage +} + +async function runCodex({ repo, authHome, prompt, evidenceDir, outputSchema, outputFile }) { + const args = [ + "exec", "--ephemeral", "--ignore-user-config", "--ignore-rules", + "--disable", "plugins", "--disable", "remote_plugin", "--disable", "apps", + "--disable", "memories", "--disable", "goals", "--disable", "multi_agent", + "--disable", "browser_use", "--disable", "computer_use", "--disable", "image_generation", + "--disable", "skill_search", "--disable", "workspace_dependencies", + "--json", "--sandbox", "danger-full-access", "-C", repo, + "--model", model, "-c", `model_reasoning_effort=\"${reasoning}\"`, + ] + if (outputSchema) args.push("--output-schema", outputSchema) + args.push("--output-last-message", outputFile, prompt) + const execution = spawnSync("codex", args, { + cwd: repo, + encoding: "utf8", + timeout: 12 * 60 * 1000, + maxBuffer: 64 * 1024 * 1024, + env: { ...process.env, CODEX_HOME: authHome }, + }) + await mkdir(evidenceDir, { recursive: true }) + await writeFile(join(evidenceDir, "transcript.jsonl"), execution.stdout ?? "") + await writeFile(join(evidenceDir, "stderr.log"), execution.stderr ?? "") + if (execution.status !== 0) throw new Error(`Codex exited ${execution.status}; see ${evidenceDir}`) + return parseUsage(execution.stdout ?? "") +} + +async function prepareRepository(session, yskill) { + const unresolved = join(session, "repository") + await mkdir(unresolved) + const repo = await realpath(unresolved) + await mkdir(join(repo, "skills/source-release"), { recursive: true }) + await cp(join(conversionRoot, "fixtures/source-skill/SKILL.md"), join(repo, "skills/source-release/SKILL.md")) + await writeFile(join(repo, "go.mod"), "module example.com/conversion-eval\n\ngo 1.26.5\n") + command("git", ["init", "-b", "main"], repo) + command("git", ["config", "user.email", "eval@operatorstack.systems"], repo) + command("git", ["config", "user.name", "Yield Eval"], repo) + command(yskill, ["bootstrap", "--root", repo, "--language", "go", "--agent", "codex", "--yes"], repo) + command("git", ["add", "."], repo) + command("git", ["commit", "-m", "conversion evaluation fixture"], repo) + return repo +} + +async function readEvents(path) { + return (await readFile(path, "utf8")).split("\n").filter(Boolean).map((line) => JSON.parse(line)) +} + +async function builderEvidence(repo) { + const directory = join(repo, "skills/yield-workflow-builder/.yield/runs") + const logs = (await readdir(directory)).filter((name) => name.endsWith(".jsonl")) + let selected + for (const name of logs.sort()) { + const path = join(directory, name) + const events = await readEvents(path) + const semantic = events.some((event) => event.type === "operation.completed" && event.data?.request_id === "project-semantics") + const complete = events.some((event) => event.type === "run.completed") + if (semantic && (complete || !selected)) selected = { path, events } + } + if (!selected) throw new Error("builder produced no semantic-conversion run log") + const { path, events } = selected + const terminal = events.findLast((event) => event.type === "run.completed" || event.type === "run.blocked" || event.type === "run.refused") + if (terminal?.type !== "run.completed") throw new Error(`builder terminal was ${terminal?.type ?? "missing"}`) + const projected = events.find((event) => event.type === "operation.completed" && event.data?.request_id === "project-semantics") + if (!projected?.data?.result) throw new Error("builder run log has no semantic projection") + return { path, projection: projected.data.result } +} + +function addUsage(left, right) { + return Object.fromEntries(["input_tokens", "cached_input_tokens", "output_tokens", "reasoning_output_tokens"].map((key) => [key, (left[key] ?? 0) + (right[key] ?? 0)])) +} + +async function main() { + const session = await mkdtemp(join(tmpdir(), "yield-conversion-eval-")) + const stamp = new Date().toISOString().replaceAll(":", "-") + const evidenceRoot = join(runsRoot, stamp) + const yskill = join(session, "yskill") + command("go", ["build", "-ldflags", "-X main.version=0.1.38", "-o", yskill, "./cmd/yskill"]) + const authHome = await prepareAuthHome(session) + try { + const repo = await prepareRepository(session, yskill) + const candidateOutput = join(repo, ".yield/candidate-session.txt") + await mkdir(join(repo, ".yield"), { recursive: true }) + const candidateUsage = await runCodex({ + repo, + authHome, + evidenceDir: join(evidenceRoot, "candidate"), + outputFile: candidateOutput, + prompt: "Read .agents/skills/yield-workflow-builder/SKILL.md and use the real builder. Convert skills/source-release into a new Go skill workflow at skills/converted-release. Drive every Yield operation to a terminal result. Do not imitate the workflow or bypass yskill. Use the current request as the specification.", + }) + const evidence = await builderEvidence(repo) + await cp(evidence.path, join(evidenceRoot, "candidate/builder-run.jsonl")) + await mkdir(join(repo, ".eval/negative-control"), { recursive: true }) + await cp(join(conversionRoot, "fixtures/negative-control"), join(repo, ".eval/negative-control"), { recursive: true }) + await cp(join(conversionRoot, "fixtures/fault-probes.json"), join(repo, ".eval/fault-probes.json")) + await writeFile(join(repo, ".eval/projection.json"), JSON.stringify(evidence.projection, null, 2) + "\n") + const judgeOutput = join(repo, ".eval/judge.json") + const judgeUsage = await runCodex({ + repo, + authHome, + evidenceDir: join(evidenceRoot, "judge"), + outputSchema: join(conversionRoot, "judge-schema.json"), + outputFile: judgeOutput, + prompt: "Act as an independent semantic-disposition judge. Read skills/source-release/SKILL.md, .eval/projection.json, every file in skills/converted-release, every file in .eval/negative-control, and .eval/fault-probes.json. Accept the generated candidate only if every source clause has exactly one valid disposition, each required destination exists and remains reachable by a coding agent, control is enforced in code, useful guidance remains model-facing, both reaches both, and exclusions have no destination plus a reason. Reject the static negative control because it deliberately drops guidance. Mark each defect probe true only when you recognize that it must be rejected. Return only schema-valid JSON.", + }) + const judge = JSON.parse(await readFile(judgeOutput, "utf8")) + const clauses = evidence.projection.clauses ?? [] + const counts = Object.fromEntries(["control", "guidance", "both", "excluded"].map((kind) => [kind, clauses.filter((clause) => clause.disposition === kind).length])) + const findings = judge.clause_findings ?? [] + const passed = judge.candidate?.verdict === "accept" && judge.negative_control?.verdict === "reject" && + clauses.length === 4 && Object.values(counts).every((count) => count === 1) && + findings.length === 4 && findings.every((finding) => finding.preserved === true && finding.reachable === true) && + Object.values(judge.defect_detection ?? {}).every((value) => value === true) + const source = await readFile(join(conversionRoot, "fixtures/source-skill/SKILL.md")) + const receipt = { + schema_version: 1, + methodology_version: "semantic-disposition-v1", + generated_at: new Date().toISOString(), + source_hash: await sourceHash(), + fixture_source_hash: createHash("sha256").update(source).digest("hex"), + status: passed ? "passed" : "failed", + model: { product: "Codex CLI", cli_version: command("codex", ["--version"]), name: model, reasoning }, + sessions: 2, + token_usage: addUsage(candidateUsage, judgeUsage), + clause_counts: { total: clauses.length, ...counts }, + candidate_verdict: judge.candidate?.verdict, + negative_control_verdict: judge.negative_control?.verdict, + defect_detection: judge.defect_detection, + claim_boundary: "Advisory evidence for this four-clause fixture. The contract is stable; model projections can differ.", + } + if (process.argv.includes("--write")) await writeFile(resultPath, JSON.stringify(receipt, null, 2) + "\n") + console.log(JSON.stringify({ status: receipt.status, clauses: receipt.clause_counts, evidence: relative(evalRoot, evidenceRoot) }, null, 2)) + if (!passed) process.exitCode = 1 + } finally { + await rm(session, { recursive: true, force: true }) + } +} + +await main() diff --git a/evals/conversion/scripts/surface.mjs b/evals/conversion/scripts/surface.mjs new file mode 100644 index 0000000..610b779 --- /dev/null +++ b/evals/conversion/scripts/surface.mjs @@ -0,0 +1,58 @@ +import { createHash } from "node:crypto" +import { execFileSync } from "node:child_process" +import { readFile, readdir, stat } from "node:fs/promises" +import { dirname, join, relative, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +export const conversionRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..") +export const evalRoot = resolve(conversionRoot, "..") +export const yieldRoot = resolve(evalRoot, "..") + +export const semanticSurface = Object.freeze([ + "cmd/yskill/bootstrap_templates.go", + "examples/convert-skill/", + "evals/conversion/fixtures/", + "evals/conversion/judge-schema.json", + "evals/conversion/scripts/", +]) +export const receiptSurface = "evals/results/latest-conversion.json" + +export function isSemanticPath(path) { + const normalized = path.replaceAll("\\", "/") + return normalized === receiptSurface || semanticSurface.some((entry) => entry.endsWith("/") ? normalized.startsWith(entry) : normalized === entry) +} + +async function filesUnder(path) { + if ((await stat(path)).isFile()) return [path] + const files = [] + for (const entry of await readdir(path, { withFileTypes: true })) { + const child = join(path, entry.name) + if (entry.isDirectory()) files.push(...await filesUnder(child)) + else files.push(child) + } + return files +} + +export async function semanticFiles() { + const files = [] + for (const entry of semanticSurface) files.push(...await filesUnder(join(yieldRoot, entry))) + return [...new Set(files)].sort() +} + +export async function sourceHash() { + const hash = createHash("sha256") + for (const path of await semanticFiles()) { + hash.update(relative(yieldRoot, path).replaceAll("\\", "/")) + hash.update("\0") + hash.update(await readFile(path)) + hash.update("\0") + } + return hash.digest("hex") +} + +export function changedPaths(base, head) { + if (!base || !head) return null + const effectiveBase = /^0+$/.test(base) ? `${head}^` : base + return execFileSync("git", ["diff", "--name-only", effectiveBase, head], { cwd: yieldRoot, encoding: "utf8" }) + .split("\n").map((path) => path.trim()).filter(Boolean) +} diff --git a/evals/conversion/scripts/validate.mjs b/evals/conversion/scripts/validate.mjs new file mode 100644 index 0000000..2d3f856 --- /dev/null +++ b/evals/conversion/scripts/validate.mjs @@ -0,0 +1,19 @@ +import { changedPaths, isSemanticPath } from "./surface.mjs" +import { validateReceipt } from "./receipt.mjs" + +function valueAfter(flag) { + const index = process.argv.indexOf(flag) + return index === -1 ? undefined : process.argv[index + 1] +} + +const base = valueAfter("--base") ?? process.env.EVAL_BASE_SHA +const head = valueAfter("--head") ?? process.env.EVAL_HEAD_SHA +const paths = process.argv.includes("--force") ? null : changedPaths(base, head) +const relevant = paths === null || paths.some(isSemanticPath) + +if (!relevant) { + console.log("conversion receipt skipped: no semantic-conversion source changed") +} else { + const receipt = await validateReceipt() + console.log(`validated conversion receipt ${receipt.source_hash.slice(0, 12)}`) +} diff --git a/evals/package.json b/evals/package.json index 4f74746..e1245ea 100644 --- a/evals/package.json +++ b/evals/package.json @@ -7,8 +7,10 @@ "eval": "node scripts/run.mjs --write", "eval:agent": "node agent/scripts/run.mjs --write", "eval:agent:smoke": "node agent/scripts/run.mjs --case success --repeat 1", + "eval:conversion": "node conversion/scripts/run.mjs --write", "test": "node scripts/validate.mjs && node scripts/run.mjs --check", "test:agent": "node agent/scripts/validate.mjs", + "test:conversion": "node --test conversion/scripts/conversion.test.mjs && node conversion/scripts/validate.mjs", "validate": "node scripts/validate.mjs" } } diff --git a/evals/results/latest-conversion.json b/evals/results/latest-conversion.json new file mode 100644 index 0000000..8848b31 --- /dev/null +++ b/evals/results/latest-conversion.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "methodology_version": "semantic-disposition-v1", + "generated_at": "2026-08-08T10:48:36.169Z", + "source_hash": "0f911c060c888838408aa0e5c4a229cc6b6ee32631ac5ed70bdc6a2ecc097405", + "fixture_source_hash": "9ab03fffe6716da8298b461f79c9eebae7c7bb01151328686ec51ed9c0b77fe5", + "status": "passed", + "model": { + "product": "Codex CLI", + "cli_version": "codex-cli 0.145.0", + "name": "gpt-5.6-sol", + "reasoning": "medium" + }, + "sessions": 2, + "token_usage": { + "input_tokens": 613869, + "cached_input_tokens": 555676, + "output_tokens": 8875, + "reasoning_output_tokens": 2776 + }, + "clause_counts": { + "total": 4, + "control": 1, + "guidance": 1, + "both": 1, + "excluded": 1 + }, + "candidate_verdict": "accept", + "negative_control_verdict": "reject", + "defect_detection": { + "missing": true, + "contradictory": true, + "incorrectly_duplicated": true, + "excluded_without_reason": true, + "unreachable": true + }, + "claim_boundary": "Advisory evidence for this four-clause fixture. The contract is stable; model projections can differ." +} diff --git a/evals/results/latest.json b/evals/results/latest.json index b3ba639..0192246 100644 --- a/evals/results/latest.json +++ b/evals/results/latest.json @@ -1,8 +1,8 @@ { "schema_version": 2, "methodology_version": "1.1", - "generated_at": "2026-08-08T09:53:12.706Z", - "source_digest": "ec4d252c049a05e5031794ad7966587196e4749a209e4139916472401ec311c8", + "generated_at": "2026-08-08T10:49:07.813Z", + "source_digest": "e82087e147677abbe53882e06888ecbdd36ade96c25412d9b2e815ff8428be61", "status": "passed", "workflow_conformance": { "passed": 40, diff --git a/examples/convert-skill/SKILL.md b/examples/convert-skill/SKILL.md index 06a40c0..0e135c6 100644 --- a/examples/convert-skill/SKILL.md +++ b/examples/convert-skill/SKILL.md @@ -20,7 +20,17 @@ Resume the run after each operation: yskill resume --response response.json --skill . Do not skip an operation or invent its response. The program owns the -pipeline: read the prose, extract the flow, pick the language, write the -program — and completion requires the generated skill to pass its own -fixture run under `yskill test`. A conversion that was never executed is -never "done". +pipeline: read the prose, pick the language, project its meaning, extract the +flow, and write the program. The projection is: + + C = clauses(S) + Pi(S) = {(c, d, T, r) | c in C} + +Each clause is control, guidance, both, or excluded. Control reaches code. +Guidance stays in the canonical SKILL.md or a relevant `agent_task`. Both +reaches both. An excluded clause has no destination and has a reason. Every +destination remains reachable by the coding agent. The map stays in the Yield +run log. It is not a destination artifact. + +Completion requires the generated skill to pass its own fixture run under +`yskill test`. A conversion that was never executed is never "done". diff --git a/examples/convert-skill/fixtures/responses.json b/examples/convert-skill/fixtures/responses.json index 2fe70eb..e65e1f6 100644 --- a/examples/convert-skill/fixtures/responses.json +++ b/examples/convert-skill/fixtures/responses.json @@ -1,5 +1,20 @@ { "source-path": { "value": "../investigate" }, + "project-semantics": { + "clauses": [ + { + "source_clause": "Investigate failures with bounded evidence gathering and model judgment.", + "disposition": "both", + "destinations": [ + { "kind": "code", "target": "main.py investigation sequence" }, + { "kind": "skill", "target": "SKILL.md investigation guidance" } + ], + "reason": "" + } + ], + "ready": true, + "unresolved": [] + }, "extract-flow": { "summary": "Bounded incident investigation: collect evidence, form at least three hypotheses, probe cheapest-first, conclude with a causal chain or block honestly.", "steps": [ diff --git a/examples/convert-skill/main.go b/examples/convert-skill/main.go index f0e4cd8..97cda29 100644 --- a/examples/convert-skill/main.go +++ b/examples/convert-skill/main.go @@ -5,8 +5,8 @@ // conversion. // // Division of labor: the program owns the pipeline order, the language -// menu, the retry bound, and the evidence gate. The model owns reading -// the prose, extracting the flow, and writing the code. +// menu, the retry bound, and the evidence gate. The model owns projecting +// each source clause, extracting the flow, and writing the code and guidance. package main import ( @@ -38,6 +38,42 @@ const flowSchema = `{ } }` +const projectionSchema = `{ + "type": "object", + "required": ["clauses", "ready", "unresolved"], + "additionalProperties": false, + "properties": { + "clauses": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["source_clause", "disposition", "destinations", "reason"], + "additionalProperties": false, + "properties": { + "source_clause": {"type": "string", "minLength": 1}, + "disposition": {"enum": ["control", "guidance", "both", "excluded"]}, + "destinations": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "target"], + "additionalProperties": false, + "properties": { + "kind": {"enum": ["code", "skill", "agent_task"]}, + "target": {"type": "string", "minLength": 1} + } + } + }, + "reason": {"type": "string"} + } + } + }, + "ready": {"type": "boolean"}, + "unresolved": {"type": "array", "items": {"type": "string", "minLength": 1}} + } +}` + const filesSchema = `{ "type": "object", "required": ["files"], @@ -58,29 +94,51 @@ func main() { prose := ctx.RunCommand("read-prose", fmt.Sprintf("cat %s/SKILL.md", shellQuote(source)), 60) ctx.Require(prose.ExitCode == 0, "the source SKILL.md is readable", map[string]int{"exit_code": prose.ExitCode}) - flowRaw := ctx.AgentTask("extract-flow", - "Read the prose skill below and extract its implicit control flow as ordered steps: "+ - "questions to the user (ask_user), model judgment (agent_task), commands (run_command), "+ - "branches, and verification points (require). Preserve the skill's intent; do not invent steps.", - map[string]string{"skill_md": prose.Stdout}, - json.RawMessage(flowSchema)) - lang := ctx.AskUser("pick-language", "Target language for the generated program?", yield.Option{Value: "go", Label: "Go"}, yield.Option{Value: "typescript", Label: "TypeScript"}, yield.Option{Value: "python", Label: "Python"}, yield.Option{Value: "rust", Label: "Rust"}) + projectionRaw := ctx.AgentTask("project-semantics", + "Map every source clause exactly once using Pi(S)={(c,d,T,r)|c in clauses(S)}. "+ + "YAML frontmatter is metadata, not a clause. Each top-level bullet is exactly one clause, including a compound sentence. For prose without bullets, treat each paragraph as one clause. "+ + "Use disposition control, guidance, both, or excluded. Control needs a reachable code destination. "+ + "Guidance needs a reachable skill or agent_task destination. Both needs both kinds. "+ + "Excluded needs no destination and a non-empty reason. Name concrete destinations that the writer can create. "+ + "Use the target language's canonical entrypoint for code: main.ts, main.py, main.go, or src/main.rs. "+ + "Report uncertainty in unresolved and set ready false. Do not write files.", + map[string]string{"skill_md": prose.Stdout, "language": lang}, + json.RawMessage(projectionSchema)) + var projection struct { + Ready bool `json:"ready"` + Unresolved []string `json:"unresolved"` + } + if err := json.Unmarshal(projectionRaw, &projection); err != nil { + return yield.Outcome{}, err + } + if !projection.Ready || len(projection.Unresolved) > 0 { + return yield.Outcome{}, ctx.Blocked("the semantic projection has unresolved source clauses") + } + + flowRaw := ctx.AgentTask("extract-flow", + "Read the prose skill and its semantic projection below. Extract its implicit control flow as ordered steps: "+ + "questions to the user (ask_user), model judgment (agent_task), commands (run_command), "+ + "branches, and verification points (require). Preserve the skill's intent; do not invent steps.", + map[string]any{"skill_md": prose.Stdout, "projection": json.RawMessage(projectionRaw)}, + json.RawMessage(flowSchema)) + dest := ctx.AskUser("dest-path", "Directory to write the converted skill into?") written := ctx.AgentTask("write-skill", "Write the converted Yield skill into the destination directory: the program "+ "(main.go / main.ts / main.py / src/main.rs per the chosen language, using that language's SDK "+ - "from this repository), a THIN SKILL.md (keep the original prose voice, delegate sequencing to "+ - "`yskill run .`), a skill.json runner manifest (omit for Go), and fixtures/responses.json with a "+ - "happy-path scripted response for every ask_user and agent_task operation. "+ + "from this repository), a thin SKILL.md (keep useful judgment, examples, tone, and tool advice; delegate sequencing to "+ + "`yskill run .`), a skill.json runner manifest, and fixtures/responses.json with a "+ + "happy-path scripted response for every ask_user and agent_task operation. Follow every semantic disposition. "+ + "Thin does not mean deleting useful guidance. "+ "Return {\"files\": [paths you actually wrote]}.", - map[string]any{"flow": json.RawMessage(flowRaw), "language": lang, "destination": dest}, + map[string]any{"source": prose.Stdout, "projection": json.RawMessage(projectionRaw), "flow": json.RawMessage(flowRaw), "language": lang, "destination": dest}, json.RawMessage(filesSchema)) // The evidence gate with a bounded repair loop: at most two repair @@ -95,6 +153,8 @@ func main() { "{\"files\": [paths you changed]}.", map[string]any{ "destination": dest, + "source": prose.Stdout, + "projection": json.RawMessage(projectionRaw), "stdout": test.Stdout, "stderr": test.Stderr, },