diff --git a/docs/BUILD_LOG.md b/docs/BUILD_LOG.md index 859c5fc..4b07108 100644 --- a/docs/BUILD_LOG.md +++ b/docs/BUILD_LOG.md @@ -2,6 +2,15 @@ This is a concise chronological record of material implementation work and runtime evidence. It is not a substitute for Git history; it captures intent, verification, and blockers that a commit alone may not explain. +## 2026-07-31 — R4 Experience Compiler and Phase-A Judge accepted + +- Added deterministic seam selection over fully supported R2 evidence and compiled the real R1/R2 fixture path into separate internal and participant recovery manifests. The participant projection is exact-schema and omits source revisions, run identity, setup, judge internals, hidden repair, production paths, and controller handles. +- Added a Phase-A judge that reopens the project-scoped snapshot and immutable command registry, verifies the participant Git boundary, computes the candidate diff, rejects protected or out-of-scope edits, and evaluates only an exact catalog-owned known repair in a separate clean evaluation twin. No-op and unrelated candidates never reach command execution. +- Added project-scoped command-evidence persistence with canonical metadata, bounded base64 payloads, hash/size verification, and tamper rejection. Reveal and abandonment remain explicit outcomes without executable readiness evidence. +- Seven R4 tests pass locally on Windows, including three byte-identical judge replays and evidence-store reopening/tamper cases. The complete extension suite passes 57/57; `npm run check`, production build, and VSIX packaging pass. Protected PR #14 run `30670322126` passed the required Linux, Windows, contract, web, and policy checks, so R4 acceptance is complete. + +Evidence: `extension/src/experience/`, `extension/src/judge/`, `extension/src/twin/{commands,snapshot}.ts`, `extension/test/experience-judge.test.ts`, local command/package output on 2026-07-31, and protected GitHub Actions run `30670322126` on PR #14. + ## 2026-07-31 — R3 Takeover Twin local candidate - Implemented a controller-owned `SnapshotStore` that starts from the verified fixture target, applies the catalog mutation, creates a sanitized standalone one-commit participant repository, and re-verifies tree hashes, commit identity, ownership, remotes, reflogs, and Git alternates before materialization. diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index 78c19f5..a82922c 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -2,7 +2,7 @@ Last updated: 2026-07-31 -## Current branch milestone — R3 Takeover Twin complete +## Current branch milestone — R4 Experience Compiler and Judge accepted Branch `codex/shadow-cockpit-rnd` resets the product R&D thesis around **Dual-Control Development**. @@ -21,7 +21,10 @@ Branch `codex/shadow-cockpit-rnd` resets the product R&D thesis around **Dual-Co - Six R2 tests cover the real cache-key fixture, cross-run determinism and golden IDs, file rename, multi-file changes, added/deleted functions, class boundaries, Git hunk parsing, unsupported syntax/language, missing checks, unsafe paths, and revision/run drift. The full local Windows extension suite passes 43/43 with TypeScript, build, and VSIX packaging. Protected PR #12 run `30666648522` passed `extension`, `extension-windows`, `contract`, `web`, and `jules-rnd-policy`, so R2 acceptance is complete. - R3 now implements the fixture-only sanitized snapshot store, standalone one-commit participant repository, opaque Twin Manager, immutable command-registry snapshots, extension-owned trusted catalog, and `TrustedFixtureRunner`. The runner accepts only the pinned manifest, exact declared state/tree, catalog command, hash-verified standalone Node runtime, and opaque twin handle; non-fixture execution remains explicitly unsupported. - Seven R3 tests cover production file/index/HEAD/ref/remote/worktree invariants, hidden-answer and source-history absence, cross-project ownership, exact-state execution, tamper/unknown-command rejection, concurrent execution IDs, single-run cancellation, immutable command snapshots, Windows paths with spaces, descendant-process termination, and exact cleanup. The full local Windows extension suite passes 50/50; build and VSIX packaging pass, and the runtime fixture assets are present in the VSIX. Protected PR #13 run `30668675359` passed `extension`, `extension-windows`, `contract`, `web`, and `jules-rnd-policy`, so R3 acceptance is complete. -- The Experience Compiler, Evidence Judge, Control Pulse runtime, readiness ledger, and v0.3 cockpit do not exist yet. R3 proves only the closed reviewed-fixture boundary; it does not execute arbitrary participant or workspace code. +- R4 now compiles one fully attributed, bounded, test-backed seam from real R1 replay plus R2 change evidence into separate internal and participant recovery manifests. The participant projection excludes source revisions, run identity, judge internals, setup, hidden repair, production paths, and controller handles. +- The Phase-A Evidence Judge reopens the immutable snapshot and command registry, verifies the standalone Git boundary, computes the participant diff, rejects protected or out-of-scope changes, and evaluates only the exact catalog-owned known repair in a clean evaluation twin. No-op and unrelated candidates fail without execution; reveal and abandonment produce no executable readiness evidence. +- Seven R4 tests cover deterministic compilation, strict projection/schema rejection, unsupported and fixture-drift rejection, protected-path integrity, no-op/unrelated non-execution, three identical clean replays, reveal/abandon honesty, and project-scoped persistent command evidence with tamper detection. The full local Windows extension suite passes 57/57; `npm run check`, production build, and VSIX packaging pass. Protected PR #14 run `30670322126` passed `extension`, `extension-windows`, `contract`, `web`, and `jules-rnd-policy`, so R4 acceptance is complete. +- The Control Pulse runtime, readiness ledger, and v0.3 cockpit do not exist yet. R4 remains a closed reviewed-fixture mechanism and does not execute arbitrary participant or workspace code. - No skill-retention or speed metric has been measured. Values in the PRD are predeclared R&D targets. - A new implementation audit found five R0 ambiguities: candidate-diff identity, pre-store fixture blobs, runtime identity, check IDs, and Git object format. The normative contract closes them with structured diffs, catalog-owned blobs, standalone Node `v22.17.0`, declared test IDs, and SHA-1 Git initialization; R0a/R0b now implement and verify that complete substrate. - A guarded Jules dispatcher and PR policy are defined as a finite R0→R4 queue. They create at most one session after a successful preflight, stop after merged R4, remain inert unless dispatch is explicitly enabled, and keep plan approval on by default. Merges remain manual because the current project tests are not an independent immutable verifier. Full scheduled continuation still requires the dispatcher workflow to be reviewed into the default branch. @@ -139,11 +142,10 @@ No external input blocks the repository-owned fixture R0–R4.5 mechanism in `do ## Next ordered actions -1. Integrate R4: one compiled recovery episode and deterministic Evidence Judge. -2. Pass R4.5: one bounded, catalog-only Explain-to-Break Pulse with replay/error fail-closed tests. -3. Run the 30-patch recovery-plus-probe technical corpus audit before expanding the product surface. -4. Add the local readiness ledger and minimal cockpit only after the vertical slice is reliable. -5. Run the preregistered delayed-transfer pilot before making any skill-retention claim. +1. Pass R4.5: one bounded, catalog-only Explain-to-Break Pulse with replay/error fail-closed tests. +2. Run the 30-patch recovery-plus-probe technical corpus audit before expanding the product surface. +3. Add the local readiness ledger and minimal cockpit only after the vertical slice is reliable. +4. Run the preregistered delayed-transfer pilot before making any skill-retention claim. ## Recent milestone commits diff --git a/docs/v0.3/CONTRACTS.md b/docs/v0.3/CONTRACTS.md index 01f2b54..96df421 100644 --- a/docs/v0.3/CONTRACTS.md +++ b/docs/v0.3/CONTRACTS.md @@ -840,6 +840,8 @@ interface JudgeResult { } ``` +`JudgeResult.resultHash` is `canonicalHash("judge-result", result without resultHash)`. + The judge establishes only the declared behavior under the declared oracle. It is not proof of total correctness or human mastery. ## 11. Readiness evidence versus verified readiness diff --git a/extension/src/experience/compile.ts b/extension/src/experience/compile.ts new file mode 100644 index 0000000..cb03d29 --- /dev/null +++ b/extension/src/experience/compile.ts @@ -0,0 +1,157 @@ +import { randomUUID } from "node:crypto"; +import { assertBoundedText, assertToken } from "../agent/types"; +import type { ExtractionResult } from "../change/evidence"; +import { canonicalJson, compareUtf8 } from "../rnd/canonical"; +import { assertEvidenceRef, type EvidenceRef } from "../recorder/events"; +import { BuiltinFixtureCatalog } from "../twin/catalog"; +import { FixtureCommandRegistry } from "../twin/commands"; +import { FixtureSnapshotStore } from "../twin/snapshot"; +import type { FixtureBlobRef } from "../twin/fixture-contract"; +import { + assertInternalExperience, + assertParticipantExperience, + type CompiledExperience, + type InternalExperience, + type ParticipantExperience, +} from "./types"; +import { selectRecoverySeam } from "./select"; + +export interface CompileExperienceInput { + extraction: ExtractionResult; + fixtureId: string; + manifestHash: string; +} + +export class FixtureExperienceCompiler { + constructor( + private readonly catalog: BuiltinFixtureCatalog, + private readonly snapshots: FixtureSnapshotStore, + private readonly registry: FixtureCommandRegistry, + private readonly options: { id?: () => string } = {}, + ) {} + + async compile(input: CompileExperienceInput): Promise { + const record = await this.catalog.open(input.fixtureId, input.manifestHash); + if (!record) throw new Error("Experience fixture is not in the trusted catalog"); + const selected = selectRecoverySeam(input.extraction); + const seam = selected.seam; + if ( + seam.projectId !== input.extraction.projectId || + seam.sourceRunId !== input.extraction.sourceRunId || + seam.baseRevision !== input.extraction.baseRevision || + seam.targetRevision !== input.extraction.targetRevision || + seam.baseRevision !== record.manifest.baseRevision || + seam.targetRevision !== record.manifest.targetRevision + ) { + throw new Error("Selected seam identity does not match the trusted fixture"); + } + if (!record.manifest.changedSymbols.some(({ path, symbol }) => path === seam.unit.path && symbol === seam.unit.symbol)) { + throw new Error("Selected seam is not declared by the trusted fixture"); + } + if (seam.linkedChecks.some((id) => !record.manifest.targetChecks.includes(id))) { + throw new Error("Selected seam references a check outside the trusted fixture target"); + } + + const mutation = evidenceFromFixture(record.manifest.mutation.changeRef, "controller"); + const target = record.manifest.states.find((state) => state.id === "target")!; + const snapshot = await this.snapshots.create({ + projectId: seam.projectId, + sourceRevision: seam.targetRevision, + mutationId: record.manifest.mutation.id, + mutation, + allowedFiles: target.files.map((file) => file.path), + }); + const commandIds = seam.linkedChecks.map((checkId) => { + const check = record.manifest.checks.find((item) => item.id === checkId); + if (!check) throw new Error(`Trusted fixture check is missing: ${checkId}`); + return check.commandId; + }).sort(compareUtf8); + const registry = await this.registry.freeze(seam.projectId, commandIds); + const editable = [...record.manifest.mutation.editablePaths]; + const id = (this.options.id ?? (() => randomUUID().replaceAll("-", "")))(); + assertToken(id, "experienceId"); + const symbols = [`${seam.unit.path}#${seam.unit.symbol}`]; + const concepts = ["tested-boundary-recovery"]; + const task = `Recover the declared behavior at ${seam.unit.symbol}. Make ${seam.linkedChecks.join(", ")} pass without changing protected files.`; + assertBoundedText(task, 4096, "experience task"); + const hiddenAnswer = evidenceFromFixture(record.manifest.knownRepair.changeRef, "controller"); + const internal: InternalExperience = { + schemaVersion: 1, + id, + sourceRunId: seam.sourceRunId, + projectId: seam.projectId, + sourceBaseRevision: seam.baseRevision, + sourceTargetRevision: seam.targetRevision, + snapshotId: snapshot.id, + snapshotTreeHash: snapshot.treeHash, + commandRegistryHash: registry.sha256, + scope: { symbols, tests: [...seam.linkedChecks], concepts }, + kind: "recover", + task, + setup: [ + { kind: "materialize-snapshot", snapshotId: snapshot.id }, + ...commandIds.map((commandId) => ({ kind: "run-approved-command" as const, commandId })), + ], + judge: { + timeoutMs: 10_000, + commandRegistryHash: registry.sha256, + editablePaths: editable, + protectedFiles: snapshot.files + .filter((file) => !editable.includes(file.path)) + .map(({ path, sha256 }) => ({ path, sha256 })), + checks: seam.linkedChecks.map((checkId) => ({ + id: checkId, + kind: "approved-command" as const, + commandId: record.manifest.checks.find((item) => item.id === checkId)!.commandId, + expectedExitCode: 0, + })), + }, + hiddenAnswer, + budgetMinutes: seam.factors.estimatedAttentionMinutes, + }; + assertInternalExperience(internal); + const participant = projectParticipant(internal); + return { + internal, + participant, + snapshot, + registry, + selectionReasons: [...selected.reasons, "fixture-supported"].sort(compareUtf8), + }; + } +} + +export function serializeParticipantExperience(value: ParticipantExperience): string { + assertParticipantExperience(value); + return canonicalJson(value); +} + +function projectParticipant(value: InternalExperience): ParticipantExperience { + return { + schemaVersion: 1, + id: value.id, + snapshotId: value.snapshotId, + snapshotTreeHash: value.snapshotTreeHash, + scope: { symbols: [...value.scope.symbols], concepts: [...value.scope.concepts] }, + kind: "recover", + task: value.task, + visibleChecks: [...value.scope.tests], + budgetMinutes: value.budgetMinutes, + }; +} + +function evidenceFromFixture(ref: FixtureBlobRef, visibility: EvidenceRef["visibility"]): EvidenceRef { + const value: EvidenceRef = { + id: ref.id, + kind: "diff", + sha256: ref.sha256, + storedBytes: ref.storedBytes, + originalBytes: ref.storedBytes, + truncated: false, + redactions: [], + mediaType: ref.mediaType, + visibility, + }; + assertEvidenceRef(value, "diff"); + return value; +} diff --git a/extension/src/experience/select.ts b/extension/src/experience/select.ts new file mode 100644 index 0000000..e51fbbb --- /dev/null +++ b/extension/src/experience/select.ts @@ -0,0 +1,33 @@ +import { compareUtf8 } from "../rnd/canonical"; +import type { CandidateSeam, ExtractionResult } from "../change/evidence"; + +export interface SeamSelection { + seam: CandidateSeam; + reasons: string[]; +} + +export function selectRecoverySeam(result: ExtractionResult): SeamSelection { + if (result.status !== "supported") { + throw new Error("Recovery compilation requires fully supported change evidence"); + } + const eligible = result.seams.filter((seam) => + seam.linkedChecks.length > 0 && + seam.unit.attribution.status === "attributed" && + seam.factors.evidenceGap === 0 && + seam.factors.estimatedAttentionMinutes > 0 && + seam.factors.estimatedAttentionMinutes <= 10, + ); + if (eligible.length === 0) throw new Error("No attributed, bounded, test-backed seam is eligible"); + + eligible.sort((left, right) => + left.factors.estimatedAttentionMinutes - right.factors.estimatedAttentionMinutes || + right.linkedChecks.length - left.linkedChecks.length || + compareUtf8(left.unit.path, right.unit.path) || + compareUtf8(left.unit.symbol, right.unit.symbol) || + compareUtf8(left.id, right.id), + ); + return { + seam: structuredClone(eligible[0]!), + reasons: ["attributed-change", "bounded-attention", "test-backed"], + }; +} diff --git a/extension/src/experience/types.ts b/extension/src/experience/types.ts new file mode 100644 index 0000000..9dcb455 --- /dev/null +++ b/extension/src/experience/types.ts @@ -0,0 +1,201 @@ +import type { EvidenceRef } from "../recorder/events"; +import { assertEvidenceRef } from "../recorder/events"; +import { assertBoundedText, assertGitOid, assertRecord, assertToken } from "../agent/types"; +import { assertExactKeys, assertRelPath, assertSha256, compareUtf8 } from "../rnd/canonical"; +import type { CommandRegistrySnapshot, SanitizedSnapshot } from "../twin/types"; + +export type ExperienceStep = + | { kind: "materialize-snapshot"; snapshotId: string } + | { kind: "apply-change"; changeRef: EvidenceRef } + | { kind: "run-approved-command"; commandId: string }; + +export interface JudgeSpec { + timeoutMs: number; + commandRegistryHash: string; + editablePaths: string[]; + protectedFiles: Array<{ path: string; sha256: string }>; + checks: Array< + | { id: string; kind: "approved-command"; commandId: string; expectedExitCode: number } + | { id: string; kind: "file-hash"; path: string; expectedHash: string } + | { id: string; kind: "trace"; traceId: string; expectedRef: EvidenceRef } + >; +} + +export interface InternalExperience { + schemaVersion: 1; + id: string; + sourceRunId: string; + projectId: string; + sourceBaseRevision: string; + sourceTargetRevision: string; + snapshotId: string; + snapshotTreeHash: string; + commandRegistryHash: string; + scope: { symbols: string[]; tests: string[]; concepts: string[] }; + kind: "recover"; + task: string; + setup: ExperienceStep[]; + judge: JudgeSpec; + hiddenAnswer: EvidenceRef; + budgetMinutes: number; +} + +export interface ParticipantExperience { + schemaVersion: 1; + id: string; + snapshotId: string; + snapshotTreeHash: string; + scope: { symbols: string[]; concepts: string[] }; + kind: "recover"; + task: string; + visibleChecks: string[]; + budgetMinutes: number; +} + +export interface CompiledExperience { + internal: InternalExperience; + participant: ParticipantExperience; + snapshot: SanitizedSnapshot; + registry: CommandRegistrySnapshot; + selectionReasons: string[]; +} + +export function assertInternalExperience(value: unknown): asserts value is InternalExperience { + assertRecord(value, "internal experience"); + assertExactKeys(value, [ + "schemaVersion", "id", "sourceRunId", "projectId", "sourceBaseRevision", "sourceTargetRevision", + "snapshotId", "snapshotTreeHash", "commandRegistryHash", "scope", "kind", "task", "setup", "judge", + "hiddenAnswer", "budgetMinutes", + ], "internal experience"); + if (value.schemaVersion !== 1 || value.kind !== "recover") throw new Error("Unsupported internal experience"); + assertToken(value.id, "experienceId"); + assertToken(value.sourceRunId, "sourceRunId"); + assertToken(value.projectId, "projectId"); + assertGitOid(value.sourceBaseRevision, "sourceBaseRevision"); + assertGitOid(value.sourceTargetRevision, "sourceTargetRevision"); + assertToken(value.snapshotId, "snapshotId"); + assertSha256(String(value.snapshotTreeHash), "snapshotTreeHash"); + assertSha256(String(value.commandRegistryHash), "commandRegistryHash"); + assertScope(value.scope, true); + assertBoundedText(value.task, 4096, "experience task"); + if (!Number.isSafeInteger(value.budgetMinutes) || (value.budgetMinutes as number) < 1 || (value.budgetMinutes as number) > 60) { + throw new Error("Experience budget is invalid"); + } + if (!Array.isArray(value.setup) || value.setup.length === 0 || value.setup.length > 16) { + throw new Error("Experience setup is invalid"); + } + for (const step of value.setup) assertStep(step); + assertJudgeSpec(value.judge); + assertEvidenceRef(value.hiddenAnswer, "diff"); + if (value.hiddenAnswer.visibility !== "controller") throw new Error("Hidden answer must remain controller-only"); +} + +export function assertParticipantExperience(value: unknown): asserts value is ParticipantExperience { + assertRecord(value, "participant experience"); + assertExactKeys( + value, + ["schemaVersion", "id", "snapshotId", "snapshotTreeHash", "scope", "kind", "task", "visibleChecks", "budgetMinutes"], + "participant experience", + ); + if (value.schemaVersion !== 1 || value.kind !== "recover") throw new Error("Unsupported participant experience"); + assertToken(value.id, "experienceId"); + assertToken(value.snapshotId, "snapshotId"); + assertSha256(String(value.snapshotTreeHash), "snapshotTreeHash"); + assertBoundedText(value.task, 4096, "experience task"); + assertScope(value.scope, false); + assertSortedStrings(value.visibleChecks, "visible checks", true); + if (!Number.isSafeInteger(value.budgetMinutes) || (value.budgetMinutes as number) < 1 || (value.budgetMinutes as number) > 60) { + throw new Error("Experience budget is invalid"); + } +} + +function assertScope(value: unknown, internal: boolean): void { + assertRecord(value, "experience scope"); + assertExactKeys(value, internal ? ["symbols", "tests", "concepts"] : ["symbols", "concepts"], "experience scope"); + assertSortedStrings(value.symbols, "scope symbols", false); + assertSortedStrings(value.concepts, "scope concepts", true); + if (internal) assertSortedStrings(value.tests, "scope tests", true); +} + +function assertSortedStrings(value: unknown, label: string, tokens: boolean): void { + if (!Array.isArray(value) || value.length === 0 || value.length > 32) throw new Error(`${label} must be a bounded array`); + let previous: string | undefined; + for (const item of value) { + if (tokens) assertToken(item, label); + else { + assertBoundedText(item, 512, label); + if (item.length === 0) throw new Error(`${label} cannot contain empty values`); + } + if (previous !== undefined && compareUtf8(previous, item) >= 0) throw new Error(`${label} must be sorted and unique`); + previous = item; + } +} + +function assertStep(value: unknown): void { + assertRecord(value, "experience step"); + if (value.kind === "materialize-snapshot") { + assertExactKeys(value, ["kind", "snapshotId"], "materialize step"); + assertToken(value.snapshotId, "snapshotId"); + } else if (value.kind === "apply-change") { + assertExactKeys(value, ["kind", "changeRef"], "apply-change step"); + assertEvidenceRef(value.changeRef, "diff"); + } else if (value.kind === "run-approved-command") { + assertExactKeys(value, ["kind", "commandId"], "command step"); + assertToken(value.commandId, "commandId"); + } else { + throw new Error("Unknown experience step"); + } +} + +function assertJudgeSpec(value: unknown): void { + assertRecord(value, "judge spec"); + assertExactKeys(value, ["timeoutMs", "commandRegistryHash", "editablePaths", "protectedFiles", "checks"], "judge spec"); + if (!Number.isSafeInteger(value.timeoutMs) || (value.timeoutMs as number) < 1000 || (value.timeoutMs as number) > 600_000) { + throw new Error("Judge timeout is invalid"); + } + assertSha256(String(value.commandRegistryHash), "judge commandRegistryHash"); + assertSortedPaths(value.editablePaths, "editable paths"); + if (!Array.isArray(value.protectedFiles) || value.protectedFiles.length > 256) throw new Error("Protected files are invalid"); + let previous: string | undefined; + for (const file of value.protectedFiles) { + assertRecord(file, "protected file"); + assertExactKeys(file, ["path", "sha256"], "protected file"); + assertRelPath(String(file.path)); + assertSha256(String(file.sha256), "protected file hash"); + if (previous !== undefined && compareUtf8(previous, String(file.path)) >= 0) throw new Error("Protected files must be sorted and unique"); + previous = String(file.path); + } + if (!Array.isArray(value.checks) || value.checks.length === 0 || value.checks.length > 32) throw new Error("Judge checks are invalid"); + const ids: string[] = []; + for (const check of value.checks) { + assertRecord(check, "judge check"); + assertToken(check.id, "judge check ID"); + ids.push(check.id); + if (check.kind === "approved-command") { + assertExactKeys(check, ["id", "kind", "commandId", "expectedExitCode"], "approved command check"); + assertToken(check.commandId, "commandId"); + if (!Number.isSafeInteger(check.expectedExitCode)) throw new Error("Expected exit code is invalid"); + } else if (check.kind === "file-hash") { + assertExactKeys(check, ["id", "kind", "path", "expectedHash"], "file hash check"); + assertRelPath(String(check.path)); + assertSha256(String(check.expectedHash), "expected file hash"); + } else if (check.kind === "trace") { + assertExactKeys(check, ["id", "kind", "traceId", "expectedRef"], "trace check"); + assertToken(check.traceId, "traceId"); + assertEvidenceRef(check.expectedRef, "trace"); + } else { + throw new Error("Unknown judge check"); + } + } + if (new Set(ids).size !== ids.length) throw new Error("Judge check IDs must be unique"); +} + +function assertSortedPaths(value: unknown, label: string): void { + if (!Array.isArray(value) || value.length === 0 || value.length > 256) throw new Error(`${label} are invalid`); + let previous: string | undefined; + for (const path of value) { + assertRelPath(String(path)); + if (previous !== undefined && compareUtf8(previous, String(path)) >= 0) throw new Error(`${label} must be sorted and unique`); + previous = String(path); + } +} diff --git a/extension/src/judge/run.ts b/extension/src/judge/run.ts new file mode 100644 index 0000000..b50ea6a --- /dev/null +++ b/extension/src/judge/run.ts @@ -0,0 +1,272 @@ +import { randomUUID } from "node:crypto"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { assertToken } from "../agent/types"; +import { assertInternalExperience, type InternalExperience } from "../experience/types"; +import { canonicalHash, canonicalJson, compareUtf8, rawSha256, treeHash, type TreeFile } from "../rnd/canonical"; +import { BuiltinFixtureCatalog } from "../twin/catalog"; +import { + FixtureCommandRegistry, + type CommandEvidenceStore, + TrustedFixtureProcessRunner, +} from "../twin/commands"; +import { candidateDiffHash, type CandidateChange, type CandidateDiff } from "../twin/fixture-contract"; +import { TwinManager } from "../twin/manager"; +import { + FixtureSnapshotStore, + readCandidateTree, + verifyParticipantGitBoundary, +} from "../twin/snapshot"; +import type { CommandResult, SanitizedSnapshot, TrustedFixtureRecord } from "../twin/types"; +import type { JudgeAttempt, JudgeResult } from "./types"; + +const runFile = promisify(execFile); + +export class FixturePhaseAJudge { + constructor( + private readonly catalog: BuiltinFixtureCatalog, + private readonly snapshots: FixtureSnapshotStore, + private readonly twins: TwinManager, + private readonly registry: FixtureCommandRegistry, + private readonly runner: TrustedFixtureProcessRunner, + private readonly evidence: CommandEvidenceStore, + ) {} + + async evaluate(attempt: JudgeAttempt): Promise { + validateAttempt(attempt); + const { experience } = attempt; + const snapshot = await this.snapshots.get(experience.projectId, experience.snapshotId); + if (!snapshot || snapshot.treeHash !== experience.snapshotTreeHash || snapshot.state !== "mutated") { + return this.integrity(experience, attempt, "snapshot-integrity"); + } + if (experience.commandRegistryHash !== experience.judge.commandRegistryHash) { + return this.integrity(experience, attempt, "registry-binding"); + } + const registry = await this.registry.open(experience.projectId, experience.commandRegistryHash); + if (!registry) return this.integrity(experience, attempt, "registry-integrity"); + + const record = await this.catalog.openForSnapshot(experience.sourceTargetRevision, snapshot.mutationId); + if (!record || !matchesHiddenAnswer(experience, record)) { + return this.integrity(experience, attempt, "hidden-answer-integrity"); + } + const root = this.twins.resolveReadyRoot(experience.projectId, attempt.twinHandle); + try { + await verifyParticipantGitBoundary(root, snapshot.participantCommit); + } catch { + return this.integrity(experience, attempt, "participant-git-integrity"); + } + const candidate = await readCandidateTree(root); + const diff = createCandidateDiff(snapshot, candidate); + const diffHash = diff.changes.length === 0 ? undefined : candidateDiffHash(diff); + const integrityFailure = verifyCandidateBoundary(experience, snapshot, candidate, diff.changes); + if (integrityFailure) return this.integrity(experience, attempt, integrityFailure, diffHash); + + if (diffHash === undefined || diffHash !== record.manifest.knownRepair.candidateDiffHash) { + const evidence = await this.judgeEvidence(experience.projectId, "candidate-diff", { + expected: record.manifest.knownRepair.candidateDiffHash, + actual: diffHash ?? null, + executed: false, + }); + return result({ + experienceId: experience.id, + outcome: "failed", + checks: [{ id: "candidate-diff", status: "failed", evidence }], + hintsUsed: attempt.hintsUsed, + elapsedMs: attempt.elapsedMs, + ...(diffHash === undefined ? {} : { candidateDiffHash: diffHash }), + }); + } + + return this.evaluateKnownRepair(attempt, snapshot, record, registry.commands.map((command) => command.id), diffHash); + } + + async close( + experience: InternalExperience, + outcome: "abandoned" | "revealed", + hintsUsed: number, + elapsedMs: number, + ): Promise { + validateCounters(hintsUsed, elapsedMs); + assertInternalExperience(experience); + return result({ + experienceId: experience.id, + outcome, + checks: [], + hintsUsed, + elapsedMs, + }); + } + + private async evaluateKnownRepair( + attempt: JudgeAttempt, + snapshot: SanitizedSnapshot, + record: TrustedFixtureRecord, + registryCommandIds: string[], + diffHash: string, + ): Promise { + const { experience } = attempt; + const twin = await this.twins.prepare(experience.projectId, snapshot.id); + const root = this.twins.resolveReadyRoot(experience.projectId, twin.handle); + const checks: JudgeResult["checks"] = []; + + try { + await applyRepair(root, record.blobs.repair.localHandle); + const target = record.manifest.states.find((state) => state.id === "target")!; + if (treeHash(await readCandidateTree(root)) !== target.treeHash) { + return this.integrity(experience, attempt, "known-repair-tree", diffHash); + } + + for (const check of experience.judge.checks) { + if (check.kind !== "approved-command") { + return this.integrity(experience, attempt, "unsupported-phase-a-check", diffHash); + } + if (!registryCommandIds.includes(check.commandId)) { + return this.integrity(experience, attempt, "registry-command-integrity", diffHash); + } + const command = await this.runner.run({ + executionId: randomUUID().replaceAll("-", ""), + projectId: experience.projectId, + fixtureId: record.manifest.fixtureId, + manifestHash: record.manifestHash, + stateId: "target", + commandId: check.commandId, + twinHandle: twin.handle, + }); + const passed = command.exitCode === check.expectedExitCode && !command.timedOut && !command.cancelled; + checks.push({ + id: check.id, + status: passed ? "passed" : "failed", + evidence: await this.commandEvidence(experience.projectId, check.id, command), + }); + } + } finally { + await this.twins.cleanup(experience.projectId, twin.handle); + } + + const passed = checks.filter((check) => check.status === "passed").length; + return result({ + experienceId: experience.id, + outcome: passed === checks.length ? "passed" : passed > 0 ? "partial" : "failed", + checks, + hintsUsed: attempt.hintsUsed, + elapsedMs: attempt.elapsedMs, + candidateDiffHash: diffHash, + }); + } + + private async integrity( + experience: InternalExperience, + attempt: Pick, + reason: string, + diffHash?: string, + ): Promise { + const evidence = await this.judgeEvidence(experience.projectId, "integrity", { reason, executed: false }); + return result({ + experienceId: experience.id, + outcome: "failed-integrity", + checks: [{ id: "integrity", status: "failed", evidence }], + hintsUsed: attempt.hintsUsed, + elapsedMs: attempt.elapsedMs, + ...(diffHash === undefined ? {} : { candidateDiffHash: diffHash }), + }); + } + + private async commandEvidence(projectId: string, checkId: string, command: CommandResult) { + return this.judgeEvidence(projectId, checkId, { + exitCode: command.exitCode, + timedOut: command.timedOut, + cancelled: command.cancelled, + }); + } + + private async judgeEvidence(projectId: string, checkId: string, value: object) { + const content = Buffer.from(canonicalJson(value), "utf8"); + const id = `judge_${rawSha256(`${projectId}\0${checkId}\0${content.toString("utf8")}`)}`; + return this.evidence.putNamed(projectId, id, content, content.byteLength); + } +} + +function createCandidateDiff(snapshot: SanitizedSnapshot, candidate: TreeFile[]): CandidateDiff { + const before = new Map(snapshot.files.map((file) => [file.path, file])); + const after = new Map(candidate.map((file) => [file.path, file])); + const paths = [...new Set([...before.keys(), ...after.keys()])].sort(compareUtf8); + const changes: CandidateChange[] = []; + for (const path of paths) { + const left = before.get(path); + const right = after.get(path); + if (left?.sha256 === right?.sha256 && left?.mode === right?.mode) continue; + changes.push({ + path, + beforeSha256: left?.sha256 ?? null, + afterSha256: right?.sha256 ?? null, + beforeMode: left?.mode ?? null, + afterMode: right?.mode ?? null, + }); + } + return { + schemaVersion: 1, + baseTreeHash: snapshot.treeHash, + resultTreeHash: treeHash(candidate), + changes, + }; +} + +function verifyCandidateBoundary( + experience: InternalExperience, + snapshot: SanitizedSnapshot, + candidate: TreeFile[], + changes: CandidateChange[], +): string | undefined { + const editable = new Set(experience.judge.editablePaths); + if (changes.some((change) => !editable.has(change.path))) return "change-outside-editable-paths"; + const files = new Map(candidate.map((file) => [file.path, file])); + for (const protectedFile of experience.judge.protectedFiles) { + if (files.get(protectedFile.path)?.sha256 !== protectedFile.sha256) return "protected-file-changed"; + } + const original = new Set(snapshot.files.map((file) => file.path)); + if (candidate.some((file) => !original.has(file.path) && !editable.has(file.path))) return "unexpected-file"; + return undefined; +} + +function matchesHiddenAnswer(experience: InternalExperience, record: TrustedFixtureRecord): boolean { + const ref = record.manifest.knownRepair.changeRef; + return experience.hiddenAnswer.id === ref.id && + experience.hiddenAnswer.sha256 === ref.sha256 && + experience.hiddenAnswer.storedBytes === ref.storedBytes && + experience.hiddenAnswer.originalBytes === ref.storedBytes && + experience.hiddenAnswer.truncated === false && + experience.hiddenAnswer.redactions.length === 0 && + experience.hiddenAnswer.mediaType === ref.mediaType && + experience.hiddenAnswer.visibility === "controller"; +} + +async function applyRepair(root: string, patch: string): Promise { + await runFile("git", ["apply", "--whitespace=nowarn", patch], { + cwd: root, + encoding: "utf8", + windowsHide: true, + timeout: 30_000, + env: { + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: process.platform === "win32" ? "NUL" : "/dev/null", + PATH: process.env.PATH ?? "", + SYSTEMROOT: process.env.SYSTEMROOT ?? "", + }, + }); +} + +function result(value: Omit): JudgeResult { + const core = { schemaVersion: 1 as const, ...value }; + return { ...core, resultHash: canonicalHash("judge-result", core) }; +} + +function validateAttempt(attempt: JudgeAttempt): void { + assertInternalExperience(attempt.experience); + assertToken(attempt.twinHandle, "twinHandle"); + validateCounters(attempt.hintsUsed, attempt.elapsedMs); +} + +function validateCounters(hintsUsed: number, elapsedMs: number): void { + if (!Number.isSafeInteger(hintsUsed) || hintsUsed < 0 || hintsUsed > 100) throw new Error("hintsUsed is invalid"); + if (!Number.isSafeInteger(elapsedMs) || elapsedMs < 0 || elapsedMs > 86_400_000) throw new Error("elapsedMs is invalid"); +} diff --git a/extension/src/judge/types.ts b/extension/src/judge/types.ts new file mode 100644 index 0000000..63ff0e4 --- /dev/null +++ b/extension/src/judge/types.ts @@ -0,0 +1,20 @@ +import type { EvidenceRef } from "../recorder/events"; +import type { InternalExperience } from "../experience/types"; + +export interface JudgeResult { + schemaVersion: 1; + experienceId: string; + outcome: "passed" | "partial" | "failed" | "failed-integrity" | "abandoned" | "revealed"; + checks: Array<{ id: string; status: "passed" | "failed"; evidence: EvidenceRef }>; + hintsUsed: number; + elapsedMs: number; + candidateDiffHash?: string; + resultHash: string; +} + +export interface JudgeAttempt { + experience: InternalExperience; + twinHandle: string; + hintsUsed: number; + elapsedMs: number; +} diff --git a/extension/src/twin/commands.ts b/extension/src/twin/commands.ts index 9f8791c..ba60d05 100644 --- a/extension/src/twin/commands.ts +++ b/extension/src/twin/commands.ts @@ -2,8 +2,9 @@ import { randomUUID } from "node:crypto"; import { execFile, spawn, type ChildProcess } from "node:child_process"; import { join, resolve, sep } from "node:path"; import { assertToken } from "../agent/types"; -import { canonicalHash, compareUtf8, rawSha256, treeHash } from "../rnd/canonical"; -import type { EvidenceRef } from "../recorder/events"; +import { canonicalHash, canonicalJson, compareUtf8, rawSha256, treeHash } from "../rnd/canonical"; +import { assertEvidenceRef, type EvidenceRef } from "../recorder/events"; +import type { LocalTextStorage } from "../recorder/store"; import { BuiltinFixtureCatalog } from "./catalog"; import { TwinManager } from "./manager"; import { readCandidateTree } from "./snapshot"; @@ -20,23 +21,32 @@ interface StoredOutput { content: string; } -export class MemoryCommandEvidenceStore { +export interface CommandEvidenceStore { + put(projectId: string, content: Buffer, originalBytes: number): Promise; + putNamed(projectId: string, id: string, content: Buffer, originalBytes: number): Promise; + open(projectId: string, ref: EvidenceRef): Promise; +} + +export class MemoryCommandEvidenceStore implements CommandEvidenceStore { private readonly values = new Map(); async put(projectId: string, content: Buffer, originalBytes: number): Promise { assertToken(projectId, "projectId"); const id = randomUUID().replaceAll("-", ""); - const ref: EvidenceRef = { - id, - kind: "command-output", - sha256: rawSha256(content), - storedBytes: content.byteLength, - originalBytes, - truncated: content.byteLength < originalBytes, - redactions: [], - mediaType: "text/plain", - visibility: "controller", - }; + return this.putNamed(projectId, id, content, originalBytes); + } + + async putNamed(projectId: string, id: string, content: Buffer, originalBytes: number): Promise { + assertToken(projectId, "projectId"); + assertToken(id, "evidenceId"); + const ref = outputRef(id, content, originalBytes); + const current = this.values.get(id); + if (current) { + if (current.projectId !== projectId || canonicalHash("evidence-ref", current.ref) !== canonicalHash("evidence-ref", ref) || current.content !== content.toString("utf8")) { + throw new Error("Named command evidence already exists with different content"); + } + return structuredClone(current.ref); + } this.values.set(id, { projectId, ref, content: content.toString("utf8") }); return structuredClone(ref); } @@ -50,6 +60,57 @@ export class MemoryCommandEvidenceStore { } } +export class LocalCommandEvidenceStore implements CommandEvidenceStore { + constructor(private readonly storage: LocalTextStorage) {} + + async put(projectId: string, content: Buffer, originalBytes: number): Promise { + return this.putNamed(projectId, randomUUID().replaceAll("-", ""), content, originalBytes); + } + + async putNamed(projectId: string, id: string, content: Buffer, originalBytes: number): Promise { + assertToken(projectId, "projectId"); + assertToken(id, "evidenceId"); + const ref = outputRef(id, content, originalBytes); + const path = outputPath(projectId, id); + const current = await this.storage.readText(`${path}.json`); + if (current !== undefined) { + const parsed = JSON.parse(current) as EvidenceRef; + assertEvidenceRef(parsed, "command-output"); + if (canonicalJson(parsed) !== current || canonicalJson(parsed) !== canonicalJson(ref)) { + throw new Error("Named command evidence already exists with different metadata"); + } + const stored = await this.storage.readText(`${path}.b64`); + if (stored !== content.toString("base64")) throw new Error("Named command evidence already exists with different content"); + return structuredClone(parsed); + } + await this.storage.writeText(`${path}.b64`, content.toString("base64")); + await this.storage.writeText(`${path}.json`, canonicalJson(ref)); + return structuredClone(ref); + } + + async open(projectId: string, ref: EvidenceRef): Promise { + assertToken(projectId, "projectId"); + assertEvidenceRef(ref, "command-output"); + const path = outputPath(projectId, ref.id); + const [meta, encoded] = await Promise.all([ + this.storage.readText(`${path}.json`), + this.storage.readText(`${path}.b64`), + ]); + if (meta === undefined && encoded === undefined) return undefined; + if (meta === undefined || encoded === undefined) throw new Error("Stored command evidence is incomplete"); + const parsed = JSON.parse(meta) as EvidenceRef; + assertEvidenceRef(parsed, "command-output"); + if (canonicalJson(parsed) !== meta || canonicalJson(parsed) !== canonicalJson(ref)) { + throw new Error("Stored command evidence metadata failed integrity"); + } + const bytes = Buffer.from(encoded, "base64"); + if (bytes.byteLength !== ref.storedBytes || rawSha256(bytes) !== ref.sha256) { + throw new Error("Stored command evidence content failed integrity"); + } + return bytes.toString("utf8"); + } +} + interface Execution { request: TrustedFixtureRequest; cancelled: boolean; @@ -63,7 +124,7 @@ export class TrustedFixtureProcessRunner { constructor( private readonly catalog: BuiltinFixtureCatalog, private readonly twins: TwinManager, - private readonly evidence: MemoryCommandEvidenceStore, + private readonly evidence: CommandEvidenceStore, ) {} async run(request: TrustedFixtureRequest): Promise { @@ -323,3 +384,26 @@ function validateRequest(request: TrustedFixtureRequest): void { assertToken(request.twinHandle, "twinHandle"); if (!["base", "target", "mutated"].includes(request.stateId)) throw new Error("Unknown fixture state"); } + +function outputRef(id: string, content: Buffer, originalBytes: number): EvidenceRef { + if (!Number.isSafeInteger(originalBytes) || originalBytes < content.byteLength) { + throw new Error("Command evidence original byte count is invalid"); + } + const ref: EvidenceRef = { + id, + kind: "command-output", + sha256: rawSha256(content), + storedBytes: content.byteLength, + originalBytes, + truncated: content.byteLength < originalBytes, + redactions: [], + mediaType: "text/plain", + visibility: "controller", + }; + assertEvidenceRef(ref, "command-output"); + return ref; +} + +function outputPath(projectId: string, id: string): string { + return `command-evidence/${projectId}/${id}`; +} diff --git a/extension/src/twin/snapshot.ts b/extension/src/twin/snapshot.ts index 19273e7..0cbb965 100644 --- a/extension/src/twin/snapshot.ts +++ b/extension/src/twin/snapshot.ts @@ -209,6 +209,12 @@ export async function verifyStandaloneRepo( participantCommit: string, files: readonly TreeFile[], ): Promise { + await verifyParticipantGitBoundary(root, participantCommit); + const actual = await readCandidateTree(root); + if (treeHash(actual) !== treeHash(files)) throw new Error("Participant tree does not match its snapshot"); +} + +export async function verifyParticipantGitBoundary(root: string, participantCommit: string): Promise { if ((await git(root, ["rev-parse", "HEAD"])) !== participantCommit) { throw new Error("Participant commit changed"); } @@ -223,8 +229,6 @@ export async function verifyStandaloneRepo( } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } - const actual = await readCandidateTree(root); - if (treeHash(actual) !== treeHash(files)) throw new Error("Participant tree does not match its snapshot"); } async function assertTree(root: string, expected: string, label: string): Promise { diff --git a/extension/test/experience-judge.test.ts b/extension/test/experience-judge.test.ts new file mode 100644 index 0000000..9bf093f --- /dev/null +++ b/extension/test/experience-judge.test.ts @@ -0,0 +1,308 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { GitRevisionDiffReader } from "../src/change/diff"; +import { extractChangeEvidence, type ExtractionResult } from "../src/change/evidence"; +import { FixtureExperienceCompiler, serializeParticipantExperience } from "../src/experience/compile"; +import { selectRecoverySeam } from "../src/experience/select"; +import { FixturePhaseAJudge } from "../src/judge/run"; +import type { ReplayTranscript } from "../src/agent/replay-driver"; +import { BuiltinFixtureCatalog } from "../src/twin/catalog"; +import { + FixtureCommandRegistry, + LocalCommandEvidenceStore, + MemoryCommandEvidenceStore, + TrustedFixtureProcessRunner, +} from "../src/twin/commands"; +import { + createTenantCacheKeyFixture, + EXPECTED_TENANT_CACHE_KEY, + type TenantCacheKeyFixture, +} from "../src/twin/fixture-factory"; +import { TwinManager } from "../src/twin/manager"; +import { FixtureSnapshotStore } from "../src/twin/snapshot"; +import { NodeLocalTextStorage } from "../src/recorder/store"; + +const transcriptPath = resolve(import.meta.dirname, "fixtures/v0.3/agent-replay/succeeded.json"); +const roots: string[] = []; +const fixtures = new Map(); + +afterEach(async () => { + await Promise.all(roots.splice(0).map(async (root) => { + const fixture = fixtures.get(root); + if (fixture) { + await fixture.dispose(); + fixtures.delete(root); + } else { + await rm(root, { recursive: true, force: true, maxRetries: 3 }); + } + })); +}); + +describe("R4 Experience Compiler and Phase-A Judge", () => { + it("compiles one deterministic recovery episode and a strict participant projection", async () => { + const env = await setup(); + const extraction = await fixtureExtraction(); + const first = await env.compiler.compile({ + extraction, + fixtureId: "tenant-cache-key", + manifestHash: EXPECTED_TENANT_CACHE_KEY.manifestHash, + }); + const second = await env.compiler.compile({ + extraction, + fixtureId: "tenant-cache-key", + manifestHash: EXPECTED_TENANT_CACHE_KEY.manifestHash, + }); + + expect(first.selectionReasons).toEqual([ + "attributed-change", + "bounded-attention", + "fixture-supported", + "test-backed", + ]); + expect(first.internal).toMatchObject({ + kind: "recover", + projectId: "project_r0_fixture", + sourceRunId: "run_tenant_cache", + sourceBaseRevision: EXPECTED_TENANT_CACHE_KEY.baseRevision, + sourceTargetRevision: EXPECTED_TENANT_CACHE_KEY.targetRevision, + snapshotTreeHash: EXPECTED_TENANT_CACHE_KEY.mutatedTreeHash, + scope: { + symbols: ["src/cache-key.ts#cacheKey"], + tests: ["cache-key.tenant-isolation"], + concepts: ["tested-boundary-recovery"], + }, + }); + expect(second.internal.commandRegistryHash).toBe(first.internal.commandRegistryHash); + expect(second.internal.snapshotTreeHash).toBe(first.internal.snapshotTreeHash); + + const serialized = serializeParticipantExperience(first.participant); + expect(JSON.parse(serialized)).toEqual(first.participant); + for (const forbidden of [ + "sourceRunId", + "sourceBaseRevision", + "sourceTargetRevision", + "commandRegistryHash", + "judge", + "hiddenAnswer", + "setup", + EXPECTED_TENANT_CACHE_KEY.targetRevision, + first.internal.hiddenAnswer.sha256, + resolve("D:/pureflow"), + ]) { + expect(serialized).not.toContain(forbidden); + } + const injected = { ...first.participant, sourceRunId: "leak" }; + expect(() => serializeParticipantExperience(injected as typeof first.participant)).toThrow("unknown"); + }, 45_000); + + it("rejects partial, untested, and fixture-mismatched seams", async () => { + const extraction = await fixtureExtraction(); + expect(selectRecoverySeam(extraction).seam).toBeDefined(); + + const partial = { ...extraction, status: "partial", reasons: ["partial-parse"] } as ExtractionResult; + expect(() => selectRecoverySeam(partial)).toThrow("supported"); + + const untested = structuredClone(extraction); + if (untested.status !== "supported") throw new Error("Expected supported fixture"); + untested.seams[0]!.linkedChecks = []; + expect(() => selectRecoverySeam(untested)).toThrow("test-backed"); + + const env = await setup(); + const drift = structuredClone(extraction); + if (drift.status !== "supported") throw new Error("Expected supported fixture"); + drift.seams[0]!.targetRevision = "0".repeat(40); + await expect(env.compiler.compile({ + extraction: drift, + fixtureId: "tenant-cache-key", + manifestHash: EXPECTED_TENANT_CACHE_KEY.manifestHash, + })).rejects.toThrow("fixture"); + + const compiled = await compile(env); + const injected = { ...compiled.internal, expectedAnswer: "trust me" }; + await expect(env.judge.evaluate({ + experience: injected as typeof compiled.internal, + twinHandle: "opaque_twin", + hintsUsed: 0, + elapsedMs: 1, + })).rejects.toThrow("unknown"); + }, 45_000); + + it("rejects no-op and unrelated candidates without executing them", async () => { + const env = await setup(); + const compiled = await compile(env); + const twin = await env.twins.prepare(compiled.internal.projectId, compiled.internal.snapshotId); + + const noOp = await env.judge.evaluate({ + experience: compiled.internal, + twinHandle: twin.handle, + hintsUsed: 0, + elapsedMs: 2_000, + }); + expect(noOp).toMatchObject({ outcome: "failed", checks: [{ id: "candidate-diff", status: "failed" }] }); + + const root = env.twins.resolveReadyRoot(compiled.internal.projectId, twin.handle); + await writeFile(join(root, "src", "cache-key.ts"), "export function cacheKey(): string { return 'unrelated'; }\n"); + const unrelated = await env.judge.evaluate({ + experience: compiled.internal, + twinHandle: twin.handle, + hintsUsed: 1, + elapsedMs: 3_000, + }); + expect(unrelated).toMatchObject({ outcome: "failed", checks: [{ id: "candidate-diff", status: "failed" }] }); + expect(env.runner.completedCount).toBe(0); + }, 45_000); + + it("fails integrity for protected or out-of-scope changes", async () => { + const env = await setup(); + const compiled = await compile(env); + const protectedTwin = await env.twins.prepare(compiled.internal.projectId, compiled.internal.snapshotId); + const protectedRoot = env.twins.resolveReadyRoot(compiled.internal.projectId, protectedTwin.handle); + await writeFile(join(protectedRoot, ".gitattributes"), "* -text\n"); + + await expect(env.judge.evaluate({ + experience: compiled.internal, + twinHandle: protectedTwin.handle, + hintsUsed: 0, + elapsedMs: 1_000, + })).resolves.toMatchObject({ outcome: "failed-integrity" }); + + const extraTwin = await env.twins.prepare(compiled.internal.projectId, compiled.internal.snapshotId); + const extraRoot = env.twins.resolveReadyRoot(compiled.internal.projectId, extraTwin.handle); + await writeFile(join(extraRoot, "answer.txt"), "bypass\n"); + await expect(env.judge.evaluate({ + experience: compiled.internal, + twinHandle: extraTwin.handle, + hintsUsed: 0, + elapsedMs: 1_000, + })).resolves.toMatchObject({ outcome: "failed-integrity" }); + expect(env.runner.completedCount).toBe(0); + }, 45_000); + + it("accepts only the exact known repair and replays deterministically", async () => { + const env = await setup(); + const compiled = await compile(env); + const results = []; + + for (let index = 0; index < 3; index += 1) { + const twin = await env.twins.prepare(compiled.internal.projectId, compiled.internal.snapshotId); + const root = env.twins.resolveReadyRoot(compiled.internal.projectId, twin.handle); + await writeFile( + join(root, "src", "cache-key.ts"), + "export function cacheKey(tenant: string, id: string): string {\n return `${tenant}:${id}`;\n}\n", + ); + results.push(await env.judge.evaluate({ + experience: compiled.internal, + twinHandle: twin.handle, + hintsUsed: 0, + elapsedMs: 4_000, + })); + } + + expect(results[0]).toMatchObject({ + outcome: "passed", + checks: [{ id: "cache-key.tenant-isolation", status: "passed" }], + hintsUsed: 0, + elapsedMs: 4_000, + }); + expect(results[1]).toEqual(results[0]); + expect(results[2]).toEqual(results[0]); + expect(results[0]!.candidateDiffHash).toBeDefined(); + expect(env.runner.completedCount).toBe(3); + }, 60_000); + + it("records reveal and abandonment without executable readiness evidence", async () => { + const env = await setup(); + const compiled = await compile(env); + + const revealed = await env.judge.close(compiled.internal, "revealed", 2, 900); + const abandoned = await env.judge.close(compiled.internal, "abandoned", 0, 500); + expect(revealed).toMatchObject({ outcome: "revealed", checks: [], hintsUsed: 2 }); + expect(abandoned).toMatchObject({ outcome: "abandoned", checks: [], hintsUsed: 0 }); + expect(revealed.resultHash).not.toBe(abandoned.resultHash); + expect(env.runner.completedCount).toBe(0); + }, 45_000); + + it("persists and re-verifies bounded judge evidence locally", async () => { + const root = await mkdtemp(join(tmpdir(), "pureflow-r4-evidence-")); + roots.push(root); + const first = new LocalCommandEvidenceStore(new NodeLocalTextStorage(root)); + const content = Buffer.from("{\"exitCode\":0}", "utf8"); + const ref = await first.putNamed("project-a", "judge_fixture_check", content, content.byteLength); + const reopened = new LocalCommandEvidenceStore(new NodeLocalTextStorage(root)); + + await expect(reopened.open("project-a", ref)).resolves.toBe(content.toString("utf8")); + await expect(reopened.open("project-b", ref)).resolves.toBeUndefined(); + await writeFile(join(root, "command-evidence", "project-a", "judge_fixture_check.b64"), "dGFtcGVyZWQ="); + await expect(reopened.open("project-a", ref)).rejects.toThrow("integrity"); + }); +}); + +class CountingRunner extends TrustedFixtureProcessRunner { + completedCount = 0; + + override async run(request: Parameters[0]) { + const result = await super.run(request); + this.completedCount += 1; + return result; + } +} + +async function setup() { + const root = await mkdtemp(join(tmpdir(), "pureflow-r4-test-")); + roots.push(root); + const catalog = new BuiltinFixtureCatalog(); + const snapshots = new FixtureSnapshotStore(join(root, "snapshots"), catalog, { + now: () => "2026-01-03T00:00:00.000Z", + }); + const twins = new TwinManager(join(root, "twins"), snapshots); + const evidence = new MemoryCommandEvidenceStore(); + const runner = new CountingRunner(catalog, twins, evidence); + const registry = new FixtureCommandRegistry( + catalog, + "tenant-cache-key", + EXPECTED_TENANT_CACHE_KEY.manifestHash, + ); + const compiler = new FixtureExperienceCompiler(catalog, snapshots, registry, { + id: () => "experience_fixture_recovery_1", + }); + const judge = new FixturePhaseAJudge(catalog, snapshots, twins, registry, runner, evidence); + return { catalog, snapshots, twins, evidence, runner, registry, compiler, judge }; +} + +async function compile(env: Awaited>) { + return env.compiler.compile({ + extraction: await fixtureExtraction(), + fixtureId: "tenant-cache-key", + manifestHash: EXPECTED_TENANT_CACHE_KEY.manifestHash, + }); +} + +async function fixtureExtraction(): Promise { + const fixture = await createFixture(); + const diff = await new GitRevisionDiffReader().read(fixture.root, fixture.baseRevision, fixture.targetRevision); + const transcript = JSON.parse(await readFile(transcriptPath, "utf8")) as ReplayTranscript; + return extractChangeEvidence({ + schemaVersion: 1, + projectId: transcript.run.projectId, + sourceRunId: transcript.run.runId, + baseRevision: fixture.baseRevision, + targetRevision: fixture.targetRevision, + diff, + links: fixture.manifest.changedSymbols.map(({ path, symbol }) => ({ + path, + symbol, + checks: fixture.manifest.targetChecks, + })), + events: transcript.events, + }); +} + +async function createFixture(): Promise { + const root = await mkdtemp(join(tmpdir(), "pureflow-r0b-")); + roots.push(root); + const fixture = await createTenantCacheKeyFixture(root); + fixtures.set(root, fixture); + return fixture; +}