diff --git a/CHALLENGE.md b/CHALLENGE.md index 878a0a0..57f87c1 100644 --- a/CHALLENGE.md +++ b/CHALLENGE.md @@ -77,7 +77,7 @@ Registration occurs in the top-level document. If `document.modelContext` is una | `compare_policies` | Read-only | Compares up to three registered policies; by default it uses `observable_adaptive` and the calibration-frozen `periodic_local` comparator | | `stage_conclusion` | Staging write | Places the artifact's typed `abstain_without_policy_claim` conclusion plus one to eight evidence IDs in the pending tray; free-form agent claims are rejected and it cannot approve or commit the conclusion | -The first seven tools are annotated read-only. `stage_conclusion` requires the current state version, refuses to overwrite an existing pending review, and affects only local pending-review state—never the source experiment JSON. No WebMCP tool can approve, reject, edit, or undo a conclusion: those actions are page-only human controls. The agent can explore broadly and prepare a coherent evidence bundle, but it must stop at the judgment boundary. +The first seven tools are annotated read-only. `stage_conclusion` requires the current state version, a failed frozen gate, and an adaptive family or run with controller abstentions; it performs a final compare-and-swap and refuses to overwrite an existing pending review. It affects only local pending-review state—never the source experiment JSON. No WebMCP tool can approve, reject, edit, or undo a conclusion: those actions are page-only human controls. A human edit is recorded explicitly as an override, and the approved claim, evidence, artifact code, timestamp, and override status remain visible until undone. The agent can explore broadly and prepare a coherent evidence bundle, but it must stop at the judgment boundary. ## Testing diff --git a/docs/observatory.html b/docs/observatory.html index 91d2037..4190715 100644 --- a/docs/observatory.html +++ b/docs/observatory.html @@ -15,10 +15,10 @@ - - - - + + + + diff --git a/docs/webmcp-mission.js b/docs/webmcp-mission.js index 691a4e8..285a972 100644 --- a/docs/webmcp-mission.js +++ b/docs/webmcp-mission.js @@ -47,6 +47,7 @@ let screeningPromise = null; let projectionPromise = null; let tourRunning = false; + let tourGeneration = 0; let editingProposal = false; let missionState = loadMissionState(); @@ -95,16 +96,36 @@ if (signal && signal.aborted) throw signal.reason || new DOMException("Aborted", "AbortError"); } - async function fetchArtifact(url, signal) { + function waitForShared(promise, signal) { abortIfNeeded(signal); - const response = await fetch(url, { cache: "no-store", headers: { Accept: "application/json" }, signal }); + if (!signal || typeof signal.addEventListener !== "function") return promise; + return new Promise((resolve, reject) => { + const onAbort = () => { + signal.removeEventListener?.("abort", onAbort); + reject(signal.reason || new DOMException("Aborted", "AbortError")); + }; + signal.addEventListener("abort", onAbort, { once: true }); + promise.then( + (value) => { + signal.removeEventListener?.("abort", onAbort); + resolve(value); + }, + (error) => { + signal.removeEventListener?.("abort", onAbort); + reject(error); + }, + ); + }); + } + + async function fetchArtifact(url) { + const response = await fetch(url, { cache: "no-store", headers: { Accept: "application/json" } }); if (!response.ok) throw new MissionError("ARTIFACT_UNAVAILABLE", `${url} returned HTTP ${response.status}.`); return response.json(); } - async function fetchGzipArtifact(url, signal) { - abortIfNeeded(signal); - const response = await fetch(url, { cache: "no-store", headers: { Accept: "application/gzip" }, signal }); + async function fetchGzipArtifact(url) { + const response = await fetch(url, { cache: "no-store", headers: { Accept: "application/gzip" } }); if (!response.ok) throw new MissionError("ARTIFACT_UNAVAILABLE", `${url} returned HTTP ${response.status}.`); if (typeof DecompressionStream !== "function") { throw new MissionError("DECOMPRESSION_UNAVAILABLE", "This browser cannot open the bounded gzip projection."); @@ -119,7 +140,7 @@ } function semanticArtifact(signal) { - semanticPromise ||= fetchArtifact(SEMANTIC_URL, signal).then((value) => { + semanticPromise ||= fetchArtifact(SEMANTIC_URL).then((value) => { if (value?.schema !== "gpu-stack.causal-observatory.e001-semantic-consistency.v1") { throw new MissionError("ARTIFACT_INVALID", "The E001-SC1 compact artifact has an unsupported schema."); } @@ -128,11 +149,11 @@ semanticPromise = null; throw error; }); - return semanticPromise; + return waitForShared(semanticPromise, signal); } function screeningArtifact(signal) { - screeningPromise ||= fetchArtifact(SCREENING_URL, signal).then((value) => { + screeningPromise ||= fetchArtifact(SCREENING_URL).then((value) => { if (value?.schema !== "gpu-stack.causal-observatory.e001.v1") { throw new MissionError("ARTIFACT_INVALID", "The E001 screening artifact has an unsupported schema."); } @@ -141,11 +162,11 @@ screeningPromise = null; throw error; }); - return screeningPromise; + return waitForShared(screeningPromise, signal); } function runProjection(signal) { - projectionPromise ||= fetchGzipArtifact(PROJECTION_URL, signal).then((value) => { + projectionPromise ||= fetchGzipArtifact(PROJECTION_URL).then((value) => { if (value?.schema !== "gpustack.webmcp-run-projection.v1" || !Array.isArray(value.epoch_columns)) { throw new MissionError("PROJECTION_INVALID", "The bounded epoch projection has an unsupported schema."); } @@ -154,7 +175,7 @@ projectionPromise = null; throw error; }); - return projectionPromise; + return waitForShared(projectionPromise, signal); } async function observatory() { @@ -268,6 +289,7 @@ const effects = semantic.researcher.paired_effects || []; const families = familyList(semantic); const view = bridge.getState(); + const latestApproved = missionState.approved[missionState.approved.length - 1] || null; const receipt = addReceipt("get_observatory_state", "complete", "Read the immutable audit state and registered evidence IDs.", { evidenceIds: [semantic.artifact_sha256], origin: context.origin, @@ -304,7 +326,11 @@ causal_nodes: screening.causal_graph.nodes.map((node) => node.node_id), failed_gates: effects.filter((effect) => effect.passed === false).map((effect) => effect.effect_id), }, - pending_proposal: missionState.pending ? missionState.pending.proposalId : null, + review: { + pending: missionState.pending ? missionState.pending.proposalId : null, + latest_approved: latestApproved ? latestApproved.proposalId : null, + human_override: latestApproved ? Boolean(latestApproved.humanOverride) : false, + }, suggested_next: "compare_stress_families", }, receipt); } @@ -416,6 +442,10 @@ throw new MissionError("HASH_MISMATCH", "The bounded projection is not bound to this compact artifact's raw trace."); } const page = projectedEpochRows(projection, run.run_id, args.epoch_offset, args.epoch_limit); + if (run.policy_id !== "observable_adaptive") { + const flagColumn = page.columns.indexOf("abstained"); + if (flagColumn >= 0) page.columns[flagColumn] = "support_envelope_flag"; + } const bridge = await observatory(); await bridge.selectView({ experiment: "E001-SC1", @@ -682,7 +712,10 @@ : median(values); } }); - return { policy_id: policyId, evaluation_runs: runs.length, metrics }; + const role = policyId === "observable_adaptive" + ? "candidate" + : (policyId === semantic.comparison.selected_fixed_policy_id ? "calibration_frozen_comparator" : "registered_reference"); + return { policy_id: policyId, role, evaluation_runs: runs.length, metrics }; }); const bridge = await observatory(); await bridge.selectView({ experiment: "E001-SC1", depth: "researcher" }); @@ -695,7 +728,10 @@ return resultWithReceipt({ ok: true, evaluation_split_only: true, - comparator_frozen_before_evaluation: semantic.comparison.selection.frozen_before_evaluation, + comparator_contract: { + policy_id: semantic.comparison.selected_fixed_policy_id, + frozen_before_evaluation: semantic.comparison.selection.frozen_before_evaluation, + }, policies: rows, frozen_gates: semantic.researcher.paired_effects.map((effect) => ({ effect_id: effect.effect_id, @@ -726,11 +762,33 @@ for (const evidenceId of args.evidence_ids) { resolved.push(await resolveEvidence(evidenceId, "researcher", false, context.signal)); } + const hasFailedGate = resolved.some((entry) => entry.kind === "frozen_gate" && entry.passed === false); + const hasAdaptiveAbstention = resolved.some((entry) => ( + (entry.kind === "held_out_family" && Number(entry.abstentions) > 0) + || (entry.kind === "run" && entry.policy_id === "observable_adaptive" && Number(entry.controller_abstentions) > 0) + )); + if (!hasFailedGate || !hasAdaptiveAbstention) { + throw new MissionError("EVIDENCE_INSUFFICIENT", "The typed abstain conclusion requires at least one failed frozen gate and one adaptive family or run with controller abstentions.", { + required_evidence: ["failed_frozen_gate", "adaptive_abstention_family_or_run"], + }); + } abortIfNeeded(context.signal); + if (args.expected_state_version !== missionState.stateVersion) { + throw new MissionError("STALE_STATE", `Expected state version ${args.expected_state_version}, but current version is ${missionState.stateVersion}.`, { + current_state_version: missionState.stateVersion, + }); + } + if (missionState.pending) { + throw new MissionError("PENDING_REVIEW_EXISTS", `Human review is already pending for ${missionState.pending.proposalId}. Approve or reject it before staging another conclusion.`, { + pending_proposal: missionState.pending.proposalId, + }); + } const proposal = { proposalId: `proposal-${String(missionState.nextProposal).padStart(3, "0")}`, claim: CANONICAL_CONCLUSIONS[args.conclusion_code], + originalClaim: CANONICAL_CONCLUSIONS[args.conclusion_code], conclusionCode: args.conclusion_code, + humanOverride: false, evidenceIds: [...args.evidence_ids], evidenceKinds: resolved.map((entry) => entry.kind), confidence: "abstain", @@ -807,9 +865,29 @@ function renderPending() { if (!dom.pending) return; const proposal = missionState.pending; + const latestApproved = missionState.approved[missionState.approved.length - 1] || null; dom.pending.replaceChildren(); if (!proposal) { - dom.pending.innerHTML = EMPTY_PENDING_HTML; + if (!latestApproved) { + dom.pending.innerHTML = EMPTY_PENDING_HTML; + } else { + const card = document.createElement("article"); + card.className = "mission-change"; + card.dataset.kind = "approved-conclusion"; + card.dataset.changeId = latestApproved.proposalId; + const title = document.createElement("h3"); + title.textContent = `${latestApproved.proposalId} · human recorded${latestApproved.humanOverride ? " · override" : ""}`; + const claim = document.createElement("p"); + claim.textContent = latestApproved.claim; + const evidence = document.createElement("p"); + const evidenceLabel = document.createElement("strong"); + evidenceLabel.textContent = "Evidence: "; + evidence.append(evidenceLabel, document.createTextNode(latestApproved.evidenceIds.join(" · "))); + const boundary = document.createElement("p"); + boundary.textContent = `Artifact code: ${latestApproved.conclusionCode}. Human override: ${latestApproved.humanOverride ? "yes" : "no"}. Recorded ${new Date(latestApproved.approvedAt).toLocaleString()}.`; + card.append(title, claim, evidence, boundary); + dom.pending.append(card); + } } else { const card = document.createElement("article"); card.className = "mission-change"; @@ -833,7 +911,7 @@ evidenceLabel.textContent = "Evidence: "; evidence.append(evidenceLabel, document.createTextNode(proposal.evidenceIds.join(" · "))); const boundary = document.createElement("p"); - boundary.textContent = `Frozen result: ${proposal.frozenConclusion}. Approval has not occurred.`; + boundary.textContent = `Frozen result: ${proposal.frozenConclusion}. Human override: ${proposal.humanOverride ? "yes" : "no"}. Approval has not occurred.`; card.append(title, claim, evidence, boundary); dom.pending.append(card); } @@ -895,7 +973,7 @@ missionState.stateVersion += 1; editingProposal = false; persist(); - addReceipt("human_approve", "approved", `Human recorded ${approved.proposalId}.`, { + addReceipt("human_approve", "approved", `Human recorded ${approved.proposalId}: ${approved.claim}`, { evidenceIds: approved.evidenceIds, delta: `Recorded conclusion → ${approved.proposalId}`, origin: "human", @@ -934,6 +1012,7 @@ return; } missionState.pending.claim = claim.slice(0, 600); + missionState.pending.humanOverride = missionState.pending.claim !== missionState.pending.originalClaim; missionState.stateVersion += 1; editingProposal = false; persist(); @@ -958,9 +1037,12 @@ } async function resetMission() { + tourGeneration += 1; + const nextStateVersion = missionState.stateVersion + 1; missionState = initialMissionState(); + missionState.stateVersion = nextStateVersion; editingProposal = false; - try { window.sessionStorage.removeItem(STORAGE_KEY); } catch (_error) { /* no-op */ } + persist(); renderMission(); clearHighlights(); const bridge = await observatory(); @@ -974,19 +1056,25 @@ async function runTour() { if (tourRunning) return; + const generation = ++tourGeneration; tourRunning = true; dom.tour.disabled = true; try { await invoke("get_observatory_state", {}, { origin: "local_tour" }); await pause(220); + if (generation !== tourGeneration) return; await invoke("compare_stress_families", {}, { origin: "local_tour" }); await pause(220); + if (generation !== tourGeneration) return; await invoke("inspect_stress_family", { family_id: "E6-repeated-membership-loss", include_regions: false }, { origin: "local_tour" }); await pause(220); + if (generation !== tourGeneration) return; await invoke("trace_causal_path", { from_node: "site_availability", to_node: "time_to_target", max_nodes: 7 }, { origin: "local_tour" }); await pause(220); + if (generation !== tourGeneration) return; await invoke("open_evidence", { evidence_id: "adaptive_minus_best_fixed_final_nll", semantic_depth: "researcher" }, { origin: "local_tour" }); await pause(220); + if (generation !== tourGeneration) return; await invoke("stage_conclusion", { conclusion_code: "abstain_without_policy_claim", evidence_ids: [ @@ -1019,7 +1107,7 @@ } function refreshRegistrationStatus(forcedState) { - if (!forcedState && missionState.pending) { + if (missionState.pending) { setStatus("Audit staged · waiting for human approval", "waiting"); return; } diff --git a/evals/webmcp-evals.json b/evals/webmcp-evals.json index 059af6b..b06294d 100644 --- a/evals/webmcp-evals.json +++ b/evals/webmcp-evals.json @@ -56,7 +56,7 @@ { "role": "user", "type": "message", - "content": "Inspect the first eight projected epochs of the observable_adaptive run in the E6 held-out family. Keep the result bounded and preserve its link to the authoritative trace." + "content": "Inspect a six-row transition slice starting at epoch 158 of the observable_adaptive run in the E6 held-out family. Keep the result bounded and preserve its link to the authoritative trace." } ], "expectedCall": [ diff --git a/tests/test_webmcp_contract.py b/tests/test_webmcp_contract.py index 579fca2..51d39f0 100644 --- a/tests/test_webmcp_contract.py +++ b/tests/test_webmcp_contract.py @@ -377,9 +377,9 @@ def test_adapter_documents_late_bound_bridge_and_human_only_approval() -> None: def test_observatory_load_order_and_cache_keys_include_the_bridge_release() -> None: html = OBSERVATORY_HTML.read_text(encoding="utf-8") scripts = [ - 'observatory.js?v=20260903.3', - 'webmcp-tools.js?v=20260903.3', - 'webmcp-mission.js?v=20260903.3', + 'observatory.js?v=20260903.5', + 'webmcp-tools.js?v=20260903.5', + 'webmcp-mission.js?v=20260903.5', ] assert all(script in html for script in scripts) diff --git a/tests/test_webmcp_mission_runtime.mjs b/tests/test_webmcp_mission_runtime.mjs index 7e70c10..312a16f 100644 --- a/tests/test_webmcp_mission_runtime.mjs +++ b/tests/test_webmcp_mission_runtime.mjs @@ -9,7 +9,7 @@ import { fileURLToPath } from "node:url"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -async function makeRuntime() { +async function makeRuntime({ fetchDelayMs = 0 } = {}) { const registrations = new Map(); const selections = []; const paths = []; @@ -72,6 +72,7 @@ async function makeRuntime() { console, document, fetch: async (url) => { + if (fetchDelayMs) await new Promise((resolve) => setTimeout(resolve, fetchDelayMs)); const filename = path.join(ROOT, "docs", String(url)); try { const body = await fs.readFile(filename); @@ -102,10 +103,10 @@ async function makeRuntime() { function executor(runtime) { - return (name, args) => { + return (name, args, signal = new AbortController().signal) => { runtime.context.__args = JSON.stringify(args); const realmArgs = vm.runInContext("JSON.parse(__args)", runtime.context); - return runtime.registrations.get(name).execute(realmArgs, { signal: new AbortController().signal }); + return runtime.registrations.get(name).execute(realmArgs, { signal }); }; } @@ -128,6 +129,11 @@ test("all eight WebMCP registrations execute against immutable evidence", async assert.equal(state.registered_ids.causal_nodes.length, 7); assert.equal(state.registered_ids.failed_gates.length, 4); assert.equal(state.evidence_boundary.unresolved, "frontier/facility transfer"); + assert.deepEqual(JSON.parse(JSON.stringify(state.review)), { + pending: null, + latest_approved: null, + human_override: false, + }); assert.ok(JSON.stringify(state).length <= 1500, `state result exceeded budget: ${JSON.stringify(state).length}`); assert.equal(state.truncated, undefined); @@ -158,6 +164,17 @@ test("all eight WebMCP registrations execute against immutable evidence", async assert.ok(JSON.stringify(run).length <= 1450, `run result left too little budget headroom: ${JSON.stringify(run).length}`); assert.equal(run.truncated, undefined); + const fixedRun = await execute("inspect_run", { + run_id: "e001-sc1:evaluation:E6-repeated-membership-loss:periodic_local", + epoch_offset: 158, + epoch_limit: 6, + }); + assert.equal(fixedRun.ok, true); + assert.equal(fixedRun.run.controller_abstentions, null); + assert.equal(fixedRun.run.support_envelope_flag_count, 24); + assert.ok(fixedRun.epoch_page.columns.includes("support_envelope_flag")); + assert.ok(!fixedRun.epoch_page.columns.includes("abstained")); + const trace = await execute("trace_causal_path", { from_node: "site_availability", to_node: "time_to_target", @@ -189,13 +206,21 @@ test("all eight WebMCP registrations execute against immutable evidence", async const policies = await execute("compare_policies", {}); assert.equal(policies.ok, true); assert.equal(policies.policies.length, 2); - assert.equal(policies.comparator_frozen_before_evaluation, true); + assert.equal(policies.comparator_contract.policy_id, "periodic_local"); + assert.equal(policies.comparator_contract.frozen_before_evaluation, true); + assert.equal(policies.policies[0].role, "candidate"); + assert.equal(policies.policies[1].role, "calibration_frozen_comparator"); assert.equal(policies.policies[0].metrics.controller_abstentions, 104); assert.equal(policies.policies[1].metrics.controller_abstentions, null); assert.equal(policies.policies[0].metrics.support_envelope_flag_count, 104); assert.equal(policies.policies[1].metrics.support_envelope_flag_count, 104); assert.equal(policies.truncated, undefined); + const referencePolicy = await execute("compare_policies", { policy_ids: ["future_trace_oracle"] }); + assert.equal(referencePolicy.ok, true); + assert.equal(referencePolicy.policies[0].role, "registered_reference"); + assert.equal(referencePolicy.comparator_contract.policy_id, "periodic_local"); + const rejectedOverclaim = await execute("stage_conclusion", { conclusion_code: "transferable_winner", evidence_ids: ["E6-repeated-membership-loss"], @@ -238,16 +263,24 @@ test("adapter rejects invalid and stale calls without mutating approval state", assert.equal(unknown.ok, false); assert.equal(unknown.code, "UNKNOWN_FAMILY"); + const insufficient = await execute("stage_conclusion", { + conclusion_code: "abstain_without_policy_claim", + evidence_ids: ["site_availability"], + expected_state_version: 1, + }); + assert.equal(insufficient.ok, false); + assert.equal(insufficient.code, "EVIDENCE_INSUFFICIENT"); + const staged = await execute("stage_conclusion", { conclusion_code: "abstain_without_policy_claim", - evidence_ids: ["adaptive_minus_best_fixed_final_nll"], + evidence_ids: ["adaptive_minus_best_fixed_final_nll", "E6-repeated-membership-loss"], expected_state_version: 1, }); assert.equal(staged.ok, true); const occupied = await execute("stage_conclusion", { conclusion_code: "abstain_without_policy_claim", - evidence_ids: ["adaptive_minus_best_fixed_final_nll"], + evidence_ids: ["adaptive_minus_best_fixed_final_nll", "E6-repeated-membership-loss"], expected_state_version: 2, }); assert.equal(occupied.ok, false); @@ -255,10 +288,48 @@ test("adapter rejects invalid and stale calls without mutating approval state", const stale = await execute("stage_conclusion", { conclusion_code: "abstain_without_policy_claim", - evidence_ids: ["adaptive_minus_best_fixed_final_nll"], + evidence_ids: ["adaptive_minus_best_fixed_final_nll", "E6-repeated-membership-loss"], expected_state_version: 1, }); assert.equal(stale.ok, false); assert.equal(stale.code, "STALE_STATE"); assert.equal(runtime.window.GPUStackMission.getState().approved.length, 0); }); + + +test("concurrent staging uses a final compare-and-swap and cannot overwrite pending review", async () => { + const runtime = await makeRuntime(); + const execute = executor(runtime); + const state = await execute("get_observatory_state", {}); + const args = { + conclusion_code: "abstain_without_policy_claim", + evidence_ids: ["adaptive_minus_best_fixed_final_nll", "E6-repeated-membership-loss"], + expected_state_version: state.state_version, + }; + + const results = await Promise.all([ + execute("stage_conclusion", args), + execute("stage_conclusion", args), + ]); + assert.equal(results.filter((result) => result.ok).length, 1); + assert.equal(results.filter((result) => !result.ok).length, 1); + assert.ok(["STALE_STATE", "PENDING_REVIEW_EXISTS"].includes(results.find((result) => !result.ok).code)); + assert.equal(runtime.window.GPUStackMission.getState().pending.proposalId, "proposal-001"); +}); + + +test("one caller aborting a cold shared artifact load does not cancel another caller", async () => { + const runtime = await makeRuntime({ fetchDelayMs: 20 }); + const execute = executor(runtime); + const cancelled = new AbortController(); + const surviving = new AbortController(); + const first = execute("get_observatory_state", {}, cancelled.signal); + const second = execute("get_observatory_state", {}, surviving.signal); + cancelled.abort(new DOMException("cancelled", "AbortError")); + + const [firstResult, secondResult] = await Promise.allSettled([first, second]); + assert.equal(firstResult.status, "rejected"); + assert.equal(secondResult.status, "fulfilled"); + assert.equal(secondResult.value.ok, true); + assert.equal(secondResult.value.artifact.runs, 56); +});