Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions CHALLENGE.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,9 +75,9 @@ Registration occurs in the top-level document. If `document.modelContext` is una
| `trace_causal_path` | Read-only | Finds and highlights a bounded path through the seven-node conceptual evidence graph without collapsing its branches or evidence classes |
| `open_evidence` | Read-only | Opens a registered artifact, source result, assumption, uncertainty item, or missing-evidence boundary at a chosen semantic depth |
| `compare_policies` | Read-only | Compares up to three registered policies; by default it uses `observable_adaptive` and the calibration-frozen `periodic_local` comparator |
| `stage_conclusion` | Staging write | Places a supported, qualified, or abstain claim plus one to eight evidence IDs in the pending tray; it cannot approve or commit the claim |
| `stage_conclusion` | Staging write | Places the artifact's typed `abstain_without_policy_claim` conclusion plus one to eight evidence IDs in the pending tray; free-form agent claims are rejected and it cannot approve or commit the conclusion |

The first seven tools are annotated read-only. `stage_conclusion` affects only local pending-review state, never the source experiment JSON. No WebMCP tool can approve, reject, or undo a conclusion: those actions are page-only human controls. The agent can explore broadly and prepare a coherent evidence bundle, but it must stop at the judgment boundary.
The first seven tools are annotated read-only. `stage_conclusion` requires the current state version, refuses to overwrite an existing pending review, and affects only local pending-review statenever the source experiment JSON. No WebMCP tool can approve, reject, edit, or undo a conclusion: those actions are page-only human controls. The agent can explore broadly and prepare a coherent evidence bundle, but it must stop at the judgment boundary.

## Testing

Expand Down
10 changes: 5 additions & 5 deletions docs/observatory.html
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
<link rel="stylesheet" href="styles/60-taskbar.css">
<link rel="stylesheet" href="styles/70-pixel-icons.css">
<link rel="stylesheet" href="styles/95-observatory-cuperos.css?v=20260717.2">
<link rel="stylesheet" href="styles/99-webmcp-mission.css?v=20260903.1">
<script src="observatory.js?v=20260903.1" defer></script>
<script src="webmcp-tools.js?v=20260903.1" defer></script>
<script src="webmcp-mission.js?v=20260903.1" defer></script>
<link rel="stylesheet" href="styles/99-webmcp-mission.css?v=20260903.2">
<script src="observatory.js?v=20260903.2" defer></script>
<script src="webmcp-tools.js?v=20260903.2" defer></script>
<script src="webmcp-mission.js?v=20260903.2" defer></script>
</head>
<body data-depth="freshman" data-data-state="loading" data-mission-mode="active">
<a class="skip-link" href="#observatory-main">Skip to experiment</a>
Expand Down Expand Up @@ -97,7 +97,7 @@ <h2 class="visually-hidden" id="mission-control-title">WebMCP causal mission con
</div>

<div class="mission-utility-actions" aria-label="Mission controls">
<button class="mission-button mission-button--compact" id="mission-tour" type="button" aria-controls="observatory-main pending-changes webmcp-receipts">Tour</button>
<button class="mission-button mission-button--compact" id="mission-tour" type="button" aria-controls="observatory-main pending-changes webmcp-receipts">Local tour</button>
<button class="mission-button mission-button--compact" id="mission-reset" type="button" aria-controls="mission-control pending-changes webmcp-receipts">Reset</button>
</div>
</section>
Expand Down
85 changes: 59 additions & 26 deletions docs/webmcp-mission.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,20 @@
const PROJECTION_URL = "data/webmcp-run-projection-v1.json.gz";
const STORAGE_KEY = "gpustack.webmcp.mission.v1";
const MAX_RECEIPTS = 40;
const CANONICAL_CONCLUSIONS = Object.freeze({
abstain_without_policy_claim: "The observable adaptive controller does not earn a transferable winner claim: it abstained under out-of-distribution stress, and every frozen aggregate gate failed.",
});
const RECEIPT_ORIGIN_LABELS = Object.freeze({
webmcp: "WEBMCP",
local_tour: "LOCAL TOUR",
human: "HUMAN",
});
const REGISTERED_METRICS = Object.freeze([
"final_held_out_nll",
"modeled_completion_seconds",
"inter_site_payload_bytes",
"abstention_count",
"controller_abstentions",
"support_envelope_flag_count",
"replayed_tokens",
"divergence_count",
]);
Expand Down Expand Up @@ -274,12 +283,15 @@
artifact: {
experiment_id: semantic.experiment_id,
sha256: semantic.artifact_sha256,
raw_sha256: semantic.full_trace.raw_trace_artifact.artifact_sha256,
families: families.length,
runs: runLedger(semantic).length,
epochs: semantic.full_trace.raw_trace_artifact.epoch_count || 12981,
},
evidence_boundary: "Measured learning + exact accounting; modeled infrastructure; frontier transfer unresolved.",
evidence_boundary: {
measured: "learning + exact accounting",
modeled: "virtual infrastructure",
unresolved: "frontier/facility transfer",
},
frozen_result: {
conclusion: semantic.status.conclusion,
all_falsifiers_pass: semantic.status.all_falsifiers_pass,
Expand All @@ -288,9 +300,9 @@
},
registered_ids: {
families: families.map((family) => family.family_id),
policies: [...new Set(runLedger(semantic).map((run) => run.policy_id))],
policies: ["observable_adaptive", semantic.comparison.selected_fixed_policy_id],
causal_nodes: screening.causal_graph.nodes.map((node) => node.node_id),
effect_ids: effects.map((effect) => effect.effect_id),
failed_gates: effects.filter((effect) => effect.passed === false).map((effect) => effect.effect_id),
},
pending_proposal: missionState.pending ? missionState.pending.proposalId : null,
suggested_next: "compare_stress_families",
Expand Down Expand Up @@ -426,8 +438,8 @@
split: run.split,
seed: run.seed,
epoch_count: run.epoch_count,
abstention_count: run.abstention_count,
ood_epoch_count: run.out_of_distribution_epoch_count,
controller_abstentions: run.policy_id === "observable_adaptive" ? run.abstention_count : null,
support_envelope_flag_count: run.out_of_distribution_epoch_count,
final_held_out_nll: run.final_held_out_nll,
completion_seconds: run.modeled_infrastructure?.completion_seconds,
inter_site_payload_bytes: run.modeled_infrastructure?.inter_site_payload_bytes,
Expand Down Expand Up @@ -512,7 +524,7 @@
}
if (rawAliases.has(evidenceId)) {
if (navigate) {
await bridge.selectView({ experiment: "E001-SC1", depth: "researcher" });
await bridge.selectView({ experiment: "E001-SC1", depth: "full_trace" });
scrollToId("semantic-consistency-raw-details");
}
return {
Expand Down Expand Up @@ -562,7 +574,8 @@
policy_id: run.policy_id,
final_held_out_nll: run.final_held_out_nll,
completion_seconds: run.modeled_infrastructure?.completion_seconds,
abstentions: run.abstention_count,
controller_abstentions: run.policy_id === "observable_adaptive" ? run.abstention_count : null,
support_envelope_flag_count: run.out_of_distribution_epoch_count,
work_contract_violations: run.exact_accounting?.work_contract_violations || [],
};
}
Expand Down Expand Up @@ -602,7 +615,7 @@
const ledgerEntry = ledgers.find((entry) => [entry.assumption_id, entry.evidence_id, entry.id].includes(evidenceId));
if (ledgerEntry) {
if (navigate) {
await bridge.selectView({ experiment: "E001-SC1", depth: "researcher" });
await bridge.selectView({ experiment: "E001-SC1", depth: "full_trace" });
scrollToId("semantic-consistency-assumptions");
}
return { evidence_id: evidenceId, kind: "evidence_boundary", entry: ledgerEntry };
Expand Down Expand Up @@ -634,7 +647,8 @@
final_held_out_nll: run.final_held_out_nll,
modeled_completion_seconds: run.modeled_infrastructure?.completion_seconds,
inter_site_payload_bytes: run.modeled_infrastructure?.inter_site_payload_bytes,
abstention_count: run.abstention_count,
controller_abstentions: run.policy_id === "observable_adaptive" ? run.abstention_count : null,
support_envelope_flag_count: run.out_of_distribution_epoch_count,
replayed_tokens: run.exact_accounting?.replayed_tokens,
divergence_count: run.diverged ? 1 : 0,
};
Expand All @@ -646,7 +660,7 @@
const ledger = runLedger(semantic);
const availablePolicies = [...new Set(ledger.map((run) => run.policy_id))];
const policyIds = args.policy_ids || ["observable_adaptive", semantic.comparison.selected_fixed_policy_id];
const metricIds = args.metric_ids || REGISTERED_METRICS.slice(0, 5);
const metricIds = args.metric_ids || REGISTERED_METRICS.slice(0, 6);
const unknownPolicy = policyIds.find((id) => !availablePolicies.includes(id));
const unknownMetric = metricIds.find((id) => !REGISTERED_METRICS.includes(id));
if (unknownPolicy) throw new MissionError("UNKNOWN_POLICY", `No registered policy named ${unknownPolicy}.`, { available_ids: availablePolicies });
Expand All @@ -655,10 +669,18 @@
const runs = ledger.filter((run) => run.split === "evaluation" && run.policy_id === policyId);
const metrics = {};
metricIds.forEach((metricId) => {
const values = runs.map((run) => Number(metricValue(run, metricId))).filter(Number.isFinite);
metrics[metricId] = ["abstention_count", "divergence_count", "replayed_tokens"].includes(metricId)
? values.reduce((sum, value) => sum + value, 0)
: median(values);
const values = runs
.map((run) => metricValue(run, metricId))
.filter((value) => value !== null && value !== undefined)
.map(Number)
.filter(Number.isFinite);
if (!values.length) {
metrics[metricId] = null;
} else {
metrics[metricId] = ["controller_abstentions", "support_envelope_flag_count", "divergence_count", "replayed_tokens"].includes(metricId)
? values.reduce((sum, value) => sum + value, 0)
: median(values);
}
});
return { policy_id: policyId, evaluation_runs: runs.length, metrics };
});
Expand All @@ -685,27 +707,33 @@

async function stageConclusion(args, context) {
const semantic = await semanticArtifact(context.signal);
if (args.expected_state_version !== undefined && args.expected_state_version !== missionState.stateVersion) {
if (args.expected_state_version !== missionState.stateVersion) {
throw new MissionError("STALE_STATE", `Expected state version ${args.expected_state_version}, but current version is ${missionState.stateVersion}.`, {
current_state_version: missionState.stateVersion,
});
}
if (args.confidence === "supported" && semantic.status.all_falsifiers_pass !== true) {
throw new MissionError("EVIDENCE_CONFLICT", "A supported conclusion is not admissible: all four frozen aggregate gates failed. Use qualified or abstain and cite the failed gates.", {
if (missionState.pending) {
throw new MissionError("PENDING_REVIEW_EXISTS", `Human review is already pending for ${missionState.pending.proposalId}. Approve or reject it before staging another conclusion.`, {
pending_proposal: missionState.pending.proposalId,
});
}
if (args.conclusion_code !== semantic.status.conclusion || !CANONICAL_CONCLUSIONS[args.conclusion_code]) {
throw new MissionError("EVIDENCE_CONFLICT", `The immutable artifact serializes ${semantic.status.conclusion}; a different conclusion cannot be staged.`, {
frozen_conclusion: semantic.status.conclusion,
recommended_confidence: "abstain",
});
}
const resolved = [];
for (const evidenceId of args.evidence_ids) {
resolved.push(await resolveEvidence(evidenceId, "researcher", false, context.signal));
}
abortIfNeeded(context.signal);
const proposal = {
proposalId: `proposal-${String(missionState.nextProposal).padStart(3, "0")}`,
claim: args.claim,
claim: CANONICAL_CONCLUSIONS[args.conclusion_code],
conclusionCode: args.conclusion_code,
evidenceIds: [...args.evidence_ids],
evidenceKinds: resolved.map((entry) => entry.kind),
confidence: args.confidence,
confidence: "abstain",
frozenConclusion: semantic.status.conclusion,
createdAt: new Date().toISOString(),
};
Expand All @@ -725,6 +753,7 @@
ok: true,
proposal_id: proposal.proposalId,
status: "pending_human_review",
conclusion_code: proposal.conclusionCode,
confidence: proposal.confidence,
evidence_ids: proposal.evidenceIds,
frozen_conclusion: proposal.frozenConclusion,
Expand All @@ -751,7 +780,7 @@
try {
abortIfNeeded(context.signal);
const result = await handler(args, context);
abortIfNeeded(context.signal);
if (toolName !== "stage_conclusion") abortIfNeeded(context.signal);
refreshRegistrationStatus();
return result;
} catch (error) {
Expand Down Expand Up @@ -827,7 +856,8 @@
item.className = "webmcp-receipt";
item.dataset.status = receipt.status;
const heading = document.createElement("strong");
heading.textContent = `${receipt.tool.replaceAll("_", " ")} · ${receipt.status}`;
const originLabel = RECEIPT_ORIGIN_LABELS[receipt.origin] || String(receipt.origin || "webmcp").toUpperCase();
heading.textContent = `${originLabel} · ${receipt.tool.replaceAll("_", " ")} · ${receipt.status}`;
const summary = document.createElement("p");
summary.textContent = receipt.summary;
const metadata = document.createElement("p");
Expand Down Expand Up @@ -958,14 +988,13 @@
await invoke("open_evidence", { evidence_id: "adaptive_minus_best_fixed_final_nll", semantic_depth: "researcher" }, { origin: "local_tour" });
await pause(220);
await invoke("stage_conclusion", {
claim: "The observable adaptive controller does not earn a transferable winner claim: it abstained under out-of-distribution stress, and every frozen aggregate gate failed.",
conclusion_code: "abstain_without_policy_claim",
evidence_ids: [
"E6-repeated-membership-loss",
"adaptive_minus_best_fixed_final_nll",
"adaptive_to_best_fixed_inter_site_payload_ratio",
"adaptive_to_best_fixed_modeled_completion_time_ratio",
],
confidence: "abstain",
expected_state_version: missionState.stateVersion,
}, { origin: "local_tour" });
setStatus("Audit staged · waiting for human approval", "waiting");
Expand All @@ -990,6 +1019,10 @@
}

function refreshRegistrationStatus(forcedState) {
if (!forcedState && missionState.pending) {
setStatus("Audit staged · waiting for human approval", "waiting");
return;
}
const webmcp = window.GPUStackWebMCP;
if (webmcp?.supported) {
webmcp.ready.then((status) => {
Expand Down
40 changes: 15 additions & 25 deletions docs/webmcp-tools.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
const MAX_RESULT_CHARS = 1500;
const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:/-]{0,179}$/;
const SEMANTIC_DEPTHS = ["freshman", "researcher", "full_trace"];
const CONFIDENCE_LEVELS = ["supported", "qualified", "abstain"];
const CONCLUSION_CODES = ["abstain_without_policy_claim"];

class ArgumentError extends Error {
constructor(field, message, expected) {
Expand Down Expand Up @@ -137,28 +137,22 @@

stage_conclusion: objectSchema(
{
claim: {
conclusion_code: {
type: "string",
minLength: 1,
maxLength: 600,
description: "Concise proposed conclusion grounded only in the cited evidence IDs.",
enum: CONCLUSION_CODES,
description: "Typed conclusion serialized by the immutable artifact. Free-form agent claims are not accepted.",
},
evidence_ids: idArraySchema(
"One to eight evidence IDs that directly support or qualify the proposed claim.",
"One to eight evidence IDs that directly support the typed artifact conclusion.",
8,
),
confidence: {
type: "string",
enum: CONFIDENCE_LEVELS,
description: "Supported, qualified, or abstain. This is evidence confidence, not approval.",
},
expected_state_version: {
type: "integer",
minimum: 0,
description: "Optional optimistic-concurrency version returned by a prior tool call.",
description: "Required optimistic-concurrency version returned by get_observatory_state.",
},
},
["claim", "evidence_ids", "confidence"],
["conclusion_code", "evidence_ids", "expected_state_version"],
),
});

Expand Down Expand Up @@ -313,16 +307,12 @@
},

stage_conclusion(args) {
checkObject(args, ["claim", "evidence_ids", "confidence", "expected_state_version"]);
const result = {
claim: cleanString(args.claim, "claim", { required: true, min: 1, max: 600 }),
checkObject(args, ["conclusion_code", "evidence_ids", "expected_state_version"]);
return {
conclusion_code: cleanEnum(args.conclusion_code, "conclusion_code", CONCLUSION_CODES),
evidence_ids: cleanIdArray(args.evidence_ids, "evidence_ids", 8),
confidence: cleanEnum(args.confidence, "confidence", CONFIDENCE_LEVELS),
expected_state_version: cleanInteger(args.expected_state_version, "expected_state_version", 0),
};
if (args.expected_state_version !== undefined) {
result.expected_state_version = cleanInteger(args.expected_state_version, "expected_state_version", 0);
}
return result;
},
});

Expand Down Expand Up @@ -354,7 +344,7 @@
{
name: "inspect_run",
title: "Inspect experiment run",
description: "Inspect one exact E001-SC1 run and a bounded page of scalar-projected optimizer-commit epochs. Returns mode choice, OOD and abstention state, completion, and event markers while preserving the authoritative raw-trace hash.",
description: "Inspect one exact E001-SC1 run and a bounded page of scalar-projected optimizer-commit epochs. Separates support-envelope flags from controller-only abstentions while preserving the authoritative raw-trace hash.",
inputSchema: SCHEMAS.inspect_run,
annotations: READ_ONLY,
},
Expand All @@ -375,14 +365,14 @@
{
name: "compare_policies",
title: "Compare registered policies",
description: "Compare up to three policies from the immutable experiment artifact. Omit IDs for observable_adaptive versus the calibration-frozen periodic_local comparator across registered metrics.",
description: "Compare up to three policies from the immutable artifact. Separates controller-only abstentions from cross-policy support-envelope flags; defaults to observable_adaptive versus frozen periodic_local.",
inputSchema: SCHEMAS.compare_policies,
annotations: READ_ONLY,
},
{
name: "stage_conclusion",
title: "Stage evidence conclusion",
description: "Stage a supported, qualified, or abstain conclusion with explicit evidence IDs in the visible pending tray. This never approves or commits it; only the human can approve or reject it in the page.",
description: "Stage the artifact's typed abstain conclusion with explicit evidence IDs and a current state version. Free-form agent claims are rejected. Only the human can approve, edit, or reject it.",
inputSchema: SCHEMAS.stage_conclusion,
annotations: STAGING_WRITE,
},
Expand Down Expand Up @@ -485,7 +475,7 @@

try {
const result = await bridge.invoke(toolName, validated, { signal });
abortIfNeeded(signal);
if (toolName !== "stage_conclusion") abortIfNeeded(signal);
return compactResult(toolName, result);
} catch (error) {
abortIfNeeded(signal);
Expand Down
Loading
Loading