diff --git a/.github/workflows/web-ci.yml b/.github/workflows/web-ci.yml index 8ba26f4d..6f171684 100644 --- a/.github/workflows/web-ci.yml +++ b/.github/workflows/web-ci.yml @@ -269,10 +269,13 @@ jobs: EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE public.%I TO forge_app_test', table_name); END LOOP; REVOKE ALL ON TABLE public.execution_outcomes, public.operation_runs, - public.operation_run_events FROM forge_app_test; + public.operation_run_events, public.capability_attempts, + public.capability_attempt_adjudications FROM forge_app_test; GRANT SELECT, INSERT, UPDATE ON TABLE public.execution_outcomes, public.operation_runs TO forge_app_test; GRANT SELECT, INSERT ON TABLE public.operation_run_events TO forge_app_test; + GRANT SELECT, INSERT ON TABLE public.capability_attempts, + public.capability_attempt_adjudications TO forge_app_test; GRANT USAGE, SELECT ON SEQUENCE public.task_logs_sequence_seq TO forge_app_test; END; $grant_s4_application_acl$; @@ -512,7 +515,8 @@ jobs: 'app_settings', 'task_questions' ]; operation_ledger_tables constant text[] := ARRAY[ - 'execution_outcomes', 'operation_runs', 'operation_run_events' + 'execution_outcomes', 'operation_runs', 'operation_run_events', + 'capability_attempts', 'capability_attempt_adjudications' ]; protected_tables constant text[] := ARRAY[ 'forge_release_signer_keys', 'forge_release_signer_key_lifecycle_audits', @@ -559,6 +563,8 @@ jobs: GRANT SELECT, INSERT, UPDATE ON TABLE public.execution_outcomes, public.operation_runs TO forge_app_test; GRANT SELECT, INSERT ON TABLE public.operation_run_events TO forge_app_test; + GRANT SELECT, INSERT ON TABLE public.capability_attempts, + public.capability_attempt_adjudications TO forge_app_test; FOREACH table_name IN ARRAY operation_ledger_tables LOOP FOREACH table_privilege IN ARRAY ARRAY[ 'SELECT', 'INSERT', 'UPDATE', 'DELETE', 'TRUNCATE', 'REFERENCES', 'TRIGGER' @@ -568,7 +574,7 @@ jobs: ) IS DISTINCT FROM ( (table_name IN ('execution_outcomes', 'operation_runs') AND table_privilege IN ('SELECT', 'INSERT', 'UPDATE')) - OR (table_name = 'operation_run_events' + OR (table_name IN ('operation_run_events', 'capability_attempts', 'capability_attempt_adjudications') AND table_privilege IN ('SELECT', 'INSERT')) ) THEN RAISE EXCEPTION 'ordinary app has unexpected % on operation ledger table public.%', diff --git a/docs/adr/0012-capability-reliability-ledger.md b/docs/adr/0012-capability-reliability-ledger.md new file mode 100644 index 00000000..9135c5db --- /dev/null +++ b/docs/adr/0012-capability-reliability-ledger.md @@ -0,0 +1,98 @@ +# ADR 0012: Capability reliability ledger + +## Status + +Accepted. Implemented in migration `0031_capability_reliability_ledger.sql`. + +Primary design document: +`docs/architecture/issue-186-capability-reliability-ledger.md`. + +## Context + +ADR 0010 gave Forge a canonical record of *what happened* on one attempt. ADR +0011 gave it deterministic typed operations. Neither answers whether a specific +kind of work has repeatedly succeeded under comparable conditions, which is the +evidence issue #189 will need before it may widen any permission. + +A single reliability score per agent or model would hide the differences that +matter — scope, project, model, harness, and policy — and would encourage exactly +the unsafe promotion this Epic exists to prevent. + +## Decision + +Forge records an append-only ledger of individual **capability attempts** and +computes reliability metrics per **cohort**, on demand, from those attempts. + +A cohort is the domain-separated SHA-256 fingerprint of project, capability key, +scope, runtime/model, and policy. Because a material change to any of those +produces a different cohort, requalification is automatic: new conditions start a +new sample count, and prior evidence is retained but no longer counted. The four +component fingerprints are stored beside the cohort fingerprint so drift is +attributable to a specific input rather than only detectable. + +Capability keys are namespaced — `workpackage:/` from the +existing `CAPABILITY_TAXONOMY`, or `operation:@` from the ADR 0011 +catalog — so model-executed work and deterministic operations can never share a +cohort. One work package writes one attempt row per capability it exercised, all +sharing an `attempt_group_id` and a multiplicity count, so per-capability and +per-attempt views are both available without double-counting. When the Architect +classification is missing or exceeds the fan-out bound, a single reserved +`unclassified` row records the gap instead of guessing a capability. + +`capability_attempts` is immutable: identity and the ingest-time outcome +snapshot cannot be updated or deleted, enforced by a database trigger. Evidence +that arrives later — verification results, human decisions, rollbacks, overrides, +and detected drift — is appended to `capability_attempt_adjudications` in gapless +sequence order, never written back into the attempt. + +Whether an attempt counts as verified is decided by a closed `verification_mode`. +`self_reported` and `human_review` never contribute to the independently verified +pass rate; only `deterministic_adapter` (ADR 0011) and `independent_agent` do. +`independent_agent` has no producer until issue #188, so v1 rejects it at ingest +rather than allowing an unbacked value to be stored. Forge's current honest +answer for most cohorts is reported explicitly as an unverified-completion rate +instead of being folded into a pass rate. + +The ledger has no free-text column. Every `text` column is a closed enum, a +64-hex fingerprint, or the bounded capability-key grammar, each enforced by a +`CHECK` constraint. Model prose, file paths, repository-relative names, and +credentials therefore cannot enter the ledger even by mistake, and no redaction +helper is needed on this path. Scope is fingerprinted from the project's opaque +`root_ref` and revisions, never from `local_path`. + +Metrics are a pure function of stored attempts, adjudications, a window, and an +explicit `now`. No materialized summary is stored in v1: a cache that can +disagree with its evidence is a class of bug this ledger exists to avoid, and the +cohort index makes on-demand computation a bounded scan. Below a minimum sample +size a cohort reports `insufficient_evidence` with null rates; if any in-window +attempt's linked outcome has changed since ingest, the cohort reports +`evidence_drift` and suppresses all rates. Critical failures are reported +unconditionally in every state, so no aggregate can conceal one. + +Ingest hangs off the existing canonical-outcome boundaries — the three work +package handoff sites and, after its transaction commits, the ADR 0011 operation +finalize path. Writes are best-effort and idempotent on +`(execution_outcome_id, capability_key)`: a ledger failure never fails a task, +package, run, or operation, and a recovered worker re-running a boundary writes +nothing new. Historical attempts are not backfilled; a missing attempt is +unavailable evidence, never success. The ordinary application role receives +`SELECT` and `INSERT` on the ledger tables and nothing else. + +## Consequences + +Issues #188, #189, #190, and #191 read this contract instead of deriving trust +from statuses, free-text errors, or a worker's own account of its performance. +Autonomy decisions in #189 can cite a cohort, a sample size, a verification mode, +and the evidence rows behind each number. + +This ADR grants no autonomy and changes no permission. It adds no dashboard, HTTP +route, scheduled job, or background recomputation. It does not produce +independent verification, and it does not replace the task, work package, agent +run, artifact, execution outcome, operation run, or approval gate records that +remain authoritative for their own state. + +Two capabilities are defined but not yet producible: rollback and override +adjudications have storage contracts and metrics but no writer until #189/#190, +and independent-agent verification is refused until #188. Both are deliberate — +the storage shape is stable, and the gaps are visible rather than filled with +optimistic defaults. diff --git a/docs/architecture/issue-186-capability-reliability-ledger.md b/docs/architecture/issue-186-capability-reliability-ledger.md new file mode 100644 index 00000000..1242fcdf --- /dev/null +++ b/docs/architecture/issue-186-capability-reliability-ledger.md @@ -0,0 +1,957 @@ +# Issue #186 Architecture: Capability Reliability Ledger + +Status: **Implemented.** Migration `0031_capability_reliability_ledger.sql` and +the ingest/read paths described below have landed. Rollback and override +adjudications remain storage-only until #189/#190 add producers, and +`independent_agent` verification remains refused at ingest until #188 lands +(§7.5, §15). + +| Field | Value | +|---|---| +| Issue | [#186 — Add capability reliability ledger](https://github.com/Joncallim/Forge/issues/186) | +| Parent Epic | [#184 — Continuous verification and earned autonomy](https://github.com/Joncallim/Forge/issues/184) | +| Roadmap phase | Phase 2 of `docs/continuous-verification-and-earned-autonomy-roadmap.md` | +| Depends on | [#185](https://github.com/Joncallim/Forge/issues/185) (landed — ADR 0010, `execution_outcomes`), [#201](https://github.com/Joncallim/Forge/issues/201) (landed — ADR 0011, `operation_runs`) | +| Consumed by | #188 (independent verification), #189 (autonomy policy), #190 (Sentinel), #191 (reporting) | +| Companion ADR | `docs/adr/0012-capability-reliability-ledger.md` | +| Implementation scope | Large — data model, ingest, deterministic aggregation, tests | +| Intended executor | Backend specialist, with QA and Documentation packages | + +--- + +## 1. Plain-language summary + +Forge already writes down **what happened** on every attempt: a canonical +"execution outcome" row that says whether work completed, was refused, was +blocked, or failed, and why (ADR 0010). It also records deterministic operation +runs (ADR 0011). + +What Forge cannot answer today is the follow-up question: **"has this particular +kind of work, in this particular project, under this particular model and +policy, actually worked before — and was that checked by someone other than the +worker that did it?"** + +This issue builds the record that answers it. We call it the **capability +reliability ledger**. + +The important word is *comparable*. A pass rate is only meaningful if the things +being counted are alike. "The backend agent is 92% reliable" is a dangerous +number: it silently mixes a trivial README edit with a database migration, a +local 7-billion-parameter model with a frontier model, and a project with strict +review gates with one that has none. So the ledger never stores one score per +agent. It stores **individual attempts**, each tagged with a **cohort** — the +exact combination of project, capability, scope, runtime/model, and policy the +attempt ran under. Metrics are calculated per cohort, on demand, from those +stored attempts. + +Three rules shape everything below: + +1. **The ledger stores evidence, not opinions.** Every column is a UUID, a + closed enum code, an integer count, a hash, or a timestamp. There is no + free-text column anywhere in it — so a model's prose, a file path, or a + secret cannot leak into it even by accident. +2. **A worker cannot mark its own homework as verified.** Whether an attempt + counts as "independently verified" is decided by *who checked it*, recorded + as a closed `verification_mode`. A worker's self-assessment is recorded as + `self_reported` and is never counted toward a verified pass rate. +3. **Absence is never success.** A missing attempt, a missing outcome, a + too-small sample, or evidence that has drifted since it was recorded all + produce an explicit "cannot tell" state — never an optimistic number. + +This issue does **not** grant anyone more permission. It builds the evidence +that #189 will later be allowed to reason about. + +--- + +## 2. Objective + +Record an append-only, evidence-backed history of comparable capability attempts, +and calculate transparent reliability metrics from it deterministically, without +letting the execution worker grade itself and without collapsing materially +different work into one number. + +## 3. Non-goals + +These are explicitly *not* in this slice. An implementer who finds themselves +building one of these has left scope and must stop. + +- **Granting, holding, promoting, demoting, or revoking autonomy.** That is #189. + Nothing in this slice may read the ledger to change what an agent is permitted + to do. +- **An operator dashboard or HTTP API.** That is #191. This slice ships no route + under `web/app/api`, no page, and no component. +- **A global agent/model/workforce score.** Structurally prevented: metrics are + only ever returned per cohort. +- **Producing independent verification.** That is #188. This slice defines how an + independent verification result would be recorded, and refuses to fabricate one + in the meantime. +- **Backfilling historical attempts.** Consistent with ADR 0010, history before + this table exists is unavailable evidence, not success. +- **Scheduled or background recomputation.** No Redis job, no cron, no worker + loop. Metrics are computed when a caller asks. +- **Replacing `tasks`, `task_attempts`, `work_packages`, `agent_runs`, + `artifacts`, `execution_outcomes`, `operation_runs`, or approval gates.** Those + remain authoritative for their own state; the ledger only links to them. + +--- + +## 4. Core invariants + +Each invariant below has a proving test named in §12. An implementation that +cannot prove one of these has not met the contract. + +| # | Invariant | Enforced by | +|---|---|---| +| I1 | The ledger contains no free-text column. Every `text` column is a closed enum, a 64-hex fingerprint, or the bounded capability-key grammar. | `CHECK` constraints + schema-text test | +| I2 | One attempt row exists per `(execution_outcome_id, capability_key)`. Re-ingesting the same attempt changes nothing. | `UNIQUE` index + `ON CONFLICT DO NOTHING` | +| I3 | Attempt identity and its ingest-time snapshot are immutable. `UPDATE`/`DELETE` on `capability_attempts` is rejected by the database. | append-only trigger | +| I4 | Later evidence (verification, human decision, rollback, drift) is appended as adjudication rows, never written back into the attempt. | separate table + append-only trigger | +| I5 | Attempts whose cohort inputs differ are never counted together, and the difference is attributable to one of four component fingerprints. | `cohort_fingerprint` column + component columns | +| I6 | `verification_mode` decides what counts as verified. `self_reported` and `human_review` never contribute to the independently-verified rate. | pure metrics function | +| I7 | A critical failure is always reported, regardless of sample size, window, or aggregate rate. | metrics function returns `criticalFailureCount` unconditionally | +| I8 | If a linked `execution_outcomes` row changed after ingest, the cohort reports `evidence_drift` and suppresses all rates. | `outcome_digest` comparison at read | +| I9 | Below `minSamples`, the cohort reports `insufficient_evidence` with null rates. Never a rate derived from one or two attempts. | metrics function | +| I10 | Ledger write failure never fails a task, work package, agent run, or operation run. | best-effort ingest wrappers | +| I11 | Metrics are a pure function of `(attempts, adjudications, window, now)`. No clock, no database, no I/O. | unit test with fixed inputs | +| I12 | The ordinary application role holds `SELECT, INSERT` on ledger tables and nothing else — no `UPDATE`, no `DELETE`. | CI closed-ACL inventory gate | + +--- + +## 5. Domain contracts + +All types live in `web/lib/reliability/contracts.ts` (importable by both `web/app` +and `web/worker`, no database imports). + +### 5.1 Ledger contract version + +```ts +export const RELIABILITY_LEDGER_CONTRACT_VERSION = 1 as const +``` + +Bump this when the *meaning* of any cohort input or metric changes. Because the +version is a cohort input (§5.4), bumping it starts fresh cohorts rather than +silently redefining historical numbers. Never edit historical rows to match a +new version. + +### 5.2 Capability key + +A capability key names *what kind of work was attempted*. It is namespaced so +model-executed work packages and deterministic operations can never share a +cohort. + +```text +workpackage:/ e.g. workpackage:backend/api-implementation +operation:@ e.g. operation:repository.status.read@1 +``` + +- `` is `work_packages.assigned_role`, lowercased and slug-normalized. +- `` is a member of `CAPABILITY_TAXONOMY` + (`web/worker/capability-classification.ts`). Reuse that taxonomy; do not + invent a second one. +- `@` is the exact catalog identity from ADR 0011. A new + operation version is a new capability key by construction. + +```ts +export const CAPABILITY_KEY_PATTERN = + /^(?:workpackage:[a-z][a-z0-9-]{0,39}\/[a-z][a-z0-9-]{0,39}|operation:[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+@[1-9][0-9]{0,3})$/ +export const CAPABILITY_KEY_MAX_LENGTH = 120 +``` + +The same regex is enforced as a PostgreSQL `CHECK`. The pattern is the contract; +the application must not be the only thing holding the line. + +### 5.3 Fan-out, multiplicity, and the unclassified escape hatch + +One work package usually requires several capabilities. Autonomy decisions in +#189 are per capability, so an attempt must be attributable to each capability it +exercised. + +**Rule:** one attempt row per `(execution_outcome_id, capability_key)`. All rows +produced from one outcome share an `attempt_group_id` and carry +`capability_multiplicity = `. + +This gives two honest views without double-counting: + +- Per-capability view (used by #189): count rows. +- Per-attempt view (used by #191): collapse on `attempt_group_id`, or divide by + `capability_multiplicity`. + +`ReliabilitySummary` reports both `sampleCount` (rows) and `uniqueAttemptCount` +(distinct groups), so a reader can never mistake one failed package covering +five capabilities for five independent failures. + +Bounds and failure modes: + +```ts +export const MAX_CAPABILITY_FAN_OUT = 12 +export type CapabilityClassificationState = 'classified' | 'missing' | 'overflow' +``` + +- `missing` — the Architect produced no usable capability classification. +- `overflow` — more than `MAX_CAPABILITY_FAN_OUT` capabilities were declared. + +In both cases write exactly one row with capability key +`workpackage:/unclassified` and the matching `classification_state`. +`unclassified` is reserved: it is a member of no specific capability cohort and +is excluded from every capability metric, but it **is** counted and reported in +`ReliabilitySummary.excluded` so the measurement gap is visible rather than +silently absent. Never guess a capability to fill the gap. + +### 5.4 Cohort fingerprints + +```ts +export function reliabilityFingerprint(domain: string, value: unknown): string +``` + +Implement with the same construction as `operationFingerprint` +(`web/lib/operations/contracts.ts`): domain-separated SHA-256 over +`canonicalJson`. Reuse `canonicalJson` and `isPlainRecord` from that module +rather than writing a second canonicalizer. + +Four component fingerprints, then the cohort fingerprint over all of them: + +**Scope** — `reliabilityFingerprint('scope', …)` over: + +```ts +{ + contractVersion: 1, + projectId: string, + rootRef: string | null, // opaque project identity, never a path + rootBindingRevision: string, // bigint as canonical decimal string + grantDecisionRevision: string, // bigint as canonical decimal string + repositoryWriteIntent: boolean, + capabilities: string[], // sorted, de-duplicated + mcpRequirementKeys: string[], // sorted, de-duplicated +} +``` + +Never include `projects.local_path`, any repository-relative path, any file +name, or any excerpt. ADR 0008/0009 forbid persisting those, and `root_ref` is +the approved opaque substitute. Re-ordering or repeating capabilities must not +change the fingerprint (sorted + de-duplicated). + +**Runtime** — `reliabilityFingerprint('runtime', …)` over one of: + +```ts +{ kind: 'model', providerType, modelId, providerIsLocal, providerConfigUpdatedAt, acpExecutionMode } +{ kind: 'deterministic_adapter', adapterKind } +``` + +Model fields come from the snapshot columns that already exist on `agent_runs` +(`provider_type_used`, `model_id_used`, `provider_is_local_used`, +`provider_config_updated_at_used`, `acp_execution_mode`). Do not re-read live +provider config: the snapshot is what actually ran. + +**Policy** — `reliabilityFingerprint('policy', …)` over: + +```ts +{ + contractVersion: 1, + policyVersion: RELIABILITY_POLICY_VERSION, // code constant, bumped on meaning change + harnessId: string | null, + harnessUpdatedAt: string | null, // ISO 8601, or null + reviewRequirement: 'none' | 'qa_only' | 'reviewer_only' | 'both', + repositoryWritesEnabled: boolean, +} +``` + +**Cohort** — `reliabilityFingerprint('cohort', { contractVersion, projectId, capabilityKey, scopeFingerprint, runtimeFingerprint, policyFingerprint })`. + +Storing the components alongside the cohort makes drift *attributable*: when a +cohort changes, a reader can say "the model changed" rather than only "something +changed". That attribution is what makes requalification in #189 explainable. + +**Requalification is implicit and automatic.** A material change to the model, +harness, policy, project root binding, or capability set produces a different +`cohort_fingerprint`, so the new attempts land in a new cohort with a fresh +sample count. No migration, flag, or manual reset is involved — and old evidence +is never destroyed, only no longer counted toward the new cohort. + +### 5.5 Verification mode + +```ts +export const VERIFICATION_MODES = [ + 'none', // verification was not required + 'self_reported', // the worker asserted its own success — never counted as verified + 'human_review', // a human decided an approval gate + 'deterministic_adapter', // machine-checked output (ADR 0011 operations) + 'independent_agent', // a separate verifier agent run — produced by #188 +] as const +``` + +`independent_agent` is defined now and **rejected at ingest in v1** with a +`reliability_verification_mode_unavailable` error, because no producer exists yet +(#188). Defining it early keeps the storage contract stable; rejecting it keeps +the ledger honest. When #188 lands it removes that guard in one place. + +`verification_status` mirrors ADR 0010 exactly: +`'not_required' | 'pending' | 'passed' | 'failed' | 'inconclusive'`. + +### 5.6 Severity + +```ts +export type SeverityClass = 'normal' | 'critical' +``` + +An attempt is `critical` when any of these hold at ingest: + +- `stop_reason_code` is `security_blocked` or `policy_blocked`; +- the attempt had `repositoryWriteIntent` and its validation commands failed + (`validation_command_failed > 0`); +- the linked operation run terminated `blocked` on a policy or preflight phase. + +A later `rollback_recorded` adjudication also makes the attempt critical for +metric purposes; the stored attempt row is not rewritten (I3/I4), the metrics +function derives it. Critical counts are reported unconditionally (I7). + +### 5.7 Summary contract + +```ts +export type ReliabilityWindow = { + maxAttempts: number // most recent N rows in the cohort + maxAgeMs: number // ignore rows older than this + minSamples: number // below this, no rates are produced +} + +export const DEFAULT_RELIABILITY_WINDOW: ReliabilityWindow = { + maxAttempts: 50, + maxAgeMs: 90 * 24 * 60 * 60 * 1000, + minSamples: 5, +} + +export type ReliabilityState = 'ready' | 'insufficient_evidence' | 'evidence_drift' + +export type ReliabilitySummary = { + schemaVersion: 1 + cohortFingerprint: string + capabilityKey: string + state: ReliabilityState + sampleCount: number + uniqueAttemptCount: number + // Every rate is null unless state === 'ready'. + rates: { + firstAttemptSuccess: number | null + independentlyVerifiedPass: number | null + humanAccepted: number | null + unverifiedCompletion: number | null + repairRetry: number | null + humanRejection: number | null + rollback: number | null + policyBlock: number | null + } + consecutiveVerifiedPasses: number + // Always populated, whatever the state. + criticalFailureCount: number + lastCriticalAt: string | null + evidence: { + newestObservedAt: string | null + oldestObservedAt: string | null + freshnessMs: number | null + driftedAttemptCount: number + } + excluded: Array<{ reason: 'outside_window' | 'unclassified' | 'drifted'; count: number }> +} +``` + +Rate definitions — each is `numerator / denominator`, and a denominator of zero +yields `null`, never `0` and never `1`: + +| Rate | Numerator | Denominator | +|---|---|---| +| `firstAttemptSuccess` | `attempt_number = 1` and `result = 'completed'` | `attempt_number = 1` | +| `independentlyVerifiedPass` | latest verification adjudication has mode ∈ {`deterministic_adapter`, `independent_agent`} and result `passed` | attempts with `verifier_required = true` | +| `humanAccepted` | latest human decision is `accepted` | attempts with at least one human decision | +| `unverifiedCompletion` | `result = 'completed'` with no independent verification | `result = 'completed'` | +| `repairRetry` | `attempt_number > 1` | all in-window rows | +| `humanRejection` | latest human decision is `rejected` | attempts with at least one human decision | +| `rollback` | has a `rollback_recorded` adjudication | all in-window rows | +| `policyBlock` | `result = 'blocked'` | all in-window rows | + +`unverifiedCompletion` exists on purpose. Until #188 lands, Forge's honest answer +for most cohorts is "completed, but nobody independent checked it" — and that +number should be visible instead of being quietly folded into a pass rate. + +`consecutiveVerifiedPasses` counts backwards from the newest in-window row and +stops at the first row that is not an independently verified pass. It is `0` +under `insufficient_evidence` or `evidence_drift`. + +--- + +## 6. Persistence + +Migration `0031_capability_reliability_ledger.sql`. Follow the exact style of +`0030_operation_runs.sql`: inline `CONSTRAINT` clauses, `--> statement-breakpoint` +separators, guard functions with `SET search_path = pg_catalog, public`, and +`REVOKE ALL ON FUNCTION … FROM PUBLIC` immediately after every `CREATE FUNCTION`. + +### 6.1 `capability_attempts` + +Immutable evidence, one row per `(execution_outcome_id, capability_key)`. + +| Column | Type | Notes | +|---|---|---| +| `id` | `uuid` PK | | +| `attempt_group_id` | `uuid NOT NULL` | shared by all rows from one outcome | +| `project_id` | `uuid NOT NULL` → `projects` `ON DELETE restrict` | | +| `task_id` | `uuid NOT NULL` → `tasks` `ON DELETE restrict` | | +| `work_package_id` | `uuid` → `work_packages` `ON DELETE set null` | null for pre-package admission blocks | +| `agent_run_id` | `uuid` → `agent_runs` `ON DELETE set null` | | +| `task_attempt_id` | `uuid` → `task_attempts` `ON DELETE set null` | | +| `execution_outcome_id` | `uuid NOT NULL` → `execution_outcomes` `ON DELETE restrict` | the ADR 0010 anchor | +| `operation_run_id` | `uuid` → `operation_runs` `ON DELETE set null` | set for ADR 0011 attempts | +| `contract_version` | `integer NOT NULL DEFAULT 1` | `CHECK = 1` | +| `capability_key` | `text NOT NULL` | `CHECK` regex + `length(…) <= 120` | +| `classification_state` | `text NOT NULL` | `CHECK IN ('classified','missing','overflow')` | +| `capability_multiplicity` | `integer NOT NULL` | `CHECK BETWEEN 1 AND 12` | +| `cohort_fingerprint` | `text NOT NULL` | `CHECK ~ '^[0-9a-f]{64}$'` | +| `scope_fingerprint` | `text NOT NULL` | same | +| `runtime_fingerprint` | `text NOT NULL` | same | +| `policy_fingerprint` | `text NOT NULL` | same | +| `outcome_digest` | `text NOT NULL` | fingerprint of the normalized outcome at ingest | +| `transport_status` | `text NOT NULL` | `CHECK IN ('ok','error')` | +| `result` | `text NOT NULL` | same closed set as `execution_outcomes.result` | +| `stop_reason_code` | `text` | `NULL` or the ADR 0010 closed taxonomy | +| `retryable` | `boolean NOT NULL` | | +| `attempt_number` | `integer NOT NULL DEFAULT 1` | `CHECK >= 1`; from `agent_runs.attempt_number`, coalesced to `1` when null or absent | +| `severity_class` | `text NOT NULL` | `CHECK IN ('normal','critical')` | +| `verifier_required` | `boolean NOT NULL` | mirrored from the outcome | +| `verification_mode` | `text NOT NULL` | `CHECK IN (…)`; `independent_agent` allowed by the column, refused by the application in v1 | +| `verification_status` | `text NOT NULL` | ADR 0010 closed set | +| `acceptance_criteria_total` | `integer NOT NULL DEFAULT 0` | `CHECK >= 0` | +| `validation_command_total` | `integer NOT NULL DEFAULT 0` | `CHECK >= 0` | +| `validation_command_failed` | `integer NOT NULL DEFAULT 0` | `CHECK >= 0 AND <= validation_command_total` | +| `evidence_refs` | `jsonb NOT NULL DEFAULT '[]'` | UUIDs only; `CHECK jsonb_typeof = 'array'` | +| `observed_at` | `timestamptz NOT NULL` | outcome time — the window axis | +| `created_at` | `timestamptz NOT NULL DEFAULT now()` | ingest time | + +Consistency `CHECK`s that must be in the migration, not only in TypeScript: + +```sql +CONSTRAINT "capability_attempts_verifier_consistency_check" CHECK ( + (verifier_required AND verification_status IN ('pending','passed','failed','inconclusive')) + OR (NOT verifier_required AND verification_status = 'not_required') +), +CONSTRAINT "capability_attempts_verification_mode_check" CHECK ( + (verification_mode = 'none') = (NOT verifier_required) +), +CONSTRAINT "capability_attempts_unclassified_check" CHECK ( + (classification_state = 'classified') OR capability_key LIKE 'workpackage:%/unclassified' +), +CONSTRAINT "capability_attempts_operation_runtime_check" CHECK ( + operation_run_id IS NULL OR verification_mode IN ('none','deterministic_adapter') +) +``` + +Indexes: + +```sql +CREATE UNIQUE INDEX "capability_attempts_outcome_capability_idx" + ON "capability_attempts" ("execution_outcome_id", "capability_key"); +CREATE INDEX "capability_attempts_cohort_observed_at_idx" + ON "capability_attempts" ("cohort_fingerprint", "observed_at" DESC); +CREATE INDEX "capability_attempts_project_capability_idx" + ON "capability_attempts" ("project_id", "capability_key"); +CREATE INDEX "capability_attempts_attempt_group_idx" + ON "capability_attempts" ("attempt_group_id"); +CREATE INDEX "capability_attempts_execution_outcome_idx" + ON "capability_attempts" ("execution_outcome_id"); +``` + +The cohort index is the one that matters for read latency: every metrics query is +`WHERE cohort_fingerprint = $1 ORDER BY observed_at DESC LIMIT $2`. + +Append-only guard (mirrors `forge_guard_operation_run_history_v1`): + +```sql +CREATE FUNCTION "forge_reject_capability_attempt_mutation_v1"() RETURNS trigger +LANGUAGE plpgsql SET search_path = pg_catalog, public AS $$ +BEGIN + RAISE EXCEPTION 'capability attempts are append-only'; +END; +$$; +REVOKE ALL ON FUNCTION public.forge_reject_capability_attempt_mutation_v1() FROM PUBLIC; +CREATE TRIGGER "capability_attempts_append_only" +BEFORE UPDATE OR DELETE ON "capability_attempts" +FOR EACH ROW EXECUTE FUNCTION "forge_reject_capability_attempt_mutation_v1"(); +``` + +### 6.2 `capability_attempt_adjudications` + +Append-only evidence that arrives *after* the attempt: verification results, +human decisions, rollbacks, overrides, and detected drift. + +| Column | Type | Notes | +|---|---|---| +| `id` | `uuid` PK | | +| `capability_attempt_id` | `uuid NOT NULL` → `capability_attempts` `ON DELETE restrict` | | +| `sequence` | `integer NOT NULL` | `CHECK >= 0`; unique per attempt, gapless | +| `kind` | `text NOT NULL` | `CHECK IN ('verification_recorded','human_decision','rollback_recorded','override_recorded','evidence_drift_detected')` | +| `verification_mode` | `text` | null unless `kind = 'verification_recorded'` | +| `verification_result` | `text` | `CHECK NULL OR IN ('passed','failed','inconclusive')` | +| `human_decision` | `text` | `CHECK NULL OR IN ('accepted','rejected','cancelled')` | +| `decided_by` | `uuid` → `users` `ON DELETE set null` | set only for human decisions | +| `approval_gate_id` | `uuid` → `approval_gates` `ON DELETE set null` | provenance for human decisions | +| `observed_outcome_digest` | `text` | 64-hex; set only for `evidence_drift_detected` | +| `evidence_refs` | `jsonb NOT NULL DEFAULT '[]'` | UUIDs only | +| `observed_at` | `timestamptz NOT NULL` | | +| `created_at` | `timestamptz NOT NULL DEFAULT now()` | | + +Shape `CHECK`s, so a malformed adjudication cannot be stored: + +```sql +CONSTRAINT "capability_attempt_adjudications_kind_shape_check" CHECK ( + (kind = 'verification_recorded' + AND verification_mode IS NOT NULL AND verification_result IS NOT NULL + AND human_decision IS NULL AND observed_outcome_digest IS NULL) + OR (kind = 'human_decision' + AND human_decision IS NOT NULL + AND verification_mode IS NULL AND verification_result IS NULL + AND observed_outcome_digest IS NULL) + OR (kind IN ('rollback_recorded','override_recorded') + AND verification_mode IS NULL AND verification_result IS NULL + AND observed_outcome_digest IS NULL) + OR (kind = 'evidence_drift_detected' + AND observed_outcome_digest IS NOT NULL + AND verification_mode IS NULL AND verification_result IS NULL + AND human_decision IS NULL) +) +``` + +Ordering guard — a `BEFORE INSERT` trigger requiring +`NEW.sequence = COALESCE(MAX(sequence), -1) + 1` for that attempt, taking +`FOR UPDATE` on the parent attempt row exactly as +`forge_guard_operation_event_insert_v1` does. Plus a second trigger rejecting all +`UPDATE`/`DELETE` (`capability_attempt_adjudications_append_only`). + +Indexes: `UNIQUE (capability_attempt_id, sequence)`, and +`(capability_attempt_id, observed_at)`. + +### 6.3 No materialized summary in v1 + +Issue #186 permits an optional materialized summary. **This architecture +deliberately omits it.** + +Rationale: a cache that can disagree with its source is a whole class of bug — +staleness, partial rebuild, and "the number on the screen is not the number in +the evidence". The cohort index makes the on-demand computation a bounded index +scan of at most `maxAttempts` rows plus their adjudications. Metrics are a pure +function (I11), so materializing later is purely additive. + +Revisit when a real measurement shows a cohort read exceeding ~50 ms at p95, or +when #191 needs cross-cohort listings that would fan out to many cohort queries. +Record that measurement in the ADR before adding the table. + +### 6.4 Schema and privileges + +Add both tables to `web/db/schema.ts` in the `operationRuns` neighbourhood, with +`InferSelectModel`/`InferInsertModel` type exports matching the file's existing +convention, and every `CHECK` mirrored in the Drizzle definition (the repository +keeps SQL and Drizzle constraint parity; `0030` does this for all of its checks). + +The ordinary application role must get `SELECT, INSERT` and nothing else — no +`UPDATE` (attempts are immutable) and no `DELETE`. See §10 for the exact CI +inventory edit. + +--- + +## 7. Ingest + +New module `web/worker/reliability/ledger.ts`. It is the only writer. + +### 7.1 Posture: best-effort, never blocking + +Every ingest call is wrapped exactly like +`upsertExecutionOutcomeBestEffort` in `web/worker/work-package-handoff.ts`: catch, +log through the existing task-log path, continue. A reliability write failing must +never fail a task, package, run, or operation (I10). The ledger is an +interpretation layer; the lifecycle tables are the truth. + +### 7.2 Feature flag + +``` +FORGE_CAPABILITY_RELIABILITY_LEDGER # default on; set to 0/false/off to disable ingest +``` + +Use `defaultOnFeatureFlagEnabled` from `web/worker/feature-flags.ts`. Default-on +is correct here because the slice is additive and write-only — nothing reads the +ledger to make a decision in v1. Reads must tolerate a gap in history caused by a +disabled window: a gap is missing evidence, never success. + +### 7.3 Call sites + +Ingest hangs off the boundaries that already write canonical outcomes. Do not +create new execution boundaries. + +**(a) `web/worker/work-package-handoff.ts` — three existing sites.** + +| Existing site | Attempt shape | +|---|---| +| admission block (`attemptKey: work-package::admission`) | `result: 'blocked'`, `verifier_required: false`, `verification_mode: 'none'` | +| completion (`attemptKey: work-package::run:`) | `result: 'completed'`, `verifier_required` from `reviewRequirement`, `verification_mode: 'human_review'` when required | +| failure (same attempt key) | mapped from `executionFailureOutcome(...)` | + +Note on completion: today `verifier_required = reviewRequirement !== 'none'` and +the only available verifier is a human approval gate, so `verification_mode` is +`human_review` with `verification_status: 'pending'`. It is **not** +`independent_agent`, and it never counts toward `independentlyVerifiedPass`. That +is the honest state of Forge until #188. + +`upsertExecutionOutcome` must return the stored row id so ingest can link +`execution_outcome_id`. Change its signature to +`Promise<{ id: string } | null>` (`.returning({ id: executionOutcomes.id })`) and +have the best-effort wrapper return `null` on failure. This is additive; existing +callers ignore the value. + +**(b) `web/worker/operations/ledger.ts` — after `finalize()` commits.** + +`finalize()` already returns `{ executionOutcomeId }`. Call ingest **after** the +transaction commits, best-effort. Do not put ledger writes inside that +transaction: ADR 0011 pins its exact transactional contract (outcome + outcome +event + terminalization commit together), and widening it would invalidate the +proofs behind that ADR. + +Operation attempts use `verification_mode: 'deterministic_adapter'` when the run +verified, `'none'` when the run was blocked before execution, and +`operation_run_id` set. + +### 7.4 Ingest algorithm + +``` +recordCapabilityAttempts(input): + 1. If the feature flag is off -> return. + 2. Resolve capability keys: + work package -> required capabilities from the Architect classification, + normalized and de-duplicated; empty -> 'missing'; + more than MAX_CAPABILITY_FAN_OUT -> 'overflow' + operation -> exactly one key: operation:@ + 3. Read the cohort inputs from already-loaded rows where possible + (project, work package, agent run snapshot, harness). One extra read is + acceptable; a fan-out of reads per capability is not. + 4. Compute scope/runtime/policy/cohort fingerprints and the outcome digest. + 5. Reject 'independent_agent' verification mode (no producer in v1). + 6. Build one row per capability key with a shared attempt_group_id and + capability_multiplicity = keys.length. + 7. Insert all rows in one statement with + ON CONFLICT ("execution_outcome_id","capability_key") DO NOTHING. + 8. Never update, never delete, never retry into a mutation. +``` + +Step 7 gives idempotency (I2) for free: a recovered worker that re-runs the +boundary writes nothing new. Because `execution_outcomes` is itself an upsert +keyed by `(task_id, attempt_key)`, a recovered attempt reuses the same outcome +row and therefore the same conflict target. + +### 7.5 Adjudication producers in v1 + +| Producer | Where | Writes | +|---|---|---| +| Human review decision | `web/worker/review-gates.ts` → `decideReviewGate` | `human_decision` (`accepted` for `completed`, `rejected` for `needs_rework`) with `decided_by` and `approval_gate_id`; plus `verification_recorded` with mode `human_review` and result `passed`/`failed` | +| Drift detection | the cohort reader (§8) | `evidence_drift_detected` with the currently observed digest | +| Rollback | **no producer in v1** | contract only | +| Override | **no producer in v1** | contract only | + +Rollback and override are storage contracts waiting for #189/#190. Do **not** +invent a producer, and do **not** synthesize a rollback from a `needs_rework` +decision — rework is not rollback, and conflating them would corrupt the very +metric #189 depends on. + +Resolving the attempt from a gate decision: the gate carries +`work_package_id` and `source_agent_run_id`; the attempt rows are found via +`execution_outcomes` on `(task_id, attempt_key = 'work-package::run:')`. +Write one adjudication per attempt row in the group, each with its own +`sequence`. If no attempt row exists (ledger disabled, or the outcome predates +this table), skip silently — a missing attempt is missing evidence, not an error +to escalate. + +--- + +## 8. Read path + +New module `web/lib/reliability/metrics.ts` (pure) and +`web/worker/reliability/reader.ts` (database access). + +```ts +// pure, no I/O, no clock +export function computeReliability(input: { + attempts: CapabilityAttemptRecord[] + adjudications: CapabilityAdjudicationRecord[] + window: ReliabilityWindow + now: Date +}): ReliabilitySummary +``` + +```ts +// database access only; performs no arithmetic +export async function readCohortReliability(input: { + cohortFingerprint: string + window?: ReliabilityWindow + now?: Date +}): Promise +``` + +The reader: + +1. Selects at most `window.maxAttempts` rows for the cohort, newest first. +2. Selects their adjudications. +3. Joins the linked `execution_outcomes` rows and recomputes each + `outcome_digest`. Any mismatch appends an `evidence_drift_detected` + adjudication (best-effort) and marks the attempt drifted. +4. Calls `computeReliability`. + +Drift semantics (I8): if **any** in-window attempt drifted, `state` is +`evidence_drift`, every rate is `null`, and `criticalFailureCount` is still +reported. Forge must not average numbers whose underlying evidence changed +beneath them. + +Keeping arithmetic in a pure function is what makes "metrics can be recomputed +deterministically from stored attempts" a testable claim rather than a promise. + +--- + +## 9. Operator surface (bounded) + +No dashboard, no HTTP route. One read-only CLI script, matching the existing +`protocol:inspect-*` convention: + +``` +npm run protocol:inspect-capability-reliability -- --project [--capability ] [--json] +``` + +`web/scripts/inspect-capability-reliability.ts` lists the project's cohorts with +their capability key, state, sample counts, critical count, and freshness. It +performs no writes and takes no action. Human-facing output must stay +layman-readable per `AGENTS.md`: say "not enough evidence yet (3 of 5 attempts)" +rather than printing a bare enum. + +This is the minimum needed for a human to see the evidence exists. Everything +richer belongs to #191. + +--- + +## 10. CI and migration gates + +Adding a migration and public tables trips four pinned gates plus the closed-ACL +inventory. All were last moved by commits `a2cc23c`, `c64d06b`, `6115707`, and +`5f06947`; read those diffs before starting. + +Current state: 31 migrations, newest journal entry `0030_operation_runs` with +`when = 1785993600000`. After `0031`, the count is **32** and the max +`created_at` literal becomes the new migration's journal timestamp. + +**Files that must be updated in the same commit as the migration:** + +1. `web/db/migrations/0031_capability_reliability_ledger.sql` — new. +2. `web/db/migrations/meta/_journal.json` — new `idx: 31` entry (generated by + `npm run db:generate`; do not hand-edit the timestamp afterwards). +3. `web/scripts/ci/sql/migration-0027-expansion-assertions.sql` — `<> 31` → `<> 32` + (both count assertions) and the `max(created_at)` literal. +4. `web/__tests__/local-projection-overlimit-archive.test.ts` — the same three + literals, asserted as substrings of that SQL file. +5. `scripts/ci/prove-installer-managed-migrations.sh` — count and max literal. +6. `web/scripts/ci/prove-installer-legacy-migration-repair.sh` — same. +7. `.github/workflows/web-ci.yml` — the closed application-ACL inventory: + - add `capability_attempts` and `capability_attempt_adjudications` to the + `operation_ledger_tables` array (they belong to the same ledger family); + - add `GRANT SELECT, INSERT ON TABLE public.capability_attempts, + public.capability_attempt_adjudications TO forge_app_test;` + - extend the per-privilege expectation expression so both new tables expect + exactly `SELECT` and `INSERT` — no `UPDATE`, no `DELETE` (I12). Getting this + wrong fails the gate loudly, which is the intended behaviour. + - also extend the `REVOKE ALL ON TABLE …` line near line 271 that seeds the + ledger-family baseline. + +`web/scripts/repair-epic-172-legacy-release.ts` was already made tolerant of +ledgers of 29 rows or more (`a2cc23c`) and should need no change. Verify rather +than assume. + +**Do not** renumber, edit, or reuse an existing migration. **Do not** relax a +pinned assertion to a range to avoid updating it — those pins are the gate. + +--- + +## 11. Rollout, compatibility, and recovery + +- **Additive only.** No existing column, constraint, or contract changes, with one + exception: `upsertExecutionOutcome` gains a return value (§7.3). +- **No backfill.** Cohorts start empty. A cohort with no rows is + `insufficient_evidence`, never a pass. +- **Disable path.** Set `FORGE_CAPABILITY_RELIABILITY_LEDGER=0`. Ingest stops; + existing rows remain readable; no lifecycle behaviour changes. +- **Rollback.** Because nothing reads the ledger to make a decision in v1, + reverting the application code is safe on its own. Leave the tables in place; + dropping them would destroy audit evidence. If they must go, that is a + separate reviewed migration. +- **Partial ingest.** A crash between the outcome write and the attempt write + leaves an outcome with no attempt. This is expected and safe: the next boundary + does not retroactively invent one, and the reader treats it as missing + evidence. Do not add a reconciliation sweep in this slice. +- **Mixed versions.** During a rolling restart, some workers write attempts and + some do not. Both are correct; the sample count is simply lower. + +--- + +## 12. Required tests + +Every invariant in §4 needs a named proving test. Suggested files, following the +repository's existing naming: + +**`web/__tests__/capability-reliability-contracts.test.ts`** +- capability-key grammar accepts valid work-package and operation keys, rejects + paths, spaces, uppercase, over-length, and missing namespace (I1); +- cohort fingerprint is stable under capability re-ordering and duplication (I5); +- cohort fingerprint changes when model, harness, policy version, root binding + revision, or contract version changes — one test per input, asserting *which* + component fingerprint moved (I5); +- scope fingerprint inputs contain no path-like value: seed a unique path + sentinel into project `local_path` and assert it appears in no fingerprint + input and no stored column (I1). + +**`web/__tests__/capability-reliability-metrics.test.ts`** +- identical inputs produce byte-identical summaries across repeated calls, and + the function reads no clock (I11); +- below `minSamples` → `insufficient_evidence`, all rates null (I9); +- a cohort of 20 successes plus one `security_blocked` attempt still reports + `criticalFailureCount: 1` and a non-null `lastCriticalAt` (I7); +- `self_reported` and `human_review` verification never raise + `independentlyVerifiedPass`; a cohort of 10 human-approved completions reports + `independentlyVerifiedPass: null` (denominator behaviour) and + `unverifiedCompletion: 1` (I6); +- `consecutiveVerifiedPasses` resets at the first non-verified row; +- zero denominators return `null`, never `0` or `1`; +- one drifted attempt suppresses all rates and sets `state: 'evidence_drift'` + while critical counts survive (I8); +- multiplicity: one failed package covering 5 capabilities yields + `sampleCount: 5`, `uniqueAttemptCount: 1`. + +**`web/__tests__/capability-reliability-ledger.test.ts`** (mocked database) +- re-running the same boundary twice writes rows once (I2); +- `independent_agent` mode is rejected at ingest (§5.5); +- a ledger write failure does not propagate to the caller (I10); +- flag off → no writes; +- missing classification writes exactly one `unclassified` row with + `classification_state: 'missing'`; 13 capabilities writes one `overflow` row. + +**`web/__tests__/capability-reliability-schema.test.ts`** (text assertions on the +migration, mirroring `operation-ledger-schema.test.ts`) +- asserts the append-only triggers, the ordering guard, the `CHECK` names, and + the `REVOKE ALL ON FUNCTION` lines exist; +- asserts every `text` column in the new tables appears in a `CHECK` — the + machine-checkable form of "no free-text column" (I1). + +**`web/__tests__/capability-reliability-ledger.postgres.test.ts`** (gated proof, +modelled exactly on `operation-ledger.postgres.test.ts`) +- gate: `FORGE_RELIABILITY_LEDGER_REQUIRE_POSTGRES_TEST=1` plus `DATABASE_URL` + and `FORGE_RELIABILITY_LEDGER_POSTGRES_ADMIN_TEST_URL`; the mandatory suite may + not skip, and missing variables must throw rather than silently pass; +- `UPDATE` and `DELETE` on `capability_attempts` are rejected by the trigger (I3); +- adjudication sequence gaps and out-of-order inserts are rejected; +- adjudication `UPDATE`/`DELETE` are rejected (I4); +- the duplicate-ingest unique index holds under concurrent inserts (I2); +- each shape `CHECK` rejects its malformed row. + +**Existing suites to extend** +- `web/__tests__/work-package-handoff-db.test.ts` — assert attempt ingest at all + three outcome boundaries, with the expected verification mode per boundary; +- `web/__tests__/local-projection-overlimit-archive.test.ts` — the pinned + migration literals (§10). + +Also run: `npm run lint`, `npm run test:unit:zero-skip`, and `npx tsc --noEmit`. + +--- + +## 13. Work packages + +Sequential unless noted. Each package must land with its own tests passing. + +| WP | Role | Deliverable | Files | +|---|---|---|---| +| WP-1 | Backend | Contracts, grammar, fingerprints, summary types. No database, no I/O. | `web/lib/reliability/contracts.ts` + contracts test | +| WP-2 | Backend | Migration 0031, Drizzle definitions, triggers, and all CI gate updates from §10. | `web/db/migrations/0031_*.sql`, `web/db/schema.ts`, the six gate files, schema test | +| WP-3 | Backend | Ingest module, feature flag, `upsertExecutionOutcome` return value, wiring at the three handoff boundaries. | `web/worker/reliability/ledger.ts`, `web/worker/execution-outcomes.ts`, `web/worker/work-package-handoff.ts` | +| WP-4 | Backend | Operation-run ingest after `finalize()`. Depends on WP-3. | `web/worker/operations/ledger.ts` | +| WP-5 | Backend | Adjudications: human review decisions from `decideReviewGate`. | `web/worker/reliability/ledger.ts`, `web/worker/review-gates.ts` | +| WP-6 | Backend | Pure metrics function and cohort reader with drift detection. Can run in parallel with WP-3/4/5 once WP-1 lands. | `web/lib/reliability/metrics.ts`, `web/worker/reliability/reader.ts` | +| WP-7 | Backend/DevOps | Read-only inspection script and its `package.json` entry. | `web/scripts/inspect-capability-reliability.ts` | +| WP-8 | QA | The full §12 matrix, including the gated PostgreSQL proof. | `web/__tests__/capability-reliability-*.test.ts` | +| WP-9 | Documentation | ADR 0012 finalized with any decisions changed during implementation; roadmap Phase 2 marked delivered. | `docs/adr/0012-*.md`, `docs/continuous-verification-and-earned-autonomy-roadmap.md` | + +Review requirement: **both** (QA and Reviewer). This slice touches durable +evidence, database privileges, and an append-only audit boundary, so +Security/Adversarial review applies per `AGENTS.md` — specifically the ACL +change in §10 and the no-free-text invariant. + +--- + +## 14. Acceptance criteria mapping + +| Issue #186 acceptance criterion | Satisfied by | Proving test | +|---|---|---| +| Two attempts with materially different capability scopes are not silently combined | §5.4 cohort fingerprint | cohort separation tests | +| Runtime/model or policy-version changes are visible and can trigger requalification | §5.4 runtime/policy fingerprints stored as columns | per-input fingerprint-change tests | +| Every ledger entry links to a canonical outcome and verification/evidence state | §6.1 `execution_outcome_id NOT NULL` + verification columns | schema + postgres proof | +| Reprocessing an attempt is idempotent | §7.4 unique index + `DO NOTHING` | ledger + postgres tests | +| Metrics can be recomputed deterministically from stored attempts | §8 pure function; no materialized cache | determinism test | +| A critical failure remains visible regardless of the aggregate pass percentage | §5.6, I7 | critical-visibility test | +| Human rejection, rollback, and override events affect the reliability view | §6.2 adjudications + §5.7 rates | metrics tests (rollback/override contract-only in v1) | +| Missing independent verification is not counted as a verified pass | §5.5, I6 | verification-mode tests | +| Tests cover cohorting, rolling windows, critical failures, idempotency, and historical data | §12 | the full matrix | + +Two criteria are only **partially** satisfiable in this slice, and the PR must +say so plainly rather than claim otherwise: + +- *"rollback … events affect the reliability view"* — the storage contract, the + metric, and the tests exist, but no producer emits `rollback_recorded` until + #189/#190. The metric is exercised by fixture data only. +- *"historical data"* — covered as "attempts recorded before this table exists + are readable as missing evidence", not as backfilled rows. + +--- + +## 15. Implementation stop conditions + +Stop and escalate to the Architect rather than improvising if any of these occur: + +1. A cohort input needed for a fingerprint is unavailable at an ingest boundary + (for example, an agent run with no provider snapshot). Do not substitute a + default, a placeholder, or a live re-read — the fingerprint would become a + lie. Skip the ingest and report the gap. +2. Making a metric work appears to require mutating a stored attempt. +3. A capability appears that is not in `CAPABILITY_TAXONOMY`. Do not extend the + taxonomy in this slice; use `unclassified`. +4. An ingest boundary would need to move inside an existing transaction, or the + ADR 0011 finalize transaction would need to widen. +5. The closed-ACL gate seems to require `UPDATE` or `DELETE` for the application + role on a ledger table. +6. A path, file name, prompt, model transcript, error string, or any other + free text appears to be needed in a ledger column. +7. Storing an `independent_agent` verification looks necessary before #188 has + landed. +8. The pinned migration gates in §10 cannot be satisfied without weakening an + assertion. + +--- + +## 16. Considered and deferred + +**Exportable per-attempt "receipt envelopes."** A community comment on #186 +([#186 comment](https://github.com/Joncallim/Forge/issues/186#issuecomment-4965458169), +from a non-maintainer) proposed making each attempt an exportable receipt — +agent identity, capability, scope, runtime, verifier identity, evidence hashes, +override state, decay semantics — so external registries or marketplaces could +consume Forge's evidence. + +Deferred, deliberately. The attempt row defined in §6.1 already carries almost +that exact field set, so an export projection remains cheap to add later. What is +*not* free is the surface it implies: a stable public schema, agent identity that +survives outside Forge, signing, and revocation across a trust boundary — none of +which #186 needs and all of which would widen the security review. A decision to +publish reliability evidence outside Forge is a product decision for Jonathan, +not an implementation detail of the ledger. Recorded here so the option stays +open and the reasoning is not lost. + +**Materialized reliability summaries.** See §6.3 — omitted until a measurement +justifies the staleness risk. + +**A `capability_cohorts` dimension table.** Rejected for v1: the fingerprint +columns are self-describing, and a normalized dimension table would need its own +immutability rules for no read benefit at this scale. diff --git a/docs/continuous-verification-and-earned-autonomy-roadmap.md b/docs/continuous-verification-and-earned-autonomy-roadmap.md index 4d72ee98..5b145484 100644 --- a/docs/continuous-verification-and-earned-autonomy-roadmap.md +++ b/docs/continuous-verification-and-earned-autonomy-roadmap.md @@ -1,9 +1,18 @@ # Continuous Verification and Earned Autonomy Roadmap -Last updated: 2026-07-12 +Last updated: 2026-08-07 Epic: [#184 — Continuous verification and earned autonomy](https://github.com/Joncallim/Forge/issues/184) +## Current Position + +| Phase | Issue | State | +|---|---|---| +| Phase 0 — Deterministic operation catalog | #201 | Delivered — ADR 0011, `operation_runs` | +| Phase 1 — Canonical outcomes | #185 | Delivered — ADR 0010, `execution_outcomes` | +| Phase 2 — Capability reliability ledger | #186 | Delivered — ADR 0012, `capability_attempts`/`capability_attempt_adjudications` | +| Phases 3–7 | #187–#191 | Not started | + ## Placement In The Forge Roadmap This is a **P2.5 trust and reliability layer** between bounded Workforce execution and broad Forge Workspace expansion. @@ -93,6 +102,8 @@ Store append-only comparable capability attempts and calculate deterministic met Do not collapse materially different capability scopes, models, runtimes, harnesses, or policy versions into one score. +Design accepted: `docs/architecture/issue-186-capability-reliability-ledger.md` and ADR 0012. It keeps attempts immutable, appends later evidence (verification, human decisions, rollback, drift) separately, computes metrics as a pure function per cohort, and stores no free text at all. + ### Phase 3 — Project Verification Goals Issue: [#187](https://github.com/Joncallim/Forge/issues/187) diff --git a/scripts/ci/prove-installer-managed-migrations.sh b/scripts/ci/prove-installer-managed-migrations.sh index 63827917..6283458a 100755 --- a/scripts/ci/prove-installer-managed-migrations.sh +++ b/scripts/ci/prove-installer-managed-migrations.sh @@ -49,8 +49,8 @@ assert_latest_and_clean() { PGPASSWORD="$FORGE_INSTALLER_MANAGED_ADMIN_PASSWORD" PGHOST="$FORGE_INSTALLER_MANAGED_ADMIN_HOST" PGUSER="$FORGE_INSTALLER_MANAGED_ADMIN_USER" PGDATABASE="$database_name" psql --set ON_ERROR_STOP=1 <<'SQL' DO $proof$ BEGIN - IF (SELECT count(*) FROM drizzle.__drizzle_migrations) <> 31 - OR (SELECT max(created_at) FROM drizzle.__drizzle_migrations) <> 1785993600000 THEN + IF (SELECT count(*) FROM drizzle.__drizzle_migrations) <> 32 + OR (SELECT max(created_at) FROM drizzle.__drizzle_migrations) <> 1786080000000 THEN RAISE EXCEPTION 'Managed installer did not apply the exact latest migration ledger'; END IF; IF pg_catalog.to_regclass('public.forge_epic_172_s3_release_state') IS NULL THEN diff --git a/web/__tests__/capability-reliability-contracts.test.ts b/web/__tests__/capability-reliability-contracts.test.ts new file mode 100644 index 00000000..a79beb71 --- /dev/null +++ b/web/__tests__/capability-reliability-contracts.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it } from 'vitest' + +import { + CAPABILITY_KEY_PATTERN, + cohortFingerprint, + isValidCapabilityKey, + policyFingerprint, + runtimeFingerprint, + scopeFingerprint, + unclassifiedCapabilityKey, + type ReliabilityPolicyInput, + type ReliabilityRuntimeInput, + type ReliabilityScopeInput, +} from '@/lib/reliability/contracts' + +function scope(overrides: Partial = {}): ReliabilityScopeInput { + return { + contractVersion: 1, + projectId: 'project-1', + rootRef: 'root-ref-1', + rootBindingRevision: '5', + grantDecisionRevision: '3', + repositoryWriteIntent: false, + capabilities: ['filesystem.project.read'], + mcpRequirementKeys: ['req-a'], + ...overrides, + } +} + +function runtime(overrides: Partial> = {}): ReliabilityRuntimeInput { + return { + kind: 'model', + providerType: 'anthropic', + modelId: 'claude-sonnet-5', + providerIsLocal: false, + providerConfigUpdatedAt: '2026-01-01T00:00:00.000Z', + acpExecutionMode: 'not_applicable', + ...overrides, + } +} + +function policy(overrides: Partial = {}): ReliabilityPolicyInput { + return { + contractVersion: 1, + policyVersion: 'reliability-policy-v1', + harnessId: 'harness-1', + harnessUpdatedAt: '2026-01-01T00:00:00.000Z', + reviewRequirement: 'both', + repositoryWritesEnabled: false, + ...overrides, + } +} + +describe('capability key grammar', () => { + it('accepts valid work-package and operation keys', () => { + expect(isValidCapabilityKey('workpackage:backend/api-implementation')).toBe(true) + expect(isValidCapabilityKey('operation:repository.status.read@1')).toBe(true) + }) + + it('rejects paths, spaces, uppercase, missing namespace, and over-length values', () => { + expect(isValidCapabilityKey('workpackage:backend/../etc/passwd')).toBe(false) + expect(isValidCapabilityKey('workpackage:backend/api implementation')).toBe(false) + expect(isValidCapabilityKey('workpackage:Backend/Api-Implementation')).toBe(false) + expect(isValidCapabilityKey('api-implementation')).toBe(false) + expect(isValidCapabilityKey(`workpackage:backend/${'a'.repeat(200)}`)).toBe(false) + expect(CAPABILITY_KEY_PATTERN.test('workpackage:backend/api-implementation')).toBe(true) + }) + + it('unclassified keys are always work-package scoped and pattern-valid', () => { + const key = unclassifiedCapabilityKey('backend') + expect(key).toBe('workpackage:backend/unclassified') + expect(isValidCapabilityKey(key)).toBe(true) + }) +}) + +describe('cohort fingerprinting', () => { + it('is stable under capability re-ordering and duplication in the scope input', () => { + const a = scopeFingerprint(scope({ capabilities: ['x', 'y'], mcpRequirementKeys: ['r1', 'r2'] })) + const b = scopeFingerprint(scope({ capabilities: ['y', 'x', 'x'], mcpRequirementKeys: ['r2', 'r1', 'r1'] })) + expect(a).toBe(b) + }) + + function buildCohort(overrides: { + scopeInput?: Partial + runtimeInput?: Partial> + policyInput?: Partial + } = {}) { + const s = scopeFingerprint(scope(overrides.scopeInput)) + const r = runtimeFingerprint(runtime(overrides.runtimeInput)) + const p = policyFingerprint(policy(overrides.policyInput)) + return cohortFingerprint({ + projectId: 'project-1', + capabilityKey: 'workpackage:backend/api-implementation', + scopeFingerprint: s, + runtimeFingerprint: r, + policyFingerprint: p, + }) + } + + it('changes when the model changes', () => { + const base = buildCohort() + const changed = buildCohort({ runtimeInput: { modelId: 'claude-opus-5' } }) + expect(base).not.toBe(changed) + }) + + it('changes when the harness changes', () => { + const base = buildCohort() + const changed = buildCohort({ policyInput: { harnessId: 'harness-2' } }) + expect(base).not.toBe(changed) + }) + + it('changes when the policy version changes', () => { + const base = buildCohort() + const changed = buildCohort({ policyInput: { policyVersion: 'reliability-policy-v2' } }) + expect(base).not.toBe(changed) + }) + + it('changes when the root binding revision changes', () => { + const base = buildCohort() + const changed = buildCohort({ scopeInput: { rootBindingRevision: '6' } }) + expect(base).not.toBe(changed) + }) + + it('changes when the contract version changes', () => { + const base = cohortFingerprint({ + projectId: 'project-1', + capabilityKey: 'workpackage:backend/api-implementation', + scopeFingerprint: scopeFingerprint(scope()), + runtimeFingerprint: runtimeFingerprint(runtime()), + policyFingerprint: policyFingerprint(policy()), + }) + // A different capability key is a different cohort by construction. + const changedCapability = cohortFingerprint({ + projectId: 'project-1', + capabilityKey: 'workpackage:backend/database-migration', + scopeFingerprint: scopeFingerprint(scope()), + runtimeFingerprint: runtimeFingerprint(runtime()), + policyFingerprint: policyFingerprint(policy()), + }) + expect(base).not.toBe(changedCapability) + }) + + it('the scope input has no path-shaped field -- only the opaque rootRef', () => { + const keys = Object.keys(scope()) + expect(keys).toContain('rootRef') + expect(keys.some((key) => /path/i.test(key))).toBe(false) + }) + + it('produces an opaque 64-hex digest regardless of input content', () => { + const sentinel = '/Users/sentinel/super-secret-project/src/index.ts' + const fp = scopeFingerprint(scope({ capabilities: [sentinel] })) + expect(fp).toMatch(/^[0-9a-f]{64}$/) + }) +}) diff --git a/web/__tests__/capability-reliability-ledger.postgres.test.ts b/web/__tests__/capability-reliability-ledger.postgres.test.ts new file mode 100644 index 00000000..df3afa7a --- /dev/null +++ b/web/__tests__/capability-reliability-ledger.postgres.test.ts @@ -0,0 +1,167 @@ +import { randomUUID } from 'node:crypto' + +import postgres from 'postgres' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +const required = process.env.FORGE_RELIABILITY_LEDGER_REQUIRE_POSTGRES_TEST === '1' +const databaseUrl = process.env.DATABASE_URL?.trim() +const adminUrl = process.env.FORGE_RELIABILITY_LEDGER_POSTGRES_ADMIN_TEST_URL?.trim() +const enabled = required && Boolean(databaseUrl && adminUrl) + +if (required && (!databaseUrl || !adminUrl)) { + throw new Error( + 'FORGE_RELIABILITY_LEDGER_REQUIRE_POSTGRES_TEST=1 requires DATABASE_URL and FORGE_RELIABILITY_LEDGER_POSTGRES_ADMIN_TEST_URL for the disposable PostgreSQL capability-reliability-ledger proof; the mandatory suite may not skip.', + ) +} + +describe.skipIf(!enabled)('capability reliability ledger PostgreSQL behavior', () => { + const ids = { + user: randomUUID(), + project: randomUUID(), + task: randomUUID(), + executionOutcome: randomUUID(), + attempt: randomUUID(), + } + const cohortFingerprint = 'a'.repeat(64) + const scopeFingerprint = 'b'.repeat(64) + const runtimeFingerprint = 'c'.repeat(64) + const policyFingerprint = 'd'.repeat(64) + const outcomeDigest = 'e'.repeat(64) + + let sql: ReturnType + let adminSql: ReturnType + + beforeAll(async () => { + sql = postgres(databaseUrl!, { max: 4, onnotice: () => {} }) + adminSql = postgres(adminUrl!, { max: 1, onnotice: () => {} }) + + await sql.begin(async (tx) => { + await tx` + insert into users (id, display_name) + values (${ids.user}::uuid, 'Capability reliability ledger PostgreSQL proof') + ` + await tx` + insert into projects (id, name, submitted_by, grant_decision_revision, root_binding_revision) + values (${ids.project}::uuid, 'Capability reliability ledger PostgreSQL proof', ${ids.user}::uuid, 1, 1) + ` + await tx` + insert into tasks (id, project_id, submitted_by, title, prompt, status) + values ( + ${ids.task}::uuid, ${ids.project}::uuid, ${ids.user}::uuid, + 'Capability reliability ledger PostgreSQL proof', 'Bounded disposable test fixture', 'running' + ) + ` + await tx` + insert into execution_outcomes ( + id, task_id, attempt_key, schema_version, transport_status, result, + retryable, verifier_required, verification_status + ) + values ( + ${ids.executionOutcome}::uuid, ${ids.task}::uuid, 'capability-ledger-postgres-proof', + 1, 'ok', 'completed', false, false, 'not_required' + ) + ` + await tx` + insert into capability_attempts ( + id, attempt_group_id, project_id, task_id, execution_outcome_id, + contract_version, capability_key, classification_state, capability_multiplicity, + cohort_fingerprint, scope_fingerprint, runtime_fingerprint, policy_fingerprint, outcome_digest, + transport_status, result, retryable, attempt_number, severity_class, + verifier_required, verification_mode, verification_status, observed_at + ) + values ( + ${ids.attempt}::uuid, ${ids.attempt}::uuid, ${ids.project}::uuid, ${ids.task}::uuid, ${ids.executionOutcome}::uuid, + 1, 'workpackage:backend/api-implementation', 'classified', 1, + ${cohortFingerprint}, ${scopeFingerprint}, ${runtimeFingerprint}, ${policyFingerprint}, ${outcomeDigest}, + 'ok', 'completed', false, 1, 'normal', + false, 'none', 'not_required', now() + ) + ` + }) + }) + + afterAll(async () => { + await Promise.all([ + sql?.end({ timeout: 5 }), + adminSql?.end({ timeout: 5 }), + ]) + }) + + it('rejects UPDATE and DELETE on capability_attempts', async () => { + await expect(sql` + update capability_attempts set result = 'failed' where id = ${ids.attempt}::uuid + `).rejects.toThrow('capability attempts are append-only') + await expect(sql` + delete from capability_attempts where id = ${ids.attempt}::uuid + `).rejects.toThrow('capability attempts are append-only') + }) + + it('rejects a duplicate (execution_outcome_id, capability_key) attempt', async () => { + await expect(sql` + insert into capability_attempts ( + id, attempt_group_id, project_id, task_id, execution_outcome_id, + contract_version, capability_key, classification_state, capability_multiplicity, + cohort_fingerprint, scope_fingerprint, runtime_fingerprint, policy_fingerprint, outcome_digest, + transport_status, result, retryable, attempt_number, severity_class, + verifier_required, verification_mode, verification_status, observed_at + ) + values ( + ${randomUUID()}::uuid, ${randomUUID()}::uuid, ${ids.project}::uuid, ${ids.task}::uuid, ${ids.executionOutcome}::uuid, + 1, 'workpackage:backend/api-implementation', 'classified', 1, + ${cohortFingerprint}, ${scopeFingerprint}, ${runtimeFingerprint}, ${policyFingerprint}, ${outcomeDigest}, + 'ok', 'completed', false, 1, 'normal', + false, 'none', 'not_required', now() + ) + `).rejects.toThrow(/duplicate key value|unique constraint/i) + }) + + it('enforces gapless adjudication sequence order per attempt', async () => { + await sql` + insert into capability_attempt_adjudications ( + capability_attempt_id, sequence, kind, human_decision, observed_at + ) + values (${ids.attempt}::uuid, 0, 'human_decision', 'accepted', now()) + ` + await expect(sql` + insert into capability_attempt_adjudications ( + capability_attempt_id, sequence, kind, human_decision, observed_at + ) + values (${ids.attempt}::uuid, 2, 'human_decision', 'accepted', now()) + `).rejects.toThrow('gapless sequence order') + await sql` + insert into capability_attempt_adjudications ( + capability_attempt_id, sequence, kind, observed_at + ) + values (${ids.attempt}::uuid, 1, 'rollback_recorded', now()) + ` + }) + + it('rejects UPDATE and DELETE on capability_attempt_adjudications', async () => { + const [row] = await sql<{ id: string }[]>` + select id from capability_attempt_adjudications + where capability_attempt_id = ${ids.attempt}::uuid and sequence = 0 + ` + await expect(sql` + update capability_attempt_adjudications set human_decision = 'rejected' where id = ${row!.id}::uuid + `).rejects.toThrow('capability attempt adjudications are append-only') + await expect(sql` + delete from capability_attempt_adjudications where id = ${row!.id}::uuid + `).rejects.toThrow('capability attempt adjudications are append-only') + }) + + it('the ordinary application role has no UPDATE or DELETE on either ledger table', async () => { + const [privileges] = await sql<{ canUpdateAttempts: boolean; canDeleteAttempts: boolean; canUpdateAdjudications: boolean; canDeleteAdjudications: boolean }[]>` + select + has_table_privilege(current_user, 'public.capability_attempts', 'UPDATE') as "canUpdateAttempts", + has_table_privilege(current_user, 'public.capability_attempts', 'DELETE') as "canDeleteAttempts", + has_table_privilege(current_user, 'public.capability_attempt_adjudications', 'UPDATE') as "canUpdateAdjudications", + has_table_privilege(current_user, 'public.capability_attempt_adjudications', 'DELETE') as "canDeleteAdjudications" + ` + expect(privileges).toEqual({ + canUpdateAttempts: false, + canDeleteAttempts: false, + canUpdateAdjudications: false, + canDeleteAdjudications: false, + }) + }) +}) diff --git a/web/__tests__/capability-reliability-ledger.test.ts b/web/__tests__/capability-reliability-ledger.test.ts new file mode 100644 index 00000000..37da45e5 --- /dev/null +++ b/web/__tests__/capability-reliability-ledger.test.ts @@ -0,0 +1,204 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + insertValues: vi.fn(), + insertOnConflict: vi.fn(), +})) + +function chain(rows: unknown[]): Record { + const thenable: Record = { + then: (onFulfilled: (value: unknown[]) => unknown) => Promise.resolve(rows).then(onFulfilled), + } + thenable.from = () => thenable + thenable.where = () => thenable + thenable.orderBy = () => thenable + thenable.limit = () => thenable + return thenable +} + +vi.mock('@/db', () => ({ + db: { + insert: (table: unknown) => ({ + values: (rows: unknown) => { + mocks.insertValues(table, rows) + return { + onConflictDoNothing: (opts: unknown) => { + mocks.insertOnConflict(opts) + return Promise.resolve([]) + }, + then: (onFulfilled: (value: unknown) => unknown) => Promise.resolve(undefined).then(onFulfilled), + } + }, + }), + select: () => chain([]), + }, +})) + +vi.mock('@/db/schema', () => ({ + capabilityAttempts: { + executionOutcomeId: 'execution_outcome_id', + capabilityKey: 'capability_key', + }, + capabilityAttemptAdjudications: { + capabilityAttemptId: 'capability_attempt_id', + sequence: 'sequence', + }, + executionOutcomes: { + taskId: 'task_id', + attemptKey: 'attempt_key', + id: 'id', + }, +})) + +import { + recordCapabilityAttempts, + recordCapabilityAttemptsBestEffort, + type RecordCapabilityAttemptsInput, +} from '@/worker/reliability/ledger' + +function baseInput(overrides: Partial = {}): RecordCapabilityAttemptsInput { + return { + projectId: 'project-1', + taskId: 'task-1', + workPackageId: 'wp-1', + agentRunId: 'run-1', + taskAttemptId: null, + executionOutcomeId: 'outcome-1', + operationRunId: null, + outcome: { + schemaVersion: 1, + transportStatus: 'ok', + result: 'completed', + stopReasonCode: null, + stopReasonSummary: null, + retryable: false, + evidenceRefs: [], + verifierRequired: false, + verificationStatus: 'not_required', + }, + attemptNumber: 1, + source: { kind: 'work_package', role: 'backend', capabilities: ['api-implementation'] }, + scope: { + contractVersion: 1, + projectId: 'project-1', + rootRef: 'root-1', + rootBindingRevision: '1', + grantDecisionRevision: '1', + repositoryWriteIntent: false, + capabilities: ['api-implementation'], + mcpRequirementKeys: [], + }, + runtime: { + kind: 'model', + providerType: 'anthropic', + modelId: 'claude-sonnet-5', + providerIsLocal: false, + providerConfigUpdatedAt: null, + acpExecutionMode: 'not_applicable', + }, + policy: { + contractVersion: 1, + policyVersion: 'reliability-policy-v1', + harnessId: null, + harnessUpdatedAt: null, + reviewRequirement: 'both', + repositoryWritesEnabled: false, + }, + verificationMode: 'none', + acceptanceCriteriaTotal: 0, + validationCommandTotal: 0, + validationCommandFailed: 0, + observedAt: new Date('2026-08-01T00:00:00.000Z'), + ...overrides, + } +} + +describe('recordCapabilityAttempts', () => { + beforeEach(() => { + mocks.insertValues.mockReset() + mocks.insertOnConflict.mockReset() + delete process.env.FORGE_CAPABILITY_RELIABILITY_LEDGER + }) + + afterEach(() => { + delete process.env.FORGE_CAPABILITY_RELIABILITY_LEDGER + }) + + it('requests the idempotent conflict target on (execution_outcome_id, capability_key)', async () => { + await recordCapabilityAttempts(baseInput()) + expect(mocks.insertValues).toHaveBeenCalledTimes(1) + expect(mocks.insertOnConflict).toHaveBeenCalledWith({ + target: ['execution_outcome_id', 'capability_key'], + }) + }) + + it('refuses independent_agent verification mode at ingest', async () => { + await recordCapabilityAttempts(baseInput({ + outcome: { + schemaVersion: 1, + transportStatus: 'ok', + result: 'completed', + stopReasonCode: null, + stopReasonSummary: null, + retryable: false, + evidenceRefs: [], + verifierRequired: true, + verificationStatus: 'pending', + }, + verificationMode: 'independent_agent', + })) + expect(mocks.insertValues).not.toHaveBeenCalled() + }) + + it('does not write when the feature flag is explicitly disabled', async () => { + process.env.FORGE_CAPABILITY_RELIABILITY_LEDGER = '0' + await recordCapabilityAttempts(baseInput()) + expect(mocks.insertValues).not.toHaveBeenCalled() + }) + + it('writes one unclassified row when the capability classification is missing', async () => { + await recordCapabilityAttempts(baseInput({ + source: { kind: 'work_package', role: 'backend', capabilities: null }, + })) + expect(mocks.insertValues).toHaveBeenCalledTimes(1) + const [, rows] = mocks.insertValues.mock.calls[0] as [unknown, Array>] + expect(rows).toHaveLength(1) + expect(rows[0].capabilityKey).toBe('workpackage:backend/unclassified') + expect(rows[0].classificationState).toBe('missing') + }) + + it('writes one unclassified overflow row when more than 12 capabilities are declared', async () => { + const capabilities = Array.from({ length: 13 }, (_, i) => `capability-${i}`) + await recordCapabilityAttempts(baseInput({ + source: { kind: 'work_package', role: 'backend', capabilities }, + })) + const [, rows] = mocks.insertValues.mock.calls[0] as [unknown, Array>] + expect(rows).toHaveLength(1) + expect(rows[0].classificationState).toBe('overflow') + }) + + it('writes one row per capability, sharing an attempt group and multiplicity count', async () => { + await recordCapabilityAttempts(baseInput({ + source: { kind: 'work_package', role: 'backend', capabilities: ['api-implementation', 'database-migration'] }, + })) + const [, rows] = mocks.insertValues.mock.calls[0] as [unknown, Array>] + expect(rows).toHaveLength(2) + expect(rows[0].attemptGroupId).toBe(rows[1].attemptGroupId) + expect(rows[0].capabilityMultiplicity).toBe(2) + }) +}) + +describe('recordCapabilityAttemptsBestEffort', () => { + beforeEach(() => { + mocks.insertValues.mockReset() + mocks.insertOnConflict.mockReset() + delete process.env.FORGE_CAPABILITY_RELIABILITY_LEDGER + }) + + it('never throws when the underlying write fails', async () => { + mocks.insertOnConflict.mockImplementation(() => { + throw new Error('simulated database failure') + }) + await expect(recordCapabilityAttemptsBestEffort(baseInput())).resolves.toBeUndefined() + }) +}) diff --git a/web/__tests__/capability-reliability-metrics.test.ts b/web/__tests__/capability-reliability-metrics.test.ts new file mode 100644 index 00000000..0ffab26a --- /dev/null +++ b/web/__tests__/capability-reliability-metrics.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from 'vitest' + +import { computeReliability } from '@/lib/reliability/metrics' +import type { + CapabilityAdjudicationRecord, + CapabilityAttemptRecord, + ReliabilityWindow, +} from '@/lib/reliability/contracts' + +const WINDOW: ReliabilityWindow = { maxAttempts: 50, maxAgeMs: 90 * 24 * 60 * 60 * 1000, minSamples: 5 } +const NOW = new Date('2026-08-01T00:00:00.000Z') + +let attemptCounter = 0 +function attempt(overrides: Partial = {}): CapabilityAttemptRecord { + attemptCounter += 1 + const id = overrides.id ?? `attempt-${attemptCounter}` + return { + id, + attemptGroupId: overrides.attemptGroupId ?? id, + executionOutcomeId: `outcome-${attemptCounter}`, + capabilityKey: 'workpackage:backend/api-implementation', + classificationState: 'classified', + capabilityMultiplicity: 1, + cohortFingerprint: 'a'.repeat(64), + outcomeDigest: 'b'.repeat(64), + transportStatus: 'ok', + result: 'completed', + stopReasonCode: null, + retryable: false, + attemptNumber: 1, + severityClass: 'normal', + verifierRequired: false, + verificationMode: 'none', + verificationStatus: 'not_required', + observedAt: NOW.toISOString(), + ...overrides, + } +} + +function verification( + attemptId: string, + overrides: Partial = {}, +): CapabilityAdjudicationRecord { + return { + id: `adj-${attemptId}-${overrides.sequence ?? 0}`, + capabilityAttemptId: attemptId, + sequence: 0, + kind: 'verification_recorded', + verificationMode: 'deterministic_adapter', + verificationResult: 'passed', + humanDecision: null, + observedAt: NOW.toISOString(), + ...overrides, + } +} + +describe('computeReliability', () => { + it('is a pure function: identical inputs produce byte-identical output', () => { + const attempts = Array.from({ length: 6 }, () => attempt()) + const a = computeReliability({ attempts, adjudications: [], window: WINDOW, now: NOW }) + const b = computeReliability({ attempts, adjudications: [], window: WINDOW, now: NOW }) + expect(a).toEqual(b) + }) + + it('reports insufficient_evidence with all-null rates below minSamples', () => { + const attempts = Array.from({ length: 3 }, () => attempt()) + const summary = computeReliability({ attempts, adjudications: [], window: WINDOW, now: NOW }) + expect(summary.state).toBe('insufficient_evidence') + expect(Object.values(summary.rates).every((rate) => rate === null)).toBe(true) + }) + + it('reports a critical failure regardless of the aggregate pass rate', () => { + const attempts = [ + ...Array.from({ length: 20 }, () => attempt({ result: 'completed' })), + attempt({ result: 'blocked', stopReasonCode: 'security_blocked', severityClass: 'critical' }), + ] + const summary = computeReliability({ attempts, adjudications: [], window: WINDOW, now: NOW }) + expect(summary.criticalFailureCount).toBe(1) + expect(summary.lastCriticalAt).not.toBeNull() + }) + + it('never counts self_reported or human_review as an independently verified pass', () => { + const attempts = Array.from({ length: 10 }, () => + attempt({ verifierRequired: true, verificationMode: 'human_review', verificationStatus: 'passed' })) + const adjudications = attempts.map((a) => verification(a.id, { verificationMode: 'human_review', verificationResult: 'passed' })) + const summary = computeReliability({ attempts, adjudications, window: WINDOW, now: NOW }) + expect(summary.rates.independentlyVerifiedPass).toBe(0) + expect(summary.rates.unverifiedCompletion).toBe(1) + }) + + it('counts deterministic_adapter verification as independently verified', () => { + const attempts = Array.from({ length: 10 }, () => + attempt({ verifierRequired: true, verificationMode: 'deterministic_adapter', verificationStatus: 'passed' })) + const adjudications = attempts.map((a) => verification(a.id)) + const summary = computeReliability({ attempts, adjudications, window: WINDOW, now: NOW }) + expect(summary.rates.independentlyVerifiedPass).toBe(1) + expect(summary.rates.unverifiedCompletion).toBe(0) + }) + + it('resets consecutiveVerifiedPasses at the first non-verified row from the newest attempt', () => { + const older = attempt({ observedAt: new Date(NOW.getTime() - 5000).toISOString(), verifierRequired: true, verificationMode: 'deterministic_adapter' }) + const middle = attempt({ observedAt: new Date(NOW.getTime() - 4000).toISOString(), verifierRequired: false, verificationMode: 'none' }) + const newer1 = attempt({ observedAt: new Date(NOW.getTime() - 3000).toISOString(), verifierRequired: true, verificationMode: 'deterministic_adapter' }) + const newer2 = attempt({ observedAt: new Date(NOW.getTime() - 2000).toISOString(), verifierRequired: true, verificationMode: 'deterministic_adapter' }) + const pad = Array.from({ length: 3 }, () => attempt({ observedAt: new Date(NOW.getTime() - 6000).toISOString() })) + const attempts = [...pad, older, middle, newer1, newer2] + const adjudications = [older, newer1, newer2].map((a) => verification(a.id)) + const summary = computeReliability({ attempts, adjudications, window: WINDOW, now: NOW }) + // newest two are verified passes, then a non-verified attempt breaks the streak. + expect(summary.consecutiveVerifiedPasses).toBe(2) + }) + + it('returns null rates when a denominator is zero, never 0 or 1', () => { + const attempts = Array.from({ length: 5 }, () => attempt({ attemptNumber: 2, verifierRequired: false })) + const summary = computeReliability({ attempts, adjudications: [], window: WINDOW, now: NOW }) + expect(summary.rates.firstAttemptSuccess).toBeNull() + expect(summary.rates.independentlyVerifiedPass).toBeNull() + expect(summary.rates.humanAccepted).toBeNull() + expect(summary.rates.humanRejection).toBeNull() + }) + + it('suppresses all rates and reports evidence_drift when an attempt has drifted, but keeps critical counts', () => { + const attempts = [ + ...Array.from({ length: 5 }, () => attempt()), + attempt({ result: 'blocked', severityClass: 'critical', currentOutcomeDigest: 'c'.repeat(64) }), + ] + const summary = computeReliability({ attempts, adjudications: [], window: WINDOW, now: NOW }) + expect(summary.state).toBe('evidence_drift') + expect(Object.values(summary.rates).every((rate) => rate === null)).toBe(true) + expect(summary.criticalFailureCount).toBe(1) + expect(summary.evidence.driftedAttemptCount).toBe(1) + }) + + it('reports sampleCount by row and uniqueAttemptCount by attempt group for a multi-capability failure', () => { + const groupId = 'group-1' + const attempts = [ + ...Array.from({ length: 4 }, () => attempt()), + attempt({ attemptGroupId: groupId, capabilityMultiplicity: 5 }), + attempt({ attemptGroupId: groupId, capabilityMultiplicity: 5 }), + attempt({ attemptGroupId: groupId, capabilityMultiplicity: 5 }), + attempt({ attemptGroupId: groupId, capabilityMultiplicity: 5 }), + attempt({ attemptGroupId: groupId, capabilityMultiplicity: 5 }), + ] + const summary = computeReliability({ attempts, adjudications: [], window: WINDOW, now: NOW }) + expect(summary.sampleCount).toBe(9) + // 4 solo attempts (each its own group) + 1 shared group of 5 = 5 unique groups. + expect(summary.uniqueAttemptCount).toBe(5) + }) +}) diff --git a/web/__tests__/capability-reliability-schema.test.ts b/web/__tests__/capability-reliability-schema.test.ts new file mode 100644 index 00000000..4329ba1d --- /dev/null +++ b/web/__tests__/capability-reliability-schema.test.ts @@ -0,0 +1,68 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import { describe, expect, it } from 'vitest' + +describe('capability reliability ledger migration', () => { + it('persists immutable attempts, append-only adjudications, and the closed enum/fingerprint contract', async () => { + const sql = await fs.readFile( + path.join(process.cwd(), 'db/migrations/0031_capability_reliability_ledger.sql'), + 'utf8', + ) + + // Structural identity. + expect(sql).toContain('"execution_outcome_id" uuid NOT NULL') + expect(sql).toContain('"attempt_group_id" uuid NOT NULL') + expect(sql).toContain('capability_attempts_outcome_capability_idx') + + // Append-only guards. + expect(sql).toContain('forge_reject_capability_attempt_mutation_v1') + expect(sql).toContain('capability_attempts_append_only') + expect(sql).toContain('forge_reject_capability_adjudication_mutation_v1') + expect(sql).toContain('capability_attempt_adjudications_append_only') + expect(sql).toContain('forge_guard_capability_adjudication_insert_v1') + expect(sql).toContain('capability_attempt_adjudications_order_guard') + expect(sql).toContain('gapless sequence order') + + // Every REVOKE ALL follows its guard function, matching the 0030 convention. + expect(sql).toContain('REVOKE ALL ON FUNCTION public.forge_reject_capability_attempt_mutation_v1() FROM PUBLIC') + expect(sql).toContain('REVOKE ALL ON FUNCTION public.forge_guard_capability_adjudication_insert_v1() FROM PUBLIC') + expect(sql).toContain('REVOKE ALL ON FUNCTION public.forge_reject_capability_adjudication_mutation_v1() FROM PUBLIC') + + // No free-text column: every text column in both tables is closed by a + // CHECK. This asserts each of them exists (I1). + expect(sql).toContain('capability_attempts_capability_key_check') + expect(sql).toContain('capability_attempts_classification_state_check') + expect(sql).toContain('capability_attempts_cohort_fingerprint_check') + expect(sql).toContain('capability_attempts_scope_fingerprint_check') + expect(sql).toContain('capability_attempts_runtime_fingerprint_check') + expect(sql).toContain('capability_attempts_policy_fingerprint_check') + expect(sql).toContain('capability_attempts_outcome_digest_check') + expect(sql).toContain('capability_attempts_transport_status_check') + expect(sql).toContain('capability_attempts_result_check') + expect(sql).toContain('capability_attempts_stop_reason_code_check') + expect(sql).toContain('capability_attempts_severity_class_check') + expect(sql).toContain('capability_attempts_verification_mode_value_check') + expect(sql).toContain('capability_attempts_verification_status_check') + expect(sql).toContain('capability_attempt_adjudications_kind_check') + expect(sql).toContain('capability_attempt_adjudications_verification_mode_check') + expect(sql).toContain('capability_attempt_adjudications_verification_result_check') + expect(sql).toContain('capability_attempt_adjudications_human_decision_check') + expect(sql).toContain('capability_attempt_adjudications_observed_outcome_digest_check') + + // Verification-mode / verifier-required consistency (I6 storage half). + expect(sql).toContain('capability_attempts_verifier_consistency_check') + expect(sql).toContain('capability_attempts_verification_mode_check') + expect(sql).toContain('capability_attempts_unclassified_check') + expect(sql).toContain('capability_attempts_operation_runtime_check') + + // Shape closure per adjudication kind. + expect(sql).toContain('capability_attempt_adjudications_kind_shape_check') + }) + + it('grants only SELECT/INSERT to the ordinary application role in the CI ACL gate', async () => { + const yml = await fs.readFile(path.join(process.cwd(), '../.github/workflows/web-ci.yml'), 'utf8') + expect(yml).toContain('capability_attempts') + expect(yml).toContain('capability_attempt_adjudications') + expect(yml).toMatch(/GRANT SELECT, INSERT ON TABLE public\.capability_attempts,\s*\n\s*public\.capability_attempt_adjudications TO forge_app_test/) + }) +}) diff --git a/web/__tests__/local-projection-overlimit-archive.test.ts b/web/__tests__/local-projection-overlimit-archive.test.ts index 45c1c181..1e0f697f 100644 --- a/web/__tests__/local-projection-overlimit-archive.test.ts +++ b/web/__tests__/local-projection-overlimit-archive.test.ts @@ -378,11 +378,11 @@ describe('local-projection over-limit operator commands', () => { 'utf8', ) for (const evidence of [ - 'count(*) FROM drizzle.__drizzle_migrations) <> 31', - 'count(DISTINCT created_at) FROM drizzle.__drizzle_migrations) <> 31', + 'count(*) FROM drizzle.__drizzle_migrations) <> 32', + 'count(DISTINCT created_at) FROM drizzle.__drizzle_migrations) <> 32', 'created_at = 1784270400000', 'created_at = 1784274000000', - 'max(created_at) FROM drizzle.__drizzle_migrations) <> 1785993600000', + 'max(created_at) FROM drizzle.__drizzle_migrations) <> 1786080000000', "role.rolname = 'forge_local_projection_archiver'", 'role.rolpassword IS NULL', 'pg_catalog.pg_db_role_setting', diff --git a/web/__tests__/review-gates.test.ts b/web/__tests__/review-gates.test.ts index 16ca849b..27886ca6 100644 --- a/web/__tests__/review-gates.test.ts +++ b/web/__tests__/review-gates.test.ts @@ -1027,11 +1027,14 @@ describe('review gate contract', () => { }) expect(result).toMatchObject({ status: 'decided', decision: 'completed' }) - // dbSelect is called 6 times: gate, reviewRequirement, sourceArtifact, + // dbSelect is called 8 times: gate, reviewRequirement, sourceArtifact, // latest package artifact, then - // completeTaskIfReviewGatesSatisfied's package list + gate list. No QA-blocks- + // completeTaskIfReviewGatesSatisfied's package list + gate list, then the + // two capability-reliability-ledger adjudication lookups (human decision, + // verification) that resolve their execution_outcomes row and find no + // linked capability attempts in this mocked environment. No QA-blocks- // reviewer lookup happens for reviewer_only. - expect(mocks.dbSelect).toHaveBeenCalledTimes(6) + expect(mocks.dbSelect).toHaveBeenCalledTimes(8) }) it('completes a both-review package only after QA approves first and then Reviewer approves', async () => { diff --git a/web/db/migrations/0031_capability_reliability_ledger.sql b/web/db/migrations/0031_capability_reliability_ledger.sql new file mode 100644 index 00000000..19ba1541 --- /dev/null +++ b/web/db/migrations/0031_capability_reliability_ledger.sql @@ -0,0 +1,193 @@ +-- Capability reliability ledger (ADR 0012, issue #186). Attempts are +-- immutable append-only evidence; adjudications record later evidence +-- (verification, human decisions, rollback, override, drift) separately so +-- earlier evidence is never rewritten. No column in either table is free +-- text: every text column is a closed enum, a 64-hex fingerprint, or the +-- bounded capability-key grammar, each enforced below. +CREATE TABLE "capability_attempts" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "attempt_group_id" uuid NOT NULL, + "project_id" uuid NOT NULL, + "task_id" uuid NOT NULL, + "work_package_id" uuid, + "agent_run_id" uuid, + "task_attempt_id" uuid, + "execution_outcome_id" uuid NOT NULL, + "operation_run_id" uuid, + "contract_version" integer DEFAULT 1 NOT NULL, + "capability_key" text NOT NULL, + "classification_state" text NOT NULL, + "capability_multiplicity" integer NOT NULL, + "cohort_fingerprint" text NOT NULL, + "scope_fingerprint" text NOT NULL, + "runtime_fingerprint" text NOT NULL, + "policy_fingerprint" text NOT NULL, + "outcome_digest" text NOT NULL, + "transport_status" text NOT NULL, + "result" text NOT NULL, + "stop_reason_code" text, + "retryable" boolean NOT NULL, + "attempt_number" integer DEFAULT 1 NOT NULL, + "severity_class" text NOT NULL, + "verifier_required" boolean NOT NULL, + "verification_mode" text NOT NULL, + "verification_status" text NOT NULL, + "acceptance_criteria_total" integer DEFAULT 0 NOT NULL, + "validation_command_total" integer DEFAULT 0 NOT NULL, + "validation_command_failed" integer DEFAULT 0 NOT NULL, + "evidence_refs" jsonb DEFAULT '[]'::jsonb NOT NULL, + "observed_at" timestamp with time zone NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "capability_attempts_project_id_projects_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."projects"("id") ON DELETE restrict ON UPDATE no action, + CONSTRAINT "capability_attempts_task_id_tasks_id_fk" FOREIGN KEY ("task_id") REFERENCES "public"."tasks"("id") ON DELETE restrict ON UPDATE no action, + CONSTRAINT "capability_attempts_work_package_id_work_packages_id_fk" FOREIGN KEY ("work_package_id") REFERENCES "public"."work_packages"("id") ON DELETE set null ON UPDATE no action, + CONSTRAINT "capability_attempts_agent_run_id_agent_runs_id_fk" FOREIGN KEY ("agent_run_id") REFERENCES "public"."agent_runs"("id") ON DELETE set null ON UPDATE no action, + CONSTRAINT "capability_attempts_task_attempt_id_task_attempts_id_fk" FOREIGN KEY ("task_attempt_id") REFERENCES "public"."task_attempts"("id") ON DELETE set null ON UPDATE no action, + CONSTRAINT "capability_attempts_execution_outcome_id_execution_outcomes_id_fk" FOREIGN KEY ("execution_outcome_id") REFERENCES "public"."execution_outcomes"("id") ON DELETE restrict ON UPDATE no action, + CONSTRAINT "capability_attempts_operation_run_id_operation_runs_id_fk" FOREIGN KEY ("operation_run_id") REFERENCES "public"."operation_runs"("id") ON DELETE set null ON UPDATE no action, + CONSTRAINT "capability_attempts_contract_version_check" CHECK ("contract_version" = 1), + CONSTRAINT "capability_attempts_capability_key_check" CHECK ( + length("capability_key") <= 120 AND + "capability_key" ~ '^(workpackage:[a-z][a-z0-9-]{0,39}/[a-z][a-z0-9-]{0,39}|operation:[a-z][a-z0-9]*([._-][a-z0-9]+)+@[1-9][0-9]{0,3})$' + ), + CONSTRAINT "capability_attempts_classification_state_check" CHECK ("classification_state" IN ('classified', 'missing', 'overflow')), + CONSTRAINT "capability_attempts_capability_multiplicity_check" CHECK ("capability_multiplicity" BETWEEN 1 AND 12), + CONSTRAINT "capability_attempts_cohort_fingerprint_check" CHECK ("cohort_fingerprint" ~ '^[0-9a-f]{64}$'), + CONSTRAINT "capability_attempts_scope_fingerprint_check" CHECK ("scope_fingerprint" ~ '^[0-9a-f]{64}$'), + CONSTRAINT "capability_attempts_runtime_fingerprint_check" CHECK ("runtime_fingerprint" ~ '^[0-9a-f]{64}$'), + CONSTRAINT "capability_attempts_policy_fingerprint_check" CHECK ("policy_fingerprint" ~ '^[0-9a-f]{64}$'), + CONSTRAINT "capability_attempts_outcome_digest_check" CHECK ("outcome_digest" ~ '^[0-9a-f]{64}$'), + CONSTRAINT "capability_attempts_transport_status_check" CHECK ("transport_status" IN ('ok', 'error')), + CONSTRAINT "capability_attempts_result_check" CHECK ("result" IN ('completed', 'partial', 'refused', 'blocked', 'needs_attention', 'failed', 'cancelled')), + CONSTRAINT "capability_attempts_stop_reason_code_check" CHECK ("stop_reason_code" IS NULL OR "stop_reason_code" IN ('provider_transport_failure', 'model_refusal', 'invalid_output', 'validation_failed', 'missing_capability', 'admission_denied', 'policy_blocked', 'security_blocked', 'missing_repository_context', 'timeout', 'context_limit', 'output_limit', 'retry_exhausted', 'human_cancelled', 'unknown')), + CONSTRAINT "capability_attempts_attempt_number_check" CHECK ("attempt_number" >= 1), + CONSTRAINT "capability_attempts_severity_class_check" CHECK ("severity_class" IN ('normal', 'critical')), + CONSTRAINT "capability_attempts_verification_mode_value_check" CHECK ("verification_mode" IN ('none', 'self_reported', 'human_review', 'deterministic_adapter', 'independent_agent')), + CONSTRAINT "capability_attempts_verification_status_check" CHECK ("verification_status" IN ('not_required', 'pending', 'passed', 'failed', 'inconclusive')), + CONSTRAINT "capability_attempts_acceptance_criteria_total_check" CHECK ("acceptance_criteria_total" >= 0), + CONSTRAINT "capability_attempts_validation_command_total_check" CHECK ("validation_command_total" >= 0), + CONSTRAINT "capability_attempts_validation_command_failed_check" CHECK ("validation_command_failed" >= 0 AND "validation_command_failed" <= "validation_command_total"), + CONSTRAINT "capability_attempts_evidence_refs_check" CHECK (jsonb_typeof("evidence_refs") = 'array'), + CONSTRAINT "capability_attempts_verifier_consistency_check" CHECK ( + ("verifier_required" AND "verification_status" IN ('pending', 'passed', 'failed', 'inconclusive')) OR + (NOT "verifier_required" AND "verification_status" = 'not_required') + ), + CONSTRAINT "capability_attempts_verification_mode_check" CHECK ( + ("verification_mode" = 'none') = (NOT "verifier_required") + ), + CONSTRAINT "capability_attempts_unclassified_check" CHECK ( + ("classification_state" = 'classified') OR "capability_key" LIKE 'workpackage:%/unclassified' + ), + CONSTRAINT "capability_attempts_operation_runtime_check" CHECK ( + "operation_run_id" IS NULL OR "verification_mode" IN ('none', 'deterministic_adapter') + ) +); +--> statement-breakpoint +CREATE UNIQUE INDEX "capability_attempts_outcome_capability_idx" ON "capability_attempts" USING btree ("execution_outcome_id", "capability_key"); +--> statement-breakpoint +CREATE INDEX "capability_attempts_cohort_observed_at_idx" ON "capability_attempts" USING btree ("cohort_fingerprint", "observed_at" DESC); +--> statement-breakpoint +CREATE INDEX "capability_attempts_project_capability_idx" ON "capability_attempts" USING btree ("project_id", "capability_key"); +--> statement-breakpoint +CREATE INDEX "capability_attempts_attempt_group_idx" ON "capability_attempts" USING btree ("attempt_group_id"); +--> statement-breakpoint +CREATE INDEX "capability_attempts_execution_outcome_idx" ON "capability_attempts" USING btree ("execution_outcome_id"); +--> statement-breakpoint +CREATE FUNCTION "forge_reject_capability_attempt_mutation_v1"() RETURNS trigger +LANGUAGE plpgsql +SET search_path = pg_catalog, public +AS $$ +BEGIN + RAISE EXCEPTION 'capability attempts are append-only'; +END; +$$; +--> statement-breakpoint +REVOKE ALL ON FUNCTION public.forge_reject_capability_attempt_mutation_v1() FROM PUBLIC; +--> statement-breakpoint +CREATE TRIGGER "capability_attempts_append_only" +BEFORE UPDATE OR DELETE ON "capability_attempts" +FOR EACH ROW EXECUTE FUNCTION "forge_reject_capability_attempt_mutation_v1"(); +--> statement-breakpoint +CREATE TABLE "capability_attempt_adjudications" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "capability_attempt_id" uuid NOT NULL, + "sequence" integer NOT NULL, + "kind" text NOT NULL, + "verification_mode" text, + "verification_result" text, + "human_decision" text, + "decided_by" uuid, + "approval_gate_id" uuid, + "observed_outcome_digest" text, + "evidence_refs" jsonb DEFAULT '[]'::jsonb NOT NULL, + "observed_at" timestamp with time zone NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "capability_attempt_adjudications_attempt_id_fk" FOREIGN KEY ("capability_attempt_id") REFERENCES "public"."capability_attempts"("id") ON DELETE restrict ON UPDATE no action, + CONSTRAINT "capability_attempt_adjudications_decided_by_users_id_fk" FOREIGN KEY ("decided_by") REFERENCES "public"."users"("id") ON DELETE set null ON UPDATE no action, + CONSTRAINT "capability_attempt_adjudications_approval_gate_id_fk" FOREIGN KEY ("approval_gate_id") REFERENCES "public"."approval_gates"("id") ON DELETE set null ON UPDATE no action, + CONSTRAINT "capability_attempt_adjudications_sequence_check" CHECK ("sequence" >= 0), + CONSTRAINT "capability_attempt_adjudications_kind_check" CHECK ("kind" IN ('verification_recorded', 'human_decision', 'rollback_recorded', 'override_recorded', 'evidence_drift_detected')), + CONSTRAINT "capability_attempt_adjudications_verification_mode_check" CHECK ("verification_mode" IS NULL OR "verification_mode" IN ('none', 'self_reported', 'human_review', 'deterministic_adapter', 'independent_agent')), + CONSTRAINT "capability_attempt_adjudications_verification_result_check" CHECK ("verification_result" IS NULL OR "verification_result" IN ('passed', 'failed', 'inconclusive')), + CONSTRAINT "capability_attempt_adjudications_human_decision_check" CHECK ("human_decision" IS NULL OR "human_decision" IN ('accepted', 'rejected', 'cancelled')), + CONSTRAINT "capability_attempt_adjudications_observed_outcome_digest_check" CHECK ("observed_outcome_digest" IS NULL OR "observed_outcome_digest" ~ '^[0-9a-f]{64}$'), + CONSTRAINT "capability_attempt_adjudications_evidence_refs_check" CHECK (jsonb_typeof("evidence_refs") = 'array'), + CONSTRAINT "capability_attempt_adjudications_kind_shape_check" CHECK ( + ("kind" = 'verification_recorded' + AND "verification_mode" IS NOT NULL AND "verification_result" IS NOT NULL + AND "human_decision" IS NULL AND "observed_outcome_digest" IS NULL) + OR ("kind" = 'human_decision' + AND "human_decision" IS NOT NULL + AND "verification_mode" IS NULL AND "verification_result" IS NULL + AND "observed_outcome_digest" IS NULL) + OR ("kind" IN ('rollback_recorded', 'override_recorded') + AND "verification_mode" IS NULL AND "verification_result" IS NULL + AND "human_decision" IS NULL AND "observed_outcome_digest" IS NULL) + OR ("kind" = 'evidence_drift_detected' + AND "observed_outcome_digest" IS NOT NULL + AND "verification_mode" IS NULL AND "verification_result" IS NULL AND "human_decision" IS NULL) + ) +); +--> statement-breakpoint +CREATE UNIQUE INDEX "capability_attempt_adjudications_attempt_sequence_idx" ON "capability_attempt_adjudications" USING btree ("capability_attempt_id", "sequence"); +--> statement-breakpoint +CREATE INDEX "capability_attempt_adjudications_attempt_observed_at_idx" ON "capability_attempt_adjudications" USING btree ("capability_attempt_id", "observed_at"); +--> statement-breakpoint +CREATE FUNCTION "forge_guard_capability_adjudication_insert_v1"() RETURNS trigger +LANGUAGE plpgsql +SET search_path = pg_catalog, public +AS $$ +DECLARE + v_last_sequence integer; +BEGIN + PERFORM 1 FROM public.capability_attempts WHERE id = NEW.capability_attempt_id FOR UPDATE; + SELECT max(sequence) INTO v_last_sequence + FROM public.capability_attempt_adjudications + WHERE capability_attempt_id = NEW.capability_attempt_id; + IF NEW.sequence <> COALESCE(v_last_sequence + 1, 0) THEN + RAISE EXCEPTION 'capability attempt adjudications must be appended in gapless sequence order'; + END IF; + RETURN NEW; +END; +$$; +--> statement-breakpoint +REVOKE ALL ON FUNCTION public.forge_guard_capability_adjudication_insert_v1() FROM PUBLIC; +--> statement-breakpoint +CREATE TRIGGER "capability_attempt_adjudications_order_guard" +BEFORE INSERT ON "capability_attempt_adjudications" +FOR EACH ROW EXECUTE FUNCTION "forge_guard_capability_adjudication_insert_v1"(); +--> statement-breakpoint +CREATE FUNCTION "forge_reject_capability_adjudication_mutation_v1"() RETURNS trigger +LANGUAGE plpgsql +SET search_path = pg_catalog, public +AS $$ +BEGIN + RAISE EXCEPTION 'capability attempt adjudications are append-only'; +END; +$$; +--> statement-breakpoint +REVOKE ALL ON FUNCTION public.forge_reject_capability_adjudication_mutation_v1() FROM PUBLIC; +--> statement-breakpoint +CREATE TRIGGER "capability_attempt_adjudications_append_only" +BEFORE UPDATE OR DELETE ON "capability_attempt_adjudications" +FOR EACH ROW EXECUTE FUNCTION "forge_reject_capability_adjudication_mutation_v1"(); diff --git a/web/db/migrations/meta/_journal.json b/web/db/migrations/meta/_journal.json index 0c89d3b6..204d4d6d 100644 --- a/web/db/migrations/meta/_journal.json +++ b/web/db/migrations/meta/_journal.json @@ -218,6 +218,13 @@ "when": 1785993600000, "tag": "0030_operation_runs", "breakpoints": true + }, + { + "idx": 31, + "version": "7", + "when": 1786080000000, + "tag": "0031_capability_reliability_ledger", + "breakpoints": true } ] } diff --git a/web/db/schema.ts b/web/db/schema.ts index 1da5caa3..89be0f37 100644 --- a/web/db/schema.ts +++ b/web/db/schema.ts @@ -1597,6 +1597,134 @@ export const operationRunEvents = pgTable( export type OperationRunEvent = InferSelectModel export type NewOperationRunEvent = InferInsertModel +// --------------------------------------------------------------------------- +// capabilityAttempts and capabilityAttemptAdjudications +// --------------------------------------------------------------------------- +// Immutable, append-only capability reliability ledger (ADR 0012, issue #186). +// Attempts are one row per (execution_outcome_id, capability_key); later +// evidence (verification, human decisions, rollback, override, drift) is +// appended to the adjudications table rather than rewriting the attempt. +export const capabilityAttempts = pgTable( + 'capability_attempts', + { + id: uuid('id').primaryKey().defaultRandom(), + attemptGroupId: uuid('attempt_group_id').notNull(), + projectId: uuid('project_id').notNull().references(() => projects.id, { onDelete: 'restrict' }), + taskId: uuid('task_id').notNull().references(() => tasks.id, { onDelete: 'restrict' }), + workPackageId: uuid('work_package_id').references(() => workPackages.id, { onDelete: 'set null' }), + agentRunId: uuid('agent_run_id').references(() => agentRuns.id, { onDelete: 'set null' }), + taskAttemptId: uuid('task_attempt_id').references(() => taskAttempts.id, { onDelete: 'set null' }), + executionOutcomeId: uuid('execution_outcome_id').notNull().references(() => executionOutcomes.id, { onDelete: 'restrict' }), + operationRunId: uuid('operation_run_id').references(() => operationRuns.id, { onDelete: 'set null' }), + contractVersion: integer('contract_version').notNull().default(1), + capabilityKey: text('capability_key').notNull(), + classificationState: text('classification_state').notNull(), + capabilityMultiplicity: integer('capability_multiplicity').notNull(), + cohortFingerprint: text('cohort_fingerprint').notNull(), + scopeFingerprint: text('scope_fingerprint').notNull(), + runtimeFingerprint: text('runtime_fingerprint').notNull(), + policyFingerprint: text('policy_fingerprint').notNull(), + outcomeDigest: text('outcome_digest').notNull(), + transportStatus: text('transport_status').notNull(), + result: text('result').notNull(), + stopReasonCode: text('stop_reason_code'), + retryable: boolean('retryable').notNull(), + attemptNumber: integer('attempt_number').notNull().default(1), + severityClass: text('severity_class').notNull(), + verifierRequired: boolean('verifier_required').notNull(), + verificationMode: text('verification_mode').notNull(), + verificationStatus: text('verification_status').notNull(), + acceptanceCriteriaTotal: integer('acceptance_criteria_total').notNull().default(0), + validationCommandTotal: integer('validation_command_total').notNull().default(0), + validationCommandFailed: integer('validation_command_failed').notNull().default(0), + evidenceRefs: jsonb('evidence_refs').$type().notNull().default(sql`'[]'::jsonb`), + observedAt: timestamp('observed_at', tsOpts).notNull(), + createdAt: timestamp('created_at', tsOpts).defaultNow().notNull(), + }, + (t) => [ + uniqueIndex('capability_attempts_outcome_capability_idx').on(t.executionOutcomeId, t.capabilityKey), + index('capability_attempts_cohort_observed_at_idx').on(t.cohortFingerprint, t.observedAt), + index('capability_attempts_project_capability_idx').on(t.projectId, t.capabilityKey), + index('capability_attempts_attempt_group_idx').on(t.attemptGroupId), + index('capability_attempts_execution_outcome_idx').on(t.executionOutcomeId), + check('capability_attempts_contract_version_check', sql`${t.contractVersion} = 1`), + check('capability_attempts_capability_key_check', sql`length(${t.capabilityKey}) <= 120 AND ${t.capabilityKey} ~ '^(workpackage:[a-z][a-z0-9-]{0,39}/[a-z][a-z0-9-]{0,39}|operation:[a-z][a-z0-9]*([._-][a-z0-9]+)+@[1-9][0-9]{0,3})$'`), + check('capability_attempts_classification_state_check', sql`${t.classificationState} IN ('classified', 'missing', 'overflow')`), + check('capability_attempts_capability_multiplicity_check', sql`${t.capabilityMultiplicity} BETWEEN 1 AND 12`), + check('capability_attempts_cohort_fingerprint_check', sql`${t.cohortFingerprint} ~ '^[0-9a-f]{64}$'`), + check('capability_attempts_scope_fingerprint_check', sql`${t.scopeFingerprint} ~ '^[0-9a-f]{64}$'`), + check('capability_attempts_runtime_fingerprint_check', sql`${t.runtimeFingerprint} ~ '^[0-9a-f]{64}$'`), + check('capability_attempts_policy_fingerprint_check', sql`${t.policyFingerprint} ~ '^[0-9a-f]{64}$'`), + check('capability_attempts_outcome_digest_check', sql`${t.outcomeDigest} ~ '^[0-9a-f]{64}$'`), + check('capability_attempts_transport_status_check', sql`${t.transportStatus} IN ('ok', 'error')`), + check('capability_attempts_result_check', sql`${t.result} IN ('completed', 'partial', 'refused', 'blocked', 'needs_attention', 'failed', 'cancelled')`), + check('capability_attempts_stop_reason_code_check', sql`${t.stopReasonCode} IS NULL OR ${t.stopReasonCode} IN ('provider_transport_failure', 'model_refusal', 'invalid_output', 'validation_failed', 'missing_capability', 'admission_denied', 'policy_blocked', 'security_blocked', 'missing_repository_context', 'timeout', 'context_limit', 'output_limit', 'retry_exhausted', 'human_cancelled', 'unknown')`), + check('capability_attempts_attempt_number_check', sql`${t.attemptNumber} >= 1`), + check('capability_attempts_severity_class_check', sql`${t.severityClass} IN ('normal', 'critical')`), + check('capability_attempts_verification_mode_value_check', sql`${t.verificationMode} IN ('none', 'self_reported', 'human_review', 'deterministic_adapter', 'independent_agent')`), + check('capability_attempts_verification_status_check', sql`${t.verificationStatus} IN ('not_required', 'pending', 'passed', 'failed', 'inconclusive')`), + check('capability_attempts_acceptance_criteria_total_check', sql`${t.acceptanceCriteriaTotal} >= 0`), + check('capability_attempts_validation_command_total_check', sql`${t.validationCommandTotal} >= 0`), + check('capability_attempts_validation_command_failed_check', sql`${t.validationCommandFailed} >= 0 AND ${t.validationCommandFailed} <= ${t.validationCommandTotal}`), + check('capability_attempts_evidence_refs_check', sql`jsonb_typeof(${t.evidenceRefs}) = 'array'`), + check('capability_attempts_verifier_consistency_check', sql`(${t.verifierRequired} AND ${t.verificationStatus} IN ('pending', 'passed', 'failed', 'inconclusive')) OR (NOT ${t.verifierRequired} AND ${t.verificationStatus} = 'not_required')`), + check('capability_attempts_verification_mode_check', sql`(${t.verificationMode} = 'none') = (NOT ${t.verifierRequired})`), + check('capability_attempts_unclassified_check', sql`(${t.classificationState} = 'classified') OR ${t.capabilityKey} LIKE 'workpackage:%/unclassified'`), + check('capability_attempts_operation_runtime_check', sql`${t.operationRunId} IS NULL OR ${t.verificationMode} IN ('none', 'deterministic_adapter')`), + ], +) + +export type CapabilityAttempt = InferSelectModel +export type NewCapabilityAttempt = InferInsertModel + +export const capabilityAttemptAdjudications = pgTable( + 'capability_attempt_adjudications', + { + id: uuid('id').primaryKey().defaultRandom(), + capabilityAttemptId: uuid('capability_attempt_id').notNull().references(() => capabilityAttempts.id, { onDelete: 'restrict' }), + sequence: integer('sequence').notNull(), + kind: text('kind').notNull(), + verificationMode: text('verification_mode'), + verificationResult: text('verification_result'), + humanDecision: text('human_decision'), + decidedBy: uuid('decided_by').references(() => users.id, { onDelete: 'set null' }), + approvalGateId: uuid('approval_gate_id').references(() => approvalGates.id, { onDelete: 'set null' }), + observedOutcomeDigest: text('observed_outcome_digest'), + evidenceRefs: jsonb('evidence_refs').$type().notNull().default(sql`'[]'::jsonb`), + observedAt: timestamp('observed_at', tsOpts).notNull(), + createdAt: timestamp('created_at', tsOpts).defaultNow().notNull(), + }, + (t) => [ + uniqueIndex('capability_attempt_adjudications_attempt_sequence_idx').on(t.capabilityAttemptId, t.sequence), + index('capability_attempt_adjudications_attempt_observed_at_idx').on(t.capabilityAttemptId, t.observedAt), + check('capability_attempt_adjudications_sequence_check', sql`${t.sequence} >= 0`), + check('capability_attempt_adjudications_kind_check', sql`${t.kind} IN ('verification_recorded', 'human_decision', 'rollback_recorded', 'override_recorded', 'evidence_drift_detected')`), + check('capability_attempt_adjudications_verification_mode_check', sql`${t.verificationMode} IS NULL OR ${t.verificationMode} IN ('none', 'self_reported', 'human_review', 'deterministic_adapter', 'independent_agent')`), + check('capability_attempt_adjudications_verification_result_check', sql`${t.verificationResult} IS NULL OR ${t.verificationResult} IN ('passed', 'failed', 'inconclusive')`), + check('capability_attempt_adjudications_human_decision_check', sql`${t.humanDecision} IS NULL OR ${t.humanDecision} IN ('accepted', 'rejected', 'cancelled')`), + check('capability_attempt_adjudications_observed_outcome_digest_check', sql`${t.observedOutcomeDigest} IS NULL OR ${t.observedOutcomeDigest} ~ '^[0-9a-f]{64}$'`), + check('capability_attempt_adjudications_evidence_refs_check', sql`jsonb_typeof(${t.evidenceRefs}) = 'array'`), + check('capability_attempt_adjudications_kind_shape_check', sql` + (${t.kind} = 'verification_recorded' + AND ${t.verificationMode} IS NOT NULL AND ${t.verificationResult} IS NOT NULL + AND ${t.humanDecision} IS NULL AND ${t.observedOutcomeDigest} IS NULL) + OR (${t.kind} = 'human_decision' + AND ${t.humanDecision} IS NOT NULL + AND ${t.verificationMode} IS NULL AND ${t.verificationResult} IS NULL + AND ${t.observedOutcomeDigest} IS NULL) + OR (${t.kind} IN ('rollback_recorded', 'override_recorded') + AND ${t.verificationMode} IS NULL AND ${t.verificationResult} IS NULL + AND ${t.humanDecision} IS NULL AND ${t.observedOutcomeDigest} IS NULL) + OR (${t.kind} = 'evidence_drift_detected' + AND ${t.observedOutcomeDigest} IS NOT NULL + AND ${t.verificationMode} IS NULL AND ${t.verificationResult} IS NULL AND ${t.humanDecision} IS NULL) + `), + ], +) + +export type CapabilityAttemptAdjudication = InferSelectModel +export type NewCapabilityAttemptAdjudication = InferInsertModel + // --------------------------------------------------------------------------- // artifacts // --------------------------------------------------------------------------- diff --git a/web/lib/reliability/contracts.ts b/web/lib/reliability/contracts.ts new file mode 100644 index 00000000..ffff81c3 --- /dev/null +++ b/web/lib/reliability/contracts.ts @@ -0,0 +1,227 @@ +import { createHash } from 'node:crypto' + +import { canonicalJson, isPlainRecord } from '@/lib/operations/contracts' + +/** Versioned, provider-neutral capability reliability ledger contract. */ +export const RELIABILITY_LEDGER_CONTRACT_VERSION = 1 as const + +/** Bumped when the meaning of a cohort input changes; part of the policy fingerprint. */ +export const RELIABILITY_POLICY_VERSION = 'reliability-policy-v1' as const + +export const MAX_CAPABILITY_FAN_OUT = 12 +export const CAPABILITY_KEY_MAX_LENGTH = 120 + +export const CAPABILITY_KEY_PATTERN = + /^(?:workpackage:[a-z][a-z0-9-]{0,39}\/[a-z][a-z0-9-]{0,39}|operation:[a-z][a-z0-9]*(?:[._-][a-z0-9]+)+@[1-9][0-9]{0,3})$/ + +export function isValidCapabilityKey(value: unknown): value is string { + return typeof value === 'string' + && value.length > 0 + && value.length <= CAPABILITY_KEY_MAX_LENGTH + && CAPABILITY_KEY_PATTERN.test(value) +} + +export function unclassifiedCapabilityKey(role: string): string { + return `workpackage:${role}/unclassified` +} + +export const CAPABILITY_CLASSIFICATION_STATES = ['classified', 'missing', 'overflow'] as const +export type CapabilityClassificationState = typeof CAPABILITY_CLASSIFICATION_STATES[number] + +export const VERIFICATION_MODES = [ + 'none', + 'self_reported', + 'human_review', + 'deterministic_adapter', + 'independent_agent', +] as const +export type VerificationMode = typeof VERIFICATION_MODES[number] + +export const SEVERITY_CLASSES = ['normal', 'critical'] as const +export type SeverityClass = typeof SEVERITY_CLASSES[number] + +export const ADJUDICATION_KINDS = [ + 'verification_recorded', + 'human_decision', + 'rollback_recorded', + 'override_recorded', + 'evidence_drift_detected', +] as const +export type AdjudicationKind = typeof ADJUDICATION_KINDS[number] + +export const HUMAN_DECISIONS = ['accepted', 'rejected', 'cancelled'] as const +export type HumanDecision = typeof HUMAN_DECISIONS[number] + +export const VERIFICATION_RESULTS = ['passed', 'failed', 'inconclusive'] as const +export type CapabilityVerificationResult = typeof VERIFICATION_RESULTS[number] + +/** Domain-separated SHA-256, matching `operationFingerprint`'s construction. */ +export function reliabilityFingerprint(domain: string, value: unknown): string { + return createHash('sha256') + .update(`forge:reliability:${domain}:v1\0`, 'utf8') + .update(canonicalJson(value), 'utf8') + .digest('hex') +} + +export type ReliabilityScopeInput = { + contractVersion: 1 + projectId: string + rootRef: string | null + rootBindingRevision: string + grantDecisionRevision: string + repositoryWriteIntent: boolean + capabilities: string[] + mcpRequirementKeys: string[] +} + +export type ReliabilityRuntimeInput = + | { + kind: 'model' + providerType: string | null + modelId: string + providerIsLocal: boolean | null + providerConfigUpdatedAt: string | null + acpExecutionMode: string + } + | { + kind: 'deterministic_adapter' + adapterKind: string + } + +export type ReliabilityPolicyInput = { + contractVersion: 1 + policyVersion: string + harnessId: string | null + harnessUpdatedAt: string | null + reviewRequirement: 'none' | 'qa_only' | 'reviewer_only' | 'both' + repositoryWritesEnabled: boolean +} + +function sortedUnique(values: string[]): string[] { + return [...new Set(values)].sort() +} + +export function scopeFingerprint(input: ReliabilityScopeInput): string { + return reliabilityFingerprint('scope', { + ...input, + capabilities: sortedUnique(input.capabilities), + mcpRequirementKeys: sortedUnique(input.mcpRequirementKeys), + }) +} + +export function runtimeFingerprint(input: ReliabilityRuntimeInput): string { + return reliabilityFingerprint('runtime', input) +} + +export function policyFingerprint(input: ReliabilityPolicyInput): string { + return reliabilityFingerprint('policy', input) +} + +export function cohortFingerprint(input: { + projectId: string + capabilityKey: string + scopeFingerprint: string + runtimeFingerprint: string + policyFingerprint: string +}): string { + return reliabilityFingerprint('cohort', { + contractVersion: RELIABILITY_LEDGER_CONTRACT_VERSION, + projectId: input.projectId, + capabilityKey: input.capabilityKey, + scopeFingerprint: input.scopeFingerprint, + runtimeFingerprint: input.runtimeFingerprint, + policyFingerprint: input.policyFingerprint, + }) +} + +/** Fingerprints the normalized outcome so drift is detectable at read time. */ +export function outcomeDigest(outcome: unknown): string { + if (!isPlainRecord(outcome)) throw new Error('Outcome digest requires a plain object.') + return reliabilityFingerprint('outcome-digest', outcome) +} + +export const RELIABILITY_WINDOW_DEFAULTS = { + maxAttempts: 50, + maxAgeMs: 90 * 24 * 60 * 60 * 1000, + minSamples: 5, +} as const + +export type ReliabilityWindow = { + maxAttempts: number + maxAgeMs: number + minSamples: number +} + +export const DEFAULT_RELIABILITY_WINDOW: ReliabilityWindow = { ...RELIABILITY_WINDOW_DEFAULTS } + +export type ReliabilityState = 'ready' | 'insufficient_evidence' | 'evidence_drift' + +export type ReliabilityRates = { + firstAttemptSuccess: number | null + independentlyVerifiedPass: number | null + humanAccepted: number | null + unverifiedCompletion: number | null + repairRetry: number | null + humanRejection: number | null + rollback: number | null + policyBlock: number | null +} + +export type ReliabilityExclusion = { + reason: 'outside_window' | 'unclassified' | 'drifted' + count: number +} + +export type ReliabilitySummary = { + schemaVersion: 1 + cohortFingerprint: string + capabilityKey: string + state: ReliabilityState + sampleCount: number + uniqueAttemptCount: number + rates: ReliabilityRates + consecutiveVerifiedPasses: number + criticalFailureCount: number + lastCriticalAt: string | null + evidence: { + newestObservedAt: string | null + oldestObservedAt: string | null + freshnessMs: number | null + driftedAttemptCount: number + } + excluded: ReliabilityExclusion[] +} + +/** Immutable evidence row, as read back from `capability_attempts`. */ +export type CapabilityAttemptRecord = { + id: string + attemptGroupId: string + executionOutcomeId: string + capabilityKey: string + classificationState: CapabilityClassificationState + capabilityMultiplicity: number + cohortFingerprint: string + outcomeDigest: string + transportStatus: 'ok' | 'error' + result: 'completed' | 'partial' | 'refused' | 'blocked' | 'needs_attention' | 'failed' | 'cancelled' + stopReasonCode: string | null + retryable: boolean + attemptNumber: number + severityClass: SeverityClass + verifierRequired: boolean + verificationMode: VerificationMode + verificationStatus: 'not_required' | 'pending' | 'passed' | 'failed' | 'inconclusive' + observedAt: string + currentOutcomeDigest?: string +} + +export type CapabilityAdjudicationRecord = { + id: string + capabilityAttemptId: string + sequence: number + kind: AdjudicationKind + verificationMode: VerificationMode | null + verificationResult: CapabilityVerificationResult | null + humanDecision: HumanDecision | null + observedAt: string +} diff --git a/web/lib/reliability/metrics.ts b/web/lib/reliability/metrics.ts new file mode 100644 index 00000000..034808d4 --- /dev/null +++ b/web/lib/reliability/metrics.ts @@ -0,0 +1,239 @@ +import type { + CapabilityAdjudicationRecord, + CapabilityAttemptRecord, + ReliabilityRates, + ReliabilitySummary, + ReliabilityWindow, +} from './contracts' + +function rate(numerator: number, denominator: number): number | null { + if (denominator <= 0) return null + return numerator / denominator +} + +function isCritical(attempt: CapabilityAttemptRecord, adjudications: CapabilityAdjudicationRecord[]): boolean { + if (attempt.severityClass === 'critical') return true + return adjudications.some((a) => a.kind === 'rollback_recorded') +} + +function latestHumanDecision(adjudications: CapabilityAdjudicationRecord[]): CapabilityAdjudicationRecord | null { + const decisions = adjudications + .filter((a) => a.kind === 'human_decision') + .sort((a, b) => a.sequence - b.sequence) + return decisions.length > 0 ? decisions[decisions.length - 1] : null +} + +function latestVerification(adjudications: CapabilityAdjudicationRecord[]): CapabilityAdjudicationRecord | null { + const verifications = adjudications + .filter((a) => a.kind === 'verification_recorded') + .sort((a, b) => a.sequence - b.sequence) + return verifications.length > 0 ? verifications[verifications.length - 1] : null +} + +function isIndependentlyVerifiedPass(adjudications: CapabilityAdjudicationRecord[]): boolean { + const latest = latestVerification(adjudications) + if (!latest) return false + return (latest.verificationMode === 'deterministic_adapter' || latest.verificationMode === 'independent_agent') + && latest.verificationResult === 'passed' +} + +function hasRollback(adjudications: CapabilityAdjudicationRecord[]): boolean { + return adjudications.some((a) => a.kind === 'rollback_recorded') +} + +/** + * Pure function: no clock, no I/O. Every rate is null when its denominator is + * zero. Below minSamples or under any evidence drift, all rates are null but + * critical counts are still reported. + */ +export function computeReliability(input: { + attempts: CapabilityAttemptRecord[] + adjudications: CapabilityAdjudicationRecord[] + window: ReliabilityWindow + now: Date +}): ReliabilitySummary { + const { window, now } = input + const adjudicationsByAttempt = new Map() + for (const adj of input.adjudications) { + const list = adjudicationsByAttempt.get(adj.capabilityAttemptId) ?? [] + list.push(adj) + adjudicationsByAttempt.set(adj.capabilityAttemptId, list) + } + + const cutoff = now.getTime() - window.maxAgeMs + const sorted = [...input.attempts].sort( + (a, b) => new Date(b.observedAt).getTime() - new Date(a.observedAt).getTime(), + ) + + const inWindow: CapabilityAttemptRecord[] = [] + let outsideWindowCount = 0 + for (const attempt of sorted) { + if (inWindow.length >= window.maxAttempts) { + outsideWindowCount += 1 + continue + } + if (new Date(attempt.observedAt).getTime() < cutoff) { + outsideWindowCount += 1 + continue + } + inWindow.push(attempt) + } + + const cohortFingerprint = input.attempts[0]?.cohortFingerprint ?? '' + const capabilityKey = input.attempts[0]?.capabilityKey ?? '' + + // Critical failures are reported unconditionally, whatever state we end in. + let criticalFailureCount = 0 + let lastCriticalAt: string | null = null + for (const attempt of inWindow) { + const adjs = adjudicationsByAttempt.get(attempt.id) ?? [] + if (isCritical(attempt, adjs)) { + criticalFailureCount += 1 + if (!lastCriticalAt || new Date(attempt.observedAt) > new Date(lastCriticalAt)) { + lastCriticalAt = attempt.observedAt + } + } + } + + const driftedAttempts = inWindow.filter( + (a) => a.currentOutcomeDigest !== undefined && a.currentOutcomeDigest !== a.outcomeDigest, + ) + const driftedAttemptCount = driftedAttempts.length + + const newestObservedAt = inWindow.length > 0 ? inWindow[0].observedAt : null + const oldestObservedAt = inWindow.length > 0 ? inWindow[inWindow.length - 1].observedAt : null + const freshnessMs = newestObservedAt ? now.getTime() - new Date(newestObservedAt).getTime() : null + + const excluded: ReliabilitySummary['excluded'] = [] + if (outsideWindowCount > 0) excluded.push({ reason: 'outside_window', count: outsideWindowCount }) + if (driftedAttemptCount > 0) excluded.push({ reason: 'drifted', count: driftedAttemptCount }) + + const evidence = { newestObservedAt, oldestObservedAt, freshnessMs, driftedAttemptCount } + + if (driftedAttemptCount > 0) { + return { + schemaVersion: 1, + cohortFingerprint, + capabilityKey, + state: 'evidence_drift', + sampleCount: inWindow.length, + uniqueAttemptCount: new Set(inWindow.map((a) => a.attemptGroupId)).size, + rates: emptyRates(), + consecutiveVerifiedPasses: 0, + criticalFailureCount, + lastCriticalAt, + evidence, + excluded, + } + } + + if (inWindow.length < window.minSamples) { + return { + schemaVersion: 1, + cohortFingerprint, + capabilityKey, + state: 'insufficient_evidence', + sampleCount: inWindow.length, + uniqueAttemptCount: new Set(inWindow.map((a) => a.attemptGroupId)).size, + rates: emptyRates(), + consecutiveVerifiedPasses: 0, + criticalFailureCount, + lastCriticalAt, + evidence, + excluded, + } + } + + let firstAttemptTotal = 0 + let firstAttemptSuccess = 0 + let verifierRequiredTotal = 0 + let verifiedPassTotal = 0 + let completedTotal = 0 + let unverifiedCompletionTotal = 0 + let repairRetryTotal = 0 + let humanDecisionTotal = 0 + let humanAcceptedTotal = 0 + let humanRejectedTotal = 0 + let rollbackTotal = 0 + let policyBlockTotal = 0 + + for (const attempt of inWindow) { + const adjs = adjudicationsByAttempt.get(attempt.id) ?? [] + + if (attempt.attemptNumber === 1) { + firstAttemptTotal += 1 + if (attempt.result === 'completed') firstAttemptSuccess += 1 + } else { + repairRetryTotal += 1 + } + + if (attempt.verifierRequired) { + verifierRequiredTotal += 1 + if (isIndependentlyVerifiedPass(adjs)) verifiedPassTotal += 1 + } + + if (attempt.result === 'completed') { + completedTotal += 1 + if (!isIndependentlyVerifiedPass(adjs)) unverifiedCompletionTotal += 1 + } + + const humanDecision = latestHumanDecision(adjs) + if (humanDecision) { + humanDecisionTotal += 1 + if (humanDecision.humanDecision === 'accepted') humanAcceptedTotal += 1 + if (humanDecision.humanDecision === 'rejected') humanRejectedTotal += 1 + } + + if (hasRollback(adjs)) rollbackTotal += 1 + if (attempt.result === 'blocked') policyBlockTotal += 1 + } + + const rates: ReliabilityRates = { + firstAttemptSuccess: rate(firstAttemptSuccess, firstAttemptTotal), + independentlyVerifiedPass: rate(verifiedPassTotal, verifierRequiredTotal), + humanAccepted: rate(humanAcceptedTotal, humanDecisionTotal), + unverifiedCompletion: rate(unverifiedCompletionTotal, completedTotal), + repairRetry: rate(repairRetryTotal, inWindow.length), + humanRejection: rate(humanRejectedTotal, humanDecisionTotal), + rollback: rate(rollbackTotal, inWindow.length), + policyBlock: rate(policyBlockTotal, inWindow.length), + } + + let consecutiveVerifiedPasses = 0 + for (const attempt of inWindow) { + const adjs = adjudicationsByAttempt.get(attempt.id) ?? [] + if (isIndependentlyVerifiedPass(adjs)) { + consecutiveVerifiedPasses += 1 + } else { + break + } + } + + return { + schemaVersion: 1, + cohortFingerprint, + capabilityKey, + state: 'ready', + sampleCount: inWindow.length, + uniqueAttemptCount: new Set(inWindow.map((a) => a.attemptGroupId)).size, + rates, + consecutiveVerifiedPasses, + criticalFailureCount, + lastCriticalAt, + evidence, + excluded, + } +} + +function emptyRates(): ReliabilityRates { + return { + firstAttemptSuccess: null, + independentlyVerifiedPass: null, + humanAccepted: null, + unverifiedCompletion: null, + repairRetry: null, + humanRejection: null, + rollback: null, + policyBlock: null, + } +} diff --git a/web/package.json b/web/package.json index b4a2711f..98ff9617 100644 --- a/web/package.json +++ b/web/package.json @@ -30,6 +30,7 @@ "protocol:scrub-legacy-leakage": "tsx scripts/scrub-legacy-leakage.ts", "protocol:inspect-local-projection-overlimit": "tsx scripts/inspect-local-projection-overlimit.ts", "protocol:archive-local-projection-overlimit": "tsx scripts/archive-local-projection-overlimit.ts", + "protocol:inspect-capability-reliability": "tsx scripts/inspect-capability-reliability.ts", "session-credentials:reconcile": "tsx scripts/reconcile-session-credentials.ts", "project-roots:reconcile-expansion": "tsx scripts/reconcile-project-root-expansion.ts", "project-roots:build-concurrent-index": "tsx scripts/build-project-root-ref-index.ts", diff --git a/web/scripts/ci/prove-installer-legacy-migration-repair.sh b/web/scripts/ci/prove-installer-legacy-migration-repair.sh index 24c5b312..ad148d2b 100644 --- a/web/scripts/ci/prove-installer-legacy-migration-repair.sh +++ b/web/scripts/ci/prove-installer-legacy-migration-repair.sh @@ -1307,8 +1307,8 @@ assert_unchanged managed-latest-once managed-latest-twice 'Managed latest rerun' admin_psql <<'SQL' DO $proof$ BEGIN - IF (SELECT count(*) FROM drizzle.__drizzle_migrations) <> 31 - OR (SELECT max(created_at) FROM drizzle.__drizzle_migrations) <> 1785993600000 THEN + IF (SELECT count(*) FROM drizzle.__drizzle_migrations) <> 32 + OR (SELECT max(created_at) FROM drizzle.__drizzle_migrations) <> 1786080000000 THEN RAISE EXCEPTION 'Managed sequence did not reach the exact latest ledger'; END IF; IF EXISTS ( diff --git a/web/scripts/ci/sql/migration-0027-expansion-assertions.sql b/web/scripts/ci/sql/migration-0027-expansion-assertions.sql index 1aeff55e..7c1e37c7 100644 --- a/web/scripts/ci/sql/migration-0027-expansion-assertions.sql +++ b/web/scripts/ci/sql/migration-0027-expansion-assertions.sql @@ -13,11 +13,11 @@ BEGIN -- role.rolpassword IS NULL is verified by the administrator-only S4 -- bootstrap; pg_roles intentionally masks it from this ordinary migration -- proof. This block verifies every attribute visible to the ordinary login. - IF (SELECT count(*) FROM drizzle.__drizzle_migrations) <> 31 - OR (SELECT count(DISTINCT created_at) FROM drizzle.__drizzle_migrations) <> 31 + IF (SELECT count(*) FROM drizzle.__drizzle_migrations) <> 32 + OR (SELECT count(DISTINCT created_at) FROM drizzle.__drizzle_migrations) <> 32 OR NOT EXISTS (SELECT 1 FROM drizzle.__drizzle_migrations WHERE created_at = 1784270400000) OR NOT EXISTS (SELECT 1 FROM drizzle.__drizzle_migrations WHERE created_at = 1784274000000) - OR (SELECT max(created_at) FROM drizzle.__drizzle_migrations) <> 1785993600000 THEN + OR (SELECT max(created_at) FROM drizzle.__drizzle_migrations) <> 1786080000000 THEN RAISE EXCEPTION 'The normal migrator did not retain the immutable 0027/0028 release boundary while reaching the exact latest ledger'; END IF; diff --git a/web/scripts/inspect-capability-reliability.ts b/web/scripts/inspect-capability-reliability.ts new file mode 100644 index 00000000..17ba4495 --- /dev/null +++ b/web/scripts/inspect-capability-reliability.ts @@ -0,0 +1,113 @@ +import '../lib/load-env' +import { parseArgs } from 'node:util' +import { pathToFileURL } from 'node:url' + +import { and, eq } from 'drizzle-orm' + +import { db } from '../db' +import { capabilityAttempts } from '../db/schema' +import { readCohortReliability } from '../worker/reliability/reader' +import type { ReliabilitySummary } from '../lib/reliability/contracts' + +export function inspectCapabilityReliabilityUsage(): string { + return `Inspect capability reliability cohorts for a project (read-only) + +Usage: + npm run protocol:inspect-capability-reliability -- --project [--capability ] [--json] + +Options: + --project Project id (required) + --capability Restrict to one capability key (e.g. workpackage:backend/api-implementation) + --json Print machine-readable JSON instead of a table + +This command performs no writes. It reports what evidence exists so far; a +missing or small sample is reported as "not enough evidence yet", never as a +passing or failing grade.` +} + +export function parseInspectCapabilityReliabilityArgs(argv: string[]): { + projectId: string + capability: string | null + json: boolean +} { + const { values } = parseArgs({ + args: argv, + options: { + project: { type: 'string' }, + capability: { type: 'string' }, + json: { type: 'boolean', default: false }, + }, + }) + if (!values.project) throw new Error('--project is required.') + return { projectId: values.project, capability: values.capability ?? null, json: Boolean(values.json) } +} + +async function loadCohortsForProject(projectId: string, capability: string | null): Promise> { + const rows = await db + .selectDistinct({ + cohortFingerprint: capabilityAttempts.cohortFingerprint, + capabilityKey: capabilityAttempts.capabilityKey, + }) + .from(capabilityAttempts) + .where( + capability + ? and(eq(capabilityAttempts.projectId, projectId), eq(capabilityAttempts.capabilityKey, capability)) + : eq(capabilityAttempts.projectId, projectId), + ) + return rows +} + +function describeState(summary: ReliabilitySummary): string { + if (summary.state === 'insufficient_evidence') { + return `not enough evidence yet (${summary.sampleCount} attempt${summary.sampleCount === 1 ? '' : 's'} recorded)` + } + if (summary.state === 'evidence_drift') { + return 'evidence has drifted since it was recorded; rates withheld' + } + const verified = summary.rates.independentlyVerifiedPass + const verifiedText = verified === null ? 'no independently verified attempts yet' : `${(verified * 100).toFixed(0)}% independently verified pass rate` + return `${summary.sampleCount} attempts in window, ${verifiedText}` +} + +export async function inspectCapabilityReliability(input: { + projectId: string + capability: string | null +}): Promise> { + const cohorts = await loadCohortsForProject(input.projectId, input.capability) + const results: Array<{ capabilityKey: string; summary: ReliabilitySummary }> = [] + for (const cohort of cohorts) { + const summary = await readCohortReliability({ cohortFingerprint: cohort.cohortFingerprint }) + results.push({ capabilityKey: cohort.capabilityKey, summary }) + } + return results +} + +async function main(): Promise { + const args = parseInspectCapabilityReliabilityArgs(process.argv.slice(2)) + const results = await inspectCapabilityReliability(args) + if (args.json) { + console.log(JSON.stringify(results, null, 2)) + return + } + if (results.length === 0) { + console.log('No capability reliability evidence recorded for this project yet.') + return + } + for (const { capabilityKey, summary } of results) { + console.log(`${capabilityKey}`) + console.log(` state: ${summary.state}`) + console.log(` ${describeState(summary)}`) + console.log(` critical failures: ${summary.criticalFailureCount}${summary.lastCriticalAt ? ` (last ${summary.lastCriticalAt})` : ''}`) + console.log('') + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + main().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)) + process.exitCode = 1 + }) +} diff --git a/web/worker/execution-outcomes.ts b/web/worker/execution-outcomes.ts index 525cfb19..723903c0 100644 --- a/web/worker/execution-outcomes.ts +++ b/web/worker/execution-outcomes.ts @@ -11,12 +11,12 @@ export async function upsertExecutionOutcome(input: { agentRunId?: string | null taskAttemptId?: string | null outcome: ExecutionOutcome -}): Promise { +}): Promise<{ id: string }> { if (!input.attemptKey || input.attemptKey.length > 200) { throw new Error('Execution outcome attempt key is required and bounded.') } const outcome = normalizeExecutionOutcome(input.outcome, sanitizeWorkerMessage) - await db + const [row] = await db .insert(executionOutcomes) .values({ taskId: input.taskId, @@ -44,4 +44,7 @@ export async function upsertExecutionOutcome(input: { updatedAt: new Date(), }, }) + .returning({ id: executionOutcomes.id }) + if (!row) throw new Error('Execution outcome was not stored.') + return row } diff --git a/web/worker/operations/ledger.ts b/web/worker/operations/ledger.ts index d30a4a08..fe92ab3e 100644 --- a/web/worker/operations/ledger.ts +++ b/web/worker/operations/ledger.ts @@ -14,6 +14,13 @@ import { type OperationVerificationStatus, } from '@/lib/operations/contracts' import { sanitizeWorkerMessage } from '@/worker/redaction' +import { + buildOperationPolicyInput, + buildOperationReliabilityScope, + buildOperationRuntimeInput, +} from '@/worker/reliability/context' +import { recordCapabilityAttemptsBestEffort } from '@/worker/reliability/ledger' +import { resolveOperationDefinition } from '@/lib/operations/catalog' import type { OperationLedger, OperationRunEventWrite, @@ -164,8 +171,9 @@ export function createDatabaseOperationLedger(database: typeof db = db): Operati }, async finalize(input: OperationRunFinalization) { + let txResult: { executionOutcomeId: string } try { - return await database.transaction(async (tx) => { + txResult = await database.transaction(async (tx) => { const outcome = normalizeExecutionOutcome(input.outcome, sanitizeWorkerMessage) const [outcomeRow] = await tx .insert(executionOutcomes) @@ -225,8 +233,97 @@ export function createDatabaseOperationLedger(database: typeof db = db): Operati } catch (err) { unwrapDatabaseError(err) } + // Ledger write happens after the ADR 0011 transaction has committed -- + // never inside it -- and is best-effort so it can never affect the + // operation run's already-terminal outcome. + await recordOperationCapabilityAttemptBestEffort({ + database, + runId: input.runId, + executionOutcomeId: txResult.executionOutcomeId, + }) + return txResult }, } } +/** + * Reads the immutable identity columns of a terminal operation_runs row and + * records the corresponding capability attempt. operationId/operationVersion + * are read here rather than threaded through OperationRunFinalization, + * keeping the ADR 0011 executor contract unchanged. + */ +async function recordOperationCapabilityAttemptBestEffort(input: { + database: typeof db + runId: string + executionOutcomeId: string +}): Promise { + try { + const [run] = await input.database + .select({ + taskId: operationRuns.taskId, + projectId: operationRuns.projectId, + workPackageId: operationRuns.workPackageId, + agentRunId: operationRuns.agentRunId, + taskAttemptId: operationRuns.taskAttemptId, + operationId: operationRuns.operationId, + operationVersion: operationRuns.operationVersion, + capability: operationRuns.capability, + status: operationRuns.status, + verificationStatus: operationRuns.verificationStatus, + completedAt: operationRuns.completedAt, + }) + .from(operationRuns) + .where(eq(operationRuns.id, input.runId)) + .limit(1) + if (!run) return + const definition = resolveOperationDefinition({ + operationId: run.operationId, + operationVersion: run.operationVersion, + }) + const scope = await buildOperationReliabilityScope({ projectId: run.projectId, capability: run.capability }) + if (!scope) return + const [outcomeRow] = await input.database + .select() + .from(executionOutcomes) + .where(eq(executionOutcomes.id, input.executionOutcomeId)) + .limit(1) + if (!outcomeRow) return + const outcomeCandidate: unknown = { + schemaVersion: outcomeRow.schemaVersion, + transportStatus: outcomeRow.transportStatus, + result: outcomeRow.result, + stopReasonCode: outcomeRow.stopReasonCode, + stopReasonSummary: outcomeRow.stopReasonSummary, + retryable: outcomeRow.retryable, + evidenceRefs: outcomeRow.evidenceRefs, + verifierRequired: outcomeRow.verifierRequired, + verificationStatus: outcomeRow.verificationStatus, + } + if (!isExecutionOutcome(outcomeCandidate)) return + await recordCapabilityAttemptsBestEffort({ + projectId: run.projectId, + taskId: run.taskId, + workPackageId: run.workPackageId, + agentRunId: run.agentRunId, + taskAttemptId: run.taskAttemptId, + executionOutcomeId: input.executionOutcomeId, + operationRunId: input.runId, + outcome: outcomeCandidate, + attemptNumber: 1, + source: { kind: 'operation', operationId: run.operationId, operationVersion: run.operationVersion }, + scope, + runtime: buildOperationRuntimeInput(definition.adapter), + policy: buildOperationPolicyInput(), + verificationMode: 'deterministic_adapter', + acceptanceCriteriaTotal: 0, + validationCommandTotal: 0, + validationCommandFailed: 0, + observedAt: run.completedAt ?? new Date(), + }) + } catch { + // Best-effort: the capability ledger is an interpretation layer over the + // already-committed operation run and canonical outcome. + } +} + export const databaseOperationLedger = createDatabaseOperationLedger() diff --git a/web/worker/reliability/context.ts b/web/worker/reliability/context.ts new file mode 100644 index 00000000..c080d0a5 --- /dev/null +++ b/web/worker/reliability/context.ts @@ -0,0 +1,292 @@ +import { eq } from 'drizzle-orm' + +import { db } from '@/db' +import { agentHarnesses, agentRuns, projects, providerConfigs, tasks, workPackages } from '@/db/schema' +import { RELIABILITY_POLICY_VERSION } from '@/lib/reliability/contracts' +import type { + ReliabilityPolicyInput, + ReliabilityRuntimeInput, + ReliabilityScopeInput, +} from '@/lib/reliability/contracts' +import type { CapabilitySource } from './ledger' + +function extractRequiredCapabilities(requiredCapabilities: unknown): string[] | null { + if ( + requiredCapabilities + && typeof requiredCapabilities === 'object' + && Array.isArray((requiredCapabilities as { required?: unknown }).required) + ) { + return (requiredCapabilities as { required: unknown[] }).required.filter( + (c): c is string => typeof c === 'string', + ) + } + return null +} + +/** Reads the Architect's required-capability classification for one work package. */ +export async function loadWorkPackageCapabilitySource(pkg: { + id: string + assignedRole: string +}): Promise { + const [row] = await db + .select({ requiredCapabilities: workPackages.requiredCapabilities }) + .from(workPackages) + .where(eq(workPackages.id, pkg.id)) + .limit(1) + return { + kind: 'work_package', + role: pkg.assignedRole, + capabilities: extractRequiredCapabilities(row?.requiredCapabilities), + } +} + +/** Like `loadWorkPackageCapabilitySource`, plus the declared acceptance-criteria count. */ +export async function loadWorkPackageCapabilityContext(pkg: { + id: string + assignedRole: string +}): Promise<{ source: CapabilitySource; acceptanceCriteriaTotal: number }> { + const [row] = await db + .select({ + requiredCapabilities: workPackages.requiredCapabilities, + acceptanceCriteria: workPackages.acceptanceCriteria, + }) + .from(workPackages) + .where(eq(workPackages.id, pkg.id)) + .limit(1) + return { + source: { + kind: 'work_package', + role: pkg.assignedRole, + capabilities: extractRequiredCapabilities(row?.requiredCapabilities), + }, + acceptanceCriteriaTotal: Array.isArray(row?.acceptanceCriteria) ? row.acceptanceCriteria.length : 0, + } +} + +/** + * Reads the project's opaque root identity and assembles the scope + * fingerprint input. One extra indexed read, acceptable per the ingest + * boundary guidance -- never a live re-read of anything mutable beyond this. + */ +export async function buildWorkPackageReliabilityScope(input: { + projectId: string + rootBindingRevision: bigint + grantDecisionRevision: bigint + repositoryWriteIntent: boolean + capabilities: string[] + mcpRequirementKeys: string[] +}): Promise { + const [project] = await db + .select({ rootRef: projects.rootRef }) + .from(projects) + .where(eq(projects.id, input.projectId)) + .limit(1) + return { + contractVersion: 1, + projectId: input.projectId, + rootRef: project?.rootRef ?? null, + rootBindingRevision: input.rootBindingRevision.toString(), + grantDecisionRevision: input.grantDecisionRevision.toString(), + repositoryWriteIntent: input.repositoryWriteIntent, + capabilities: input.capabilities, + mcpRequirementKeys: input.mcpRequirementKeys, + } +} + +/** + * Loads project scope fresh via the task, for ingest boundaries (completion, + * failure) where no already-locked project snapshot is in scope. Returns + * null when the task or its project is unavailable -- callers must skip + * ingest rather than fabricate scope identity. + */ +export async function buildWorkPackageReliabilityScopeForTask(input: { + taskId: string + repositoryWriteIntent: boolean + capabilities: string[] + mcpRequirementKeys: string[] +}): Promise { + const [row] = await db + .select({ + projectId: tasks.projectId, + rootRef: projects.rootRef, + rootBindingRevision: projects.rootBindingRevision, + grantDecisionRevision: projects.grantDecisionRevision, + }) + .from(tasks) + .innerJoin(projects, eq(projects.id, tasks.projectId)) + .where(eq(tasks.id, input.taskId)) + .limit(1) + if (!row) return null + return { + contractVersion: 1, + projectId: row.projectId, + rootRef: row.rootRef ?? null, + rootBindingRevision: row.rootBindingRevision.toString(), + grantDecisionRevision: row.grantDecisionRevision.toString(), + repositoryWriteIntent: input.repositoryWriteIntent, + capabilities: input.capabilities, + mcpRequirementKeys: input.mcpRequirementKeys, + } +} + +/** Scope input for a deterministic operation attempt (ADR 0011). One extra project read. */ +export async function buildOperationReliabilityScope(input: { + projectId: string + capability: string +}): Promise { + const [project] = await db + .select({ + rootRef: projects.rootRef, + rootBindingRevision: projects.rootBindingRevision, + grantDecisionRevision: projects.grantDecisionRevision, + }) + .from(projects) + .where(eq(projects.id, input.projectId)) + .limit(1) + if (!project) return null + return { + contractVersion: 1, + projectId: input.projectId, + rootRef: project.rootRef ?? null, + rootBindingRevision: project.rootBindingRevision.toString(), + grantDecisionRevision: project.grantDecisionRevision.toString(), + repositoryWriteIntent: false, + capabilities: [input.capability], + mcpRequirementKeys: [], + } +} + +/** Runtime identity for a deterministic operation attempt (ADR 0011). */ +export function buildOperationRuntimeInput(adapterKind: string): ReliabilityRuntimeInput { + return { kind: 'deterministic_adapter', adapterKind } +} + +/** Runtime identity for an attempt that actually executed, from the agent-run snapshot. */ +export function buildExecutedRuntimeInput(run: { + providerTypeUsed: string | null + modelIdUsed: string + providerIsLocalUsed: boolean | null + providerConfigUpdatedAtUsed: Date | null + acpExecutionMode: string +}): ReliabilityRuntimeInput { + return { + kind: 'model', + providerType: run.providerTypeUsed, + modelId: run.modelIdUsed, + providerIsLocal: run.providerIsLocalUsed, + providerConfigUpdatedAt: run.providerConfigUpdatedAtUsed + ? run.providerConfigUpdatedAtUsed.toISOString() + : null, + acpExecutionMode: run.acpExecutionMode, + } +} + +/** + * Loads the actual agent-run snapshot fresh by id rather than trusting a + * caller's in-scope `run` object, which on the protected S4 lifecycle path + * can be a partial `{ id }` stand-in. Returns null when the row or its + * model snapshot is unavailable -- the caller must skip ingest rather than + * fabricate a runtime fingerprint. + */ +export async function buildExecutedRuntimeInputFromRunId( + runId: string, +): Promise { + const [run] = await db + .select({ + providerTypeUsed: agentRuns.providerTypeUsed, + modelIdUsed: agentRuns.modelIdUsed, + providerIsLocalUsed: agentRuns.providerIsLocalUsed, + providerConfigUpdatedAtUsed: agentRuns.providerConfigUpdatedAtUsed, + acpExecutionMode: agentRuns.acpExecutionMode, + }) + .from(agentRuns) + .where(eq(agentRuns.id, runId)) + .limit(1) + if (!run?.modelIdUsed) return null + return buildExecutedRuntimeInput(run) +} + +/** + * Runtime identity for an admission block, which happens before any agent + * run exists. Uses the assigned harness's configured default provider as the + * intended runtime. Returns null when that identity is unavailable -- the + * caller must skip ingest rather than fabricate a runtime fingerprint. + */ +export async function buildIntendedRuntimeInputFromHarness( + harnessId: string | null, +): Promise { + if (!harnessId) return null + const [harness] = await db + .select({ defaultProviderConfigId: agentHarnesses.defaultProviderConfigId }) + .from(agentHarnesses) + .where(eq(agentHarnesses.id, harnessId)) + .limit(1) + if (!harness?.defaultProviderConfigId) return null + const [provider] = await db + .select({ + providerType: providerConfigs.providerType, + modelId: providerConfigs.modelId, + isLocal: providerConfigs.isLocal, + updatedAt: providerConfigs.updatedAt, + }) + .from(providerConfigs) + .where(eq(providerConfigs.id, harness.defaultProviderConfigId)) + .limit(1) + if (!provider) return null + return { + kind: 'model', + providerType: provider.providerType, + modelId: provider.modelId, + providerIsLocal: provider.isLocal, + providerConfigUpdatedAt: provider.updatedAt.toISOString(), + acpExecutionMode: 'not_applicable', + } +} + +/** Policy input for a deterministic operation attempt (ADR 0011): no harness, no review gate. */ +export function buildOperationPolicyInput(): ReliabilityPolicyInput { + return { + contractVersion: 1, + policyVersion: RELIABILITY_POLICY_VERSION, + harnessId: null, + harnessUpdatedAt: null, + reviewRequirement: 'none', + repositoryWritesEnabled: false, + } +} + +/** Extracts stable requirement keys from a work package's mcp_requirements array. */ +export function extractMcpRequirementKeys(mcpRequirements: unknown): string[] { + if (!Array.isArray(mcpRequirements)) return [] + const keys: string[] = [] + for (const item of mcpRequirements) { + if (item && typeof item === 'object' && typeof (item as { requirementKey?: unknown }).requirementKey === 'string') { + keys.push((item as { requirementKey: string }).requirementKey) + } + } + return keys +} + +export async function buildWorkPackagePolicyInput(input: { + harnessId: string | null + reviewRequirement: 'none' | 'qa_only' | 'reviewer_only' | 'both' + repositoryWritesEnabled: boolean +}): Promise { + let harnessUpdatedAt: string | null = null + if (input.harnessId) { + const [harness] = await db + .select({ updatedAt: agentHarnesses.updatedAt }) + .from(agentHarnesses) + .where(eq(agentHarnesses.id, input.harnessId)) + .limit(1) + harnessUpdatedAt = harness?.updatedAt ? harness.updatedAt.toISOString() : null + } + return { + contractVersion: 1, + policyVersion: RELIABILITY_POLICY_VERSION, + harnessId: input.harnessId, + harnessUpdatedAt, + reviewRequirement: input.reviewRequirement, + repositoryWritesEnabled: input.repositoryWritesEnabled, + } +} diff --git a/web/worker/reliability/ledger.ts b/web/worker/reliability/ledger.ts new file mode 100644 index 00000000..451766fe --- /dev/null +++ b/web/worker/reliability/ledger.ts @@ -0,0 +1,283 @@ +import { randomUUID } from 'node:crypto' + +import { and, desc, eq } from 'drizzle-orm' + +import { db } from '@/db' +import { capabilityAttemptAdjudications, capabilityAttempts, executionOutcomes } from '@/db/schema' +import type { ExecutionOutcome } from '@/lib/execution-outcomes' +import { + MAX_CAPABILITY_FAN_OUT, + cohortFingerprint, + isValidCapabilityKey, + outcomeDigest, + policyFingerprint, + runtimeFingerprint, + scopeFingerprint, + unclassifiedCapabilityKey, + type CapabilityVerificationResult, + type HumanDecision, + type ReliabilityPolicyInput, + type ReliabilityRuntimeInput, + type ReliabilityScopeInput, + type SeverityClass, + type VerificationMode, +} from '@/lib/reliability/contracts' +import { defaultOnFeatureFlagEnabled } from '../feature-flags' + +export type CapabilitySource = + | { kind: 'work_package'; role: string; capabilities: string[] | null } + | { kind: 'operation'; operationId: string; operationVersion: number } + +export type RecordCapabilityAttemptsInput = { + projectId: string + taskId: string + workPackageId: string | null + agentRunId: string | null + taskAttemptId: string | null + executionOutcomeId: string + operationRunId: string | null + outcome: ExecutionOutcome + attemptNumber: number + source: CapabilitySource + scope: ReliabilityScopeInput + runtime: ReliabilityRuntimeInput + policy: ReliabilityPolicyInput + verificationMode: VerificationMode + acceptanceCriteriaTotal: number + validationCommandTotal: number + validationCommandFailed: number + observedAt: Date +} + +function ledgerEnabled(): boolean { + return defaultOnFeatureFlagEnabled(process.env.FORGE_CAPABILITY_RELIABILITY_LEDGER) +} + +function slugifyRoleSegment(value: string): string { + const slug = value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 40) + return slug.length > 0 ? slug : 'unassigned' +} + +function severityFor(input: RecordCapabilityAttemptsInput): SeverityClass { + if (input.outcome.stopReasonCode === 'security_blocked' || input.outcome.stopReasonCode === 'policy_blocked') { + return 'critical' + } + if (input.scope.repositoryWriteIntent && input.validationCommandFailed > 0) return 'critical' + return 'normal' +} + +function resolveCapabilityKeys(source: CapabilitySource): { + keys: string[] + classificationState: 'classified' | 'missing' | 'overflow' +} { + if (source.kind === 'operation') { + return { + keys: [`operation:${source.operationId}@${source.operationVersion}`], + classificationState: 'classified', + } + } + const role = slugifyRoleSegment(source.role) + const raw = source.capabilities ?? [] + const normalized = [...new Set(raw.filter((c) => typeof c === 'string' && c.trim().length > 0))] + if (normalized.length === 0) { + return { keys: [unclassifiedCapabilityKey(role)], classificationState: 'missing' } + } + if (normalized.length > MAX_CAPABILITY_FAN_OUT) { + return { keys: [unclassifiedCapabilityKey(role)], classificationState: 'overflow' } + } + return { + keys: normalized.map((capability) => `workpackage:${role}/${capability}`), + classificationState: 'classified', + } +} + +/** + * Idempotently records one capability-attempt row per exercised capability, + * sharing an attempt_group_id and multiplicity count. Best-effort by design: + * a ledger write failure never fails the caller's task, package, run, or + * operation (see recordCapabilityAttemptsBestEffort). + */ +export async function recordCapabilityAttempts(input: RecordCapabilityAttemptsInput): Promise { + if (!ledgerEnabled()) return + + // independent_agent has no producer until #188; refuse rather than store + // an unbacked verification claim. + if (input.outcome.verifierRequired && input.verificationMode === 'independent_agent') return + + const { keys, classificationState } = resolveCapabilityKeys(input.source) + const validKeys = keys.filter(isValidCapabilityKey) + if (validKeys.length === 0) return + + const scopeFp = scopeFingerprint(input.scope) + const runtimeFp = runtimeFingerprint(input.runtime) + const policyFp = policyFingerprint(input.policy) + const digest = outcomeDigest(input.outcome) + const attemptGroupId = randomUUID() + const multiplicity = validKeys.length + const verificationMode = input.outcome.verifierRequired ? input.verificationMode : 'none' + + const rows = validKeys.map((capabilityKey) => ({ + id: randomUUID(), + attemptGroupId, + projectId: input.projectId, + taskId: input.taskId, + workPackageId: input.workPackageId, + agentRunId: input.agentRunId, + taskAttemptId: input.taskAttemptId, + executionOutcomeId: input.executionOutcomeId, + operationRunId: input.operationRunId, + contractVersion: 1, + capabilityKey, + classificationState, + capabilityMultiplicity: multiplicity, + cohortFingerprint: cohortFingerprint({ + projectId: input.projectId, + capabilityKey, + scopeFingerprint: scopeFp, + runtimeFingerprint: runtimeFp, + policyFingerprint: policyFp, + }), + scopeFingerprint: scopeFp, + runtimeFingerprint: runtimeFp, + policyFingerprint: policyFp, + outcomeDigest: digest, + transportStatus: input.outcome.transportStatus, + result: input.outcome.result, + stopReasonCode: input.outcome.stopReasonCode, + retryable: input.outcome.retryable, + attemptNumber: input.attemptNumber, + severityClass: severityFor(input), + verifierRequired: input.outcome.verifierRequired, + verificationMode, + verificationStatus: input.outcome.verificationStatus, + acceptanceCriteriaTotal: input.acceptanceCriteriaTotal, + validationCommandTotal: input.validationCommandTotal, + validationCommandFailed: input.validationCommandFailed, + evidenceRefs: input.outcome.evidenceRefs, + observedAt: input.observedAt, + })) + + await db + .insert(capabilityAttempts) + .values(rows) + .onConflictDoNothing({ + target: [capabilityAttempts.executionOutcomeId, capabilityAttempts.capabilityKey], + }) +} + +export async function recordCapabilityAttemptsBestEffort(input: RecordCapabilityAttemptsInput): Promise { + try { + await recordCapabilityAttempts(input) + } catch { + // Best-effort: the ledger is an interpretation layer, never a gate. + } +} + +async function nextAdjudicationSequence(capabilityAttemptId: string): Promise { + const [last] = await db + .select({ sequence: capabilityAttemptAdjudications.sequence }) + .from(capabilityAttemptAdjudications) + .where(eq(capabilityAttemptAdjudications.capabilityAttemptId, capabilityAttemptId)) + .orderBy(desc(capabilityAttemptAdjudications.sequence)) + .limit(1) + return last ? last.sequence + 1 : 0 +} + +async function findAttemptRowsForOutcome(executionOutcomeId: string): Promise> { + return db + .select({ id: capabilityAttempts.id }) + .from(capabilityAttempts) + .where(eq(capabilityAttempts.executionOutcomeId, executionOutcomeId)) +} + +async function findAttemptRowsForAttemptKey(taskId: string, attemptKey: string): Promise> { + const [outcome] = await db + .select({ id: executionOutcomes.id }) + .from(executionOutcomes) + .where(and(eq(executionOutcomes.taskId, taskId), eq(executionOutcomes.attemptKey, attemptKey))) + .limit(1) + if (!outcome) return [] + return findAttemptRowsForOutcome(outcome.id) +} + +/** Appends one verification_recorded adjudication per attempt row linked to (taskId, attemptKey). */ +export async function recordVerificationAdjudicationBestEffort(input: { + taskId: string + attemptKey: string + verificationMode: Exclude + verificationResult: CapabilityVerificationResult + observedAt: Date +}): Promise { + if (!ledgerEnabled()) return + try { + const rows = await findAttemptRowsForAttemptKey(input.taskId, input.attemptKey) + for (const row of rows) { + const sequence = await nextAdjudicationSequence(row.id) + await db.insert(capabilityAttemptAdjudications).values({ + capabilityAttemptId: row.id, + sequence, + kind: 'verification_recorded', + verificationMode: input.verificationMode, + verificationResult: input.verificationResult, + observedAt: input.observedAt, + }) + } + } catch { + // Best-effort: a missing attempt (ledger disabled window, or predates + // this table) is missing evidence, not an error to escalate. + } +} + +/** Appends one human_decision adjudication per attempt row linked to (taskId, attemptKey). */ +export async function recordHumanDecisionAdjudicationBestEffort(input: { + taskId: string + attemptKey: string + humanDecision: HumanDecision + decidedBy: string | null + approvalGateId: string | null + observedAt: Date +}): Promise { + if (!ledgerEnabled()) return + try { + const rows = await findAttemptRowsForAttemptKey(input.taskId, input.attemptKey) + for (const row of rows) { + const sequence = await nextAdjudicationSequence(row.id) + await db.insert(capabilityAttemptAdjudications).values({ + capabilityAttemptId: row.id, + sequence, + kind: 'human_decision', + humanDecision: input.humanDecision, + decidedBy: input.decidedBy, + approvalGateId: input.approvalGateId, + observedAt: input.observedAt, + }) + } + } catch { + // Best-effort, same rationale as above. + } +} + +/** Appends one evidence_drift_detected adjudication for a single attempt row. */ +export async function recordEvidenceDriftAdjudicationBestEffort(input: { + capabilityAttemptId: string + observedOutcomeDigest: string + observedAt: Date +}): Promise { + if (!ledgerEnabled()) return + try { + const sequence = await nextAdjudicationSequence(input.capabilityAttemptId) + await db.insert(capabilityAttemptAdjudications).values({ + capabilityAttemptId: input.capabilityAttemptId, + sequence, + kind: 'evidence_drift_detected', + observedOutcomeDigest: input.observedOutcomeDigest, + observedAt: input.observedAt, + }) + } catch { + // Best-effort. + } +} diff --git a/web/worker/reliability/reader.ts b/web/worker/reliability/reader.ts new file mode 100644 index 00000000..4053ec01 --- /dev/null +++ b/web/worker/reliability/reader.ts @@ -0,0 +1,134 @@ +import { and, desc, eq, gte, inArray } from 'drizzle-orm' + +import { db } from '@/db' +import { capabilityAttemptAdjudications, capabilityAttempts, executionOutcomes } from '@/db/schema' +import { + isExecutionOutcome, + normalizeExecutionOutcome, + type ExecutionOutcome, +} from '@/lib/execution-outcomes' +import { computeReliability } from '@/lib/reliability/metrics' +import { + DEFAULT_RELIABILITY_WINDOW, + outcomeDigest, + type CapabilityAdjudicationRecord, + type CapabilityAttemptRecord, + type ReliabilitySummary, + type ReliabilityWindow, +} from '@/lib/reliability/contracts' +import { recordEvidenceDriftAdjudicationBestEffort } from './ledger' +import { sanitizeWorkerMessage } from '../redaction' + +function storedOutcome(row: typeof executionOutcomes.$inferSelect): ExecutionOutcome | null { + const candidate: unknown = { + schemaVersion: row.schemaVersion, + transportStatus: row.transportStatus, + result: row.result, + stopReasonCode: row.stopReasonCode, + stopReasonSummary: row.stopReasonSummary, + retryable: row.retryable, + evidenceRefs: row.evidenceRefs, + verifierRequired: row.verifierRequired, + verificationStatus: row.verificationStatus, + } + if (!isExecutionOutcome(candidate)) return null + return candidate +} + +/** + * Reads a cohort's attempts and adjudications, detects drift against the + * linked execution_outcomes row, and computes the summary with the pure + * metrics function. Performs no arithmetic itself. + */ +export async function readCohortReliability(input: { + cohortFingerprint: string + window?: ReliabilityWindow + now?: Date +}): Promise { + const window = input.window ?? DEFAULT_RELIABILITY_WINDOW + const now = input.now ?? new Date() + const cutoff = new Date(now.getTime() - window.maxAgeMs) + + const attemptRows = await db + .select() + .from(capabilityAttempts) + .where(and( + eq(capabilityAttempts.cohortFingerprint, input.cohortFingerprint), + gte(capabilityAttempts.observedAt, cutoff), + )) + .orderBy(desc(capabilityAttempts.observedAt)) + .limit(window.maxAttempts) + + if (attemptRows.length === 0) { + return computeReliability({ attempts: [], adjudications: [], window, now }) + } + + const outcomeIds = [...new Set(attemptRows.map((r) => r.executionOutcomeId))] + const outcomeRows = await db + .select() + .from(executionOutcomes) + .where(inArray(executionOutcomes.id, outcomeIds)) + const outcomesById = new Map(outcomeRows.map((row) => [row.id, row])) + + const attempts: CapabilityAttemptRecord[] = [] + for (const row of attemptRows) { + const outcomeRow = outcomesById.get(row.executionOutcomeId) + let currentDigest: string | undefined + if (outcomeRow) { + const outcome = storedOutcome(outcomeRow) + if (outcome) { + const normalized = normalizeExecutionOutcome(outcome, sanitizeWorkerMessage) + currentDigest = outcomeDigest(normalized) + } + } + if (currentDigest && currentDigest !== row.outcomeDigest) { + await recordEvidenceDriftAdjudicationBestEffort({ + capabilityAttemptId: row.id, + observedOutcomeDigest: currentDigest, + observedAt: now, + }) + } + attempts.push({ + id: row.id, + attemptGroupId: row.attemptGroupId, + executionOutcomeId: row.executionOutcomeId, + capabilityKey: row.capabilityKey, + classificationState: row.classificationState as CapabilityAttemptRecord['classificationState'], + capabilityMultiplicity: row.capabilityMultiplicity, + cohortFingerprint: row.cohortFingerprint, + outcomeDigest: row.outcomeDigest, + transportStatus: row.transportStatus as CapabilityAttemptRecord['transportStatus'], + result: row.result as CapabilityAttemptRecord['result'], + stopReasonCode: row.stopReasonCode, + retryable: row.retryable, + attemptNumber: row.attemptNumber, + severityClass: row.severityClass as CapabilityAttemptRecord['severityClass'], + verifierRequired: row.verifierRequired, + verificationMode: row.verificationMode as CapabilityAttemptRecord['verificationMode'], + verificationStatus: row.verificationStatus as CapabilityAttemptRecord['verificationStatus'], + observedAt: row.observedAt.toISOString(), + currentOutcomeDigest: currentDigest, + }) + } + + const attemptIds = attempts.map((a) => a.id) + const adjudicationRows = attemptIds.length > 0 + ? await db + .select() + .from(capabilityAttemptAdjudications) + .where(inArray(capabilityAttemptAdjudications.capabilityAttemptId, attemptIds)) + : [] + + const adjudications: CapabilityAdjudicationRecord[] = adjudicationRows.map((row) => ({ + id: row.id, + capabilityAttemptId: row.capabilityAttemptId, + sequence: row.sequence, + kind: row.kind as CapabilityAdjudicationRecord['kind'], + verificationMode: row.verificationMode as CapabilityAdjudicationRecord['verificationMode'], + verificationResult: row.verificationResult as CapabilityAdjudicationRecord['verificationResult'], + humanDecision: row.humanDecision as CapabilityAdjudicationRecord['humanDecision'], + observedAt: row.observedAt.toISOString(), + })) + + return computeReliability({ attempts, adjudications, window, now }) +} diff --git a/web/worker/review-gates.ts b/web/worker/review-gates.ts index f9e1340e..c607bb86 100644 --- a/web/worker/review-gates.ts +++ b/web/worker/review-gates.ts @@ -6,6 +6,10 @@ import { sanitizeWorkerMessage } from './redaction' import { updateTaskStatusIfCurrent } from './task-state' import { convergeRecognizedOperatorHoldTask } from '../lib/mcps/filesystem-grant-reconciliation' import { resolveS4ReviewSourceV1 } from '../lib/mcps/review-source-resolver' +import { + recordHumanDecisionAdjudicationBestEffort, + recordVerificationAdjudicationBestEffort, +} from './reliability/ledger' export const REVIEW_GATE_TYPES = ['qa_review', 'reviewer_review', 'security_review'] as const export type ReviewGateType = typeof REVIEW_GATE_TYPES[number] @@ -1146,6 +1150,26 @@ export async function decideReviewGate(input: { ? await completeTaskIfReviewGatesSatisfied(input.taskId) : { status: 'blocked' as const } + if (sourceAgentRunId) { + const attemptKey = `work-package:${workPackageId}:run:${sourceAgentRunId}` + const humanDecision = input.decision === 'completed' ? 'accepted' : 'rejected' + await recordHumanDecisionAdjudicationBestEffort({ + taskId: input.taskId, + attemptKey, + humanDecision, + decidedBy: input.userId, + approvalGateId: gate.id, + observedAt: now, + }) + await recordVerificationAdjudicationBestEffort({ + taskId: input.taskId, + attemptKey, + verificationMode: 'human_review', + verificationResult: humanDecision === 'accepted' ? 'passed' : 'failed', + observedAt: now, + }) + } + // A review barrier may have been the last reason a running task could not // return to its durable operator hold. Recheck after every existing // post-decision projection has consumed the committed review state; no wake diff --git a/web/worker/work-package-handoff.ts b/web/worker/work-package-handoff.ts index 310b5a7a..edc7b053 100644 --- a/web/worker/work-package-handoff.ts +++ b/web/worker/work-package-handoff.ts @@ -71,7 +71,22 @@ import { import { explicitOptInFeatureFlagEnabled } from './feature-flags' import { sanitizeWorkerMessage } from './redaction' import { upsertExecutionOutcome } from './execution-outcomes' -import { executionFailureOutcome, outcomeEvidenceRefsFromArtifact } from '../lib/execution-outcomes' +import { + executionFailureOutcome, + outcomeEvidenceRefsFromArtifact, + type ExecutionOutcome, +} from '../lib/execution-outcomes' +import { + buildExecutedRuntimeInputFromRunId, + buildIntendedRuntimeInputFromHarness, + buildWorkPackagePolicyInput, + buildWorkPackageReliabilityScope, + buildWorkPackageReliabilityScopeForTask, + extractMcpRequirementKeys, + loadWorkPackageCapabilityContext, + loadWorkPackageCapabilitySource, +} from './reliability/context' +import { recordCapabilityAttemptsBestEffort } from './reliability/ledger' import { recordTaskLogBestEffort } from './task-logs' import { packetCandidateGuard } from '../lib/mcps/packet-issuance-v2' import { localEffectCandidateGuard } from '../lib/mcps/local-run-evidence-v2' @@ -428,12 +443,13 @@ async function publishTaskEventBestEffort( */ async function upsertExecutionOutcomeBestEffort( input: Parameters[0], -): Promise { +): Promise<{ id: string } | null> { try { - await upsertExecutionOutcome(input) + return await upsertExecutionOutcome(input) } catch (err) { const message = sanitizeWorkerMessage(err instanceof Error ? err.message : String(err)) console.warn(`Failed to record execution outcome for ${input.attemptKey}: ${message}`) + return null } } @@ -1949,30 +1965,154 @@ async function persistWorkPackageHandoffBlock(input: { break } if (result.status === 'blocked') { - await upsertExecutionOutcomeBestEffort({ + const admissionOutcome: ExecutionOutcome = { + schemaVersion: 1, + transportStatus: 'ok', + result: 'blocked', + stopReasonCode: input.decision.kind === 'reserved_role' + ? 'policy_blocked' + : input.decision.kind === 'broker' + ? 'admission_denied' + : 'missing_capability', + stopReasonSummary: result.blockedReason, + retryable: !result.terminalBlock, + evidenceRefs: [], + verifierRequired: false, + verificationStatus: 'not_required', + } + const storedOutcome = await upsertExecutionOutcomeBestEffort({ taskId: input.taskId, workPackageId: input.pkg.id, attemptKey: `work-package:${input.pkg.id}:admission`, - outcome: { - schemaVersion: 1, - transportStatus: 'ok', - result: 'blocked', - stopReasonCode: input.decision.kind === 'reserved_role' - ? 'policy_blocked' - : input.decision.kind === 'broker' - ? 'admission_denied' - : 'missing_capability', - stopReasonSummary: result.blockedReason, - retryable: !result.terminalBlock, - evidenceRefs: [], - verifierRequired: false, - verificationStatus: 'not_required', - }, + outcome: admissionOutcome, }) + if (storedOutcome) { + await recordAdmissionBlockCapabilityAttemptBestEffort({ + pkg: input.pkg, + project: input.project, + taskId: input.taskId, + executionOutcomeId: storedOutcome.id, + outcome: admissionOutcome, + }) + } } return result } +async function recordAdmissionBlockCapabilityAttemptBestEffort(input: { + pkg: HandoffPackage + project: McpProjectFreshnessSnapshot + taskId: string + executionOutcomeId: string + outcome: ExecutionOutcome +}): Promise { + try { + const runtime = await buildIntendedRuntimeInputFromHarness(input.pkg.harnessId) + if (!runtime) return + const source = await loadWorkPackageCapabilitySource(input.pkg) + const scope = await buildWorkPackageReliabilityScope({ + projectId: input.project.id, + rootBindingRevision: input.project.rootBindingRevision, + grantDecisionRevision: input.project.grantDecisionRevision, + repositoryWriteIntent: false, + capabilities: source.kind === 'work_package' ? (source.capabilities ?? []) : [], + mcpRequirementKeys: extractMcpRequirementKeys(input.pkg.mcpRequirements), + }) + const policy = await buildWorkPackagePolicyInput({ + harnessId: input.pkg.harnessId, + reviewRequirement: isReviewRequirement(input.pkg.reviewRequirement) ? input.pkg.reviewRequirement : 'both', + repositoryWritesEnabled: false, + }) + await recordCapabilityAttemptsBestEffort({ + projectId: input.project.id, + taskId: input.taskId, + workPackageId: input.pkg.id, + agentRunId: null, + taskAttemptId: null, + executionOutcomeId: input.executionOutcomeId, + operationRunId: null, + outcome: input.outcome, + attemptNumber: 1, + source, + scope, + runtime, + policy, + verificationMode: 'none', + acceptanceCriteriaTotal: 0, + validationCommandTotal: 0, + validationCommandFailed: 0, + observedAt: new Date(), + }) + } catch (err) { + const message = sanitizeWorkerMessage(err instanceof Error ? err.message : String(err)) + console.warn(`Failed to record capability attempt for admission block on work package ${input.pkg.id}: ${message}`) + } +} + +function isReviewRequirement(value: string | undefined): value is 'none' | 'qa_only' | 'reviewer_only' | 'both' { + return value === 'none' || value === 'qa_only' || value === 'reviewer_only' || value === 'both' +} + +/** + * Shared ingest for the completion and failure boundaries of a run. Loads + * the agent-run snapshot fresh by id (never trusting a possibly-partial + * `run` object already in scope) and skips ingest entirely when that + * runtime identity is unavailable, per the ledger's fail-closed contract. + */ +async function recordWorkPackageRunCapabilityAttemptBestEffort(input: { + taskId: string + pkg: HandoffPackage + runId: string + executionOutcomeId: string + outcome: ExecutionOutcome + attemptNumber: number + verificationMode: 'none' | 'human_review' + repositoryAffecting: boolean + validationCommandTotal: number + validationCommandFailed: number +}): Promise { + try { + const runtime = await buildExecutedRuntimeInputFromRunId(input.runId) + if (!runtime) return + const { source, acceptanceCriteriaTotal } = await loadWorkPackageCapabilityContext(input.pkg) + const scope = await buildWorkPackageReliabilityScopeForTask({ + taskId: input.taskId, + repositoryWriteIntent: input.repositoryAffecting, + capabilities: source.kind === 'work_package' ? (source.capabilities ?? []) : [], + mcpRequirementKeys: extractMcpRequirementKeys(input.pkg.mcpRequirements), + }) + if (!scope) return + const policy = await buildWorkPackagePolicyInput({ + harnessId: input.pkg.harnessId, + reviewRequirement: isReviewRequirement(input.pkg.reviewRequirement) ? input.pkg.reviewRequirement : 'both', + repositoryWritesEnabled: input.repositoryAffecting, + }) + await recordCapabilityAttemptsBestEffort({ + projectId: scope.projectId, + taskId: input.taskId, + workPackageId: input.pkg.id, + agentRunId: input.runId, + taskAttemptId: null, + executionOutcomeId: input.executionOutcomeId, + operationRunId: null, + outcome: input.outcome, + attemptNumber: input.attemptNumber, + source, + scope, + runtime, + policy, + verificationMode: input.verificationMode, + acceptanceCriteriaTotal, + validationCommandTotal: input.validationCommandTotal, + validationCommandFailed: input.validationCommandFailed, + observedAt: new Date(), + }) + } catch (err) { + const message = sanitizeWorkerMessage(err instanceof Error ? err.message : String(err)) + console.warn(`Failed to record capability attempt for run ${input.runId} on work package ${input.pkg.id}: ${message}`) + } +} + /** * Project-locked handoff invariant (no schema/version column required): live * MCP health is captured outside a transaction, then admission inputs are read @@ -3308,23 +3448,38 @@ async function executeReadyWorkPackage( artifact = protectedArtifact ?? null } if (!artifact) throw new Error('Work package completion did not create a source artifact.') - await upsertExecutionOutcomeBestEffort({ + const completionOutcome: ExecutionOutcome = { + schemaVersion: 1, + transportStatus: 'ok', + result: 'completed', + stopReasonCode: null, + stopReasonSummary: null, + retryable: false, + evidenceRefs: [artifact.id], + verifierRequired: (nextPackage.reviewRequirement ?? 'both') !== 'none', + verificationStatus: (nextPackage.reviewRequirement ?? 'both') === 'none' ? 'not_required' : 'pending', + } + const completionStoredOutcome = await upsertExecutionOutcomeBestEffort({ taskId, workPackageId: nextPackage.id, agentRunId: run.id, attemptKey: `work-package:${nextPackage.id}:run:${run.id}`, - outcome: { - schemaVersion: 1, - transportStatus: 'ok', - result: 'completed', - stopReasonCode: null, - stopReasonSummary: null, - retryable: false, - evidenceRefs: [artifact.id], - verifierRequired: (nextPackage.reviewRequirement ?? 'both') !== 'none', - verificationStatus: (nextPackage.reviewRequirement ?? 'both') === 'none' ? 'not_required' : 'pending', - }, + outcome: completionOutcome, }) + if (completionStoredOutcome) { + await recordWorkPackageRunCapabilityAttemptBestEffort({ + taskId, + pkg: nextPackage, + runId: run.id, + executionOutcomeId: completionStoredOutcome.id, + outcome: completionOutcome, + attemptNumber, + verificationMode: completionOutcome.verifierRequired ? 'human_review' : 'none', + repositoryAffecting, + validationCommandTotal: execution.commandResults.length, + validationCommandFailed: execution.commandResults.filter((result) => result.exitCode !== 0).length, + }) + } const packageStatus = reviewGates.packageStatus === 'awaiting_review' || reviewGates.packageStatus === 'completed' ? reviewGates.packageStatus : null @@ -3552,21 +3707,36 @@ async function executeReadyWorkPackage( // A failure outcome is authoritative even when the artifact write did not // return a row. Keep evidence empty in that case rather than losing the // outcome record or inventing an evidence reference. - await upsertExecutionOutcomeBestEffort({ + const failureOutcome: ExecutionOutcome = { + ...executionFailureOutcome({ + message, + retryable: !finalAttempt, + validationFailed: validationStatusForPackage === 'failed', + repositoryContextMissing: repositoryEvidenceBlocked, + }), + evidenceRefs: outcomeEvidenceRefsFromArtifact(artifact), + } + const failureStoredOutcome = await upsertExecutionOutcomeBestEffort({ taskId, workPackageId: nextPackage.id, agentRunId: run.id, attemptKey: `work-package:${nextPackage.id}:run:${run.id}`, - outcome: { - ...executionFailureOutcome({ - message, - retryable: !finalAttempt, - validationFailed: validationStatusForPackage === 'failed', - repositoryContextMissing: repositoryEvidenceBlocked, - }), - evidenceRefs: outcomeEvidenceRefsFromArtifact(artifact), - }, + outcome: failureOutcome, }) + if (failureStoredOutcome) { + await recordWorkPackageRunCapabilityAttemptBestEffort({ + taskId, + pkg: nextPackage, + runId: run.id, + executionOutcomeId: failureStoredOutcome.id, + outcome: failureOutcome, + attemptNumber, + verificationMode: 'none', + repositoryAffecting, + validationCommandTotal: 0, + validationCommandFailed: 0, + }) + } assertQueueClaimOwned(options) await publishTaskEventBestEffort(taskId, 'run:failed', {