From 7b41bf6738281b24413bf19d28767789a379d6e7 Mon Sep 17 00:00:00 2001 From: "Admilson B. F. Cossa" Date: Mon, 17 Aug 2026 18:51:42 +0200 Subject: [PATCH 1/6] feat(candidates): add deterministic candidate policy Add firstAcceptable, conservative WorkIt failure taxonomy, bounded redacted candidate evidence, package exports, API and size locks, adversarial/property coverage, runtime consumers, and manifest-driven claim verification. --- packages/core/evidence/README.md | 13 + packages/core/evidence/claims.json | 198 +++++++++ packages/core/package.json | 17 +- .../api-snapshots/candidates.index.d.ts | 15 + .../api-snapshots/candidates.types.d.ts | 67 +++ packages/core/scripts/build-cjs.mjs | 28 ++ packages/core/scripts/check-api-surface.mjs | 8 + packages/core/scripts/check-bundle-size.mjs | 15 + .../scripts/check-candidate-scenarios.mjs | 8 + .../core/scripts/check-candidates-api.mjs | 31 ++ .../scripts/check-candidates-performance.mjs | 8 + .../core/scripts/check-evidence-ledger.mjs | 78 ++++ .../core/scripts/check-package-consumer.mjs | 133 ++++-- .../core/scripts/evidence-source-digest.mjs | 67 +++ packages/core/scripts/report-bundle-size.mjs | 4 + .../core/src/candidates/classification.ts | 172 ++++++++ packages/core/src/candidates/evidence.ts | 51 +++ packages/core/src/candidates/index.ts | 328 ++++++++++++++ packages/core/src/candidates/types.ts | 97 +++++ packages/core/src/replay/index.ts | 8 +- packages/core/src/runtime/unsupported.ts | 2 + .../evidence/correctness/candidate-policy.mjs | 387 +++++++++++++++++ .../correctness/candidate-scenarios.mjs | 155 +++++++ .../correctness/runtime-resilience.mjs | 48 +++ .../lifecycle/candidate-lifecycle.mjs | 90 ++++ packages/core/tests/evidence/manifest.mjs | 44 ++ .../evidence/performance/candidate-bounds.mjs | 108 +++++ .../release/candidate-package-contract.mjs | 46 ++ .../release/core-release-contracts.mjs | 58 +++ packages/core/tests/evidence/run-all.mjs | 83 ++-- .../evidence/security/candidate-boundary.mjs | 44 ++ .../property/candidates.property.test.js | 110 +++++ .../tests/unit/candidates-adversarial.test.js | 220 ++++++++++ packages/core/tests/unit/candidates.test.js | 405 ++++++++++++++++++ 34 files changed, 3083 insertions(+), 63 deletions(-) create mode 100644 packages/core/scripts/api-snapshots/candidates.index.d.ts create mode 100644 packages/core/scripts/api-snapshots/candidates.types.d.ts create mode 100644 packages/core/scripts/check-candidate-scenarios.mjs create mode 100644 packages/core/scripts/check-candidates-api.mjs create mode 100644 packages/core/scripts/check-candidates-performance.mjs create mode 100644 packages/core/scripts/check-evidence-ledger.mjs create mode 100644 packages/core/scripts/evidence-source-digest.mjs create mode 100644 packages/core/src/candidates/classification.ts create mode 100644 packages/core/src/candidates/evidence.ts create mode 100644 packages/core/src/candidates/index.ts create mode 100644 packages/core/src/candidates/types.ts create mode 100644 packages/core/tests/evidence/correctness/candidate-policy.mjs create mode 100644 packages/core/tests/evidence/correctness/candidate-scenarios.mjs create mode 100644 packages/core/tests/evidence/correctness/runtime-resilience.mjs create mode 100644 packages/core/tests/evidence/lifecycle/candidate-lifecycle.mjs create mode 100644 packages/core/tests/evidence/manifest.mjs create mode 100644 packages/core/tests/evidence/performance/candidate-bounds.mjs create mode 100644 packages/core/tests/evidence/release/candidate-package-contract.mjs create mode 100644 packages/core/tests/evidence/release/core-release-contracts.mjs create mode 100644 packages/core/tests/evidence/security/candidate-boundary.mjs create mode 100644 packages/core/tests/property/candidates.property.test.js create mode 100644 packages/core/tests/unit/candidates-adversarial.test.js create mode 100644 packages/core/tests/unit/candidates.test.js diff --git a/packages/core/evidence/README.md b/packages/core/evidence/README.md index db76f84..2f8993c 100644 --- a/packages/core/evidence/README.md +++ b/packages/core/evidence/README.md @@ -15,6 +15,7 @@ runtime source + npm run verify benchmarks/articles/run-all.mjs tests/evidence/run-all.mjs evidence/claims.json +coverage/evidence/latest.json (temporary, ignored) -> README, articles ``` @@ -38,12 +39,20 @@ the impact and invariant are security-relevant. npm run verify npm run bench:articles npm run test:evidence +npm run check:evidence-ledger ``` `benchmarks/results/articles.latest.json` stores the captured article benchmark run used by README and articles for representative values. The benchmark assertions remain the portable proof. +`test:evidence` rebuilds the package, executes every proof declared in +`tests/evidence/manifest.mjs`, and writes the per-claim actual results to the +ignored `coverage/evidence/latest.json` artifact. The capture includes a SHA-256 digest over the +claim ledger, package contract, API snapshots, runtime source, and executable +evidence. `check:evidence-ledger` rejects missing claims, missing proof files, +unregistered evidence scripts, failing results, or a stale digest. + ## Evidence Stack | Layer | Source of truth | Role | @@ -53,6 +62,7 @@ assertions remain the portable proof. | Captured bench run | `benchmarks/results/articles.latest.json` | representative publication values for this revision | | Claim ledger | `evidence/claims.json` | claim IDs, class, proof path, invariant, status, and limitation | | Evidence tests | `tests/evidence/run-all.mjs` | curated lifecycle, correctness, security, release, and performance proofs | +| Temporary captured results | `coverage/evidence/latest.json` | ignored environment, source digest, elapsed time, and actual result used by the current verification run | ## Publication Rule @@ -60,3 +70,6 @@ README summarizes. Articles teach. Neither invents claim status. Public prose must cite one of the executable sources above, and security claims must stay security-specific rather than using "security" as a label for every adversarial or lifecycle proof. + +Historical tags are never rewritten: later backfills remain explicitly labeled +as backfills in the ledger. diff --git a/packages/core/evidence/claims.json b/packages/core/evidence/claims.json index a727195..95eed2d 100644 --- a/packages/core/evidence/claims.json +++ b/packages/core/evidence/claims.json @@ -107,6 +107,26 @@ "expectedInvariant": "each admitted invocation records its attempt number and outcome while bounded metadata applies default secret redaction", "limitations": "Attempt recording wraps explicit task bodies. It does not infer provider policy, prove remote cancellation, or provide deterministic scheduler replay." }, + { + "id": "LIFE-013", + "title": "candidate deadline is aggregate and terminal across the chain", + "class": "lifecycle", + "status": "proven", + "proof": "tests/evidence/lifecycle/candidate-lifecycle.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "one absolute deadline reaches the admitted candidate context and timeout admits no fallback candidate", + "limitations": "Timer precision depends on the JavaScript event loop, and remote providers stop only when they honor AbortSignal." + }, + { + "id": "LIFE-014", + "title": "candidate callback cancellation remains authoritative", + "class": "lifecycle", + "status": "proven", + "proof": "tests/evidence/lifecycle/candidate-lifecycle.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "cancellation thrown by quality or classification callbacks propagates unchanged and admits no fallback", + "limitations": "The proof covers typed in-process WorkIt cancellation, not provider-side termination without cooperative AbortSignal support." + }, { "id": "CORR-001", "title": "budget inputs are immutable boundary values", @@ -307,6 +327,108 @@ "expectedInvariant": "nested retry policies referencing one budget key are inadmissible when aggregate demand exceeds the supplied runtime budget snapshot", "limitations": "Planning uses an explicit point-in-time budget snapshot. Concurrent runtime consumption after planning can reduce the remaining budget before execution." }, + { + "id": "CORR-028", + "title": "candidate policy separates retries, quality rejection, and fallback", + "class": "correctness", + "status": "proven", + "proof": "tests/evidence/correctness/candidate-policy.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "a retryable transport failure, successful low-quality result, and accepted fallback produce distinct ordered bounded evidence", + "limitations": "Candidate execution is sequential and in-process. Provider side effects remain caller-owned and require caller-supplied idempotency." + }, + { + "id": "CORR-029", + "title": "candidate policy preserves authoritative cancellation", + "class": "correctness", + "status": "proven", + "proof": "tests/evidence/correctness/candidate-policy.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "a real WorkIt cancellation bypasses provider classification and admits no later candidate", + "limitations": "Remote work stops only when the underlying provider cooperates with AbortSignal." + }, + { + "id": "CORR-030", + "title": "candidate retries share the WorkIt retry budget", + "class": "correctness", + "status": "proven", + "proof": "tests/evidence/correctness/candidate-policy.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "candidate fallback cannot admit a retry after the shared scope retry budget is exhausted", + "limitations": "The budget covers retries admitted by WorkIt. Calls performed outside the candidate task remain application responsibility." + }, + { + "id": "CORR-031", + "title": "candidate admission and aggregate attempts are bounded", + "class": "correctness", + "status": "proven", + "proof": "tests/evidence/correctness/candidate-policy.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "callbacks cannot expand the admitted candidate snapshot and unsafe candidate-count by retry-count products fail before execution", + "limitations": "The configured maximum bounds WorkIt admissions; it does not account for provider-internal retries outside the candidate task." + }, + { + "id": "CORR-032", + "title": "candidate decisions are normalized exactly once", + "class": "correctness", + "status": "proven", + "proof": "tests/evidence/correctness/candidate-policy.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "mutable getters cannot change failure disposition, reason, or acceptance after validation", + "limitations": "Normalization protects WorkIt policy reads; callers still own mutable candidate and result objects passed to their callbacks." + }, + { + "id": "CORR-033", + "title": "candidate error taxonomy is conservative and provider-neutral", + "class": "correctness", + "status": "proven", + "proof": "tests/evidence/correctness/candidate-policy.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "known WorkIt cancellation, timeout, and budget failures have stable decisions while unknown provider errors remain unclassified", + "limitations": "Provider-specific errors require a caller classifier; WorkIt does not infer HTTP, SDK, billing, or safety policy." + }, + { + "id": "CORR-034", + "title": "candidate callback failures do not become fallback", + "class": "correctness", + "status": "proven", + "proof": "tests/evidence/correctness/candidate-policy.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "quality and classifier callback bugs propagate unchanged and admit no later candidate", + "limitations": "Applications remain responsible for testing their classifier and quality predicate implementations." + }, + { + "id": "CORR-035", + "title": "telemetry context and observer failures remain isolated", + "class": "correctness", + "status": "proven", + "proof": "tests/evidence/correctness/runtime-resilience.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "throwing telemetry context access and observers cannot escape event emission or block another observer", + "limitations": "Observer exceptions are intentionally discarded. Exporter diagnostics and delivery guarantees belong to opt-in observability adapters.", + "verifiedFor": "0.5.0", + "ledgerNote": "Backfilled after the 0.5.0 tag because its changelog claim had unit coverage but no dedicated claim-ledger entry." + }, + { + "id": "CORR-036", + "title": "hostile thrown values cannot bypass candidate classification", + "class": "correctness", + "status": "proven", + "proof": "tests/evidence/correctness/candidate-policy.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "proxy traps and throwing Error accessors remain bounded while the caller classifier retains policy authority", + "limitations": "This is JavaScript object hardening, not isolation from arbitrary code executing in the same process." + }, + { + "id": "CORR-037", + "title": "bounded operational candidate scenarios preserve declared policy", + "class": "correctness", + "status": "proven", + "proof": "tests/evidence/correctness/candidate-scenarios.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "quality fallback, caller-owned idempotency, user-input stops, and cooperative cancellation preserve their declared boundaries", + "limitations": "These are bounded in-process fixtures, not a real provider, durable-store, distributed-system, or Oryn production canary." + }, { "id": "SEC-001", "title": "worker offload rejects remote and executable URL schemes", @@ -327,6 +449,16 @@ "expectedInvariant": "late marker is not written after offload timeout", "limitations": "The hard boundary applies to worker offload; JavaScript cannot preempt a CPU loop on the main thread." }, + { + "id": "SEC-003", + "title": "candidate metadata normalization keeps prototype-shaped fields inert", + "class": "security", + "status": "proven", + "proof": "tests/evidence/security/candidate-boundary.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "an own __proto__ metadata field is retained as data without changing Object.prototype and secret fields are redacted", + "limitations": "This proof covers candidate evidence normalization. It is not a general object sandbox or protection for caller code that mutates global prototypes." + }, { "id": "REL-001", "title": "public proof artifact exposes release evidence", @@ -387,6 +519,40 @@ "expectedInvariant": "a receipt appended through node:sqlite is readable after database close and reopen, while conflicting content is rejected", "limitations": "This proof uses node:sqlite when the active Node runtime provides it. Postgres remains an environment-gated integration path because WorkIt does not take a runtime pg dependency." }, + { + "id": "REL-008", + "title": "candidates subpath has locked ESM CommonJS and type artifacts", + "class": "release", + "status": "proven", + "proof": "tests/evidence/release/candidate-package-contract.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "dedicated ESM, CommonJS, and declaration artifacts expose firstAcceptable while the root API remains unchanged", + "limitations": "Installed-tarball and runtime-matrix behavior is verified by the separate package-consumer release gate." + }, + { + "id": "REL-009", + "title": "published core has zero runtime dependencies", + "class": "release", + "status": "proven", + "proof": "tests/evidence/release/core-release-contracts.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "package dependencies are absent and the generated CycloneDX SBOM contains no runtime components", + "limitations": "Development and optional peer dependencies remain outside the zero-runtime-dependency claim.", + "verifiedFor": "0.5.0", + "ledgerNote": "Backfilled after the 0.5.0 tag because the release changelog claim was enforced by SBOM gates but absent from the claim ledger." + }, + { + "id": "REL-010", + "title": "compiled package entrypoints stay within enforced size budgets", + "class": "release", + "status": "proven", + "proof": "tests/evidence/release/core-release-contracts.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "the size gate passes for compiled root, group, candidates ESM, and candidates CommonJS artifacts", + "limitations": "Bundle results depend on the pinned build toolchain and measure declared fixtures rather than every consumer bundler configuration.", + "verifiedFor": "0.5.0", + "ledgerNote": "The root and group measurements backfill the 0.5.0 claim; candidate measurements apply to the 0.6.0 target." + }, { "id": "PERF-001", "title": "article benchmark suite has expected executable coverage", @@ -406,6 +572,38 @@ "command": "npm run test:evidence", "expectedInvariant": "benchmarks/results/articles.latest.json records 19 passing benches", "limitations": "Timing values are representative for the captured machine and run; semantic assertions are the portable claim." + }, + { + "id": "PERF-003", + "title": "candidate execution and retained evidence remain bounded", + "class": "performance", + "status": "proven", + "proof": "tests/evidence/performance/candidate-bounds.mjs", + "command": "npm run test:evidence", + "expectedInvariant": "large quality and retry chains stay inside time and retained-heap budgets while evidence retains only its configured window", + "limitations": "Timings and heap deltas are environment-sensitive regression bounds, not universal latency or memory guarantees." + }, + { + "id": "PROD-001", + "title": "scope idempotency is not durable idempotency", + "class": "product-decision", + "status": "product-decision", + "proof": "README.md", + "command": "npm run check:evidence-ledger", + "expectedInvariant": "TaskOpts.idempotencyKey is documented as live-scope coalescing and durable restart behavior remains caller-owned", + "limitations": "Applications requiring restart-safe side effects must use a durable idempotency store or an explicit activity boundary.", + "verifiedFor": "0.5.0", + "ledgerNote": "Backfilled after the 0.5.0 tag as an explicit product-decision claim." + }, + { + "id": "PROD-002", + "title": "candidate side-effect idempotency remains caller-owned", + "class": "product-decision", + "status": "product-decision", + "proof": "src/candidates/index.ts", + "command": "npm run check:evidence-ledger", + "expectedInvariant": "firstAcceptable remains a sequential in-process policy helper and does not imply durable idempotency or background handoff", + "limitations": "Retries and fallback can repeat side effects; callers must supply durable idempotency where operations are not naturally safe." } ] } diff --git a/packages/core/package.json b/packages/core/package.json index 4f5c3f3..f8fd923 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -82,6 +82,14 @@ "import": "./dist/channel/index.js", "require": "./dist-cjs/channel/index.cjs" }, + "./candidates": { + "types": "./dist/candidates/index.d.ts", + "node": { + "import": "./dist/candidates/index.js", + "require": "./dist-cjs/candidates/index.cjs" + }, + "default": "./dist/runtime/unsupported.js" + }, "./contracts": { "types": "./dist/contracts/index.d.ts", "import": "./dist/contracts/index.js", @@ -164,9 +172,12 @@ "check:sbom": "npm run build && node scripts/check-sbom.mjs", "check:tests": "node scripts/check-tests.mjs", "check:api": "npm run build && node scripts/check-api-surface.mjs", + "check:candidates-api": "npm run build && node scripts/check-candidates-api.mjs", "check:size": "npm run build && node scripts/check-bundle-size.mjs", "report:size": "npm run build && node scripts/report-bundle-size.mjs", "check:benchmark": "npm run build && node scripts/check-benchmark.mjs", + "check:candidates-performance": "npm run build && node --expose-gc scripts/check-candidates-performance.mjs", + "check:candidate-scenarios": "npm run build && node scripts/check-candidate-scenarios.mjs", "check:context-performance": "npm run build && node scripts/check-context-performance.mjs", "check:1b": "npm run build && node scripts/check-1b-benchmark.mjs", "check:leak": "npm run build && node --expose-gc scripts/check-leak.mjs", @@ -174,14 +185,16 @@ "check:soak": "npm run build && node --expose-gc scripts/check-soak.mjs", "check:exporter-stress": "npm run build && node --expose-gc scripts/check-exporter-stress.mjs", "check:package-consumer": "npm run build && node scripts/check-package-consumer.mjs", + "check:package-consumer:node": "npm run build && node scripts/check-package-consumer.mjs --node-only", "check:claims": "npm run build && node scripts/check-claim-fixtures.mjs", + "check:evidence-ledger": "node scripts/check-evidence-ledger.mjs", "check:public-proof": "node scripts/check-public-proof.mjs", "check:worker-contract": "node scripts/check-worker-contract-docs.mjs", "check:release-policy": "npm run build && node scripts/check-release-provenance.mjs", "check:release": "npm run build && node scripts/check-release-provenance.mjs --registry-dry-run", "bench:articles": "node benchmarks/articles/run-all.mjs", "bench:articles:repeated": "node benchmarks/articles/run-repeated.mjs", - "test:evidence": "node tests/evidence/run-all.mjs", + "test:evidence": "npm run build && node tests/evidence/run-all.mjs --output coverage/evidence/latest.json", "test:property": "npm run build && vitest run tests/property", "pack:dry": "npm run build && npm pack --dry-run --json", "sample:1b": "npm run build && node samples/1b-stream.sample.js", @@ -209,7 +222,7 @@ "soak:24h": "npm run build && node --expose-gc scripts/soak-24h.mjs", "test": "npm run build && vitest run --maxWorkers=1", "test:coverage": "npm run build && vitest run --coverage --maxWorkers=1", - "verify": "npm run typecheck && npm run check:no-network && npm run check:headers && npm run check:tests && npm test && npm run check:security && npm run check:vulnerabilities && npm run check:sbom && npm run check:api && npm run check:size && npm run check:benchmark && npm run check:context-performance && npm run check:1b && npm run check:leak && npm run check:stream-memory && npm run check:soak && npm run check:exporter-stress && npm run check:package-consumer && npm run check:claims && npm run check:public-proof && npm run check:worker-contract && npm run check:release-policy && npm run pack:dry" + "verify": "npm run typecheck && npm run check:no-network && npm run check:headers && npm run check:tests && npm test && npm run test:evidence && npm run check:evidence-ledger && npm run check:security && npm run check:vulnerabilities && npm run check:sbom && npm run check:api && npm run check:candidates-api && npm run check:size && npm run check:benchmark && npm run check:candidates-performance && npm run check:candidate-scenarios && npm run check:context-performance && npm run check:1b && npm run check:leak && npm run check:stream-memory && npm run check:soak && npm run check:exporter-stress && npm run check:package-consumer && npm run check:claims && npm run check:public-proof && npm run check:worker-contract && npm run check:release-policy && npm run pack:dry" }, "engines": { "node": ">=20.11" diff --git a/packages/core/scripts/api-snapshots/candidates.index.d.ts b/packages/core/scripts/api-snapshots/candidates.index.d.ts new file mode 100644 index 0000000..d39d915 --- /dev/null +++ b/packages/core/scripts/api-snapshots/candidates.index.d.ts @@ -0,0 +1,15 @@ +/** + * Deterministic candidate selection with typed failure and quality policy. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + * + * Candidate execution is sequential and may repeat side effects. Callers own + * idempotency for side-effecting operations. + */ +import type { CandidateFailureDecision, CandidateRunResult, FirstAcceptableOptions } from "./types.js"; +export type { AcceptanceDecision, CandidateActionDecision, CandidateAttemptDecision, CandidateAttemptEvidence, CandidateFailureDecision, CandidateRetryPolicy, CandidateRunResult, FailureDisposition, FirstAcceptableOptions, } from "./types.js"; +/** Returns the safe built-in classification for a known WorkIt runtime error. */ +export declare function classifyWorkItFailure(error: unknown): CandidateFailureDecision | undefined; +/** Selects the first semantically acceptable result in deterministic candidate order. */ +export declare function firstAcceptable(candidates: readonly C[], opts: FirstAcceptableOptions): Promise>; diff --git a/packages/core/scripts/api-snapshots/candidates.types.d.ts b/packages/core/scripts/api-snapshots/candidates.types.d.ts new file mode 100644 index 0000000..9a11f51 --- /dev/null +++ b/packages/core/scripts/api-snapshots/candidates.types.d.ts @@ -0,0 +1,67 @@ +/** + * Public contracts for deterministic candidate selection. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ +import type { AttemptRecorderOptions, WorkItAttemptEvidence } from "../replay/index.js"; +import type { RetryOpts, TaskContext } from "../types/index.js"; +/** Policy outcome assigned to a failed candidate attempt. */ +export type FailureDisposition = "retry_same_candidate" | "try_next_candidate" | "terminal" | "cancelled" | "requires_user_input"; +/** Bounded, machine-readable classification of a candidate failure. */ +export interface CandidateFailureDecision { + readonly disposition: FailureDisposition; + readonly reasonCode: string; +} +/** Failure decisions a provider classifier may return. Real cancellation is detected by WorkIt. */ +export interface CandidateActionDecision extends CandidateFailureDecision { + readonly disposition: Exclude; +} +/** Semantic quality decision kept separate from transport success. */ +export type AcceptanceDecision = { + readonly accepted: true; +} | { + readonly accepted: false; + readonly reasonCode: string; +}; +/** Candidate-level interpretation of one recorded task-body invocation. */ +export type CandidateAttemptDecision = CandidateActionDecision["disposition"] | "quality_rejected" | "accepted"; +/** Bounded attempt evidence enriched with candidate and quality policy facts. */ +export interface CandidateAttemptEvidence extends WorkItAttemptEvidence { + readonly candidateIndex: number; + readonly decision: CandidateAttemptDecision; +} +interface CandidateResultEvidence { + readonly evidence: readonly CandidateAttemptEvidence[]; + readonly droppedEvidence: number; +} +/** Exhaustive expected outcome of a candidate selection run. */ +export type CandidateRunResult = (CandidateResultEvidence & { + readonly status: "accepted"; + readonly candidate: C; + readonly candidateIndex: number; + readonly value: T; +}) | (CandidateResultEvidence & { + readonly status: "exhausted"; +}) | (CandidateResultEvidence & { + readonly status: "terminal"; + readonly reasonCode: string; + readonly error: unknown; +}) | (CandidateResultEvidence & { + readonly status: "requires_user_input"; + readonly reasonCode: string; +}); +/** Retry policy whose admission predicate is owned by the failure classifier. */ +export type CandidateRetryPolicy = number | Omit; +/** Candidate selection policy and bounded evidence controls. */ +export interface FirstAcceptableOptions { + readonly execute: (candidate: C, ctx: TaskContext) => T | Promise; + readonly accept: (value: T, candidate: C, ctx: TaskContext) => AcceptanceDecision | Promise; + readonly classifyFailure: (error: unknown, candidate: C, ctx: TaskContext) => CandidateActionDecision | Promise; + readonly retry?: CandidateRetryPolicy; + readonly maxCandidates?: number; + readonly deadlineAt?: number | Date; + readonly evidence?: AttemptRecorderOptions; + readonly candidateMetadata?: (candidate: C, candidateIndex: number) => Readonly> | undefined; +} +export {}; diff --git a/packages/core/scripts/build-cjs.mjs b/packages/core/scripts/build-cjs.mjs index c20489e..1017158 100644 --- a/packages/core/scripts/build-cjs.mjs +++ b/packages/core/scripts/build-cjs.mjs @@ -12,12 +12,35 @@ import { rm } from "node:fs/promises"; import { build } from "esbuild"; +const candidateSharedRuntimePlugin = { + name: "candidate-shared-runtime", + setup(buildContext) { + buildContext.onResolve({ filter: /^\.\.\/run\/index\.js$/ }, (args) => + isCandidateModule(args.importer, "index.js") + ? { path: "@workit/core", external: true } + : undefined); + buildContext.onResolve({ filter: /^\.\.\/replay\/index\.js$/ }, (args) => + isCandidateModule(args.importer, "index.js") + ? { path: "@workit/core/replay", external: true } + : undefined); + buildContext.onResolve({ filter: /^\.\.\/types\/index\.js$/ }, (args) => + isCandidateModule(args.importer, "classification.js") + ? { path: "@workit/core", external: true } + : undefined); + }, +}; + const ENTRIES = [ { entry: "dist/index.js", outfile: "dist-cjs/index.cjs" }, { entry: "dist/activity/index.js", outfile: "dist-cjs/activity/index.cjs" }, { entry: "dist/ai/index.js", outfile: "dist-cjs/ai/index.cjs" }, { entry: "dist/analysis/index.js", outfile: "dist-cjs/analysis/index.cjs" }, { entry: "dist/channel/index.js", outfile: "dist-cjs/channel/index.cjs" }, + { + entry: "dist/candidates/index.js", + outfile: "dist-cjs/candidates/index.cjs", + plugins: [candidateSharedRuntimePlugin], + }, { entry: "dist/contracts/index.js", outfile: "dist-cjs/contracts/index.cjs" }, { entry: "dist/diagnostics/index.js", outfile: "dist-cjs/diagnostics/index.cjs" }, { entry: "dist/fault/index.js", outfile: "dist-cjs/fault/index.cjs" }, @@ -40,6 +63,11 @@ for (const target of ENTRIES) { platform: "node", target: "node20", external: ["@opentelemetry/api"], + plugins: target.plugins ?? [], logLevel: "silent", }); } + +function isCandidateModule(importer, filename) { + return importer.replaceAll("\\", "/").endsWith(`/candidates/${filename}`); +} diff --git a/packages/core/scripts/check-api-surface.mjs b/packages/core/scripts/check-api-surface.mjs index a939a03..dda8e19 100644 --- a/packages/core/scripts/check-api-surface.mjs +++ b/packages/core/scripts/check-api-surface.mjs @@ -19,6 +19,7 @@ const EXPECTED_EXPORT_MAP = [ "./activity", "./ai", "./analysis", + "./candidates", "./channel", "./contracts", "./diagnostics", @@ -83,6 +84,10 @@ const EXPECTED_RUNTIME_EXPORTS = { "ChannelClosedError", "createChannel", ], + "./candidates": [ + "classifyWorkItFailure", + "firstAcceptable", + ], "./contracts": [ "cancellable", "discardCancellation", @@ -145,6 +150,7 @@ const EXPECTED_EXPORT_CONDITIONS = { "./ai": ["default", "node", "types"], "./analysis": ["import", "require", "types"], "./channel": ["import", "require", "types"], + "./candidates": ["default", "node", "types"], "./contracts": ["import", "require", "types"], "./diagnostics": ["import", "require", "types"], "./fault": ["default", "node", "types"], @@ -163,6 +169,7 @@ const MODULE_PATHS = { "./ai": "../dist/ai/index.js", "./analysis": "../dist/analysis/index.js", "./channel": "../dist/channel/index.js", + "./candidates": "../dist/candidates/index.js", "./contracts": "../dist/contracts/index.js", "./diagnostics": "../dist/diagnostics/index.js", "./fault": "../dist/fault/index.js", @@ -181,6 +188,7 @@ const CJS_MODULE_PATHS = { "./ai": "../dist-cjs/ai/index.cjs", "./analysis": "../dist-cjs/analysis/index.cjs", "./channel": "../dist-cjs/channel/index.cjs", + "./candidates": "../dist-cjs/candidates/index.cjs", "./contracts": "../dist-cjs/contracts/index.cjs", "./diagnostics": "../dist-cjs/diagnostics/index.cjs", "./fault": "../dist-cjs/fault/index.cjs", diff --git a/packages/core/scripts/check-bundle-size.mjs b/packages/core/scripts/check-bundle-size.mjs index 3bfefe5..e41aaaf 100644 --- a/packages/core/scripts/check-bundle-size.mjs +++ b/packages/core/scripts/check-bundle-size.mjs @@ -11,6 +11,7 @@ */ import { gzipSync } from "node:zlib"; +import { stat } from "node:fs/promises"; import { build } from "esbuild"; const DIST_ENTRY = "./dist/index.js"; @@ -28,8 +29,16 @@ const BUDGETS = [ maxMinifiedBytes: 14_000, maxGzipBytes: 4_900, }, + { + name: "candidates-subpath", + source: 'export * from "./dist/candidates/index.js";', + maxMinifiedBytes: 29_450, + maxGzipBytes: 9_900, + }, ]; +const CANDIDATES_CJS_MAX_BYTES = 18_000; + const failures = []; for (const budget of BUDGETS) { @@ -62,6 +71,12 @@ for (const budget of BUDGETS) { } } +const candidatesCjsBytes = (await stat("./dist-cjs/candidates/index.cjs")).size; +console.log(`candidates-commonjs: ${candidatesCjsBytes} B (limit ${CANDIDATES_CJS_MAX_BYTES} B)`); +if (candidatesCjsBytes > CANDIDATES_CJS_MAX_BYTES) { + failures.push("candidates-commonjs exceeded configured bundle budget"); +} + if (failures.length > 0) { for (const failure of failures) console.error(failure); process.exit(1); diff --git a/packages/core/scripts/check-candidate-scenarios.mjs b/packages/core/scripts/check-candidate-scenarios.mjs new file mode 100644 index 0000000..7d0907e --- /dev/null +++ b/packages/core/scripts/check-candidate-scenarios.mjs @@ -0,0 +1,8 @@ +/** + * Runs the canonical bounded candidate operational evidence proof. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +await import("../tests/evidence/correctness/candidate-scenarios.mjs"); diff --git a/packages/core/scripts/check-candidates-api.mjs b/packages/core/scripts/check-candidates-api.mjs new file mode 100644 index 0000000..2378d0d --- /dev/null +++ b/packages/core/scripts/check-candidates-api.mjs @@ -0,0 +1,31 @@ +/** + * Candidate subpath declaration API snapshot gate. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; + +const DECLARATIONS = [ + ["../dist/candidates/index.d.ts", "./api-snapshots/candidates.index.d.ts"], + ["../dist/candidates/types.d.ts", "./api-snapshots/candidates.types.d.ts"], +]; + +for (const [actualPath, snapshotPath] of DECLARATIONS) { + const actual = await readFile(new URL(actualPath, import.meta.url), "utf8"); + const snapshot = await readFile(new URL(snapshotPath, import.meta.url), "utf8"); + assert.equal( + normalizeNewlines(actual), + normalizeNewlines(snapshot), + `${actualPath} changed without an intentional candidate API snapshot update`, + ); +} + +process.stdout.write(JSON.stringify({ candidateDeclarationApi: "locked", files: DECLARATIONS.length }) + "\n"); + +function normalizeNewlines(value) { + return value.replaceAll("\r\n", "\n"); +} + diff --git a/packages/core/scripts/check-candidates-performance.mjs b/packages/core/scripts/check-candidates-performance.mjs new file mode 100644 index 0000000..182dca1 --- /dev/null +++ b/packages/core/scripts/check-candidates-performance.mjs @@ -0,0 +1,8 @@ +/** + * Runs the canonical candidate performance evidence proof. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +await import("../tests/evidence/performance/candidate-bounds.mjs"); diff --git a/packages/core/scripts/check-evidence-ledger.mjs b/packages/core/scripts/check-evidence-ledger.mjs new file mode 100644 index 0000000..cdeb8a9 --- /dev/null +++ b/packages/core/scripts/check-evidence-ledger.mjs @@ -0,0 +1,78 @@ +/** + * Validates the claim ledger against the proof manifest and captured results. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from "node:assert/strict"; +import { access, readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { evidenceProofs } from "../tests/evidence/manifest.mjs"; +import { computeEvidenceSourceDigest } from "./evidence-source-digest.mjs"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const ledger = await readJson("evidence/claims.json"); +const capture = await readJson("coverage/evidence/latest.json"); +const allowedStatuses = new Set([ + "deferred", + "environment-blocked", + "product-decision", + "proven", + "unproven", +]); +const proofManifest = new Set(evidenceProofs.map(({ file }) => `tests/evidence/${file}`)); +const claimIds = new Set(); + +assert.equal(ledger.artifact, "workit-claim-ledger"); +assert.ok(Array.isArray(ledger.allowedClasses)); +assert.ok(Array.isArray(ledger.claims)); + +for (const claim of ledger.claims) { + assert.match(claim.id, /^[A-Z]+-[0-9]{3}$/u, `invalid claim id: ${claim.id}`); + assert.ok(!claimIds.has(claim.id), `duplicate claim id: ${claim.id}`); + claimIds.add(claim.id); + assert.ok(ledger.allowedClasses.includes(claim.class), `${claim.id} has invalid class`); + assert.ok(allowedStatuses.has(claim.status), `${claim.id} has invalid status`); + for (const field of ["title", "proof", "command", "expectedInvariant", "limitations"]) { + assert.equal(typeof claim[field], "string", `${claim.id} is missing ${field}`); + assert.ok(claim[field].length > 0, `${claim.id} has empty ${field}`); + } + await access(resolve(packageRoot, claim.proof)); + if (claim.command === "npm run test:evidence") { + assert.ok(proofManifest.has(claim.proof), `${claim.id} proof is absent from evidence manifest`); + } +} + +assert.equal(capture.artifact, "workit-publication-evidence"); +assert.equal(capture.schemaVersion, 2); +assert.equal(capture.failed, 0, "captured evidence contains failed proof files"); +assert.equal(capture.sourceDigest, await computeEvidenceSourceDigest(packageRoot), "captured evidence is stale"); + +const resultById = new Map(); +for (const result of capture.claimResults) { + assert.ok(!resultById.has(result.id), `duplicate captured claim result: ${result.id}`); + assert.ok(claimIds.has(result.id), `captured result has no ledger claim: ${result.id}`); + resultById.set(result.id, result); +} + +for (const claim of ledger.claims) { + if (claim.status !== "proven" || claim.command !== "npm run test:evidence") continue; + const result = resultById.get(claim.id); + assert.ok(result, `${claim.id} has no captured actual result`); + assert.equal(result.status, "pass", `${claim.id} captured result did not pass`); + assert.equal(result.proof, claim.proof, `${claim.id} captured proof path differs from ledger`); + assert.ok(result.actualResult !== undefined, `${claim.id} captured actual result is missing`); +} + +process.stdout.write(JSON.stringify({ + evidenceLedger: "ok", + claims: ledger.claims.length, + capturedResults: capture.claimResults.length, + sourceDigest: capture.sourceDigest, +}) + "\n"); + +async function readJson(path) { + return JSON.parse(await readFile(resolve(packageRoot, path), "utf8")); +} diff --git a/packages/core/scripts/check-package-consumer.mjs b/packages/core/scripts/check-package-consumer.mjs index 879029a..24570a1 100644 --- a/packages/core/scripts/check-package-consumer.mjs +++ b/packages/core/scripts/check-package-consumer.mjs @@ -20,6 +20,7 @@ import { build } from "esbuild"; const execFileAsync = promisify(execFile); const require = createRequire(import.meta.url); const ROOT = resolve(fileURLToPath(new URL("..", import.meta.url))); +const nodeOnly = process.argv.includes("--node-only"); const tscCli = require.resolve("typescript/bin/tsc"); const wranglerPackagePath = require.resolve("wrangler/package.json"); const wranglerPackage = JSON.parse(await readFile(wranglerPackagePath, "utf8")); @@ -39,9 +40,9 @@ const wranglerCli = await findExecutable( ] ); -if (bunCli === null) throw new Error("Bun compatibility fixture requires a Bun executable."); -if (denoCli === null) throw new Error("Deno compatibility fixture requires a Deno executable."); -if (wranglerCli === null) throw new Error("Cloudflare Worker dry-run fixture requires Wrangler."); +if (!nodeOnly && bunCli === null) throw new Error("Bun compatibility fixture requires a Bun executable."); +if (!nodeOnly && denoCli === null) throw new Error("Deno compatibility fixture requires a Deno executable."); +if (!nodeOnly && wranglerCli === null) throw new Error("Cloudflare Worker dry-run fixture requires Wrangler."); const temp = await mkdtemp(join(tmpdir(), "workit-consumer-")); @@ -117,6 +118,7 @@ try { import { ActivitySerializationError, createFileActivityStore, createMemoryActivityStore, runActivity } from "@workit/core/activity"; import { analyzeReceipt, verifyReceipt, verifySourceProtocol } from "@workit/core/analysis"; import { AgentCapabilityError, embedAll, runAgent, streamWithBackpressure } from "@workit/core/ai"; + import { firstAcceptable } from "@workit/core/candidates"; import { cancellable, getTaskContract, shielded, typedGroup } from "@workit/core/contracts"; import { cleanupHang, runFaultScenario } from "@workit/core/fault"; import { createMemoryReceiptLedger, createPostgresReceiptLedger, createSqliteReceiptLedger } from "@workit/core/ledger"; @@ -128,6 +130,11 @@ try { import { offload } from "@workit/core/worker"; const result = await run.all([async () => "sdk", async () => "ok"]); + const candidateResult = await firstAcceptable(["primary"], { + execute: async (candidate) => candidate, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "consumer_failure" }), + }); const deadlineAt = Date.now() + 1_000; const observedDeadline = await group(async (task) => task(run.deadline(async (ctx) => ctx.deadlineAt, deadlineAt)) @@ -197,6 +204,9 @@ try { }); if (result.join(":") !== "sdk:ok") throw new Error("root import failed"); + if (candidateResult.status !== "accepted" || candidateResult.value !== "primary") { + throw new Error("candidates import failed"); + } if (observedDeadline !== deadlineAt) throw new Error("deadline introspection failed"); if (retried !== "retried" || retryAttempts !== 2) throw new Error("retry budget failed"); if (retryPlan.retryBudgets[0]?.status !== "admissible") throw new Error("retry budget planning failed"); @@ -298,11 +308,36 @@ try { await writeFile(join(temp, "cjs-smoke.cjs"), ` const { createBudget, run, work } = require("@workit/core"); + const { firstAcceptable } = require("@workit/core/candidates"); const { cancellable, typedGroup } = require("@workit/core/contracts"); const { planTimePolicy } = require("@workit/core/time-policy"); (async () => { const values = await run.all([async () => "cjs", async () => "ok"]); + const candidateResult = await firstAcceptable(["primary"], { + execute: async (candidate) => candidate, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "consumer_failure" }), + }); + const CandidateRetryBudget = createBudget("CjsCandidateRetryBudget", { unit: "retries" }); + let candidateAttempts = 0; + const budgetedCandidateResult = await run.context.with( + CandidateRetryBudget, + { limit: 1, spent: 0, unit: "retries" }, + async () => firstAcceptable(["primary"], { + execute: async (candidate) => { + candidateAttempts++; + if (candidateAttempts === 1) throw new Error("retry candidate"); + return candidate; + }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ + disposition: "retry_same_candidate", + reasonCode: "candidate_retry", + }), + retry: { times: 2, initialDelay: 0, retryBudget: CandidateRetryBudget }, + }), + ); const deadlineAt = Date.now() + 1_000; const observedDeadline = await run.group(async (task) => task(run.deadline(async (ctx) => ctx.deadlineAt, deadlineAt)) @@ -328,6 +363,12 @@ try { const output = await work([1, 2, 3]).inParallel(2).do(async (item) => item + 1); const typed = await typedGroup(async (spawn) => await spawn(cancellable(async () => "contracts"))); if (values.join(":") !== "cjs:ok") throw new Error("CommonJS root import failed"); + if (candidateResult.status !== "accepted" || candidateResult.value !== "primary") { + throw new Error("CommonJS candidates import failed"); + } + if (budgetedCandidateResult.status !== "accepted" || candidateAttempts !== 2) { + throw new Error("CommonJS candidates did not share the root retry budget context"); + } if (observedDeadline !== deadlineAt) throw new Error("CommonJS deadline introspection failed"); if (retried !== "retried" || retryAttempts !== 2) throw new Error("CommonJS retry budget failed"); if (retryPlan.retryBudgets[0]?.status !== "admissible") throw new Error("CommonJS retry budget planning failed"); @@ -393,6 +434,10 @@ try { type SourceProtocolAnalysisReport, } from "@workit/core/analysis"; import { AgentCapabilityError, embedAll, runAgent, streamWithBackpressure } from "@workit/core/ai"; + import { + firstAcceptable, + type CandidateRunResult, + } from "@workit/core/candidates"; import { cancellable, discardCancellation, @@ -428,6 +473,21 @@ try { async () => 1, async () => "typed", ] as const); + const candidateResult: CandidateRunResult = await firstAcceptable(["primary"], { + execute: async (candidate) => candidate, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "strict_failure" }), + }); + if (candidateResult.status === "accepted" && candidateResult.value !== "primary") { + throw new Error("candidate result inference failed"); + } + const candidateStatuses: Record["status"], true> = { + accepted: true, + exhausted: true, + terminal: true, + requires_user_input: true, + }; + void candidateStatuses; const value = await group(async (task) => { return await task(async (ctx: TaskContext) => { @@ -705,10 +765,12 @@ try { if (result.join(":") !== "bun:ok") throw new Error("Bun runtime fixture failed"); `, "utf8"); - await execFileAsync(bunCli, ["bun-fixture.mjs"], { - cwd: temp, - timeout: 120_000, - }); + if (!nodeOnly) { + await execFileAsync(bunCli, ["bun-fixture.mjs"], { + cwd: temp, + timeout: 120_000, + }); + } await writeFile(join(temp, "deno-fixture.mjs"), ` import { run } from "@workit/core"; @@ -717,10 +779,12 @@ try { if (result.join(":") !== "deno:ok") throw new Error("Deno runtime fixture failed"); `, "utf8"); - await execFileAsync(denoCli, ["run", "--allow-read", "--allow-env", "--allow-sys", "deno-fixture.mjs"], { - cwd: temp, - timeout: 120_000, - }); + if (!nodeOnly) { + await execFileAsync(denoCli, ["run", "--allow-read", "--allow-env", "--allow-sys", "deno-fixture.mjs"], { + cwd: temp, + timeout: 120_000, + }); + } await writeFile(join(temp, "aws-fixture.mjs"), ` import { work } from "@workit/core"; @@ -962,8 +1026,9 @@ try { await writeFile(join(temp, "browser-entry.mjs"), ` import { group } from "@workit/core"; + import { firstAcceptable } from "@workit/core/candidates"; import { offload } from "@workit/core/worker"; - globalThis.__workitBrowserSmoke = [typeof group, typeof offload]; + globalThis.__workitBrowserSmoke = [typeof group, typeof firstAcceptable, typeof offload]; `, "utf8"); const browserBundle = await build({ @@ -997,32 +1062,34 @@ try { }; `, "utf8"); - await execCli(wranglerCli, [ - "deploy", - "cloudflare-worker.mjs", - "--name", - "workit-compat-smoke", - "--dry-run", - "--outdir", - "wrangler-out", - "--compatibility-date", - "2026-05-07", - ], { - cwd: temp, - timeout: 120_000, - }); + if (!nodeOnly) { + await execCli(wranglerCli, [ + "deploy", + "cloudflare-worker.mjs", + "--name", + "workit-compat-smoke", + "--dry-run", + "--outdir", + "wrangler-out", + "--compatibility-date", + "2026-05-07", + ], { + cwd: temp, + timeout: 120_000, + }); - const workerBundle = await readFile(join(temp, "wrangler-out", "cloudflare-worker.js"), "utf8"); - if (workerBundle.includes("node:async_hooks") || workerBundle.includes("node:worker_threads")) { - throw new Error("Cloudflare Worker dry-run pulled in Node-only WorkIt modules"); - } - if (!workerBundle.includes("UnsupportedRuntimeError")) { - throw new Error("Cloudflare Worker dry-run did not resolve to the unsupported runtime split"); + const workerBundle = await readFile(join(temp, "wrangler-out", "cloudflare-worker.js"), "utf8"); + if (workerBundle.includes("node:async_hooks") || workerBundle.includes("node:worker_threads")) { + throw new Error("Cloudflare Worker dry-run pulled in Node-only WorkIt modules"); + } + if (!workerBundle.includes("UnsupportedRuntimeError")) { + throw new Error("Cloudflare Worker dry-run did not resolve to the unsupported runtime split"); + } } console.log(JSON.stringify({ packageConsumer: "ok", - runtimeFixtures: "ok", + runtimeFixtures: nodeOnly ? "node-only" : "ok", frameworkFixtures: "ok", frameworks: ["express", "fastify", "trpc", "next", "vercel-ai"], tarball: pack.filename, diff --git a/packages/core/scripts/evidence-source-digest.mjs b/packages/core/scripts/evidence-source-digest.mjs new file mode 100644 index 0000000..1dc487e --- /dev/null +++ b/packages/core/scripts/evidence-source-digest.mjs @@ -0,0 +1,67 @@ +/** + * Computes the source digest bound to a captured evidence run. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createHash } from "node:crypto"; +import { readdir, readFile } from "node:fs/promises"; +import { relative, resolve } from "node:path"; + +const EVIDENCE_SOURCE_PATHS = Object.freeze([ + "../../package-lock.json", + "../../package.json", + "README.md", + "SECURITY.md", + "benchmarks", + "evidence/claims.json", + "package.json", + "samples", + "scripts", + "src", + "tests", + "tsconfig.json", + "vitest.config.ts", +]); + +/** Returns a stable digest over runtime, public API, ledger, and proof sources. */ +export async function computeEvidenceSourceDigest(packageRoot) { + const root = resolve(packageRoot); + const ledger = JSON.parse(await readFile(resolve(root, "evidence/claims.json"), "utf8")); + const paths = new Set([ + ...EVIDENCE_SOURCE_PATHS, + ...ledger.claims.map(({ proof }) => proof), + ]); + const files = new Set(); + for (const path of paths) { + await collectFiles(resolve(root, path), files); + } + const orderedFiles = [...files].sort((left, right) => left.localeCompare(right)); + + const digest = createHash("sha256"); + for (const file of orderedFiles) { + const logicalPath = relative(root, file).replaceAll("\\", "/"); + digest.update(logicalPath); + digest.update("\0"); + digest.update(await readFile(file)); + digest.update("\0"); + } + return digest.digest("hex"); +} + +async function collectFiles(path, files) { + const entries = await readdir(path, { withFileTypes: true }).catch((error) => { + if (error?.code === "ENOTDIR") return null; + throw error; + }); + if (entries === null) { + files.add(path); + return; + } + for (const entry of entries) { + const child = resolve(path, entry.name); + if (entry.isDirectory()) await collectFiles(child, files); + if (entry.isFile()) files.add(child); + } +} diff --git a/packages/core/scripts/report-bundle-size.mjs b/packages/core/scripts/report-bundle-size.mjs index 3158677..de2dbce 100644 --- a/packages/core/scripts/report-bundle-size.mjs +++ b/packages/core/scripts/report-bundle-size.mjs @@ -34,6 +34,10 @@ const ENTRIES = [ name: "direct-work-import", source: 'export { work } from "./dist/work/index.js";', }, + { + name: "candidates-subpath", + source: 'export * from "./dist/candidates/index.js";', + }, ]; for (const entry of ENTRIES) { diff --git a/packages/core/src/candidates/classification.ts b/packages/core/src/candidates/classification.ts new file mode 100644 index 0000000..48a859e --- /dev/null +++ b/packages/core/src/candidates/classification.ts @@ -0,0 +1,172 @@ +/** + * Candidate failure and quality decision validation. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + BudgetExceededError, + CancellationError, + TimeoutError, + type TaskContext, +} from "../types/index.js"; +import type { + AcceptanceDecision, + CandidateActionDecision, + CandidateFailureDecision, + FirstAcceptableOptions, +} from "./types.js"; + +const REASON_CODE_PATTERN = /^[a-z][a-z0-9_]{0,127}$/; +const ACTION_DISPOSITIONS = new Set([ + "retry_same_candidate", + "try_next_candidate", + "terminal", + "requires_user_input", +]); + +const WORKIT_BUDGET_EXHAUSTED = Object.freeze({ + disposition: "terminal", + reasonCode: "workit_budget_exhausted", +}); +const WORKIT_TIMEOUT = Object.freeze({ + disposition: "terminal", + reasonCode: "workit_timeout", +}); +const WORKIT_CANCELLED = Object.freeze({ + disposition: "cancelled", + reasonCode: "workit_cancelled", +}); + +const WORKIT_FAILURE_CLASSIFIERS: ReadonlyArray<{ + readonly matches: (error: unknown) => boolean; + readonly decision: CandidateFailureDecision; +}> = Object.freeze([ + { + matches: (error) => safeInstanceOf(error, BudgetExceededError), + decision: WORKIT_BUDGET_EXHAUSTED, + }, + { + matches: (error) => safeInstanceOf(error, TimeoutError), + decision: WORKIT_TIMEOUT, + }, + { + matches: (error) => safeInstanceOf(error, CancellationError), + decision: WORKIT_CANCELLED, + }, +]); + +/** Returns the safe built-in classification for a known WorkIt runtime error. */ +export function classifyWorkItFailure(error: unknown): CandidateFailureDecision | undefined { + return WORKIT_FAILURE_CLASSIFIERS.find((classifier) => classifier.matches(error))?.decision; +} + +export class ClassifiedCandidateError extends Error { + readonly decision: CandidateActionDecision; + readonly original: unknown; + + constructor(original: unknown, decision: CandidateActionDecision) { + super(readErrorMessage(original)); + this.name = readErrorName(original); + this.original = original; + this.decision = decision; + } +} + +export class CandidateCallbackError extends Error { + readonly original: unknown; + + constructor(original: unknown) { + super("candidate policy callback failed"); + this.name = "CandidateCallbackError"; + this.original = original; + } +} + +export async function classifyCandidateError( + error: unknown, + candidate: C, + ctx: TaskContext, + classify: FirstAcceptableOptions["classifyFailure"], +): Promise { + const builtIn = classifyWorkItFailure(error); + if (builtIn?.disposition === "cancelled") throw error; + + let decision: CandidateActionDecision; + try { + decision = builtIn as CandidateActionDecision | undefined + ?? normalizeActionDecision(await classify(error, candidate, ctx)); + } catch (callbackError) { + if (classifyWorkItFailure(callbackError) !== undefined) throw callbackError; + throw new CandidateCallbackError(callbackError); + } + return new ClassifiedCandidateError(error, decision); +} + +export function normalizeAcceptanceDecision(value: unknown): AcceptanceDecision { + if (!isRecord(value)) { + throw new TypeError("accept must return an AcceptanceDecision"); + } + const accepted = value.accepted; + if (typeof accepted !== "boolean") throw new TypeError("accept must return an AcceptanceDecision"); + if (accepted) return Object.freeze({ accepted: true }); + const reasonCode = value.reasonCode; + assertReasonCode(reasonCode); + return Object.freeze({ accepted: false, reasonCode }); +} + +function normalizeActionDecision(value: unknown): CandidateActionDecision { + if (!isRecord(value)) { + throw new TypeError("classifyFailure returned an invalid disposition"); + } + const disposition = value.disposition; + if (!ACTION_DISPOSITIONS.has(disposition as CandidateActionDecision["disposition"])) { + throw new TypeError("classifyFailure returned an invalid disposition"); + } + const reasonCode = value.reasonCode; + assertReasonCode(reasonCode); + return Object.freeze({ + disposition: disposition as CandidateActionDecision["disposition"], + reasonCode, + }); +} + +function assertReasonCode(value: unknown): asserts value is string { + if (typeof value !== "string" || !REASON_CODE_PATTERN.test(value)) { + throw new RangeError("reasonCode must match /^[a-z][a-z0-9_]{0,127}$/"); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function readErrorName(error: unknown): string { + return readErrorProperty(error, "name", "Error"); +} + +function readErrorMessage(error: unknown): string { + if (safeInstanceOf(error, Error)) return readErrorProperty(error, "message", "Candidate attempt failed"); + return typeof error === "string" ? error : "Candidate attempt failed"; +} + +function readErrorProperty(error: unknown, key: "name" | "message", fallback: string): string { + try { + const value = (error as Error)[key]; + return typeof value === "string" ? value.slice(0, 1_024) : fallback; + } catch { + return fallback; + } +} + +function safeInstanceOf( + value: unknown, + errorType: { readonly [Symbol.hasInstance]: (candidate: unknown) => boolean }, +): boolean { + try { + return errorType[Symbol.hasInstance](value); + } catch { + return false; + } +} diff --git a/packages/core/src/candidates/evidence.ts b/packages/core/src/candidates/evidence.ts new file mode 100644 index 0000000..3843d2d --- /dev/null +++ b/packages/core/src/candidates/evidence.ts @@ -0,0 +1,51 @@ +/** + * Bounded candidate evidence mapping over the shared attempt recorder. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { + WorkItAttemptEvidence, + WorkItReceiptError, +} from "../replay/index.js"; +import type { + CandidateAttemptDecision, + CandidateAttemptEvidence, +} from "./types.js"; + +const MAX_ERROR_NAME_LENGTH = 128; +const MAX_ERROR_MESSAGE_LENGTH = 1_024; + +export interface CandidateDecisionRecord { + readonly candidateIndex: number; + readonly decision: CandidateAttemptDecision; + readonly reasonCode?: string; +} + +export function mapCandidateEvidence( + attempts: readonly WorkItAttemptEvidence[], + decisions: readonly CandidateDecisionRecord[], +): readonly CandidateAttemptEvidence[] { + return attempts.map((attempt, index) => { + const decision = decisions[index]!; + return { + ...attempt, + candidateIndex: decision.candidateIndex, + decision: decision.decision, + ...(decision.reasonCode !== undefined ? { reasonCode: decision.reasonCode } : {}), + ...(attempt.error !== undefined ? { error: boundError(attempt.error) } : {}), + }; + }); +} + +function boundError(error: WorkItReceiptError): WorkItReceiptError { + return { + name: truncate(error.name, MAX_ERROR_NAME_LENGTH), + message: truncate(error.message, MAX_ERROR_MESSAGE_LENGTH), + }; +} + +function truncate(value: string, maxLength: number): string { + return value.length <= maxLength ? value : value.slice(0, maxLength); +} diff --git a/packages/core/src/candidates/index.ts b/packages/core/src/candidates/index.ts new file mode 100644 index 0000000..7b437d7 --- /dev/null +++ b/packages/core/src/candidates/index.ts @@ -0,0 +1,328 @@ +/** + * Deterministic candidate selection with typed failure and quality policy. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + * + * Candidate execution is sequential and may repeat side effects. Callers own + * idempotency for side-effecting operations. + */ + +import { createAttemptRecorder } from "../replay/index.js"; +import { run } from "../run/index.js"; +import { validateRetryPolicy } from "../engine/retry.js"; +import type { RetryOpts, TaskFn } from "../types/index.js"; +import { + CandidateCallbackError, + ClassifiedCandidateError, + classifyCandidateError, + classifyWorkItFailure as classifyKnownWorkItFailure, + normalizeAcceptanceDecision, +} from "./classification.js"; +import { + mapCandidateEvidence, + type CandidateDecisionRecord, +} from "./evidence.js"; +import type { + CandidateActionDecision, + CandidateFailureDecision, + CandidateRetryPolicy, + CandidateRunResult, + FirstAcceptableOptions, +} from "./types.js"; + +export type { + AcceptanceDecision, + CandidateActionDecision, + CandidateAttemptDecision, + CandidateAttemptEvidence, + CandidateFailureDecision, + CandidateRetryPolicy, + CandidateRunResult, + FailureDisposition, + FirstAcceptableOptions, +} from "./types.js"; + +const DEFAULT_MAX_CANDIDATES = 16; +const MAX_CANDIDATES = 1_000; +const DEFAULT_MAX_EVIDENCE = 256; +const DEFAULT_RETRY_POLICY: CandidateRetryPolicy = Object.freeze({ times: 1 }); +const MAX_TOTAL_CANDIDATE_ATTEMPTS = 10_000; +const CANDIDATE_TASK_NAME = "candidate"; + +/** Returns the safe built-in classification for a known WorkIt runtime error. */ +export function classifyWorkItFailure(error: unknown): CandidateFailureDecision | undefined { + return classifyKnownWorkItFailure(error); +} + +type AttemptResult = + | { readonly accepted: true; readonly value: T } + | { readonly accepted: false }; + +interface CandidatePolicy { + readonly execute: FirstAcceptableOptions["execute"]; + readonly accept: FirstAcceptableOptions["accept"]; + readonly classifyFailure: FirstAcceptableOptions["classifyFailure"]; + readonly retry: RetryOpts; + readonly deadlineAt?: number; + readonly candidateMetadata?: FirstAcceptableOptions["candidateMetadata"]; +} + +interface CandidateDecisionBuffer { + readonly records: CandidateDecisionRecord[]; + readonly limit: number; +} + +/** Selects the first semantically acceptable result in deterministic candidate order. */ +export async function firstAcceptable( + candidates: readonly C[], + opts: FirstAcceptableOptions, +): Promise> { + const admittedCandidates = snapshotCandidates(candidates); + const policy = normalizeCandidatePolicy(admittedCandidates, opts); + const evidenceOptions = opts.evidence; + const evidenceLimit = evidenceOptions?.maxAttempts ?? DEFAULT_MAX_EVIDENCE; + const recorder = createAttemptRecorder({ + ...evidenceOptions, + maxAttempts: evidenceLimit, + }); + const decisions: CandidateDecisionBuffer = { records: [], limit: evidenceLimit }; + + for (let candidateIndex = 0; candidateIndex < admittedCandidates.length; candidateIndex++) { + const candidate = admittedCandidates[candidateIndex]!; + const task = buildCandidateTask(candidate, candidateIndex, policy, recorder, decisions); + try { + const attemptResult = await run.group(async (spawn) => spawn(task, { name: CANDIDATE_TASK_NAME })); + if (!attemptResult.accepted) continue; + return { + status: "accepted", + candidate, + candidateIndex, + value: attemptResult.value, + ...readEvidence(recorder, decisions.records), + }; + } catch (error) { + if (!(error instanceof ClassifiedCandidateError)) throw unwrapCallbackError(error); + const result = resultFromFailure(error, recorder, decisions.records); + if (result !== undefined) return result; + } + } + + return { + status: "exhausted", + ...readEvidence(recorder, decisions.records), + }; +} + +function buildCandidateTask( + candidate: C, + candidateIndex: number, + policy: CandidatePolicy, + recorder: ReturnType, + decisions: CandidateDecisionBuffer, +): TaskFn> { + const candidateMetadata = policy.candidateMetadata?.(candidate, candidateIndex); + if (candidateMetadata !== undefined && !isMetadataRecord(candidateMetadata)) { + throw new TypeError("candidateMetadata must return an object when defined"); + } + const metadata = { + ...candidateMetadata, + candidateIndex, + }; + const attempt = createAttemptTask(candidate, candidateIndex, policy, decisions); + const boundedAttempt = policy.deadlineAt === undefined ? attempt : run.deadline(attempt, policy.deadlineAt); + const classifiedAttempt = classifyDeadlineFailure(boundedAttempt, candidate, candidateIndex, policy, decisions); + const recordedAttempt = recorder.wrap(classifiedAttempt, { + metadata, + reasonCode: (error) => error instanceof ClassifiedCandidateError + ? error.decision.reasonCode + : undefined, + }); + return run.retry(recordedAttempt, withClassifierRetry(policy.retry)); +} + +function createAttemptTask( + candidate: C, + candidateIndex: number, + policy: CandidatePolicy, + decisions: CandidateDecisionBuffer, +): TaskFn> { + return async (ctx) => { + let value: T; + try { + value = await policy.execute(candidate, ctx); + } catch (error) { + const classified = await classifyCandidateError(error, candidate, ctx, policy.classifyFailure); + recordFailure(decisions, candidateIndex, classified.decision); + throw classified; + } + + try { + const acceptance = normalizeAcceptanceDecision(await policy.accept(value, candidate, ctx)); + if (acceptance.accepted) { + recordDecision(decisions, { candidateIndex, decision: "accepted" }); + return { accepted: true, value }; + } + recordDecision(decisions, { + candidateIndex, + decision: "quality_rejected", + reasonCode: acceptance.reasonCode, + }); + return { accepted: false }; + } catch (error) { + if (classifyKnownWorkItFailure(error) !== undefined) throw error; + throw new CandidateCallbackError(error); + } + }; +} + +function classifyDeadlineFailure( + task: TaskFn>, + candidate: C, + candidateIndex: number, + policy: CandidatePolicy, + decisions: CandidateDecisionBuffer, +): TaskFn> { + return async (ctx) => { + try { + return await task(ctx); + } catch (error) { + if (error instanceof ClassifiedCandidateError || error instanceof CandidateCallbackError) throw error; + const classified = await classifyCandidateError(error, candidate, ctx, policy.classifyFailure); + recordFailure(decisions, candidateIndex, classified.decision); + throw classified; + } + }; +} + +function withClassifierRetry(retry: RetryOpts): RetryOpts & { + readonly retryIf: (error: unknown) => boolean; +} { + const retryIf = (error: unknown) => error instanceof ClassifiedCandidateError + && error.decision.disposition === "retry_same_candidate"; + return { ...retry, retryIf }; +} + +function resultFromFailure( + error: ClassifiedCandidateError, + recorder: ReturnType, + decisions: readonly CandidateDecisionRecord[], +): CandidateRunResult | undefined { + const evidence = readEvidence(recorder, decisions); + if (error.decision.disposition === "terminal") { + return { + status: "terminal", + reasonCode: error.decision.reasonCode, + error: error.original, + ...evidence, + }; + } + if (error.decision.disposition === "requires_user_input") { + return { + status: "requires_user_input", + reasonCode: error.decision.reasonCode, + ...evidence, + }; + } + return undefined; +} + +function recordFailure( + decisions: CandidateDecisionBuffer, + candidateIndex: number, + decision: CandidateActionDecision, +): void { + recordDecision(decisions, { + candidateIndex, + decision: decision.disposition, + reasonCode: decision.reasonCode, + }); +} + +function recordDecision(buffer: CandidateDecisionBuffer, decision: CandidateDecisionRecord): void { + if (buffer.records.length < buffer.limit) buffer.records.push(decision); +} + +function readEvidence( + recorder: ReturnType, + decisions: readonly CandidateDecisionRecord[], +): Pick, "evidence" | "droppedEvidence"> { + return { + evidence: mapCandidateEvidence(recorder.attempts, decisions), + droppedEvidence: recorder.droppedAttempts, + }; +} + +function snapshotCandidates(candidates: readonly C[]): readonly C[] { + if (!Array.isArray(candidates)) throw new TypeError("candidates must be an array"); + return Object.freeze(Array.prototype.slice.call(candidates) as C[]); +} + +function normalizeCandidatePolicy( + candidates: readonly C[], + opts: FirstAcceptableOptions, +): CandidatePolicy { + if (typeof opts !== "object" || opts === null) throw new TypeError("options are required"); + const execute = opts.execute; + const accept = opts.accept; + const classifyFailure = opts.classifyFailure; + if (typeof execute !== "function" || typeof accept !== "function" || typeof classifyFailure !== "function") { + throw new TypeError("execute, accept, and classifyFailure must be functions"); + } + const maxCandidates = opts.maxCandidates ?? DEFAULT_MAX_CANDIDATES; + if (!Number.isInteger(maxCandidates) || maxCandidates < 1 || maxCandidates > MAX_CANDIDATES) { + throw new RangeError(`maxCandidates must be an integer between 1 and ${MAX_CANDIDATES}`); + } + if (candidates.length > maxCandidates) throw new RangeError("candidates length exceeds maxCandidates"); + const retry = snapshotRetryPolicy(opts.retry); + if (candidates.length * retry.times > MAX_TOTAL_CANDIDATE_ATTEMPTS) { + throw new RangeError(`total candidate attempts must not exceed ${MAX_TOTAL_CANDIDATE_ATTEMPTS}`); + } + const deadlineAt = normalizeDeadline(opts.deadlineAt); + const candidateMetadata = opts.candidateMetadata; + if (candidateMetadata !== undefined && typeof candidateMetadata !== "function") { + throw new TypeError("candidateMetadata must be a function"); + } + return Object.freeze({ + execute, + accept, + classifyFailure, + retry, + ...(deadlineAt !== undefined ? { deadlineAt } : {}), + ...(candidateMetadata !== undefined ? { candidateMetadata } : {}), + }); +} + +function snapshotRetryPolicy(value: CandidateRetryPolicy | undefined): RetryOpts { + const retry = value ?? DEFAULT_RETRY_POLICY; + if (typeof retry === "number") { + validateRetryPolicy(retry); + return Object.freeze({ times: retry }); + } + if (Object.hasOwn(retry, "retryIf")) throw new TypeError("candidate retry policy must not define retryIf"); + const snapshot: RetryOpts = { + times: retry.times, + ...(retry.backoff !== undefined ? { backoff: retry.backoff } : {}), + ...(retry.initialDelay !== undefined ? { initialDelay: retry.initialDelay } : {}), + ...(retry.maxDelay !== undefined ? { maxDelay: retry.maxDelay } : {}), + ...(retry.jitter !== undefined ? { jitter: retry.jitter } : {}), + ...(retry.retryBudget !== undefined ? { retryBudget: retry.retryBudget } : {}), + }; + validateRetryPolicy(snapshot); + return Object.freeze(snapshot); +} + +function normalizeDeadline(value: number | Date | undefined): number | undefined { + if (value === undefined) return undefined; + const deadlineAt = typeof value === "number" ? value : value.getTime(); + if (!Number.isFinite(deadlineAt)) throw new RangeError("deadlineAt must be a finite timestamp or valid Date"); + return deadlineAt; +} + +function unwrapCallbackError(error: unknown): unknown { + return error instanceof CandidateCallbackError ? error.original : error; +} + +function isMetadataRecord(value: unknown): value is Readonly> { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/core/src/candidates/types.ts b/packages/core/src/candidates/types.ts new file mode 100644 index 0000000..875f3fe --- /dev/null +++ b/packages/core/src/candidates/types.ts @@ -0,0 +1,97 @@ +/** + * Public contracts for deterministic candidate selection. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { + AttemptRecorderOptions, + WorkItAttemptEvidence, +} from "../replay/index.js"; +import type { RetryOpts, TaskContext } from "../types/index.js"; + +/** Policy outcome assigned to a failed candidate attempt. */ +export type FailureDisposition = + | "retry_same_candidate" + | "try_next_candidate" + | "terminal" + | "cancelled" + | "requires_user_input"; + +/** Bounded, machine-readable classification of a candidate failure. */ +export interface CandidateFailureDecision { + readonly disposition: FailureDisposition; + readonly reasonCode: string; +} + +/** Failure decisions a provider classifier may return. Real cancellation is detected by WorkIt. */ +export interface CandidateActionDecision extends CandidateFailureDecision { + readonly disposition: Exclude; +} + +/** Semantic quality decision kept separate from transport success. */ +export type AcceptanceDecision = + | { readonly accepted: true } + | { readonly accepted: false; readonly reasonCode: string }; + +/** Candidate-level interpretation of one recorded task-body invocation. */ +export type CandidateAttemptDecision = + | CandidateActionDecision["disposition"] + | "quality_rejected" + | "accepted"; + +/** Bounded attempt evidence enriched with candidate and quality policy facts. */ +export interface CandidateAttemptEvidence extends WorkItAttemptEvidence { + readonly candidateIndex: number; + readonly decision: CandidateAttemptDecision; +} + +interface CandidateResultEvidence { + readonly evidence: readonly CandidateAttemptEvidence[]; + readonly droppedEvidence: number; +} + +/** Exhaustive expected outcome of a candidate selection run. */ +export type CandidateRunResult = + | (CandidateResultEvidence & { + readonly status: "accepted"; + readonly candidate: C; + readonly candidateIndex: number; + readonly value: T; + }) + | (CandidateResultEvidence & { + readonly status: "exhausted"; + }) + | (CandidateResultEvidence & { + readonly status: "terminal"; + readonly reasonCode: string; + readonly error: unknown; + }) + | (CandidateResultEvidence & { + readonly status: "requires_user_input"; + readonly reasonCode: string; + }); + +/** Retry policy whose admission predicate is owned by the failure classifier. */ +export type CandidateRetryPolicy = number | Omit; + +/** Candidate selection policy and bounded evidence controls. */ +export interface FirstAcceptableOptions { + readonly execute: (candidate: C, ctx: TaskContext) => T | Promise; + readonly accept: (value: T, candidate: C, ctx: TaskContext) => AcceptanceDecision | Promise; + readonly classifyFailure: ( + error: unknown, + candidate: C, + ctx: TaskContext, + ) => CandidateActionDecision | Promise; + readonly retry?: CandidateRetryPolicy; + readonly maxCandidates?: number; + readonly deadlineAt?: number | Date; + readonly evidence?: AttemptRecorderOptions; + readonly candidateMetadata?: ( + candidate: C, + candidateIndex: number, + ) => Readonly> | undefined; +} + diff --git a/packages/core/src/replay/index.ts b/packages/core/src/replay/index.ts index 6c70253..0b9f817 100644 --- a/packages/core/src/replay/index.ts +++ b/packages/core/src/replay/index.ts @@ -662,9 +662,15 @@ function redactValue(value: unknown, policy: RedactionRuntimePolicy, depth: numb for (const [key, item] of Object.entries(input)) { const normalizedKey = key.toLowerCase(); if (policy.removeFields.has(normalizedKey)) continue; - output[key] = policy.redactFields.has(normalizedKey) + const redactedItem = policy.redactFields.has(normalizedKey) ? "[redacted]" : redactValue(item, policy, depth + 1); + Object.defineProperty(output, key, { + value: redactedItem, + enumerable: true, + configurable: true, + writable: true, + }); } return output; } diff --git a/packages/core/src/runtime/unsupported.ts b/packages/core/src/runtime/unsupported.ts index 36462bd..963b377 100644 --- a/packages/core/src/runtime/unsupported.ts +++ b/packages/core/src/runtime/unsupported.ts @@ -90,6 +90,8 @@ export const streamWithBackpressure = unsupportedFunction("streamWithBackpressur export const transcribeStream = unsupportedFunction("transcribeStream"); export const wrapAI = unsupportedFunction("wrapAI"); export const offload = unsupportedFunction("offload"); +export const firstAcceptable = unsupportedFunction("firstAcceptable"); +export const classifyWorkItFailure = unsupportedFunction("classifyWorkItFailure"); export const run = Object.freeze({ all: unsupportedFunction("run.all"), diff --git a/packages/core/tests/evidence/correctness/candidate-policy.mjs b/packages/core/tests/evidence/correctness/candidate-policy.mjs new file mode 100644 index 0000000..12600a3 --- /dev/null +++ b/packages/core/tests/evidence/correctness/candidate-policy.mjs @@ -0,0 +1,387 @@ +/** + * Correctness evidence: candidate failure, quality, cancellation, and budget policy. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + BudgetExceededError, + CancellationError, + TimeoutError, + createBudget, + run, +} from "../../../dist/index.js"; +import { + classifyWorkItFailure, + firstAcceptable, +} from "../../../dist/candidates/index.js"; +import { createSuite } from "../harness.mjs"; + +const suite = createSuite("correctness"); + +await suite.proof( + "CORR-028", + "candidate policy composes retry, quality rejection, and fallback", + "transport retry and semantic rejection remain distinct before the first acceptable fallback is selected", + async () => { + let primaryCalls = 0; + const result = await firstAcceptable(["primary", "fallback"], { + execute: async (candidate) => { + if (candidate === "primary" && ++primaryCalls === 1) throw new Error("temporary"); + return { candidate, score: candidate === "primary" ? 0.4 : 0.9 }; + }, + accept: async (value) => value.score >= 0.8 + ? { accepted: true } + : { accepted: false, reasonCode: "score_too_low" }, + classifyFailure: async () => ({ + disposition: "retry_same_candidate", + reasonCode: "temporary_failure", + }), + retry: { times: 2, initialDelay: 0, jitter: false }, + evidence: { maxAttempts: 8, maxMetadataBytes: 256 }, + candidateMetadata: (candidate) => ({ candidate, token: "secret" }), + }); + + const evidence = result.evidence.map((attempt) => ({ + candidateIndex: attempt.candidateIndex, + attempt: attempt.attempt, + decision: attempt.decision, + outcome: attempt.outcome, + reasonCode: attempt.reasonCode, + token: attempt.metadata?.token, + })); + return { + ok: result.status === "accepted" + && result.candidate === "fallback" + && result.droppedEvidence === 0 + && JSON.stringify(evidence) === JSON.stringify([ + { + candidateIndex: 0, + attempt: 1, + decision: "retry_same_candidate", + outcome: "failed", + reasonCode: "temporary_failure", + token: "[redacted]", + }, + { + candidateIndex: 0, + attempt: 2, + decision: "quality_rejected", + outcome: "succeeded", + reasonCode: "score_too_low", + token: "[redacted]", + }, + { + candidateIndex: 1, + attempt: 1, + decision: "accepted", + outcome: "succeeded", + token: "[redacted]", + }, + ]), + status: result.status, + evidence, + droppedEvidence: result.droppedEvidence, + }; + }, +); + +await suite.proof( + "CORR-029", + "candidate policy cannot convert cancellation into fallback", + "real cancellation bypasses the provider classifier and no later candidate starts", + async () => { + let classifications = 0; + const admitted = []; + let observed; + try { + await firstAcceptable(["primary", "fallback"], { + execute: async (candidate) => { + admitted.push(candidate); + throw new CancellationError({ kind: "manual", tag: "evidence-stop" }); + }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => { + classifications++; + return { disposition: "try_next_candidate", reasonCode: "incorrect_fallback" }; + }, + }); + } catch (error) { + observed = error; + } + + return { + ok: observed instanceof CancellationError + && observed.reason.kind === "manual" + && observed.reason.tag === "evidence-stop" + && classifications === 0 + && JSON.stringify(admitted) === JSON.stringify(["primary"]), + classifications, + admitted, + cancellationReason: observed?.reason, + }; + }, +); + +await suite.proof( + "CORR-030", + "candidate retries share the existing scope retry budget", + "fallback candidates cannot admit a retry after the shared WorkIt budget is exhausted", + async () => { + const RetryBudget = createBudget("CandidateEvidenceRetryBudget", { unit: "retries" }); + let attempts = 0; + let observed; + try { + await run.context.with( + RetryBudget, + { limit: 1, spent: 0, unit: "retries" }, + async () => firstAcceptable(["primary", "fallback"], { + execute: async () => { + attempts++; + throw new Error("retry"); + }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ + disposition: "retry_same_candidate", + reasonCode: "retryable", + }), + retry: { times: 2, initialDelay: 0, jitter: false, retryBudget: RetryBudget }, + }), + ); + } catch (error) { + observed = error; + } + + return { + ok: observed instanceof BudgetExceededError && attempts === 3, + attempts, + errorName: observed?.name, + budgetKey: observed?.budgetKey, + }; + }, +); + +await suite.proof( + "CORR-031", + "candidate admission snapshots inputs and enforces an aggregate attempt cap", + "callbacks cannot expand the admitted candidate set and unsafe candidate-count by retry-count products fail before execution", + async () => { + const candidates = ["original"]; + const admitted = []; + const result = await firstAcceptable(candidates, { + execute: async (candidate) => { + admitted.push(candidate); + candidates.push("injected"); + return candidate; + }, + accept: async () => ({ accepted: false, reasonCode: "quality_rejected" }), + classifyFailure: async () => ({ disposition: "try_next_candidate", reasonCode: "candidate_failed" }), + }); + + let unsafeError; + try { + await firstAcceptable(Array.from({ length: 11 }, (_, index) => index), { + execute: async (candidate) => candidate, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "unexpected_failure" }), + maxCandidates: 11, + retry: { times: 1_000 }, + }); + } catch (error) { + unsafeError = error; + } + + return { + ok: result.status === "exhausted" + && JSON.stringify(admitted) === JSON.stringify(["original"]) + && unsafeError instanceof RangeError, + admitted, + unsafeError: unsafeError?.message, + }; + }, +); + +await suite.proof( + "CORR-032", + "candidate decisions are normalized once before policy action", + "mutable decision getters cannot change disposition, reason, or acceptance after validation", + async () => { + let dispositionReads = 0; + let reasonReads = 0; + const failureDecision = { + get disposition() { + dispositionReads++; + return dispositionReads === 1 ? "terminal" : "retry_same_candidate"; + }, + get reasonCode() { + reasonReads++; + return reasonReads === 1 ? "stable_reason" : "changed_reason"; + }, + }; + const failureResult = await firstAcceptable(["failure"], { + execute: async () => { throw new Error("provider failed"); }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => failureDecision, + retry: { times: 3, initialDelay: 0 }, + }); + + let acceptedReads = 0; + const acceptanceDecision = { + get accepted() { + acceptedReads++; + return acceptedReads === 1; + }, + }; + const acceptanceResult = await firstAcceptable(["accepted"], { + execute: async (candidate) => candidate, + accept: async () => acceptanceDecision, + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "unexpected_failure" }), + }); + + return { + ok: failureResult.status === "terminal" + && failureResult.reasonCode === "stable_reason" + && dispositionReads === 1 + && reasonReads === 1 + && acceptanceResult.status === "accepted" + && acceptedReads === 1, + failureStatus: failureResult.status, + failureReason: failureResult.status === "terminal" ? failureResult.reasonCode : null, + dispositionReads, + reasonReads, + acceptanceStatus: acceptanceResult.status, + acceptedReads, + }; + }, +); + +await suite.proof( + "CORR-033", + "candidate error taxonomy classifies known WorkIt failures conservatively", + "cancellation, timeout, and budget errors have stable built-in decisions while unknown provider errors remain caller-owned", + async () => { + const cancellation = classifyWorkItFailure(new CancellationError({ kind: "manual", tag: "taxonomy" })); + const timeout = classifyWorkItFailure(new TimeoutError(10)); + const budget = classifyWorkItFailure(new BudgetExceededError({ + budgetKey: "TaxonomyBudget", + limit: 1, + spent: 2, + attempted: 1, + })); + const unknown = classifyWorkItFailure(new Error("provider")); + return { + ok: JSON.stringify(cancellation) === JSON.stringify({ + disposition: "cancelled", + reasonCode: "workit_cancelled", + }) + && JSON.stringify(timeout) === JSON.stringify({ + disposition: "terminal", + reasonCode: "workit_timeout", + }) + && JSON.stringify(budget) === JSON.stringify({ + disposition: "terminal", + reasonCode: "workit_budget_exhausted", + }) + && unknown === undefined + && Object.isFrozen(cancellation), + cancellation, + timeout, + budget, + unknown: unknown ?? null, + }; + }, +); + +await suite.proof( + "CORR-034", + "candidate policy callback failures cannot silently become fallback", + "quality and classifier callback bugs propagate unchanged and admit no later candidate", + async () => { + const qualityFailure = new Error("quality callback failed"); + const classifierFailure = new Error("classifier callback failed"); + const qualityAdmitted = []; + const classifierAdmitted = []; + let observedQuality; + let observedClassifier; + + try { + await firstAcceptable(["primary", "fallback"], { + execute: async (candidate) => { + qualityAdmitted.push(candidate); + return candidate; + }, + accept: async () => { throw qualityFailure; }, + classifyFailure: async () => ({ disposition: "try_next_candidate", reasonCode: "provider_failure" }), + }); + } catch (error) { + observedQuality = error; + } + + try { + await firstAcceptable(["primary", "fallback"], { + execute: async (candidate) => { + classifierAdmitted.push(candidate); + throw new Error("provider failed"); + }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => { throw classifierFailure; }, + }); + } catch (error) { + observedClassifier = error; + } + + return { + ok: observedQuality === qualityFailure + && observedClassifier === classifierFailure + && JSON.stringify(qualityAdmitted) === JSON.stringify(["primary"]) + && JSON.stringify(classifierAdmitted) === JSON.stringify(["primary"]), + qualityFailurePreserved: observedQuality === qualityFailure, + classifierFailurePreserved: observedClassifier === classifierFailure, + qualityAdmitted, + classifierAdmitted, + }; + }, +); + +await suite.proof( + "CORR-036", + "hostile thrown values cannot bypass candidate classification", + "proxy traps and throwing Error accessors are bounded while the caller classifier retains policy authority", + async () => { + const hostileProxy = new Proxy({}, { + get() { throw new Error("hostile get"); }, + getPrototypeOf() { throw new Error("hostile prototype"); }, + }); + const hostileError = new Error("hidden"); + Object.defineProperties(hostileError, { + name: { get: () => { throw new Error("hostile name"); } }, + message: { get: () => { throw new Error("hostile message"); } }, + }); + + const reports = []; + for (const failure of [hostileProxy, hostileError]) { + const result = await firstAcceptable(["candidate"], { + execute: async () => { throw failure; }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "hostile_failure" }), + }); + reports.push({ + status: result.status, + reasonCode: result.status === "terminal" ? result.reasonCode : null, + originalPreserved: result.status === "terminal" && result.error === failure, + }); + } + + return { + ok: reports.every((report) => report.status === "terminal" + && report.reasonCode === "hostile_failure" + && report.originalPreserved), + reports, + }; + }, +); + +const summary = suite.summary(); +process.stdout.write(JSON.stringify(summary, null, 2) + "\n"); +process.exit(summary.failed > 0 ? 1 : 0); diff --git a/packages/core/tests/evidence/correctness/candidate-scenarios.mjs b/packages/core/tests/evidence/correctness/candidate-scenarios.mjs new file mode 100644 index 0000000..1853cd8 --- /dev/null +++ b/packages/core/tests/evidence/correctness/candidate-scenarios.mjs @@ -0,0 +1,155 @@ +/** + * Bounded operational scenarios for the candidate policy contract. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CancellationError, group } from "../../../dist/index.js"; +import { firstAcceptable } from "../../../dist/candidates/index.js"; +import { createSuite } from "../harness.mjs"; + +const suite = createSuite("correctness"); + +await suite.proof( + "CORR-037", + "candidate policy composes bounded operational decision scenarios", + "quality fallback, caller-owned idempotency, user-input stops, and cooperative cancellation preserve their declared boundaries", + async () => { + const reports = [ + await runQualityFallback(), + await runCallerOwnedIdempotency(), + await runUserInputStop(), + await runCancellation(), + ]; + return { + ok: reports.every((report) => report.status === "pass"), + reports, + limitation: "These are bounded in-process operational fixtures, not a provider, durable-store, or Oryn production canary.", + }; + }, +); + +const summary = suite.summary(); +process.stdout.write(JSON.stringify(summary, null, 2) + "\n"); +process.exit(summary.failed > 0 ? 1 : 0); + +async function runQualityFallback() { + const calls = []; + const result = await firstAcceptable(["reasoning", "balanced"], { + execute: async (model, ctx) => { + calls.push({ model, attempt: ctx.attempt, deadlineAt: ctx.deadlineAt }); + if (model === "reasoning" && ctx.attempt === 1) throw new Error("provider overloaded"); + return { model, groundedness: model === "reasoning" ? 0.63 : 0.91 }; + }, + accept: async (answer) => answer.groundedness >= 0.85 + ? { accepted: true } + : { accepted: false, reasonCode: "groundedness_too_low" }, + classifyFailure: async () => ({ + disposition: "retry_same_candidate", + reasonCode: "provider_overloaded", + }), + retry: { times: 2, initialDelay: 0, jitter: false }, + deadlineAt: Date.now() + 5_000, + }); + const decisions = result.evidence.map((attempt) => attempt.decision); + return { + name: "quality-fallback", + status: result.status === "accepted" + && result.candidate === "balanced" + && JSON.stringify(decisions) === JSON.stringify([ + "retry_same_candidate", + "quality_rejected", + "accepted", + ]) + && calls.every((call) => typeof call.deadlineAt === "number") + ? "pass" + : "fail", + selected: result.status === "accepted" ? result.candidate : null, + decisions, + }; +} + +async function runCallerOwnedIdempotency() { + const inProcessStore = new Map(); + let providerCalls = 0; + const result = await firstAcceptable(["payments-primary"], { + execute: async (_provider, ctx) => { + providerCalls++; + const idempotencyKey = "order-1042"; + const existing = inProcessStore.get(idempotencyKey); + if (existing !== undefined) return existing; + const charge = { chargeId: "charge-1042", settled: true }; + inProcessStore.set(idempotencyKey, charge); + if (ctx.attempt === 1) throw new Error("response lost after commit"); + return charge; + }, + accept: async (charge) => charge.settled + ? { accepted: true } + : { accepted: false, reasonCode: "charge_not_settled" }, + classifyFailure: async () => ({ disposition: "retry_same_candidate", reasonCode: "response_lost" }), + retry: { times: 2, initialDelay: 0, jitter: false }, + }); + return { + name: "caller-owned-idempotency", + status: result.status === "accepted" && providerCalls === 2 && inProcessStore.size === 1 ? "pass" : "fail", + providerCalls, + sideEffects: inProcessStore.size, + store: "bounded-in-process-fixture", + }; +} + +async function runUserInputStop() { + const admitted = []; + const result = await firstAcceptable(["restricted-tool", "unsafe-fallback"], { + execute: async (candidate) => { + admitted.push(candidate); + throw new Error("approval required"); + }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ + disposition: "requires_user_input", + reasonCode: "tool_approval_required", + }), + }); + return { + name: "user-input-stop", + status: result.status === "requires_user_input" + && JSON.stringify(admitted) === JSON.stringify(["restricted-tool"]) + ? "pass" + : "fail", + admitted, + reasonCode: result.status === "requires_user_input" ? result.reasonCode : null, + }; +} + +async function runCancellation() { + const admitted = []; + let observed; + try { + await group(async (spawn) => spawn(async (ctx) => { + setTimeout(() => ctx.scope.cancel("request-disconnected"), 5); + await firstAcceptable(["slow-primary", "fallback"], { + execute: async (candidate, candidateCtx) => { + admitted.push(candidate); + await new Promise((_resolve, reject) => { + candidateCtx.signal.addEventListener("abort", () => reject(candidateCtx.signal.reason), { once: true }); + }); + }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ disposition: "try_next_candidate", reasonCode: "provider_failed" }), + }); + })); + } catch (error) { + observed = error; + } + return { + name: "cooperative-cancellation", + status: observed instanceof CancellationError + && JSON.stringify(admitted) === JSON.stringify(["slow-primary"]) + ? "pass" + : "fail", + admitted, + cancellationKind: observed?.reason?.kind, + }; +} diff --git a/packages/core/tests/evidence/correctness/runtime-resilience.mjs b/packages/core/tests/evidence/correctness/runtime-resilience.mjs new file mode 100644 index 0000000..0db9ab6 --- /dev/null +++ b/packages/core/tests/evidence/correctness/runtime-resilience.mjs @@ -0,0 +1,48 @@ +/** + * Correctness evidence for telemetry failure isolation introduced in 0.5.0. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { EventBus } from "../../../dist/engine/event-bus.js"; +import { createSuite } from "../harness.mjs"; + +const suite = createSuite("correctness"); + +await suite.proof( + "CORR-035", + "telemetry context and observer failures remain isolated", + "throwing context access and one throwing observer cannot interrupt delivery to another observer or escape event emission", + async () => { + const bus = new EventBus(); + const delivered = []; + bus.on(() => { throw new Error("observer failed"); }); + bus.on((event) => delivered.push(event.type)); + const hostileContext = { + get() { throw new Error("context failed"); }, + }; + const event = { + type: "scope:closed", + scopeId: "scope-evidence", + durationMs: 1, + at: Date.now(), + }; + + let contextEscaped = false; + let observerEscaped = false; + try { bus.emit(event, hostileContext); } catch { contextEscaped = true; } + try { bus.emit(event); } catch { observerEscaped = true; } + + return { + ok: !contextEscaped && !observerEscaped && JSON.stringify(delivered) === JSON.stringify(["scope:closed"]), + contextEscaped, + observerEscaped, + delivered, + }; + }, +); + +const summary = suite.summary(); +process.stdout.write(JSON.stringify(summary, null, 2) + "\n"); +process.exit(summary.failed > 0 ? 1 : 0); diff --git a/packages/core/tests/evidence/lifecycle/candidate-lifecycle.mjs b/packages/core/tests/evidence/lifecycle/candidate-lifecycle.mjs new file mode 100644 index 0000000..3ce8ebc --- /dev/null +++ b/packages/core/tests/evidence/lifecycle/candidate-lifecycle.mjs @@ -0,0 +1,90 @@ +/** + * Lifecycle evidence for candidate deadline and callback cancellation authority. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CancellationError } from "../../../dist/index.js"; +import { firstAcceptable } from "../../../dist/candidates/index.js"; +import { createSuite } from "../harness.mjs"; + +const suite = createSuite("lifecycle"); + +await suite.proof( + "LIFE-013", + "candidate deadline is aggregate and terminal across the chain", + "one absolute deadline reaches the admitted candidate context and timeout admits no fallback candidate", + async () => { + const deadlineAt = Date.now() + 15; + const admitted = []; + const observedDeadlines = []; + const result = await firstAcceptable(["slow", "fallback"], { + execute: async (candidate, ctx) => { + admitted.push(candidate); + observedDeadlines.push(ctx.deadlineAt); + await new Promise((_resolve, reject) => { + ctx.signal.addEventListener("abort", () => reject(ctx.signal.reason), { once: true }); + }); + }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ disposition: "try_next_candidate", reasonCode: "provider_failure" }), + deadlineAt, + }); + + return { + ok: result.status === "terminal" + && result.reasonCode === "workit_timeout" + && JSON.stringify(admitted) === JSON.stringify(["slow"]) + && JSON.stringify(observedDeadlines) === JSON.stringify([deadlineAt]), + status: result.status, + reasonCode: result.status === "terminal" ? result.reasonCode : null, + admitted, + observedDeadlines, + deadlineAt, + }; + }, +); + +await suite.proof( + "LIFE-014", + "candidate callback cancellation remains authoritative", + "cancellation thrown by quality or classification callbacks propagates unchanged and admits no fallback", + async () => { + const qualityCancellation = new CancellationError({ kind: "manual", tag: "quality-stop" }); + const classifierCancellation = new CancellationError({ kind: "manual", tag: "classifier-stop" }); + const admitted = []; + const observed = []; + + for (const scenario of ["quality", "classifier"]) { + try { + await firstAcceptable(["primary", "fallback"], { + execute: async (candidate) => { + admitted.push(`${scenario}:${candidate}`); + if (scenario === "classifier") throw new Error("provider failure"); + return candidate; + }, + accept: async () => { + if (scenario === "quality") throw qualityCancellation; + return { accepted: true }; + }, + classifyFailure: async () => { throw classifierCancellation; }, + }); + } catch (error) { + observed.push(error); + } + } + + return { + ok: observed[0] === qualityCancellation + && observed[1] === classifierCancellation + && JSON.stringify(admitted) === JSON.stringify(["quality:primary", "classifier:primary"]), + admitted, + cancellationTags: observed.map((error) => error?.reason?.tag), + }; + }, +); + +const summary = suite.summary(); +process.stdout.write(JSON.stringify(summary, null, 2) + "\n"); +process.exit(summary.failed > 0 ? 1 : 0); diff --git a/packages/core/tests/evidence/manifest.mjs b/packages/core/tests/evidence/manifest.mjs new file mode 100644 index 0000000..700666f --- /dev/null +++ b/packages/core/tests/evidence/manifest.mjs @@ -0,0 +1,44 @@ +/** + * Single manifest of publication evidence proof processes. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +/** Proof processes executed by the publication evidence runner. */ +export const evidenceProofs = Object.freeze([ + proof("lifecycle/owned-work.mjs"), + proof("lifecycle/activity-restart.mjs"), + proof("lifecycle/resource-audit.mjs"), + proof("lifecycle/resource-ownership.mjs"), + proof("lifecycle/replay-receipts.mjs"), + proof("lifecycle/candidate-lifecycle.mjs"), + proof("correctness/agent-authority.mjs"), + proof("correctness/analysis-verifiers.mjs"), + proof("correctness/activity-boundary.mjs"), + proof("correctness/candidate-policy.mjs"), + proof("correctness/candidate-scenarios.mjs"), + proof("correctness/fault-injection.mjs"), + proof("correctness/resource-ownership-model.mjs"), + proof("correctness/runtime-contracts.mjs"), + proof("correctness/runtime-resilience.mjs"), + proof("correctness/source-protocol-analysis.mjs"), + proof("correctness/time-policy-planner.mjs"), + proof("correctness/formal-time-policy-model.mjs"), + proof("correctness/nested-time-policy-composition.mjs"), + proof("correctness/typed-cancellation-contracts.mjs"), + proof("security/worker-boundary.mjs"), + proof("security/candidate-boundary.mjs"), + proof("release/candidate-package-contract.mjs"), + proof("release/core-release-contracts.mjs"), + proof("release/release-integrity.mjs"), + proof("release/receipt-ledger.mjs"), + proof("release/sql-receipt-ledger.mjs"), + proof("release/sql-ledger-integration.mjs"), + proof("performance/benchmark-contracts.mjs"), + proof("performance/candidate-bounds.mjs", ["--expose-gc"]), +]); + +function proof(file, nodeArguments = []) { + return Object.freeze({ file, nodeArguments: Object.freeze([...nodeArguments]) }); +} diff --git a/packages/core/tests/evidence/performance/candidate-bounds.mjs b/packages/core/tests/evidence/performance/candidate-bounds.mjs new file mode 100644 index 0000000..20efef9 --- /dev/null +++ b/packages/core/tests/evidence/performance/candidate-bounds.mjs @@ -0,0 +1,108 @@ +/** + * Performance evidence for bounded candidate attempts and retained evidence. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { performance } from "node:perf_hooks"; +import { firstAcceptable } from "../../../dist/candidates/index.js"; +import { createSuite } from "../harness.mjs"; + +const QUALITY_CANDIDATES = 1_000; +const RETRY_CANDIDATES = 25; +const RETRIES_PER_CANDIDATE = 10; +const EVIDENCE_LIMIT = 32; +const MAX_QUALITY_CHAIN_MS = 5_000; +const MAX_RETRY_CHAIN_MS = 8_000; +const MAX_RETAINED_HEAP_BYTES = 32 * 1024 * 1024; + +if (typeof globalThis.gc !== "function") { + throw new Error("candidate performance evidence requires node --expose-gc"); +} + +const suite = createSuite("performance"); + +await suite.proof( + "PERF-003", + "candidate execution and retained evidence remain bounded", + "large quality and retry chains stay inside time and retained-heap budgets while evidence retains only its configured window", + async () => { + const quality = await measure(async () => firstAcceptable( + Array.from({ length: QUALITY_CANDIDATES }, (_, index) => index), + { + execute: async (candidate) => candidate, + accept: async () => ({ accepted: false, reasonCode: "quality_rejected" }), + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "unexpected_failure" }), + maxCandidates: QUALITY_CANDIDATES, + evidence: { maxAttempts: EVIDENCE_LIMIT }, + }, + )); + + let retryAttempts = 0; + const retry = await measure(async () => firstAcceptable( + Array.from({ length: RETRY_CANDIDATES }, (_, index) => index), + { + execute: async () => { + retryAttempts++; + throw new Error("retryable"); + }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ disposition: "retry_same_candidate", reasonCode: "retryable" }), + retry: { times: RETRIES_PER_CANDIDATE, initialDelay: 0, jitter: false }, + maxCandidates: RETRY_CANDIDATES, + evidence: { maxAttempts: EVIDENCE_LIMIT }, + }, + )); + + const expectedRetryAttempts = RETRY_CANDIDATES * RETRIES_PER_CANDIDATE; + const qualityReport = summarize(quality, QUALITY_CANDIDATES); + const retryReport = summarize(retry, expectedRetryAttempts); + return { + ok: quality.value.status === "exhausted" + && qualityReport.evidenceEntries === EVIDENCE_LIMIT + && qualityReport.droppedEvidence === QUALITY_CANDIDATES - EVIDENCE_LIMIT + && qualityReport.elapsedMs <= MAX_QUALITY_CHAIN_MS + && qualityReport.retainedHeapBytes <= MAX_RETAINED_HEAP_BYTES + && retry.value.status === "exhausted" + && retryAttempts === expectedRetryAttempts + && retryReport.evidenceEntries === EVIDENCE_LIMIT + && retryReport.droppedEvidence === expectedRetryAttempts - EVIDENCE_LIMIT + && retryReport.elapsedMs <= MAX_RETRY_CHAIN_MS + && retryReport.retainedHeapBytes <= MAX_RETAINED_HEAP_BYTES, + quality: qualityReport, + retry: retryReport, + limits: { + evidenceEntries: EVIDENCE_LIMIT, + retainedHeapBytes: MAX_RETAINED_HEAP_BYTES, + qualityChainMs: MAX_QUALITY_CHAIN_MS, + retryChainMs: MAX_RETRY_CHAIN_MS, + }, + }; + }, +); + +const summary = suite.summary(); +process.stdout.write(JSON.stringify(summary, null, 2) + "\n"); +process.exit(summary.failed > 0 ? 1 : 0); + +async function measure(operation) { + globalThis.gc(); + const heapBefore = process.memoryUsage().heapUsed; + const startedAt = performance.now(); + const value = await operation(); + const elapsedMs = Math.round(performance.now() - startedAt); + globalThis.gc(); + const retainedHeapBytes = Math.max(0, process.memoryUsage().heapUsed - heapBefore); + return { value, elapsedMs, retainedHeapBytes }; +} + +function summarize(measurement, attempts) { + return { + attempts, + elapsedMs: measurement.elapsedMs, + retainedHeapBytes: measurement.retainedHeapBytes, + evidenceEntries: measurement.value.evidence.length, + droppedEvidence: measurement.value.droppedEvidence, + }; +} diff --git a/packages/core/tests/evidence/release/candidate-package-contract.mjs b/packages/core/tests/evidence/release/candidate-package-contract.mjs new file mode 100644 index 0000000..5bf625a --- /dev/null +++ b/packages/core/tests/evidence/release/candidate-package-contract.mjs @@ -0,0 +1,46 @@ +/** + * Release evidence for the built candidates subpath contract. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { readFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { createSuite } from "../harness.mjs"; + +const require = createRequire(import.meta.url); +const suite = createSuite("release"); + +await suite.proof( + "REL-008", + "candidates subpath has locked ESM CommonJS and type artifacts", + "the package export resolves dedicated ESM, CommonJS, and declaration artifacts without adding firstAcceptable to the root API", + async () => { + const packageJson = JSON.parse(await readFile(new URL("../../../package.json", import.meta.url), "utf8")); + const esm = await import("../../../dist/candidates/index.js"); + const cjs = require("../../../dist-cjs/candidates/index.cjs"); + const root = await import("../../../dist/index.js"); + const declaration = await readFile(new URL("../../../dist/candidates/index.d.ts", import.meta.url), "utf8"); + const candidateExport = packageJson.exports?.["./candidates"]; + return { + ok: candidateExport?.types === "./dist/candidates/index.d.ts" + && candidateExport?.node?.import === "./dist/candidates/index.js" + && candidateExport?.node?.require === "./dist-cjs/candidates/index.cjs" + && candidateExport?.default === "./dist/runtime/unsupported.js" + && typeof esm.firstAcceptable === "function" + && typeof cjs.firstAcceptable === "function" + && !("firstAcceptable" in root) + && declaration.includes("firstAcceptable"), + export: candidateExport, + esm: typeof esm.firstAcceptable, + commonjs: typeof cjs.firstAcceptable, + rootHasFirstAcceptable: "firstAcceptable" in root, + declarationsContainFirstAcceptable: declaration.includes("firstAcceptable"), + }; + }, +); + +const summary = suite.summary(); +process.stdout.write(JSON.stringify(summary, null, 2) + "\n"); +process.exit(summary.failed > 0 ? 1 : 0); diff --git a/packages/core/tests/evidence/release/core-release-contracts.mjs b/packages/core/tests/evidence/release/core-release-contracts.mjs new file mode 100644 index 0000000..0431cb1 --- /dev/null +++ b/packages/core/tests/evidence/release/core-release-contracts.mjs @@ -0,0 +1,58 @@ +/** + * Release evidence for zero runtime dependencies and enforced size budgets. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { execFile } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { createSuite } from "../harness.mjs"; + +const execFileAsync = promisify(execFile); +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); +const suite = createSuite("release"); + +await suite.proof( + "REL-009", + "published core has zero runtime dependencies", + "package dependencies are absent and the generated CycloneDX SBOM contains no runtime components", + async () => { + const packageJson = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8")); + const sbom = JSON.parse(await readFile(resolve(packageRoot, "dist/workit-core.sbom.cdx.json"), "utf8")); + const runtimeDependencies = Object.keys(packageJson.dependencies ?? {}); + return { + ok: runtimeDependencies.length === 0 && Array.isArray(sbom.components) && sbom.components.length === 0, + runtimeDependencies, + sbomComponents: sbom.components?.length, + }; + }, +); + +await suite.proof( + "REL-010", + "compiled package entrypoints stay within enforced size budgets", + "the release size gate passes against compiled root, group, candidates ESM, and candidates CommonJS artifacts", + async () => { + const { stdout } = await execFileAsync(process.execPath, ["scripts/check-bundle-size.mjs"], { + cwd: packageRoot, + timeout: 120_000, + maxBuffer: 1024 * 1024, + }); + const measurements = stdout.trim().split(/\r?\n/u).filter(Boolean); + return { + ok: measurements.some((line) => line.startsWith("public-api:")) + && measurements.some((line) => line.startsWith("core-group-import:")) + && measurements.some((line) => line.startsWith("candidates-subpath:")) + && measurements.some((line) => line.startsWith("candidates-commonjs:")), + measurements, + }; + }, +); + +const summary = suite.summary(); +process.stdout.write(JSON.stringify(summary, null, 2) + "\n"); +process.exit(summary.failed > 0 ? 1 : 0); diff --git a/packages/core/tests/evidence/run-all.mjs b/packages/core/tests/evidence/run-all.mjs index 6489819..4ea5cf3 100644 --- a/packages/core/tests/evidence/run-all.mjs +++ b/packages/core/tests/evidence/run-all.mjs @@ -6,46 +6,36 @@ */ import { spawn } from "node:child_process"; +import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { computeEvidenceSourceDigest } from "../../scripts/evidence-source-digest.mjs"; +import { evidenceProofs } from "./manifest.mjs"; const here = path.dirname(fileURLToPath(import.meta.url)); -const files = [ - "lifecycle/owned-work.mjs", - "lifecycle/activity-restart.mjs", - "lifecycle/resource-audit.mjs", - "lifecycle/resource-ownership.mjs", - "lifecycle/replay-receipts.mjs", - "correctness/agent-authority.mjs", - "correctness/analysis-verifiers.mjs", - "correctness/activity-boundary.mjs", - "correctness/fault-injection.mjs", - "correctness/resource-ownership-model.mjs", - "correctness/runtime-contracts.mjs", - "correctness/source-protocol-analysis.mjs", - "correctness/time-policy-planner.mjs", - "correctness/formal-time-policy-model.mjs", - "correctness/nested-time-policy-composition.mjs", - "correctness/typed-cancellation-contracts.mjs", - "security/worker-boundary.mjs", - "release/release-integrity.mjs", - "release/receipt-ledger.mjs", - "release/sql-receipt-ledger.mjs", - "release/sql-ledger-integration.mjs", - "performance/benchmark-contracts.mjs", -]; +const packageRoot = path.resolve(here, "../.."); +const options = parseArguments(process.argv.slice(2)); const summary = { author: "Admilson B. F. Cossa", spdxLicense: "Apache-2.0", artifact: "workit-publication-evidence", + schemaVersion: 2, + releaseTarget: options.releaseTarget, + sourceDigest: await computeEvidenceSourceDigest(packageRoot), + environment: { + node: process.version, + platform: process.platform, + arch: process.arch, + }, proofs: [], }; -for (const file of files) { +for (const proof of evidenceProofs) { + const { file, nodeArguments } = proof; const startedAt = Date.now(); const childResult = await new Promise((resolve, reject) => { - const child = spawn(process.execPath, [path.join(here, file)], { + const child = spawn(process.execPath, [...nodeArguments, path.join(here, file)], { stdio: ["ignore", "pipe", "pipe"], }); let stdout = ""; @@ -81,6 +71,45 @@ for (const file of files) { const failures = summary.proofs.filter((proof) => proof.exitCode !== 0).length; summary.passed = summary.proofs.length - failures; summary.failed = failures; +summary.claimResults = summary.proofs.flatMap((proof) => + (proof.report?.results ?? []).map((result) => ({ + id: result.id, + proof: `tests/evidence/${proof.file}`, + status: result.status, + actualResult: result.evidence, + })) +); -process.stdout.write(JSON.stringify(summary, null, 2) + "\n"); +const serialized = JSON.stringify(summary, null, 2) + "\n"; +if (options.output !== undefined) { + const output = resolveOutput(options.output); + await mkdir(path.dirname(output), { recursive: true }); + await writeFile(output, serialized, "utf8"); +} +process.stdout.write(serialized); process.exit(failures > 0 ? 1 : 0); + +function parseArguments(args) { + const values = new Map(); + for (let index = 0; index < args.length; index += 2) { + const name = args[index]; + const value = args[index + 1]; + if ((name !== "--output" && name !== "--release-target") || value === undefined) { + throw new Error("Usage: run-all.mjs [--output ] [--release-target ]"); + } + values.set(name, value); + } + return { + output: values.get("--output"), + releaseTarget: values.get("--release-target") ?? null, + }; +} + +function resolveOutput(value) { + const output = path.resolve(packageRoot, value); + const relative = path.relative(packageRoot, output); + if (relative.startsWith("..") || path.isAbsolute(relative)) { + throw new Error("Evidence output must remain inside the package root"); + } + return output; +} diff --git a/packages/core/tests/evidence/security/candidate-boundary.mjs b/packages/core/tests/evidence/security/candidate-boundary.mjs new file mode 100644 index 0000000..657d365 --- /dev/null +++ b/packages/core/tests/evidence/security/candidate-boundary.mjs @@ -0,0 +1,44 @@ +/** + * Security evidence for candidate metadata normalization. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { firstAcceptable } from "../../../dist/candidates/index.js"; +import { createSuite } from "../harness.mjs"; + +const suite = createSuite("security"); + +await suite.proof( + "SEC-003", + "candidate evidence keeps prototype-shaped metadata inert and redacted", + "metadata normalization preserves an own __proto__ field without changing Object.prototype and redacts secret fields", + async () => { + const metadata = JSON.parse('{"__proto__":{"polluted":true},"token":"secret","provider":"primary"}'); + const result = await firstAcceptable(["candidate"], { + execute: async (candidate) => candidate, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "unexpected_failure" }), + candidateMetadata: () => metadata, + evidence: { maxAttempts: 1, maxMetadataBytes: 256 }, + }); + const captured = result.evidence[0]?.metadata; + return { + ok: result.status === "accepted" + && Object.hasOwn(captured ?? {}, "__proto__") + && captured?.__proto__?.polluted === true + && captured?.token === "[redacted]" + && Object.prototype.polluted === undefined + && ({}).polluted === undefined, + status: result.status, + hasOwnProto: Object.hasOwn(captured ?? {}, "__proto__"), + token: captured?.token, + objectPrototypePolluted: Object.prototype.polluted ?? null, + }; + }, +); + +const summary = suite.summary(); +process.stdout.write(JSON.stringify(summary, null, 2) + "\n"); +process.exit(summary.failed > 0 ? 1 : 0); diff --git a/packages/core/tests/property/candidates.property.test.js b/packages/core/tests/property/candidates.property.test.js new file mode 100644 index 0000000..05462ae --- /dev/null +++ b/packages/core/tests/property/candidates.property.test.js @@ -0,0 +1,110 @@ +/** + * Property tests for candidate admission and evidence invariants. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { test } from "vitest"; +import assert from "node:assert/strict"; +import fc from "fast-check"; +import { firstAcceptable } from "../../dist/candidates/index.js"; + +const PROPERTY_RUNS = 35; + +test("property: candidate attempts never exceed maxCandidates times retry attempts", async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + candidateCount: fc.integer({ min: 0, max: 8 }), + retryTimes: fc.integer({ min: 1, max: 4 }), + }), + async ({ candidateCount, retryTimes }) => { + const candidates = Array.from({ length: candidateCount }, (_, index) => index); + let attempts = 0; + const result = await firstAcceptable(candidates, { + execute: async () => { + attempts++; + throw new Error("retryable"); + }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ + disposition: "retry_same_candidate", + reasonCode: "retryable", + }), + retry: { times: retryTimes, initialDelay: 0, jitter: false }, + maxCandidates: Math.max(1, candidateCount), + evidence: { maxAttempts: Math.max(1, candidateCount * retryTimes) }, + }); + + assert.equal(result.status, "exhausted"); + assert.equal(attempts, candidateCount * retryTimes); + assert.ok(attempts <= Math.max(1, candidateCount) * retryTimes); + assert.equal(result.evidence.length, attempts); + }, + ), + { numRuns: PROPERTY_RUNS, seed: 0xCA7D01 }, + ); +}); + +test("property: accepted results come from the first admitted acceptable candidate", async () => { + await fc.assert( + fc.asyncProperty( + fc.integer({ min: 1, max: 12 }).chain((candidateCount) => fc.record({ + candidateCount: fc.constant(candidateCount), + acceptedIndex: fc.integer({ min: 0, max: candidateCount - 1 }), + })), + async ({ candidateCount, acceptedIndex }) => { + const candidates = Array.from({ length: candidateCount }, (_, index) => `candidate-${index}`); + const admitted = []; + const result = await firstAcceptable(candidates, { + execute: async (candidate) => { + admitted.push(candidate); + return candidate; + }, + accept: async (_value, _candidate, ctx) => ctx.name === "candidate" + && admitted.length - 1 === acceptedIndex + ? { accepted: true } + : { accepted: false, reasonCode: "quality_rejected" }, + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "unexpected_failure" }), + maxCandidates: candidateCount, + }); + + assert.equal(result.status, "accepted"); + assert.equal(result.candidateIndex, acceptedIndex); + assert.equal(result.candidate, candidates[acceptedIndex]); + assert.deepEqual(admitted, candidates.slice(0, acceptedIndex + 1)); + assert.deepEqual( + result.evidence.map((attempt) => attempt.candidateIndex), + Array.from({ length: acceptedIndex + 1 }, (_, index) => index), + ); + }, + ), + { numRuns: PROPERTY_RUNS, seed: 0xCA7D02 }, + ); +}); + +test("property: stopping dispositions never admit a later candidate", async () => { + await fc.assert( + fc.asyncProperty( + fc.constantFrom("terminal", "requires_user_input"), + async (disposition) => { + const admitted = []; + const result = await firstAcceptable([0, 1, 2], { + execute: async (candidate) => { + admitted.push(candidate); + throw new Error("stop"); + }, + accept: async () => ({ accepted: true }), + classifyFailure: async () => ({ disposition, reasonCode: "selection_stopped" }), + }); + + assert.equal(result.status, disposition); + assert.deepEqual(admitted, [0]); + assert.deepEqual(result.evidence.map((attempt) => attempt.candidateIndex), [0]); + }, + ), + { numRuns: PROPERTY_RUNS, seed: 0xCA7D03 }, + ); +}); + diff --git a/packages/core/tests/unit/candidates-adversarial.test.js b/packages/core/tests/unit/candidates-adversarial.test.js new file mode 100644 index 0000000..2163d7e --- /dev/null +++ b/packages/core/tests/unit/candidates-adversarial.test.js @@ -0,0 +1,220 @@ +/** + * Adversarial candidate policy tests - exercises mutable and hostile inputs. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { test } from "vitest"; +import assert from "node:assert/strict"; +import { + classifyWorkItFailure, + firstAcceptable, +} from "../../dist/candidates/index.js"; +import { CancellationError } from "../../dist/index.js"; + +const ACCEPT = Object.freeze({ accepted: true }); + +function base(overrides = {}) { + return { + execute: async (candidate) => candidate, + accept: async () => ACCEPT, + classifyFailure: async () => ({ disposition: "try_next_candidate", reasonCode: "next_candidate" }), + ...overrides, + }; +} + +test("candidate admission snapshots the input array before callbacks can expand it", async () => { + const candidates = ["original"]; + const admitted = []; + const result = await firstAcceptable(candidates, base({ + execute: async (candidate) => { + admitted.push(candidate); + candidates.push(...Array.from({ length: 100 }, (_, index) => `injected-${index}`)); + return candidate; + }, + accept: async () => ({ accepted: false, reasonCode: "quality_rejected" }), + })); + + assert.equal(result.status, "exhausted"); + assert.deepEqual(admitted, ["original"]); +}); + +test("failure decisions are normalized once and cannot change after validation", async () => { + let dispositionReads = 0; + let reasonReads = 0; + let attempts = 0; + const mutableDecision = { + get disposition() { + dispositionReads++; + return dispositionReads === 1 ? "terminal" : "retry_same_candidate"; + }, + get reasonCode() { + reasonReads++; + return reasonReads === 1 ? "stable_reason" : "changed_reason"; + }, + }; + + const result = await firstAcceptable(["candidate"], base({ + execute: async () => { + attempts++; + throw new Error("failure"); + }, + classifyFailure: async () => mutableDecision, + retry: { times: 3, initialDelay: 0 }, + })); + + assert.equal(result.status, "terminal"); + assert.equal(result.reasonCode, "stable_reason"); + assert.equal(attempts, 1); + assert.equal(dispositionReads, 1); + assert.equal(reasonReads, 1); +}); + +test("acceptance decisions are normalized once and resist getter time-of-check changes", async () => { + let acceptedReads = 0; + const decision = { + get accepted() { + acceptedReads++; + return acceptedReads === 1; + }, + }; + + const result = await firstAcceptable(["candidate"], base({ + accept: async () => decision, + })); + + assert.equal(result.status, "accepted"); + assert.equal(acceptedReads, 1); +}); + +test("built-in classifications are immutable shared constants", () => { + const cancellation = new CancellationError({ kind: "manual", tag: "stop" }); + const first = classifyWorkItFailure(cancellation); + + assert.equal(Object.isFrozen(first), true); + assert.throws(() => { + first.disposition = "try_next_candidate"; + }, TypeError); + assert.deepEqual(classifyWorkItFailure(cancellation), { + disposition: "cancelled", + reasonCode: "workit_cancelled", + }); +}); + +test("candidate execution snapshots callback and retry policy references", async () => { + const retry = { times: 1, initialDelay: 0 }; + const opts = base({ + execute: async () => { + opts.classifyFailure = async () => ({ disposition: "terminal", reasonCode: "mutated_classifier" }); + retry.times = 100; + throw new Error("failure"); + }, + classifyFailure: async () => ({ disposition: "try_next_candidate", reasonCode: "snapshotted_classifier" }), + retry, + }); + + const result = await firstAcceptable(["candidate"], opts); + + assert.equal(result.status, "exhausted"); + assert.equal(result.evidence.length, 1); + assert.equal(result.evidence[0].reasonCode, "snapshotted_classifier"); +}); + +test("aggregate attempt admission rejects unsafe products and invalid empty-chain retry policy", async () => { + await assert.rejects( + firstAcceptable(Array.from({ length: 11 }, (_, index) => index), base({ + retry: { times: 1_000 }, + maxCandidates: 11, + })), + /total candidate attempts/, + ); + await assert.rejects( + firstAcceptable([], base({ retry: { times: 0 } })), + /retry attempts/, + ); + await assert.rejects( + firstAcceptable([], base({ retry: { times: 1, retryIf: () => true } })), + /retryIf/, + ); + await assert.rejects( + firstAcceptable([], base({ candidateMetadata: "invalid" })), + /candidateMetadata/, + ); + await assert.rejects( + firstAcceptable(["candidate"], base({ candidateMetadata: () => [] })), + /candidateMetadata must return an object/, + ); + + const validFullRetryPolicy = await firstAcceptable(["candidate"], base({ + retry: { + times: 1, + backoff: "fixed", + initialDelay: 0, + maxDelay: 0, + jitter: false, + }, + })); + assert.equal(validFullRetryPolicy.status, "accepted"); +}); + +test("hostile thrown proxies and Error accessors cannot bypass caller classification", async () => { + const hostileProxy = new Proxy({}, { + get() { + throw new Error("hostile get"); + }, + getPrototypeOf() { + throw new Error("hostile prototype"); + }, + }); + const hostileError = new Error("hidden"); + Object.defineProperties(hostileError, { + name: { get: () => { throw new Error("hostile name"); } }, + message: { get: () => { throw new Error("hostile message"); } }, + }); + let classifications = 0; + + for (const failure of [hostileProxy, hostileError]) { + const result = await firstAcceptable(["candidate"], base({ + execute: async () => { throw failure; }, + classifyFailure: async () => { + classifications++; + return { disposition: "terminal", reasonCode: "hostile_failure" }; + }, + })); + assert.equal(result.status, "terminal"); + assert.equal(result.error, failure); + } + assert.equal(classifications, 2); +}); + +test("cancellation raised inside quality or classification callbacks remains authoritative", async () => { + const qualityCancellation = new CancellationError({ kind: "manual", tag: "quality-cancel" }); + const classifierCancellation = new CancellationError({ kind: "manual", tag: "classifier-cancel" }); + + await assert.rejects( + firstAcceptable(["candidate"], base({ + accept: async () => { throw qualityCancellation; }, + })), + (error) => error === qualityCancellation, + ); + await assert.rejects( + firstAcceptable(["candidate"], base({ + execute: async () => { throw new Error("provider"); }, + classifyFailure: async () => { throw classifierCancellation; }, + })), + (error) => error === classifierCancellation, + ); +}); + +test("metadata named __proto__ stays inert after normalization", async () => { + const metadata = JSON.parse('{"__proto__":{"polluted":true},"constructor":{"prototype":{"polluted":true}}}'); + const result = await firstAcceptable(["candidate"], base({ + candidateMetadata: () => metadata, + })); + + assert.equal(result.status, "accepted"); + assert.equal({}.polluted, undefined); + assert.equal(Object.prototype.polluted, undefined); + assert.equal(result.evidence[0].metadata.__proto__.polluted, true); +}); diff --git a/packages/core/tests/unit/candidates.test.js b/packages/core/tests/unit/candidates.test.js new file mode 100644 index 0000000..25a9e17 --- /dev/null +++ b/packages/core/tests/unit/candidates.test.js @@ -0,0 +1,405 @@ +/** + * Candidate policy tests - verifies deterministic admission, classification, + * quality decisions, and bounded evidence. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { test } from "vitest"; +import assert from "node:assert/strict"; +import { + firstAcceptable, + classifyWorkItFailure, +} from "../../dist/candidates/index.js"; +import { + BudgetExceededError, + CancellationError, + TimeoutError, + createBudget, + group, + run, +} from "../../dist/index.js"; + +const ACCEPT = Object.freeze({ accepted: true }); +const RETRY = Object.freeze({ disposition: "retry_same_candidate", reasonCode: "temporary_failure" }); +const NEXT = Object.freeze({ disposition: "try_next_candidate", reasonCode: "candidate_unavailable" }); + +function options(overrides = {}) { + return { + execute: async (candidate) => candidate, + accept: async () => ACCEPT, + classifyFailure: async () => NEXT, + ...overrides, + }; +} + +test("firstAcceptable returns the first semantically accepted candidate", async () => { + const result = await firstAcceptable(["primary", "secondary"], options({ + execute: async (candidate, ctx) => ({ candidate, deadlineAt: ctx.deadlineAt }), + accept: async (value) => value.candidate === "primary" ? ACCEPT : { accepted: false, reasonCode: "low_quality" }, + })); + + assert.equal(result.status, "accepted"); + assert.equal(result.candidate, "primary"); + assert.equal(result.candidateIndex, 0); + assert.equal(result.value.candidate, "primary"); + assert.deepEqual(result.evidence.map(({ candidateIndex, decision, outcome }) => ({ + candidateIndex, + decision, + outcome, + })), [{ candidateIndex: 0, decision: "accepted", outcome: "succeeded" }]); + assert.equal(result.droppedEvidence, 0); +}); + +test("firstAcceptable retries only retry_same_candidate failures", async () => { + const calls = []; + const result = await firstAcceptable(["primary", "secondary"], options({ + execute: async (candidate, ctx) => { + calls.push([candidate, ctx.attempt]); + if (candidate === "primary") throw new Error("temporary"); + return candidate; + }, + classifyFailure: async () => RETRY, + retry: { times: 2, initialDelay: 0, jitter: false }, + })); + + assert.equal(result.status, "accepted"); + assert.equal(result.candidate, "secondary"); + assert.deepEqual(calls, [["primary", 1], ["primary", 2], ["secondary", 1]]); + assert.deepEqual(result.evidence.map((attempt) => [attempt.candidateIndex, attempt.attempt, attempt.decision]), [ + [0, 1, "retry_same_candidate"], + [0, 2, "retry_same_candidate"], + [1, 1, "accepted"], + ]); +}); + +test("firstAcceptable advances immediately after try_next_candidate", async () => { + const calls = []; + const result = await firstAcceptable(["primary", "secondary"], options({ + execute: async (candidate) => { + calls.push(candidate); + if (candidate === "primary") throw new Error("quota"); + return "ok"; + }, + retry: { times: 5, initialDelay: 0 }, + })); + + assert.equal(result.status, "accepted"); + assert.deepEqual(calls, ["primary", "secondary"]); + assert.equal(result.evidence[0].reasonCode, "candidate_unavailable"); +}); + +test("quality rejection is distinct from transport failure and advances", async () => { + const result = await firstAcceptable([1, 2], options({ + accept: async (value) => value >= 2 ? ACCEPT : { accepted: false, reasonCode: "quality_too_low" }, + })); + + assert.equal(result.status, "accepted"); + assert.equal(result.value, 2); + assert.deepEqual(result.evidence.map(({ decision, outcome, reasonCode }) => ({ decision, outcome, reasonCode })), [ + { decision: "quality_rejected", outcome: "succeeded", reasonCode: "quality_too_low" }, + { decision: "accepted", outcome: "succeeded", reasonCode: undefined }, + ]); +}); + +test("firstAcceptable returns exhausted for empty and fully rejected chains", async () => { + const empty = await firstAcceptable([], options()); + const rejected = await firstAcceptable([1, 2], options({ + accept: async () => ({ accepted: false, reasonCode: "not_enough_evidence" }), + })); + + assert.deepEqual(empty, { status: "exhausted", evidence: [], droppedEvidence: 0 }); + assert.equal(rejected.status, "exhausted"); + assert.deepEqual(rejected.evidence.map((attempt) => attempt.reasonCode), [ + "not_enough_evidence", + "not_enough_evidence", + ]); +}); + +test("terminal and requires_user_input decisions stop later candidates", async () => { + const admitted = []; + const terminalError = new Error("invalid request"); + const terminal = await firstAcceptable(["first", "never"], options({ + execute: async (candidate) => { + admitted.push(candidate); + throw terminalError; + }, + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "invalid_request" }), + })); + + const userInput = await firstAcceptable(["first", "never"], options({ + execute: async () => { throw new Error("approval required"); }, + classifyFailure: async () => ({ disposition: "requires_user_input", reasonCode: "approval_required" }), + })); + + assert.equal(terminal.status, "terminal"); + assert.equal(terminal.reasonCode, "invalid_request"); + assert.equal(terminal.error, terminalError); + assert.deepEqual(admitted, ["first"]); + assert.equal(userInput.status, "requires_user_input"); + assert.equal(userInput.reasonCode, "approval_required"); +}); + +test("CancellationError always wins over caller classification", async () => { + let classifications = 0; + let secondStarted = false; + const cancellation = new CancellationError({ kind: "manual", tag: "stop" }); + + await assert.rejects( + firstAcceptable(["first", "second"], options({ + execute: async (candidate) => { + if (candidate === "second") secondStarted = true; + throw cancellation; + }, + classifyFailure: async () => { + classifications++; + return NEXT; + }, + })), + (error) => error === cancellation, + ); + + assert.equal(classifications, 0); + assert.equal(secondStarted, false); +}); + +test("cancellation during retry backoff admits no next candidate", async () => { + let scope; + let secondStarted = false; + + await assert.rejects( + group(async (task) => { + await task(async (ctx) => { + scope = ctx.scope; + setTimeout(() => scope.cancel("stop-backoff"), 5); + await firstAcceptable(["first", "second"], options({ + execute: async (candidate) => { + if (candidate === "second") secondStarted = true; + throw new Error("retry me"); + }, + classifyFailure: async () => RETRY, + retry: { times: 2, initialDelay: 1_000, jitter: false }, + })); + }); + }), + CancellationError, + ); + + assert.equal(secondStarted, false); +}); + +test("one absolute deadline is inherited without increasing across candidates", async () => { + const deadlineAt = Date.now() + 1_000; + const observed = []; + const result = await firstAcceptable(["first", "second"], options({ + deadlineAt, + execute: async (candidate, ctx) => { + observed.push(ctx.deadlineAt); + if (candidate === "first") throw new Error("next"); + return candidate; + }, + })); + + assert.equal(result.status, "accepted"); + assert.deepEqual(observed, [deadlineAt, deadlineAt]); +}); + +test("aggregate deadline timeout is terminal and does not call the provider classifier", async () => { + let classifications = 0; + const result = await firstAcceptable(["slow"], options({ + deadlineAt: new Date(Date.now() + 10), + execute: async (_candidate, ctx) => await new Promise((_resolve, reject) => { + ctx.signal.addEventListener("abort", () => reject(ctx.signal.reason), { once: true }); + }), + classifyFailure: async () => { + classifications++; + return NEXT; + }, + })); + + assert.equal(result.status, "terminal"); + assert.equal(result.reasonCode, "workit_timeout"); + assert.equal(classifications, 0); +}); + +test("shared retry budgets stop attempts before exceeding their invariant", async () => { + const RetryBudget = createBudget("CandidateRetryBudget", { unit: "retries" }); + let calls = 0; + + await assert.rejects( + run.context.with( + RetryBudget, + { limit: 1, spent: 0, unit: "retries" }, + async () => firstAcceptable(["first"], options({ + execute: async () => { + calls++; + throw new Error("retry"); + }, + classifyFailure: async () => RETRY, + retry: { times: 3, initialDelay: 0, retryBudget: RetryBudget }, + })), + ), + BudgetExceededError, + ); + + assert.equal(calls, 2); +}); + +test("classifyWorkItFailure safely classifies known runtime errors", () => { + assert.deepEqual( + classifyWorkItFailure(new CancellationError({ kind: "manual", tag: "stop" })), + { disposition: "cancelled", reasonCode: "workit_cancelled" }, + ); + assert.deepEqual( + classifyWorkItFailure(new TimeoutError(10)), + { disposition: "terminal", reasonCode: "workit_timeout" }, + ); + assert.deepEqual( + classifyWorkItFailure(new BudgetExceededError({ + budgetKey: "CostBudget", + limit: 1, + spent: 2, + attempted: 1, + })), + { disposition: "terminal", reasonCode: "workit_budget_exhausted" }, + ); + assert.equal(classifyWorkItFailure(new Error("provider")), undefined); +}); + +test("candidate evidence is bounded, redacted, cloned, and reports drops", async () => { + const metadata = { provider: "primary", token: "secret", nested: { password: "hidden" } }; + const result = await firstAcceptable(["first"], options({ + execute: async () => { throw new Error("temporary"); }, + classifyFailure: async () => RETRY, + retry: { times: 3, initialDelay: 0 }, + evidence: { maxAttempts: 1, maxMetadataBytes: 256 }, + candidateMetadata: () => metadata, + })); + + metadata.provider = "mutated"; + assert.equal(result.status, "exhausted"); + assert.equal(result.evidence.length, 1); + assert.equal(result.droppedEvidence, 2); + assert.deepEqual(result.evidence[0].metadata, { + provider: "primary", + token: "[redacted]", + nested: { password: "[redacted]" }, + candidateIndex: 0, + }); +}); + +test("candidate evidence bounds error text and normalizes non-Error failures", async () => { + const longMessage = "x".repeat(2_000); + const stringFailure = await firstAcceptable(["first"], options({ + execute: async () => { throw longMessage; }, + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "string_failure" }), + retry: 1, + })); + const objectFailure = await firstAcceptable(["first"], options({ + execute: async () => { throw { code: 503 }; }, + classifyFailure: async () => ({ disposition: "terminal", reasonCode: "object_failure" }), + })); + + assert.equal(stringFailure.status, "terminal"); + assert.equal(stringFailure.evidence[0].error.name, "Error"); + assert.equal(stringFailure.evidence[0].error.message.length, 1_024); + assert.equal(objectFailure.status, "terminal"); + assert.equal(objectFailure.evidence[0].error.message, "Candidate attempt failed"); +}); + +test("candidate contract rejects unsafe bounds, malformed decisions, and metadata", async () => { + await assert.rejects( + firstAcceptable([1, 2], options({ maxCandidates: 1 })), + /candidates length exceeds maxCandidates/, + ); + await assert.rejects( + firstAcceptable([1], options({ maxCandidates: 0 })), + /maxCandidates/, + ); + await assert.rejects( + firstAcceptable([1], options({ maxCandidates: 1.5 })), + /maxCandidates/, + ); + await assert.rejects( + firstAcceptable([1], options({ maxCandidates: 1_001 })), + /maxCandidates/, + ); + await assert.rejects( + firstAcceptable("not-an-array", options()), + /candidates must be an array/, + ); + await assert.rejects( + firstAcceptable([], null), + /options are required/, + ); + await assert.rejects( + firstAcceptable([], { execute: undefined, accept: undefined, classifyFailure: undefined }), + /must be functions/, + ); + await assert.rejects( + firstAcceptable([1], options({ deadlineAt: Number.NaN })), + /deadlineAt/, + ); + await assert.rejects( + firstAcceptable([1], options({ + accept: async () => ({ accepted: false, reasonCode: "NOT VALID" }), + })), + /reasonCode/, + ); + await assert.rejects( + firstAcceptable([1], options({ accept: async () => null })), + /AcceptanceDecision/, + ); + await assert.rejects( + firstAcceptable([1], options({ accept: async () => ({ accepted: "yes" }) })), + /AcceptanceDecision/, + ); + await assert.rejects( + firstAcceptable([1], options({ + execute: async () => { throw new Error("failure"); }, + classifyFailure: async () => ({ disposition: "unknown", reasonCode: "unknown" }), + })), + /disposition/, + ); + await assert.rejects( + firstAcceptable([1], options({ + execute: async () => { throw new Error("failure"); }, + classifyFailure: async () => null, + })), + /disposition/, + ); + await assert.rejects( + firstAcceptable([1], options({ + execute: async () => { throw new Error("failure"); }, + classifyFailure: async () => ({ disposition: "terminal", reasonCode: 1 }), + })), + /reasonCode/, + ); + await assert.rejects( + firstAcceptable([1], options({ + candidateMetadata: () => ({ payload: "x".repeat(1_000) }), + evidence: { maxMetadataBytes: 32 }, + })), + /maxMetadataBytes/, + ); +}); + +test("callback failures propagate as programmer errors instead of fallback", async () => { + const classifierError = new Error("classifier bug"); + const acceptanceError = new Error("acceptance bug"); + + await assert.rejects( + firstAcceptable([1], options({ + execute: async () => { throw new Error("provider"); }, + classifyFailure: async () => { throw classifierError; }, + })), + (error) => error === classifierError, + ); + await assert.rejects( + firstAcceptable([1], options({ + accept: async () => { throw acceptanceError; }, + })), + (error) => error === acceptanceError, + ); +}); From 26c891c5ab11ab5115a03f45fa12a03924e60741 Mon Sep 17 00:00:00 2001 From: "Admilson B. F. Cossa" Date: Mon, 17 Aug 2026 18:51:46 +0200 Subject: [PATCH 2/6] docs(candidates): publish the unreleased contract Document taxonomy, outcomes, deadlines, evidence boundaries, compatibility-tested runtimes, release status, and enforce the expanded public-proof matrix. --- CHANGELOG.md | 15 +++ README.md | 9 +- packages/core/README.md | 114 +++++++++++++++++-- packages/core/benchmarks/public-proof.json | 19 +++- packages/core/scripts/check-public-proof.mjs | 7 ++ 5 files changed, 153 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22f1400..d3c331c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ SPDX-License-Identifier: Apache-2.0 ## Unreleased +- Add the explicit `@workit/core/candidates` subpath with deterministic + `firstAcceptable()` selection, provider-neutral failure dispositions, typed + semantic quality decisions, existing retry/deadline/budget reuse, and bounded + redacted candidate-attempt evidence. +- Add conservative built-in taxonomy for WorkIt cancellation, timeout, and + retry-budget exhaustion while leaving unknown provider failures under the + caller's explicit classifier policy. +- Keep real cancellation authoritative, return terminal and user-input stops as + exhaustive typed results, and leave root runtime exports unchanged. +- Add ESM, CommonJS, strict TypeScript, unsupported-runtime, unit, property, and + installed-package coverage for candidate policy contracts. +- Add a manifest-driven evidence runner, per-claim actual-result capture in the + ignored coverage directory, and a source-digest ledger gate that rejects + missing, failing, unregistered, or stale evidence. + ## 0.5.0 Add runtime deadline introspection, shared retry admission limits, and bounded diff --git a/README.md b/README.md index adc5935..4db1991 100644 --- a/README.md +++ b/README.md @@ -87,13 +87,16 @@ owned by `packages/core`. The monorepo layout must not change how users install or import WorkIt. -Stable consumer paths for this release line: +Consumer paths in the current checkout are listed below. Paths described under +`Unreleased` in the changelog are not part of the npm `latest` version until the +corresponding minor release is published. ```txt @workit/core @workit/core/activity @workit/core/ai @workit/core/analysis +@workit/core/candidates @workit/core/channel @workit/core/contracts @workit/core/diagnostics @@ -118,6 +121,10 @@ npm run check:size npm run check:package-consumer ``` +The current checkout's candidate-policy subpath is documented in the +[`@workit/core` package README](packages/core/README.md#candidate-selection). +It remains unreleased while the package version is `0.5.0`. + Run the site gates from the repository root: ```sh diff --git a/packages/core/README.md b/packages/core/README.md index aea5038..1b28eb2 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -318,6 +318,7 @@ ownership helpers live behind explicit subpaths. | Build lifecycle receipts from scope events and snapshots | `@workit/core/replay` | audit evidence, not deterministic scheduler replay | | Persist receipts in caller-owned stores | `@workit/core/ledger` | memory, file, and caller-owned SQL receipt ledgers, not a database framework | | Verify receipts and caller-provided protocol specs | `@workit/core/analysis` | bounded verification over supplied evidence, not whole-program analysis | +| Select the first semantically acceptable candidate | `@workit/core/candidates` | sequential candidate policy, not implicit racing or durable idempotency | | Record explicit terminal activity boundaries | `@workit/core/activity` | completed activity replay, not in-flight workflow recovery | | Compose lazy, shared, and scope-owned resources | `@workit/core/resources` | cleanup ownership through WorkIt scopes, not automatic resource detection | | Plan declared retry, hedge, timeout, deadline, series, and parallel time bounds | `@workit/core/time-policy` | conservative planning over declared policies, not wall-clock execution proof | @@ -332,6 +333,93 @@ deduplication record. Use `@workit/core/activity` with a caller-owned store for terminal replay across process restarts, and `@workit/core/ledger` when lifecycle receipts must be persisted independently. +### Candidate Selection + +> **Unreleased:** this subpath exists in the current checkout and is planned for +> the next minor release. It is not included in npm `latest` while the package +> version remains `0.5.0`. + +`@workit/core/candidates` separates transport success, semantic quality, and +failure policy. Candidates run sequentially in caller order. WorkIt's built-in +taxonomy handles its own cancellation, timeout, and retry-budget errors first. +The caller classifier owns unknown failures such as provider responses and +decides whether to retry the same candidate, advance, stop terminally, or +require user input. Real WorkIt cancellation always throws and cannot be +converted into fallback. + +```ts +import { + classifyWorkItFailure, + firstAcceptable, +} from "@workit/core/candidates"; +import { TimeoutError } from "@workit/core"; + +const result = await firstAcceptable([primaryModel, fallbackModel], { + execute: (model, ctx) => model.generate({ signal: ctx.signal }), + accept: (value) => value.confidence >= 0.8 + ? { accepted: true } + : { accepted: false, reasonCode: "confidence_too_low" }, + classifyFailure: (error) => isTransient(error) + ? { disposition: "retry_same_candidate", reasonCode: "transient_provider_failure" } + : { disposition: "try_next_candidate", reasonCode: "candidate_unavailable" }, + retry: { times: 2, initialDelay: "100ms" }, + maxCandidates: 2, + evidence: { maxAttempts: 8 }, + candidateMetadata: (_model, candidateIndex) => ({ candidateIndex }), +}); + +if (result.status === "accepted") { + useResult(result.value); +} + +// The same conservative taxonomy is available for policy inspection. +const knownFailure = classifyWorkItFailure(new TimeoutError(1_000)); +``` + +`classifyWorkItFailure()` returns these stable built-in decisions: + +| Error | Decision | `reasonCode` | `firstAcceptable()` behavior | +|---|---|---|---| +| `CancellationError` | `cancelled` | `workit_cancelled` | rethrows the original cancellation; no fallback is admitted | +| `TimeoutError` | `terminal` | `workit_timeout` | returns a terminal result; no fallback is admitted | +| `BudgetExceededError` | `terminal` | `workit_budget_exhausted` | returns a terminal result; no fallback is admitted | +| unknown/provider error | `undefined` | caller-defined | invokes `classifyFailure` exactly once for the normalized decision | + +The result is an exhaustive discriminated union: + +| `status` | Meaning | Additional fields | +|---|---|---| +| `accepted` | first transport-successful value accepted by the quality predicate | `candidate`, `candidateIndex`, `value` | +| `exhausted` | no candidate produced an acceptable value and no policy stop occurred | none | +| `terminal` | a built-in or caller terminal decision stopped the chain | `reasonCode`, original `error` | +| `requires_user_input` | caller policy stopped for approval or missing input | `reasonCode` | + +Every result includes bounded `evidence` and `droppedEvidence`. Each evidence +entry records candidate index, attempt, outcome, decision, optional reason code, +timing, normalized error, and redacted metadata. + +`deadlineAt` is one absolute aggregate deadline for the entire candidate chain, +not a fresh timeout per candidate. Every admitted task context observes the same +effective deadline. When it expires, the result is terminal and later candidates +are not admitted. + +Quality rejection is recorded as a successful transport attempt with the +separate `quality_rejected` decision. Evidence uses the existing attempt +recorder, validates lowercase bounded reason codes, bounds and redacts metadata, +truncates error text, and reports dropped attempts. Defaults admit at most 16 +candidates and retain at most 256 evidence entries; larger candidate lists must +raise `maxCandidates` explicitly, up to the hard cap of 1,000. Configuration is +rejected when `candidateCount * retry.times` exceeds 10,000 admitted attempts. + +The helper does not race candidates and does not make side effects idempotent. +Use caller-owned idempotency for operations that may be repeated by retries or +fallback. `retryIf` is intentionally not accepted in the retry option because +`classifyFailure` is the single retry admission policy. + +Exceptions thrown by `accept`, `classifyFailure`, or `candidateMetadata` are +configuration/callback failures and are not silently converted into fallback. +WorkIt cancellation thrown by a callback remains authoritative. + ### Attempt Evidence For a scheduled task wrapped by `run.retry()`, the outer retry boundary emits @@ -671,9 +759,9 @@ thresholds, not exact milliseconds. | Evidence | Current result | |---|---:| -| Unit tests | 375 passing | -| Coverage gate | 100% statements, branches, functions, lines | -| Evidence proof files | 22 passing | +| Unit and property tests | 403 passing | +| Coverage gate | 100% statements (2,902/2,902), branches (1,837/1,837), functions (688/688), lines (2,797/2,797) | +| Evidence proof files | 30 passing / 56 executable claims captured | | Runtime dependencies | 0 | | Article benchmark suite | 19/19 passing | | Core group import | 13,807 B minified / 4,842 B gzip | @@ -688,7 +776,7 @@ Representative article-benchmark results: |---|---:|---:| | Provider race losers after winner | losers continue until their sleeps finish | losers cancelled in scope close | | Retry after cancellation | 7 extra attempts, 622 ms latency | 0 extra attempts, 1 ms latency | -| Context `.with()` over 5,000 keys | 31.68 ms | 0.014 ms | +| Context `.with()` over 5,000 keys | 31.253 ms | 0.011 ms | | 1B-row source, take 25 | 281 items pulled | 40 items pulled | | Sampling volume | 1,300 events | 36 events | @@ -699,10 +787,11 @@ npm run verify ``` `npm run verify` runs type-checking, header and test hygiene, unit tests, -security checks, vulnerability audit, SBOM validation, API and bundle-size -locks, runtime benchmarks, stream and soak gates, exporter stress, -package-consumer fixtures, public-proof validation, worker-contract checks, -release-policy checks, and `npm pack --dry-run`. +manifest-driven evidence proofs, source-digest ledger validation, security +checks, vulnerability audit, SBOM validation, API and bundle-size locks, +runtime benchmarks, stream and soak gates, exporter stress, package-consumer +fixtures, public-proof validation, worker-contract checks, release-policy +checks, and `npm pack --dry-run`. Run the article benchmark suite: @@ -758,6 +847,15 @@ Supported: - Next.js route-shaped handlers - Express, Fastify, tRPC, and Vercel AI SDK fixtures +Compatibility-tested, but outside the primary Node.js support commitment: + +- Bun `1.3.13` installed-package fixture +- Deno `2.2.7` installed-package fixture + +These fixtures prove the tested package surface on the pinned versions. They do +not broaden the documented primary runtime target beyond Node.js server +runtimes. + Unsupported today: - browser client runtime diff --git a/packages/core/benchmarks/public-proof.json b/packages/core/benchmarks/public-proof.json index c15dffb..a16a120 100644 --- a/packages/core/benchmarks/public-proof.json +++ b/packages/core/benchmarks/public-proof.json @@ -9,7 +9,12 @@ "npm run check:stream-memory", "npm run check:soak", "npm run check:package-consumer", - "npm run check:claims" + "npm run check:claims", + "npm run test:evidence", + "npm run check:evidence-ledger", + "npm run check:candidates-api", + "npm run check:candidates-performance", + "npm run check:candidate-scenarios" ], "benchmarkFixtures": [ { @@ -55,6 +60,16 @@ "status": "supported", "evidence": "npm run check:package-consumer" }, + { + "runtime": "Bun", + "status": "pinned compatibility fixture; not primary support contract", + "evidence": "npm run check:package-consumer" + }, + { + "runtime": "Deno", + "status": "pinned compatibility fixture; not primary support contract", + "evidence": "npm run check:package-consumer" + }, { "runtime": "Express", "status": "supported fixture", @@ -73,7 +88,7 @@ { "runtime": "Next.js route", "status": "supported fixture", - "evidence": "npm run check:claims" + "evidence": "npm run check:package-consumer" }, { "runtime": "Vercel AI SDK handler", diff --git a/packages/core/scripts/check-public-proof.mjs b/packages/core/scripts/check-public-proof.mjs index b5247bd..9ced9a5 100644 --- a/packages/core/scripts/check-public-proof.mjs +++ b/packages/core/scripts/check-public-proof.mjs @@ -23,12 +23,19 @@ const REQUIRED_COMMANDS = [ "npm run check:soak", "npm run check:package-consumer", "npm run check:claims", + "npm run test:evidence", + "npm run check:evidence-ledger", + "npm run check:candidates-api", + "npm run check:candidates-performance", + "npm run check:candidate-scenarios", ]; const REQUIRED_MIGRATIONS = ["p-limit", "p-map", "RxJS", "Bottleneck"]; const REQUIRED_RUNTIMES = [ "Node.js ESM", "Node.js CommonJS", + "Bun", + "Deno", "Express", "Fastify", "tRPC", From baac2182a905b7f2f180755f7f54fb7d17956665 Mon Sep 17 00:00:00 2001 From: "Admilson B. F. Cossa" Date: Mon, 17 Aug 2026 18:54:56 +0200 Subject: [PATCH 3/6] release: prepare @workit/core 0.6.0 --- CHANGELOG.md | 2 ++ README.md | 12 ++++++------ package-lock.json | 6 +++--- package.json | 2 +- packages/core/README.md | 8 ++++---- packages/core/package.json | 2 +- 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3c331c..86e596d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ SPDX-License-Identifier: Apache-2.0 ## Unreleased +## 0.6.0 + - Add the explicit `@workit/core/candidates` subpath with deterministic `firstAcceptable()` selection, provider-neutral failure dispositions, typed semantic quality decisions, existing retry/deadline/budget reuse, and bounded diff --git a/README.md b/README.md index 4db1991..8709c4e 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ cite the software release you used: title = {WorkIt: A TypeScript Structured Concurrency Runtime for Node.js Server Runtimes}, year = {2026}, url = {https://github.com/WorkRuntime/workit}, - version = {0.5.0}, + version = {0.6.0}, license = {Apache-2.0} } ``` @@ -87,9 +87,9 @@ owned by `packages/core`. The monorepo layout must not change how users install or import WorkIt. -Consumer paths in the current checkout are listed below. Paths described under -`Unreleased` in the changelog are not part of the npm `latest` version until the -corresponding minor release is published. +Consumer paths in the `0.6.0` release candidate are listed below. The new +candidate-policy subpath is not part of npm `latest` until the signed `v0.6.0` +tag and provenance-backed publish complete. ```txt @workit/core @@ -121,9 +121,9 @@ npm run check:size npm run check:package-consumer ``` -The current checkout's candidate-policy subpath is documented in the +The `0.6.0` release candidate's candidate-policy subpath is documented in the [`@workit/core` package README](packages/core/README.md#candidate-selection). -It remains unreleased while the package version is `0.5.0`. +It remains unavailable from npm `latest` until the release is published. Run the site gates from the repository root: diff --git a/package-lock.json b/package-lock.json index 5646cd2..8d6b841 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "workit", - "version": "0.5.0", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "workit", - "version": "0.5.0", + "version": "0.6.0", "license": "Apache-2.0", "workspaces": [ "packages/core" @@ -3018,7 +3018,7 @@ }, "packages/core": { "name": "@workit/core", - "version": "0.5.0", + "version": "0.6.0", "license": "Apache-2.0", "devDependencies": { "@opentelemetry/api": "1.9.1", diff --git a/package.json b/package.json index e223289..4d46fc3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "workit", - "version": "0.5.0", + "version": "0.6.0", "private": true, "description": "WorkIt monorepo.", "type": "module", diff --git a/packages/core/README.md b/packages/core/README.md index 1b28eb2..65d0b4d 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -335,9 +335,9 @@ receipts must be persisted independently. ### Candidate Selection -> **Unreleased:** this subpath exists in the current checkout and is planned for -> the next minor release. It is not included in npm `latest` while the package -> version remains `0.5.0`. +> **0.6.0 release candidate:** this subpath is present in the prepared package, +> but it is not included in npm `latest` until the signed `v0.6.0` tag and +> provenance-backed publish complete. `@workit/core/candidates` separates transport success, semantic quality, and failure policy. Candidates run sequentially in caller order. WorkIt's built-in @@ -950,7 +950,7 @@ cite the software release you used: title = {WorkIt: A TypeScript Structured Concurrency Runtime for Node.js Server Runtimes}, year = {2026}, url = {https://github.com/WorkRuntime/workit}, - version = {0.5.0}, + version = {0.6.0}, license = {Apache-2.0} } ``` diff --git a/packages/core/package.json b/packages/core/package.json index f8fd923..35b4d73 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@workit/core", - "version": "0.5.0", + "version": "0.6.0", "description": "Structured concurrency runtime for TypeScript: owned async work, cancellation, budgets, retries, timeouts, worker offload, scopes.", "keywords": [ "structured-concurrency", From bde83eeab847d5175fb5248cdc5f3cff5e02194d Mon Sep 17 00:00:00 2001 From: "Admilson B. F. Cossa" Date: Mon, 17 Aug 2026 19:36:09 +0200 Subject: [PATCH 4/6] fix(evidence): block release on deferred Oryn canary --- .github/workflows/release-provenance.yml | 3 + package.json | 1 + packages/core/README.md | 4 ++ packages/core/SECURITY.md | 1 + packages/core/evidence/claims.json | 11 ++++ packages/core/package.json | 3 +- .../core/scripts/check-evidence-ledger.mjs | 18 ++--- .../core/scripts/check-release-provenance.mjs | 7 +- .../core/scripts/check-release-readiness.mjs | 25 +++++++ .../core/scripts/evidence-ledger-status.mjs | 47 +++++++++++++ .../tests/unit/evidence-ledger-status.test.js | 66 +++++++++++++++++++ 11 files changed, 176 insertions(+), 10 deletions(-) create mode 100644 packages/core/scripts/check-release-readiness.mjs create mode 100644 packages/core/scripts/evidence-ledger-status.mjs create mode 100644 packages/core/tests/unit/evidence-ledger-status.test.js diff --git a/.github/workflows/release-provenance.yml b/.github/workflows/release-provenance.yml index 1b20797..e8eaad6 100644 --- a/.github/workflows/release-provenance.yml +++ b/.github/workflows/release-provenance.yml @@ -60,6 +60,9 @@ jobs: - name: Verify coverage run: npm run test:coverage + - name: Verify release evidence readiness + run: npm run check:release-readiness + - name: Verify release tag if: inputs.dry_run == 'false' shell: bash diff --git a/package.json b/package.json index 4d46fc3..2a470f0 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "check:public-proof": "npm --workspace @workit/core run check:public-proof", "check:worker-contract": "npm --workspace @workit/core run check:worker-contract", "check:release-policy": "npm --workspace @workit/core run check:release-policy", + "check:release-readiness": "npm --workspace @workit/core run check:release-readiness", "check:release": "npm --workspace @workit/core run check:release", "bench:articles": "npm --workspace @workit/core run bench:articles", "bench:articles:repeated": "npm --workspace @workit/core run bench:articles:repeated", diff --git a/packages/core/README.md b/packages/core/README.md index 65d0b4d..771286d 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -338,6 +338,10 @@ receipts must be persisted independently. > **0.6.0 release candidate:** this subpath is present in the prepared package, > but it is not included in npm `latest` until the signed `v0.6.0` tag and > provenance-backed publish complete. +> +> Publication is also blocked by evidence claim `REL-011` until Oryn passes a +> real integration canary at its provider and durable-idempotency boundaries. +> The bounded fixtures in this repository do not satisfy that claim. `@workit/core/candidates` separates transport success, semantic quality, and failure policy. Candidates run sequentially in caller order. WorkIt's built-in diff --git a/packages/core/SECURITY.md b/packages/core/SECURITY.md index 7c796bf..4a72424 100644 --- a/packages/core/SECURITY.md +++ b/packages/core/SECURITY.md @@ -57,6 +57,7 @@ provenance enabled. A release is not approved unless these commands pass: ```sh npm run verify npm run test:coverage +npm run check:release-readiness npm run check:vulnerabilities npm run check:sbom npm pack --dry-run --json diff --git a/packages/core/evidence/claims.json b/packages/core/evidence/claims.json index 95eed2d..7baec72 100644 --- a/packages/core/evidence/claims.json +++ b/packages/core/evidence/claims.json @@ -553,6 +553,17 @@ "verifiedFor": "0.5.0", "ledgerNote": "The root and group measurements backfill the 0.5.0 claim; candidate measurements apply to the 0.6.0 target." }, + { + "id": "REL-011", + "title": "Oryn real integration canary gates the 0.6.0 publication", + "class": "release", + "status": "deferred", + "releaseBlocking": true, + "proof": "README.md", + "command": "Oryn release environment: run the real candidates integration canary and attach its redacted receipt", + "expectedInvariant": "Oryn exercises firstAcceptable against its real provider boundary and caller-owned durable idempotency, preserving quality fallback, retry budget, aggregate deadline, cancellation, user-input stop, and bounded redacted evidence before v0.6.0 is tagged", + "limitations": "Deferred because this repository has no authority over the Oryn deployment, provider credentials, or durable store. Bounded in-process fixtures and package-consumer tests do not satisfy this claim." + }, { "id": "PERF-001", "title": "article benchmark suite has expected executable coverage", diff --git a/packages/core/package.json b/packages/core/package.json index 35b4d73..2bd50d0 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -188,10 +188,11 @@ "check:package-consumer:node": "npm run build && node scripts/check-package-consumer.mjs --node-only", "check:claims": "npm run build && node scripts/check-claim-fixtures.mjs", "check:evidence-ledger": "node scripts/check-evidence-ledger.mjs", + "check:release-readiness": "node scripts/check-release-readiness.mjs", "check:public-proof": "node scripts/check-public-proof.mjs", "check:worker-contract": "node scripts/check-worker-contract-docs.mjs", "check:release-policy": "npm run build && node scripts/check-release-provenance.mjs", - "check:release": "npm run build && node scripts/check-release-provenance.mjs --registry-dry-run", + "check:release": "npm run check:release-readiness && npm run build && node scripts/check-release-provenance.mjs --registry-dry-run", "bench:articles": "node benchmarks/articles/run-all.mjs", "bench:articles:repeated": "node benchmarks/articles/run-repeated.mjs", "test:evidence": "npm run build && node tests/evidence/run-all.mjs --output coverage/evidence/latest.json", diff --git a/packages/core/scripts/check-evidence-ledger.mjs b/packages/core/scripts/check-evidence-ledger.mjs index cdeb8a9..01272fa 100644 --- a/packages/core/scripts/check-evidence-ledger.mjs +++ b/packages/core/scripts/check-evidence-ledger.mjs @@ -11,17 +11,11 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { evidenceProofs } from "../tests/evidence/manifest.mjs"; import { computeEvidenceSourceDigest } from "./evidence-source-digest.mjs"; +import { summarizeEvidenceLedger } from "./evidence-ledger-status.mjs"; const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const ledger = await readJson("evidence/claims.json"); const capture = await readJson("coverage/evidence/latest.json"); -const allowedStatuses = new Set([ - "deferred", - "environment-blocked", - "product-decision", - "proven", - "unproven", -]); const proofManifest = new Set(evidenceProofs.map(({ file }) => `tests/evidence/${file}`)); const claimIds = new Set(); @@ -34,7 +28,6 @@ for (const claim of ledger.claims) { assert.ok(!claimIds.has(claim.id), `duplicate claim id: ${claim.id}`); claimIds.add(claim.id); assert.ok(ledger.allowedClasses.includes(claim.class), `${claim.id} has invalid class`); - assert.ok(allowedStatuses.has(claim.status), `${claim.id} has invalid status`); for (const field of ["title", "proof", "command", "expectedInvariant", "limitations"]) { assert.equal(typeof claim[field], "string", `${claim.id} is missing ${field}`); assert.ok(claim[field].length > 0, `${claim.id} has empty ${field}`); @@ -43,6 +36,13 @@ for (const claim of ledger.claims) { if (claim.command === "npm run test:evidence") { assert.ok(proofManifest.has(claim.proof), `${claim.id} proof is absent from evidence manifest`); } + if (claim.releaseBlocking === true && claim.status === "proven") { + assert.equal( + claim.command, + "npm run test:evidence", + `${claim.id} cannot resolve a release blocker without executable captured evidence`, + ); + } } assert.equal(capture.artifact, "workit-publication-evidence"); @@ -66,11 +66,13 @@ for (const claim of ledger.claims) { assert.ok(result.actualResult !== undefined, `${claim.id} captured actual result is missing`); } +const releaseStatus = summarizeEvidenceLedger(ledger.claims); process.stdout.write(JSON.stringify({ evidenceLedger: "ok", claims: ledger.claims.length, capturedResults: capture.claimResults.length, sourceDigest: capture.sourceDigest, + ...releaseStatus, }) + "\n"); async function readJson(path) { diff --git a/packages/core/scripts/check-release-provenance.mjs b/packages/core/scripts/check-release-provenance.mjs index 3f3a930..eb4a7c5 100644 --- a/packages/core/scripts/check-release-provenance.mjs +++ b/packages/core/scripts/check-release-provenance.mjs @@ -55,6 +55,11 @@ assert.match(workflow, /attestations:\s*write/u, "release workflow must allow Gi assert.match(workflow, /npm publish --workspace @workit\/core --provenance --access public/u, "release workflow must publish @workit/core with npm provenance"); assert.match(workflow, /npm run verify/u, "release workflow must run full verification before publish"); assert.match(workflow, /npm run test:coverage/u, "release workflow must run coverage before publish"); +assert.match( + workflow, + /npm run check:release-readiness/u, + "release workflow must reject unresolved release-blocking evidence" +); assert.match(workflow, /gpg\.ssh\.allowedSignersFile/u, "release workflow must configure SSH allowed signers before tag verification"); assert.match(workflow, /fetch-depth:\s*0/u, "release workflow must fetch signed tag objects and release history"); assert.ok( @@ -87,7 +92,7 @@ assert.match(allowedSigners, /admilsoncossa@gmail\.com ssh-ed25519 /u, "release await assertExistingTagsAreSigned(); if (!requireRegistryDryRun) { - console.log("release-policy-gate: provenance workflow validated and package is publishable"); + console.log("release-policy-gate: provenance workflow and publication shape validated; evidence readiness is a separate gate"); process.exit(0); } diff --git a/packages/core/scripts/check-release-readiness.mjs b/packages/core/scripts/check-release-readiness.mjs new file mode 100644 index 0000000..aa3d227 --- /dev/null +++ b/packages/core/scripts/check-release-readiness.mjs @@ -0,0 +1,25 @@ +/** + * Rejects publication while the evidence ledger has unresolved release blockers. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { summarizeEvidenceLedger } from "./evidence-ledger-status.mjs"; + +const ledger = JSON.parse(await readFile(new URL("../evidence/claims.json", import.meta.url), "utf8")); +assert.equal(ledger.artifact, "workit-claim-ledger", "release readiness requires the claim ledger"); + +const releaseStatus = summarizeEvidenceLedger(ledger.claims); +process.stdout.write(JSON.stringify({ + releaseEvidence: releaseStatus.releaseReadiness, + releaseBlockers: releaseStatus.releaseBlockers, +}) + "\n"); + +assert.equal( + releaseStatus.releaseBlockers.length, + 0, + `release evidence is blocked by ${releaseStatus.releaseBlockers.map(({ id, status }) => `${id}:${status}`).join(", ")}`, +); diff --git a/packages/core/scripts/evidence-ledger-status.mjs b/packages/core/scripts/evidence-ledger-status.mjs new file mode 100644 index 0000000..db27052 --- /dev/null +++ b/packages/core/scripts/evidence-ledger-status.mjs @@ -0,0 +1,47 @@ +/** + * Computes evidence-ledger status without performing file-system I/O. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +export const CLAIM_STATUSES = Object.freeze([ + "deferred", + "environment-blocked", + "product-decision", + "proven", + "unproven", +]); + +export const RELEASE_READINESS = Object.freeze({ + BLOCKED: "blocked", + READY: "ready", +}); + +const RESOLVED_RELEASE_STATUSES = new Set(["proven"]); + +/** Returns deterministic status counts and unresolved release blockers. */ +export function summarizeEvidenceLedger(claims) { + const statusCounts = Object.fromEntries(CLAIM_STATUSES.map((status) => [status, 0])); + for (const claim of claims) { + if (!Object.hasOwn(statusCounts, claim.status)) { + throw new TypeError(`${claim.id} has invalid status`); + } + if (claim.releaseBlocking !== undefined && typeof claim.releaseBlocking !== "boolean") { + throw new TypeError(`${claim.id} has invalid releaseBlocking`); + } + statusCounts[claim.status] += 1; + } + + const releaseBlockers = claims + .filter((claim) => claim.releaseBlocking === true && !RESOLVED_RELEASE_STATUSES.has(claim.status)) + .map(({ id, status, title }) => ({ id, status, title })); + + return { + statusCounts, + releaseReadiness: releaseBlockers.length === 0 + ? RELEASE_READINESS.READY + : RELEASE_READINESS.BLOCKED, + releaseBlockers, + }; +} diff --git a/packages/core/tests/unit/evidence-ledger-status.test.js b/packages/core/tests/unit/evidence-ledger-status.test.js new file mode 100644 index 0000000..9dff99e --- /dev/null +++ b/packages/core/tests/unit/evidence-ledger-status.test.js @@ -0,0 +1,66 @@ +/** + * Evidence-ledger release-readiness policy tests. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, test } from "vitest"; +import { + CLAIM_STATUSES, + RELEASE_READINESS, + summarizeEvidenceLedger, +} from "../../scripts/evidence-ledger-status.mjs"; + +describe("evidence ledger release readiness", () => { + test.each(["deferred", "environment-blocked", "product-decision", "unproven"])( + "treats a release-blocking %s claim as unresolved", + (status) => { + const summary = summarizeEvidenceLedger([ + { id: "REL-011", status, title: "real integration canary", releaseBlocking: true }, + ]); + + expect(summary.releaseReadiness).toBe(RELEASE_READINESS.BLOCKED); + expect(summary.releaseBlockers).toEqual([ + { id: "REL-011", status, title: "real integration canary" }, + ]); + }, + ); + + test("does not let non-blocking or resolved claims prevent a release", () => { + const claims = [ + { id: "CORR-001", status: "unproven", title: "future research" }, + { id: "REL-001", status: "proven", title: "release proof", releaseBlocking: true }, + { + id: "PROD-001", + status: "product-decision", + title: "declared boundary", + }, + ]; + + const summary = summarizeEvidenceLedger(claims); + + expect(summary.releaseReadiness).toBe(RELEASE_READINESS.READY); + expect(summary.releaseBlockers).toEqual([]); + expect(summary.statusCounts).toEqual({ + deferred: 0, + "environment-blocked": 0, + "product-decision": 1, + proven: 1, + unproven: 1, + }); + expect(Object.keys(summary.statusCounts)).toEqual(CLAIM_STATUSES); + }); + + test("rejects unknown claim statuses", () => { + expect(() => summarizeEvidenceLedger([ + { id: "REL-011", status: "passing", title: "invalid state", releaseBlocking: true }, + ])).toThrow("REL-011 has invalid status"); + }); + + test("rejects a non-boolean release-blocking marker", () => { + expect(() => summarizeEvidenceLedger([ + { id: "REL-011", status: "deferred", title: "invalid marker", releaseBlocking: "yes" }, + ])).toThrow("REL-011 has invalid releaseBlocking"); + }); +}); From 077b5b138a591bb30218fb3e961a911e3bc74276 Mon Sep 17 00:00:00 2001 From: "Admilson B. F. Cossa" Date: Mon, 17 Aug 2026 20:27:45 +0200 Subject: [PATCH 5/6] chore(security): rotate release signing key --- .github/allowed_signers | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/allowed_signers b/.github/allowed_signers index 263372e..2812552 100644 --- a/.github/allowed_signers +++ b/.github/allowed_signers @@ -1 +1,2 @@ admilsoncossa@gmail.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP9aimezLMuhjkApDOumfx/GXhW4IwdEw79kLzlaIHZ7 admilsoncossa@gmail.com +admilsoncossa@gmail.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICxMA5TIcWbUzpPXkmwc2xQfFRmFGyokIuegaF86Ydum admilsoncossa@gmail.com WorkIt signing 2026 From e7554517db1a25f3ecedfb70ef4d4b06cab25058 Mon Sep 17 00:00:00 2001 From: "Admilson B. F. Cossa" Date: Mon, 17 Aug 2026 21:10:48 +0200 Subject: [PATCH 6/6] test(evidence): prove Oryn candidate canary --- packages/core/evidence/README.md | 10 ++ packages/core/evidence/claims.json | 10 +- .../oryn-candidate-canary.v0.6.0.json | 132 ++++++++++++++++++ .../core/scripts/evidence-source-digest.mjs | 1 + packages/core/tests/evidence/manifest.mjs | 1 + .../release/oryn-candidate-canary.mjs | 85 +++++++++++ 6 files changed, 235 insertions(+), 4 deletions(-) create mode 100644 packages/core/evidence/oryn-candidate-canary.v0.6.0.json create mode 100644 packages/core/tests/evidence/release/oryn-candidate-canary.mjs diff --git a/packages/core/evidence/README.md b/packages/core/evidence/README.md index 2f8993c..a0c29ef 100644 --- a/packages/core/evidence/README.md +++ b/packages/core/evidence/README.md @@ -73,3 +73,13 @@ or lifecycle proof. Historical tags are never rewritten: later backfills remain explicitly labeled as backfills in the ledger. + +## Oryn 0.6.0 Canary + +`oryn-candidate-canary.v0.6.0.json` is the redacted external-integration receipt +for `REL-011`. It binds the packed `0.6.0` tarball hash, WorkIt and Oryn commits, +real provider routing decisions, daemon-backed receipt round-trip, durable replay, +and controlled retry-budget, deadline, and user-input-stop scenarios. The receipt +retains environment warnings and limitations; it contains neither provider response +bodies nor credentials. `release/oryn-candidate-canary.mjs` validates this receipt +as part of `test:evidence`, and the evidence-source digest includes the receipt. diff --git a/packages/core/evidence/claims.json b/packages/core/evidence/claims.json index 7baec72..d01e112 100644 --- a/packages/core/evidence/claims.json +++ b/packages/core/evidence/claims.json @@ -557,12 +557,14 @@ "id": "REL-011", "title": "Oryn real integration canary gates the 0.6.0 publication", "class": "release", - "status": "deferred", + "status": "proven", "releaseBlocking": true, - "proof": "README.md", - "command": "Oryn release environment: run the real candidates integration canary and attach its redacted receipt", + "proof": "tests/evidence/release/oryn-candidate-canary.mjs", + "command": "npm run test:evidence", "expectedInvariant": "Oryn exercises firstAcceptable against its real provider boundary and caller-owned durable idempotency, preserving quality fallback, retry budget, aggregate deadline, cancellation, user-input stop, and bounded redacted evidence before v0.6.0 is tagged", - "limitations": "Deferred because this repository has no authority over the Oryn deployment, provider credentials, or durable store. Bounded in-process fixtures and package-consumer tests do not satisfy this claim." + "limitations": "One local Oryn edge/daemon canary crossed real Ollama provider and daemon receipt boundaries. It is not a production-load, multi-host, or long-duration soak result; retry-budget, deadline, and requires-user-input faults remain controlled boundary scenarios.", + "verifiedFor": "0.6.0", + "ledgerNote": "Resolved by the redacted receipt in evidence/oryn-candidate-canary.v0.6.0.json, bound to WorkIt commit 077b5b1, Oryn commit 2e8c7b2, and the packed tarball SHA-256." }, { "id": "PERF-001", diff --git a/packages/core/evidence/oryn-candidate-canary.v0.6.0.json b/packages/core/evidence/oryn-candidate-canary.v0.6.0.json new file mode 100644 index 0000000..b2ea301 --- /dev/null +++ b/packages/core/evidence/oryn-candidate-canary.v0.6.0.json @@ -0,0 +1,132 @@ +{ + "artifact": "workit-oryn-candidate-canary", + "schemaVersion": 1, + "release": "0.6.0", + "capturedAt": "2026-08-17T19:00:29.581Z", + "source": { + "workitCommit": "077b5b138a591bb30218fb3e961a911e3bc74276", + "workitTarball": { + "file": "workit-core-0.6.0.tgz", + "sha256": "944746fd62814229401502b9e0c6003f126932ace31e0fd7fae45eda4f0e3256" + }, + "orynCommit": "2e8c7b243351527e8d8fd979085b6d91cd5a5c97", + "canaryScriptSha256": "b0570f2a22ee2dfe12dd5a2e8e0c055c4a3b828bd730be49fe079b01d0af2846" + }, + "toolchain": { + "node": "24.19.0", + "pnpm": "10.27.0", + "platform": "win32" + }, + "environment": { + "runtimeProfile": "edge", + "executionBackend": "daemon", + "daemonAddress": "127.0.0.1:7777", + "daemonEnabled": true, + "syntheticProvider": false, + "availableProviders": [ + "gemini", + "ollama", + "openrouter" + ], + "warnings": [ + { + "code": "oryn_entitlement_lease_invalid", + "detail": "Oryn bootstrap reported an invalid entitlement lease audience or issuer.", + "workitReleaseImpact": "The provider and daemon canary remained executable; this is an Oryn local-environment warning, not evidence of a WorkIt runtime failure." + }, + { + "code": "oryn_local_storage_secret_dev_fallback", + "detail": "LOCAL_STORAGE_SECRET was not configured and Oryn reported its documented development fallback.", + "workitReleaseImpact": "No storage payload or credential was used by the candidate canary; production Oryn deployment must not use this fallback." + } + ] + }, + "route": { + "taskKey": "intent.classify", + "providerHint": "ollama", + "primaryModel": "glm-5.2:cloud", + "fallbackModels": [ + "kimi-k2.6:cloud", + "minimax-m2.7:cloud", + "gpt-oss:20b-cloud", + "glm-5:cloud" + ] + }, + "liveProviderBoundary": { + "canaryRunId": "f336ab91-9acf-4f90-8193-c4c91a0ca2d5", + "providerCalls": 2, + "acceptedCandidateIndex": 1, + "acceptedModel": "kimi-k2.6:cloud", + "decisions": [ + { + "candidateIndex": 0, + "modelId": "glm-5.2:cloud", + "provider": "ollama", + "attempt": 1, + "outcome": "succeeded", + "decision": "quality_rejected", + "reasonCode": "canary_quality_ordinal_mismatch", + "token": "[redacted]" + }, + { + "candidateIndex": 1, + "modelId": "kimi-k2.6:cloud", + "provider": "ollama", + "attempt": 1, + "outcome": "succeeded", + "decision": "accepted", + "reasonCode": null, + "token": "[redacted]" + } + ], + "evidenceLimit": 4, + "droppedEvidence": 0 + }, + "durableReplay": { + "activityDirectoryPersistedAcrossRuntimeInstances": true, + "recreatedRuntime": true, + "replayOperationCalls": 0, + "providerCallsAfterLiveExecution": 2, + "providerCallsAfterReplay": 2, + "daemonReceiptRoundTrip": true, + "receiptId": "work-attempt:attempt-f336ab91-9acf-4f90-8193-c4c91a0ca2d5", + "replayReceiptId": "work-attempt:attempt-f336ab91-9acf-4f90-8193-c4c91a0ca2d5" + }, + "controlledBoundaryScenarios": { + "sharedRetryBudget": { + "status": "accepted", + "attempts": 2, + "spent": 1, + "limit": 1 + }, + "aggregateDeadline": { + "status": "terminal", + "reasonCode": "workit_timeout", + "elapsedMs": 32, + "configuredMs": 25 + }, + "requiresUserInput": { + "status": "requires_user_input", + "reasonCode": "provider_credential_required", + "candidateCalls": 1, + "fallbackStopped": true + } + }, + "assertions": { + "liveAcceptedAfterQualityFallback": true, + "evidenceBounded": true, + "evidenceRedacted": true, + "durableReplaySkippedProvider": true, + "daemonReceiptRoundTrip": true, + "sharedRetryBudgetChargedOnce": true, + "aggregateDeadlineStoppedWork": true, + "userInputStoppedFallback": true, + "noUnhandledProcessFailure": true + }, + "unexpectedProcessFailures": [], + "limitations": [ + "This is one local Oryn edge/daemon canary, not a production-load, multi-host, or long-duration soak result.", + "Provider response bodies are intentionally excluded; only routing facts, bounded decisions, redacted metadata, and output-independent assertions are retained.", + "Retry-budget, deadline, and requires-user-input faults are controlled boundary scenarios executed from the packed 0.6.0 artifact inside the real Oryn process; provider quality fallback and durable replay use the real provider and daemon boundaries." + ] +} diff --git a/packages/core/scripts/evidence-source-digest.mjs b/packages/core/scripts/evidence-source-digest.mjs index 1dc487e..6229e48 100644 --- a/packages/core/scripts/evidence-source-digest.mjs +++ b/packages/core/scripts/evidence-source-digest.mjs @@ -16,6 +16,7 @@ const EVIDENCE_SOURCE_PATHS = Object.freeze([ "SECURITY.md", "benchmarks", "evidence/claims.json", + "evidence/oryn-candidate-canary.v0.6.0.json", "package.json", "samples", "scripts", diff --git a/packages/core/tests/evidence/manifest.mjs b/packages/core/tests/evidence/manifest.mjs index 700666f..1afa03c 100644 --- a/packages/core/tests/evidence/manifest.mjs +++ b/packages/core/tests/evidence/manifest.mjs @@ -31,6 +31,7 @@ export const evidenceProofs = Object.freeze([ proof("security/candidate-boundary.mjs"), proof("release/candidate-package-contract.mjs"), proof("release/core-release-contracts.mjs"), + proof("release/oryn-candidate-canary.mjs"), proof("release/release-integrity.mjs"), proof("release/receipt-ledger.mjs"), proof("release/sql-receipt-ledger.mjs"), diff --git a/packages/core/tests/evidence/release/oryn-candidate-canary.mjs b/packages/core/tests/evidence/release/oryn-candidate-canary.mjs new file mode 100644 index 0000000..76f2c16 --- /dev/null +++ b/packages/core/tests/evidence/release/oryn-candidate-canary.mjs @@ -0,0 +1,85 @@ +/** + * Release evidence for the Oryn real-provider and durable-replay canary. + * + * @author Admilson B. F. Cossa + * SPDX-License-Identifier: Apache-2.0 + */ + +import { readFile } from "node:fs/promises"; +import { createSuite } from "../harness.mjs"; + +const suite = createSuite("release"); + +await suite.proof( + "REL-011", + "Oryn real integration canary gates the 0.6.0 publication", + "the packed 0.6.0 candidate policy crosses Oryn's real provider and daemon boundaries while preserving quality fallback, durable replay, bounded redaction, retry budget, deadline cancellation, and user-input stop", + async () => { + const receipt = JSON.parse(await readFile( + new URL("../../../evidence/oryn-candidate-canary.v0.6.0.json", import.meta.url), + "utf8", + )); + const live = receipt.liveProviderBoundary; + const replay = receipt.durableReplay; + const controlled = receipt.controlledBoundaryScenarios; + const assertions = Object.values(receipt.assertions ?? {}); + const serialized = JSON.stringify(receipt); + const decisions = live?.decisions ?? []; + + return { + ok: receipt.artifact === "workit-oryn-candidate-canary" + && receipt.schemaVersion === 1 + && receipt.release === "0.6.0" + && /^[0-9a-f]{40}$/u.test(receipt.source?.workitCommit ?? "") + && /^[0-9a-f]{40}$/u.test(receipt.source?.orynCommit ?? "") + && /^[0-9a-f]{64}$/u.test(receipt.source?.workitTarball?.sha256 ?? "") + && /^[0-9a-f]{64}$/u.test(receipt.source?.canaryScriptSha256 ?? "") + && receipt.environment?.syntheticProvider === false + && receipt.environment?.daemonEnabled === true + && receipt.route?.providerHint === "ollama" + && live?.providerCalls === 2 + && live?.acceptedCandidateIndex === 1 + && decisions.some(({ decision }) => decision === "quality_rejected") + && decisions.some(({ decision }) => decision === "accepted") + && decisions.every(({ token }) => token === "[redacted]") + && live?.droppedEvidence === 0 + && decisions.length <= live?.evidenceLimit + && replay?.recreatedRuntime === true + && replay?.replayOperationCalls === 0 + && replay?.providerCallsAfterLiveExecution === replay?.providerCallsAfterReplay + && replay?.daemonReceiptRoundTrip === true + && replay?.receiptId === replay?.replayReceiptId + && controlled?.sharedRetryBudget?.spent === controlled?.sharedRetryBudget?.limit + && controlled?.sharedRetryBudget?.attempts === 2 + && controlled?.aggregateDeadline?.reasonCode === "workit_timeout" + && controlled?.requiresUserInput?.fallbackStopped === true + && controlled?.requiresUserInput?.candidateCalls === 1 + && assertions.length > 0 + && assertions.every(Boolean) + && receipt.unexpectedProcessFailures?.length === 0 + && !serialized.includes("canary-secret-must-not-escape"), + workitVersion: receipt.release, + workitCommit: receipt.source?.workitCommit, + orynCommit: receipt.source?.orynCommit, + tarballSha256: receipt.source?.workitTarball?.sha256, + providerCalls: live?.providerCalls, + decisions: decisions.map(({ candidateIndex, modelId, decision, reasonCode }) => ({ + candidateIndex, + modelId, + decision, + reasonCode, + })), + replayOperationCalls: replay?.replayOperationCalls, + providerCallsAfterReplay: replay?.providerCallsAfterReplay, + sharedRetryBudget: controlled?.sharedRetryBudget, + aggregateDeadline: controlled?.aggregateDeadline, + requiresUserInput: controlled?.requiresUserInput, + environmentWarnings: receipt.environment?.warnings?.map(({ code }) => code), + limitations: receipt.limitations, + }; + }, +); + +const summary = suite.summary(); +process.stdout.write(JSON.stringify(summary, null, 2) + "\n"); +process.exit(summary.failed > 0 ? 1 : 0);