From ec8dbfb5526d46c23de4f3df1593a80994f646c9 Mon Sep 17 00:00:00 2001 From: claude-bot-go Date: Thu, 23 Jul 2026 21:59:19 +0800 Subject: [PATCH] Fix #25: Fix WasmAgent/wasmagent-protocol#13 ([milestone Milestone 3 ] Publish an `aep-conformance/` fixture set in the package ( --- aep-conformance/README.md | 69 +++++++++ .../invalid/bad-capability-decision.json | 13 ++ .../invalid/bad-schema-version.json | 5 + .../invalid/bad-timestamp-type.json | 13 ++ .../invalid/missing-action-required.json | 12 ++ .../invalid/missing-created-at-ms.json | 4 + aep-conformance/invalid/missing-run-id.json | 4 + .../invalid/missing-signature-required.json | 9 ++ aep-conformance/manifest.json | 79 ++++++++++ aep-conformance/valid/minimal.json | 5 + aep-conformance/valid/v0.1.json | 9 ++ aep-conformance/valid/with-budget-ledger.json | 22 +++ aep-conformance/valid/with-capabilities.json | 50 ++++++ aep-conformance/valid/with-refs.json | 24 +++ package.json | 6 +- pyproject.toml | 3 + tests/test_aep_conformance.py | 144 ++++++++++++++++++ 17 files changed, 469 insertions(+), 2 deletions(-) create mode 100644 aep-conformance/README.md create mode 100644 aep-conformance/invalid/bad-capability-decision.json create mode 100644 aep-conformance/invalid/bad-schema-version.json create mode 100644 aep-conformance/invalid/bad-timestamp-type.json create mode 100644 aep-conformance/invalid/missing-action-required.json create mode 100644 aep-conformance/invalid/missing-created-at-ms.json create mode 100644 aep-conformance/invalid/missing-run-id.json create mode 100644 aep-conformance/invalid/missing-signature-required.json create mode 100644 aep-conformance/manifest.json create mode 100644 aep-conformance/valid/minimal.json create mode 100644 aep-conformance/valid/v0.1.json create mode 100644 aep-conformance/valid/with-budget-ledger.json create mode 100644 aep-conformance/valid/with-capabilities.json create mode 100644 aep-conformance/valid/with-refs.json create mode 100644 tests/test_aep_conformance.py diff --git a/aep-conformance/README.md b/aep-conformance/README.md new file mode 100644 index 0000000..7d694cc --- /dev/null +++ b/aep-conformance/README.md @@ -0,0 +1,69 @@ +# `aep-conformance/` — AEP record conformance fixture set + +A versioned corpus of **valid** and **invalid** `aep-record` instances, published +inside the [`@wasmagent/protocol`](https://www.npmjs.com/package/@wasmagent/protocol) +package so that any Agent Evidence Protocol emitter can self-check: + +> _Does my output validate against the canonical record contract?_ + +This is the consumer-facing counterpart to the repository's internal +[`tests/fixtures/`](../tests/fixtures/) harness. Where `tests/fixtures/` holds one +example per schema to guard the repo itself, `aep-conformance/` is a richer, +deliberately enumerated set covering the distinct constraint regions of +`aep-record`, intended to be pointed at a real emitter's output. + +## Layout + +| Path | Purpose | +| --- | --- | +| [`manifest.json`](manifest.json) | Machine-readable index of every fixture: path, `valid` flag, description, and (for invalid cases) the `expect_keyword` the violation should trip. | +| `valid/*.json` | Records that **MUST pass** validation against [`schemas/aep/aep-record.schema.json`](../schemas/aep/aep-record.schema.json). | +| `invalid/*.json` | Records that **MUST fail**, each violating exactly one constraint, annotated in the manifest. | + +## Coverage + +- **Valid** — minimal record, the `aep/v0.1` legacy version, capability decisions + + actions + verifier results, a full budget ledger + run context, and input/output + refs with taint labels and redaction profiles. +- **Invalid** — `enum` (schema version, capability decision), `required` (missing + top-level / action / signature fields), and `type` (numeric field as string) + violations. + +## Using it + +Iterate `manifest.json`, load the schema from the package, and validate each +fixture — asserting the opposite result for `valid` vs `invalid`. + +### JavaScript / TypeScript + +```ts +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; + +// Fixture corpus + manifest ship inside the package under aep-conformance/. +const manifest = JSON.parse( + readFileSync(require.resolve("@wasmagent/protocol/aep-conformance/manifest.json"), "utf8"), +); + +// Resolve the canonical schema and a validator from your toolchain (e.g. ajv). +const require = createRequire(import.meta.url); +const schema = require("@wasmagent/protocol/schemas/aep/aep-record.schema.json"); +``` + +### Python + +```python +import json +from importlib import resources +from wasmagent_protocol import get_schema + +schema = get_schema("aep-record") # parsed canonical schema +manifest = json.loads( + (resources.files("wasmagent_protocol") / "aep_conformance" / "manifest.json") + .read_text("utf-8") +) +``` + +> The npm package exposes the corpus via `@wasmagent/protocol/aep-conformance/*`. +> Consumers iterate the manifest rather than hard-coding paths so the set can grow +> additively across versions. diff --git a/aep-conformance/invalid/bad-capability-decision.json b/aep-conformance/invalid/bad-capability-decision.json new file mode 100644 index 0000000..6bbc98f --- /dev/null +++ b/aep-conformance/invalid/bad-capability-decision.json @@ -0,0 +1,13 @@ +{ + "schema_version": "aep/v0.2", + "run_id": "run-bad-decision", + "created_at_ms": 1737600000000, + "capability_decisions": [ + { + "capability": "fs.write", + "subject": "agent-1", + "resource": "/tmp/x", + "decision": "blocked" + } + ] +} diff --git a/aep-conformance/invalid/bad-schema-version.json b/aep-conformance/invalid/bad-schema-version.json new file mode 100644 index 0000000..bd22b04 --- /dev/null +++ b/aep-conformance/invalid/bad-schema-version.json @@ -0,0 +1,5 @@ +{ + "schema_version": "aep/v9.9", + "run_id": "run-bad-version", + "created_at_ms": 1737600000000 +} diff --git a/aep-conformance/invalid/bad-timestamp-type.json b/aep-conformance/invalid/bad-timestamp-type.json new file mode 100644 index 0000000..00ffa85 --- /dev/null +++ b/aep-conformance/invalid/bad-timestamp-type.json @@ -0,0 +1,13 @@ +{ + "schema_version": "aep/v0.2", + "run_id": "run-bad-ts-type", + "created_at_ms": 1737600000000, + "actions": [ + { + "action_id": "a1", + "tool_name": "write_file", + "state_changing": true, + "timestamp_ms": "1737600000100" + } + ] +} diff --git a/aep-conformance/invalid/missing-action-required.json b/aep-conformance/invalid/missing-action-required.json new file mode 100644 index 0000000..52e4921 --- /dev/null +++ b/aep-conformance/invalid/missing-action-required.json @@ -0,0 +1,12 @@ +{ + "schema_version": "aep/v0.2", + "run_id": "run-action-missing-tool", + "created_at_ms": 1737600000000, + "actions": [ + { + "action_id": "a1", + "state_changing": true, + "timestamp_ms": 1737600000100 + } + ] +} diff --git a/aep-conformance/invalid/missing-created-at-ms.json b/aep-conformance/invalid/missing-created-at-ms.json new file mode 100644 index 0000000..ace7018 --- /dev/null +++ b/aep-conformance/invalid/missing-created-at-ms.json @@ -0,0 +1,4 @@ +{ + "schema_version": "aep/v0.2", + "run_id": "run-no-created" +} diff --git a/aep-conformance/invalid/missing-run-id.json b/aep-conformance/invalid/missing-run-id.json new file mode 100644 index 0000000..96b7a5f --- /dev/null +++ b/aep-conformance/invalid/missing-run-id.json @@ -0,0 +1,4 @@ +{ + "schema_version": "aep/v0.2", + "created_at_ms": 1737600000000 +} diff --git a/aep-conformance/invalid/missing-signature-required.json b/aep-conformance/invalid/missing-signature-required.json new file mode 100644 index 0000000..057528f --- /dev/null +++ b/aep-conformance/invalid/missing-signature-required.json @@ -0,0 +1,9 @@ +{ + "schema_version": "aep/v0.2", + "run_id": "run-sig-incomplete", + "created_at_ms": 1737600000000, + "signature": { + "alg": "ed25519", + "key_id": "k1" + } +} diff --git a/aep-conformance/manifest.json b/aep-conformance/manifest.json new file mode 100644 index 0000000..dd87319 --- /dev/null +++ b/aep-conformance/manifest.json @@ -0,0 +1,79 @@ +{ + "name": "aep-conformance", + "description": "Machine-readable index of the aep-record conformance fixture set. Each entry names a fixture, whether it MUST pass (valid) or MUST fail (invalid) against the canonical aep-record schema, and for invalid cases the JSON Schema keyword the violation is expected to trip. Consumers iterate these fixtures to self-check their emitter output.", + "schema": { + "id": "aep-record", + "version": "aep/v0.2", + "canonical_id": "https://wasmagent.dev/schemas/aep/aep-record.schema.json", + "path": "schemas/aep/aep-record.schema.json" + }, + "fixtures": [ + { + "path": "valid/minimal.json", + "valid": true, + "description": "Minimal record carrying only the three required top-level fields (schema_version, run_id, created_at_ms)." + }, + { + "path": "valid/v0.1.json", + "valid": true, + "description": "Record using the legacy aep/v0.1 schema_version plus optional scalar provenance fields." + }, + { + "path": "valid/with-capabilities.json", + "valid": true, + "description": "Record exercising capability_decisions, actions, verifier_results, and the signature envelope." + }, + { + "path": "valid/with-budget-ledger.json", + "valid": true, + "description": "Record exercising a fully-populated budget_ledger (all six budget kinds) and run_context." + }, + { + "path": "valid/with-refs.json", + "valid": true, + "description": "Record exercising input_refs (with taint_labels) and output_refs (with redaction_profile)." + }, + { + "path": "invalid/bad-schema-version.json", + "valid": false, + "description": "schema_version is outside the allowed aep/v0.1, aep/v0.2 enum.", + "expect_keyword": "enum" + }, + { + "path": "invalid/missing-run-id.json", + "valid": false, + "description": "Missing the required run_id top-level field.", + "expect_keyword": "required" + }, + { + "path": "invalid/missing-created-at-ms.json", + "valid": false, + "description": "Missing the required created_at_ms top-level field.", + "expect_keyword": "required" + }, + { + "path": "invalid/bad-capability-decision.json", + "valid": false, + "description": "A capability decision uses a value outside the allow/deny/ask_user/dry_run enum.", + "expect_keyword": "enum" + }, + { + "path": "invalid/missing-action-required.json", + "valid": false, + "description": "An action is missing the required tool_name field.", + "expect_keyword": "required" + }, + { + "path": "invalid/bad-timestamp-type.json", + "valid": false, + "description": "An action timestamp_ms is a string rather than a number.", + "expect_keyword": "type" + }, + { + "path": "invalid/missing-signature-required.json", + "valid": false, + "description": "The signature block is present but missing its required sig field.", + "expect_keyword": "required" + } + ] +} diff --git a/aep-conformance/valid/minimal.json b/aep-conformance/valid/minimal.json new file mode 100644 index 0000000..c7040ab --- /dev/null +++ b/aep-conformance/valid/minimal.json @@ -0,0 +1,5 @@ +{ + "schema_version": "aep/v0.2", + "run_id": "run-minimal", + "created_at_ms": 1737600000000 +} diff --git a/aep-conformance/valid/v0.1.json b/aep-conformance/valid/v0.1.json new file mode 100644 index 0000000..13bf6d6 --- /dev/null +++ b/aep-conformance/valid/v0.1.json @@ -0,0 +1,9 @@ +{ + "schema_version": "aep/v0.1", + "run_id": "run-v0-1", + "trace_id": "trace-v0-1", + "runtime_version": "wasmagent-js@1.19.0", + "model_provider": "anthropic", + "model_id": "claude-sonnet-5", + "created_at_ms": 1737600000000 +} diff --git a/aep-conformance/valid/with-budget-ledger.json b/aep-conformance/valid/with-budget-ledger.json new file mode 100644 index 0000000..1034a20 --- /dev/null +++ b/aep-conformance/valid/with-budget-ledger.json @@ -0,0 +1,22 @@ +{ + "schema_version": "aep/v0.2", + "run_id": "run-budget", + "trace_id": "trace-budget-001", + "created_at_ms": 1737600000000, + "budget_ledger": { + "token_budget": { "limit": 100000, "spent": 42300 }, + "latency_budget": { "limit_ms": 30000, "actual_ms": 8421 }, + "tool_budget": { "limit": 50, "spent": 3 }, + "risk_budget": { "limit": 100, "spent": 12 }, + "retry_budget": { "limit": 5, "spent": 0 }, + "human_approval_budget": { "limit": 3, "spent": 1 } + }, + "run_context": { + "agent_id": "agent-1", + "agent_version": "0.4.2", + "subagent_id": "sub-1", + "delegation_chain": ["agent-1", "sub-1"], + "environment_digest": "sha256:env", + "dependency_lock_digest": "sha256:lock" + } +} diff --git a/aep-conformance/valid/with-capabilities.json b/aep-conformance/valid/with-capabilities.json new file mode 100644 index 0000000..d2f5a1f --- /dev/null +++ b/aep-conformance/valid/with-capabilities.json @@ -0,0 +1,50 @@ +{ + "schema_version": "aep/v0.2", + "run_id": "run-capabilities", + "trace_id": "trace-capabilities-001", + "parent_trace_id": null, + "repo_commit": "deadbeef", + "runtime_version": "wasmagent-js@1.20.0", + "created_at_ms": 1737600000000, + "capability_decisions": [ + { + "capability": "fs.write", + "subject": "agent-1", + "resource": "/tmp/out.txt", + "decision": "allow" + }, + { + "capability": "net.request", + "subject": "agent-1", + "resource": "https://example.com", + "decision": "deny", + "reason_code": "no_network_scope" + } + ], + "actions": [ + { + "action_id": "a1", + "tool_name": "write_file", + "state_changing": true, + "precondition_digest": "sha256:pre", + "result_digest": "sha256:post", + "evidence_refs": ["ev-1"], + "parent_action_id": "a0", + "causal_chain_id": "chain-1", + "timestamp_ms": 1737600000100 + } + ], + "verifier_results": [ + { + "verifier_id": "symkernel-cel", + "passed": true, + "score": 1.0, + "claim_ids": ["claim-1"] + } + ], + "signature": { + "alg": "ed25519", + "key_id": "k1", + "sig": "base64sig==" + } +} diff --git a/aep-conformance/valid/with-refs.json b/aep-conformance/valid/with-refs.json new file mode 100644 index 0000000..ae31aac --- /dev/null +++ b/aep-conformance/valid/with-refs.json @@ -0,0 +1,24 @@ +{ + "schema_version": "aep/v0.2", + "run_id": "run-refs", + "trace_id": "trace-refs-001", + "runtime_version": "wasmagent-js@1.20.0", + "policy_bundle_digest": "sha256:policy", + "tool_manifest_digest": "sha256:manifest", + "mcp_server_card_digest": null, + "created_at_ms": 1737600000000, + "input_refs": [ + { + "uri": "file:///workspace/src/app.ts", + "digest": "sha256:in1", + "taint_labels": ["user-input"] + } + ], + "output_refs": [ + { + "uri": "file:///workspace/dist/app.js", + "digest": "sha256:out1", + "redaction_profile": "default" + } + ] +} diff --git a/package.json b/package.json index 8bd4f37..b749d24 100644 --- a/package.json +++ b/package.json @@ -20,12 +20,14 @@ "types": "./index.d.ts", "import": "./index.js" }, - "./schemas/*": "./schemas/*" + "./schemas/*": "./schemas/*", + "./aep-conformance/*": "./aep-conformance/*" }, "files": [ "index.js", "index.d.ts", - "schemas/" + "schemas/", + "aep-conformance/" ], "scripts": { "test": "node --test" diff --git a/pyproject.toml b/pyproject.toml index 110041a..0aa55fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,3 +35,6 @@ packages = ["src/wasmagent_protocol"] # from an installed wheel. [tool.hatch.build.targets.wheel.force-include] "schemas" = "wasmagent_protocol/schemas" +# Ship the aep-conformance fixture corpus so consumers can self-check their +# emitter output from an installed wheel (mirrors how schemas ship). +"aep-conformance" = "wasmagent_protocol/aep_conformance" diff --git a/tests/test_aep_conformance.py b/tests/test_aep_conformance.py new file mode 100644 index 0000000..40f02cd --- /dev/null +++ b/tests/test_aep_conformance.py @@ -0,0 +1,144 @@ +"""Pytest harness for the published ``aep-conformance/`` fixture corpus. + +Issue #13 / #25 ship a versioned corpus of valid + invalid ``aep-record`` +instances *inside the published package* so any Agent Evidence Protocol emitter +can self-check its output against the canonical contract. These tests lock that +contract in: + +- every fixture named in ``manifest.json`` behaves exactly as the manifest + claims (valid -> passes; invalid -> fails, and the failing JSON Schema keyword + matches ``expect_keyword``); +- the manifest and the on-disk fixtures stay in sync (no missing or orphan + files); +- the manifest's own ``schema`` block points at the real canonical + ``aep-record`` schema; +- the corpus is actually declared as shipped package data in both ``package.json`` + and ``pyproject.toml``. + +This complements ``tests/conformance.py`` (the repo-internal per-schema harness, +run as a script) by exercising the richer, consumer-facing ``aep-record`` corpus. +""" +from __future__ import annotations + +import json +import re +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +ROOT = Path(__file__).resolve().parent.parent +CORPUS = ROOT / "aep-conformance" + +MANIFEST = json.loads((CORPUS / "manifest.json").read_text(encoding="utf-8")) +FIXTURES = MANIFEST["fixtures"] + +SCHEMA_PATH = ROOT / "schemas" / "aep" / "aep-record.schema.json" +SCHEMA = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) +VALIDATOR = Draft202012Validator(SCHEMA) + + +def _load_fixture(rel: str) -> object: + return json.loads((CORPUS / rel).read_text(encoding="utf-8")) + + +# --- per-fixture validation --------------------------------------------------- + +VALID_FIXTURES = [f for f in FIXTURES if f["valid"]] +INVALID_FIXTURES = [f for f in FIXTURES if not f["valid"]] + + +@pytest.mark.parametrize("fixture", VALID_FIXTURES, ids=lambda f: f["path"]) +def test_valid_fixture_passes(fixture: dict) -> None: + """Every valid fixture MUST validate against the canonical aep-record schema.""" + errors = list(VALIDATOR.iter_errors(_load_fixture(fixture["path"]))) + assert not errors, ( + f"valid fixture {fixture['path']} was rejected: {errors[0].message}" + ) + + +@pytest.mark.parametrize("fixture", INVALID_FIXTURES, ids=lambda f: f["path"]) +def test_invalid_fixture_fails_with_expected_keyword(fixture: dict) -> None: + """Each invalid fixture MUST fail, and on the keyword the manifest predicts.""" + errors = list(VALIDATOR.iter_errors(_load_fixture(fixture["path"]))) + assert errors, ( + f"invalid fixture {fixture['path']} was accepted (should have failed)" + ) + keywords = {error.validator for error in errors} + expected = fixture["expect_keyword"] + assert expected in keywords, ( + f"invalid fixture {fixture['path']} tripped {sorted(keywords)}, " + f"but manifest expected {expected!r}" + ) + + +# --- corpus completeness ------------------------------------------------------ + +def test_corpus_has_both_valid_and_invalid_fixtures() -> None: + """The published set must carry at least one valid and one invalid record.""" + assert VALID_FIXTURES, "manifest lists no valid fixtures" + assert INVALID_FIXTURES, "manifest lists no invalid fixtures" + + +def test_manifest_and_files_are_in_sync() -> None: + """Every manifest path exists, and every on-disk fixture is listed.""" + listed = {f["path"] for f in FIXTURES} + + for rel in sorted(listed): + assert (CORPUS / rel).is_file(), f"manifest names a missing fixture: {rel}" + + on_disk = { + str(p.relative_to(CORPUS)) + for p in (CORPUS / "valid").glob("*.json") + } | { + str(p.relative_to(CORPUS)) + for p in (CORPUS / "invalid").glob("*.json") + } + unlisted = on_disk - listed + assert not unlisted, f"fixtures on disk but absent from manifest: {sorted(unlisted)}" + + +def test_invalid_fixtures_each_violate_exactly_one_keyword() -> None: + """An invalid fixture isolates one constraint region — assert it fails for + exactly the keyword family its description claims (no accidental coverage).""" + for fixture in INVALID_FIXTURES: + errors = list(VALIDATOR.iter_errors(_load_fixture(fixture["path"]))) + keywords = {error.validator for error in errors} + assert keywords == {fixture["expect_keyword"]}, ( + f"{fixture['path']} tripped {sorted(keywords)}; " + f"expected only {fixture['expect_keyword']!r}" + ) + + +# --- manifest metadata + package publication ---------------------------------- + +def test_manifest_schema_block_matches_canonical_record() -> None: + """The manifest's ``schema`` block must point at the real canonical schema.""" + block = MANIFEST["schema"] + assert block["id"] == "aep-record" + assert block["canonical_id"] == SCHEMA["$id"] + assert block["path"] == "schemas/aep/aep-record.schema.json" + + index = json.loads((ROOT / "schemas" / "index.json").read_text(encoding="utf-8")) + entry = next(s for s in index["schemas"] if s["id"] == "aep-record") + assert block["version"] == entry["version"] + + +def test_corpus_is_published_in_npm_package() -> None: + """package.json must ship aep-conformance/ and expose it under exports.""" + pkg = json.loads((ROOT / "package.json").read_text(encoding="utf-8")) + assert "aep-conformance/" in pkg["files"], ( + "package.json 'files' does not ship aep-conformance/" + ) + assert any( + k.startswith("./aep-conformance") for k in pkg["exports"] + ), "package.json 'exports' does not expose aep-conformance/*" + + +def test_corpus_is_published_in_python_package() -> None: + """pyproject.toml must force-include aep-conformance into the wheel.""" + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + assert re.search( + r'"aep-conformance"\s*=\s*"wasmagent_protocol/aep_conformance"', + text, + ), "pyproject.toml does not force-include aep-conformance into the wheel"