test: add cross-language contract fixtures and validators (#241)

.github/workflows/tests.yml

@@ -67,6 +67,8 @@ jobs:
           python scripts/validate_docs.py
           python scripts/validate_changelog.py
           python scripts/validate_schemas.py
+          python scripts/validate_contract_fixtures.py
+          node scripts/validate_contract_fixtures.mjs
           python scripts/benchmark_runtime.py --check
           python -m compileall -q examples
       - name: Run tests with coverage threshold

CHANGELOG.md

@@ -49,6 +49,8 @@ and versions are tracked in the repo-root `VERSION` file.
 
 ### Added
 
+- Add golden success, error, inspection, log, NDJSON, and command-protocol
+  fixtures with Python and Node.js validators for cross-language consumers.
 - Publish versioned JSON Schema artifacts for output, error, inspection, log,
   NDJSON, and decoded command-protocol contracts in the package and docs site.
 - Add a framework choice guide, five-minute evaluation path, and clearer

docs/json-contracts.md

@@ -56,6 +56,12 @@ their own structured `details` records. Secret-looking keys (`token`,
 `password`, `secret`, `api_key`, and `authorization`) and credential-bearing
 URLs are redacted recursively.
 
+Golden payloads for each public contract live in
+[`tests/fixtures/contracts`](https://github.com/basefoundry/base-cli/tree/main/tests/fixtures/contracts).
+CI validates them against the packaged schemas with both a Python validator and
+a dependency-free Node.js reader; consumers can use the same fixtures as
+cross-language conformance tests.
+
 ## Inspection envelopes
 
 Read-only inspection commands can use the stable inspection helpers when their

scripts/validate_contract_fixtures.mjs

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+/** Validate golden contract fixtures with a non-Python reference reader. */
+
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const schemaDir = path.join(root, "lib", "python", "base_cli", "schemas", "v1");
+const fixtureDir = path.join(root, "tests", "fixtures", "contracts");
+const fixtures = new Map([
+  ["output-success.json", "output.schema.json"],
+  ["error-usage.json", "error.schema.json"],
+  ["inspection-warn.json", "inspection.schema.json"],
+  ["log-record.json", "log.schema.json"],
+  ["ndjson-record.json", "ndjson.schema.json"],
+  ["command-protocol.json", "command-protocol.schema.json"],
+]);
+
+function readJson(file) {
+  return JSON.parse(fs.readFileSync(file, "utf8"));
+}
+
+function validate(schema, value, label) {
+  if (schema.type === "object" && (value === null || Array.isArray(value) || typeof value !== "object")) {
+    throw new Error(`${label} must be an object`);
+  }
+  for (const field of schema.required ?? []) {
+    if (!Object.prototype.hasOwnProperty.call(value, field)) throw new Error(`${label} is missing ${field}`);
+  }
+  if (schema.additionalProperties === false) {
+    for (const field of Object.keys(value)) {
+      if (!Object.prototype.hasOwnProperty.call(schema.properties ?? {}, field)) throw new Error(`${label} has unexpected field ${field}`);
+    }
+  }
+  for (const [field, rule] of Object.entries(schema.properties ?? {})) {
+    if (!Object.prototype.hasOwnProperty.call(value, field)) continue;
+    if (Object.prototype.hasOwnProperty.call(rule, "const") && value[field] !== rule.const) throw new Error(`${label}.${field} constant mismatch`);
+    if (rule.enum && !rule.enum.includes(value[field])) throw new Error(`${label}.${field} enum mismatch`);
+  }
+}
+
+for (const [fixtureName, schemaName] of fixtures) {
+  validate(readJson(path.join(schemaDir, schemaName)), readJson(path.join(fixtureDir, fixtureName)), fixtureName);
+  console.log(`Validated ${fixtureName} against ${schemaName}`);
+}
+
+let rejected = false;
+try {
+  validate(readJson(path.join(schemaDir, "output.schema.json")), readJson(path.join(fixtureDir, "invalid-output-extra-field.json")), "invalid fixture");
+} catch {
+  rejected = true;
+}
+if (!rejected) throw new Error("invalid fixture was accepted");
+console.log("Rejected invalid-output-extra-field.json as expected");

scripts/validate_contract_fixtures.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python3
+"""Validate golden contract fixtures against the versioned JSON Schemas."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any, NoReturn
+
+SCHEMA_DIR = Path("lib/python/base_cli/schemas/v1")
+FIXTURE_DIR = Path("tests/fixtures/contracts")
+FIXTURE_SCHEMAS = {
+    "output-success.json": "output.schema.json",
+    "error-usage.json": "error.schema.json",
+    "inspection-warn.json": "inspection.schema.json",
+    "log-record.json": "log.schema.json",
+    "ndjson-record.json": "ndjson.schema.json",
+    "command-protocol.json": "command-protocol.schema.json",
+}
+
+
+def fail(message: str) -> NoReturn:
+    print(f"contract fixture validation failed: {message}", file=sys.stderr)
+    raise SystemExit(1)
+
+
+def load_json(path: Path) -> Any:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        fail(f"cannot parse {path}: {exc}")
+
+
+def _matches_type(value: Any, schema_type: str) -> bool:
+    return {
+        "object": isinstance(value, dict),
+        "array": isinstance(value, list),
+        "string": isinstance(value, str),
+        "integer": isinstance(value, int) and not isinstance(value, bool),
+        "boolean": isinstance(value, bool),
+        "null": value is None,
+    }.get(schema_type, True)
+
+
+def validate_instance(schema: dict[str, Any], value: Any, label: str) -> None:
+    if schema.get("type") == "object" and not isinstance(value, dict):
+        fail(f"{label} must be an object")
+    for field in schema.get("required", []):
+        if field not in value:
+            fail(f"{label} is missing required field {field!r}")
+    if schema.get("additionalProperties") is False:
+        unknown = set(value) - set(schema.get("properties", {}))
+        if unknown:
+            fail(f"{label} has unexpected fields: {', '.join(sorted(unknown))}")
+    for field, field_schema in schema.get("properties", {}).items():
+        if field not in value:
+            continue
+        field_value = value[field]
+        if "const" in field_schema and field_value != field_schema["const"]:
+            fail(f"{label}.{field} does not equal its contract constant")
+        allowed = field_schema.get("type")
+        if isinstance(allowed, str):
+            allowed = [allowed]
+        if isinstance(allowed, list) and not any(_matches_type(field_value, item) for item in allowed):
+            fail(f"{label}.{field} has an invalid type")
+        if "enum" in field_schema and field_value not in field_schema["enum"]:
+            fail(f"{label}.{field} is outside its contract enum")
+
+
+def validate(root: Path) -> None:
+    for fixture_name, schema_name in FIXTURE_SCHEMAS.items():
+        schema = load_json(root / SCHEMA_DIR / schema_name)
+        fixture = load_json(root / FIXTURE_DIR / fixture_name)
+        if not isinstance(schema, dict) or not isinstance(fixture, dict):
+            fail(f"{fixture_name} and {schema_name} must contain JSON objects")
+        validate_instance(schema, fixture, fixture_name)
+        print(f"Validated {fixture_name} against {schema_name}")
+
+    invalid = load_json(root / FIXTURE_DIR / "invalid-output-extra-field.json")
+    output_schema = load_json(root / SCHEMA_DIR / "output.schema.json")
+    try:
+        validate_instance(output_schema, invalid, "invalid-output-extra-field.json")
+    except SystemExit:
+        print("Rejected invalid-output-extra-field.json as expected")
+        return
+    fail("invalid-output-extra-field.json was accepted")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("root", type=Path, nargs="?", default=Path(__file__).resolve().parents[1])
+    validate(parser.parse_args().root)
+
+
+if __name__ == "__main__":
+    main()

tests/fixtures/contracts/command-protocol.json

@@ -0,0 +1,6 @@
+{
+  "protocol_header": "COMMAND_PROTOCOL_V1",
+  "record_type": "status",
+  "record_count": 1,
+  "records": [{"name": "ready", "ok": true}]
+}

tests/fixtures/contracts/error-usage.json

@@ -0,0 +1,9 @@
+{
+  "schema_version": 1,
+  "schema": "base-cli.error",
+  "code": "usage_error",
+  "type": "error",
+  "message": "Usage error",
+  "details": {"exit_code": 2, "stdout": ""},
+  "run_id": null
+}

tests/fixtures/contracts/inspection-warn.json

@@ -0,0 +1,7 @@
+{
+  "schema_version": 1,
+  "command": "doctor",
+  "status": "warn",
+  "data": {"checks": 3},
+  "error": null
+}

tests/fixtures/contracts/invalid-output-extra-field.json

@@ -0,0 +1,10 @@
+{
+  "schema_version": 1,
+  "schema": "base-cli.output",
+  "code": "ok",
+  "type": "success",
+  "message": "Success",
+  "details": {},
+  "run_id": null,
+  "unexpected": true
+}

tests/fixtures/contracts/log-record.json

@@ -0,0 +1,9 @@
+{
+  "schema_version": 1,
+  "schema": "base-cli.log",
+  "timestamp": "2026-08-28T07:00:00.000Z",
+  "level": "INFO",
+  "logger": "demo",
+  "message": "completed",
+  "run_id": "run-log"
+}

tests/fixtures/contracts/ndjson-record.json

@@ -0,0 +1,5 @@
+{
+  "schema_version": 1,
+  "schema": "base-cli.record",
+  "record": {"name": "base", "count": 1}
+}

tests/fixtures/contracts/output-success.json

@@ -0,0 +1,9 @@
+{
+  "schema_version": 1,
+  "schema": "base-cli.output",
+  "code": "ok",
+  "type": "success",
+  "message": "Success",
+  "details": {"exit_code": 0, "stdout": "hello\n"},
+  "run_id": "run-success"
+}

tests/test_validate_schemas.py

@@ -7,13 +7,16 @@
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
-from scripts import validate_schemas
+from scripts import validate_contract_fixtures, validate_schemas
 
 
 class SchemaValidationTests(unittest.TestCase):
     def test_repository_schemas_are_valid_and_in_sync(self) -> None:
         validate_schemas.validate(Path(__file__).resolve().parents[1])
 
+    def test_golden_contract_fixtures_are_valid(self) -> None:
+        validate_contract_fixtures.validate(Path(__file__).resolve().parents[1])
+
     def test_schema_drift_is_rejected(self) -> None:
         with tempfile.TemporaryDirectory() as tmpdir:
             root = Path(tmpdir)