diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6551c83..752f5fb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,6 +67,8 @@ jobs: python scripts/validate_docs.py python scripts/validate_changelog.py python scripts/validate_schemas.py + python scripts/validate_contract_fixtures.py + node scripts/validate_contract_fixtures.mjs python scripts/benchmark_runtime.py --check python -m compileall -q examples - name: Run tests with coverage threshold diff --git a/CHANGELOG.md b/CHANGELOG.md index 73bd6b0..d06f446 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,8 @@ and versions are tracked in the repo-root `VERSION` file. ### Added +- Add golden success, error, inspection, log, NDJSON, and command-protocol + fixtures with Python and Node.js validators for cross-language consumers. - Publish versioned JSON Schema artifacts for output, error, inspection, log, NDJSON, and decoded command-protocol contracts in the package and docs site. - Add a framework choice guide, five-minute evaluation path, and clearer diff --git a/docs/json-contracts.md b/docs/json-contracts.md index b75826f..9523f22 100644 --- a/docs/json-contracts.md +++ b/docs/json-contracts.md @@ -56,6 +56,12 @@ their own structured `details` records. Secret-looking keys (`token`, `password`, `secret`, `api_key`, and `authorization`) and credential-bearing URLs are redacted recursively. +Golden payloads for each public contract live in +[`tests/fixtures/contracts`](https://github.com/basefoundry/base-cli/tree/main/tests/fixtures/contracts). +CI validates them against the packaged schemas with both a Python validator and +a dependency-free Node.js reader; consumers can use the same fixtures as +cross-language conformance tests. + ## Inspection envelopes Read-only inspection commands can use the stable inspection helpers when their diff --git a/scripts/validate_contract_fixtures.mjs b/scripts/validate_contract_fixtures.mjs new file mode 100644 index 0000000..67d6669 --- /dev/null +++ b/scripts/validate_contract_fixtures.mjs @@ -0,0 +1,55 @@ +#!/usr/bin/env node +/** Validate golden contract fixtures with a non-Python reference reader. */ + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const schemaDir = path.join(root, "lib", "python", "base_cli", "schemas", "v1"); +const fixtureDir = path.join(root, "tests", "fixtures", "contracts"); +const fixtures = new Map([ + ["output-success.json", "output.schema.json"], + ["error-usage.json", "error.schema.json"], + ["inspection-warn.json", "inspection.schema.json"], + ["log-record.json", "log.schema.json"], + ["ndjson-record.json", "ndjson.schema.json"], + ["command-protocol.json", "command-protocol.schema.json"], +]); + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function validate(schema, value, label) { + if (schema.type === "object" && (value === null || Array.isArray(value) || typeof value !== "object")) { + throw new Error(`${label} must be an object`); + } + for (const field of schema.required ?? []) { + if (!Object.prototype.hasOwnProperty.call(value, field)) throw new Error(`${label} is missing ${field}`); + } + if (schema.additionalProperties === false) { + for (const field of Object.keys(value)) { + if (!Object.prototype.hasOwnProperty.call(schema.properties ?? {}, field)) throw new Error(`${label} has unexpected field ${field}`); + } + } + for (const [field, rule] of Object.entries(schema.properties ?? {})) { + if (!Object.prototype.hasOwnProperty.call(value, field)) continue; + if (Object.prototype.hasOwnProperty.call(rule, "const") && value[field] !== rule.const) throw new Error(`${label}.${field} constant mismatch`); + if (rule.enum && !rule.enum.includes(value[field])) throw new Error(`${label}.${field} enum mismatch`); + } +} + +for (const [fixtureName, schemaName] of fixtures) { + validate(readJson(path.join(schemaDir, schemaName)), readJson(path.join(fixtureDir, fixtureName)), fixtureName); + console.log(`Validated ${fixtureName} against ${schemaName}`); +} + +let rejected = false; +try { + validate(readJson(path.join(schemaDir, "output.schema.json")), readJson(path.join(fixtureDir, "invalid-output-extra-field.json")), "invalid fixture"); +} catch { + rejected = true; +} +if (!rejected) throw new Error("invalid fixture was accepted"); +console.log("Rejected invalid-output-extra-field.json as expected"); diff --git a/scripts/validate_contract_fixtures.py b/scripts/validate_contract_fixtures.py new file mode 100644 index 0000000..a32ef7f --- /dev/null +++ b/scripts/validate_contract_fixtures.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Validate golden contract fixtures against the versioned JSON Schemas.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, NoReturn + +SCHEMA_DIR = Path("lib/python/base_cli/schemas/v1") +FIXTURE_DIR = Path("tests/fixtures/contracts") +FIXTURE_SCHEMAS = { + "output-success.json": "output.schema.json", + "error-usage.json": "error.schema.json", + "inspection-warn.json": "inspection.schema.json", + "log-record.json": "log.schema.json", + "ndjson-record.json": "ndjson.schema.json", + "command-protocol.json": "command-protocol.schema.json", +} + + +def fail(message: str) -> NoReturn: + print(f"contract fixture validation failed: {message}", file=sys.stderr) + raise SystemExit(1) + + +def load_json(path: Path) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + fail(f"cannot parse {path}: {exc}") + + +def _matches_type(value: Any, schema_type: str) -> bool: + return { + "object": isinstance(value, dict), + "array": isinstance(value, list), + "string": isinstance(value, str), + "integer": isinstance(value, int) and not isinstance(value, bool), + "boolean": isinstance(value, bool), + "null": value is None, + }.get(schema_type, True) + + +def validate_instance(schema: dict[str, Any], value: Any, label: str) -> None: + if schema.get("type") == "object" and not isinstance(value, dict): + fail(f"{label} must be an object") + for field in schema.get("required", []): + if field not in value: + fail(f"{label} is missing required field {field!r}") + if schema.get("additionalProperties") is False: + unknown = set(value) - set(schema.get("properties", {})) + if unknown: + fail(f"{label} has unexpected fields: {', '.join(sorted(unknown))}") + for field, field_schema in schema.get("properties", {}).items(): + if field not in value: + continue + field_value = value[field] + if "const" in field_schema and field_value != field_schema["const"]: + fail(f"{label}.{field} does not equal its contract constant") + allowed = field_schema.get("type") + if isinstance(allowed, str): + allowed = [allowed] + if isinstance(allowed, list) and not any(_matches_type(field_value, item) for item in allowed): + fail(f"{label}.{field} has an invalid type") + if "enum" in field_schema and field_value not in field_schema["enum"]: + fail(f"{label}.{field} is outside its contract enum") + + +def validate(root: Path) -> None: + for fixture_name, schema_name in FIXTURE_SCHEMAS.items(): + schema = load_json(root / SCHEMA_DIR / schema_name) + fixture = load_json(root / FIXTURE_DIR / fixture_name) + if not isinstance(schema, dict) or not isinstance(fixture, dict): + fail(f"{fixture_name} and {schema_name} must contain JSON objects") + validate_instance(schema, fixture, fixture_name) + print(f"Validated {fixture_name} against {schema_name}") + + invalid = load_json(root / FIXTURE_DIR / "invalid-output-extra-field.json") + output_schema = load_json(root / SCHEMA_DIR / "output.schema.json") + try: + validate_instance(output_schema, invalid, "invalid-output-extra-field.json") + except SystemExit: + print("Rejected invalid-output-extra-field.json as expected") + return + fail("invalid-output-extra-field.json was accepted") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", type=Path, nargs="?", default=Path(__file__).resolve().parents[1]) + validate(parser.parse_args().root) + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/contracts/command-protocol.json b/tests/fixtures/contracts/command-protocol.json new file mode 100644 index 0000000..07d53ad --- /dev/null +++ b/tests/fixtures/contracts/command-protocol.json @@ -0,0 +1,6 @@ +{ + "protocol_header": "COMMAND_PROTOCOL_V1", + "record_type": "status", + "record_count": 1, + "records": [{"name": "ready", "ok": true}] +} diff --git a/tests/fixtures/contracts/error-usage.json b/tests/fixtures/contracts/error-usage.json new file mode 100644 index 0000000..c5def88 --- /dev/null +++ b/tests/fixtures/contracts/error-usage.json @@ -0,0 +1,9 @@ +{ + "schema_version": 1, + "schema": "base-cli.error", + "code": "usage_error", + "type": "error", + "message": "Usage error", + "details": {"exit_code": 2, "stdout": ""}, + "run_id": null +} diff --git a/tests/fixtures/contracts/inspection-warn.json b/tests/fixtures/contracts/inspection-warn.json new file mode 100644 index 0000000..0a52740 --- /dev/null +++ b/tests/fixtures/contracts/inspection-warn.json @@ -0,0 +1,7 @@ +{ + "schema_version": 1, + "command": "doctor", + "status": "warn", + "data": {"checks": 3}, + "error": null +} diff --git a/tests/fixtures/contracts/invalid-output-extra-field.json b/tests/fixtures/contracts/invalid-output-extra-field.json new file mode 100644 index 0000000..98a031c --- /dev/null +++ b/tests/fixtures/contracts/invalid-output-extra-field.json @@ -0,0 +1,10 @@ +{ + "schema_version": 1, + "schema": "base-cli.output", + "code": "ok", + "type": "success", + "message": "Success", + "details": {}, + "run_id": null, + "unexpected": true +} diff --git a/tests/fixtures/contracts/log-record.json b/tests/fixtures/contracts/log-record.json new file mode 100644 index 0000000..41899fa --- /dev/null +++ b/tests/fixtures/contracts/log-record.json @@ -0,0 +1,9 @@ +{ + "schema_version": 1, + "schema": "base-cli.log", + "timestamp": "2026-08-28T07:00:00.000Z", + "level": "INFO", + "logger": "demo", + "message": "completed", + "run_id": "run-log" +} diff --git a/tests/fixtures/contracts/ndjson-record.json b/tests/fixtures/contracts/ndjson-record.json new file mode 100644 index 0000000..7ce62bd --- /dev/null +++ b/tests/fixtures/contracts/ndjson-record.json @@ -0,0 +1,5 @@ +{ + "schema_version": 1, + "schema": "base-cli.record", + "record": {"name": "base", "count": 1} +} diff --git a/tests/fixtures/contracts/output-success.json b/tests/fixtures/contracts/output-success.json new file mode 100644 index 0000000..22d7833 --- /dev/null +++ b/tests/fixtures/contracts/output-success.json @@ -0,0 +1,9 @@ +{ + "schema_version": 1, + "schema": "base-cli.output", + "code": "ok", + "type": "success", + "message": "Success", + "details": {"exit_code": 0, "stdout": "hello\n"}, + "run_id": "run-success" +} diff --git a/tests/test_validate_schemas.py b/tests/test_validate_schemas.py index d871851..e53f5bc 100644 --- a/tests/test_validate_schemas.py +++ b/tests/test_validate_schemas.py @@ -7,13 +7,16 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from scripts import validate_schemas +from scripts import validate_contract_fixtures, validate_schemas class SchemaValidationTests(unittest.TestCase): def test_repository_schemas_are_valid_and_in_sync(self) -> None: validate_schemas.validate(Path(__file__).resolve().parents[1]) + def test_golden_contract_fixtures_are_valid(self) -> None: + validate_contract_fixtures.validate(Path(__file__).resolve().parents[1]) + def test_schema_drift_is_rejected(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: root = Path(tmpdir)