From 11c5d9a63dd00dc55b7239be200726be0711743f Mon Sep 17 00:00:00 2001 From: Ario Barin Ostovary Date: Tue, 4 Aug 2026 13:48:48 -0400 Subject: [PATCH] validate selected portable definitions --- scripts/README.md | 1 + scripts/test-all.ps1 | 66 ++++++++++++ scripts/validate-portable-definitions.py | 129 +++++++++++++++++++++++ 3 files changed, 196 insertions(+) create mode 100644 scripts/validate-portable-definitions.py diff --git a/scripts/README.md b/scripts/README.md index 82587155..da7a01fe 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,6 +2,7 @@ - `install.ps1` previews or installs the manifest allowlist. - `verify-live.ps1` checks selected sources against live agent homes. +- `validate-portable-definitions.py` checks selected skill and agent identities. - `test-all.ps1` runs one focused install and preservation round trip. `common.ps1` provides shared manifest validation, mapping, path safety, copy, diff --git a/scripts/test-all.ps1 b/scripts/test-all.ps1 index 1670d4be..fd9fb8d9 100644 --- a/scripts/test-all.ps1 +++ b/scripts/test-all.ps1 @@ -36,12 +36,77 @@ function Assert-TestFileContains { } } +function Invoke-PortableDefinitionValidation { + param([string]$Root) + + $python = Get-Command python -ErrorAction SilentlyContinue + if (-not $python) { + $python = Get-Command python3 -ErrorAction SilentlyContinue + } + if (-not $python) { + throw "Python 3.11 or newer is required to validate portable definitions" + } + + $validator = Join-Path $PSScriptRoot "validate-portable-definitions.py" + $previousErrorAction = $ErrorActionPreference + try { + $ErrorActionPreference = "Continue" + $output = @(& $python.Source $validator $Root 2>&1) + $exitCode = $LASTEXITCODE + } + finally { + $ErrorActionPreference = $previousErrorAction + } + return [pscustomobject]@{ + ExitCode = $exitCode + Output = $output -join "`n" + } +} + $testRoot = Join-Path ([System.IO.Path]::GetTempPath()) "compass-test-$([guid]::NewGuid().ToString('N'))" $codexHome = Join-Path $testRoot "codex" $agentsHome = Join-Path $testRoot "agents" $claudeHome = Join-Path $testRoot "claude" try { + $repoRoot = Split-Path -Parent $PSScriptRoot + $invalidRoot = Join-Path $testRoot "invalid-definitions" + Write-TestFile -Path (Join-Path $invalidRoot "manifests\portable-files.json") -Content @" +{ + "codex": { "agents": ["broken-agent"] }, + "agents": { "skills": ["broken-skill"] }, + "claude": { "skills": [], "agents": [] } +} +"@ + Write-TestFile -Path (Join-Path $invalidRoot "codex\skills\broken-skill\SKILL.md") -Content @" +--- +name: wrong-name +description: Deliberately invalid fixture. +--- +"@ + Write-TestFile -Path (Join-Path $invalidRoot "codex\agents\broken-agent.toml") -Content @" +name = ["not", "a", "string"] +description = "Deliberately invalid fixture." +developer_instructions = "Do nothing." +"@ + $invalidDefinitions = Invoke-PortableDefinitionValidation -Root $invalidRoot + if ($invalidDefinitions.ExitCode -eq 0) { + throw "portable definition validation accepted invalid definitions" + } + foreach ($expected in @( + "Codex agent broken-agent: name must be a non-empty string", + "shared skill broken-skill: name must match its manifest entry" + )) { + if (-not $invalidDefinitions.Output.Contains($expected)) { + throw "portable definition validation did not report: $expected" + } + } + + $validDefinitions = Invoke-PortableDefinitionValidation -Root $repoRoot + if ($validDefinitions.ExitCode -ne 0) { + throw $validDefinitions.Output + } + $machineLabel = "preserve caf$([char]0x00E9)" Write-TestFile -Path (Join-Path $codexHome "AGENTS.md\local.txt") -Content "preserve this backup`n" Write-TestFile -Path (Join-Path $codexHome "auth.json") -Content "leave unlisted state alone`n" @@ -88,6 +153,7 @@ trust_level = "trusted" throw "expected one live config backup" } + Write-Host $validDefinitions.Output Write-Host "portable tests: ok" } finally { diff --git a/scripts/validate-portable-definitions.py b/scripts/validate-portable-definitions.py new file mode 100644 index 00000000..253314b3 --- /dev/null +++ b/scripts/validate-portable-definitions.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Validate the identity and syntax of manifest-selected definitions.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +try: + import tomllib +except ModuleNotFoundError: + print("portable definition validation requires Python 3.11 or newer", file=sys.stderr) + raise SystemExit(2) + + +def selected(manifest: dict[str, Any], section: str, key: str) -> list[str]: + table = manifest.get(section) + if not isinstance(table, dict): + raise ValueError(f"manifest {section} must be an object") + value = table.get(key) + if not isinstance(value, list) or any(not isinstance(item, str) for item in value): + raise ValueError(f"manifest {section}.{key} must be a string array") + return value + + +def frontmatter(path: Path) -> dict[str, str]: + lines = path.read_text(encoding="utf-8-sig").splitlines() + if not lines or lines[0] != "---": + raise ValueError("frontmatter must start on the first line") + try: + closing = lines.index("---", 1) + except ValueError as error: + raise ValueError("frontmatter is missing its closing delimiter") from error + + values: dict[str, str] = {} + for line in lines[1:closing]: + if not line.strip() or line.lstrip().startswith("#"): + continue + key, separator, value = line.partition(":") + key = key.strip() + value = value.strip() + if not separator or not key or not value: + raise ValueError(f"frontmatter entry is not a non-empty scalar: {line!r}") + if key in values: + raise ValueError(f"frontmatter field is duplicated: {key}") + if value[:1] in {'"', "'"}: + if len(value) < 2 or value[-1] != value[0]: + raise ValueError(f"frontmatter scalar has an unmatched quote: {key}") + value = value[1:-1].strip() + values[key] = value + return values + + +def validate_markdown(path: Path, expected_name: str, kind: str) -> list[str]: + try: + values = frontmatter(path) + except (OSError, UnicodeError, ValueError) as error: + return [f"{kind} {expected_name}: {error}"] + + errors = [] + if values.get("name") != expected_name: + errors.append(f"{kind} {expected_name}: name must match its manifest entry") + if not values.get("description"): + errors.append(f"{kind} {expected_name}: description must be a non-empty scalar") + return errors + + +def validate_agent(path: Path, expected_name: str) -> list[str]: + try: + with path.open("rb") as handle: + values = tomllib.load(handle) + except (OSError, tomllib.TOMLDecodeError) as error: + return [f"Codex agent {expected_name}: invalid TOML: {error}"] + + errors = [] + for field in ("name", "description", "developer_instructions"): + if not isinstance(values.get(field), str) or not values[field].strip(): + errors.append(f"Codex agent {expected_name}: {field} must be a non-empty string") + if isinstance(values.get("name"), str) and values["name"] != expected_name: + errors.append(f"Codex agent {expected_name}: name must match its manifest entry") + return errors + + +def validate(root: Path) -> list[str]: + manifest_path = root / "manifests" / "portable-files.json" + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8-sig")) + if not isinstance(manifest, dict): + raise ValueError("root must be an object") + codex_agents = selected(manifest, "codex", "agents") + shared_skills = selected(manifest, "agents", "skills") + claude_skills = selected(manifest, "claude", "skills") + claude_agents = selected(manifest, "claude", "agents") + except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as error: + return [f"portable manifest: {error}"] + + errors = [] + for name in codex_agents: + errors.extend(validate_agent(root / "codex" / "agents" / f"{name}.toml", name)) + for name in shared_skills: + path = root / "codex" / "skills" / name / "SKILL.md" + errors.extend(validate_markdown(path, name, "shared skill")) + for name in claude_skills: + path = root / "claude" / "skills" / name / "SKILL.md" + errors.extend(validate_markdown(path, name, "Claude skill")) + for name in claude_agents: + path = root / "claude" / "agents" / f"{name}.md" + errors.extend(validate_markdown(path, name, "Claude agent")) + return errors + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: validate-portable-definitions.py REPO_ROOT", file=sys.stderr) + return 2 + + errors = validate(Path(sys.argv[1]).resolve()) + if errors: + for error in errors: + print(f"error: {error}", file=sys.stderr) + return 1 + print("portable definitions: ok") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())