From 0750bc89d41baf7447dbba4b2874664f2d190c9d Mon Sep 17 00:00:00 2001 From: creayma Date: Wed, 29 Jul 2026 09:52:09 +0800 Subject: [PATCH] test(scripts): add platform support-declaration consistency tests The supported platform set (qoder, codex, claude, cursor, qwen, copilot) is declared independently in CLI help, the agent-customize provider registry, the session-analysis platform loader, the Harness report platform gate, the asset-baseline provider gate, and the host adapter matrix. Nothing kept those declarations in agreement, so adding or dropping a host in one owner silently left the others stale. Add test/support-declarations.test.mjs, which derives the declared list from each surface (help output, registry keys, fail-closed error messages, and documented module paths) through public routes only and asserts set equality against one canonical list. To let the contract test assert the report platform gate's whitelist rather than only that it rejects unknown input, extract the reportPlatform whitelist to a file-local REPORT_PLATFORMS constant and have its rejection error name the full supported set, matching the session-analysis and asset-baseline gates. This is the only runtime change; the error prefix and error.code are unchanged. This closes roadmap P0 item A-06. Story: roadmap.md TODO A-06 Spec: docs/specs/2026-07-28-a06-support-declaration-consistency.md Test: node --test test/support-declarations.test.mjs (6 pass) Co-authored-by: QoderAI (Qwen 3.8 Max) --- CHANGELOG.md | 7 +- ...-28-a06-support-declaration-consistency.md | 82 ++++++++++++ roadmap.md | 2 +- scripts/harness-analysis/report-run.mjs | 15 ++- test/support-declarations.test.mjs | 121 ++++++++++++++++++ 5 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 docs/specs/2026-07-28-a06-support-declaration-consistency.md create mode 100644 test/support-declarations.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 89cdcd3..6e6a1ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,12 @@ observable behavior and compatibility, not every internal refactor. ## Unreleased -No unreleased changes yet. +### Changed + +- The `harness analyze` platform gate now names the full supported set + (`qoder, codex, claude, cursor, qwen, copilot`) when it rejects an unsupported + `--platform`, matching the session-analysis and asset-baseline gates. The + existing error prefix and exit behavior are unchanged. ## 0.3.0 - 2026-07-27 diff --git a/docs/specs/2026-07-28-a06-support-declaration-consistency.md b/docs/specs/2026-07-28-a06-support-declaration-consistency.md new file mode 100644 index 0000000..e998114 --- /dev/null +++ b/docs/specs/2026-07-28-a06-support-declaration-consistency.md @@ -0,0 +1,82 @@ +# Support-declaration Consistency Tests + +Add a contract test that keeps the platform support declarations named by +the roadmap Definition of Done in agreement: CLI help, the agent-customize +provider registry, session-analysis platform loading, Harness report +platform gating, and the host adapter matrix. This is roadmap P0 item A-06. + +## Traceability + +- Spec ID: 2026-07-28-a06-support-declaration-consistency +- Story: roadmap.md TODO A-06 +- Status: Implemented + +## Intent + +The supported platform set (`qoder`, `codex`, `claude`, `cursor`, `qwen`, +`copilot`) is declared +independently in at least five places: + +- `scripts/session-analysis.mjs` and `scripts/session-analysis/analyzer.mjs` + help text and `loadPlatform` gates; +- `scripts/agent-customize/providers/index.mjs` `PROVIDER_COLLECTORS`; +- `scripts/harness-analysis/report-run.mjs` `ANALYZE_HELP` and the + `reportPlatform` whitelist (whose rejection now names the supported set, + matching the other gates); +- `scripts/coding-agent-practices/asset-baseline.mjs` provider gate; +- `docs/adapters/README.md` host adapter matrix rows. + +Nothing asserts these declarations agree. Adding or dropping a host in one +owner silently leaves the others stale, so CLI help, error messages, and docs +can advertise different platform sets. The roadmap Definition of Done requires +that "CLI help, provider registry, session platforms, report platforms, and +docs agree". + +## Acceptance + +- AC-1: A test derives the platform list each surface declares — help output, + registry keys, loader and gate error messages, and adapter matrix module + paths — and asserts every list equals the canonical supported set. +- AC-2: The test exercises public routes only: the root facade CLI, the + `scripts/session-analysis/index.mjs` and + `scripts/agent-customize/providers/index.mjs` import surfaces, and the + shipped docs file. No private helper is imported. +- AC-3: Unsupported platform input keeps failing closed on each gated route, + and the failure message names the full supported set. +- AC-4: `npm test` picks the test up automatically (default `node --test` + discovery under `test/`). + +## Non-goals + +- No change to which platforms are supported. +- No new shared runtime constant across modules; each capability keeps owning + its own declaration, and the test only proves the declarations agree. The + `reportPlatform` whitelist is extracted to a file-local `REPORT_PLATFORMS` + constant so its gate and error message share one source; `ANALYZE_HELP` + keeps its own prose list, which the contract test asserts separately. +- No assertions on prose wording beyond the declared platform lists. +- Other commands that declare their own platform or provider lists + (coding-agent-practices `inventory` and `asset-integrity`, + `agent-customize`, `agent-lint`, `evidence-bundle`, `task-loop-source`, + `selection-profile`) stay out of scope; the roadmap Definition of Done + names only the surfaces above. + +## Plan + +1. Add `test/support-declarations.test.mjs` with one canonical + `SUPPORTED_PLATFORMS` list and set-equality helpers. +2. Extend `reportPlatform` in `scripts/harness-analysis/report-run.mjs` so its + rejection error names the full supported set, matching the session-analysis + and asset-baseline gates, so the contract test can assert the declared list + rather than only that an unsupported value is rejected. +3. Cover the five surfaces: session-analysis CLI help plus its platform + gate, the exported `SESSION_ANALYSIS_HELP`, `harness analyze` help plus + its platform gate, `PROVIDER_COLLECTORS`, `createAnalyzer` rejection, + the asset-baseline provider gate, and the adapter matrix module + references. +4. Mark roadmap A-06 done. + +## Test Evidence + +- `node --test test/support-declarations.test.mjs` +- `node --test test/doc-link-graph.test.mjs` diff --git a/roadmap.md b/roadmap.md index c9308b4..d65ec71 100644 --- a/roadmap.md +++ b/roadmap.md @@ -46,7 +46,7 @@ platform depth. | [ ] | P0 | A-03 | Bind source references and provider-home paths to an explicit provider. | A patch cannot resolve into another host's configuration root. | | [x] | P0 | A-04 | Add a help-only path to `agent-customize`. | `--help` returns before reading HOME, workspace, SQLite, or plugin caches. | | [ ] | P0 | A-05 | Fix stale adapter documentation and smoke commands. | The matrix uses current `harness analyze` / `harness render` commands and does not overstate Cursor output support. | -| [ ] | P0 | A-06 | Add support-declaration consistency tests. | CLI help, provider registry, session platforms, report platforms, and docs agree. | +| [x] | P0 | A-06 | Add support-declaration consistency tests. | CLI help, provider registry, session platforms, report platforms, and docs agree. | | [ ] | P1 | C-01 | Add Codex-specific configuration source precedence. | Checkup distinguishes editable sources from cache, audit, and session data. | | [ ] | P1 | C-02 | Normalize Codex model, usage, and hook evidence when present. | Missing data stays unavailable; no model, token, or hook values are invented. | | [ ] | P1 | C-03 | Add a real Codex installation smoke. | Build, install, discover Skills, analyze, render HTML, validate, and reinstall all pass. | diff --git a/scripts/harness-analysis/report-run.mjs b/scripts/harness-analysis/report-run.mjs index 004ec57..6156f93 100644 --- a/scripts/harness-analysis/report-run.mjs +++ b/scripts/harness-analysis/report-run.mjs @@ -34,12 +34,19 @@ function clone(value) { return value === undefined ? undefined : JSON.parse(JSON.stringify(value)); } +const REPORT_PLATFORMS = ["qoder", "codex", "claude", "cursor", "qwen", "copilot"]; + function reportPlatform(value = "qoder") { const platform = String(value || "qoder").toLowerCase(); - if (!["qoder", "codex", "claude", "cursor", "qwen", "copilot"].includes(platform)) { - throw Object.assign(new Error(`unsupported Harness report platform: ${platform}`), { - code: "UNSUPPORTED_REPORT_PLATFORM", - }); + if (!REPORT_PLATFORMS.includes(platform)) { + throw Object.assign( + new Error( + `unsupported Harness report platform: ${platform}. Supported platforms: ${REPORT_PLATFORMS.join(", ")}.`, + ), + { + code: "UNSUPPORTED_REPORT_PLATFORM", + }, + ); } return platform; } diff --git a/test/support-declarations.test.mjs b/test/support-declarations.test.mjs new file mode 100644 index 0000000..6310547 --- /dev/null +++ b/test/support-declarations.test.mjs @@ -0,0 +1,121 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import test from "node:test"; + +import { PROVIDER_COLLECTORS } from "../scripts/agent-customize/providers/index.mjs"; +import { createAnalyzer, SESSION_ANALYSIS_HELP } from "../scripts/session-analysis/index.mjs"; + +// Canonical support declaration (roadmap A-06): CLI help, provider registry, +// session platforms, report platforms, and docs must all agree on this set. +const SUPPORTED_PLATFORMS = ["qoder", "codex", "claude", "cursor", "qwen", "copilot"]; + +const cliPath = path.join(process.cwd(), "scripts", "better-harness.mjs"); +const adapterMatrixPath = path.join(process.cwd(), "docs", "adapters", "README.md"); + +function runBetterHarness(args) { + return spawnSync(process.execPath, [cliPath, ...args], { + cwd: process.cwd(), + encoding: "utf8", + }); +} + +function sortedSet(values) { + return [...new Set(values)].sort(); +} + +function assertSameSet(actual, label) { + assert.deepEqual(sortedSet(actual), sortedSet(SUPPORTED_PLATFORMS), `${label} disagrees with the supported platform set`); +} + +test("agent-customize provider registry declares exactly the supported platforms", () => { + assertSameSet([...PROVIDER_COLLECTORS.keys()], "PROVIDER_COLLECTORS"); + + for (const platform of SUPPORTED_PLATFORMS) { + const providerModule = path.join(process.cwd(), "scripts", "agent-customize", "providers", `${platform}.mjs`); + assert.ok(existsSync(providerModule), `missing configured-asset provider module: ${providerModule}`); + } +}); + +test("session-analysis platform loader declares exactly the supported platforms", async () => { + for (const platform of SUPPORTED_PLATFORMS) { + const platformModule = path.join(process.cwd(), "scripts", "session-analysis", "platforms", `${platform}.mjs`); + assert.ok(existsSync(platformModule), `missing session platform module: ${platformModule}`); + } + + let message = ""; + try { + await createAnalyzer("__unsupported__"); + } catch (error) { + message = error.message; + } + const declared = message.match(/Supported platforms: ([a-z, ]+)\./u)?.[1]; + assert.ok(declared, `platform loader did not fail closed with a supported list: ${message}`); + assertSameSet(declared.split(", "), "session-analysis loadPlatform error"); + + const declaredHelp = SESSION_ANALYSIS_HELP.match(/--platform <([a-z|]+)>/u)?.[1]; + assert.ok(declaredHelp, `exported session-analysis help does not declare a platform list:\n${SESSION_ANALYSIS_HELP}`); + assertSameSet(declaredHelp.split("|"), "SESSION_ANALYSIS_HELP platform list"); +}); + +test("session-analysis CLI help and platform gate agree with the supported platforms", () => { + const result = runBetterHarness(["session-analysis", "--help"]); + assert.equal(result.status, 0, result.stderr); + + const declared = result.stdout.match(/--platform <([a-z|]+)>/u)?.[1]; + assert.ok(declared, `session-analysis help does not declare a platform list:\n${result.stdout}`); + assertSameSet(declared.split("|"), "session-analysis --help platform list"); + + const gated = runBetterHarness(["session-analysis", "sources", "--platform", "__unsupported__", "--workspace", "."]); + assert.notEqual(gated.status, 0, "session-analysis CLI accepted an unsupported platform"); + const gateDeclared = `${gated.stderr}${gated.stdout}`.match(/Supported platforms: ([a-z, ]+)\./u)?.[1]; + assert.ok(gateDeclared, `session-analysis CLI did not fail closed with a supported list:\n${gated.stderr}`); + assertSameSet(gateDeclared.split(", "), "session-analysis CLI platform gate"); +}); + +test("harness analyze help and platform gate agree with the supported platforms", () => { + const help = runBetterHarness(["harness", "analyze", "--help"]); + assert.equal(help.status, 0, help.stderr); + + const declared = help.stdout.match(/--platform \s+([a-z, ]+or [a-z]+)/u)?.[1]; + assert.ok(declared, `harness analyze help does not declare a platform list:\n${help.stdout}`); + assertSameSet(declared.match(/[a-z]+/gu).filter((word) => word !== "or"), "harness analyze --help platform list"); + + const gated = runBetterHarness(["harness", "analyze", "--platform", "__unsupported__", "--workspace", ".", "--format", "json"]); + assert.notEqual(gated.status, 0, "harness analyze accepted an unsupported platform"); + const gatedOutput = `${gated.stderr}${gated.stdout}`; + assert.match(gatedOutput, /unsupported Harness report platform/u); + const gateDeclared = gatedOutput.match(/Supported platforms: ([a-z, ]+)\./u)?.[1]; + assert.ok(gateDeclared, `harness analyze did not name the supported set on rejection:\n${gatedOutput}`); + assertSameSet(gateDeclared.split(", "), "harness analyze platform gate"); +}); + +test("asset-baseline provider gate lists exactly the supported platforms", () => { + const result = runBetterHarness(["coding-agent-practices", "asset-baseline", "__unsupported__", "--workspace", "."]); + assert.notEqual(result.status, 0, "asset-baseline accepted an unsupported provider"); + + const declared = `${result.stderr}${result.stdout}`.match(/Supported providers: ([a-z, ]+)\./u)?.[1]; + assert.ok(declared, `asset-baseline did not fail closed with a supported list:\n${result.stderr}`); + assertSameSet(declared.split(", "), "asset-baseline provider gate"); +}); + +test("host adapter matrix documents exactly the supported platforms", () => { + const matrix = readFileSync(adapterMatrixPath, "utf8"); + + for (const platform of SUPPORTED_PLATFORMS) { + assert.ok( + matrix.includes(`scripts/agent-customize/providers/${platform}.mjs`), + `adapter matrix is missing the configured-asset provider for ${platform}`, + ); + assert.ok( + matrix.includes(`scripts/session-analysis/platforms/${platform}.mjs`), + `adapter matrix is missing the session platform for ${platform}`, + ); + } + + const documentedProviders = [...matrix.matchAll(/agent-customize\/providers\/([a-z-]+)\.mjs/gu)].map((match) => match[1]); + const documentedPlatforms = [...matrix.matchAll(/session-analysis\/platforms\/([a-z-]+)\.mjs/gu)].map((match) => match[1]); + assertSameSet(documentedProviders, "adapter matrix configured-asset providers"); + assertSameSet(documentedPlatforms, "adapter matrix session platforms"); +});