From e5a680b6258ade51c08de1b921662e7ddb18537f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 4 Sep 2026 04:57:53 +0000 Subject: [PATCH] test(cli): sweep every `os explain` catalog entry against its spec schema The `SCHEMAS` catalog in `packages/cli/src/commands/explain.ts` is hand-maintained and derives from nothing; until now one field of one entry was guarded by an exact-token assertion, and one entry's example was pinned by parsing it against the real schema. This generalises that parse technique across the whole catalog, deriving the entry set from `SCHEMAS` itself so a future entry cannot be added without being classified. Entries whose example does not parse today land as `it.fails` xfails naming the card filed for each; rewriting a catalog entry is operator-facing output and a separate review question, so nothing is fixed here. The two entries with no schema to parse against assert that reason rather than being silently absent. Also corrects a stale claim in this file's own comments: it is no longer outside every tsc program. `packages/cli/tsconfig.test.json` includes `test/**/*`, and this file carries no row in `test-typecheck-debt.json`, so a diagnostic it gains is red on arrival. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01D47qPfEWVPmhguWgBZCi5N --- packages/cli/test/commands.test.ts | 151 ++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 3 deletions(-) diff --git a/packages/cli/test/commands.test.ts b/packages/cli/test/commands.test.ts index d7c0db92a1..31d478239f 100644 --- a/packages/cli/test/commands.test.ts +++ b/packages/cli/test/commands.test.ts @@ -13,6 +13,18 @@ import Lint from '../src/commands/lint'; import Diff from '../src/commands/diff'; import Explain, { SCHEMAS } from '../src/commands/explain'; import { FlowSchema } from '@objectstack/spec/automation'; +// The catalog sweep below resolves each entry's schema BY NAME, so it needs the +// name surface rather than one binding. `@objectstack/spec`'s root exports none +// of these Zod schemas (measured: 129 root exports, no `ObjectSchema` / +// `FieldSchema` / … among them) — every one lives on a subpath, so the four +// metadata-authoring subpaths the catalog draws on are named here. The named +// `FlowSchema` import above stays: a value import fails loudly on a broken +// export where a namespace property read would degrade to `undefined`, and the +// sweep pays for its namespace form with an explicit resolvability assertion. +import * as specAi from '@objectstack/spec/ai'; +import * as specAutomation from '@objectstack/spec/automation'; +import * as specData from '@objectstack/spec/data'; +import * as specUi from '@objectstack/spec/ui'; describe('CLI Commands (oclif)', () => { it('should have compile command', () => { @@ -118,9 +130,13 @@ describe('os explain — schema catalog accuracy', () => { // drift — it re-derives the truth from the spec on every run, which is what // the hand-maintained catalog otherwise has no way to do. // The catalog's element shape, stated locally: `SchemaInfo` is not exported, - // and these tests must stay honest even where `SCHEMAS` widens to `any` - // (this file sits outside every tsc program — see the TEST_DEBT ledger — so - // an implicit `any` here would silently stop checking anything). + // and these tests must stay honest even where `SCHEMAS` widens to `any`. + // ⚠️ The reason recorded here has CHANGED and the discipline has not. This + // file no longer sits outside every tsc program: #14710 landed + // `packages/cli/tsconfig.test.json`, whose `include: ["test/**/*"]` puts this + // file in the program (`tsc --noEmit --listFiles -p tsconfig.test.json` + // resolves it), and it carries NO row in `test-typecheck-debt.json` — so any + // diagnostic it gains is red on arrival rather than silently unchecked. type CatalogField = { name: string; type: string }; const flowFields = (kind: 'required' | 'optional'): CatalogField[] => SCHEMAS.flow[kind]; @@ -161,3 +177,132 @@ describe('os explain — schema catalog accuracy', () => { expect(declared).toContain('edges'); }); }); + +// ── `os explain` — the WHOLE catalog, swept against the spec (#14811) ────── +// +// #14782 pinned one entry (`flow`) by parsing its `example` against the real +// schema. This generalises that technique to every entry, and derives the entry +// set from `SCHEMAS` itself: a hand-written list of entries is precisely the +// place a future entry escapes through unnoticed, which is the same defect this +// guard closes one level down. Add a catalog entry and this block goes RED +// until the entry is classified. +// +// ⛔ It does NOT fix what it turns red. Rewriting a catalog entry rewrites +// operator-facing output and is a separate change with a separate review +// question, so entries whose example does not parse today land as `it.fails` +// xfails naming the card filed for each. The day one is corrected its xfail +// fails ("expected to fail but passed") — promote it to a plain `it` then. +// +// ⛔ And it does not skip silently. Two entries resolve to no schema at all; +// they get tests that ASSERT that reason. A guard reporting green over the +// entries it never looked at is this card's own defect, one layer up. +describe('os explain — every catalog entry swept against its spec schema (#14811)', () => { + type CatalogEntry = { name: string; example: string }; + const catalog = SCHEMAS as unknown as Record; + + // The searched name surface, stated rather than assumed: absence below means + // absent from exactly these four subpaths. (`grep` over `packages/spec/src` + // finds no `export const TriggerSchema` or `WorkflowSchema` anywhere at all.) + const specSurface: Record = { + ...specData, + ...specUi, + ...specAi, + ...specAutomation, + }; + + type ParseResult = { success: boolean; error?: { issues: unknown[] } }; + type ZodLike = { safeParse: (value: unknown) => ParseResult }; + + // The catalog stores examples as authored source, so evaluate the literal — + // the same technique as the `flow` pin above. + const evaluate = (key: string): unknown => + new Function(`return (${catalog[key].example});`)() as unknown; + + // Entries with one schema to parse against. `card` marks a known-broken one + // and names where its errors are recorded; its absence means "must parse". + const BOUND: Record = { + object: { schema: 'ObjectSchema', card: 15170 }, + field: { schema: 'FieldSchema' }, + view: { schema: 'ViewSchema', card: 15171 }, + flow: { schema: 'FlowSchema' }, + agent: { schema: 'AgentSchema', card: 15172 }, + app: { schema: 'AppSchema', card: 15173 }, + query: { schema: 'QuerySchema' }, + dashboard: { schema: 'DashboardSchema', card: 15174 }, + action: { schema: 'ActionSchema', card: 15175 }, + }; + + // Entries with NO single schema to parse against, and the reason each of the + // two tests at the bottom asserts rather than merely states. + const UNBOUND: Record = { + workflow: + 'there is no standalone Workflow authoring type (ADR-0019) — the entry is a ' + + 'redirect and its example is commentary, not a literal', + trigger: + 'no `TriggerSchema` exists in the spec, and the sample is not a Hook either (#15176)', + }; + + it('classifies every entry in SCHEMAS — none is silently unswept', () => { + const classified = [...Object.keys(BOUND), ...Object.keys(UNBOUND)].sort(); + expect( + classified, + 'a new `os explain` catalog entry must be classified here: bind it to a spec ' + + 'schema, or give it an UNBOUND reason plus a test that asserts that reason', + ).toEqual(Object.keys(catalog).sort()); + }); + + // Harness health, asserted separately from the xfails: `it.fails` is green on + // ANY failure, so a broken subpath export or an unevaluable example would + // otherwise keep six xfails passing while measuring nothing at all. + it('resolves every bound entry to a real schema, and every bound example to an object', () => { + for (const [key, bound] of Object.entries(BOUND)) { + const schema = specSurface[bound.schema] as ZodLike | undefined; + expect(typeof schema?.safeParse, `${bound.schema} (for os explain ${key})`).toBe('function'); + expect(typeof evaluate(key), `os explain ${key} example`).toBe('object'); + } + }); + + for (const [key, bound] of Object.entries(BOUND)) { + const parses = (): void => { + const schema = specSurface[bound.schema] as ZodLike; + const result = schema.safeParse(evaluate(key)); + expect( + result.success, + `os explain ${key}: its example must parse as ${bound.schema}. Issues: ${ + result.success ? '' : JSON.stringify(result.error?.issues, null, 2) + }`, + ).toBe(true); + }; + + if (bound.card === undefined) { + it(`os explain ${key} — example parses as ${bound.schema}`, parses); + } else { + it.fails( + `os explain ${key} — example does NOT parse as ${bound.schema} ` + + `(known-broken, filed as #${bound.card}; promote to a plain assertion once fixed)`, + parses, + ); + } + } + + it(`os explain workflow — ${UNBOUND.workflow}`, () => { + expect('WorkflowSchema' in specSurface).toBe(false); + expect(catalog.workflow.name).toContain('no standalone type'); + // Its example is commentary about the live mechanisms, not a literal. + // Asserted, so "nothing was parsed here" is a property of this file rather + // than an omission a reader has to notice. + expect(() => evaluate('workflow')).toThrow(); + }); + + it(`os explain trigger — ${UNBOUND.trigger}`, () => { + expect('TriggerSchema' in specSurface).toBe(false); + // …and it is not `HookSchema` under another name. Ruling the one real + // candidate out is what makes the unbound classification a measurement + // instead of an assumption: `event` is a strict-object ALIAS of `events` + // (the same alias-as-a-documented-key failure the `flow` entry had), and a + // hook's code slot is `handler`, so the entry's `flow` key is unrecognised. + const hook = specSurface.HookSchema as ZodLike | undefined; + expect(typeof hook?.safeParse, 'HookSchema — the candidate this rules out').toBe('function'); + expect(hook!.safeParse(evaluate('trigger')).success).toBe(false); + }); +});