From 747dcf4d0090cee71f989f1fa990ec84d6989489 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 04:24:06 +0000 Subject: [PATCH 1/5] wip: per-branch json-schema projection --- packages/spec/authorable-defaults/data.json | 4 + packages/spec/authorable-surface/data.json | 48 +++ packages/spec/json-schema.manifest/data.json | 5 + packages/spec/scripts/build-schemas.ts | 87 ++++- .../scripts/lib/union-branch-projection.ts | 308 ++++++++++++++++++ packages/spec/unemitted-schemas.baseline.json | 36 +- 6 files changed, 456 insertions(+), 32 deletions(-) create mode 100644 packages/spec/scripts/lib/union-branch-projection.ts diff --git a/packages/spec/authorable-defaults/data.json b/packages/spec/authorable-defaults/data.json index 8d23d58d5d..7ac78fc74f 100644 --- a/packages/spec/authorable-defaults/data.json +++ b/packages/spec/authorable-defaults/data.json @@ -51,6 +51,10 @@ "data/FullTextSearch:fuzzy = false", "data/FullTextSearch:highlight = false", "data/FullTextSearch:operator = \"or\"", + "data/Hook:async = false", + "data/Hook:onError = \"abort\"", + "data/Hook:priority = 100", + "data/Hook:runAs = \"inherit\"", "data/ImportFieldMapping:transform = \"none\"", "data/Index:unique = false", "data/JSONValidation:active = true", diff --git a/packages/spec/authorable-surface/data.json b/packages/spec/authorable-surface/data.json index 5e552d2b60..5eb5fd5920 100644 --- a/packages/spec/authorable-surface/data.json +++ b/packages/spec/authorable-surface/data.json @@ -41,6 +41,10 @@ "data/AutoPersistenceConfig:path", "data/AutoPersistenceConfig:type", "data/BaseEngineOptions:context", + "data/ComparisonOperator:$gt", + "data/ComparisonOperator:$gte", + "data/ComparisonOperator:$lt", + "data/ComparisonOperator:$lte", "data/ConditionalValidation:_lock", "data/ConditionalValidation:_lockDocsUrl", "data/ConditionalValidation:_lockReason", @@ -415,6 +419,24 @@ "data/Field:widget", "data/FieldMaskingKeep:keepHead", "data/FieldMaskingKeep:keepTail", + "data/FieldOperators:$between", + "data/FieldOperators:$contains", + "data/FieldOperators:$endsWith", + "data/FieldOperators:$eq", + "data/FieldOperators:$exists", + "data/FieldOperators:$gt", + "data/FieldOperators:$gte", + "data/FieldOperators:$icontains", + "data/FieldOperators:$ilike", + "data/FieldOperators:$in", + "data/FieldOperators:$like", + "data/FieldOperators:$lt", + "data/FieldOperators:$lte", + "data/FieldOperators:$ne", + "data/FieldOperators:$nin", + "data/FieldOperators:$notContains", + "data/FieldOperators:$null", + "data/FieldOperators:$startsWith", "data/FieldReference:$field", "data/FieldReference:addDays", "data/FilePersistenceConfig:autoSaveInterval [RETIRED]", @@ -455,6 +477,28 @@ "data/FullTextSearch:minScore", "data/FullTextSearch:operator", "data/FullTextSearch:query", + "data/Hook:_lock", + "data/Hook:_lockDocsUrl", + "data/Hook:_lockReason", + "data/Hook:_lockSource", + "data/Hook:_packageId", + "data/Hook:_packageVersion", + "data/Hook:_provenance", + "data/Hook:async", + "data/Hook:body", + "data/Hook:condition", + "data/Hook:description", + "data/Hook:events", + "data/Hook:handler", + "data/Hook:label", + "data/Hook:name", + "data/Hook:object", + "data/Hook:onError", + "data/Hook:priority", + "data/Hook:retryPolicy", + "data/Hook:runAs", + "data/Hook:timeout [RETIRED]", + "data/Hook:timeoutMs", "data/HookContext:api", "data/HookContext:dispatch", "data/HookContext:event", @@ -618,6 +662,9 @@ "data/NoSQLTransactionOptions:readConcern", "data/NoSQLTransactionOptions:readPreference", "data/NoSQLTransactionOptions:writeConcern", + "data/NormalizedFilter:$and", + "data/NormalizedFilter:$not", + "data/NormalizedFilter:$or", "data/Object:_lock", "data/Object:_lockDocsUrl", "data/Object:_lockReason", @@ -736,6 +783,7 @@ "data/Query:where", "data/Query:windowFunctions [RETIRED]", "data/QueryFilter:where", + "data/RangeOperator:$between", "data/ReferenceResolution:field", "data/ReferenceResolution:fieldType", "data/ReferenceResolution:multiple", diff --git a/packages/spec/json-schema.manifest/data.json b/packages/spec/json-schema.manifest/data.json index f243440cde..8fa311f473 100644 --- a/packages/spec/json-schema.manifest/data.json +++ b/packages/spec/json-schema.manifest/data.json @@ -18,6 +18,7 @@ "data/BaseEngineOptions", "data/CalendarDateValue", "data/ClockTimeValue", + "data/ComparisonOperator", "data/ConditionalValidation", "data/ConsistencyLevel", "data/ContextToken", @@ -81,6 +82,7 @@ "data/FieldMaskingKeep", "data/FieldMaskingRule", "data/FieldNode", + "data/FieldOperators", "data/FieldReference", "data/FieldType", "data/FileLikeValue", @@ -92,6 +94,7 @@ "data/FormatValidation", "data/FullTextSearch", "data/GroupByNode", + "data/Hook", "data/HookBody", "data/HookBodyCapability", "data/HookContext", @@ -118,6 +121,7 @@ "data/NoSQLOperationType", "data/NoSQLQueryOptions", "data/NoSQLTransactionOptions", + "data/NormalizedFilter", "data/Object", "data/ObjectAccessConfig", "data/ObjectCapabilities", @@ -134,6 +138,7 @@ "data/PostgresConfig", "data/Query", "data/QueryFilter", + "data/RangeOperator", "data/ReferenceIdValue", "data/ReferenceResolution", "data/ReferenceResolutionError", diff --git a/packages/spec/scripts/build-schemas.ts b/packages/spec/scripts/build-schemas.ts index 24c2023ec3..c15199abb8 100644 --- a/packages/spec/scripts/build-schemas.ts +++ b/packages/spec/scripts/build-schemas.ts @@ -36,6 +36,10 @@ import { readUnemittedBaseline, type UnemittedSkip, } from './lib/unemitted-schemas'; +import { + projectByPruningUnionBranches, + type PrunedBranch, +} from './lib/union-branch-projection'; // Who owns what under json-schema/. This generator shares that directory with // gen:openapi, and used to clear it by deleting the directory itself (#5371). import { @@ -374,6 +378,19 @@ const emittedDefs: EmittedDef[] = []; const exportedZodKeys = new Set(); const unemittedSkips: UnemittedSkip[] = []; +// Every export that reached a published surface only because the projection +// dropped a union branch with no JSON form (#16431 (a)). Recorded here — and +// printed in the summary, and written onto the artifact itself as +// `x-unprojectable-branches` — because a schema that is NARROWER than its Zod +// type with nothing saying so is the same silence this card was filed about, +// one level down: the difference would otherwise live only in the two files +// nobody diffs against each other. +const branchPrunedProjections: Array<{ + readonly namespace: string; + readonly exportKey: string; + readonly pruned: readonly PrunedBranch[]; +}> = []; + // Error messages for schema types that inherently cannot be represented in JSON Schema. // These are expected warnings, not build-breaking errors. const KNOWN_UNSUPPORTED_PATTERNS = [ @@ -421,6 +438,7 @@ for (const [namespaceName, namespaceExports] of Object.entries(Protocol)) { // schema (that's how PageTabsProps vanished in #2967 — see #2978). let jsonSchema: Record; let io: 'output' | 'input' = 'output'; + let prunedBranches: readonly PrunedBranch[] = []; try { jsonSchema = z.toJSONSchema(value, { target: 'draft-2020-12', @@ -428,12 +446,32 @@ for (const [namespaceName, namespaceExports] of Object.entries(Protocol)) { } catch (outputError) { if (!isKnownUnsupported(outputError)) throw outputError; io = 'input'; - // Throws again for types unrepresentable in either direction - // (functions, Date, BigInt, custom) — caught by the outer skip. - jsonSchema = z.toJSONSchema(value, { - target: 'draft-2020-12', - io: 'input', - }) as Record; + try { + jsonSchema = z.toJSONSchema(value, { + target: 'draft-2020-12', + io: 'input', + }) as Record; + } catch (inputError) { + if (!isKnownUnsupported(inputError)) throw inputError; + // THIRD attempt, #16431 (a): both directions above refuse the + // WHOLE schema over one node, so try the projection that drops + // union branches with no JSON form. `orderingComparandSchema` is + // `number | Date | string | { $field }`; a JSON document can + // carry three of those and no Date instance, so the union's set + // of valid JSON documents is unchanged by the drop while the + // reference section it was costing is not. + // + // ⛔ Returns null unless the projection is faithful — a marked + // node outside a union position, or nothing to drop. The skip is + // then re-thrown with the message Zod produced, so this attempt + // can never change WHY an export is skipped, and so never the + // `cause` recorded for it in unemitted-schemas.baseline.json. + const projected = projectByPruningUnionBranches(value, { target: 'draft-2020-12' }); + if (!projected) throw inputError; + jsonSchema = projected.schema; + io = projected.io; + prunedBranches = projected.pruned; + } } // Add $id URL and version metadata for IDE autocomplete and schema resolution @@ -445,6 +483,19 @@ for (const [namespaceName, namespaceExports] of Object.entries(Protocol)) { // shape — parse-time transforms/defaults are not applied in it. jsonSchema['x-io'] = 'input'; } + if (prunedBranches.length > 0) { + // Say it on the artifact, not only in the build log (#16431 (a)). + // A reader of this file — or of the reference page rendered from it + // — can otherwise not tell that the Zod type carries a branch no + // JSON document could ever satisfy, and the `.describe()` prose + // above the union DOES name it (the ordering comparand's text says + // "a number, a Date, a string, or a { $field } reference"). + jsonSchema['x-unprojectable-branches'] = prunedBranches.map((branch) => ({ + at: branch.at, + type: branch.type, + })); + branchPrunedProjections.push({ namespace: namespaceName, exportKey: key, pruned: prunedBranches }); + } const fileName = `${schemaName}.json`; const filePath = path.join(categoryDir, fileName); @@ -482,7 +533,29 @@ for (const [namespaceName, namespaceExports] of Object.entries(Protocol)) { console.log(`\n─── Summary ───`); console.log(` Generated: ${count}${inputModeCount > 0 ? ` (${inputModeCount} as input shape)` : ''}`); if (skippedCount > 0) { - console.log(` Skipped: ${skippedCount} (unsupported types: function, date, bigint, custom)`); + // The cause list is DERIVED from this build's own skips (#16431 (a)). It was + // a hard-coded `function, date, bigint, custom` — a fixed list that had + // stopped describing the tree: `bigint` was in it and is not a cause here, + // `undefined` is a cause here and was not in it. A summary line that names + // the wrong families is worse than a bare count, because it reads as a + // measurement. + const causeList = [...countByCause(unemittedSkips).keys()].join(', '); + console.log(` Skipped: ${skippedCount} (unsupported types: ${causeList})`); +} +if (branchPrunedProjections.length > 0) { + // Never a silent widening: every entry here published ONLY the branches that + // have a JSON form, and each dropped branch is named on the artifact too. + const droppedCount = branchPrunedProjections.reduce((sum, p) => sum + p.pruned.length, 0); + console.log( + ` Projected: ${branchPrunedProjections.length} export(s) after dropping ` + + `${droppedCount} union branch(es) with no JSON form (#16431):`, + ); + for (const projection of branchPrunedProjections) { + const types = [...new Set(projection.pruned.map((b) => b.type))].sort().join(', '); + console.log( + ` ⤷ ${projection.namespace}.${projection.exportKey} — dropped ${projection.pruned.length} (${types})`, + ); + } } if (errorCount > 0) { diff --git a/packages/spec/scripts/lib/union-branch-projection.ts b/packages/spec/scripts/lib/union-branch-projection.ts new file mode 100644 index 0000000000..1b23049dcf --- /dev/null +++ b/packages/spec/scripts/lib/union-branch-projection.ts @@ -0,0 +1,308 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Per-BRANCH JSON-Schema projection — option (a) of #16431. + * + * ## The gap this closes + * + * `build-schemas.ts` projects each exported `z.ZodType` with + * `z.toJSONSchema()`, and Zod refuses a whole schema the moment ONE node in it + * has no JSON form. The refusal is per SCHEMA, so a union of four comparand + * shapes — three of them plain JSON — publishes nothing because the fourth is a + * `z.date()`. That is how `$gt` / `$gte` / `$lt` / `$lte` / `$between` reached + * no reference row at all: `orderingComparandSchema` is + * `z.union([z.number(), z.date(), z.string(), FieldReferenceSchema])`. + * + * ## What is actually true about a `z.date()` branch, and why dropping it is + * not a narrowing + * + * These artifacts describe **JSON documents** — what an author writes in + * `.yml` / `.json` metadata and what travels on the wire. A JSON document + * cannot carry a `Date` INSTANCE; the ordering docblock in + * `src/data/filter.zod.ts` states the same thing from the runtime side ("the + * DRIVER only ever sees ISO date / timestamp strings"). So the set of JSON + * documents the `z.date()` branch admits is EMPTY, and an empty branch of an + * `anyOf` contributes nothing to the union's accept set. Removing it changes + * which JSON documents validate by exactly nothing, while the difference + * between publishing and not publishing is the whole reference section. + * + * The same argument holds for every other unprojectable node in a direct union + * position — a `z.function()` member of a `functions:` entry union is a live + * callable, and no JSON document is one. + * + * ⛔ It does NOT hold for an unprojectable node anywhere else. An object + * property typed `z.function()` is REQUIRED of every document the object + * accepts, so dropping it would publish a shape no runtime value has. This + * module therefore prunes **only direct members of `anyOf` / `oneOf`** and + * refuses the projection entirely when a marked node survives anywhere else — + * in which case the caller skips the export exactly as it does today, with + * today's message. + * + * ## Why the mechanism is `unrepresentable: 'any'` + `override`, and not the + * `io: 'input'` retry the card guessed at + * + * #16431 hypothesised that the existing `io: 'input'` fallback would project a + * date branch if it were applied per branch rather than per schema, because an + * author writes an ISO string. Measured against zod 4.4.3, that is false and + * the reason is structural: `dateProcessor` + * (`zod/v4/core/json-schema-processors`) reads only `ctx.unrepresentable` and + * never `ctx.io`, so `z.toJSONSchema(z.date(), { io: 'input' })` throws exactly + * as the output direction does — at any granularity, branch or schema. The + * only switch Zod offers is `unrepresentable: 'any'`, which turns EVERY + * unrepresentable node into `{}`. + * + * ⛔ A bare `{}` is the dangerous answer, not the safe one: inside an `anyOf` + * it accepts every JSON value, so a build that simply set `unrepresentable: + * 'any'` would publish universally-permissive schemas for the 23 exports that + * are skipped today and report nothing. This module uses that mode only as a + * VEHICLE: every node that came back with no structural keyword is marked, + * marked members of a union are dropped, and a mark surviving anywhere else + * fails the projection. Nothing is ever emitted with a `{}` standing in for a + * type Zod refused. + */ +import { z } from 'zod'; + +/** + * Temporary marker key written onto a node Zod could not project. It never + * reaches an emitted file: a marked node is either dropped with its union + * branch, or its survival fails the whole projection. + */ +export const UNPROJECTABLE_MARK = 'x-os-unprojectable'; + +/** + * Keys that annotate a schema without constraining any value. A node carrying + * only these constrains nothing — which for anything but `z.any()` / + * `z.unknown()` means the processor produced no projection for it. + * + * `.describe()` lands on the node BEFORE `override` runs, so a + * `z.date().describe('…')` comes back as `{ description: '…' }` — non-empty, + * and it would escape a plain `Object.keys().length === 0` test. + */ +const ANNOTATION_ONLY_KEYS: ReadonlySet = new Set([ + '$comment', + '$id', + '$schema', + 'default', + 'deprecated', + 'description', + 'examples', + 'readOnly', + 'title', + 'writeOnly', +]); + +/** The union keywords whose members are alternatives, and so are droppable. */ +const UNION_KEYWORDS = ['anyOf', 'oneOf'] as const; + +/** One union branch this projection dropped, and what it was. */ +export interface PrunedBranch { + /** JSON Pointer to the branch in the UNPRUNED projection. */ + readonly at: string; + /** The Zod type that has no JSON form, e.g. `date`, `function`. */ + readonly type: string; +} + +/** A successful per-branch projection. */ +export interface BranchProjection { + /** The emitted JSON Schema, with every marked branch removed. */ + readonly schema: Record; + /** Which `io` direction produced it — `output` is preferred, as elsewhere. */ + readonly io: 'output' | 'input'; + /** Every dropped branch, in document order. Never empty. */ + readonly pruned: readonly PrunedBranch[]; +} + +type JsonObject = Record; + +const isObject = (v: unknown): v is JsonObject => + typeof v === 'object' && v !== null && !Array.isArray(v); + +/** True when a node carries at least one keyword that constrains a value. */ +function hasStructuralKeyword(json: JsonObject): boolean { + return Object.keys(json).some( + (key) => !ANNOTATION_ONLY_KEYS.has(key) && !key.startsWith('x-'), + ); +} + +/** + * Ask Zod itself whether one node has a projection, in the SAME direction the + * enclosing run uses. + * + * Asked only of a node that came back with no structural keyword, which is + * either a type with no JSON form or one that genuinely accepts any JSON value + * — and `z.toJSONSchema(node, { unrepresentable: 'throw' })` is the only + * authority on which. ⛔ A hard-coded list of unrepresentable type names is + * NOT that authority: it is written against one Zod version, and the direction + * it fails in is the dangerous one — a type it fails to name is emitted as `{}` + * and accepts every JSON value. + * + * Anything but a clean conversion counts as "no projection", including a + * message this repo has never seen: a node this function cannot vouch for must + * not become a `{}` in a published artifact. + */ +function projectsUnderStrictMode(node: z.ZodType, io: 'output' | 'input'): boolean { + try { + z.toJSONSchema(node, { + target: 'draft-2020-12', + unrepresentable: 'throw', + ...(io === 'input' ? { io } : {}), + }); + return true; + } catch { + return false; + } +} + +/** + * Build the `override` callback that marks every node Zod produced no + * projection for. + * + * Two signals, in this order, because each alone is wrong: + * + * - **Emitted keywords**, not a type-name list, so a Zod release that adds an + * unrepresentable type is marked without this file being edited. + * - **A strict re-conversion of the node**, because "emitted no keyword" also + * describes `z.any()` and `z.unknown()`, which accept any JSON value + * legitimately — and describes them through wrappers too. `FieldOperators`' + * `$eq` is `z.any().optional().describe(…)`: it comes back as + * `{ description }`, exactly like an unprojectable `z.date().describe(…)`, + * and marking it would have refused the whole projection for the enforced + * half of the filter contract. + */ +export function markUnprojectableNodes(io: 'output' | 'input') { + return (ctx: { zodSchema: unknown; jsonSchema: JsonObject }): void => { + if (hasStructuralKeyword(ctx.jsonSchema)) return; + const node = ctx.zodSchema as z.ZodType & { _zod: { def: { type: string } } }; + if (projectsUnderStrictMode(node, io)) return; + ctx.jsonSchema[UNPROJECTABLE_MARK] = node._zod.def.type; + }; +} + +/** JSON Pointer escaping (RFC 6901). */ +const escapeToken = (token: string): string => token.replace(/~/g, '~0').replace(/\//g, '~1'); + +/** + * Drop every marked member of every `anyOf` / `oneOf`, depth-first so a nested + * union that loses ALL its branches is itself marked before its parent decides. + * + * Mutates `node`. Records what it dropped, with the pointer the branch had + * BEFORE any sibling was removed, so the record names a location that existed. + */ +export function pruneMarkedUnionBranches( + node: unknown, + at: string, + pruned: PrunedBranch[], +): void { + if (Array.isArray(node)) { + node.forEach((item, index) => pruneMarkedUnionBranches(item, `${at}/${index}`, pruned)); + return; + } + if (!isObject(node)) return; + + for (const [key, value] of Object.entries(node)) { + pruneMarkedUnionBranches(value, `${at}/${escapeToken(key)}`, pruned); + } + + for (const keyword of UNION_KEYWORDS) { + const branches = node[keyword]; + if (!Array.isArray(branches)) continue; + + const kept: unknown[] = []; + branches.forEach((branch, index) => { + const mark = isObject(branch) ? branch[UNPROJECTABLE_MARK] : undefined; + if (typeof mark === 'string') { + pruned.push({ at: `${at}/${keyword}/${index}`, type: mark }); + return; + } + kept.push(branch); + }); + + if (kept.length === branches.length) continue; + if (kept.length === 0) { + // Every alternative was unprojectable, so the union itself admits no JSON + // document. Mark it and let the parent drop it — or, if there is no union + // above it, fail the projection. + delete node[keyword]; + node[UNPROJECTABLE_MARK] = 'union'; + continue; + } + node[keyword] = kept; + } +} + +/** The pointer of the first surviving mark, or `null` when there is none. */ +export function findSurvivingMark(node: unknown, at = '#'): string | null { + if (Array.isArray(node)) { + for (const [index, item] of node.entries()) { + const found = findSurvivingMark(item, `${at}/${index}`); + if (found !== null) return found; + } + return null; + } + if (!isObject(node)) return null; + if (typeof node[UNPROJECTABLE_MARK] === 'string') return at; + for (const [key, value] of Object.entries(node)) { + const found = findSurvivingMark(value, `${at}/${escapeToken(key)}`); + if (found !== null) return found; + } + return null; +} + +/** + * Project `value` by dropping union branches Zod cannot represent. + * + * Returns `null` when that does not produce a faithful projection — a marked + * node survived outside a union, or nothing needed dropping — and the caller + * then skips the export exactly as before, with the message Zod threw. That + * `null` is what keeps this change unable to alter the skip message, and so the + * recorded `cause` of any entry still in `unemitted-schemas.baseline.json`. + * + * BOTH directions are projected and the one that drops FEWER branches wins, + * `output` breaking a tie. ⛔ Not "output first, input only on failure", which + * is right for the strict passes and wrong here: a `.transform()` branch has no + * OUTPUT form but a perfectly good input one, so an output-first rule would + * quietly DELETE an authorable shape and publish the narrower schema — the + * measured case is `Data.HookSchema`, where output drops the deprecated + * `z.custom` handler AND a `pipe`, while input drops only the `z.custom`. + * Fewest drops is the most faithful projection available, and the `x-io` flag + * already tells a reader which shape they are looking at (#2967 / #2978). + */ +export function projectByPruningUnionBranches( + value: z.ZodType, + options: { readonly target: 'draft-2020-12' }, +): BranchProjection | null { + const candidates: BranchProjection[] = []; + + for (const io of ['output', 'input'] as const) { + let schema: JsonObject; + try { + schema = z.toJSONSchema(value, { + target: options.target, + unrepresentable: 'any', + override: markUnprojectableNodes(io), + ...(io === 'input' ? { io } : {}), + }) as JsonObject; + } catch { + // `unrepresentable: 'any'` removes the unrepresentable-type throws, so + // anything left (a `cycles: 'throw'` refusal, a generator bug) is not a + // projection question — try the other direction, then give up. + continue; + } + + const pruned: PrunedBranch[] = []; + pruneMarkedUnionBranches(schema, '#', pruned); + // A mark that survived is a node with no JSON form OUTSIDE a union — an + // object property, a record value, an array item — where dropping it would + // publish a shape no runtime value has. Refuse the direction. + if (findSurvivingMark(schema) !== null) continue; + // Nothing to drop, yet a strict pass refused this schema: whatever that is, + // it is not the case this module exists for. Leave the export as it was. + if (pruned.length === 0) continue; + candidates.push({ schema, io, pruned }); + } + + if (candidates.length === 0) return null; + return candidates.reduce((best, candidate) => + candidate.pruned.length < best.pruned.length ? candidate : best, + ); +} diff --git a/packages/spec/unemitted-schemas.baseline.json b/packages/spec/unemitted-schemas.baseline.json index e79fca3512..0110a311b0 100644 --- a/packages/spec/unemitted-schemas.baseline.json +++ b/packages/spec/unemitted-schemas.baseline.json @@ -27,8 +27,14 @@ "", "SEEDED FROM THE TREE AS IT STOOD, not as anyone wishes it were. Several entries below record", "a real documentation gap rather than blessing one — that is the point. This ratchet reports;", - "it repairs nothing and makes nothing start or stop publishing. The four `Data` filter entries", - "are #16431's own (a)/(b) remedies and remain open on that card." + "it repairs nothing and makes nothing start or stop publishing.", + "", + "FIVE ENTRIES HAVE ALREADY LEFT, which is the direction this ledger is FOR. #16431 option (a)", + "taught the generator to project a union BRANCH-BY-BRANCH (scripts/lib/union-branch-projection.ts),", + "and the four `Data` filter entries plus `Data.HookSchema` now emit; the ratchet reported each of", + "them as `repaired` and refused the build until the line went. What remains below is what a", + "per-branch projection cannot reach: an unprojectable node that is not itself a union member —", + "an object property, a record value — where dropping it would publish a shape no runtime value has." ], "entries": { "Automation.FlowFunctionDeclarationSchema": { @@ -37,32 +43,12 @@ }, "Automation.FlowFunctionEntrySchema": { "cause": "function", - "reason": "The union of all four `functions` entry shapes, and its first member is the bare `z.function()` above, so the whole union is unprojectable even though its two lowered members are plain JSON. A per-branch projection would publish those two; today the generator's fallback is per-SCHEMA, which is the same limitation #16431 records against the `Data` filter entries below." + "reason": "The union of all four `functions` entry shapes. Its first member IS the bare `z.function()` above, and #16431 (a)'s per-branch projection does drop that member — measured: 1 branch pruned — but the union still does not publish, because the member behind it is FlowFunctionDeclarationSchema, whose required `handler` is itself a live callable. An unprojectable node in a PROPERTY position is not droppable: every document the declaration accepts has that key, so publishing the record without it would describe a shape no runtime value has. Both lowered members are plain JSON and would publish on their own; what this entry now records is a union that cannot be projected while one of its branches requires a callable." }, "Cloud.EnvironmentArtifactSchema": { "cause": "function", "reason": "Re-export of System.EnvironmentArtifactSchema (system/environment-artifact.zod.ts) — see that entry. Both export names are recorded because the ratchet is keyed by EXPORT, which is the unit that reaches or fails to reach a published surface." }, - "Data.ComparisonOperatorSchema": { - "cause": "date", - "reason": "⚠ A REAL GAP, NOT A BLESSING: this is #16431's own finding. `$gt`/`$gte`/`$lt`/`$lte` carry ~1050 characters of `.describe()` each — the #5685 comparand contract — and reach no reference row at all, because `orderingComparandSchema` includes `z.date()` and the generator's `io: 'input'` fallback applies per SCHEMA rather than per BRANCH. Recorded here so the gap is counted; the remedy is #16431 (a)/(b), which stays open." - }, - "Data.FieldOperatorsSchema": { - "cause": "date", - "reason": "⚠ A REAL GAP: same cause as ComparisonOperatorSchema — it restates the ordering operators, so the whole record is unprojectable. Its `$null`/`$exists` members are described on the reference page only because #15059's prose carries a second copy of them. Remedy: #16431 (a)/(b)." - }, - "Data.NormalizedFilterSchema": { - "cause": "date", - "reason": "⚠ A REAL GAP: the normalised filter form reached through `$not` → field → `$gt`, so it inherits the same `z.date()` comparand. Remedy: #16431 (a)/(b)." - }, - "Data.RangeOperatorSchema": { - "cause": "date", - "reason": "⚠ A REAL GAP: `$between` carries the #6571 endpoint contract and the #7596 no-`{ $field }`-in-a-list rule, ~1010 characters, and reaches no reference row — `rangeEndpointSchema` includes `z.date()`. Remedy: #16431 (a)/(b)." - }, - "Data.HookSchema": { - "cause": "custom", - "reason": "⚠ A REAL GAP, and the widest one in this ledger: `hook` is a declared metadata type (`**/*.hook.ts`, `**/*.hook.yml`, `allowRuntimeCreate: true`), so authors write these documents — yet the schema is unprojectable because its DEPRECATED inline-function handler form is a `z.custom<(...args) => any>`. Measured consequence beyond the missing reference page: authorable-surface/ holds 14 `data/HookContext:` keys and zero `data/Hook:` keys, so every authorable key on this type is outside the #3855/#4650 key ratchet and the #4666 default ratchet too. Narrowing the handler union to its string form would publish it." - }, "Data.DataEngineContractSchema": { "cause": "function", "reason": "The engine method contract — `find`/`findOne`/`insert`/`update`/`delete` are `z.function()` members. A CODE interface implemented by an engine, never a document anyone authors or transmits, so no reader loses a reference page." @@ -77,11 +63,11 @@ }, "Data.MemoryConfigSchema": { "cause": "function", - "reason": "⚠ A REAL GAP: this is a memory datasource's authorable `config`, and it is unprojectable only because ONE branch of its `persistence` union is the code-only custom adapter above. Everything an author can write in YAML is representable. A per-branch projection, or excluding the custom branch from the published shape, would give this type its reference page back." + "reason": "⚠ A REAL GAP: this is a memory datasource's authorable `config`, and it is unprojectable only because ONE branch of its `persistence` union is the code-only custom adapter above. Everything an author can write in YAML is representable. #16431 (a)'s per-branch projection does NOT reach it — measured: the unprojectable node is `#/properties/persistence/anyOf/0/anyOf/4/properties/adapter/properties/load`, a function nested two levels INSIDE a union branch rather than being that branch, and the pruner drops only direct members. Excluding the custom branch from the published shape — or lifting the adapter out of the authorable `config` — would give this type its reference page back." }, "Data.MemoryPersistenceConfigSchema": { "cause": "function", - "reason": "⚠ A REAL GAP, same cause and same remedy as MemoryConfigSchema: the union of five persistence shapes, four of them plain JSON, unprojectable because the fifth carries the live adapter." + "reason": "⚠ A REAL GAP, same cause and same remedy as MemoryConfigSchema: the union of five persistence shapes, four of them plain JSON, unprojectable because the fifth carries the live adapter — as a nested property of that branch, not as the branch itself, which is why the per-branch projection leaves it here too." }, "Data.PersistenceAdapterSchema": { "cause": "function", From 2cbdadf86b244bf1a529175080422058fad83f57 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 04:25:57 +0000 Subject: [PATCH 2/5] wip: regenerate manifests, docs, import baseline --- content/docs/references/data/filter.mdx | 197 +++++++++++++++++- content/docs/references/data/hook.mdx | 66 +++++- content/docs/references/index.mdx | 12 +- .../spec/docs-import-surface.baseline.json | 2 + 4 files changed, 265 insertions(+), 12 deletions(-) diff --git a/content/docs/references/data/filter.mdx b/content/docs/references/data/filter.mdx index 43ad51f251..85714dbb1b 100644 --- a/content/docs/references/data/filter.mdx +++ b/content/docs/references/data/filter.mdx @@ -32,13 +32,55 @@ Design Principles: ## TypeScript Usage ```typescript -import { EqualityOperatorSchema, FieldReferenceSchema, FilterArraySchema, FilterConditionSchema, QueryFilterSchema, SetOperatorSchema, SpecialOperatorSchema, StringOperatorSchema } from '@objectstack/spec/data'; -import type { FieldReference, FilterArray, FilterCondition, QueryFilter } from '@objectstack/spec/data'; +import { ComparisonOperatorSchema, EqualityOperatorSchema, FieldOperatorsSchema, FieldReferenceSchema, FilterArraySchema, FilterConditionSchema, NormalizedFilterSchema, QueryFilterSchema, RangeOperatorSchema, SetOperatorSchema, SpecialOperatorSchema, StringOperatorSchema } from '@objectstack/spec/data'; +import type { FieldOperators, FieldReference, FilterArray, FilterCondition, NormalizedFilter, QueryFilter } from '@objectstack/spec/data'; // Validate data -const result = EqualityOperatorSchema.parse(data); +const result = ComparisonOperatorSchema.parse(data); ``` +--- + +## ComparisonOperator + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$gt** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Greater than. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$gte** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Greater than or equal to. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$lt** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Less than. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$lte** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Less than or equal to. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | + +### Nested Shape: `ComparisonOperator.$gt` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$field** | `string` | ✅ | Field Reference/Column Name | +| **addDays** | `integer \| { $field: string }` | optional | Whole-day offset added to the referenced column before comparing: an integer literal of any sign (negative subtracts; whole days only), or a `{ $field }` reference to a numeric column. A NULL offset column contributes zero days; a NULL referenced column makes the comparison false rather than NULL, so it stays false under $not. Compiles on SQL push-down between two temporal columns of the same class (date/date, datetime/datetime) and evaluates identically in memory. | + +### Nested Shape: `ComparisonOperator.$gte` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$field** | `string` | ✅ | Field Reference/Column Name | +| **addDays** | `integer \| { $field: string }` | optional | Whole-day offset added to the referenced column before comparing: an integer literal of any sign (negative subtracts; whole days only), or a `{ $field }` reference to a numeric column. A NULL offset column contributes zero days; a NULL referenced column makes the comparison false rather than NULL, so it stays false under $not. Compiles on SQL push-down between two temporal columns of the same class (date/date, datetime/datetime) and evaluates identically in memory. | + +### Nested Shape: `ComparisonOperator.$lt` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$field** | `string` | ✅ | Field Reference/Column Name | +| **addDays** | `integer \| { $field: string }` | optional | Whole-day offset added to the referenced column before comparing: an integer literal of any sign (negative subtracts; whole days only), or a `{ $field }` reference to a numeric column. A NULL offset column contributes zero days; a NULL referenced column makes the comparison false rather than NULL, so it stays false under $not. Compiles on SQL push-down between two temporal columns of the same class (date/date, datetime/datetime) and evaluates identically in memory. | + +### Nested Shape: `ComparisonOperator.$lte` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$field** | `string` | ✅ | Field Reference/Column Name | +| **addDays** | `integer \| { $field: string }` | optional | Whole-day offset added to the referenced column before comparing: an integer literal of any sign (negative subtracts; whole days only), or a `{ $field }` reference to a numeric column. A NULL offset column contributes zero days; a NULL referenced column makes the comparison false rather than NULL, so it stays false under $not. Compiles on SQL push-down between two temporal columns of the same class (date/date, datetime/datetime) and evaluates identically in memory. | + + --- ## EqualityOperator @@ -51,6 +93,62 @@ const result = EqualityOperatorSchema.parse(data); | **$ne** | `any` | optional | Not equal to. Lowered to `<>` / `!=` on the SQL family and to `$ne` on MongoDB. The comparand is a literal, or a `{ $field }` reference to another column of the same table ($ne is one of the six scalar comparisons a reference may be the whole comparand of). `{ "$ne": null }` is the HAS-A-VALUE predicate — the same question `{ "$exists": true }` asks: it matches rows whose field holds a value, never rows that merely carry the key. | +--- + +## FieldOperators + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$eq** | `any` | optional | Equal to — the DEFAULT operator: a bare value written against a field key is the same condition as this one. Lowered to `=` on the SQL family and to `$eq` on MongoDB. The comparand is a literal, or a `{ $field }` reference to another column of the same table ($eq is one of the six scalar comparisons a reference may be the whole comparand of). `{ "$eq": null }` is the has-NO-value predicate — the same question `{ "$null": true }` asks, and it is about the VALUE, never about whether a key is present. | +| **$ne** | `any` | optional | Not equal to. Lowered to `<>` / `!=` on the SQL family and to `$ne` on MongoDB. The comparand is a literal, or a `{ $field }` reference to another column of the same table ($ne is one of the six scalar comparisons a reference may be the whole comparand of). `{ "$ne": null }` is the HAS-A-VALUE predicate — the same question `{ "$exists": true }` asks: it matches rows whose field holds a value, never rows that merely carry the key. | +| **$gt** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Greater than. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$gte** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Greater than or equal to. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$lt** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Less than. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$lte** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Less than or equal to. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$in** | `any[]` | optional | Membership list. Members are literal values of any type the column stores. A `{ $field }` reference is NOT a member shape: no backend resolves one inside a list — put it in a scalar comparison ($eq/$ne/$gt/$gte/$lt/$lte) instead. null is NOT a member shape either: state absence explicitly with the null predicate — "one of […] OR has no value" is `{ "$or": [{ "$in": […] }, { "$null": true }] }`. | +| **$nin** | `any[]` | optional | Membership list. Members are literal values of any type the column stores. A `{ $field }` reference is NOT a member shape: no backend resolves one inside a list — put it in a scalar comparison ($eq/$ne/$gt/$gte/$lt/$lte) instead. null is NOT a member shape either: state absence explicitly with the null predicate — "one of […] OR has no value" is `{ "$or": [{ "$in": […] }, { "$null": true }] }`. | +| **$between** | `any[]` | optional | Between (inclusive). Closed interval [min, max]. Each endpoint is a number, a Date, or a string. A `{ $field }` reference is NOT an endpoint shape: no backend resolves one inside a list — put it in a scalar comparison ($gt/$gte/$lt/$lte), which does compile to a column-to-column bound. STRING is the form the platform itself produces: the date-macro resolver walks INTO arrays, so `{ $between: ["{current_year_start}", "{current_year_end}"] }` resolves to two strings. The guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column; those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees. The driver reconciles each endpoint with the column independently (a bare calendar day used as the MAX becomes the half-open next-day boundary). Ranging over NON-temporal text is permitted but NOT promised: the order is the backend collation's, and those coincide only for ASCII. | +| **$contains** | `string` | optional | Contains substring, CASE-SENSITIVELY — "acme" does NOT match "ACME". Lowered to `LIKE '%?%'` (case-exact) on the SQL family, and answered case-exactly on every JS evaluation face the platform ships. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). Case-INSENSITIVE containment is $icontains, which folds ASCII case only. | +| **$notContains** | `string` | optional | Does not contain substring, CASE-SENSITIVELY — the negation of $contains, on the same comparand contract. Lowered to `NOT LIKE '%?%'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$startsWith** | `string` | optional | Starts with prefix, CASE-SENSITIVELY. Lowered to `LIKE '?%'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$endsWith** | `string` | optional | Ends with suffix, CASE-SENSITIVELY. Lowered to `LIKE '%?'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$icontains** | `string` | optional | Contains substring, ignoring case — but ONLY ASCII case (A-Z against a-z). Every other character compares literally, so "café" does NOT match "CAFÉ" and "москва" does not match "МОСКВА". The domain is ASCII because that is the one fold all five backends can deliver: SQLite (and therefore turso and sqlite-wasm) folds ASCII only, so a Unicode promise here would be a guarantee three of the five could not keep. The comparand is matched LITERALLY — "%", "_" and regex metacharacters are ordinary characters, not wildcards. Case-SENSITIVE containment is $contains. Lowered on the SQL family (driver-sql, driver-sqlite-wasm, driver-turso on both transports) and on every JS evaluation face, so it is portable across every backend the platform ships. | +| **$like** | `string` | optional | Whole-string pattern match with CALLER-bound wildcards: "%" matches any sequence (including empty), "_" matches exactly one character, and a backslash escapes the character after it ("\\%", "\\_", "\\\\") so it matches literally. The pattern must cover the WHOLE value — a pattern with no wildcards is an exact comparison, NOT a substring search; write $contains for containment. A pattern ending in a lone unpaired backslash is refused (INVALID_FILTER). Comparison is case-SENSITIVE, same contract as $contains (Q2 = A); $ilike is the case-insensitive twin. Answered by the SQL family (driver-sql, driver-sqlite-wasm, driver-turso on both transports), by driver-memory and by @objectstack/formula. driver-mongodb, objectql `having` and service-analytics REFUSE it in the INVALID_FILTER envelope rather than approximating it — see FILTER_OPERATORS for why it is staged out of that allowlist. | +| **$ilike** | `string` | optional | Whole-string pattern match like $like — "%" / "_" wildcards bound by the caller, backslash escapes — but ignoring ASCII case (A-Z against a-z) and ONLY ASCII case: "café" does NOT match "CAFÉ", the same Q1 = A boundary $icontains declares, because SQLite's fold is ASCII-only and three of the five backends are SQLite underneath. Staged with $like — see FILTER_OPERATORS. | +| **$null** | `boolean` | optional | Is-null check. `true` matches rows where the field is null, `false` matches rows where it is not null. Lowered to `IS NULL` (true) / `IS NOT NULL` (false) on the SQL family and to `{ field: null }` (true) / `{ $ne: null }` (false) on MongoDB. | +| **$exists** | `boolean` | optional | Has-a-value check — the exact inverse of `$null`. `true` matches rows where the field holds a value (`!= null`), `false` matches rows where it holds none. Portable across every backend the platform ships: lowered to `IS NOT NULL` (true) / `IS NULL` (false) on the SQL family and to `{ $ne: null }` (true) / `{ $eq: null }` (false) on MongoDB. | + +### Nested Shape: `FieldOperators.$gt` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$field** | `string` | ✅ | Field Reference/Column Name | +| **addDays** | `integer \| { $field: string }` | optional | Whole-day offset added to the referenced column before comparing: an integer literal of any sign (negative subtracts; whole days only), or a `{ $field }` reference to a numeric column. A NULL offset column contributes zero days; a NULL referenced column makes the comparison false rather than NULL, so it stays false under $not. Compiles on SQL push-down between two temporal columns of the same class (date/date, datetime/datetime) and evaluates identically in memory. | + +### Nested Shape: `FieldOperators.$gte` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$field** | `string` | ✅ | Field Reference/Column Name | +| **addDays** | `integer \| { $field: string }` | optional | Whole-day offset added to the referenced column before comparing: an integer literal of any sign (negative subtracts; whole days only), or a `{ $field }` reference to a numeric column. A NULL offset column contributes zero days; a NULL referenced column makes the comparison false rather than NULL, so it stays false under $not. Compiles on SQL push-down between two temporal columns of the same class (date/date, datetime/datetime) and evaluates identically in memory. | + +### Nested Shape: `FieldOperators.$lt` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$field** | `string` | ✅ | Field Reference/Column Name | +| **addDays** | `integer \| { $field: string }` | optional | Whole-day offset added to the referenced column before comparing: an integer literal of any sign (negative subtracts; whole days only), or a `{ $field }` reference to a numeric column. A NULL offset column contributes zero days; a NULL referenced column makes the comparison false rather than NULL, so it stays false under $not. Compiles on SQL push-down between two temporal columns of the same class (date/date, datetime/datetime) and evaluates identically in memory. | + +### Nested Shape: `FieldOperators.$lte` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$field** | `string` | ✅ | Field Reference/Column Name | +| **addDays** | `integer \| { $field: string }` | optional | Whole-day offset added to the referenced column before comparing: an integer literal of any sign (negative subtracts; whole days only), or a `{ $field }` reference to a numeric column. A NULL offset column contributes zero days; a NULL referenced column makes the comparison false rather than NULL, so it stays false under $not. Compiles on SQL push-down between two temporal columns of the same class (date/date, datetime/datetime) and evaluates identically in memory. | + + --- ## FieldReference @@ -113,6 +211,88 @@ Type: `[FilterArray](#filterarray)[]` **Type:** `Record & { $and?: [FilterCondition](#filtercondition)[]; $or?: [FilterCondition](#filtercondition)[]; $not?: [FilterCondition](#filtercondition) }` +--- + +## NormalizedFilter + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$and** | `(Record \| [NormalizedFilter](#normalizedfilter))[]` | optional | | +| **$or** | `(Record \| [NormalizedFilter](#normalizedfilter))[]` | optional | | +| **$not** | `Record \| [NormalizedFilter](#normalizedfilter)` | optional | | + +### Nested Shape: `NormalizedFilter.$and[number][string]` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$eq** | `any` | optional | Equal to — the DEFAULT operator: a bare value written against a field key is the same condition as this one. Lowered to `=` on the SQL family and to `$eq` on MongoDB. The comparand is a literal, or a `{ $field }` reference to another column of the same table ($eq is one of the six scalar comparisons a reference may be the whole comparand of). `{ "$eq": null }` is the has-NO-value predicate — the same question `{ "$null": true }` asks, and it is about the VALUE, never about whether a key is present. | +| **$ne** | `any` | optional | Not equal to. Lowered to `<>` / `!=` on the SQL family and to `$ne` on MongoDB. The comparand is a literal, or a `{ $field }` reference to another column of the same table ($ne is one of the six scalar comparisons a reference may be the whole comparand of). `{ "$ne": null }` is the HAS-A-VALUE predicate — the same question `{ "$exists": true }` asks: it matches rows whose field holds a value, never rows that merely carry the key. | +| **$gt** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Greater than. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$gte** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Greater than or equal to. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$lt** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Less than. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$lte** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Less than or equal to. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$in** | `any[]` | optional | Membership list. Members are literal values of any type the column stores. A `{ $field }` reference is NOT a member shape: no backend resolves one inside a list — put it in a scalar comparison ($eq/$ne/$gt/$gte/$lt/$lte) instead. null is NOT a member shape either: state absence explicitly with the null predicate — "one of […] OR has no value" is `{ "$or": [{ "$in": […] }, { "$null": true }] }`. | +| **$nin** | `any[]` | optional | Membership list. Members are literal values of any type the column stores. A `{ $field }` reference is NOT a member shape: no backend resolves one inside a list — put it in a scalar comparison ($eq/$ne/$gt/$gte/$lt/$lte) instead. null is NOT a member shape either: state absence explicitly with the null predicate — "one of […] OR has no value" is `{ "$or": [{ "$in": […] }, { "$null": true }] }`. | +| **$between** | `any[]` | optional | Between (inclusive). Closed interval [min, max]. Each endpoint is a number, a Date, or a string. A `{ $field }` reference is NOT an endpoint shape: no backend resolves one inside a list — put it in a scalar comparison ($gt/$gte/$lt/$lte), which does compile to a column-to-column bound. STRING is the form the platform itself produces: the date-macro resolver walks INTO arrays, so `{ $between: ["{current_year_start}", "{current_year_end}"] }` resolves to two strings. The guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column; those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees. The driver reconciles each endpoint with the column independently (a bare calendar day used as the MAX becomes the half-open next-day boundary). Ranging over NON-temporal text is permitted but NOT promised: the order is the backend collation's, and those coincide only for ASCII. | +| **$contains** | `string` | optional | Contains substring, CASE-SENSITIVELY — "acme" does NOT match "ACME". Lowered to `LIKE '%?%'` (case-exact) on the SQL family, and answered case-exactly on every JS evaluation face the platform ships. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). Case-INSENSITIVE containment is $icontains, which folds ASCII case only. | +| **$notContains** | `string` | optional | Does not contain substring, CASE-SENSITIVELY — the negation of $contains, on the same comparand contract. Lowered to `NOT LIKE '%?%'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$startsWith** | `string` | optional | Starts with prefix, CASE-SENSITIVELY. Lowered to `LIKE '?%'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$endsWith** | `string` | optional | Ends with suffix, CASE-SENSITIVELY. Lowered to `LIKE '%?'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$icontains** | `string` | optional | Contains substring, ignoring case — but ONLY ASCII case (A-Z against a-z). Every other character compares literally, so "café" does NOT match "CAFÉ" and "москва" does not match "МОСКВА". The domain is ASCII because that is the one fold all five backends can deliver: SQLite (and therefore turso and sqlite-wasm) folds ASCII only, so a Unicode promise here would be a guarantee three of the five could not keep. The comparand is matched LITERALLY — "%", "_" and regex metacharacters are ordinary characters, not wildcards. Case-SENSITIVE containment is $contains. Lowered on the SQL family (driver-sql, driver-sqlite-wasm, driver-turso on both transports) and on every JS evaluation face, so it is portable across every backend the platform ships. | +| **$like** | `string` | optional | Whole-string pattern match with CALLER-bound wildcards: "%" matches any sequence (including empty), "_" matches exactly one character, and a backslash escapes the character after it ("\\%", "\\_", "\\\\") so it matches literally. The pattern must cover the WHOLE value — a pattern with no wildcards is an exact comparison, NOT a substring search; write $contains for containment. A pattern ending in a lone unpaired backslash is refused (INVALID_FILTER). Comparison is case-SENSITIVE, same contract as $contains (Q2 = A); $ilike is the case-insensitive twin. Answered by the SQL family (driver-sql, driver-sqlite-wasm, driver-turso on both transports), by driver-memory and by @objectstack/formula. driver-mongodb, objectql `having` and service-analytics REFUSE it in the INVALID_FILTER envelope rather than approximating it — see FILTER_OPERATORS for why it is staged out of that allowlist. | +| **$ilike** | `string` | optional | Whole-string pattern match like $like — "%" / "_" wildcards bound by the caller, backslash escapes — but ignoring ASCII case (A-Z against a-z) and ONLY ASCII case: "café" does NOT match "CAFÉ", the same Q1 = A boundary $icontains declares, because SQLite's fold is ASCII-only and three of the five backends are SQLite underneath. Staged with $like — see FILTER_OPERATORS. | +| **$null** | `boolean` | optional | Is-null check. `true` matches rows where the field is null, `false` matches rows where it is not null. Lowered to `IS NULL` (true) / `IS NOT NULL` (false) on the SQL family and to `{ field: null }` (true) / `{ $ne: null }` (false) on MongoDB. | +| **$exists** | `boolean` | optional | Has-a-value check — the exact inverse of `$null`. `true` matches rows where the field holds a value (`!= null`), `false` matches rows where it holds none. Portable across every backend the platform ships: lowered to `IS NOT NULL` (true) / `IS NULL` (false) on the SQL family and to `{ $ne: null }` (true) / `{ $eq: null }` (false) on MongoDB. | + +### Nested Shape: `NormalizedFilter.$or[number][string]` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$eq** | `any` | optional | Equal to — the DEFAULT operator: a bare value written against a field key is the same condition as this one. Lowered to `=` on the SQL family and to `$eq` on MongoDB. The comparand is a literal, or a `{ $field }` reference to another column of the same table ($eq is one of the six scalar comparisons a reference may be the whole comparand of). `{ "$eq": null }` is the has-NO-value predicate — the same question `{ "$null": true }` asks, and it is about the VALUE, never about whether a key is present. | +| **$ne** | `any` | optional | Not equal to. Lowered to `<>` / `!=` on the SQL family and to `$ne` on MongoDB. The comparand is a literal, or a `{ $field }` reference to another column of the same table ($ne is one of the six scalar comparisons a reference may be the whole comparand of). `{ "$ne": null }` is the HAS-A-VALUE predicate — the same question `{ "$exists": true }` asks: it matches rows whose field holds a value, never rows that merely carry the key. | +| **$gt** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Greater than. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$gte** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Greater than or equal to. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$lt** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Less than. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$lte** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Less than or equal to. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$in** | `any[]` | optional | Membership list. Members are literal values of any type the column stores. A `{ $field }` reference is NOT a member shape: no backend resolves one inside a list — put it in a scalar comparison ($eq/$ne/$gt/$gte/$lt/$lte) instead. null is NOT a member shape either: state absence explicitly with the null predicate — "one of […] OR has no value" is `{ "$or": [{ "$in": […] }, { "$null": true }] }`. | +| **$nin** | `any[]` | optional | Membership list. Members are literal values of any type the column stores. A `{ $field }` reference is NOT a member shape: no backend resolves one inside a list — put it in a scalar comparison ($eq/$ne/$gt/$gte/$lt/$lte) instead. null is NOT a member shape either: state absence explicitly with the null predicate — "one of […] OR has no value" is `{ "$or": [{ "$in": […] }, { "$null": true }] }`. | +| **$between** | `any[]` | optional | Between (inclusive). Closed interval [min, max]. Each endpoint is a number, a Date, or a string. A `{ $field }` reference is NOT an endpoint shape: no backend resolves one inside a list — put it in a scalar comparison ($gt/$gte/$lt/$lte), which does compile to a column-to-column bound. STRING is the form the platform itself produces: the date-macro resolver walks INTO arrays, so `{ $between: ["{current_year_start}", "{current_year_end}"] }` resolves to two strings. The guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column; those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees. The driver reconciles each endpoint with the column independently (a bare calendar day used as the MAX becomes the half-open next-day boundary). Ranging over NON-temporal text is permitted but NOT promised: the order is the backend collation's, and those coincide only for ASCII. | +| **$contains** | `string` | optional | Contains substring, CASE-SENSITIVELY — "acme" does NOT match "ACME". Lowered to `LIKE '%?%'` (case-exact) on the SQL family, and answered case-exactly on every JS evaluation face the platform ships. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). Case-INSENSITIVE containment is $icontains, which folds ASCII case only. | +| **$notContains** | `string` | optional | Does not contain substring, CASE-SENSITIVELY — the negation of $contains, on the same comparand contract. Lowered to `NOT LIKE '%?%'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$startsWith** | `string` | optional | Starts with prefix, CASE-SENSITIVELY. Lowered to `LIKE '?%'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$endsWith** | `string` | optional | Ends with suffix, CASE-SENSITIVELY. Lowered to `LIKE '%?'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$icontains** | `string` | optional | Contains substring, ignoring case — but ONLY ASCII case (A-Z against a-z). Every other character compares literally, so "café" does NOT match "CAFÉ" and "москва" does not match "МОСКВА". The domain is ASCII because that is the one fold all five backends can deliver: SQLite (and therefore turso and sqlite-wasm) folds ASCII only, so a Unicode promise here would be a guarantee three of the five could not keep. The comparand is matched LITERALLY — "%", "_" and regex metacharacters are ordinary characters, not wildcards. Case-SENSITIVE containment is $contains. Lowered on the SQL family (driver-sql, driver-sqlite-wasm, driver-turso on both transports) and on every JS evaluation face, so it is portable across every backend the platform ships. | +| **$like** | `string` | optional | Whole-string pattern match with CALLER-bound wildcards: "%" matches any sequence (including empty), "_" matches exactly one character, and a backslash escapes the character after it ("\\%", "\\_", "\\\\") so it matches literally. The pattern must cover the WHOLE value — a pattern with no wildcards is an exact comparison, NOT a substring search; write $contains for containment. A pattern ending in a lone unpaired backslash is refused (INVALID_FILTER). Comparison is case-SENSITIVE, same contract as $contains (Q2 = A); $ilike is the case-insensitive twin. Answered by the SQL family (driver-sql, driver-sqlite-wasm, driver-turso on both transports), by driver-memory and by @objectstack/formula. driver-mongodb, objectql `having` and service-analytics REFUSE it in the INVALID_FILTER envelope rather than approximating it — see FILTER_OPERATORS for why it is staged out of that allowlist. | +| **$ilike** | `string` | optional | Whole-string pattern match like $like — "%" / "_" wildcards bound by the caller, backslash escapes — but ignoring ASCII case (A-Z against a-z) and ONLY ASCII case: "café" does NOT match "CAFÉ", the same Q1 = A boundary $icontains declares, because SQLite's fold is ASCII-only and three of the five backends are SQLite underneath. Staged with $like — see FILTER_OPERATORS. | +| **$null** | `boolean` | optional | Is-null check. `true` matches rows where the field is null, `false` matches rows where it is not null. Lowered to `IS NULL` (true) / `IS NOT NULL` (false) on the SQL family and to `{ field: null }` (true) / `{ $ne: null }` (false) on MongoDB. | +| **$exists** | `boolean` | optional | Has-a-value check — the exact inverse of `$null`. `true` matches rows where the field holds a value (`!= null`), `false` matches rows where it holds none. Portable across every backend the platform ships: lowered to `IS NOT NULL` (true) / `IS NULL` (false) on the SQL family and to `{ $ne: null }` (true) / `{ $eq: null }` (false) on MongoDB. | + +### Nested Shape: `NormalizedFilter.$not[string]` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$eq** | `any` | optional | Equal to — the DEFAULT operator: a bare value written against a field key is the same condition as this one. Lowered to `=` on the SQL family and to `$eq` on MongoDB. The comparand is a literal, or a `{ $field }` reference to another column of the same table ($eq is one of the six scalar comparisons a reference may be the whole comparand of). `{ "$eq": null }` is the has-NO-value predicate — the same question `{ "$null": true }` asks, and it is about the VALUE, never about whether a key is present. | +| **$ne** | `any` | optional | Not equal to. Lowered to `<>` / `!=` on the SQL family and to `$ne` on MongoDB. The comparand is a literal, or a `{ $field }` reference to another column of the same table ($ne is one of the six scalar comparisons a reference may be the whole comparand of). `{ "$ne": null }` is the HAS-A-VALUE predicate — the same question `{ "$exists": true }` asks: it matches rows whose field holds a value, never rows that merely carry the key. | +| **$gt** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Greater than. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$gte** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Greater than or equal to. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$lt** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Less than. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$lte** | `number \| string \| { $field: string; addDays?: integer \| object }` | optional | Less than or equal to. Comparand is a number, a Date, a string, or a `{ $field }` reference (optionally carrying a whole-day addDays offset). STRING is the form the platform itself produces: the date-macro resolver returns only strings ("`{current_year_start}`" -> "2026-01-01"), and the guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column. Those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees; the driver reconciles the comparand with the column (a bare calendar day used as an upper bound becomes the half-open next-day boundary). Ordering NON-temporal text is permitted but NOT promised: the order is the backend collation's (byte-wise on SQLite, the database locale on Postgres, UTF-16 code units in the JS matchers), and those coincide only for ASCII. null is NOT a comparand: null is not ordered — state absence with the null predicate instead ($eq: null is "has no value", $ne: null is "has a value"). | +| **$in** | `any[]` | optional | Membership list. Members are literal values of any type the column stores. A `{ $field }` reference is NOT a member shape: no backend resolves one inside a list — put it in a scalar comparison ($eq/$ne/$gt/$gte/$lt/$lte) instead. null is NOT a member shape either: state absence explicitly with the null predicate — "one of […] OR has no value" is `{ "$or": [{ "$in": […] }, { "$null": true }] }`. | +| **$nin** | `any[]` | optional | Membership list. Members are literal values of any type the column stores. A `{ $field }` reference is NOT a member shape: no backend resolves one inside a list — put it in a scalar comparison ($eq/$ne/$gt/$gte/$lt/$lte) instead. null is NOT a member shape either: state absence explicitly with the null predicate — "one of […] OR has no value" is `{ "$or": [{ "$in": […] }, { "$null": true }] }`. | +| **$between** | `any[]` | optional | Between (inclusive). Closed interval [min, max]. Each endpoint is a number, a Date, or a string. A `{ $field }` reference is NOT an endpoint shape: no backend resolves one inside a list — put it in a scalar comparison ($gt/$gte/$lt/$lte), which does compile to a column-to-column bound. STRING is the form the platform itself produces: the date-macro resolver walks INTO arrays, so `{ $between: ["{current_year_start}", "{current_year_end}"] }` resolves to two strings. The guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column; those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees. The driver reconciles each endpoint with the column independently (a bare calendar day used as the MAX becomes the half-open next-day boundary). Ranging over NON-temporal text is permitted but NOT promised: the order is the backend collation's, and those coincide only for ASCII. | +| **$contains** | `string` | optional | Contains substring, CASE-SENSITIVELY — "acme" does NOT match "ACME". Lowered to `LIKE '%?%'` (case-exact) on the SQL family, and answered case-exactly on every JS evaluation face the platform ships. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). Case-INSENSITIVE containment is $icontains, which folds ASCII case only. | +| **$notContains** | `string` | optional | Does not contain substring, CASE-SENSITIVELY — the negation of $contains, on the same comparand contract. Lowered to `NOT LIKE '%?%'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$startsWith** | `string` | optional | Starts with prefix, CASE-SENSITIVELY. Lowered to `LIKE '?%'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$endsWith** | `string` | optional | Ends with suffix, CASE-SENSITIVELY. Lowered to `LIKE '%?'` (case-exact) on the SQL family. The comparand is matched LITERALLY: "%", "_" and regex metacharacters are ordinary characters, because this family escapes and anchors the comparand on the caller's behalf. To bind the wildcards yourself, write $like (case-exact) or $ilike (ASCII-folded). | +| **$icontains** | `string` | optional | Contains substring, ignoring case — but ONLY ASCII case (A-Z against a-z). Every other character compares literally, so "café" does NOT match "CAFÉ" and "москва" does not match "МОСКВА". The domain is ASCII because that is the one fold all five backends can deliver: SQLite (and therefore turso and sqlite-wasm) folds ASCII only, so a Unicode promise here would be a guarantee three of the five could not keep. The comparand is matched LITERALLY — "%", "_" and regex metacharacters are ordinary characters, not wildcards. Case-SENSITIVE containment is $contains. Lowered on the SQL family (driver-sql, driver-sqlite-wasm, driver-turso on both transports) and on every JS evaluation face, so it is portable across every backend the platform ships. | +| **$like** | `string` | optional | Whole-string pattern match with CALLER-bound wildcards: "%" matches any sequence (including empty), "_" matches exactly one character, and a backslash escapes the character after it ("\\%", "\\_", "\\\\") so it matches literally. The pattern must cover the WHOLE value — a pattern with no wildcards is an exact comparison, NOT a substring search; write $contains for containment. A pattern ending in a lone unpaired backslash is refused (INVALID_FILTER). Comparison is case-SENSITIVE, same contract as $contains (Q2 = A); $ilike is the case-insensitive twin. Answered by the SQL family (driver-sql, driver-sqlite-wasm, driver-turso on both transports), by driver-memory and by @objectstack/formula. driver-mongodb, objectql `having` and service-analytics REFUSE it in the INVALID_FILTER envelope rather than approximating it — see FILTER_OPERATORS for why it is staged out of that allowlist. | +| **$ilike** | `string` | optional | Whole-string pattern match like $like — "%" / "_" wildcards bound by the caller, backslash escapes — but ignoring ASCII case (A-Z against a-z) and ONLY ASCII case: "café" does NOT match "CAFÉ", the same Q1 = A boundary $icontains declares, because SQLite's fold is ASCII-only and three of the five backends are SQLite underneath. Staged with $like — see FILTER_OPERATORS. | +| **$null** | `boolean` | optional | Is-null check. `true` matches rows where the field is null, `false` matches rows where it is not null. Lowered to `IS NULL` (true) / `IS NOT NULL` (false) on the SQL family and to `{ field: null }` (true) / `{ $ne: null }` (false) on MongoDB. | +| **$exists** | `boolean` | optional | Has-a-value check — the exact inverse of `$null`. `true` matches rows where the field holds a value (`!= null`), `false` matches rows where it holds none. Portable across every backend the platform ships: lowered to `IS NOT NULL` (true) / `IS NULL` (false) on the SQL family and to `{ $ne: null }` (true) / `{ $eq: null }` (false) on MongoDB. | + + --- ## QueryFilter @@ -124,6 +304,17 @@ Type: `[FilterArray](#filterarray)[]` | **where** | `any` | optional | The condition tree the query filters by. A field-keyed entry is a condition on that field — a bare value is implicit equality, an object is a map of field operators — and `$and` / `$or` / `$not` combine conditions. `$not` is NULL-safe: a row whose compared column is null does NOT satisfy the negated condition and IS returned. An empty `$and` is the AND identity (no constraint); an empty `$or` is the OR identity (zero rows). | +--- + +## RangeOperator + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$between** | `any[]` | optional | Between (inclusive). Closed interval [min, max]. Each endpoint is a number, a Date, or a string. A `{ $field }` reference is NOT an endpoint shape: no backend resolves one inside a list — put it in a scalar comparison ($gt/$gte/$lt/$lte), which does compile to a column-to-column bound. STRING is the form the platform itself produces: the date-macro resolver walks INTO arrays, so `{ $between: ["{current_year_start}", "{current_year_end}"] }` resolves to two strings. The guaranteed spellings are an ISO calendar day (YYYY-MM-DD), a UTC ISO-8601 instant, or a wall-clock time of day (HH:MM[:SS[.fff]]) for a Field.time column; those are ASCII and fixed-width, so lexicographic order IS chronological order and every backend agrees. The driver reconciles each endpoint with the column independently (a bare calendar day used as the MAX becomes the half-open next-day boundary). Ranging over NON-temporal text is permitted but NOT promised: the order is the backend collation's, and those coincide only for ASCII. | + + --- ## SetOperator diff --git a/content/docs/references/data/hook.mdx b/content/docs/references/data/hook.mdx index bd12f4da9f..c91f44bf03 100644 --- a/content/docs/references/data/hook.mdx +++ b/content/docs/references/data/hook.mdx @@ -12,13 +12,73 @@ description: Hook protocol schemas ## TypeScript Usage ```typescript -import { HookContextSchema, HookEvent } from '@objectstack/spec/data'; -import type { HookContext } from '@objectstack/spec/data'; +import { HookSchema, HookContextSchema, HookEvent } from '@objectstack/spec/data'; +import type { Hook, HookContext } from '@objectstack/spec/data'; // Validate data -const result = HookContextSchema.parse(data); +const result = HookSchema.parse(data); ``` +--- + +## Hook + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **name** | `string` | ✅ | Hook unique name (snake_case) | +| **label** | `string` | optional | Description of what this hook does | +| **object** | `string \| string[]` | ✅ | Target object(s) | +| **events** | `Enum<'beforeFind' \| 'afterFind' \| 'beforeInsert' \| 'afterInsert' \| 'beforeUpdate' \| 'afterUpdate' \| 'beforeDelete' \| 'afterDelete'>[]` | ✅ | Lifecycle events | +| **handler** | `string` | optional | Handler function name (string, post-build) or inline function (pre-build) — DEPRECATED, prefer `body` | +| **body** | `{ language: 'expression'; source: string } \| { language: 'js'; source: string; capabilities?: Enum<'api.read' \| 'api.write' \| 'api.transaction' \| 'crypto.uuid' \| 'log'>[]; timeoutMs?: integer; … }` | optional | Hook body — expression (L1) or sandboxed JS (L2) | +| **priority** | `number` | optional (default: `100`) | Execution priority | +| **async** | `boolean` | optional (default: `false`) | Run specifically as fire-and-forget | +| **condition** | `string \| { dialect: Enum<'cel' \| 'cron' \| 'template'>; source?: string; ast?: any; meta?: object }` | optional | Predicate (CEL); hook runs only when TRUE. e.g. P`record.status == "closed" && record.amount > 1000` | +| **description** | `string` | optional | Human-readable description of what this hook does | +| **retryPolicy** | `{ maxRetries?: number; backoffMs?: number }` | optional | Retry policy for failed hook executions | +| **timeoutMs** | `number` | optional | Maximum execution time in milliseconds before the hook is aborted | +| **timeout** | `never` | optional | [REMOVED] `hook.timeout` was removed in @objectstack/spec 17 — its unit (milliseconds) lived only in the description, beside a body-level `timeoutMs` and a `retryPolicy.backoffMs` that spell theirs, so the same number read as two conventions on one surface. Rename the key to `timeoutMs`; the value (milliseconds) is unchanged. Run `os migrate meta --from 17` to list the mechanical edits for existing sources; apply them by hand. | +| **onError** | `Enum<'abort' \| 'log'>` | optional (default: `"abort"`) | Error handling strategy | +| **runAs** | `Enum<'system' \| 'user' \| 'inherit'>` | optional (default: `"inherit"`) | Execution identity for the hook's ctx.api data operations: system = elevated (bypasses RLS), user = the triggering user (RLS-respecting), inherit = the context of the write that fired the hook (the pre-runAs behaviour; the default). A hook with no trigger user has no identity to scope to, so under user its ctx.api data operations are REFUSED — declare system to make the elevation explicit. This covers any hook fired by a write that carried no user (an isSystem plugin/service write; a system-elevated flow node). Scope: ctx.api only — condition evaluation, the readonly strip on ctx.input, ctx.session and async are unchanged. | +| **_lock** | `Enum<'none' \| 'no-overlay' \| 'no-delete' \| 'full'>` | optional | Item-level lock — controls overlay & delete (ADR-0010). | +| **_lockReason** | `string` | optional | Human-readable reason shown when a write is refused by _lock. | +| **_lockSource** | `Enum<'artifact' \| 'package' \| 'env-forced'>` | optional | Layer that set _lock (artifact \| package \| env-forced). | +| **_provenance** | `Enum<'package' \| 'org' \| 'env-forced'>` | optional | Origin of the item (package \| org \| env-forced). | +| **_packageId** | `string` | optional | Owning package machine id. | +| **_packageVersion** | `string` | optional | Owning package version. | +| **_lockDocsUrl** | `string` | optional | Optional documentation link surfaced next to _lockReason. | + +### Nested Shape: `Hook.body[language='expression']` + +L1 expression body — pure formula, no IO + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **language** | `'expression'` | ✅ | | +| **source** | `string` | ✅ | Formula expression source | + +### Nested Shape: `Hook.body[language='js']` + +L2 sandboxed JS body — runs inside an isolated VM with declared capabilities + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **language** | `'js'` | ✅ | | +| **source** | `string` | ✅ | Function body source | +| **capabilities** | `Enum<'api.read' \| 'api.write' \| 'api.transaction' \| 'crypto.uuid' \| 'log'>[]` | optional (default: `[]`) | Granted capability tokens | +| **timeoutMs** | `integer` | optional | Per-invocation timeout (ms) | +| **memoryMb** | `integer` | optional | Per-invocation memory cap (MB) | + +### Nested Shape: `Hook.retryPolicy` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **maxRetries** | `number` | optional (default: `3`) | Maximum retry attempts on failure | +| **backoffMs** | `number` | optional (default: `1000`) | Backoff delay between retries in milliseconds | + + --- ## HookContext diff --git a/content/docs/references/index.mdx b/content/docs/references/index.mdx index 181a484f2e..2ee210f7c9 100644 --- a/content/docs/references/index.mdx +++ b/content/docs/references/index.mdx @@ -1,6 +1,6 @@ --- title: Protocol Reference -description: Every schema published by @objectstack/spec — 1577 schemas across 14 protocol modules +description: Every schema published by @objectstack/spec — 1582 schemas across 14 protocol modules --- {/* ⚠️ AUTO-GENERATED — DO NOT EDIT. Run build-docs.ts to regenerate. Hand-written docs live in the module folders under content/docs/. */} @@ -23,7 +23,7 @@ counts are sums of the rows they head. Regenerate with | [API Protocol](/docs/references/api) | 31 | 437 | REST contracts, endpoints, routing, realtime, batch, discovery. | | [Automation Protocol](/docs/references/automation) | 13 | 73 | Flows and their nodes, approvals, ETL pipelines, webhooks, state machines, execution records. | | [Cloud Protocol](/docs/references/cloud) | 11 | 94 | Environments, packages and versions, marketplace, developer portal, tenancy. | -| [Data Protocol](/docs/references/data) | 29 | 168 | Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. | +| [Data Protocol](/docs/references/data) | 29 | 173 | Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. | | [Identity Protocol](/docs/references/identity) | 5 | 27 | Users and accounts, organizations, positions, SCIM provisioning. | | [Integration Protocol](/docs/references/integration) | 1 | 24 | The single connector protocol (ADR-0097) — catalog descriptors and provider-bound instances. | | [Kernel Protocol](/docs/references/kernel) | 30 | 162 | Plugin lifecycle and manifests, capabilities and security, metadata loading, service registry. | @@ -33,7 +33,7 @@ counts are sums of the rows they head. Regenerate with | [Studio Protocol](/docs/references/studio) | 3 | 35 | Studio designer metadata — the authoring surfaces for the protocols above. | | [System Protocol](/docs/references/system) | 33 | 272 | The runtime environment — logging, jobs, cache, metrics, notifications, i18n and compliance. | | [UI Protocol](/docs/references/ui) | 16 | 153 | Apps, pages, views, dashboards, reports, actions and themes — the ObjectUI layer. | -| **Total** | **198** | **1577** | 14 protocol modules | +| **Total** | **198** | **1582** | 14 protocol modules | --- @@ -149,7 +149,7 @@ Environments, packages and versions, marketplace, developer portal, tenancy. ## Data Protocol -**Source:** `packages/spec/src/data/` · **Import:** `@objectstack/spec/data` · **29 pages, 168 schemas** +**Source:** `packages/spec/src/data/` · **Import:** `@objectstack/spec/data` · **29 pages, 173 schemas** Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. @@ -175,8 +175,8 @@ Objects, fields, queries, filters, datasources and drivers — the ObjectQL laye | [`feed.zod.ts`](/docs/references/data/feed) | `FeedFilterMode`, `FeedItemType` | | [`field.zod.ts`](/docs/references/data/field) | `CurrencyConfig`, `CurrencyValue`, `Field`, `FieldMaskingKeep`, `FieldMaskingRule`, `FieldType`, `InlineGridColumn`, `LocationCoordinates`, `SelectOption`, `UniqueScope` | | [`field-value.zod.ts`](/docs/references/data/field-value) | `Address`, `AddressValue`, `CalendarDateValue`, `ClockTimeValue`, `FileLikeValue`, `FileReferenceIdValue`, `FileValue`, `InstantValue`, `LocationValue`, `ReferenceIdValue` | -| [`filter.zod.ts`](/docs/references/data/filter) | `EqualityOperator`, `FieldReference`, `FilterArray`, `FilterCondition`, `QueryFilter`, `SetOperator`, `SpecialOperator`, `StringOperator` | -| [`hook.zod.ts`](/docs/references/data/hook) | `HookContext`, `HookEvent` | +| [`filter.zod.ts`](/docs/references/data/filter) | `ComparisonOperator`, `EqualityOperator`, `FieldOperators`, `FieldReference`, `FilterArray`, `FilterCondition`, `NormalizedFilter`, `QueryFilter`, `RangeOperator`, `SetOperator`, `SpecialOperator`, `StringOperator` | +| [`hook.zod.ts`](/docs/references/data/hook) | `Hook`, `HookContext`, `HookEvent` | | [`hook-body.zod.ts`](/docs/references/data/hook-body) | `ExpressionBody`, `HookBody`, `HookBodyCapability`, `ScriptBody` | | [`mapping.zod.ts`](/docs/references/data/mapping) | `ImportFieldMapping`, `Mapping`, `TransformType` | | [`object.zod.ts`](/docs/references/data/object) | `ApiMethod`, `ApiOperation`, `Index`, `Lifecycle`, `LifecycleClass`, `Object`, `ObjectAccessConfig`, `ObjectCapabilities`, `ObjectExtension`, `ObjectExternalBinding`, `ObjectFieldGroup`, `ObjectOwnershipEnum`, `ObjectRequiredPermissions`, `PerOperationRequiredPermissions`, `RowCrudActionOverride`, `TenancyConfig` | diff --git a/packages/spec/docs-import-surface.baseline.json b/packages/spec/docs-import-surface.baseline.json index 3b049e5b8f..e406fbb179 100644 --- a/packages/spec/docs-import-surface.baseline.json +++ b/packages/spec/docs-import-surface.baseline.json @@ -11,6 +11,7 @@ "api/MetadataQueryRequest — no type export", "automation/FlowVariable — no type export", "automation/StateMachine — no type export", + "data/ComparisonOperator — no type export", "data/DataEngineAggregateRequest — no type export", "data/DataEngineCountRequest — no type export", "data/DataEngineDeleteRequest — no type export", @@ -24,6 +25,7 @@ "data/Object — no type export", "data/ObjectOwnershipEnum — no type export", "data/Query — no type export", + "data/RangeOperator — no type export", "data/SeedMode — no type export", "data/SetOperator — no type export", "data/SpecialOperator — no type export", From cdd75438512ddcab315dfca846901f7080b75b7b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 04:27:53 +0000 Subject: [PATCH 3/5] wip: projection unit tests --- .../scripts/union-branch-projection.test.ts | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 packages/spec/scripts/union-branch-projection.test.ts diff --git a/packages/spec/scripts/union-branch-projection.test.ts b/packages/spec/scripts/union-branch-projection.test.ts new file mode 100644 index 0000000000..7e407ad51c --- /dev/null +++ b/packages/spec/scripts/union-branch-projection.test.ts @@ -0,0 +1,231 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// Pins the per-branch JSON-Schema projection (#16431 option (a)) — the third +// attempt `build-schemas.ts` makes after `z.toJSONSchema()` has refused a whole +// export in both `io` directions. +// +// The defect it exists for: `orderingComparandSchema` is +// `z.union([z.number(), z.date(), z.string(), FieldReferenceSchema])`, Zod +// refuses the WHOLE schema over the one member with no JSON form, and so +// `$gt` / `$gte` / `$lt` / `$lte` / `$between` — ~2000 characters of `.describe()` +// carrying the #5685 comparand contract and the #6571 endpoint contract — +// reached no reference row at all. Not a blank cell: no section. +// +// Two directions have to be pinned, and the second is the one that matters: +// +// 1. a union branch with no JSON form is DROPPED, so the export publishes; +// 2. an unprojectable node anywhere ELSE still refuses the projection, so the +// export is skipped exactly as before. The mechanism is Zod's +// `unrepresentable: 'any'`, which turns every unrepresentable node into +// `{}` — and `{}` inside an `anyOf` accepts EVERY JSON value. A regression +// that let one of those through would publish a universally-permissive +// schema and report nothing, which is a worse version of the silence this +// card was filed about. + +import { describe, it, expect } from 'vitest'; +import { z } from 'zod'; + +import { + UNPROJECTABLE_MARK, + findSurvivingMark, + markUnprojectableNodes, + projectByPruningUnionBranches, + pruneMarkedUnionBranches, + type PrunedBranch, +} from './lib/union-branch-projection'; +import { + ComparisonOperatorSchema, + FieldOperatorsSchema, + NormalizedFilterSchema, + PersistenceAdapterSchema, + RangeOperatorSchema, +} from '../src/data'; + +const TARGET = { target: 'draft-2020-12' } as const; + +/** Convert with the marker override, the way the projection itself does. */ +function markedProjection(schema: z.ZodType, io: 'output' | 'input' = 'output'): Record { + return z.toJSONSchema(schema, { + target: 'draft-2020-12', + unrepresentable: 'any', + override: markUnprojectableNodes(io), + ...(io === 'input' ? { io } : {}), + }) as Record; +} + +/** Every value at `key`, anywhere in a JSON tree. */ +function collect(node: unknown, key: string, into: unknown[] = []): unknown[] { + if (Array.isArray(node)) { + for (const item of node) collect(item, key, into); + return into; + } + if (typeof node !== 'object' || node === null) return into; + for (const [k, v] of Object.entries(node as Record)) { + if (k === key) into.push(v); + collect(v, key, into); + } + return into; +} + +describe('markUnprojectableNodes — what counts as "no JSON form"', () => { + it('marks a bare z.date(), in BOTH io directions', () => { + // The premise #16431 recorded — that the existing `io: 'input'` fallback + // would project a date branch if applied per branch — is false, and this is + // why: `dateProcessor` reads only `ctx.unrepresentable`, never `ctx.io`. + for (const io of ['output', 'input'] as const) { + expect(markedProjection(z.date(), io)[UNPROJECTABLE_MARK]).toBe('date'); + expect(() => z.toJSONSchema(z.date(), { ...TARGET, io })).toThrow( + /Date cannot be represented in JSON Schema/, + ); + } + }); + + it('marks a described z.date(), which an emptiness test alone would miss', () => { + // `.describe()` lands on the node BEFORE the override runs, so this comes + // back as `{ description }` — non-empty, and indistinguishable from a real + // schema to `Object.keys().length === 0`. + const marked = markedProjection(z.date().describe('a real Date instance')); + expect(marked).toMatchObject({ description: 'a real Date instance', [UNPROJECTABLE_MARK]: 'date' }); + }); + + it('does NOT mark z.any() / z.unknown(), which accept any JSON value legitimately', () => { + expect(markedProjection(z.any())).not.toHaveProperty(UNPROJECTABLE_MARK); + expect(markedProjection(z.unknown())).not.toHaveProperty(UNPROJECTABLE_MARK); + }); + + it('does NOT mark z.any() through wrappers — the `$eq` shape', () => { + // `FieldOperators.$eq` is `z.any().optional().describe(EQ_DESCRIPTION)`. It + // projects to `{ description }`, byte-shaped exactly like the unprojectable + // date above, and marking it refused the whole projection for the ENFORCED + // half of the filter contract. Only a strict re-conversion tells them apart. + const eqShape = z.any().optional().describe('Equal to — the DEFAULT operator'); + expect(markedProjection(z.object({ $eq: eqShape })).properties).toMatchObject({ + $eq: { description: 'Equal to — the DEFAULT operator' }, + }); + expect(findSurvivingMark(markedProjection(z.object({ $eq: eqShape })))).toBeNull(); + }); +}); + +describe('pruneMarkedUnionBranches — only a union member is droppable', () => { + it('drops the marked branch and names where it was', () => { + const schema = markedProjection(z.object({ $gt: z.union([z.number(), z.date(), z.string()]) })); + const pruned: PrunedBranch[] = []; + pruneMarkedUnionBranches(schema, '#', pruned); + + expect(pruned).toEqual([{ at: '#/properties/$gt/anyOf/1', type: 'date' }]); + expect(findSurvivingMark(schema)).toBeNull(); + expect((schema.properties as Record }>).$gt.anyOf) + .toEqual([{ type: 'number' }, { type: 'string' }]); + }); + + it('marks a union that loses EVERY branch, so the parent can drop it in turn', () => { + const schema = markedProjection(z.union([z.date(), z.function()])); + pruneMarkedUnionBranches(schema, '#', []); + expect(schema).not.toHaveProperty('anyOf'); + expect(schema[UNPROJECTABLE_MARK]).toBe('union'); + }); + + it('leaves a marked node in a PROPERTY position for the caller to refuse', () => { + const schema = markedProjection(z.object({ handler: z.function(), name: z.string() })); + pruneMarkedUnionBranches(schema, '#', []); + expect(findSurvivingMark(schema)).toBe('#/properties/handler'); + }); +}); + +describe('projectByPruningUnionBranches — the contract build-schemas.ts relies on', () => { + it('projects the ordering comparand union without its Date branch', () => { + const projected = projectByPruningUnionBranches( + z.object({ $gt: z.union([z.number(), z.date(), z.string()]).optional() }), + TARGET, + ); + expect(projected).not.toBeNull(); + expect(projected!.pruned).toEqual([{ at: '#/properties/$gt/anyOf/1', type: 'date' }]); + // What an author writes still validates: a JSON document can carry the + // string and the number, and can never carry a Date instance, so the set of + // valid JSON documents is unchanged by the drop. + expect((projected!.schema.properties as Record).$gt.anyOf) + .toEqual([{ type: 'number' }, { type: 'string' }]); + }); + + it('refuses when an unprojectable node is NOT a union member', () => { + // Dropping a required `handler` would publish a shape no runtime value has. + expect(projectByPruningUnionBranches(z.object({ handler: z.function() }), TARGET)).toBeNull(); + expect(projectByPruningUnionBranches(z.record(z.string(), z.function()), TARGET)).toBeNull(); + expect(projectByPruningUnionBranches(z.array(z.date()), TARGET)).toBeNull(); + }); + + it('refuses a union whose every branch is unprojectable', () => { + expect(projectByPruningUnionBranches(z.union([z.date(), z.function()]), TARGET)).toBeNull(); + }); + + it('returns null when there was nothing to drop', () => { + expect(projectByPruningUnionBranches(z.object({ a: z.string() }), TARGET)).toBeNull(); + }); + + it('prefers the direction that drops FEWER branches, not output-first', () => { + // A `.transform()` has no OUTPUT form and a perfectly good input one. An + // output-first rule would drop it as if it had no JSON form at all and + // publish the narrower schema — deleting an authorable shape silently. + const withTransform = z.union([ + z.date(), + z.string().transform((s) => s.length), + z.number(), + ]); + const projected = projectByPruningUnionBranches(withTransform, TARGET); + expect(projected).not.toBeNull(); + expect(projected!.io).toBe('input'); + expect(projected!.pruned.map((b) => b.type)).toEqual(['date']); + expect(projected!.schema.anyOf).toEqual([{ type: 'string' }, { type: 'number' }]); + }); + + it('never returns a schema still carrying a marker, and never an empty `{}` branch', () => { + const projected = projectByPruningUnionBranches(ComparisonOperatorSchema, TARGET); + expect(projected).not.toBeNull(); + expect(findSurvivingMark(projected!.schema)).toBeNull(); + expect(JSON.stringify(projected!.schema)).not.toContain(UNPROJECTABLE_MARK); + for (const branches of collect(projected!.schema, 'anyOf') as unknown[][]) { + for (const branch of branches) { + expect(Object.keys(branch as Record).length).toBeGreaterThan(0); + } + } + }); +}); + +describe('the four filter exports #16431 measured, and the boundary beside them', () => { + it.each([ + ['ComparisonOperatorSchema', ComparisonOperatorSchema, 4], + ['RangeOperatorSchema', RangeOperatorSchema, 2], + ['FieldOperatorsSchema', FieldOperatorsSchema, 6], + ['NormalizedFilterSchema', NormalizedFilterSchema, 18], + ])('%s projects, dropping only Date branches', (_name, schema, dropped) => { + // Every one of these throws in both strict directions today — that is the + // whole finding — so the strict pass is asserted first, or the projection + // could be passing for a reason that has nothing to do with this change. + for (const io of ['output', 'input'] as const) { + expect(() => z.toJSONSchema(schema as z.ZodType, { ...TARGET, io })).toThrow( + /cannot be represented in JSON Schema/, + ); + } + const projected = projectByPruningUnionBranches(schema as z.ZodType, TARGET); + expect(projected).not.toBeNull(); + expect(projected!.pruned).toHaveLength(dropped); + expect(new Set(projected!.pruned.map((b) => b.type))).toEqual(new Set(['date'])); + }); + + it('publishes the five operators the card named, with their prose intact', () => { + const comparison = projectByPruningUnionBranches(ComparisonOperatorSchema, TARGET)!; + const range = projectByPruningUnionBranches(RangeOperatorSchema, TARGET)!; + const slots = comparison.schema.properties as Record; + for (const op of ['$gt', '$gte', '$lt', '$lte']) { + expect(slots[op]?.description).toContain('null is NOT a comparand'); + } + expect((range.schema.properties as Record).$between?.description) + .toContain('$between'); + }); + + it('leaves a driver interface of z.function() members skipped', () => { + // The population the #16431 ratchet holds closed must not be emptied by a + // projection that publishes shapes nobody authors. + expect(projectByPruningUnionBranches(PersistenceAdapterSchema, TARGET)).toBeNull(); + }); +}); From 28a5549831820c61c661ff41abd9a214c0ca1817 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 04:28:38 +0000 Subject: [PATCH 4/5] wip: changeset --- .../filter-operator-schema-projection.md | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .changeset/filter-operator-schema-projection.md diff --git a/.changeset/filter-operator-schema-projection.md b/.changeset/filter-operator-schema-projection.md new file mode 100644 index 0000000000..015ca262e1 --- /dev/null +++ b/.changeset/filter-operator-schema-projection.md @@ -0,0 +1,34 @@ +--- +'@objectstack/spec': patch +--- + +fix(spec): project a union branch-by-branch, so five filter operators reach a published reference page + +`z.toJSONSchema()` refuses a whole schema the moment ONE node in it has no JSON +form, and `build-schemas.ts` applied that refusal per SCHEMA. `orderingComparandSchema` +is `z.union([z.number(), z.date(), z.string(), FieldReferenceSchema])`, so four +`data/filter.zod.ts` exports emitted nothing at all — and `$gt`, `$gte`, `$lt`, +`$lte` and `$between` reached no reference row. Not a blank Description cell: no +section. The ~2000 characters of `.describe()` on those slots — the #5685 comparand +contract, the #6571 endpoint contract, and the `{ "$gte": "2026-01-01" }` shape the +platform's own date-macro resolver produces — reached no reader. + +The generator now makes a third attempt when both strict directions refuse: it +projects with Zod's `unrepresentable: 'any'`, marks every node that came back with +no structural keyword, and DROPS the marked ones that are direct members of an +`anyOf` / `oneOf`. That is not a narrowing. These artifacts describe JSON +documents, a JSON document cannot carry a `Date` INSTANCE, so the set of JSON +documents that union accepts is unchanged by the drop. + +⛔ A marked node anywhere else — an object property, a record value, an array item +— refuses the projection and the export is skipped with the message Zod threw, so +this cannot change WHY anything is skipped. Five exports leave +`unemitted-schemas.baseline.json` (23 → 18): the four filter exports, plus +`data/Hook`, whose only unprojectable member was the deprecated inline-function +handler branch — that puts 22 `data/Hook:` authorable keys under the key ratchet +for the first time. + +Published artifacts gain `json-schema/data/{ComparisonOperator,FieldOperators, +NormalizedFilter,RangeOperator,Hook}.json`, each carrying an +`x-unprojectable-branches` record naming exactly which branch the projection +dropped and where. From 464d2f3762719a01392a0ef89303323afd1d9341 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 04:35:41 +0000 Subject: [PATCH 5/5] wip: declaration-map --- packages/spec/declaration-map/data.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/spec/declaration-map/data.json b/packages/spec/declaration-map/data.json index 3f29c42592..1db92a4e2d 100644 --- a/packages/spec/declaration-map/data.json +++ b/packages/spec/declaration-map/data.json @@ -32,6 +32,7 @@ "CalendarDateValueSchema": "data/CalendarDateValue", "ClockTimeValue": "data/ClockTimeValue", "ClockTimeValueSchema": "data/ClockTimeValue", + "ComparisonOperatorSchema": "data/ComparisonOperator", "ConditionalValidation": "data/ConditionalValidation", "ConditionalValidationSchema": "data/ConditionalValidation", "ConsistencyLevel": "data/ConsistencyLevel", @@ -145,6 +146,8 @@ "FieldMaskingRuleSchema": "data/FieldMaskingRule", "FieldNode": "data/FieldNode", "FieldNodeSchema": "data/FieldNode", + "FieldOperators": "data/FieldOperators", + "FieldOperatorsSchema": "data/FieldOperators", "FieldReference": "data/FieldReference", "FieldReferenceSchema": "data/FieldReference", "FieldSchema": "data/Field", @@ -167,12 +170,14 @@ "FullTextSearchSchema": "data/FullTextSearch", "GroupByNode": "data/GroupByNode", "GroupByNodeSchema": "data/GroupByNode", + "Hook": "data/Hook", "HookBody": "data/HookBody", "HookBodyCapability": "data/HookBodyCapability", "HookBodySchema": "data/HookBody", "HookContext": "data/HookContext", "HookContextSchema": "data/HookContext", "HookEvent": "data/HookEvent", + "HookSchema": "data/Hook", "ImportFieldMapping": "data/ImportFieldMapping", "ImportFieldMappingSchema": "data/ImportFieldMapping", "IndexSchema": "data/Index", @@ -216,6 +221,8 @@ "NoSQLQueryOptionsSchema": "data/NoSQLQueryOptions", "NoSQLTransactionOptions": "data/NoSQLTransactionOptions", "NoSQLTransactionOptionsSchema": "data/NoSQLTransactionOptions", + "NormalizedFilter": "data/NormalizedFilter", + "NormalizedFilterSchema": "data/NormalizedFilter", "ObjectAccessConfig": "data/ObjectAccessConfig", "ObjectAccessConfigSchema": "data/ObjectAccessConfig", "ObjectCapabilities": "data/ObjectCapabilities", @@ -245,6 +252,7 @@ "QueryFilter": "data/QueryFilter", "QueryFilterSchema": "data/QueryFilter", "QuerySchema": "data/Query", + "RangeOperatorSchema": "data/RangeOperator", "ReferenceIdValue": "data/ReferenceIdValue", "ReferenceIdValueSchema": "data/ReferenceIdValue", "ReferenceResolution": "data/ReferenceResolution",