diff --git a/.changeset/fix-correlated-include-routes.md b/.changeset/fix-correlated-include-routes.md new file mode 100644 index 0000000000..11f171fc16 --- /dev/null +++ b/.changeset/fix-correlated-include-routes.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Preserve parent-specific include results through nested includes, subqueries, unions, joins, ordering, projections, aggregates, and having clauses. diff --git a/packages/db/package.json b/packages/db/package.json index ebed26a22b..03e905075f 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts" + "test:oracles": "vitest --run tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/query/builder/index.ts b/packages/db/src/query/builder/index.ts index 0291c204cd..5b262ab85f 100644 --- a/packages/db/src/query/builder/index.ts +++ b/packages/db/src/query/builder/index.ts @@ -1106,14 +1106,17 @@ function buildConditionalSelect( /** * Recursively collects all PropRef nodes from an expression tree. */ -function collectRefsFromExpression(expr: BasicExpression): Array { +function collectRefsFromExpression( + expr: BasicExpression | Aggregate, +): Array { const refs: Array = [] switch (expr.type) { case `ref`: refs.push(expr) break case `func`: - for (const arg of (expr as any).args ?? []) { + case `agg`: + for (const arg of expr.args) { refs.push(...collectRefsFromExpression(arg)) } break @@ -1123,6 +1126,164 @@ function collectRefsFromExpression(expr: BasicExpression): Array { return refs } +function collectRefsFromSelectValue(value: unknown): Array { + if ( + value instanceof PropRef || + value instanceof FuncExpr || + value instanceof AggregateExpr + ) { + return collectRefsFromExpression(value) + } + if (value instanceof ConditionalSelect) { + return [ + ...value.branches.flatMap((branch) => [ + ...collectRefsFromExpression(branch.condition), + ...collectRefsFromSelectValue(branch.value), + ]), + ...(value.defaultValue === undefined + ? [] + : collectRefsFromSelectValue(value.defaultValue)), + ] + } + if (value instanceof IncludesSubquery) { + return [ + value.correlationField, + ...(value.parentProjection ?? []), + ...collectExternalRefsFromQuery(value.query), + ] + } + if (!isPlainObject(value)) return [] + return Object.values(value).flatMap(collectRefsFromSelectValue) +} + +function collectExternalRefsFromQuery(query: QueryIR): Array { + const localAliases = new Set(collectQueryAliases(query)) + const refs: Array = [] + const addExpression = (expression: BasicExpression | Aggregate) => { + refs.push(...collectRefsFromExpression(expression)) + } + const addWhere = (where: Where) => { + addExpression( + typeof where === `object` && `expression` in where + ? where.expression + : where, + ) + } + + for (const where of query.where ?? []) addWhere(where) + for (const join of query.join ?? []) { + addExpression(join.left) + addExpression(join.right) + if (join.from.type === `queryRef`) { + refs.push(...collectExternalRefsFromQuery(join.from.query)) + } + } + for (const expression of query.groupBy ?? []) addExpression(expression) + for (const having of query.having ?? []) addWhere(having) + for (const { expression } of query.orderBy ?? []) addExpression(expression) + if (query.select) refs.push(...collectRefsFromSelectValue(query.select)) + + if (query.from.type === `queryRef`) { + refs.push(...collectExternalRefsFromQuery(query.from.query)) + } else if (query.from.type === `unionFrom`) { + for (const source of query.from.sources) { + if (source.type === `queryRef`) { + refs.push(...collectExternalRefsFromQuery(source.query)) + } + } + } else if (query.from.type === `unionAll`) { + for (const branch of query.from.queries) { + refs.push(...collectExternalRefsFromQuery(branch)) + } + } + + const seen = new Set() + return refs.filter((ref) => { + const alias = ref.path.length > 1 ? ref.path[0] : undefined + const path = ref.path.join(`.`) + if ( + alias == null || + alias === `$selected` || + localAliases.has(alias) || + seen.has(path) + ) { + return false + } + seen.add(path) + return true + }) +} + +function collectParentRefsFromQuery( + query: QueryIR, + parentAliases: Array, +): Array { + const refs: Array = [] + const addExpression = (expression: BasicExpression | Aggregate) => { + refs.push(...collectRefsFromExpression(expression)) + } + const addWhere = (where: Where) => { + addExpression( + typeof where === `object` && `expression` in where + ? where.expression + : where, + ) + } + + for (const where of query.where ?? []) addWhere(where) + for (const join of query.join ?? []) { + addExpression(join.left) + addExpression(join.right) + if (join.from.type === `queryRef`) { + refs.push(...collectParentRefsFromQuery(join.from.query, parentAliases)) + } + } + for (const expression of query.groupBy ?? []) addExpression(expression) + for (const having of query.having ?? []) addWhere(having) + for (const { expression } of query.orderBy ?? []) addExpression(expression) + if (query.select) { + refs.push(...collectRefsFromSelectValue(query.select)) + } + + if (query.from.type === `queryRef`) { + refs.push(...collectParentRefsFromQuery(query.from.query, parentAliases)) + } else if (query.from.type === `unionFrom`) { + for (const source of query.from.sources) { + if (source.type === `queryRef`) { + refs.push(...collectParentRefsFromQuery(source.query, parentAliases)) + } + } + } else if (query.from.type === `unionAll`) { + for (const branch of query.from.queries) { + refs.push(...collectParentRefsFromQuery(branch, parentAliases)) + } + } + + const seen = new Set() + return refs.filter((ref) => { + const path = ref.path.join(`.`) + if ( + ref.path[0] == null || + !parentAliases.includes(ref.path[0]) || + seen.has(path) + ) { + return false + } + seen.add(path) + return true + }) +} + +function collectExternalParentAliases(query: QueryIR): Array { + return [ + ...new Set( + collectExternalRefsFromQuery(query) + .map((ref) => ref.path[0]) + .filter((alias): alias is string => alias !== undefined), + ), + ] +} + /** * Checks whether a WHERE clause references any parent alias. */ @@ -1151,6 +1312,9 @@ function buildIncludesSubquery( // Collect child's own aliases const childAliases = collectQueryAliases(childQuery) + const visibleParentAliases = [ + ...new Set([...parentAliases, ...collectExternalParentAliases(childQuery)]), + ] // Walk child's WHERE clauses to find the correlation condition. // The correlation eq() may be a standalone WHERE or nested inside a top-level and(). @@ -1176,7 +1340,7 @@ function buildIncludesSubquery( const result = extractCorrelation( expr.args[0]!, expr.args[1]!, - parentAliases, + visibleParentAliases, childAliases, ) if (result) { @@ -1203,7 +1367,7 @@ function buildIncludesSubquery( const result = extractCorrelation( arg.args[0]!, arg.args[1]!, - parentAliases, + visibleParentAliases, childAliases, ) if (result) { @@ -1264,32 +1428,21 @@ function buildIncludesSubquery( const pureChildWhere: Array = [] const parentFilters: Array = [] for (const w of modifiedWhere) { - if (referencesParent(w, parentAliases)) { + if (referencesParent(w, visibleParentAliases)) { parentFilters.push(w) } else { pureChildWhere.push(w) } } - // Collect distinct parent PropRefs from parent-referencing filters - let parentProjection: Array | undefined - if (parentFilters.length > 0) { - const seen = new Set() - parentProjection = [] - for (const w of parentFilters) { - const expr = typeof w === `object` && `expression` in w ? w.expression : w - for (const ref of collectRefsFromExpression(expr)) { - if ( - ref.path[0] != null && - parentAliases.includes(ref.path[0]) && - !seen.has(ref.path.join(`.`)) - ) { - seen.add(ref.path.join(`.`)) - parentProjection.push(ref) - } - } - } - } + // Every parent input that can affect the child plan belongs to the route + // identity, not only the main equality key or residual filters. + const projectedParentRefs = collectParentRefsFromQuery( + { ...childQuery, where: modifiedWhere }, + visibleParentAliases, + ) + const parentProjection = + projectedParentRefs.length > 0 ? projectedParentRefs : undefined const modifiedQuery: QueryIR = { ...childQuery, diff --git a/packages/db/src/query/builder/types.ts b/packages/db/src/query/builder/types.ts index db20942b59..6286403e27 100644 --- a/packages/db/src/query/builder/types.ts +++ b/packages/db/src/query/builder/types.ts @@ -93,7 +93,7 @@ export type Source = { [alias: string]: | CollectionImpl | CollectionOptionsIdentity - | QueryBuilder + | QueryBuilder } /** @@ -168,7 +168,7 @@ export type ContextFromUnionSource = : ContextFromSource type ResultFromBranch = - TBranch extends QueryBuilder ? GetResult : never + TBranch extends QueryBuilder ? GetRawResult : never type UnionBranchResult>> = ResultFromBranch diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index c670de9649..751d1e577e 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -43,6 +43,50 @@ type RowVirtualMetadata = { hasLocal: boolean } +function addCorrelationRouteToGroupKey( + key: Record, + row: NamespacedRow, + mainSource: string, +): void { + const rowRecord = row as Record + const source = rowRecord[mainSource] as Record | undefined + key.__correlationKey = source?.__correlationKey + if (rowRecord.__parentContext != null) { + key.__parentContext = rowRecord.__parentContext + } +} + +function getCorrelationRouteIdentity( + aggregatedRow: Record, +): unknown { + return aggregatedRow.__parentContext == null + ? aggregatedRow.__correlationKey + : [aggregatedRow.__correlationKey, aggregatedRow.__parentContext] +} + +function getHavingEvaluationRow(row: Record): NamespacedRow { + const parentContext = row.__parentContext + return { + ...(parentContext !== null && typeof parentContext === `object` + ? (parentContext as NamespacedRow) + : {}), + $selected: row.$selected as Record, + } +} + +function getWrappedAggregateEvaluationRow( + row: Record, + selected: Record, +): NamespacedRow { + const parentContext = row.__parentContext + return { + ...(parentContext !== null && typeof parentContext === `object` + ? (parentContext as NamespacedRow) + : {}), + $selected: selected, + } +} + function getRowVirtualMetadata(row: NamespacedRow): RowVirtualMetadata { let found = false let allSynced = true @@ -189,15 +233,13 @@ export function processGroupBy( } } - // Use a constant key for single group. - // When mainSource is set (includes mode), include __correlationKey so that - // rows from different parents aggregate separately. - const keyExtractor = mainSource - ? ([, row]: [string, NamespacedRow]) => ({ - __singleGroup: true, - __correlationKey: (row as any)?.[mainSource]?.__correlationKey, - }) - : () => ({ __singleGroup: true }) + // Use a constant key for single group. In includes mode, add the complete + // correlation route so parents with distinct projected inputs stay apart. + const keyExtractor = ([, row]: [string, NamespacedRow]) => { + const key: Record = { __singleGroup: true } + if (mainSource) addCorrelationRouteToGroupKey(key, row, mainSource) + return key + } // Apply the groupBy operator with single group pipeline = pipeline.pipe( @@ -231,9 +273,12 @@ export function processGroupBy( const correlationKey = mainSource ? (aggregatedRow as any).__correlationKey : undefined + const correlationRoute = mainSource + ? getCorrelationRouteIdentity(aggregatedRow) + : undefined const resultKey = - correlationKey !== undefined - ? `single_group_${serializeValue(correlationKey)}` + correlationRoute !== undefined + ? `single_group_${serializeValue(correlationRoute)}` : `single_group` const resultRow: Record = { ...(aggregatedRow as Record), @@ -272,8 +317,7 @@ export function processGroupBy( pipeline = pipeline.pipe( filter(([, row]) => { - // Create a namespaced row structure for HAVING evaluation - const namespacedRow = { $selected: (row as any).$selected } + const namespacedRow = getHavingEvaluationRow(row) return toBooleanPredicate(compiledHaving(namespacedRow)) }), ) @@ -285,8 +329,7 @@ export function processGroupBy( for (const fnHaving of fnHavingClauses) { pipeline = pipeline.pipe( filter(([, row]) => { - // Create a namespaced row structure for functional HAVING evaluation - const namespacedRow = { $selected: (row as any).$selected } + const namespacedRow = getHavingEvaluationRow(row) return toBooleanPredicate(fnHaving(namespacedRow)) }), ) @@ -305,9 +348,9 @@ export function processGroupBy( compileExpression(e), ) - // Create a key extractor function using simple __key_X format. - // When mainSource is set (includes mode), include __correlationKey so that - // rows from different parents with the same group key aggregate separately. + // Create a key extractor function using simple __key_X format. In includes + // mode, add the complete route so parents with distinct projected inputs do + // not aggregate together. const keyExtractor = ([, row]: [ string, NamespacedRow & { $selected?: any }, @@ -325,9 +368,7 @@ export function processGroupBy( key[`__key_${i}`] = value } - if (mainSource) { - key.__correlationKey = (row as any)?.[mainSource]?.__correlationKey - } + if (mainSource) addCorrelationRouteToGroupKey(key, row, mainSource) return key } @@ -397,17 +438,20 @@ export function processGroupBy( } // Generate a simple key for the live collection using group values. - // When in includes mode, include the correlation key so that groups - // from different parents don't collide. + // In includes mode, add the complete route so correlated groups do not + // collide. const correlationKey = mainSource ? (aggregatedRow as any).__correlationKey : undefined + const correlationRoute = mainSource + ? getCorrelationRouteIdentity(aggregatedRow) + : undefined const keyParts: Array = [] for (let i = 0; i < groupByClause.length; i++) { keyParts.push(aggregatedRow[`__key_${i}`]) } - if (correlationKey !== undefined) { - keyParts.push(correlationKey) + if (correlationRoute !== undefined) { + keyParts.push(correlationRoute) } const finalKey = keyParts.length === 1 ? keyParts[0] : serializeValue(keyParts) @@ -449,8 +493,7 @@ export function processGroupBy( pipeline = pipeline.pipe( filter(([, row]) => { - // Create a namespaced row structure for HAVING evaluation - const namespacedRow = { $selected: (row as any).$selected } + const namespacedRow = getHavingEvaluationRow(row) return compiledHaving(namespacedRow) }), ) @@ -462,8 +505,7 @@ export function processGroupBy( for (const fnHaving of fnHavingClauses) { pipeline = pipeline.pipe( filter(([, row]) => { - // Create a namespaced row structure for functional HAVING evaluation - const namespacedRow = { $selected: (row as any).$selected } + const namespacedRow = getHavingEvaluationRow(row) return toBooleanPredicate(fnHaving(namespacedRow)) }), ) @@ -647,7 +689,9 @@ function evaluateWrappedAggregates( finalResults[`${GROUP_KEY_REF_PREFIX}${i}`] = aggregatedRow[`__key_${i}`] } for (const [alias, evaluator] of Object.entries(wrappedAggExprs)) { - finalResults[alias] = evaluator({ $selected: finalResults }) + finalResults[alias] = evaluator( + getWrappedAggregateEvaluationRow(aggregatedRow, finalResults), + ) } for (const key of Object.keys(finalResults)) { if (key.startsWith(`__agg_`) || key.startsWith(GROUP_KEY_REF_PREFIX)) { diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index d075539213..a0bba00e34 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -30,6 +30,7 @@ import { } from '../ir.js' import { ensureIndexForField } from '../../indexes/auto-index.js' import { deepEquals } from '../../utils.js' +import { normalizeValue } from '../../utils/comparison.js' import { compileExpression, isCaseWhenConditionTrue, @@ -39,6 +40,12 @@ import { processJoins, registerLazyDemandPlan } from './joins.js' import { containsAggregate, processGroupBy } from './group-by.js' import { getLazyLoadTargets } from './lazy-targets.js' import { processOrderBy } from './order-by.js' +import { crossJoinParentRoutes } from './parent-routes.js' +import { + INCLUDES_PUBLIC_KEY, + attachRouteMetadataToResult, + getRoutedScalarMetadata, +} from './route-metadata.js' import { processSelect } from './select.js' import type { CollectionSubscription } from '../../collection/subscription.js' import type { OrderByOptimizationInfo } from './order-by.js' @@ -62,10 +69,10 @@ import type { import type { QueryCache, QueryMapping, WindowOptions } from './types.js' export type { WindowOptions } from './types.js' +export { INCLUDES_PUBLIC_KEY } from './route-metadata.js' /** Symbol used to tag parent $selected with routing metadata for includes */ export const INCLUDES_ROUTING = Symbol(`includesRouting`) -export const INCLUDES_PUBLIC_KEY = Symbol(`includesPublicKey`) export const FN_SELECT_STATE = Symbol(`fnSelectState`) const SKIP_INCLUDE = Symbol(`skipInclude`) @@ -84,6 +91,103 @@ type ProjectedSourceIncludePath = { guards: Array } +type CompiledParentProjection = { + alias: string + field: Array + compiled: (row: NamespacedRow) => unknown +} + +function projectParentContext( + nsRow: NamespacedRow, + projections: Array, +): Record { + const inherited = (nsRow as any).__parentContext + const parentContext: Record = + inherited != null && typeof inherited === `object` ? { ...inherited } : {} + + for (const projection of projections) { + if (projection.field.length === 0) { + const projectedAlias = projection.compiled(nsRow) + parentContext[projection.alias] = + projectedAlias != null && typeof projectedAlias === `object` + ? { ...projectedAlias } + : projectedAlias + continue + } + + const inheritedAlias = parentContext[projection.alias] + const aliasContext = + inheritedAlias != null && typeof inheritedAlias === `object` + ? { ...inheritedAlias } + : {} + parentContext[projection.alias] = aliasContext + + let target = aliasContext + for (let index = 0; index < projection.field.length - 1; index++) { + const segment = projection.field[index]! + const inheritedNested = target[segment] + const nested = + inheritedNested != null && typeof inheritedNested === `object` + ? { ...inheritedNested } + : {} + target[segment] = nested + target = nested + } + target[projection.field[projection.field.length - 1]!] = + projection.compiled(nsRow) + } + + return parentContext +} + +function parameterizeByParentRoutes( + pipeline: NamespacedAndKeyedStream, + parentKeyStream: KeyedStream, + mainSource: string, +): NamespacedAndKeyedStream { + return crossJoinParentRoutes( + pipeline, + parentKeyStream, + (rowKey, row, correlationKey, parentContext) => { + const namespaced = { + ...(row as Record), + } as Record + namespaced[mainSource] = { + ...namespaced[mainSource], + __correlationKey: correlationKey, + [INCLUDES_PUBLIC_KEY]: + namespaced[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? rowKey, + } + if (parentContext != null) Object.assign(namespaced, parentContext) + namespaced.__correlationKey = correlationKey + namespaced.__parentContext = parentContext + return [ + serializeValue([rowKey, correlationKey, parentContext]), + namespaced, + ] as [string, NamespacedRow] + }, + ) as NamespacedAndKeyedStream +} + +function getRowCorrelationKey(row: NamespacedRow, mainSource: string): unknown { + return ( + (row as any)[mainSource]?.__correlationKey ?? (row as any).__correlationKey + ) +} + +function correlationValuesEqual(left: unknown, right: unknown): boolean { + if (left == null || right == null) return false + const normalizedLeft = normalizeValue(left) + const normalizedRight = normalizeValue(right) + return ( + Object.is(normalizedLeft, normalizedRight) || + (typeof normalizedLeft === `number` && + typeof normalizedRight === `number` && + Number.isNaN(normalizedLeft) && + Number.isNaN(normalizedRight)) + ) +} + /** * Result of compiling an includes subquery, including the child pipeline * and metadata needed to route child results to parent-scoped Collections. @@ -227,6 +331,7 @@ export function compileQuery( sourceIncludes, directIncludes, isUnionFrom, + isParentRouted, } = processFromClause( query.from, allInputs, @@ -241,6 +346,7 @@ export function compileQuery( aliasToCollectionId, aliasRemapping, sourceWhereClauses, + parentKeyStream, ) Object.assign(sources, fromSources) @@ -250,17 +356,13 @@ export function compileQuery( // so the child pipeline only processes rows that match parents. let pipeline: NamespacedAndKeyedStream = initialPipeline const childCorrelationAlias = childCorrelationField?.path[0] - const joinsParentAfterJoins = + const joinsParentDirectly = !isUnionFrom && + !isParentRouted && parentKeyStream !== undefined && childCorrelationField !== undefined && - childCorrelationAlias !== mainSource - if ( - !isUnionFrom && - parentKeyStream && - childCorrelationField && - !joinsParentAfterJoins - ) { + childCorrelationAlias === mainSource + if (parentKeyStream && childCorrelationField && joinsParentDirectly) { const mainInput = sources[mainSource]! let filteredMainInput = mainInput // Re-key child input by correlation field: [correlationValue, [childKey, childRow]] @@ -302,6 +404,14 @@ export function compileQuery( sources[mainSource] = filteredMainInput pipeline = wrapInputWithAlias(filteredMainInput, mainSource) + } else if (parentKeyStream && !isParentRouted) { + // QueryRefs, unions, and joined-source correlations need the route before + // source-local joins, filters, grouping, ordering, or windows run. + pipeline = parameterizeByParentRoutes( + initialPipeline, + parentKeyStream, + mainSource, + ) } // Process JOIN clauses if they exist @@ -326,41 +436,23 @@ export function compileQuery( aliasToCollectionId, aliasRemapping, sourceWhereClauses, - parentKeyStream !== undefined && !joinsParentAfterJoins, + parentKeyStream !== undefined, + parentKeyStream, ) } - // A correlation field owned by a joined source does not exist on the main - // input. Join the fully namespaced child relation with its parent routes - // here, after the source join has made that field available. - if (joinsParentAfterJoins) { + // A recursively compiled source or a correlation owned by a joined source + // is already parameterized by route. Once the correlation field is visible, + // retain only the copy whose route key matches it. + if (parentKeyStream && childCorrelationField && !joinsParentDirectly) { const compiledChildCorrelation = compileExpression(childCorrelationField) pipeline = pipeline.pipe( - map( - ([key, row]) => - [compiledChildCorrelation(row), [key, row]] as [ - unknown, - [unknown, typeof row], - ], + filter(([, row]) => + correlationValuesEqual( + compiledChildCorrelation(row), + getRowCorrelationKey(row, mainSource), + ), ), - joinOperator(parentKeyStream, `inner`), - filter(([_correlationValue, [childSide]]) => childSide != null), - map(([correlationValue, [childSide, parentSide]]) => { - const [childKey, row] = childSide as [unknown, NamespacedRow] - const namespaced = { ...row } as Record - namespaced[mainSource] = { - ...namespaced[mainSource], - __correlationKey: correlationValue, - [INCLUDES_PUBLIC_KEY]: childKey, - } - if (parentSide != null) { - Object.assign(namespaced, parentSide) - namespaced.__parentContext = parentSide - } - const effectiveKey = - parentSide != null ? serializeValue([childKey, parentSide]) : childKey - return [effectiveKey, namespaced] - }), ) as NamespacedAndKeyedStream } @@ -515,34 +607,23 @@ export function compileQuery( condition: compileExpression(guard.condition), expected: guard.expected, })) - let parentKeys: any - if (subquery.parentProjection && subquery.parentProjection.length > 0) { - const compiledProjections = subquery.parentProjection.map((ref) => ({ + const compiledProjections: Array = + subquery.parentProjection?.map((ref) => ({ alias: ref.path[0]!, field: ref.path.slice(1), compiled: compileExpression(ref), - })) + })) ?? [] + let parentKeys: any + if (compiledProjections.length > 0) { parentKeys = pipeline.pipe( map(([_key, nsRow]: any) => { if (!matchesConditionalSelectGuards(compiledGuards, nsRow)) { return [SKIP_INCLUDE, null] as any } - const parentContext: Record> = {} - for (const proj of compiledProjections) { - if (!parentContext[proj.alias]) { - parentContext[proj.alias] = {} - } - const value = proj.compiled(nsRow) - // Set nested field in the alias namespace - let target = parentContext[proj.alias]! - for (let i = 0; i < proj.field.length - 1; i++) { - if (!target[proj.field[i]!]) { - target[proj.field[i]!] = {} - } - target = target[proj.field[i]!] - } - target[proj.field[proj.field.length - 1]!] = value - } + const parentContext = projectParentContext( + nsRow, + compiledProjections, + ) return [compiledCorrelation(nsRow), parentContext] as any }), ) @@ -696,12 +777,7 @@ export function compileQuery( }) // Capture routing function for INCLUDES_ROUTING tagging - if (subquery.parentProjection && subquery.parentProjection.length > 0) { - const compiledProjs = subquery.parentProjection.map((ref) => ({ - alias: ref.path[0]!, - field: ref.path.slice(1), - compiled: compileExpression(ref), - })) + if (compiledProjections.length > 0) { const compiledCorr = compiledCorrelation const compiledRoutingGuards = compiledGuards includesRoutingFns.push({ @@ -714,21 +790,10 @@ export function compileQuery( parentContext: null, } } - const parentContext: Record> = {} - for (const proj of compiledProjs) { - if (!parentContext[proj.alias]) { - parentContext[proj.alias] = {} - } - const value = proj.compiled(nsRow) - let target = parentContext[proj.alias]! - for (let i = 0; i < proj.field.length - 1; i++) { - if (!target[proj.field[i]!]) { - target[proj.field[i]!] = {} - } - target = target[proj.field[i]!] - } - target[proj.field[proj.field.length - 1]!] = value - } + const parentContext = projectParentContext( + nsRow, + compiledProjections, + ) return { active: true, correlationKey: compiledCorr(nsRow), @@ -820,10 +885,13 @@ export function compileQuery( // If no SELECT clause, create $selected with the main table data pipeline = pipeline.pipe( map(([key, namespacedRow]) => { + const routedScalar = getRoutedScalarMetadata(namespacedRow) const selectResults = - !isUnionFrom && !query.join && !query.groupBy - ? namespacedRow[mainSource] - : namespacedRow + isUnionFrom && routedScalar + ? routedScalar.value + : !isUnionFrom && !query.join && !query.groupBy + ? namespacedRow[mainSource] + : namespacedRow return [ key, @@ -1118,7 +1186,7 @@ function canonicalizeSelectedRows( value: row.$selected, routing: row.$selected?.[INCLUDES_ROUTING], outerCorrelation: isIncludedRelation - ? row[mainSource]?.__correlationKey + ? (row[mainSource]?.__correlationKey ?? row.__correlationKey) : undefined, parentContext: isIncludedRelation ? (row.__parentContext ?? row[mainSource]?.__parentContext ?? null) @@ -1259,6 +1327,7 @@ function processFromClause( aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, + parentKeyStream?: KeyedStream, ): { alias: string pipeline: NamespacedAndKeyedStream @@ -1267,6 +1336,7 @@ function processFromClause( sourceIncludes: Array directIncludes: Array isUnionFrom: boolean + isParentRouted: boolean } { if (from.type === `unionAll`) { return processUnionAll( @@ -1283,25 +1353,28 @@ function processFromClause( aliasToCollectionId, aliasRemapping, sourceWhereClauses, + parentKeyStream, ) } if (from.type !== `unionFrom`) { - const { alias, input, collectionId, sourceIncludes } = processFrom( - from, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - aliasToCollectionId, - aliasRemapping, - sourceWhereClauses, - ) + const { alias, input, collectionId, sourceIncludes, isParentRouted } = + processFrom( + from, + allInputs, + collections, + subscriptions, + callbacks, + lazySources, + optimizableOrderByCollections, + setWindowFn, + cache, + queryMapping, + aliasToCollectionId, + aliasRemapping, + sourceWhereClauses, + parentKeyStream, + ) return { alias, @@ -1311,6 +1384,7 @@ function processFromClause( sourceIncludes, directIncludes: [], isUnionFrom: false, + isParentRouted, } } @@ -1330,6 +1404,7 @@ function processFromClause( input, collectionId, sourceIncludes: childSourceIncludes, + isParentRouted, } = processFrom( source, allInputs, @@ -1344,6 +1419,7 @@ function processFromClause( aliasToCollectionId, aliasRemapping, sourceWhereClauses, + parentKeyStream, ) if (!mainAlias) { @@ -1353,12 +1429,31 @@ function processFromClause( sources[alias] = input sourceIncludes.push(...childSourceIncludes) - const branch = wrapInputWithAlias(input, alias).pipe( + const routedBranch = + parentKeyStream && !isParentRouted + ? parameterizeByParentRoutes( + wrapInputWithAlias(input, alias), + parentKeyStream, + alias, + ) + : wrapInputWithAlias(input, alias) + const branch = routedBranch.pipe( map(([key, row]) => { - return [`${alias}:${encodeKeyForUnionBranch(key)}`, row] as [ - string, - typeof row, - ] + const branchKey = `${alias}:${encodeKeyForUnionBranch(key)}` + const aliasRow = row[alias] as Record | undefined + const publicKey = aliasRow?.[INCLUDES_PUBLIC_KEY] ?? key + const branchPublicKey = `${alias}:${encodeKeyForUnionBranch(publicKey)}` + const branchRow = parentKeyStream + ? { + ...row, + [INCLUDES_PUBLIC_KEY]: branchPublicKey, + [alias]: { + ...row[alias], + [INCLUDES_PUBLIC_KEY]: branchPublicKey, + }, + } + : row + return [branchKey, branchRow] as [string, typeof row] }), ) @@ -1373,6 +1468,7 @@ function processFromClause( sourceIncludes, directIncludes: [], isUnionFrom: true, + isParentRouted: parentKeyStream !== undefined, } } @@ -1390,6 +1486,7 @@ function processUnionAll( aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, + parentKeyStream?: KeyedStream, ): { alias: string pipeline: NamespacedAndKeyedStream @@ -1398,6 +1495,7 @@ function processUnionAll( sourceIncludes: Array directIncludes: Array isUnionFrom: boolean + isParentRouted: boolean } { if (from.queries.length === 0) { throw new UnsupportedFromTypeError(`empty unionAll`) @@ -1432,6 +1530,7 @@ function processUnionAll( setWindowFn, cache, queryMapping, + parentKeyStream, ) if (!mainCollectionId) { @@ -1446,11 +1545,22 @@ function processUnionAll( } const branchPipeline = branchResult.pipeline.pipe( - map(([key, [row]]) => { - return [`${index}:${encodeKeyForUnionBranch(key)}`, row] as [ - string, - Record, - ] + map((data: any) => { + const [key, [row, _order, correlationKey, parentContext, publicKey]] = + data + const branchKey = `${index}:${encodeKeyForUnionBranch(key)}` + const branchPublicKey = `${index}:${encodeKeyForUnionBranch( + publicKey ?? key, + )}` + const routedRow = parentKeyStream + ? attachRouteMetadataToResult( + row, + correlationKey, + parentContext, + branchPublicKey, + ) + : row + return [branchKey, routedRow] as [string, Record] }), ) @@ -1467,6 +1577,7 @@ function processUnionAll( sourceIncludes, directIncludes, isUnionFrom: true, + isParentRouted: parentKeyStream !== undefined, } } @@ -1476,6 +1587,28 @@ function wrapInputWithAlias( ): NamespacedAndKeyedStream { return input.pipe( map(([key, row]) => { + const inputRow: unknown = row + const scalar = getRoutedScalarMetadata(inputRow) + if (scalar) { + const nsRow = { + [alias]: scalar.value, + __correlationKey: scalar.correlationKey, + __parentContext: scalar.parentContext, + [INCLUDES_PUBLIC_KEY]: scalar.publicKey, + } as unknown as NamespacedRow + if ( + scalar.parentContext != null && + typeof scalar.parentContext === `object` + ) { + Object.assign(nsRow, scalar.parentContext) + } + return [key, nsRow] as [unknown, NamespacedRow] + } + + if (inputRow == null || typeof inputRow !== `object`) { + return [key, { [alias]: inputRow }] as [unknown, NamespacedRow] + } + // Initialize the record with a nested structure. // If __parentContext exists (from parent-referencing includes), merge parent // aliases into the namespaced row so WHERE can resolve parent refs. @@ -1517,11 +1650,13 @@ function processFrom( aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, + parentKeyStream?: KeyedStream, ): { alias: string input: KeyedStream collectionId: string sourceIncludes: Array + isParentRouted: boolean } { switch (from.type) { case `collectionRef`: { @@ -1539,6 +1674,7 @@ function processFrom( input, collectionId: from.collection.id, sourceIncludes: [], + isParentRouted: false, } } case `queryRef`: { @@ -1557,6 +1693,7 @@ function processFrom( setWindowFn, cache, queryMapping, + parentKeyStream, ) // Pull up alias mappings from subquery to parent scope. @@ -1615,9 +1752,17 @@ function processFrom( // We need to extract just the value for use in parent queries const extractedInput = subQueryInput.pipe( map((data: any) => { - const [key, [value, _orderByIndex]] = data + const [ + key, + [value, _orderByIndex, correlationKey, parentContext, publicKey], + ] = data // Unwrap Value expressions that might have leaked through as the entire row - const unwrapped = unwrapValue(value) + const unwrapped = attachRouteMetadataToResult( + unwrapValue(value), + correlationKey, + parentContext, + publicKey, + ) return [key, unwrapped] as [unknown, any] }), ) @@ -1631,6 +1776,7 @@ function processFrom( sourceAlias: from.alias, include, })) ?? [], + isParentRouted: parentKeyStream !== undefined, } } default: @@ -1704,7 +1850,11 @@ function getIncludesPublicKey( mainSource: string, fallback: unknown, ): unknown { - return row[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? fallback + return ( + row[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? + (row as any)[INCLUDES_PUBLIC_KEY] ?? + fallback + ) } /** diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 6d0bee863e..45ce965878 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -20,6 +20,12 @@ import { normalizeValue } from '../../utils/comparison.js' import { ensureIndexForField } from '../../indexes/auto-index.js' import { compileExpression } from './evaluators.js' import { getLazyLoadTargets } from './lazy-targets.js' +import { crossJoinParentRoutes } from './parent-routes.js' +import { + INCLUDES_PUBLIC_KEY, + attachRouteMetadataToResult, + getRoutedScalarMetadata, +} from './route-metadata.js' import type { CompileQueryFn } from './index.js' import type { OrderByOptimizationInfo } from './order-by.js' import type { @@ -54,6 +60,69 @@ export type LazyCollectionCallbacks = { let nextLazyDemandPlanId = 0 +function parameterizeJoinInputByParentRoutes( + input: KeyedStream, + parentKeyStream: KeyedStream, +): KeyedStream { + return crossJoinParentRoutes( + input, + parentKeyStream, + (rowKey, row, correlationKey, parentContext) => { + return [ + serializeValue([rowKey, correlationKey, parentContext]), + { + ...(row as Record), + __correlationKey: correlationKey, + __parentContext: parentContext, + }, + ] + }, + ) +} + +function wrapJoinedInputRow(alias: string, row: any): NamespacedRow { + const scalar = getRoutedScalarMetadata(row) + if (scalar) { + const namespaced = { + [alias]: scalar.value, + __correlationKey: scalar.correlationKey, + __parentContext: scalar.parentContext, + [INCLUDES_PUBLIC_KEY]: scalar.publicKey, + } as unknown as NamespacedRow + if ( + scalar.parentContext != null && + typeof scalar.parentContext === `object` + ) { + Object.assign(namespaced, scalar.parentContext) + } + return namespaced + } + + if (row == null || typeof row !== `object`) { + return { [alias]: row } + } + + const { __parentContext, ...cleanRow } = row + const namespaced: NamespacedRow = { [alias]: cleanRow } + if (__parentContext != null) { + Object.assign(namespaced, __parentContext) + namespaced.__parentContext = __parentContext + } + return namespaced +} + +function getRouteJoinKey( + row: NamespacedRow, + source: string, + value: unknown, +): string { + return serializeValue([ + row[source]?.__correlationKey ?? row.__correlationKey, + row.__parentContext ?? row[source]?.__parentContext ?? null, + value, + ]) +} + export function registerLazyDemandPlan( callbacks: Record, target: { sourceId: string; path: Array; collection: Collection }, @@ -95,6 +164,7 @@ export function processJoins( aliasRemapping: Record, sourceWhereClauses: Map>, mainSourceIsParentFiltered: boolean, + parentKeyStream?: KeyedStream, ): NamespacedAndKeyedStream { let resultPipeline = pipeline @@ -120,6 +190,7 @@ export function processJoins( aliasRemapping, sourceWhereClauses, mainSourceIsParentFiltered, + parentKeyStream, ) } @@ -151,12 +222,30 @@ function processJoin( aliasRemapping: Record, sourceWhereClauses: Map>, mainSourceIsParentFiltered: boolean, + parentKeyStream?: KeyedStream, ): NamespacedAndKeyedStream { const isCollectionRef = joinClause.from.type === `collectionRef` + const joinedSource = joinClause.from.alias + const availableSources = [...Object.keys(sources), joinedSource] + const { mainExpr, joinedExpr } = analyzeJoinExpressions( + joinClause.left, + joinClause.right, + availableSources, + joinedSource, + rawQuery.from.type === `unionAll`, + ) + const joinedExpressionAliases = getSourceAliasesFromExpression(joinedExpr) + const joinedExpressionUsesParent = [...joinedExpressionAliases].some( + (alias) => alias !== joinedSource && !sources[alias], + ) + const routeJoinedSource = + parentKeyStream !== undefined && + (joinClause.from.type === `queryRef` || joinedExpressionUsesParent) + // Get the joined source alias and input stream const { - alias: joinedSource, + alias: processedJoinedSource, input: joinedInput, collectionId: joinedCollectionId, } = processJoinSource( @@ -174,8 +263,13 @@ function processJoin( aliasToCollectionId, aliasRemapping, sourceWhereClauses, + routeJoinedSource ? parentKeyStream : undefined, ) + if (processedJoinedSource !== joinedSource) { + throw new InvalidJoinConditionSourceMismatchError() + } + // Add the joined source to the sources map sources[joinedSource] = joinedInput if (isCollectionRef) { @@ -195,22 +289,16 @@ function processJoin( throw new JoinCollectionNotFoundError(joinedCollectionId) } - const { activeSource, lazySource } = getActiveAndLazySources( + const sourceActivity = getActiveAndLazySources( joinClause.type, mainCollection, joinedCollection, mainSourceIsParentFiltered, ) - - // Analyze which source each expression refers to and swap if necessary - const availableSources = Object.keys(sources) - const { mainExpr, joinedExpr } = analyzeJoinExpressions( - joinClause.left, - joinClause.right, - availableSources, - joinedSource, - rawQuery.from.type === `unionAll`, - ) + const activeSource = routeJoinedSource + ? undefined + : sourceActivity.activeSource + const lazySource = sourceActivity.lazySource // Pre-compile the join expressions const compiledMainExpr = compileExpression(mainExpr) @@ -220,7 +308,10 @@ function processJoin( let mainPipeline = pipeline.pipe( map(([currentKey, namespacedRow]) => { // Extract the join key from the main source expression - const mainKey = normalizeValue(compiledMainExpr(namespacedRow)) + const value = normalizeValue(compiledMainExpr(namespacedRow)) + const mainKey = routeJoinedSource + ? getRouteJoinKey(namespacedRow, mainSource, value) + : value // Return [joinKey, [originalKey, namespacedRow]] return [mainKey, [currentKey, namespacedRow]] as [ @@ -234,10 +325,13 @@ function processJoin( let joinedPipeline = joinedInput.pipe( map(([currentKey, row]) => { // Wrap the row in a namespaced structure - const namespacedRow: NamespacedRow = { [joinedSource]: row } + const namespacedRow = wrapJoinedInputRow(joinedSource, row) // Extract the join key from the joined source expression - const joinedKey = normalizeValue(compiledJoinedExpr(namespacedRow)) + const value = normalizeValue(compiledJoinedExpr(namespacedRow)) + const joinedKey = routeJoinedSource + ? getRouteJoinKey(namespacedRow, joinedSource, value) + : value // Return [joinKey, [originalKey, namespacedRow]] return [joinedKey, [currentKey, namespacedRow]] as [ @@ -472,6 +566,7 @@ function processJoinSource( aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, + parentKeyStream?: KeyedStream, ): { alias: string; input: KeyedStream; collectionId: string } { switch (from.type) { case `collectionRef`: { @@ -484,7 +579,13 @@ function processJoinSource( ) } aliasToCollectionId[from.alias] = from.collection.id - return { alias: from.alias, input, collectionId: from.collection.id } + return { + alias: from.alias, + input: parentKeyStream + ? parameterizeJoinInputByParentRoutes(input, parentKeyStream) + : input, + collectionId: from.collection.id, + } } case `queryRef`: { // Find the original query for caching purposes @@ -502,6 +603,7 @@ function processJoinSource( setWindowFn, cache, queryMapping, + parentKeyStream, ) // Pull up alias mappings from subquery to parent scope. @@ -559,8 +661,22 @@ function processJoinSource( // We need to extract just the value for use in parent queries const extractedInput = subQueryInput.pipe( map((data: any) => { - const [key, [value, _orderByIndex]] = data - return [key, value] as [unknown, any] + const [ + key, + [value, _orderByIndex, correlationKey, parentContext, publicKey], + ] = data + if (!parentKeyStream) { + return [key, value] as [unknown, any] + } + return [ + key, + attachRouteMetadataToResult( + value, + correlationKey, + parentContext, + publicKey, + ), + ] as [unknown, any] }), ) diff --git a/packages/db/src/query/compiler/parent-routes.ts b/packages/db/src/query/compiler/parent-routes.ts new file mode 100644 index 0000000000..b6eddb32f9 --- /dev/null +++ b/packages/db/src/query/compiler/parent-routes.ts @@ -0,0 +1,40 @@ +import { filter, join as joinOperator, map } from '@tanstack/db-ivm' +import type { KeyedStream } from '../../types.js' + +const PARENT_ROUTE_CROSS_KEY = `__tanstack_parent_route_cross__` + +export function crossJoinParentRoutes( + input: KeyedStream, + parentKeyStream: KeyedStream, + assemble: ( + rowKey: unknown, + row: unknown, + correlationKey: unknown, + parentContext: unknown, + ) => [unknown, unknown], +): KeyedStream { + // Recursive sources need their route before a correlation field is always + // available. The constant key intentionally creates one copy of each input + // row per active route; callers filter those copies once the field is visible. + const rows: any = input.pipe( + map(([rowKey, row]) => [PARENT_ROUTE_CROSS_KEY, [rowKey, row]]), + ) + const routes: any = parentKeyStream.pipe( + map(([correlationKey, parentContext]) => [ + PARENT_ROUTE_CROSS_KEY, + [correlationKey, parentContext], + ]), + ) + + return rows.pipe( + joinOperator(routes, `inner`), + filter(([, [rowSide, routeSide]]: any) => + Boolean(rowSide != null && routeSide != null), + ), + map(([, [rowSide, routeSide]]: any) => { + const [rowKey, row] = rowSide + const [correlationKey, parentContext] = routeSide + return assemble(rowKey, row, correlationKey, parentContext) + }), + ) as KeyedStream +} diff --git a/packages/db/src/query/compiler/route-metadata.ts b/packages/db/src/query/compiler/route-metadata.ts new file mode 100644 index 0000000000..703c7bd037 --- /dev/null +++ b/packages/db/src/query/compiler/route-metadata.ts @@ -0,0 +1,67 @@ +const ROUTED_SCALAR_VALUE = Symbol(`tanstack_db_routed_scalar_value`) +export const INCLUDES_PUBLIC_KEY = Symbol(`includesPublicKey`) + +type RoutedScalarResult = { + [ROUTED_SCALAR_VALUE]: unknown + __correlationKey: unknown + __parentContext: unknown + [INCLUDES_PUBLIC_KEY]: unknown +} + +export type RoutedScalarMetadata = { + value: unknown + correlationKey: unknown + parentContext: unknown + publicKey: unknown +} + +export function attachRouteMetadataToResult( + value: unknown, + correlationKey: unknown, + parentContext: unknown, + publicKey: unknown, +): unknown { + if ( + correlationKey === undefined && + parentContext === undefined && + publicKey === undefined + ) { + return value + } + + if (value != null && typeof value === `object`) { + return { + ...value, + __correlationKey: correlationKey, + __parentContext: parentContext, + [INCLUDES_PUBLIC_KEY]: publicKey, + } + } + + return { + [ROUTED_SCALAR_VALUE]: value, + __correlationKey: correlationKey, + __parentContext: parentContext, + [INCLUDES_PUBLIC_KEY]: publicKey, + } satisfies RoutedScalarResult +} + +export function getRoutedScalarMetadata( + value: unknown, +): RoutedScalarMetadata | undefined { + if ( + value == null || + typeof value !== `object` || + !(ROUTED_SCALAR_VALUE in value) + ) { + return undefined + } + + const routed = value as RoutedScalarResult + return { + value: routed[ROUTED_SCALAR_VALUE], + correlationKey: routed.__correlationKey, + parentContext: routed.__parentContext, + publicKey: routed[INCLUDES_PUBLIC_KEY], + } +} diff --git a/packages/db/src/query/compiler/select.ts b/packages/db/src/query/compiler/select.ts index 88ec842915..de428ff515 100644 --- a/packages/db/src/query/compiler/select.ts +++ b/packages/db/src/query/compiler/select.ts @@ -156,6 +156,16 @@ export function processSelect( select: Select, _allInputs: Record, ): NamespacedAndKeyedStream { + if (!isNestedSelectObject(select)) { + const compiled = compileSelectValue(select as SelectValueExpression) + return pipeline.pipe( + map(([key, namespacedRow]) => [ + key, + { ...namespacedRow, $selected: compiled(namespacedRow) }, + ]), + ) as NamespacedAndKeyedStream + } + // Build ordered operations to preserve authoring order (spreads and fields) const ops: Array = [] diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index 739d036a98..d551831afd 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -189,7 +189,7 @@ export class IncludesSubquery extends BaseExpression { public childCorrelationField: PropRef, // Child-side ref (e.g., issue.projectId) public fieldName: string, // Result field name (e.g., "issues") public parentFilters?: Array, // WHERE clauses referencing parent aliases (applied post-join) - public parentProjection?: Array, // Parent field refs used by parentFilters + public parentProjection?: Array, // Parent field refs used anywhere in the child plan public materialization: IncludesMaterialization = `collection`, public scalarField?: string, ) { diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 41bdda884e..ad44e21ae5 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -99,9 +99,10 @@ reduction that enforces public-key congruence and multiplicity. ## Identity Aliases are lexical query-language names. They are not runtime identities. The -query builder requires collection aliases to be unique across one query tree, -including includes; shadowing is rejected before compilation. Compilation then -assigns opaque IDs to the accepted plan: +query builder requires collection aliases to be unique within each lexical +scope and rejects nested queries that shadow an ancestor alias. Sibling include +scopes may reuse an alias because neither alias is visible to the other. +Compilation then assigns opaque IDs to the accepted plan: ```ts type SourceId = Brand @@ -114,8 +115,8 @@ unused name cannot change the compiled graph or its result. A `CanonicalCorrelationKey` is the canonical tuple of every evaluated parent-dependent value that can affect the child plan. This includes values -used by filters, joins, ordering, limits, and nullable predicates, not only the -obvious foreign-key equality. +used by filters, joins, grouping, aggregates, ordering, projections, limits, +and nullable predicates, not only the obvious foreign-key equality. A bucket key identifies one such correlated partition at one relation node: @@ -131,6 +132,61 @@ Implementations use canonical values, interned handles, or nested maps; they do not reconstruct array or object keys and expect JavaScript `Map` identity to match. +### Route-context transport + +A parent reference is a lexical dependency, even when it appears below the +immediate child query. For every parent reference that the builder can inspect, +the compiler must: + +1. discover it across nested includes, `QueryRef` sources, union branches, and + joined sources; +2. include its evaluated value in the route identity; +3. attach that route context before the first operator that evaluates it; and +4. preserve it through each later recursive source, join, grouping, and + materialization edge. + +The third rule fixes the evaluation order. A parent-dependent filter, join key, +aggregate wrapper, order, or window must run once per parent route. It cannot +run on a shared child relation first and receive a route after the fact. + +The route-context grammar crosses these dimensions: + +```text +lexical dependency scope + x recursive source boundary (nested include, QueryRef, union) + x recursive result shape (record, scalar, nullable scalar) + x evaluation phase (filter, join, group, aggregate, order, window) + x join side and correlation attachment point + x materialization form + x parent or child update +``` + +Adding one dimension to the query language requires checking its product with +the others. A passing one-level filter case does not prove a nested aggregate, +joined subquery, or union branch transports the same context. + +The executable oracle factors that product into valid compiler sub-grammars: + +- parent field projection by whole-row projection; +- unmatched correlation values by null correlation values; +- lexical scope, including nested outer and inner materialization forms; +- grouping mode by aggregate-expression placement; +- recursive source boundary by evaluation phase; and +- join-key side by correlation attachment point; +- union form and public-key identity; and +- derived-result boundary by selection mode and scalar nullability. + +Objects carry route metadata as hidden fields while the compiler moves them +through recursive sources. Scalars, including `null`, cannot carry fields, so +the compiler uses an internal envelope at those same edges. Namespacing and +join adapters unwrap the value, keep the route beside it, and never expose the +envelope in the public query result. + +Every valid plan is checked as a Collection, `toArray`, and `materialize` +include at initial load, after a parent-route update, and after a child update. +The grammar declarations generate the cases; individual reported defects do +not get one-off tests outside that product. + A materialization cell identifies one include field on one parent-row occurrence: @@ -466,7 +522,8 @@ create recursive Collection machinery. ## Normative laws 1. **Alpha-renaming:** changing any accepted alias to another unused name cannot - change results; alias shadowing within a query tree is rejected. + change results; aliases must be unique within one lexical scope and cannot + shadow an ancestor alias. Sibling scopes may reuse aliases. 2. **Contribution conservation:** a public row exists exactly when its reduced supporting weight and collision policy produce one. 3. **Batch partition:** equivalent valid split and atomic deliveries converge. @@ -527,6 +584,7 @@ create recursive Collection machinery. | Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | | Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | | Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | +| Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | | Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | | Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 6f001b586d..e83210c478 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1,23 +1,27 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' -import { createCollection } from '../../src/collection/index.js' import { + add, + caseWhen, concat, + count, createLiveQueryCollection, eq, + gt, + lt, materialize, + multiply, + sum, toArray, } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' import { oraclePropertyOptions } from '../oracle-config.js' -import { - flushPromises, - mockSyncCollectionOptions, - withExpectedRejection, -} from '../utils.js' +import { flushPromises, withExpectedRejection } from '../utils.js' +import { createControlledCollection as createOracleControlledCollection } from './includes-oracle-helpers.js' import type { Collection } from '../../src/collection/index.js' import type { ChangeMessage } from '../../src/types.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' +import type { ControlledCollection } from './includes-oracle-helpers.js' type ParentRow = { id: number @@ -50,19 +54,6 @@ type CollectionObservation = { publications: Array> } -type ControlledCollection = { - collection: Collection - write: (type: `insert` | `update` | `delete`, value: T) => void - writeBatch: ( - changes: ReadonlyArray<{ - type: `insert` | `update` | `delete` - value: T - }>, - ) => void - resolveSync: () => void - rejectSync: (error: Error) => void -} - type CollectionContext = { parents: ControlledCollection children: ControlledCollection @@ -75,39 +66,21 @@ type CollectionContext = { subscription?: { unsubscribe: () => void } } -let nextCollectionOracleId = 0 - function createControlledCollection( name: string, initialData: ReadonlyArray, ): ControlledCollection { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextCollectionOracleId++}`, - getKey: (row) => row.id, - initialData: initialData.map((row) => ({ ...row })), + return createOracleControlledCollection(name, initialData, { autoIndex: `eager`, + rowUpdateMode: `full`, }) - options.sync.rowUpdateMode = `full` - const collection = createCollection(options) - const writeBatch: ControlledCollection[`writeBatch`] = (changes) => { - options.utils.begin() - for (const change of changes) { - options.utils.write({ - type: change.type, - value: { ...change.value }, - }) - } - options.utils.commit() - } +} +function expectedMaterializations(rows: ReadonlyArray) { return { - collection, - write(type, value) { - writeBatch([{ type, value }]) - }, - writeBatch, - resolveSync: options.utils.resolveSync, - rejectSync: options.utils.rejectSync, + facade: [...rows], + array: [...rows], + materialized: [...rows], } } @@ -1123,6 +1096,600 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest( + `propagates an order-only child move through every materialization`, + async () => { + type OrderedChild = ChildRow & { position: number; label: string } + const parents = createControlledCollection(`order-move-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `order-move-children`, + [ + { id: 10, parentGroup: 1, value: 1, position: 0, label: `a` }, + { id: 20, parentGroup: 1, value: 2, position: 1, label: `b` }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ + id: child.id, + position: child.position, + label: child.label, + })) + + return { + id: parent.id, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + first: materialize(childRows().findOne()), + joined: concat( + toArray( + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .orderBy(({ child }) => child.id) + .select(({ child }) => child.label), + ), + ), + } + }), + ) + + const project = () => { + const row = live.get(1)! + return { + facade: row.facade.toArray.map(({ id }) => id), + array: row.array.map(({ id }) => id), + materialized: row.materialized.map(({ id }) => id), + first: row.first?.id, + joined: row.joined, + } + } + + try { + await live.preload() + const facade = live.get(1)!.facade + const revision = facade._layoutRevision + expect(project()).toEqual({ + ...expectedMaterializations([10, 20]), + first: 10, + joined: `ab`, + }) + + children.write(`update`, { + id: 10, + parentGroup: 1, + value: 1, + position: 2, + label: `a`, + }) + + expect(live.get(1)!.facade).toBe(facade) + expect(facade._layoutRevision).toBeGreaterThan(revision) + expect(project()).toEqual({ + ...expectedMaterializations([20, 10]), + first: 20, + joined: `ba`, + }) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + fcTest( + `reconstructs nested conditional includes through guard transitions`, + async () => { + type GuardedParent = ParentRow & { active: boolean } + const parents = createControlledCollection( + `guarded-parents`, + [{ id: 1, group: 1, active: true }], + ) + const children = createControlledCollection(`guarded-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + value: child.value, + })) + + return { + id: parent.id, + profile: caseWhen( + eq(parent.active, true), + { + kind: `active` as const, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + }, + { kind: `inactive` as const }, + ), + } + }), + ) + + const project = () => { + const profile = live.get(1)!.profile + if (profile.kind === `inactive`) return profile + const rows = (values: Iterable) => + [...values].map(({ id, value }) => ({ id, value })) + return { + kind: profile.kind, + facade: rows(profile.facade.values()), + array: rows(profile.array), + materialized: rows(profile.materialized), + } + } + + try { + await live.preload() + const initialProfile = live.get(1)!.profile + if (initialProfile.kind === `inactive`) { + throw new Error( + `Expected the initial conditional branch to be active`, + ) + } + const initialFacade = initialProfile.facade + expect(project()).toEqual({ + kind: `active`, + ...expectedMaterializations([{ id: 10, value: 1 }]), + }) + + parents.write(`update`, { id: 1, group: 1, active: false }) + expect(project()).toEqual({ kind: `inactive` }) + expect(initialFacade.toArray).toEqual([]) + expect(initialFacade.status).toBe(`ready`) + children.write(`insert`, { id: 20, parentGroup: 1, value: 2 }) + expect(project()).toEqual({ kind: `inactive` }) + + parents.write(`update`, { id: 1, group: 1, active: true }) + const reactivatedProfile = live.get(1)!.profile + if (reactivatedProfile.kind === `inactive`) { + throw new Error(`Expected the conditional branch to reactivate`) + } + expect(reactivatedProfile.facade).not.toBe(initialFacade) + expect(project()).toEqual({ + kind: `active`, + ...expectedMaterializations([ + { id: 10, value: 1 }, + { id: 20, value: 2 }, + ]), + }) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + fcTest( + `matches recomputation for correlated aggregate child relations`, + async () => { + const parents = createControlledCollection(`aggregate-parents`, [ + { id: 1, group: 1, factor: 1 }, + { id: 2, group: 1, factor: -1 }, + { id: 3, group: 2, factor: 2 }, + ]) + const children = createControlledCollection(`aggregate-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + { id: 30, parentGroup: 2, value: 5 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const summaries = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .groupBy(({ child }) => child.parentGroup) + .select(({ child }) => ({ + parentGroup: child.parentGroup, + count: count(child.id), + total: sum(multiply(child.value, parent.factor)), + })) + const total = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => ({ + count: count(child.id), + total: sum(multiply(child.value, parent.factor)), + })) + + return { + id: parent.id, + facade: summaries(), + array: toArray(summaries()), + materialized: materialize(summaries()), + implicit: materialize(total()), + } + }), + ) + + type Summary = { parentGroup: number; count: number; total: number } + const project = () => + live.toArray.map((row) => { + const clean = (values: Iterable) => + [...values].map(({ parentGroup, count: size, total }) => ({ + parentGroup, + count: size, + total, + })) + return { + id: row.id, + facade: clean(row.facade.values()), + array: clean(row.array), + materialized: clean(row.materialized), + implicit: row.implicit.map(({ count: size, total }) => ({ + count: size, + total, + })), + } + }) + + const expected = (groupOneTotal: number, groupOneCount: number) => [ + { + id: 1, + ...expectedMaterializations([ + { parentGroup: 1, count: groupOneCount, total: groupOneTotal }, + ]), + implicit: [{ count: groupOneCount, total: groupOneTotal }], + }, + { + id: 2, + ...expectedMaterializations([ + { + parentGroup: 1, + count: groupOneCount, + total: -groupOneTotal, + }, + ]), + implicit: [{ count: groupOneCount, total: -groupOneTotal }], + }, + { + id: 3, + ...expectedMaterializations([ + { parentGroup: 2, count: 1, total: 10 }, + ]), + implicit: [{ count: 1, total: 10 }], + }, + ] + + try { + await live.preload() + expect(project()).toEqual(expected(3, 2)) + + children.write(`update`, { id: 20, parentGroup: 1, value: 7 }) + expect(project()).toEqual(expected(8, 2)) + + children.write(`delete`, { id: 10, parentGroup: 1, value: 1 }) + expect(project()).toEqual(expected(7, 1)) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + fcTest( + `evaluates correlated having clauses with parent context`, + async () => { + type HavingParent = ParentRow & { threshold: number } + const parents = createControlledCollection( + `having-parents`, + [ + { id: 1, group: 1, threshold: 1 }, + { id: 2, group: 1, threshold: 3 }, + ], + ) + const children = createControlledCollection(`having-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const summaries = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .groupBy(({ child }) => child.parentGroup) + .having(({ child }) => gt(count(child.id), parent.threshold)) + .select(({ child }) => ({ count: count(child.id) })) + + return { + id: parent.id, + facade: summaries(), + array: toArray(summaries()), + materialized: materialize(summaries()), + } + }), + ) + + const project = () => + live.toArray.map((row) => { + const counts = (values: Iterable<{ count: number }>) => + [...values].map(({ count: size }) => size) + return { + id: row.id, + facade: counts(row.facade.values()), + array: counts(row.array), + materialized: counts(row.materialized), + } + }) + + try { + await live.preload() + expect(project()).toEqual([ + { id: 1, ...expectedMaterializations([2]) }, + { id: 2, ...expectedMaterializations([]) }, + ]) + + parents.write(`update`, { id: 2, group: 1, threshold: 1 }) + expect(project()).toEqual([ + { id: 1, ...expectedMaterializations([2]) }, + { id: 2, ...expectedMaterializations([2]) }, + ]) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + fcTest(`routes changes to non-key parent filter inputs`, async () => { + type FilterParent = ParentRow & { threshold: number } + const parents = createControlledCollection( + `parent-filter-input-parents`, + [{ id: 1, group: 1, threshold: 2 }], + ) + const children = createControlledCollection( + `parent-filter-input-children`, + [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 3 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .where(({ child }) => lt(child.value, parent.threshold)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ id: child.id, value: child.value })) + + return { + id: parent.id, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + } + }), + ) + + const project = () => { + const row = live.get(1)! + const ids = (values: Iterable<{ id: number }>) => + [...values].map(({ id }) => id) + return { + facade: ids(row.facade.values()), + array: ids(row.array), + materialized: ids(row.materialized), + } + } + + try { + await live.preload() + expect(project()).toEqual(expectedMaterializations([10])) + + parents.write(`update`, { id: 1, group: 1, threshold: 4 }) + expect(project()).toEqual(expectedMaterializations([10, 20])) + + parents.write(`update`, { id: 1, group: 1, threshold: 0 }) + expect(project()).toEqual(expectedMaterializations([])) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }) + + fcTest(`routes changes to parent-dependent child ordering`, async () => { + type OrderingParent = ParentRow & { direction: number } + const parents = createControlledCollection( + `parent-order-input-parents`, + [ + { id: 1, group: 1, direction: 1 }, + { id: 2, group: 1, direction: -1 }, + ], + ) + const children = createControlledCollection(`parent-order-input-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => multiply(child.value, parent.direction)) + .select(({ child }) => ({ id: child.id, value: child.value })) + + return { + id: parent.id, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + } + }), + ) + + const project = (id: number) => { + const row = live.get(id)! + const rows = (values: Iterable<{ id: number; value: number }>) => + [...values].map(({ id: childId, value }) => ({ id: childId, value })) + return { + facade: rows(row.facade.values()), + array: rows(row.array), + materialized: rows(row.materialized), + } + } + + try { + await live.preload() + const ascendingFacade = live.get(1)!.facade + const descendingFacade = live.get(2)!.facade + expect(project(1)).toEqual( + expectedMaterializations([ + { id: 10, value: 1 }, + { id: 20, value: 2 }, + ]), + ) + expect(project(2)).toEqual( + expectedMaterializations([ + { id: 20, value: 2 }, + { id: 10, value: 1 }, + ]), + ) + expect(ascendingFacade).not.toBe(descendingFacade) + + parents.write(`update`, { id: 1, group: 1, direction: -1 }) + expect(live.get(1)!.facade).toBe(descendingFacade) + expect(ascendingFacade.toArray).toEqual([]) + expect(project(1)).toEqual( + expectedMaterializations([ + { id: 20, value: 2 }, + { id: 10, value: 1 }, + ]), + ) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }) + + fcTest(`routes changes to parent-dependent child joins`, async () => { + type JoinParent = ParentRow & { offset: number } + type JoinedChild = ChildRow & { tagId: number } + type Tag = { id: number; label: string } + const parents = createControlledCollection( + `parent-join-parents`, + [ + { id: 1, group: 1, offset: 0 }, + { id: 2, group: 1, offset: 1 }, + ], + ) + const children = createControlledCollection( + `parent-join-children`, + [{ id: 10, parentGroup: 1, value: 1, tagId: 1 }], + ) + const tags = createControlledCollection(`parent-join-tags`, [ + { id: 1, label: `direct` }, + { id: 2, label: `offset` }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .innerJoin({ tag: tags.collection }, ({ child, tag }) => + eq(tag.id, add(child.tagId, parent.offset)), + ) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child, tag }) => ({ + id: child.id, + label: tag.label, + })) + + return { + id: parent.id, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + } + }), + ) + + const project = (id: number) => { + const row = live.get(id)! + const labels = (values: Iterable<{ label: string }>) => + [...values].map(({ label }) => label) + return { + facade: labels(row.facade.values()), + array: labels(row.array), + materialized: labels(row.materialized), + } + } + + try { + await live.preload() + const directFacade = live.get(1)!.facade + const offsetFacade = live.get(2)!.facade + expect(directFacade).not.toBe(offsetFacade) + expect(project(1)).toEqual(expectedMaterializations([`direct`])) + expect(project(2)).toEqual(expectedMaterializations([`offset`])) + + parents.write(`update`, { id: 1, group: 1, offset: 1 }) + expect(live.get(1)!.facade).toBe(offsetFacade) + expect(directFacade.toArray).toEqual([]) + expect(project(1)).toEqual(expectedMaterializations([`offset`])) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + tags.collection.cleanup(), + ]) + } + }) + fcTest.prop( [ fc.record({ diff --git a/packages/db/tests/query/includes-context-transport-oracle.test.ts b/packages/db/tests/query/includes-context-transport-oracle.test.ts new file mode 100644 index 0000000000..40d12d1ddf --- /dev/null +++ b/packages/db/tests/query/includes-context-transport-oracle.test.ts @@ -0,0 +1,1516 @@ +import { describe, expect, test } from 'vitest' +import { + add, + coalesce, + count, + createLiveQueryCollection, + eq, + gt, + lt, + lte, + materialize, + multiply, + sum, + toArray, +} from '../../src/query/index.js' +import { createControlledCollection } from './includes-oracle-helpers.js' +import type { Collection } from '../../src/collection/index.js' +import type { Context, QueryBuilder } from '../../src/query/builder/index.js' +import type { ControlledCollection } from './includes-oracle-helpers.js' + +type Cleanable = { cleanup: () => Promise } +type MaterializationForm = (typeof materializationForms)[number] + +type MaterializedForms = { + collection: { values: () => Iterable } + array: Iterable + materialized: Iterable +} + +const materializationForms = [`collection`, `array`, `materialized`] as const +const checkpoints = [`initial`, `parent-update`, `child-update`] as const + +const routeContextGrammar = { + parentProjection: { + shapes: [`field`, `whole-row`] as const, + }, + correlationDomain: { + states: [`unmatched`, `null`] as const, + }, + lexicalScope: { + scopes: [`immediate-parent`, `lexical-ancestor`] as const, + }, + aggregation: { + groupings: [`implicit`, `explicit`] as const, + placements: [`inside-aggregate`, `wrapped-aggregate`] as const, + }, + recursiveSource: { + boundaries: [`from-query-ref`, `joined-query-ref`, `union-branch`] as const, + phases: [ + `filter`, + `projection`, + `aggregate`, + `having`, + `order-window`, + ] as const, + }, + join: { + keySides: [`main`, `joined`] as const, + correlationAttachments: [`main`, `joined`] as const, + }, + unionIdentity: { + forms: [`union-from`, `union-all`] as const, + }, + derivedResult: { + boundaries: [`from-query-ref`, `joined-query-ref`, `union-all`] as const, + selections: [`expression`, `functional`] as const, + domains: [`non-null`, `nullable`] as const, + }, +} as const + +const queryRefMetadataGrammar = { + routeModes: [`plain`, `routed`] as const, +} as const + +type QueryRefMetadataMode = (typeof queryRefMetadataGrammar.routeModes)[number] + +type ParentProjectionCell = { + family: `parent-projection` + shape: (typeof routeContextGrammar.parentProjection.shapes)[number] +} + +type CorrelationDomainCell = { + family: `correlation-domain` + state: (typeof routeContextGrammar.correlationDomain.states)[number] +} + +type LexicalScopeCell = { + family: `lexical-scope` + scope: (typeof routeContextGrammar.lexicalScope.scopes)[number] +} + +type AggregationCell = { + family: `aggregation` + grouping: (typeof routeContextGrammar.aggregation.groupings)[number] + placement: (typeof routeContextGrammar.aggregation.placements)[number] +} + +type RecursiveSourceCell = { + family: `recursive-source` + boundary: (typeof routeContextGrammar.recursiveSource.boundaries)[number] + phase: (typeof routeContextGrammar.recursiveSource.phases)[number] +} + +type JoinCell = { + family: `join` + keySide: (typeof routeContextGrammar.join.keySides)[number] + correlationAttachment: (typeof routeContextGrammar.join.correlationAttachments)[number] +} + +type UnionIdentityCell = { + family: `union-identity` + form: (typeof routeContextGrammar.unionIdentity.forms)[number] +} + +type DerivedResultCell = { + family: `derived-result` + boundary: (typeof routeContextGrammar.derivedResult.boundaries)[number] + selection: (typeof routeContextGrammar.derivedResult.selections)[number] + domain: (typeof routeContextGrammar.derivedResult.domains)[number] +} + +type GrammarCell = + | ParentProjectionCell + | CorrelationDomainCell + | LexicalScopeCell + | AggregationCell + | RecursiveSourceCell + | JoinCell + | UnionIdentityCell + | DerivedResultCell + +const grammarCells: Array = [ + ...routeContextGrammar.parentProjection.shapes.map( + (shape): ParentProjectionCell => ({ + family: `parent-projection`, + shape, + }), + ), + ...routeContextGrammar.correlationDomain.states.map( + (state): CorrelationDomainCell => ({ + family: `correlation-domain`, + state, + }), + ), + ...routeContextGrammar.lexicalScope.scopes.map( + (scope): LexicalScopeCell => ({ family: `lexical-scope`, scope }), + ), + ...routeContextGrammar.aggregation.groupings.flatMap((grouping) => + routeContextGrammar.aggregation.placements.map( + (placement): AggregationCell => ({ + family: `aggregation`, + grouping, + placement, + }), + ), + ), + ...routeContextGrammar.recursiveSource.boundaries.flatMap((boundary) => + routeContextGrammar.recursiveSource.phases.map( + (phase): RecursiveSourceCell => ({ + family: `recursive-source`, + boundary, + phase, + }), + ), + ), + ...routeContextGrammar.join.keySides.flatMap((keySide) => + routeContextGrammar.join.correlationAttachments.map( + (correlationAttachment): JoinCell => ({ + family: `join`, + keySide, + correlationAttachment, + }), + ), + ), + ...routeContextGrammar.unionIdentity.forms.map( + (form): UnionIdentityCell => ({ family: `union-identity`, form }), + ), + ...routeContextGrammar.derivedResult.boundaries.flatMap((boundary) => + routeContextGrammar.derivedResult.selections.flatMap((selection) => + routeContextGrammar.derivedResult.domains.map( + (domain): DerivedResultCell => ({ + family: `derived-result`, + boundary, + selection, + domain, + }), + ), + ), + ), +] + +async function cleanup( + live: Cleanable, + sources: Array<{ collection: Cleanable }>, +): Promise { + await live.cleanup() + await Promise.all(sources.map(({ collection }) => collection.cleanup())) +} + +function createGrammarCollection( + name: string, + rows: ReadonlyArray, +): ControlledCollection { + return createControlledCollection(name, rows, { autoIndex: `eager` }) +} + +function includeInEveryForm( + query: QueryBuilder, +) { + return { + collection: query, + array: toArray(query), + materialized: materialize(query), + } +} + +function readEveryForm( + forms: MaterializedForms, + project: (rows: Iterable) => U, +): Record { + return { + collection: project(forms.collection.values()), + array: project(forms.array), + materialized: project(forms.materialized), + } +} + +function expectEveryForm( + forms: MaterializedForms, + project: (rows: Iterable) => U, + expected: U, +): void { + expect(readEveryForm(forms, project)).toEqual( + Object.fromEntries(materializationForms.map((form) => [form, expected])), + ) +} + +function ids(rows: Iterable<{ id: number }>): Array { + return [...rows].map((row) => row.id) +} + +function grammarCellName(cell: GrammarCell): string { + switch (cell.family) { + case `parent-projection`: + return `${cell.family} / ${cell.shape}` + case `correlation-domain`: + return `${cell.family} / ${cell.state}` + case `lexical-scope`: + return `${cell.family} / ${cell.scope}` + case `aggregation`: + return `${cell.family} / ${cell.grouping} / ${cell.placement}` + case `recursive-source`: + return `${cell.family} / ${cell.boundary} / ${cell.phase}` + case `join`: + return `${cell.family} / ${cell.keySide}-side key / ${cell.correlationAttachment}-side correlation` + case `union-identity`: + return `${cell.family} / ${cell.form}` + case `derived-result`: + return `${cell.family} / ${cell.boundary} / ${cell.selection} / ${cell.domain}` + } +} + +async function runParentProjectionCell({ + shape, +}: ParentProjectionCell): Promise { + type ParentRow = { id: number; group: number; token: string } + const parentRows: Array = [ + { id: 1, group: 1, token: `one` }, + { id: 2, group: 1, token: `two` }, + ] + const parents = createGrammarCollection( + `projection-${shape}-parents`, + parentRows, + ) + const children = createGrammarCollection(`projection-${shape}-children`, [ + { id: 10, parentGroup: 1 }, + { id: 20, parentGroup: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const correlated = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + const rows = correlated + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ + id: child.id, + parentSnapshot: + shape === `field` + ? { token: parent.token } + : coalesce(parent, null), + })) + return { id: parent.id, ...includeInEveryForm(rows) } + }), + ) + + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + return children.collection.toArray + .filter((child) => child.parentGroup === parent.group) + .map(({ id }) => ({ + id, + parentSnapshot: + shape === `field` ? { token: parent.token } : { ...parent }, + })) + } + const project = (rows: Iterable<{ id: number; parentSnapshot: unknown }>) => + [...rows].map(({ id, parentSnapshot }) => ({ id, parentSnapshot })) + const assertParents = () => { + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) + } + } + + try { + await live.preload() + assertParents() + + const updatedParent = { id: 1, group: 1, token: `updated` } + parents.write(`update`, updatedParent) + assertParents() + + children.write(`insert`, { id: 30, parentGroup: 1 }) + assertParents() + } finally { + await cleanup(live, [parents, children]) + } +} + +async function runCorrelationDomainCell({ + state, +}: CorrelationDomainCell): Promise { + type ParentRow = { id: number; group: number | null } + type ChildRow = { id: number; parentGroup: number | null } + const parents = createGrammarCollection( + `correlation-${state}-parents`, + [{ id: 1, group: state === `null` ? null : 999 }], + ) + const children = createGrammarCollection( + `correlation-${state}-children`, + [ + { id: 10, parentGroup: null }, + { id: 20, parentGroup: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const rows = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ id: child.id })) + return { id: parent.id, ...includeInEveryForm(rows) } + }), + ) + + const expected = () => { + const parentGroup = parents.collection.get(1)!.group + return children.collection.toArray + .filter( + (child) => + parentGroup != null && + child.parentGroup != null && + child.parentGroup === parentGroup, + ) + .map(({ id }) => id) + } + const assertResult = () => expectEveryForm(live.get(1)!, ids, expected()) + + try { + await live.preload() + assertResult() + + parents.write(`update`, { id: 1, group: 1 }) + assertResult() + + children.write(`insert`, { id: 30, parentGroup: 1 }) + assertResult() + } finally { + await cleanup(live, [parents, children]) + } +} + +async function runLexicalScopeCell({ scope }: LexicalScopeCell): Promise { + const parents = createGrammarCollection(`scope-${scope}-parents`, [ + { id: 1, group: 1, threshold: 2 }, + { id: 2, group: 1, threshold: 4 }, + ]) + const children = createGrammarCollection(`scope-${scope}-children`, [ + { id: 10, parentGroup: 1, group: 10, value: 1 }, + { id: 20, parentGroup: 1, group: 20, value: 3 }, + ]) + const grandchildren = createGrammarCollection( + `scope-${scope}-grandchildren`, + [ + { id: 100, parentGroup: 10, value: 1 }, + { id: 200, parentGroup: 10, value: 3 }, + ], + ) + + if (scope === `immediate-parent`) { + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const childRows = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .where(({ child }) => lt(child.value, parent.threshold)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ id: child.id })) + return { id: parent.id, ...includeInEveryForm(childRows) } + }), + ) + + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + return children.collection.toArray + .filter( + (child) => + child.parentGroup === parent.group && + child.value < parent.threshold, + ) + .map(({ id }) => id) + .sort((left, right) => left - right) + } + + try { + await live.preload() + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, ids, expected(parent.id)) + } + + parents.write(`update`, { id: 1, group: 1, threshold: 4 }) + expectEveryForm(live.get(1)!, ids, expected(1)) + + children.write(`insert`, { + id: 30, + parentGroup: 1, + group: 30, + value: 2, + }) + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, ids, expected(parent.id)) + } + } finally { + await cleanup(live, [parents, children, grandchildren]) + } + return + } + + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const childRows = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => { + const grandchildRows = q + .from({ grandchild: grandchildren.collection }) + .where(({ grandchild }) => + eq(grandchild.parentGroup, child.group), + ) + .where(({ grandchild }) => lt(grandchild.value, parent.threshold)) + .orderBy(({ grandchild }) => grandchild.id) + .select(({ grandchild }) => ({ id: grandchild.id })) + + return { + id: child.id, + ...includeInEveryForm(grandchildRows), + } + }) + + return { id: parent.id, ...includeInEveryForm(childRows) } + }), + ) + + const expected = (parentId: number, childGroup: number) => { + const parent = parents.collection.get(parentId)! + return grandchildren.collection.toArray + .filter( + (grandchild) => + grandchild.parentGroup === childGroup && + grandchild.value < parent.threshold, + ) + .map(({ id }) => id) + .sort((left, right) => left - right) + } + + const projectOuterForm = ( + parentId: number, + outerForm: MaterializationForm, + ) => { + const parentRow = live.get(parentId)! + const outerRows = + outerForm === `collection` + ? parentRow.collection.values() + : parentRow[outerForm] + return [...outerRows].map((child) => ({ + id: child.id, + grandchildren: readEveryForm(child, ids), + })) + } + + const assertNestedProduct = (parentId: number) => { + const expectedRows = children.collection.toArray + .filter( + (child) => + child.parentGroup === parents.collection.get(parentId)!.group, + ) + .map((child) => ({ + id: child.id, + grandchildren: Object.fromEntries( + materializationForms.map((form) => [ + form, + expected(parentId, child.group), + ]), + ), + })) + for (const outerForm of materializationForms) { + expect(projectOuterForm(parentId, outerForm)).toEqual(expectedRows) + } + } + + try { + await live.preload() + assertNestedProduct(1) + assertNestedProduct(2) + + parents.write(`update`, { id: 1, group: 1, threshold: 4 }) + assertNestedProduct(1) + + grandchildren.write(`insert`, { id: 300, parentGroup: 10, value: 2 }) + assertNestedProduct(1) + assertNestedProduct(2) + } finally { + await cleanup(live, [parents, children, grandchildren]) + } +} + +async function runAggregationCell({ + grouping, + placement, +}: AggregationCell): Promise { + const name = `${grouping}-${placement}` + const parents = createGrammarCollection(`aggregate-${name}-parents`, [ + { id: 1, group: 1, factor: 2 }, + { id: 2, group: 1, factor: -1 }, + ]) + const children = createGrammarCollection(`aggregate-${name}-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const correlated = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + const rows = + grouping === `explicit` + ? correlated + .groupBy(({ child }) => child.parentGroup) + .select(({ child }) => ({ + score: + placement === `inside-aggregate` + ? sum(add(child.value, parent.factor)) + : add(sum(child.value), parent.factor), + })) + : correlated.select(({ child }) => ({ + score: + placement === `inside-aggregate` + ? sum(add(child.value, parent.factor)) + : add(sum(child.value), parent.factor), + })) + return { id: parent.id, ...includeInEveryForm(rows) } + }), + ) + + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + const total = children.collection.toArray + .filter((child) => child.parentGroup === parent.group) + .reduce((result, child) => result + child.value, 0) + const childCount = children.collection.toArray.filter( + (child) => child.parentGroup === parent.group, + ).length + return [ + placement === `inside-aggregate` + ? total + childCount * parent.factor + : total + parent.factor, + ] + } + + const project = (rows: Iterable<{ score: number }>) => + [...rows].map(({ score }) => score) + + try { + await live.preload() + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) + } + + parents.write(`update`, { id: 2, group: 1, factor: 3 }) + expectEveryForm(live.get(2)!, project, expected(2)) + + children.write(`insert`, { id: 30, parentGroup: 1, value: 4 }) + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) + } + } finally { + await cleanup(live, [parents, children]) + } +} + +type CandidateRow = { + id: number + parentGroup: number + value: number +} + +async function runRecursiveSourceCell({ + boundary, + phase, +}: RecursiveSourceCell): Promise { + const name = `${boundary}-${phase}` + const initialParameter = + phase === `order-window` + ? [1, -1] + : phase === `projection` || phase === `aggregate` + ? [10, 20] + : phase === `having` + ? [0, 1] + : [1, 2] + const parents = createGrammarCollection(`recursive-${name}-parents`, [ + { id: 1, group: 1, parameter: initialParameter[0]! }, + { id: 2, group: 1, parameter: initialParameter[1]! }, + ]) + const initialCandidates: Array = [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + { id: 30, parentGroup: 1, value: 3 }, + { id: 40, parentGroup: 1, value: 4 }, + ] + const candidates = createGrammarCollection( + `recursive-${name}-candidates`, + initialCandidates, + ) + const left = createGrammarCollection( + `recursive-${name}-left`, + initialCandidates.filter(({ id }) => id % 20 === 10), + ) + const right = createGrammarCollection( + `recursive-${name}-right`, + initialCandidates.filter(({ id }) => id % 20 === 0), + ) + const anchors = createGrammarCollection(`recursive-${name}-anchors`, [ + ...initialCandidates.map(({ id, parentGroup }) => ({ id, parentGroup })), + { id: 50, parentGroup: 1 }, + ]) + + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const buildCandidates = (source: Collection) => { + const correlated = q + .from({ candidate: source }) + .where(({ candidate }) => eq(candidate.parentGroup, parent.group)) + switch (phase) { + case `filter`: + return correlated + .where(({ candidate }) => + lte(candidate.value, parent.parameter), + ) + .select(({ candidate }) => ({ + id: candidate.id, + parentGroup: candidate.parentGroup, + value: candidate.id, + })) + case `projection`: + return correlated.select(({ candidate }) => ({ + id: candidate.id, + parentGroup: candidate.parentGroup, + value: add(candidate.value, parent.parameter), + })) + case `aggregate`: + return correlated + .groupBy(({ candidate }) => [ + candidate.id, + candidate.parentGroup, + ]) + .select(({ candidate }) => ({ + id: candidate.id, + parentGroup: candidate.parentGroup, + value: add(count(candidate.id), parent.parameter), + })) + case `having`: + return correlated + .groupBy(({ candidate }) => [ + candidate.id, + candidate.parentGroup, + ]) + .having(({ candidate }) => + gt(count(candidate.id), parent.parameter), + ) + .select(({ candidate }) => ({ + id: candidate.id, + parentGroup: candidate.parentGroup, + value: count(candidate.id), + })) + case `order-window`: + return correlated + .orderBy(({ candidate }) => + multiply(candidate.value, parent.parameter), + ) + .orderBy(({ candidate }) => candidate.id) + .limit(1) + .select(({ candidate }) => ({ + id: candidate.id, + parentGroup: candidate.parentGroup, + value: candidate.id, + })) + } + } + + // unionAll branches must use distinct lexical aliases. Keep these two + // adapters explicit so the grammar exercises the public builder rules + // without erasing their types behind a cast. + const buildLeftCandidates = () => { + const correlated = q + .from({ leftCandidate: left.collection }) + .where(({ leftCandidate }) => + eq(leftCandidate.parentGroup, parent.group), + ) + switch (phase) { + case `filter`: + return correlated + .where(({ leftCandidate }) => + lte(leftCandidate.value, parent.parameter), + ) + .select(({ leftCandidate }) => ({ + id: leftCandidate.id, + parentGroup: leftCandidate.parentGroup, + value: leftCandidate.id, + })) + case `projection`: + return correlated.select(({ leftCandidate }) => ({ + id: leftCandidate.id, + parentGroup: leftCandidate.parentGroup, + value: add(leftCandidate.value, parent.parameter), + })) + case `aggregate`: + return correlated + .groupBy(({ leftCandidate }) => [ + leftCandidate.id, + leftCandidate.parentGroup, + ]) + .select(({ leftCandidate }) => ({ + id: leftCandidate.id, + parentGroup: leftCandidate.parentGroup, + value: add(count(leftCandidate.id), parent.parameter), + })) + case `having`: + return correlated + .groupBy(({ leftCandidate }) => [ + leftCandidate.id, + leftCandidate.parentGroup, + ]) + .having(({ leftCandidate }) => + gt(count(leftCandidate.id), parent.parameter), + ) + .select(({ leftCandidate }) => ({ + id: leftCandidate.id, + parentGroup: leftCandidate.parentGroup, + value: count(leftCandidate.id), + })) + case `order-window`: + return correlated + .orderBy(({ leftCandidate }) => + multiply(leftCandidate.value, parent.parameter), + ) + .orderBy(({ leftCandidate }) => leftCandidate.id) + .limit(1) + .select(({ leftCandidate }) => ({ + id: leftCandidate.id, + parentGroup: leftCandidate.parentGroup, + value: leftCandidate.id, + })) + } + } + + const buildRightCandidates = () => { + const correlated = q + .from({ rightCandidate: right.collection }) + .where(({ rightCandidate }) => + eq(rightCandidate.parentGroup, parent.group), + ) + switch (phase) { + case `filter`: + return correlated + .where(({ rightCandidate }) => + lte(rightCandidate.value, parent.parameter), + ) + .select(({ rightCandidate }) => ({ + id: rightCandidate.id, + parentGroup: rightCandidate.parentGroup, + value: rightCandidate.id, + })) + case `projection`: + return correlated.select(({ rightCandidate }) => ({ + id: rightCandidate.id, + parentGroup: rightCandidate.parentGroup, + value: add(rightCandidate.value, parent.parameter), + })) + case `aggregate`: + return correlated + .groupBy(({ rightCandidate }) => [ + rightCandidate.id, + rightCandidate.parentGroup, + ]) + .select(({ rightCandidate }) => ({ + id: rightCandidate.id, + parentGroup: rightCandidate.parentGroup, + value: add(count(rightCandidate.id), parent.parameter), + })) + case `having`: + return correlated + .groupBy(({ rightCandidate }) => [ + rightCandidate.id, + rightCandidate.parentGroup, + ]) + .having(({ rightCandidate }) => + gt(count(rightCandidate.id), parent.parameter), + ) + .select(({ rightCandidate }) => ({ + id: rightCandidate.id, + parentGroup: rightCandidate.parentGroup, + value: count(rightCandidate.id), + })) + case `order-window`: + return correlated + .orderBy(({ rightCandidate }) => + multiply(rightCandidate.value, parent.parameter), + ) + .orderBy(({ rightCandidate }) => rightCandidate.id) + .limit(1) + .select(({ rightCandidate }) => ({ + id: rightCandidate.id, + parentGroup: rightCandidate.parentGroup, + value: rightCandidate.id, + })) + } + } + + switch (boundary) { + case `from-query-ref`: { + const routed = q + .from({ result: buildCandidates(candidates.collection) }) + .where(({ result }) => eq(result.parentGroup, parent.group)) + .select(({ result }) => ({ + id: result.id, + value: result.value, + })) + return { id: parent.id, ...includeInEveryForm(routed) } + } + case `joined-query-ref`: { + const routed = q + .from({ anchor: anchors.collection }) + .innerJoin( + { result: buildCandidates(candidates.collection) }, + ({ anchor, result }) => eq(anchor.id, result.id), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ result }) => ({ + id: result.id, + value: result.value, + })) + return { id: parent.id, ...includeInEveryForm(routed) } + } + case `union-branch`: { + const routed = q + .unionAll(buildLeftCandidates(), buildRightCandidates()) + .innerJoin({ anchor: anchors.collection }, ({ id, anchor }) => + eq(id, anchor.id), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ id, value }) => ({ id, value })) + return { id: parent.id, ...includeInEveryForm(routed) } + } + } + }), + ) + + const modelRows = new Map(initialCandidates.map((row) => [row.id, row])) + const leftModelIds = new Set( + initialCandidates.filter(({ id }) => id % 20 === 10).map(({ id }) => id), + ) + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + const correlated = [...modelRows.values()].filter( + (candidate) => candidate.parentGroup === parent.group, + ) + switch (phase) { + case `filter`: + return correlated + .filter((candidate) => candidate.value <= parent.parameter) + .map((candidate) => ({ id: candidate.id, value: candidate.id })) + case `projection`: + return correlated.map((candidate) => ({ + id: candidate.id, + value: candidate.value + parent.parameter, + })) + case `aggregate`: + return correlated.map((candidate) => ({ + id: candidate.id, + value: 1 + parent.parameter, + })) + case `having`: + return parent.parameter < 1 + ? correlated.map((candidate) => ({ id: candidate.id, value: 1 })) + : [] + case `order-window`: { + const partitions = + boundary === `union-branch` + ? [ + correlated.filter(({ id }) => leftModelIds.has(id)), + correlated.filter(({ id }) => !leftModelIds.has(id)), + ] + : [correlated] + return partitions.flatMap((partition) => + partition + .sort( + (leftRow, rightRow) => + leftRow.value * parent.parameter - + rightRow.value * parent.parameter || leftRow.id - rightRow.id, + ) + .slice(0, 1) + .map((candidate) => ({ id: candidate.id, value: candidate.id })), + ) + } + } + } + + const project = (rows: Iterable<{ id: number; value: number }>) => + [...rows] + .map(({ id, value }) => ({ id, value })) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + const assertParents = () => { + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) + } + } + + try { + await live.preload() + assertParents() + + const updatedParameter = + phase === `order-window` + ? -1 + : phase === `projection` || phase === `aggregate` + ? 30 + : phase === `having` + ? 1 + : 4 + parents.write(`update`, { id: 1, group: 1, parameter: updatedParameter }) + assertParents() + + const inserted = { + id: 50, + parentGroup: 1, + value: phase === `filter` ? 1 : 5, + } + modelRows.set(inserted.id, inserted) + if (boundary === `union-branch`) right.write(`insert`, inserted) + else candidates.write(`insert`, inserted) + assertParents() + } finally { + await cleanup(live, [parents, candidates, left, right, anchors]) + } +} + +async function runUnionIdentityCell({ + form, +}: UnionIdentityCell): Promise { + const parents = createGrammarCollection(`identity-${form}-parents`, [ + { id: 1, group: 1, parameter: 10 }, + { id: 2, group: 1, parameter: 20 }, + ]) + const left = createGrammarCollection(`identity-${form}-left`, [ + { id: 10, parentGroup: 1, value: 1 }, + ]) + const right = createGrammarCollection( + `identity-${form}-right`, + [{ id: 20, parentGroup: 1, value: 2 }], + ) + const anchors = createGrammarCollection(`identity-${form}-anchors`, [ + { id: 10, parentGroup: 1 }, + { id: 20, parentGroup: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const unionAllRows = () => { + const leftRows = q + .from({ leftCandidate: left.collection }) + .select(({ leftCandidate }) => ({ + id: leftCandidate.id, + value: add(leftCandidate.value, parent.parameter), + })) + const rightRows = q + .from({ rightCandidate: right.collection }) + .select(({ rightCandidate }) => ({ + id: rightCandidate.id, + value: add(rightCandidate.value, parent.parameter), + })) + return q + .unionAll(leftRows, rightRows) + .innerJoin({ anchor: anchors.collection }, ({ id, anchor }) => + eq(id, anchor.id), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ id, value }) => ({ id, value })) + } + const unionFromRows = () => + q + .unionAll({ + leftCandidate: left.collection, + rightCandidate: right.collection, + }) + .innerJoin( + { anchor: anchors.collection }, + ({ leftCandidate, rightCandidate, anchor }) => + eq(coalesce(leftCandidate.id, rightCandidate.id), anchor.id), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ leftCandidate, rightCandidate }) => ({ + id: coalesce(leftCandidate.id, rightCandidate.id), + value: add( + coalesce(leftCandidate.value, rightCandidate.value), + parent.parameter, + ), + })) + return form === `union-all` + ? { id: parent.id, ...includeInEveryForm(unionAllRows()) } + : { id: parent.id, ...includeInEveryForm(unionFromRows()) } + }), + ) + + const project = (rows: Iterable<{ id: number; value: number }>) => + [...rows] + .map(({ id, value }) => ({ id, value })) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + return [...left.collection.toArray, ...right.collection.toArray] + .filter((candidate) => candidate.parentGroup === parent.group) + .map((candidate) => ({ + id: candidate.id, + value: candidate.value + parent.parameter, + })) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + } + const keys = (parentId: number) => { + const collection = live.get(parentId)!.collection + return new Map( + collection.toArray.map((row) => [row.id, collection.getKeyFromItem(row)]), + ) + } + const assertParents = () => { + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) + } + } + const assertRouteIndependentKeys = () => expect(keys(2)).toEqual(keys(1)) + + try { + await live.preload() + assertParents() + assertRouteIndependentKeys() + const initialKeys = keys(1) + + parents.write(`update`, { id: 1, group: 1, parameter: 30 }) + assertParents() + expect(keys(1)).toEqual(initialKeys) + assertRouteIndependentKeys() + + right.write(`update`, { id: 20, parentGroup: 1, value: 5 }) + assertParents() + expect(keys(1)).toEqual(initialKeys) + assertRouteIndependentKeys() + } finally { + await cleanup(live, [parents, left, right, anchors]) + } +} + +type DerivedCandidateRow = { + id: number + value: number | null +} + +async function runDerivedResultCell({ + boundary, + selection, + domain, +}: DerivedResultCell): Promise { + const name = `${boundary}-${selection}-${domain}` + const parents = createGrammarCollection(`derived-${name}-parents`, [ + { id: 1, group: 1 }, + { id: 2, group: 2 }, + ]) + const initialCandidates: Array = [ + { id: 10, value: 10 }, + { id: 15, value: domain === `nullable` ? null : 15 }, + { id: 20, value: 20 }, + ] + const candidates = createGrammarCollection( + `derived-${name}-candidates`, + initialCandidates, + ) + const left = createGrammarCollection( + `derived-${name}-left`, + initialCandidates.filter(({ id }) => id !== 20), + ) + const right = createGrammarCollection( + `derived-${name}-right`, + initialCandidates.filter(({ id }) => id === 20), + ) + const anchors = createGrammarCollection(`derived-${name}-anchors`, [ + { id: 100, parentGroup: 1, value: 10 }, + { id: 200, parentGroup: 2, value: 20 }, + { id: 300, parentGroup: 2, value: 30 }, + ]) + + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + if (boundary === `union-all`) { + const leftRows = q.from({ leftCandidate: left.collection }) + const rightRows = q.from({ rightCandidate: right.collection }) + const values = + selection === `expression` + ? q.unionAll( + leftRows.select(({ leftCandidate }) => + coalesce(leftCandidate.value, null), + ), + rightRows.select(({ rightCandidate }) => + coalesce(rightCandidate.value, null), + ), + ) + : q.unionAll( + leftRows.fn.select( + ({ leftCandidate }) => leftCandidate.value, + ), + rightRows.fn.select( + ({ rightCandidate }) => rightCandidate.value, + ), + ) + const rows = q + .from({ anchor: anchors.collection }) + .innerJoin({ value: values }, ({ anchor, value }) => + eq(anchor.value, value), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ anchor, value }) => ({ id: anchor.id, value })) + return { id: parent.id, ...includeInEveryForm(rows) } + } + + if (selection === `expression`) { + const values = q + .from({ candidate: candidates.collection }) + .select(({ candidate }) => coalesce(candidate.value, null)) + if (boundary === `from-query-ref`) { + const rows = q + .from({ value: values }) + .innerJoin({ anchor: anchors.collection }, ({ value, anchor }) => + eq(value, anchor.value), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ anchor, value }) => ({ id: anchor.id, value })) + return { id: parent.id, ...includeInEveryForm(rows) } + } + + const rows = q + .from({ anchor: anchors.collection }) + .innerJoin({ value: values }, ({ anchor, value }) => + eq(anchor.value, value), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ anchor, value }) => ({ id: anchor.id, value })) + return { id: parent.id, ...includeInEveryForm(rows) } + } + + const values = q + .from({ candidate: candidates.collection }) + .fn.select(({ candidate }) => candidate.value) + if (boundary === `from-query-ref`) { + const rows = q + .from({ value: values }) + .innerJoin({ anchor: anchors.collection }, ({ value, anchor }) => + eq(value, anchor.value), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ anchor, value }) => ({ id: anchor.id, value })) + return { id: parent.id, ...includeInEveryForm(rows) } + } + + const rows = q + .from({ anchor: anchors.collection }) + .innerJoin({ value: values }, ({ anchor, value }) => + eq(anchor.value, value), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ anchor, value }) => ({ id: anchor.id, value })) + return { id: parent.id, ...includeInEveryForm(rows) } + }), + ) + + const modelRows = new Map(initialCandidates.map((row) => [row.id, row])) + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + return [...modelRows.values()] + .flatMap((candidate) => + anchors.collection.toArray + .filter( + (anchor) => + candidate.value != null && + anchor.value === candidate.value && + anchor.parentGroup === parent.group, + ) + .map((anchor) => ({ id: anchor.id, value: candidate.value })), + ) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + } + const project = ( + rows: Iterable<{ id: number; value: number | null | undefined }>, + ) => + [...rows] + .map(({ id, value }) => ({ id, value })) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + const assertParents = () => { + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) + } + } + + try { + await live.preload() + assertParents() + + parents.write(`update`, { id: 1, group: 2 }) + assertParents() + + const updated = { id: 20, value: 30 } + modelRows.set(updated.id, updated) + if (boundary === `union-all`) right.write(`update`, updated) + else candidates.write(`update`, updated) + assertParents() + } finally { + await cleanup(live, [parents, candidates, left, right, anchors]) + } +} + +function expectNoRouteMetadata(row: object): void { + expect(Object.hasOwn(row, `__correlationKey`)).toBe(false) + expect(Object.hasOwn(row, `__parentContext`)).toBe(false) +} + +async function runQueryRefMetadataCell( + routeMode: QueryRefMetadataMode, +): Promise { + const anchors = createGrammarCollection(`metadata-${routeMode}-anchors`, [ + { id: 1, candidateId: 10, parentGroup: 1 }, + ]) + const candidates = createGrammarCollection( + `metadata-${routeMode}-candidates`, + [{ id: 10, label: `ten` }], + ) + + if (routeMode === `plain`) { + const live = createLiveQueryCollection((q) => { + const projectedCandidates = q + .from({ candidate: candidates.collection }) + .select(({ candidate }) => ({ + id: candidate.id, + label: candidate.label, + })) + return q + .from({ anchor: anchors.collection }) + .innerJoin( + { candidateResult: projectedCandidates }, + ({ anchor, candidateResult }) => + eq(anchor.candidateId, candidateResult.id), + ) + .select(({ candidateResult }) => candidateResult) + }) + + try { + await live.preload() + expectNoRouteMetadata(live.toArray[0]!) + + candidates.write(`update`, { id: 10, label: `updated` }) + expectNoRouteMetadata(live.toArray[0]!) + } finally { + await cleanup(live, [anchors, candidates]) + } + return + } + + const parents = createGrammarCollection(`metadata-routed-parents`, [ + { id: 1, group: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const projectedCandidates = q + .from({ candidate: candidates.collection }) + .select(({ candidate }) => ({ + id: candidate.id, + label: candidate.label, + })) + const rows = q + .from({ anchor: anchors.collection }) + .innerJoin( + { candidateResult: projectedCandidates }, + ({ anchor, candidateResult }) => + eq(anchor.candidateId, candidateResult.id), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ candidateResult }) => candidateResult) + return { id: parent.id, ...includeInEveryForm(rows) } + }), + ) + const assertClean = () => { + const forms = live.get(1)! + for (const rows of [ + forms.collection.values(), + forms.array, + forms.materialized, + ]) { + for (const row of rows) expectNoRouteMetadata(row) + } + } + + try { + await live.preload() + assertClean() + + parents.write(`update`, { id: 1, group: 2 }) + assertClean() + + anchors.write(`update`, { id: 1, candidateId: 10, parentGroup: 2 }) + assertClean() + } finally { + await cleanup(live, [parents, anchors, candidates]) + } +} + +async function runJoinCell({ + keySide, + correlationAttachment, +}: JoinCell): Promise { + const name = `${keySide}-${correlationAttachment}` + const parents = createGrammarCollection(`join-${name}-parents`, [ + { id: 1, group: 1, offset: 0 }, + { id: 2, group: 2, offset: 1 }, + ]) + const children = createGrammarCollection(`join-${name}-children`, [ + { id: 10, parentGroup: 1, tagId: 2 }, + { id: 20, parentGroup: 2, tagId: 2 }, + ]) + const tags = createGrammarCollection(`join-${name}-tags`, [ + { id: 1, parentGroup: 2, label: `one` }, + { id: 2, parentGroup: 1, label: `two` }, + { id: 3, parentGroup: 2, label: `three` }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const joined = + keySide === `main` + ? q + .from({ child: children.collection }) + .innerJoin({ tag: tags.collection }, ({ child, tag }) => + eq(tag.id, add(child.tagId, parent.offset)), + ) + : q + .from({ child: children.collection }) + .innerJoin({ tag: tags.collection }, ({ child, tag }) => + eq(add(tag.id, parent.offset), child.tagId), + ) + const routed = ( + correlationAttachment === `main` + ? joined.where(({ child }) => eq(child.parentGroup, parent.group)) + : joined.where(({ tag }) => eq(tag.parentGroup, parent.group)) + ) + .orderBy(({ child }) => child.id) + .select(({ child, tag }) => ({ id: child.id, value: tag.label })) + return { id: parent.id, ...includeInEveryForm(routed) } + }), + ) + + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + return children.collection.toArray + .flatMap((child) => + tags.collection.toArray + .filter((tag) => { + const keyMatches = + keySide === `main` + ? tag.id === child.tagId + parent.offset + : tag.id + parent.offset === child.tagId + const routeMatches = + correlationAttachment === `main` + ? child.parentGroup === parent.group + : tag.parentGroup === parent.group + return keyMatches && routeMatches + }) + .map((tag) => ({ id: child.id, value: tag.label })), + ) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + } + + const project = (rows: Iterable<{ id: number; value: string }>) => + [...rows].map(({ id, value }) => ({ id, value })) + const assertParents = () => { + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) + } + } + + try { + await live.preload() + assertParents() + + parents.write(`update`, { id: 1, group: 1, offset: 1 }) + assertParents() + + children.write(`insert`, { id: 30, parentGroup: 1, tagId: 2 }) + assertParents() + } finally { + await cleanup(live, [parents, children, tags]) + } +} + +async function runGrammarCell(cell: GrammarCell): Promise { + switch (cell.family) { + case `parent-projection`: + return runParentProjectionCell(cell) + case `correlation-domain`: + return runCorrelationDomainCell(cell) + case `lexical-scope`: + return runLexicalScopeCell(cell) + case `aggregation`: + return runAggregationCell(cell) + case `recursive-source`: + return runRecursiveSourceCell(cell) + case `join`: + return runJoinCell(cell) + case `union-identity`: + return runUnionIdentityCell(cell) + case `derived-result`: + return runDerivedResultCell(cell) + } +} + +describe(`correlated include route-context transport grammar`, () => { + test(`expands every declared product without duplicate cells`, () => { + const expectedCellCount = + routeContextGrammar.parentProjection.shapes.length + + routeContextGrammar.correlationDomain.states.length + + routeContextGrammar.lexicalScope.scopes.length + + routeContextGrammar.aggregation.groupings.length * + routeContextGrammar.aggregation.placements.length + + routeContextGrammar.recursiveSource.boundaries.length * + routeContextGrammar.recursiveSource.phases.length + + routeContextGrammar.join.keySides.length * + routeContextGrammar.join.correlationAttachments.length + + routeContextGrammar.unionIdentity.forms.length + + routeContextGrammar.derivedResult.boundaries.length * + routeContextGrammar.derivedResult.selections.length * + routeContextGrammar.derivedResult.domains.length + const names = grammarCells.map(grammarCellName) + + expect(grammarCells).toHaveLength(expectedCellCount) + expect(new Set(names)).toHaveLength(expectedCellCount) + expect( + grammarCells.length * materializationForms.length * checkpoints.length, + ).toBe(387) + }) + + for (const cell of grammarCells) { + test(`${grammarCellName(cell)} × every materialization form × parent/child updates`, () => + runGrammarCell(cell)) + } + + for (const routeMode of queryRefMetadataGrammar.routeModes) { + test(`query-ref metadata / ${routeMode}`, () => + runQueryRefMetadataCell(routeMode)) + } +}) diff --git a/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts b/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts index 59920e5ea1..850aeb02c2 100644 --- a/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts +++ b/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts @@ -1,6 +1,5 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' -import { createCollection } from '../../src/collection/index.js' import { createLiveQueryCollection, eq, @@ -11,8 +10,10 @@ import { toArray, } from '../../src/query/index.js' import { oraclePropertyOptions } from '../oracle-config.js' -import { flushPromises, mockSyncCollectionOptions } from '../utils.js' +import { flushPromises } from '../utils.js' +import { createControlledCollection as createOracleControlledCollection } from './includes-oracle-helpers.js' import type { Collection } from '../../src/collection/index.js' +import type { ControlledCollection } from './includes-oracle-helpers.js' type ParentRow = { id: number @@ -44,11 +45,6 @@ type NormalizedParent = ParentRow & { children: Array } -type ControlledCollection = { - collection: Collection - write: (type: `insert` | `update` | `delete`, value: T) => void -} - type FlatRow = { parentId: number parentGroup: number @@ -56,27 +52,14 @@ type FlatRow = { child: ChildRow | undefined } -let nextCrossFormulationId = 0 - function createControlledCollection( name: string, initialData: ReadonlyArray, ): ControlledCollection { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextCrossFormulationId++}`, - getKey: (row) => row.id, - initialData: initialData.map((row) => ({ ...row })), + return createOracleControlledCollection(name, initialData, { autoIndex: `eager`, + rowUpdateMode: `full`, }) - options.sync.rowUpdateMode = `full` - return { - collection: createCollection(options), - write(type, value) { - options.utils.begin() - options.utils.write({ type, value: { ...value } }) - options.utils.commit() - }, - } } function compareParents(left: ParentRow, right: ParentRow): number { @@ -170,6 +153,42 @@ function createNestedQuery( }) } +function createWindowedNestedQuery( + parents: Collection, + children: Collection, + offset: number, + limit: number, +) { + return createLiveQueryCollection({ + getKey: (row) => row.id, + query: (q) => + q + .from({ parent: parents }) + .orderBy(({ parent }) => parent.position) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => ({ + id: parent.id, + group: parent.group, + position: parent.position, + children: toArray( + q + .from({ child: children }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .orderBy(({ child }) => child.id) + .offset(offset) + .limit(limit) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + score: child.score, + position: child.position, + })), + ), + })), + }) +} + function createFlatQuery( parents: Collection, children: Collection, @@ -358,6 +377,58 @@ async function expectFormulationsEquivalent( } } +async function expectWindowedIncludeMatches( + scenario: CrossFormulationScenario, + offset: number, + limit: number, +): Promise { + const parentSource = createControlledCollection( + `windowed-cross-form-parents`, + scenario.parents, + ) + const childSource = createControlledCollection( + `windowed-cross-form-children`, + scenario.children, + ) + const parents = new Map(scenario.parents.map((row) => [row.id, { ...row }])) + const children = new Map(scenario.children.map((row) => [row.id, { ...row }])) + const nested = createWindowedNestedQuery( + parentSource.collection, + childSource.collection, + offset, + limit, + ) + + const assertEquivalent = () => { + const expected = normalizeNested( + [...parents.values()].map((parent) => ({ + ...parent, + children: [...children.values()] + .filter((child) => child.parentGroup === parent.group) + .sort(compareChildren) + .slice(offset, offset + limit), + })), + ) + expect(normalizeNested(nested.toArray)).toEqual(expected) + } + + try { + await nested.preload() + assertEquivalent() + for (const action of scenario.actions) { + applyAction(action, parentSource, childSource, parents, children) + await flushPromises() + assertEquivalent() + } + } finally { + await Promise.allSettled([ + nested.cleanup(), + parentSource.collection.cleanup(), + childSource.collection.cleanup(), + ]) + } +} + const parentRowArbitrary = (id: number) => fc.record({ id: fc.constant(id), @@ -403,6 +474,12 @@ const scenarioArbitrary: fc.Arbitrary = fc.record({ actions: fc.array(actionArbitrary, { minLength: 1, maxLength: 5 }), }) +const windowedScenarioArbitrary = fc.record({ + scenario: scenarioArbitrary, + offset: fc.integer({ min: 0, max: 2 }), + limit: fc.integer({ min: 0, max: 3 }), +}) + describe(`includes cross-formulation oracle`, () => { fcTest(`shared-route child deletion agrees across formulations`, () => expectFormulationsEquivalent({ @@ -424,4 +501,10 @@ describe(`includes cross-formulation oracle`, () => { `agrees across nested includes, flat joins, per-parent queries, and TLP partitions`, expectFormulationsEquivalent, ) + + fcTest.prop([windowedScenarioArbitrary], oraclePropertyOptions(12))( + `matches recomputation for ordered offset and limit child windows`, + ({ scenario, offset, limit }) => + expectWindowedIncludeMatches(scenario, offset, limit), + ) }) diff --git a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts index 1a190a911b..0e125cdfc8 100644 --- a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts +++ b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts @@ -1,19 +1,16 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' -import { createCollection } from '../../src/collection/index.js' import { createLiveQueryCollection, eq, toArray, } from '../../src/query/index.js' -import { - flushPromises, - mockSyncCollectionOptions, - withExpectedRejection, -} from '../utils.js' +import { flushPromises, withExpectedRejection } from '../utils.js' import { runTrace } from '../trace-runner.js' import { oraclePropertyOptions } from '../oracle-config.js' +import { createControlledCollection as createOracleControlledCollection } from './includes-oracle-helpers.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' +import type { OracleSyncChange as SyncChange } from './includes-oracle-helpers.js' type RootRow = { id: number @@ -26,11 +23,6 @@ type ChildRow = RootRow & { parentGroup: number } -type SyncChange = { - type: `insert` | `update` | `delete` - value: T -} - type ChildLevel = 1 | 2 | 3 type ChildPatch = Partial< @@ -149,37 +141,13 @@ type RouteValues = { authoritative: number } -let nextHarnessId = 0 - function createControlledCollection( name: string, initialData: ReadonlyArray, ) { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextHarnessId++}`, - getKey: (row) => row.id, - initialData: initialData.map((row) => ({ ...row })), + return createOracleControlledCollection(name, initialData, { + rowUpdateMode: `full`, }) - options.sync.rowUpdateMode = `full` - const collection = createCollection(options) - - const writeBatch = (changes: ReadonlyArray>) => { - options.utils.begin() - for (const change of changes) { - options.utils.write({ - type: change.type, - value: { ...change.value }, - }) - } - options.utils.commit() - } - - return { - collection, - writeBatch, - resolveSync: options.utils.resolveSync, - rejectSync: options.utils.rejectSync, - } } function createSources( diff --git a/packages/db/tests/query/includes-oracle-helpers.ts b/packages/db/tests/query/includes-oracle-helpers.ts new file mode 100644 index 0000000000..77f08543b5 --- /dev/null +++ b/packages/db/tests/query/includes-oracle-helpers.ts @@ -0,0 +1,58 @@ +import { createCollection } from '../../src/collection/index.js' +import { mockSyncCollectionOptions } from '../utils.js' +import type { Collection } from '../../src/collection/index.js' + +export type OracleSyncChange = { + type: `insert` | `update` | `delete` + value: T +} + +export type ControlledCollection = { + collection: Collection + write: (type: OracleSyncChange[`type`], value: T) => void + writeBatch: (changes: ReadonlyArray>) => void + resolveSync: () => void + rejectSync: (error: Error) => void +} + +type ControlledCollectionOptions = { + autoIndex?: `off` | `eager` + rowUpdateMode?: `partial` | `full` +} + +let nextControlledCollectionId = 0 + +export function createControlledCollection( + name: string, + initialData: ReadonlyArray = [], + options: ControlledCollectionOptions = {}, +): ControlledCollection { + const collectionOptions = mockSyncCollectionOptions({ + id: `${name}-${nextControlledCollectionId++}`, + getKey: (row) => row.id, + initialData: initialData.map((row) => ({ ...row })), + ...(options.autoIndex ? { autoIndex: options.autoIndex } : {}), + }) + collectionOptions.sync.rowUpdateMode = options.rowUpdateMode ?? `partial` + const collection = createCollection(collectionOptions) + const writeBatch: ControlledCollection[`writeBatch`] = (changes) => { + collectionOptions.utils.begin() + for (const change of changes) { + collectionOptions.utils.write({ + type: change.type, + value: { ...change.value }, + }) + } + collectionOptions.utils.commit() + } + + return { + collection, + write(type, value) { + writeBatch([{ type, value }]) + }, + writeBatch, + resolveSync: collectionOptions.utils.resolveSync, + rejectSync: collectionOptions.utils.rejectSync, + } +} diff --git a/packages/db/tests/query/includes-oracle.property.test.ts b/packages/db/tests/query/includes-oracle.property.test.ts index c487f4897c..8fb7625876 100644 --- a/packages/db/tests/query/includes-oracle.property.test.ts +++ b/packages/db/tests/query/includes-oracle.property.test.ts @@ -8,19 +8,16 @@ import { queryOnce, toArray, } from '../../src/query/index.js' -import { createCollection } from '../../src/collection/index.js' -import { - flushPromises, - mockSyncCollectionOptions, - withExpectedRejection, -} from '../utils.js' +import { flushPromises, withExpectedRejection } from '../utils.js' import { oraclePropertyOptions, oracleRuns } from '../oracle-config.js' import { runTrace } from '../trace-runner.js' +import { createControlledCollection as createOracleControlledCollection } from './includes-oracle-helpers.js' import type { TraceCheckpoint, TraceDriver, TraceProjection, } from '../trace-runner.js' +import type { OracleSyncChange as SyncChange } from './includes-oracle-helpers.js' type IncludeDepth = 1 | 2 | 3 | 4 @@ -35,11 +32,6 @@ type ChildRow = RootRow & { parentGroup: number } -type SyncChange = { - type: `insert` | `update` | `delete` - value: T -} - type HistoryAction = { type: `put` | `delete` | `optimisticConfirm` | `optimisticRollback` level: 0 | IncludeDepth @@ -371,39 +363,14 @@ const confirmedChildReorderSeed: Scenario = { ], } -let nextHarnessId = 0 - function createControlledCollection( name: string, initialData: Array = [], rowUpdateMode: `partial` | `full` = `partial`, ) { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextHarnessId++}`, - getKey: (row) => row.id, - initialData, + return createOracleControlledCollection(name, initialData, { + rowUpdateMode, }) - options.sync.rowUpdateMode = rowUpdateMode - const collection = createCollection(options) - const writeBatch = (changes: ReadonlyArray>): void => { - options.utils.begin() - changes.forEach((change) => options.utils.write(change)) - options.utils.commit() - } - - return { - collection, - write(type: `insert` | `update` | `delete`, value: T): void { - writeBatch([{ type, value }]) - }, - writeBatch, - resolveSync(): void { - options.utils.resolveSync() - }, - rejectSync(error: Error): void { - options.utils.rejectSync(error) - }, - } } function compareRows(left: RootRow, right: RootRow): number { diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 0201d57de0..e78656d70e 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -1,6 +1,5 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' -import { createCollection } from '../../src/collection/index.js' import { BasicIndex } from '../../src/indexes/basic-index.js' import { createLiveQueryCollection, @@ -9,11 +8,8 @@ import { } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' import { oraclePropertyOptions } from '../oracle-config.js' -import { - flushPromises, - mockSyncCollectionOptions, - withExpectedRejection, -} from '../utils.js' +import { flushPromises, withExpectedRejection } from '../utils.js' +import { createControlledCollection } from './includes-oracle-helpers.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' type ParentRow = { @@ -53,11 +49,6 @@ const initialOtherChildren: ReadonlyArray = [ { id: 400, parentGroup: 20, value: 4 }, ] -type SyncChange = { - type: `insert` | `update` | `delete` - value: T -} - type PublicationAction = | { type: `parentScalar`; value: number } | { type: `childScalar`; value: number } @@ -69,34 +60,6 @@ type PublicationAction = let nextCollectionId = 0 -function createControlledCollection( - name: string, - initialData: ReadonlyArray, -) { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextCollectionId++}`, - getKey: (row) => row.id, - initialData: initialData.map((row) => ({ ...row })), - }) - const collection = createCollection(options) - - const writeBatch = (changes: ReadonlyArray>): void => { - options.utils.begin() - for (const change of changes) options.utils.write(change) - options.utils.commit() - } - - return { - collection, - write(type: SyncChange[`type`], value: T): void { - writeBatch([{ type, value: { ...value } }]) - }, - writeBatch, - resolveSync: options.utils.resolveSync, - rejectSync: options.utils.rejectSync, - } -} - function createLayeredQuery( parents: ReturnType>, children: ReturnType>, diff --git a/packages/db/tests/query/includes-query-shape-oracle.test.ts b/packages/db/tests/query/includes-query-shape-oracle.test.ts index 6e11994420..cbf342402c 100644 --- a/packages/db/tests/query/includes-query-shape-oracle.test.ts +++ b/packages/db/tests/query/includes-query-shape-oracle.test.ts @@ -1,6 +1,5 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' -import { createCollection } from '../../src/collection/index.js' import { BasicIndex } from '../../src/indexes/basic-index.js' import { createLiveQueryCollection, @@ -9,36 +8,13 @@ import { } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' import { oracleRuns } from '../oracle-config.js' -import { mockSyncCollectionOptions } from '../utils.js' +import { createControlledCollection } from './includes-oracle-helpers.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' -let nextCollectionId = 0 - function rowsById(rows: Array): Map { return new Map(rows.map((row) => [row.id, row])) } -function createControlledCollection( - name: string, - initialData: Array = [], -) { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextCollectionId++}`, - getKey: (row) => row.id, - initialData, - }) - const collection = createCollection(options) - - return { - collection, - write(type: `insert` | `update` | `delete`, value: T): void { - options.utils.begin() - options.utils.write({ type, value }) - options.utils.commit() - }, - } -} - function stripVirtualProperties(value: unknown): unknown { if (Array.isArray(value)) { return value.map(stripVirtualProperties)