From 74e8669e7db2bb1dcbe01a1d68e57c3e08c83279 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 22 Aug 2026 14:47:57 -0600 Subject: [PATCH 1/8] fix: preserve correlated include route identity --- packages/db/src/query/builder/index.ts | 118 +++- packages/db/src/query/compiler/group-by.ts | 87 ++- packages/db/src/query/ir.ts | 2 +- packages/db/src/query/live/ARCHITECTURE.md | 14 +- ...ncludes-collection-oracle.property.test.ts | 655 ++++++++++++++++-- ...-cross-formulation-oracle.property.test.ts | 127 +++- ...ncludes-optimistic-oracle.property.test.ts | 42 +- .../db/tests/query/includes-oracle-helpers.ts | 58 ++ .../query/includes-oracle.property.test.ts | 43 +- .../query/includes-publication-oracle.test.ts | 41 +- .../query/includes-query-shape-oracle.test.ts | 26 +- 11 files changed, 951 insertions(+), 262 deletions(-) create mode 100644 packages/db/tests/query/includes-oracle-helpers.ts diff --git a/packages/db/src/query/builder/index.ts b/packages/db/src/query/builder/index.ts index 0291c204cd..0d5d6862bf 100644 --- a/packages/db/src/query/builder/index.ts +++ b/packages/db/src/query/builder/index.ts @@ -1106,14 +1106,17 @@ function buildConditionalSelect( /** * Recursively collects all PropRef nodes from an expression tree. */ -function collectRefsFromExpression(expr: BasicExpression): Array { +function collectRefsFromExpression( + expr: BasicExpression | Aggregate, +): Array { const refs: Array = [] switch (expr.type) { case `ref`: refs.push(expr) break case `func`: - for (const arg of (expr as any).args ?? []) { + case `agg`: + for (const arg of expr.args) { refs.push(...collectRefsFromExpression(arg)) } break @@ -1123,6 +1126,90 @@ function collectRefsFromExpression(expr: BasicExpression): Array { return refs } +function collectRefsFromSelectValue(value: unknown): Array { + if ( + value instanceof PropRef || + value instanceof FuncExpr || + value instanceof AggregateExpr + ) { + return collectRefsFromExpression(value) + } + if (value instanceof ConditionalSelect) { + return [ + ...value.branches.flatMap((branch) => [ + ...collectRefsFromExpression(branch.condition), + ...collectRefsFromSelectValue(branch.value), + ]), + ...(value.defaultValue === undefined + ? [] + : collectRefsFromSelectValue(value.defaultValue)), + ] + } + if (value instanceof IncludesSubquery) { + return [value.correlationField, ...(value.parentProjection ?? [])] + } + if (!isPlainObject(value)) return [] + return Object.values(value).flatMap(collectRefsFromSelectValue) +} + +function collectParentRefsFromQuery( + query: QueryIR, + parentAliases: Array, +): Array { + const refs: Array = [] + const addExpression = (expression: BasicExpression | Aggregate) => { + refs.push(...collectRefsFromExpression(expression)) + } + const addWhere = (where: Where) => { + addExpression( + typeof where === `object` && `expression` in where + ? where.expression + : where, + ) + } + + for (const where of query.where ?? []) addWhere(where) + for (const join of query.join ?? []) { + addExpression(join.left) + addExpression(join.right) + if (join.from.type === `queryRef`) { + refs.push(...collectParentRefsFromQuery(join.from.query, parentAliases)) + } + } + for (const expression of query.groupBy ?? []) addExpression(expression) + for (const having of query.having ?? []) addWhere(having) + for (const { expression } of query.orderBy ?? []) addExpression(expression) + if (query.select) refs.push(...collectRefsFromSelectValue(query.select)) + + if (query.from.type === `queryRef`) { + refs.push(...collectParentRefsFromQuery(query.from.query, parentAliases)) + } else if (query.from.type === `unionFrom`) { + for (const source of query.from.sources) { + if (source.type === `queryRef`) { + refs.push(...collectParentRefsFromQuery(source.query, parentAliases)) + } + } + } else if (query.from.type === `unionAll`) { + for (const branch of query.from.queries) { + refs.push(...collectParentRefsFromQuery(branch, parentAliases)) + } + } + + const seen = new Set() + return refs.filter((ref) => { + const path = ref.path.join(`.`) + if ( + ref.path[0] == null || + !parentAliases.includes(ref.path[0]) || + seen.has(path) + ) { + return false + } + seen.add(path) + return true + }) +} + /** * Checks whether a WHERE clause references any parent alias. */ @@ -1271,25 +1358,14 @@ function buildIncludesSubquery( } } - // Collect distinct parent PropRefs from parent-referencing filters - let parentProjection: Array | undefined - if (parentFilters.length > 0) { - const seen = new Set() - parentProjection = [] - for (const w of parentFilters) { - const expr = typeof w === `object` && `expression` in w ? w.expression : w - for (const ref of collectRefsFromExpression(expr)) { - if ( - ref.path[0] != null && - parentAliases.includes(ref.path[0]) && - !seen.has(ref.path.join(`.`)) - ) { - seen.add(ref.path.join(`.`)) - parentProjection.push(ref) - } - } - } - } + // Every parent input that can affect the child plan belongs to the route + // identity, not only the main equality key or residual filters. + const projectedParentRefs = collectParentRefsFromQuery( + { ...childQuery, where: modifiedWhere }, + parentAliases, + ) + const parentProjection = + projectedParentRefs.length > 0 ? projectedParentRefs : undefined const modifiedQuery: QueryIR = { ...childQuery, diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index c670de9649..e71d03a25f 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -43,6 +43,37 @@ type RowVirtualMetadata = { hasLocal: boolean } +function addCorrelationRouteToGroupKey( + key: Record, + row: NamespacedRow, + mainSource: string, +): void { + const rowRecord = row as Record + const source = rowRecord[mainSource] as Record | undefined + key.__correlationKey = source?.__correlationKey + if (rowRecord.__parentContext != null) { + key.__parentContext = rowRecord.__parentContext + } +} + +function getCorrelationRouteIdentity( + aggregatedRow: Record, +): unknown { + return aggregatedRow.__parentContext == null + ? aggregatedRow.__correlationKey + : [aggregatedRow.__correlationKey, aggregatedRow.__parentContext] +} + +function getHavingEvaluationRow(row: Record): NamespacedRow { + const parentContext = row.__parentContext + return { + ...(parentContext !== null && typeof parentContext === `object` + ? (parentContext as NamespacedRow) + : {}), + $selected: row.$selected as Record, + } +} + function getRowVirtualMetadata(row: NamespacedRow): RowVirtualMetadata { let found = false let allSynced = true @@ -189,15 +220,13 @@ export function processGroupBy( } } - // Use a constant key for single group. - // When mainSource is set (includes mode), include __correlationKey so that - // rows from different parents aggregate separately. - const keyExtractor = mainSource - ? ([, row]: [string, NamespacedRow]) => ({ - __singleGroup: true, - __correlationKey: (row as any)?.[mainSource]?.__correlationKey, - }) - : () => ({ __singleGroup: true }) + // Use a constant key for single group. In includes mode, add the complete + // correlation route so parents with distinct projected inputs stay apart. + const keyExtractor = ([, row]: [string, NamespacedRow]) => { + const key: Record = { __singleGroup: true } + if (mainSource) addCorrelationRouteToGroupKey(key, row, mainSource) + return key + } // Apply the groupBy operator with single group pipeline = pipeline.pipe( @@ -231,9 +260,12 @@ export function processGroupBy( const correlationKey = mainSource ? (aggregatedRow as any).__correlationKey : undefined + const correlationRoute = mainSource + ? getCorrelationRouteIdentity(aggregatedRow) + : undefined const resultKey = - correlationKey !== undefined - ? `single_group_${serializeValue(correlationKey)}` + correlationRoute !== undefined + ? `single_group_${serializeValue(correlationRoute)}` : `single_group` const resultRow: Record = { ...(aggregatedRow as Record), @@ -272,8 +304,7 @@ export function processGroupBy( pipeline = pipeline.pipe( filter(([, row]) => { - // Create a namespaced row structure for HAVING evaluation - const namespacedRow = { $selected: (row as any).$selected } + const namespacedRow = getHavingEvaluationRow(row) return toBooleanPredicate(compiledHaving(namespacedRow)) }), ) @@ -285,8 +316,7 @@ export function processGroupBy( for (const fnHaving of fnHavingClauses) { pipeline = pipeline.pipe( filter(([, row]) => { - // Create a namespaced row structure for functional HAVING evaluation - const namespacedRow = { $selected: (row as any).$selected } + const namespacedRow = getHavingEvaluationRow(row) return toBooleanPredicate(fnHaving(namespacedRow)) }), ) @@ -305,9 +335,9 @@ export function processGroupBy( compileExpression(e), ) - // Create a key extractor function using simple __key_X format. - // When mainSource is set (includes mode), include __correlationKey so that - // rows from different parents with the same group key aggregate separately. + // Create a key extractor function using simple __key_X format. In includes + // mode, add the complete route so parents with distinct projected inputs do + // not aggregate together. const keyExtractor = ([, row]: [ string, NamespacedRow & { $selected?: any }, @@ -325,9 +355,7 @@ export function processGroupBy( key[`__key_${i}`] = value } - if (mainSource) { - key.__correlationKey = (row as any)?.[mainSource]?.__correlationKey - } + if (mainSource) addCorrelationRouteToGroupKey(key, row, mainSource) return key } @@ -397,17 +425,20 @@ export function processGroupBy( } // Generate a simple key for the live collection using group values. - // When in includes mode, include the correlation key so that groups - // from different parents don't collide. + // In includes mode, add the complete route so correlated groups do not + // collide. const correlationKey = mainSource ? (aggregatedRow as any).__correlationKey : undefined + const correlationRoute = mainSource + ? getCorrelationRouteIdentity(aggregatedRow) + : undefined const keyParts: Array = [] for (let i = 0; i < groupByClause.length; i++) { keyParts.push(aggregatedRow[`__key_${i}`]) } - if (correlationKey !== undefined) { - keyParts.push(correlationKey) + if (correlationRoute !== undefined) { + keyParts.push(correlationRoute) } const finalKey = keyParts.length === 1 ? keyParts[0] : serializeValue(keyParts) @@ -449,8 +480,7 @@ export function processGroupBy( pipeline = pipeline.pipe( filter(([, row]) => { - // Create a namespaced row structure for HAVING evaluation - const namespacedRow = { $selected: (row as any).$selected } + const namespacedRow = getHavingEvaluationRow(row) return compiledHaving(namespacedRow) }), ) @@ -462,8 +492,7 @@ export function processGroupBy( for (const fnHaving of fnHavingClauses) { pipeline = pipeline.pipe( filter(([, row]) => { - // Create a namespaced row structure for functional HAVING evaluation - const namespacedRow = { $selected: (row as any).$selected } + const namespacedRow = getHavingEvaluationRow(row) return toBooleanPredicate(fnHaving(namespacedRow)) }), ) diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index 739d036a98..d551831afd 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -189,7 +189,7 @@ export class IncludesSubquery extends BaseExpression { public childCorrelationField: PropRef, // Child-side ref (e.g., issue.projectId) public fieldName: string, // Result field name (e.g., "issues") public parentFilters?: Array, // WHERE clauses referencing parent aliases (applied post-join) - public parentProjection?: Array, // Parent field refs used by parentFilters + public parentProjection?: Array, // Parent field refs used anywhere in the child plan public materialization: IncludesMaterialization = `collection`, public scalarField?: string, ) { diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 41bdda884e..f0a390ce6f 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -99,9 +99,10 @@ reduction that enforces public-key congruence and multiplicity. ## Identity Aliases are lexical query-language names. They are not runtime identities. The -query builder requires collection aliases to be unique across one query tree, -including includes; shadowing is rejected before compilation. Compilation then -assigns opaque IDs to the accepted plan: +query builder requires collection aliases to be unique within each lexical +scope and rejects nested queries that shadow an ancestor alias. Sibling include +scopes may reuse an alias because neither alias is visible to the other. +Compilation then assigns opaque IDs to the accepted plan: ```ts type SourceId = Brand @@ -114,8 +115,8 @@ unused name cannot change the compiled graph or its result. A `CanonicalCorrelationKey` is the canonical tuple of every evaluated parent-dependent value that can affect the child plan. This includes values -used by filters, joins, ordering, limits, and nullable predicates, not only the -obvious foreign-key equality. +used by filters, joins, grouping, aggregates, ordering, projections, limits, +and nullable predicates, not only the obvious foreign-key equality. A bucket key identifies one such correlated partition at one relation node: @@ -466,7 +467,8 @@ create recursive Collection machinery. ## Normative laws 1. **Alpha-renaming:** changing any accepted alias to another unused name cannot - change results; alias shadowing within a query tree is rejected. + change results; aliases must be unique within one lexical scope and cannot + shadow an ancestor alias. Sibling scopes may reuse aliases. 2. **Contribution conservation:** a public row exists exactly when its reduced supporting weight and collision policy produce one. 3. **Batch partition:** equivalent valid split and atomic deliveries converge. diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 6f001b586d..e83210c478 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1,23 +1,27 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' -import { createCollection } from '../../src/collection/index.js' import { + add, + caseWhen, concat, + count, createLiveQueryCollection, eq, + gt, + lt, materialize, + multiply, + sum, toArray, } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' import { oraclePropertyOptions } from '../oracle-config.js' -import { - flushPromises, - mockSyncCollectionOptions, - withExpectedRejection, -} from '../utils.js' +import { flushPromises, withExpectedRejection } from '../utils.js' +import { createControlledCollection as createOracleControlledCollection } from './includes-oracle-helpers.js' import type { Collection } from '../../src/collection/index.js' import type { ChangeMessage } from '../../src/types.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' +import type { ControlledCollection } from './includes-oracle-helpers.js' type ParentRow = { id: number @@ -50,19 +54,6 @@ type CollectionObservation = { publications: Array> } -type ControlledCollection = { - collection: Collection - write: (type: `insert` | `update` | `delete`, value: T) => void - writeBatch: ( - changes: ReadonlyArray<{ - type: `insert` | `update` | `delete` - value: T - }>, - ) => void - resolveSync: () => void - rejectSync: (error: Error) => void -} - type CollectionContext = { parents: ControlledCollection children: ControlledCollection @@ -75,39 +66,21 @@ type CollectionContext = { subscription?: { unsubscribe: () => void } } -let nextCollectionOracleId = 0 - function createControlledCollection( name: string, initialData: ReadonlyArray, ): ControlledCollection { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextCollectionOracleId++}`, - getKey: (row) => row.id, - initialData: initialData.map((row) => ({ ...row })), + return createOracleControlledCollection(name, initialData, { autoIndex: `eager`, + rowUpdateMode: `full`, }) - options.sync.rowUpdateMode = `full` - const collection = createCollection(options) - const writeBatch: ControlledCollection[`writeBatch`] = (changes) => { - options.utils.begin() - for (const change of changes) { - options.utils.write({ - type: change.type, - value: { ...change.value }, - }) - } - options.utils.commit() - } +} +function expectedMaterializations(rows: ReadonlyArray) { return { - collection, - write(type, value) { - writeBatch([{ type, value }]) - }, - writeBatch, - resolveSync: options.utils.resolveSync, - rejectSync: options.utils.rejectSync, + facade: [...rows], + array: [...rows], + materialized: [...rows], } } @@ -1123,6 +1096,600 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest( + `propagates an order-only child move through every materialization`, + async () => { + type OrderedChild = ChildRow & { position: number; label: string } + const parents = createControlledCollection(`order-move-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `order-move-children`, + [ + { id: 10, parentGroup: 1, value: 1, position: 0, label: `a` }, + { id: 20, parentGroup: 1, value: 2, position: 1, label: `b` }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ + id: child.id, + position: child.position, + label: child.label, + })) + + return { + id: parent.id, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + first: materialize(childRows().findOne()), + joined: concat( + toArray( + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .orderBy(({ child }) => child.id) + .select(({ child }) => child.label), + ), + ), + } + }), + ) + + const project = () => { + const row = live.get(1)! + return { + facade: row.facade.toArray.map(({ id }) => id), + array: row.array.map(({ id }) => id), + materialized: row.materialized.map(({ id }) => id), + first: row.first?.id, + joined: row.joined, + } + } + + try { + await live.preload() + const facade = live.get(1)!.facade + const revision = facade._layoutRevision + expect(project()).toEqual({ + ...expectedMaterializations([10, 20]), + first: 10, + joined: `ab`, + }) + + children.write(`update`, { + id: 10, + parentGroup: 1, + value: 1, + position: 2, + label: `a`, + }) + + expect(live.get(1)!.facade).toBe(facade) + expect(facade._layoutRevision).toBeGreaterThan(revision) + expect(project()).toEqual({ + ...expectedMaterializations([20, 10]), + first: 20, + joined: `ba`, + }) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + fcTest( + `reconstructs nested conditional includes through guard transitions`, + async () => { + type GuardedParent = ParentRow & { active: boolean } + const parents = createControlledCollection( + `guarded-parents`, + [{ id: 1, group: 1, active: true }], + ) + const children = createControlledCollection(`guarded-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + value: child.value, + })) + + return { + id: parent.id, + profile: caseWhen( + eq(parent.active, true), + { + kind: `active` as const, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + }, + { kind: `inactive` as const }, + ), + } + }), + ) + + const project = () => { + const profile = live.get(1)!.profile + if (profile.kind === `inactive`) return profile + const rows = (values: Iterable) => + [...values].map(({ id, value }) => ({ id, value })) + return { + kind: profile.kind, + facade: rows(profile.facade.values()), + array: rows(profile.array), + materialized: rows(profile.materialized), + } + } + + try { + await live.preload() + const initialProfile = live.get(1)!.profile + if (initialProfile.kind === `inactive`) { + throw new Error( + `Expected the initial conditional branch to be active`, + ) + } + const initialFacade = initialProfile.facade + expect(project()).toEqual({ + kind: `active`, + ...expectedMaterializations([{ id: 10, value: 1 }]), + }) + + parents.write(`update`, { id: 1, group: 1, active: false }) + expect(project()).toEqual({ kind: `inactive` }) + expect(initialFacade.toArray).toEqual([]) + expect(initialFacade.status).toBe(`ready`) + children.write(`insert`, { id: 20, parentGroup: 1, value: 2 }) + expect(project()).toEqual({ kind: `inactive` }) + + parents.write(`update`, { id: 1, group: 1, active: true }) + const reactivatedProfile = live.get(1)!.profile + if (reactivatedProfile.kind === `inactive`) { + throw new Error(`Expected the conditional branch to reactivate`) + } + expect(reactivatedProfile.facade).not.toBe(initialFacade) + expect(project()).toEqual({ + kind: `active`, + ...expectedMaterializations([ + { id: 10, value: 1 }, + { id: 20, value: 2 }, + ]), + }) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + fcTest( + `matches recomputation for correlated aggregate child relations`, + async () => { + const parents = createControlledCollection(`aggregate-parents`, [ + { id: 1, group: 1, factor: 1 }, + { id: 2, group: 1, factor: -1 }, + { id: 3, group: 2, factor: 2 }, + ]) + const children = createControlledCollection(`aggregate-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + { id: 30, parentGroup: 2, value: 5 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const summaries = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .groupBy(({ child }) => child.parentGroup) + .select(({ child }) => ({ + parentGroup: child.parentGroup, + count: count(child.id), + total: sum(multiply(child.value, parent.factor)), + })) + const total = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => ({ + count: count(child.id), + total: sum(multiply(child.value, parent.factor)), + })) + + return { + id: parent.id, + facade: summaries(), + array: toArray(summaries()), + materialized: materialize(summaries()), + implicit: materialize(total()), + } + }), + ) + + type Summary = { parentGroup: number; count: number; total: number } + const project = () => + live.toArray.map((row) => { + const clean = (values: Iterable) => + [...values].map(({ parentGroup, count: size, total }) => ({ + parentGroup, + count: size, + total, + })) + return { + id: row.id, + facade: clean(row.facade.values()), + array: clean(row.array), + materialized: clean(row.materialized), + implicit: row.implicit.map(({ count: size, total }) => ({ + count: size, + total, + })), + } + }) + + const expected = (groupOneTotal: number, groupOneCount: number) => [ + { + id: 1, + ...expectedMaterializations([ + { parentGroup: 1, count: groupOneCount, total: groupOneTotal }, + ]), + implicit: [{ count: groupOneCount, total: groupOneTotal }], + }, + { + id: 2, + ...expectedMaterializations([ + { + parentGroup: 1, + count: groupOneCount, + total: -groupOneTotal, + }, + ]), + implicit: [{ count: groupOneCount, total: -groupOneTotal }], + }, + { + id: 3, + ...expectedMaterializations([ + { parentGroup: 2, count: 1, total: 10 }, + ]), + implicit: [{ count: 1, total: 10 }], + }, + ] + + try { + await live.preload() + expect(project()).toEqual(expected(3, 2)) + + children.write(`update`, { id: 20, parentGroup: 1, value: 7 }) + expect(project()).toEqual(expected(8, 2)) + + children.write(`delete`, { id: 10, parentGroup: 1, value: 1 }) + expect(project()).toEqual(expected(7, 1)) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + fcTest( + `evaluates correlated having clauses with parent context`, + async () => { + type HavingParent = ParentRow & { threshold: number } + const parents = createControlledCollection( + `having-parents`, + [ + { id: 1, group: 1, threshold: 1 }, + { id: 2, group: 1, threshold: 3 }, + ], + ) + const children = createControlledCollection(`having-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const summaries = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .groupBy(({ child }) => child.parentGroup) + .having(({ child }) => gt(count(child.id), parent.threshold)) + .select(({ child }) => ({ count: count(child.id) })) + + return { + id: parent.id, + facade: summaries(), + array: toArray(summaries()), + materialized: materialize(summaries()), + } + }), + ) + + const project = () => + live.toArray.map((row) => { + const counts = (values: Iterable<{ count: number }>) => + [...values].map(({ count: size }) => size) + return { + id: row.id, + facade: counts(row.facade.values()), + array: counts(row.array), + materialized: counts(row.materialized), + } + }) + + try { + await live.preload() + expect(project()).toEqual([ + { id: 1, ...expectedMaterializations([2]) }, + { id: 2, ...expectedMaterializations([]) }, + ]) + + parents.write(`update`, { id: 2, group: 1, threshold: 1 }) + expect(project()).toEqual([ + { id: 1, ...expectedMaterializations([2]) }, + { id: 2, ...expectedMaterializations([2]) }, + ]) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + fcTest(`routes changes to non-key parent filter inputs`, async () => { + type FilterParent = ParentRow & { threshold: number } + const parents = createControlledCollection( + `parent-filter-input-parents`, + [{ id: 1, group: 1, threshold: 2 }], + ) + const children = createControlledCollection( + `parent-filter-input-children`, + [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 3 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .where(({ child }) => lt(child.value, parent.threshold)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ id: child.id, value: child.value })) + + return { + id: parent.id, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + } + }), + ) + + const project = () => { + const row = live.get(1)! + const ids = (values: Iterable<{ id: number }>) => + [...values].map(({ id }) => id) + return { + facade: ids(row.facade.values()), + array: ids(row.array), + materialized: ids(row.materialized), + } + } + + try { + await live.preload() + expect(project()).toEqual(expectedMaterializations([10])) + + parents.write(`update`, { id: 1, group: 1, threshold: 4 }) + expect(project()).toEqual(expectedMaterializations([10, 20])) + + parents.write(`update`, { id: 1, group: 1, threshold: 0 }) + expect(project()).toEqual(expectedMaterializations([])) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }) + + fcTest(`routes changes to parent-dependent child ordering`, async () => { + type OrderingParent = ParentRow & { direction: number } + const parents = createControlledCollection( + `parent-order-input-parents`, + [ + { id: 1, group: 1, direction: 1 }, + { id: 2, group: 1, direction: -1 }, + ], + ) + const children = createControlledCollection(`parent-order-input-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => multiply(child.value, parent.direction)) + .select(({ child }) => ({ id: child.id, value: child.value })) + + return { + id: parent.id, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + } + }), + ) + + const project = (id: number) => { + const row = live.get(id)! + const rows = (values: Iterable<{ id: number; value: number }>) => + [...values].map(({ id: childId, value }) => ({ id: childId, value })) + return { + facade: rows(row.facade.values()), + array: rows(row.array), + materialized: rows(row.materialized), + } + } + + try { + await live.preload() + const ascendingFacade = live.get(1)!.facade + const descendingFacade = live.get(2)!.facade + expect(project(1)).toEqual( + expectedMaterializations([ + { id: 10, value: 1 }, + { id: 20, value: 2 }, + ]), + ) + expect(project(2)).toEqual( + expectedMaterializations([ + { id: 20, value: 2 }, + { id: 10, value: 1 }, + ]), + ) + expect(ascendingFacade).not.toBe(descendingFacade) + + parents.write(`update`, { id: 1, group: 1, direction: -1 }) + expect(live.get(1)!.facade).toBe(descendingFacade) + expect(ascendingFacade.toArray).toEqual([]) + expect(project(1)).toEqual( + expectedMaterializations([ + { id: 20, value: 2 }, + { id: 10, value: 1 }, + ]), + ) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }) + + fcTest(`routes changes to parent-dependent child joins`, async () => { + type JoinParent = ParentRow & { offset: number } + type JoinedChild = ChildRow & { tagId: number } + type Tag = { id: number; label: string } + const parents = createControlledCollection( + `parent-join-parents`, + [ + { id: 1, group: 1, offset: 0 }, + { id: 2, group: 1, offset: 1 }, + ], + ) + const children = createControlledCollection( + `parent-join-children`, + [{ id: 10, parentGroup: 1, value: 1, tagId: 1 }], + ) + const tags = createControlledCollection(`parent-join-tags`, [ + { id: 1, label: `direct` }, + { id: 2, label: `offset` }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .innerJoin({ tag: tags.collection }, ({ child, tag }) => + eq(tag.id, add(child.tagId, parent.offset)), + ) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child, tag }) => ({ + id: child.id, + label: tag.label, + })) + + return { + id: parent.id, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + } + }), + ) + + const project = (id: number) => { + const row = live.get(id)! + const labels = (values: Iterable<{ label: string }>) => + [...values].map(({ label }) => label) + return { + facade: labels(row.facade.values()), + array: labels(row.array), + materialized: labels(row.materialized), + } + } + + try { + await live.preload() + const directFacade = live.get(1)!.facade + const offsetFacade = live.get(2)!.facade + expect(directFacade).not.toBe(offsetFacade) + expect(project(1)).toEqual(expectedMaterializations([`direct`])) + expect(project(2)).toEqual(expectedMaterializations([`offset`])) + + parents.write(`update`, { id: 1, group: 1, offset: 1 }) + expect(live.get(1)!.facade).toBe(offsetFacade) + expect(directFacade.toArray).toEqual([]) + expect(project(1)).toEqual(expectedMaterializations([`offset`])) + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + tags.collection.cleanup(), + ]) + } + }) + fcTest.prop( [ fc.record({ diff --git a/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts b/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts index 59920e5ea1..850aeb02c2 100644 --- a/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts +++ b/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts @@ -1,6 +1,5 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' -import { createCollection } from '../../src/collection/index.js' import { createLiveQueryCollection, eq, @@ -11,8 +10,10 @@ import { toArray, } from '../../src/query/index.js' import { oraclePropertyOptions } from '../oracle-config.js' -import { flushPromises, mockSyncCollectionOptions } from '../utils.js' +import { flushPromises } from '../utils.js' +import { createControlledCollection as createOracleControlledCollection } from './includes-oracle-helpers.js' import type { Collection } from '../../src/collection/index.js' +import type { ControlledCollection } from './includes-oracle-helpers.js' type ParentRow = { id: number @@ -44,11 +45,6 @@ type NormalizedParent = ParentRow & { children: Array } -type ControlledCollection = { - collection: Collection - write: (type: `insert` | `update` | `delete`, value: T) => void -} - type FlatRow = { parentId: number parentGroup: number @@ -56,27 +52,14 @@ type FlatRow = { child: ChildRow | undefined } -let nextCrossFormulationId = 0 - function createControlledCollection( name: string, initialData: ReadonlyArray, ): ControlledCollection { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextCrossFormulationId++}`, - getKey: (row) => row.id, - initialData: initialData.map((row) => ({ ...row })), + return createOracleControlledCollection(name, initialData, { autoIndex: `eager`, + rowUpdateMode: `full`, }) - options.sync.rowUpdateMode = `full` - return { - collection: createCollection(options), - write(type, value) { - options.utils.begin() - options.utils.write({ type, value: { ...value } }) - options.utils.commit() - }, - } } function compareParents(left: ParentRow, right: ParentRow): number { @@ -170,6 +153,42 @@ function createNestedQuery( }) } +function createWindowedNestedQuery( + parents: Collection, + children: Collection, + offset: number, + limit: number, +) { + return createLiveQueryCollection({ + getKey: (row) => row.id, + query: (q) => + q + .from({ parent: parents }) + .orderBy(({ parent }) => parent.position) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => ({ + id: parent.id, + group: parent.group, + position: parent.position, + children: toArray( + q + .from({ child: children }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .orderBy(({ child }) => child.id) + .offset(offset) + .limit(limit) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + score: child.score, + position: child.position, + })), + ), + })), + }) +} + function createFlatQuery( parents: Collection, children: Collection, @@ -358,6 +377,58 @@ async function expectFormulationsEquivalent( } } +async function expectWindowedIncludeMatches( + scenario: CrossFormulationScenario, + offset: number, + limit: number, +): Promise { + const parentSource = createControlledCollection( + `windowed-cross-form-parents`, + scenario.parents, + ) + const childSource = createControlledCollection( + `windowed-cross-form-children`, + scenario.children, + ) + const parents = new Map(scenario.parents.map((row) => [row.id, { ...row }])) + const children = new Map(scenario.children.map((row) => [row.id, { ...row }])) + const nested = createWindowedNestedQuery( + parentSource.collection, + childSource.collection, + offset, + limit, + ) + + const assertEquivalent = () => { + const expected = normalizeNested( + [...parents.values()].map((parent) => ({ + ...parent, + children: [...children.values()] + .filter((child) => child.parentGroup === parent.group) + .sort(compareChildren) + .slice(offset, offset + limit), + })), + ) + expect(normalizeNested(nested.toArray)).toEqual(expected) + } + + try { + await nested.preload() + assertEquivalent() + for (const action of scenario.actions) { + applyAction(action, parentSource, childSource, parents, children) + await flushPromises() + assertEquivalent() + } + } finally { + await Promise.allSettled([ + nested.cleanup(), + parentSource.collection.cleanup(), + childSource.collection.cleanup(), + ]) + } +} + const parentRowArbitrary = (id: number) => fc.record({ id: fc.constant(id), @@ -403,6 +474,12 @@ const scenarioArbitrary: fc.Arbitrary = fc.record({ actions: fc.array(actionArbitrary, { minLength: 1, maxLength: 5 }), }) +const windowedScenarioArbitrary = fc.record({ + scenario: scenarioArbitrary, + offset: fc.integer({ min: 0, max: 2 }), + limit: fc.integer({ min: 0, max: 3 }), +}) + describe(`includes cross-formulation oracle`, () => { fcTest(`shared-route child deletion agrees across formulations`, () => expectFormulationsEquivalent({ @@ -424,4 +501,10 @@ describe(`includes cross-formulation oracle`, () => { `agrees across nested includes, flat joins, per-parent queries, and TLP partitions`, expectFormulationsEquivalent, ) + + fcTest.prop([windowedScenarioArbitrary], oraclePropertyOptions(12))( + `matches recomputation for ordered offset and limit child windows`, + ({ scenario, offset, limit }) => + expectWindowedIncludeMatches(scenario, offset, limit), + ) }) diff --git a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts index 1a190a911b..0e125cdfc8 100644 --- a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts +++ b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts @@ -1,19 +1,16 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' -import { createCollection } from '../../src/collection/index.js' import { createLiveQueryCollection, eq, toArray, } from '../../src/query/index.js' -import { - flushPromises, - mockSyncCollectionOptions, - withExpectedRejection, -} from '../utils.js' +import { flushPromises, withExpectedRejection } from '../utils.js' import { runTrace } from '../trace-runner.js' import { oraclePropertyOptions } from '../oracle-config.js' +import { createControlledCollection as createOracleControlledCollection } from './includes-oracle-helpers.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' +import type { OracleSyncChange as SyncChange } from './includes-oracle-helpers.js' type RootRow = { id: number @@ -26,11 +23,6 @@ type ChildRow = RootRow & { parentGroup: number } -type SyncChange = { - type: `insert` | `update` | `delete` - value: T -} - type ChildLevel = 1 | 2 | 3 type ChildPatch = Partial< @@ -149,37 +141,13 @@ type RouteValues = { authoritative: number } -let nextHarnessId = 0 - function createControlledCollection( name: string, initialData: ReadonlyArray, ) { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextHarnessId++}`, - getKey: (row) => row.id, - initialData: initialData.map((row) => ({ ...row })), + return createOracleControlledCollection(name, initialData, { + rowUpdateMode: `full`, }) - options.sync.rowUpdateMode = `full` - const collection = createCollection(options) - - const writeBatch = (changes: ReadonlyArray>) => { - options.utils.begin() - for (const change of changes) { - options.utils.write({ - type: change.type, - value: { ...change.value }, - }) - } - options.utils.commit() - } - - return { - collection, - writeBatch, - resolveSync: options.utils.resolveSync, - rejectSync: options.utils.rejectSync, - } } function createSources( diff --git a/packages/db/tests/query/includes-oracle-helpers.ts b/packages/db/tests/query/includes-oracle-helpers.ts new file mode 100644 index 0000000000..77f08543b5 --- /dev/null +++ b/packages/db/tests/query/includes-oracle-helpers.ts @@ -0,0 +1,58 @@ +import { createCollection } from '../../src/collection/index.js' +import { mockSyncCollectionOptions } from '../utils.js' +import type { Collection } from '../../src/collection/index.js' + +export type OracleSyncChange = { + type: `insert` | `update` | `delete` + value: T +} + +export type ControlledCollection = { + collection: Collection + write: (type: OracleSyncChange[`type`], value: T) => void + writeBatch: (changes: ReadonlyArray>) => void + resolveSync: () => void + rejectSync: (error: Error) => void +} + +type ControlledCollectionOptions = { + autoIndex?: `off` | `eager` + rowUpdateMode?: `partial` | `full` +} + +let nextControlledCollectionId = 0 + +export function createControlledCollection( + name: string, + initialData: ReadonlyArray = [], + options: ControlledCollectionOptions = {}, +): ControlledCollection { + const collectionOptions = mockSyncCollectionOptions({ + id: `${name}-${nextControlledCollectionId++}`, + getKey: (row) => row.id, + initialData: initialData.map((row) => ({ ...row })), + ...(options.autoIndex ? { autoIndex: options.autoIndex } : {}), + }) + collectionOptions.sync.rowUpdateMode = options.rowUpdateMode ?? `partial` + const collection = createCollection(collectionOptions) + const writeBatch: ControlledCollection[`writeBatch`] = (changes) => { + collectionOptions.utils.begin() + for (const change of changes) { + collectionOptions.utils.write({ + type: change.type, + value: { ...change.value }, + }) + } + collectionOptions.utils.commit() + } + + return { + collection, + write(type, value) { + writeBatch([{ type, value }]) + }, + writeBatch, + resolveSync: collectionOptions.utils.resolveSync, + rejectSync: collectionOptions.utils.rejectSync, + } +} diff --git a/packages/db/tests/query/includes-oracle.property.test.ts b/packages/db/tests/query/includes-oracle.property.test.ts index c487f4897c..8fb7625876 100644 --- a/packages/db/tests/query/includes-oracle.property.test.ts +++ b/packages/db/tests/query/includes-oracle.property.test.ts @@ -8,19 +8,16 @@ import { queryOnce, toArray, } from '../../src/query/index.js' -import { createCollection } from '../../src/collection/index.js' -import { - flushPromises, - mockSyncCollectionOptions, - withExpectedRejection, -} from '../utils.js' +import { flushPromises, withExpectedRejection } from '../utils.js' import { oraclePropertyOptions, oracleRuns } from '../oracle-config.js' import { runTrace } from '../trace-runner.js' +import { createControlledCollection as createOracleControlledCollection } from './includes-oracle-helpers.js' import type { TraceCheckpoint, TraceDriver, TraceProjection, } from '../trace-runner.js' +import type { OracleSyncChange as SyncChange } from './includes-oracle-helpers.js' type IncludeDepth = 1 | 2 | 3 | 4 @@ -35,11 +32,6 @@ type ChildRow = RootRow & { parentGroup: number } -type SyncChange = { - type: `insert` | `update` | `delete` - value: T -} - type HistoryAction = { type: `put` | `delete` | `optimisticConfirm` | `optimisticRollback` level: 0 | IncludeDepth @@ -371,39 +363,14 @@ const confirmedChildReorderSeed: Scenario = { ], } -let nextHarnessId = 0 - function createControlledCollection( name: string, initialData: Array = [], rowUpdateMode: `partial` | `full` = `partial`, ) { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextHarnessId++}`, - getKey: (row) => row.id, - initialData, + return createOracleControlledCollection(name, initialData, { + rowUpdateMode, }) - options.sync.rowUpdateMode = rowUpdateMode - const collection = createCollection(options) - const writeBatch = (changes: ReadonlyArray>): void => { - options.utils.begin() - changes.forEach((change) => options.utils.write(change)) - options.utils.commit() - } - - return { - collection, - write(type: `insert` | `update` | `delete`, value: T): void { - writeBatch([{ type, value }]) - }, - writeBatch, - resolveSync(): void { - options.utils.resolveSync() - }, - rejectSync(error: Error): void { - options.utils.rejectSync(error) - }, - } } function compareRows(left: RootRow, right: RootRow): number { diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 0201d57de0..e78656d70e 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -1,6 +1,5 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' -import { createCollection } from '../../src/collection/index.js' import { BasicIndex } from '../../src/indexes/basic-index.js' import { createLiveQueryCollection, @@ -9,11 +8,8 @@ import { } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' import { oraclePropertyOptions } from '../oracle-config.js' -import { - flushPromises, - mockSyncCollectionOptions, - withExpectedRejection, -} from '../utils.js' +import { flushPromises, withExpectedRejection } from '../utils.js' +import { createControlledCollection } from './includes-oracle-helpers.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' type ParentRow = { @@ -53,11 +49,6 @@ const initialOtherChildren: ReadonlyArray = [ { id: 400, parentGroup: 20, value: 4 }, ] -type SyncChange = { - type: `insert` | `update` | `delete` - value: T -} - type PublicationAction = | { type: `parentScalar`; value: number } | { type: `childScalar`; value: number } @@ -69,34 +60,6 @@ type PublicationAction = let nextCollectionId = 0 -function createControlledCollection( - name: string, - initialData: ReadonlyArray, -) { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextCollectionId++}`, - getKey: (row) => row.id, - initialData: initialData.map((row) => ({ ...row })), - }) - const collection = createCollection(options) - - const writeBatch = (changes: ReadonlyArray>): void => { - options.utils.begin() - for (const change of changes) options.utils.write(change) - options.utils.commit() - } - - return { - collection, - write(type: SyncChange[`type`], value: T): void { - writeBatch([{ type, value: { ...value } }]) - }, - writeBatch, - resolveSync: options.utils.resolveSync, - rejectSync: options.utils.rejectSync, - } -} - function createLayeredQuery( parents: ReturnType>, children: ReturnType>, diff --git a/packages/db/tests/query/includes-query-shape-oracle.test.ts b/packages/db/tests/query/includes-query-shape-oracle.test.ts index 6e11994420..cbf342402c 100644 --- a/packages/db/tests/query/includes-query-shape-oracle.test.ts +++ b/packages/db/tests/query/includes-query-shape-oracle.test.ts @@ -1,6 +1,5 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' -import { createCollection } from '../../src/collection/index.js' import { BasicIndex } from '../../src/indexes/basic-index.js' import { createLiveQueryCollection, @@ -9,36 +8,13 @@ import { } from '../../src/query/index.js' import { runTrace } from '../trace-runner.js' import { oracleRuns } from '../oracle-config.js' -import { mockSyncCollectionOptions } from '../utils.js' +import { createControlledCollection } from './includes-oracle-helpers.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' -let nextCollectionId = 0 - function rowsById(rows: Array): Map { return new Map(rows.map((row) => [row.id, row])) } -function createControlledCollection( - name: string, - initialData: Array = [], -) { - const options = mockSyncCollectionOptions({ - id: `${name}-${nextCollectionId++}`, - getKey: (row) => row.id, - initialData, - }) - const collection = createCollection(options) - - return { - collection, - write(type: `insert` | `update` | `delete`, value: T): void { - options.utils.begin() - options.utils.write({ type, value }) - options.utils.commit() - }, - } -} - function stripVirtualProperties(value: unknown): unknown { if (Array.isArray(value)) { return value.map(stripVirtualProperties) From 95e6391a39607d4ecf8b0fb80e21ffd6cc4bdf90 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 22 Aug 2026 14:49:20 -0600 Subject: [PATCH 2/8] chore: add changeset for correlated includes --- .changeset/fix-correlated-include-routes.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fix-correlated-include-routes.md diff --git a/.changeset/fix-correlated-include-routes.md b/.changeset/fix-correlated-include-routes.md new file mode 100644 index 0000000000..69e0c14fdf --- /dev/null +++ b/.changeset/fix-correlated-include-routes.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Preserve distinct correlated include routes when parent values affect child joins, ordering, projections, aggregates, or having clauses. From 813e3696bba80447de70e93b32d77ab704389784 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 22 Aug 2026 15:59:29 -0600 Subject: [PATCH 3/8] fix: thread correlated routes through query boundaries --- .changeset/fix-correlated-include-routes.md | 2 +- packages/db/package.json | 2 +- packages/db/src/query/builder/index.ts | 89 +++- packages/db/src/query/compiler/group-by.ts | 17 +- packages/db/src/query/compiler/index.ts | 328 ++++++++++----- packages/db/src/query/compiler/joins.ts | 136 +++++- packages/db/src/query/live/ARCHITECTURE.md | 33 ++ .../includes-context-transport-oracle.test.ts | 387 ++++++++++++++++++ 8 files changed, 869 insertions(+), 125 deletions(-) create mode 100644 packages/db/tests/query/includes-context-transport-oracle.test.ts diff --git a/.changeset/fix-correlated-include-routes.md b/.changeset/fix-correlated-include-routes.md index 69e0c14fdf..11f171fc16 100644 --- a/.changeset/fix-correlated-include-routes.md +++ b/.changeset/fix-correlated-include-routes.md @@ -2,4 +2,4 @@ '@tanstack/db': patch --- -Preserve distinct correlated include routes when parent values affect child joins, ordering, projections, aggregates, or having clauses. +Preserve parent-specific include results through nested includes, subqueries, unions, joins, ordering, projections, aggregates, and having clauses. diff --git a/packages/db/package.json b/packages/db/package.json index ebed26a22b..03e905075f 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts" + "test:oracles": "vitest --run tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/query/builder/index.ts b/packages/db/src/query/builder/index.ts index 0d5d6862bf..5b262ab85f 100644 --- a/packages/db/src/query/builder/index.ts +++ b/packages/db/src/query/builder/index.ts @@ -1146,12 +1146,74 @@ function collectRefsFromSelectValue(value: unknown): Array { ] } if (value instanceof IncludesSubquery) { - return [value.correlationField, ...(value.parentProjection ?? [])] + return [ + value.correlationField, + ...(value.parentProjection ?? []), + ...collectExternalRefsFromQuery(value.query), + ] } if (!isPlainObject(value)) return [] return Object.values(value).flatMap(collectRefsFromSelectValue) } +function collectExternalRefsFromQuery(query: QueryIR): Array { + const localAliases = new Set(collectQueryAliases(query)) + const refs: Array = [] + const addExpression = (expression: BasicExpression | Aggregate) => { + refs.push(...collectRefsFromExpression(expression)) + } + const addWhere = (where: Where) => { + addExpression( + typeof where === `object` && `expression` in where + ? where.expression + : where, + ) + } + + for (const where of query.where ?? []) addWhere(where) + for (const join of query.join ?? []) { + addExpression(join.left) + addExpression(join.right) + if (join.from.type === `queryRef`) { + refs.push(...collectExternalRefsFromQuery(join.from.query)) + } + } + for (const expression of query.groupBy ?? []) addExpression(expression) + for (const having of query.having ?? []) addWhere(having) + for (const { expression } of query.orderBy ?? []) addExpression(expression) + if (query.select) refs.push(...collectRefsFromSelectValue(query.select)) + + if (query.from.type === `queryRef`) { + refs.push(...collectExternalRefsFromQuery(query.from.query)) + } else if (query.from.type === `unionFrom`) { + for (const source of query.from.sources) { + if (source.type === `queryRef`) { + refs.push(...collectExternalRefsFromQuery(source.query)) + } + } + } else if (query.from.type === `unionAll`) { + for (const branch of query.from.queries) { + refs.push(...collectExternalRefsFromQuery(branch)) + } + } + + const seen = new Set() + return refs.filter((ref) => { + const alias = ref.path.length > 1 ? ref.path[0] : undefined + const path = ref.path.join(`.`) + if ( + alias == null || + alias === `$selected` || + localAliases.has(alias) || + seen.has(path) + ) { + return false + } + seen.add(path) + return true + }) +} + function collectParentRefsFromQuery( query: QueryIR, parentAliases: Array, @@ -1179,7 +1241,9 @@ function collectParentRefsFromQuery( for (const expression of query.groupBy ?? []) addExpression(expression) for (const having of query.having ?? []) addWhere(having) for (const { expression } of query.orderBy ?? []) addExpression(expression) - if (query.select) refs.push(...collectRefsFromSelectValue(query.select)) + if (query.select) { + refs.push(...collectRefsFromSelectValue(query.select)) + } if (query.from.type === `queryRef`) { refs.push(...collectParentRefsFromQuery(query.from.query, parentAliases)) @@ -1210,6 +1274,16 @@ function collectParentRefsFromQuery( }) } +function collectExternalParentAliases(query: QueryIR): Array { + return [ + ...new Set( + collectExternalRefsFromQuery(query) + .map((ref) => ref.path[0]) + .filter((alias): alias is string => alias !== undefined), + ), + ] +} + /** * Checks whether a WHERE clause references any parent alias. */ @@ -1238,6 +1312,9 @@ function buildIncludesSubquery( // Collect child's own aliases const childAliases = collectQueryAliases(childQuery) + const visibleParentAliases = [ + ...new Set([...parentAliases, ...collectExternalParentAliases(childQuery)]), + ] // Walk child's WHERE clauses to find the correlation condition. // The correlation eq() may be a standalone WHERE or nested inside a top-level and(). @@ -1263,7 +1340,7 @@ function buildIncludesSubquery( const result = extractCorrelation( expr.args[0]!, expr.args[1]!, - parentAliases, + visibleParentAliases, childAliases, ) if (result) { @@ -1290,7 +1367,7 @@ function buildIncludesSubquery( const result = extractCorrelation( arg.args[0]!, arg.args[1]!, - parentAliases, + visibleParentAliases, childAliases, ) if (result) { @@ -1351,7 +1428,7 @@ function buildIncludesSubquery( const pureChildWhere: Array = [] const parentFilters: Array = [] for (const w of modifiedWhere) { - if (referencesParent(w, parentAliases)) { + if (referencesParent(w, visibleParentAliases)) { parentFilters.push(w) } else { pureChildWhere.push(w) @@ -1362,7 +1439,7 @@ function buildIncludesSubquery( // identity, not only the main equality key or residual filters. const projectedParentRefs = collectParentRefsFromQuery( { ...childQuery, where: modifiedWhere }, - parentAliases, + visibleParentAliases, ) const parentProjection = projectedParentRefs.length > 0 ? projectedParentRefs : undefined diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index e71d03a25f..751d1e577e 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -74,6 +74,19 @@ function getHavingEvaluationRow(row: Record): NamespacedRow { } } +function getWrappedAggregateEvaluationRow( + row: Record, + selected: Record, +): NamespacedRow { + const parentContext = row.__parentContext + return { + ...(parentContext !== null && typeof parentContext === `object` + ? (parentContext as NamespacedRow) + : {}), + $selected: selected, + } +} + function getRowVirtualMetadata(row: NamespacedRow): RowVirtualMetadata { let found = false let allSynced = true @@ -676,7 +689,9 @@ function evaluateWrappedAggregates( finalResults[`${GROUP_KEY_REF_PREFIX}${i}`] = aggregatedRow[`__key_${i}`] } for (const [alias, evaluator] of Object.entries(wrappedAggExprs)) { - finalResults[alias] = evaluator({ $selected: finalResults }) + finalResults[alias] = evaluator( + getWrappedAggregateEvaluationRow(aggregatedRow, finalResults), + ) } for (const key of Object.keys(finalResults)) { if (key.startsWith(`__agg_`) || key.startsWith(GROUP_KEY_REF_PREFIX)) { diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index d075539213..0f2ff39ad2 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -30,6 +30,7 @@ import { } from '../ir.js' import { ensureIndexForField } from '../../indexes/auto-index.js' import { deepEquals } from '../../utils.js' +import { normalizeValue } from '../../utils/comparison.js' import { compileExpression, isCaseWhenConditionTrue, @@ -68,6 +69,7 @@ export const INCLUDES_ROUTING = Symbol(`includesRouting`) export const INCLUDES_PUBLIC_KEY = Symbol(`includesPublicKey`) export const FN_SELECT_STATE = Symbol(`fnSelectState`) const SKIP_INCLUDE = Symbol(`skipInclude`) +const PARENT_ROUTE_CROSS_KEY = `__tanstack_parent_route_cross__` type ConditionalSelectGuard = { condition: BasicExpression @@ -84,6 +86,130 @@ type ProjectedSourceIncludePath = { guards: Array } +type CompiledParentProjection = { + alias: string + field: Array + compiled: (row: NamespacedRow) => unknown +} + +function projectParentContext( + nsRow: NamespacedRow, + projections: Array, +): Record> { + const inherited = (nsRow as any).__parentContext + const parentContext: Record> = inherited != + null && typeof inherited === `object` + ? { ...inherited } + : {} + + for (const projection of projections) { + const inheritedAlias = parentContext[projection.alias] + const aliasContext = + inheritedAlias != null && typeof inheritedAlias === `object` + ? { ...inheritedAlias } + : {} + parentContext[projection.alias] = aliasContext + + let target = aliasContext + for (let index = 0; index < projection.field.length - 1; index++) { + const segment = projection.field[index]! + const inheritedNested = target[segment] + const nested = + inheritedNested != null && typeof inheritedNested === `object` + ? { ...inheritedNested } + : {} + target[segment] = nested + target = nested + } + target[projection.field[projection.field.length - 1]!] = + projection.compiled(nsRow) + } + + return parentContext +} + +function parameterizeByParentRoutes( + pipeline: NamespacedAndKeyedStream, + parentKeyStream: KeyedStream, + mainSource: string, +): NamespacedAndKeyedStream { + const rows: any = pipeline.pipe( + map(([rowKey, row]) => [PARENT_ROUTE_CROSS_KEY, [rowKey, row]]), + ) + const routes: any = parentKeyStream.pipe( + map(([correlationKey, parentContext]) => [ + PARENT_ROUTE_CROSS_KEY, + [correlationKey, parentContext], + ]), + ) + + return rows.pipe( + joinOperator(routes, `inner`), + filter(([, [rowSide, routeSide]]: any) => + Boolean(rowSide != null && routeSide != null), + ), + map(([, [rowSide, routeSide]]: any) => { + const [rowKey, row] = rowSide! + const [correlationKey, parentContext] = routeSide! + const namespaced = { ...row } as Record + namespaced[mainSource] = { + ...namespaced[mainSource], + __correlationKey: correlationKey, + [INCLUDES_PUBLIC_KEY]: + namespaced[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? rowKey, + } + if (parentContext != null) Object.assign(namespaced, parentContext) + namespaced.__parentContext = parentContext + return [ + serializeValue([rowKey, correlationKey, parentContext]), + namespaced, + ] as [string, NamespacedRow] + }), + ) as NamespacedAndKeyedStream +} + +function getRowCorrelationKey(row: NamespacedRow, mainSource: string): unknown { + return ( + (row as any)[mainSource]?.__correlationKey ?? (row as any).__correlationKey + ) +} + +function correlationValuesEqual(left: unknown, right: unknown): boolean { + if (left == null || right == null) return false + const normalizedLeft = normalizeValue(left) + const normalizedRight = normalizeValue(right) + return ( + Object.is(normalizedLeft, normalizedRight) || + (typeof normalizedLeft === `number` && + typeof normalizedRight === `number` && + Number.isNaN(normalizedLeft) && + Number.isNaN(normalizedRight)) + ) +} + +function attachRouteMetadataToResult( + value: any, + correlationKey: unknown, + parentContext: unknown, + publicKey: unknown, +): any { + if ( + value == null || + typeof value !== `object` || + (correlationKey === undefined && + parentContext === undefined && + publicKey === undefined) + ) { + return value + } + return { + ...value, + __correlationKey: correlationKey, + __parentContext: parentContext, + [INCLUDES_PUBLIC_KEY]: publicKey, + } +} + /** * Result of compiling an includes subquery, including the child pipeline * and metadata needed to route child results to parent-scoped Collections. @@ -227,6 +353,7 @@ export function compileQuery( sourceIncludes, directIncludes, isUnionFrom, + isParentRouted, } = processFromClause( query.from, allInputs, @@ -241,6 +368,7 @@ export function compileQuery( aliasToCollectionId, aliasRemapping, sourceWhereClauses, + parentKeyStream, ) Object.assign(sources, fromSources) @@ -250,17 +378,13 @@ export function compileQuery( // so the child pipeline only processes rows that match parents. let pipeline: NamespacedAndKeyedStream = initialPipeline const childCorrelationAlias = childCorrelationField?.path[0] - const joinsParentAfterJoins = + const joinsParentDirectly = !isUnionFrom && + !isParentRouted && parentKeyStream !== undefined && childCorrelationField !== undefined && - childCorrelationAlias !== mainSource - if ( - !isUnionFrom && - parentKeyStream && - childCorrelationField && - !joinsParentAfterJoins - ) { + childCorrelationAlias === mainSource + if (parentKeyStream && childCorrelationField && joinsParentDirectly) { const mainInput = sources[mainSource]! let filteredMainInput = mainInput // Re-key child input by correlation field: [correlationValue, [childKey, childRow]] @@ -302,6 +426,14 @@ export function compileQuery( sources[mainSource] = filteredMainInput pipeline = wrapInputWithAlias(filteredMainInput, mainSource) + } else if (parentKeyStream && !isParentRouted) { + // QueryRefs, unions, and joined-source correlations need the route before + // source-local joins, filters, grouping, ordering, or windows run. + pipeline = parameterizeByParentRoutes( + initialPipeline, + parentKeyStream, + mainSource, + ) } // Process JOIN clauses if they exist @@ -326,41 +458,23 @@ export function compileQuery( aliasToCollectionId, aliasRemapping, sourceWhereClauses, - parentKeyStream !== undefined && !joinsParentAfterJoins, + parentKeyStream !== undefined, + parentKeyStream, ) } - // A correlation field owned by a joined source does not exist on the main - // input. Join the fully namespaced child relation with its parent routes - // here, after the source join has made that field available. - if (joinsParentAfterJoins) { + // A recursively compiled source or a correlation owned by a joined source + // is already parameterized by route. Once the correlation field is visible, + // retain only the copy whose route key matches it. + if (parentKeyStream && childCorrelationField && !joinsParentDirectly) { const compiledChildCorrelation = compileExpression(childCorrelationField) pipeline = pipeline.pipe( - map( - ([key, row]) => - [compiledChildCorrelation(row), [key, row]] as [ - unknown, - [unknown, typeof row], - ], + filter(([, row]) => + correlationValuesEqual( + compiledChildCorrelation(row), + getRowCorrelationKey(row, mainSource), + ), ), - joinOperator(parentKeyStream, `inner`), - filter(([_correlationValue, [childSide]]) => childSide != null), - map(([correlationValue, [childSide, parentSide]]) => { - const [childKey, row] = childSide as [unknown, NamespacedRow] - const namespaced = { ...row } as Record - namespaced[mainSource] = { - ...namespaced[mainSource], - __correlationKey: correlationValue, - [INCLUDES_PUBLIC_KEY]: childKey, - } - if (parentSide != null) { - Object.assign(namespaced, parentSide) - namespaced.__parentContext = parentSide - } - const effectiveKey = - parentSide != null ? serializeValue([childKey, parentSide]) : childKey - return [effectiveKey, namespaced] - }), ) as NamespacedAndKeyedStream } @@ -517,32 +631,21 @@ export function compileQuery( })) let parentKeys: any if (subquery.parentProjection && subquery.parentProjection.length > 0) { - const compiledProjections = subquery.parentProjection.map((ref) => ({ - alias: ref.path[0]!, - field: ref.path.slice(1), - compiled: compileExpression(ref), - })) + const compiledProjections: Array = + subquery.parentProjection.map((ref) => ({ + alias: ref.path[0]!, + field: ref.path.slice(1), + compiled: compileExpression(ref), + })) parentKeys = pipeline.pipe( map(([_key, nsRow]: any) => { if (!matchesConditionalSelectGuards(compiledGuards, nsRow)) { return [SKIP_INCLUDE, null] as any } - const parentContext: Record> = {} - for (const proj of compiledProjections) { - if (!parentContext[proj.alias]) { - parentContext[proj.alias] = {} - } - const value = proj.compiled(nsRow) - // Set nested field in the alias namespace - let target = parentContext[proj.alias]! - for (let i = 0; i < proj.field.length - 1; i++) { - if (!target[proj.field[i]!]) { - target[proj.field[i]!] = {} - } - target = target[proj.field[i]!] - } - target[proj.field[proj.field.length - 1]!] = value - } + const parentContext = projectParentContext( + nsRow, + compiledProjections, + ) return [compiledCorrelation(nsRow), parentContext] as any }), ) @@ -697,11 +800,12 @@ export function compileQuery( // Capture routing function for INCLUDES_ROUTING tagging if (subquery.parentProjection && subquery.parentProjection.length > 0) { - const compiledProjs = subquery.parentProjection.map((ref) => ({ - alias: ref.path[0]!, - field: ref.path.slice(1), - compiled: compileExpression(ref), - })) + const compiledProjs: Array = + subquery.parentProjection.map((ref) => ({ + alias: ref.path[0]!, + field: ref.path.slice(1), + compiled: compileExpression(ref), + })) const compiledCorr = compiledCorrelation const compiledRoutingGuards = compiledGuards includesRoutingFns.push({ @@ -714,21 +818,7 @@ export function compileQuery( parentContext: null, } } - const parentContext: Record> = {} - for (const proj of compiledProjs) { - if (!parentContext[proj.alias]) { - parentContext[proj.alias] = {} - } - const value = proj.compiled(nsRow) - let target = parentContext[proj.alias]! - for (let i = 0; i < proj.field.length - 1; i++) { - if (!target[proj.field[i]!]) { - target[proj.field[i]!] = {} - } - target = target[proj.field[i]!] - } - target[proj.field[proj.field.length - 1]!] = value - } + const parentContext = projectParentContext(nsRow, compiledProjs) return { active: true, correlationKey: compiledCorr(nsRow), @@ -1259,6 +1349,7 @@ function processFromClause( aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, + parentKeyStream?: KeyedStream, ): { alias: string pipeline: NamespacedAndKeyedStream @@ -1267,6 +1358,7 @@ function processFromClause( sourceIncludes: Array directIncludes: Array isUnionFrom: boolean + isParentRouted: boolean } { if (from.type === `unionAll`) { return processUnionAll( @@ -1283,25 +1375,28 @@ function processFromClause( aliasToCollectionId, aliasRemapping, sourceWhereClauses, + parentKeyStream, ) } if (from.type !== `unionFrom`) { - const { alias, input, collectionId, sourceIncludes } = processFrom( - from, - allInputs, - collections, - subscriptions, - callbacks, - lazySources, - optimizableOrderByCollections, - setWindowFn, - cache, - queryMapping, - aliasToCollectionId, - aliasRemapping, - sourceWhereClauses, - ) + const { alias, input, collectionId, sourceIncludes, isParentRouted } = + processFrom( + from, + allInputs, + collections, + subscriptions, + callbacks, + lazySources, + optimizableOrderByCollections, + setWindowFn, + cache, + queryMapping, + aliasToCollectionId, + aliasRemapping, + sourceWhereClauses, + parentKeyStream, + ) return { alias, @@ -1311,6 +1406,7 @@ function processFromClause( sourceIncludes, directIncludes: [], isUnionFrom: false, + isParentRouted, } } @@ -1330,6 +1426,7 @@ function processFromClause( input, collectionId, sourceIncludes: childSourceIncludes, + isParentRouted, } = processFrom( source, allInputs, @@ -1344,6 +1441,7 @@ function processFromClause( aliasToCollectionId, aliasRemapping, sourceWhereClauses, + parentKeyStream, ) if (!mainAlias) { @@ -1353,7 +1451,15 @@ function processFromClause( sources[alias] = input sourceIncludes.push(...childSourceIncludes) - const branch = wrapInputWithAlias(input, alias).pipe( + const routedBranch = + parentKeyStream && !isParentRouted + ? parameterizeByParentRoutes( + wrapInputWithAlias(input, alias), + parentKeyStream, + alias, + ) + : wrapInputWithAlias(input, alias) + const branch = routedBranch.pipe( map(([key, row]) => { return [`${alias}:${encodeKeyForUnionBranch(key)}`, row] as [ string, @@ -1373,6 +1479,7 @@ function processFromClause( sourceIncludes, directIncludes: [], isUnionFrom: true, + isParentRouted: parentKeyStream !== undefined, } } @@ -1390,6 +1497,7 @@ function processUnionAll( aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, + parentKeyStream?: KeyedStream, ): { alias: string pipeline: NamespacedAndKeyedStream @@ -1398,6 +1506,7 @@ function processUnionAll( sourceIncludes: Array directIncludes: Array isUnionFrom: boolean + isParentRouted: boolean } { if (from.queries.length === 0) { throw new UnsupportedFromTypeError(`empty unionAll`) @@ -1432,6 +1541,7 @@ function processUnionAll( setWindowFn, cache, queryMapping, + parentKeyStream, ) if (!mainCollectionId) { @@ -1446,8 +1556,16 @@ function processUnionAll( } const branchPipeline = branchResult.pipeline.pipe( - map(([key, [row]]) => { - return [`${index}:${encodeKeyForUnionBranch(key)}`, row] as [ + map((data: any) => { + const [key, [row, _order, correlationKey, parentContext, publicKey]] = + data + const routedRow = attachRouteMetadataToResult( + row, + correlationKey, + parentContext, + publicKey, + ) + return [`${index}:${encodeKeyForUnionBranch(key)}`, routedRow] as [ string, Record, ] @@ -1467,6 +1585,7 @@ function processUnionAll( sourceIncludes, directIncludes, isUnionFrom: true, + isParentRouted: parentKeyStream !== undefined, } } @@ -1517,11 +1636,13 @@ function processFrom( aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, + parentKeyStream?: KeyedStream, ): { alias: string input: KeyedStream collectionId: string sourceIncludes: Array + isParentRouted: boolean } { switch (from.type) { case `collectionRef`: { @@ -1539,6 +1660,7 @@ function processFrom( input, collectionId: from.collection.id, sourceIncludes: [], + isParentRouted: false, } } case `queryRef`: { @@ -1557,6 +1679,7 @@ function processFrom( setWindowFn, cache, queryMapping, + parentKeyStream, ) // Pull up alias mappings from subquery to parent scope. @@ -1615,9 +1738,17 @@ function processFrom( // We need to extract just the value for use in parent queries const extractedInput = subQueryInput.pipe( map((data: any) => { - const [key, [value, _orderByIndex]] = data + const [ + key, + [value, _orderByIndex, correlationKey, parentContext, publicKey], + ] = data // Unwrap Value expressions that might have leaked through as the entire row - const unwrapped = unwrapValue(value) + const unwrapped = attachRouteMetadataToResult( + unwrapValue(value), + correlationKey, + parentContext, + publicKey, + ) return [key, unwrapped] as [unknown, any] }), ) @@ -1631,6 +1762,7 @@ function processFrom( sourceAlias: from.alias, include, })) ?? [], + isParentRouted: parentKeyStream !== undefined, } } default: diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 6d0bee863e..d389b2fb2d 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -53,6 +53,63 @@ export type LazyCollectionCallbacks = { } let nextLazyDemandPlanId = 0 +const PARENT_ROUTE_CROSS_KEY = `__tanstack_join_parent_route_cross__` + +function parameterizeJoinInputByParentRoutes( + input: KeyedStream, + parentKeyStream: KeyedStream, +): KeyedStream { + const rows: any = input.pipe( + map(([rowKey, row]) => [PARENT_ROUTE_CROSS_KEY, [rowKey, row]]), + ) + const routes: any = parentKeyStream.pipe( + map(([correlationKey, parentContext]) => [ + PARENT_ROUTE_CROSS_KEY, + [correlationKey, parentContext], + ]), + ) + + return rows.pipe( + joinOperator(routes, `inner`), + filter(([, [rowSide, routeSide]]: any) => + Boolean(rowSide != null && routeSide != null), + ), + map(([, [rowSide, routeSide]]: any) => { + const [rowKey, row] = rowSide + const [correlationKey, parentContext] = routeSide + return [ + serializeValue([rowKey, correlationKey, parentContext]), + { + ...row, + __correlationKey: correlationKey, + __parentContext: parentContext, + }, + ] + }), + ) as KeyedStream +} + +function wrapJoinedInputRow(alias: string, row: any): NamespacedRow { + const { __parentContext, ...cleanRow } = row + const namespaced: NamespacedRow = { [alias]: cleanRow } + if (__parentContext != null) { + Object.assign(namespaced, __parentContext) + namespaced.__parentContext = __parentContext + } + return namespaced +} + +function getRouteJoinKey( + row: NamespacedRow, + source: string, + value: unknown, +): string { + return serializeValue([ + row[source]?.__correlationKey ?? row.__correlationKey, + row.__parentContext ?? row[source]?.__parentContext ?? null, + value, + ]) +} export function registerLazyDemandPlan( callbacks: Record, @@ -95,6 +152,7 @@ export function processJoins( aliasRemapping: Record, sourceWhereClauses: Map>, mainSourceIsParentFiltered: boolean, + parentKeyStream?: KeyedStream, ): NamespacedAndKeyedStream { let resultPipeline = pipeline @@ -120,6 +178,7 @@ export function processJoins( aliasRemapping, sourceWhereClauses, mainSourceIsParentFiltered, + parentKeyStream, ) } @@ -151,12 +210,30 @@ function processJoin( aliasRemapping: Record, sourceWhereClauses: Map>, mainSourceIsParentFiltered: boolean, + parentKeyStream?: KeyedStream, ): NamespacedAndKeyedStream { const isCollectionRef = joinClause.from.type === `collectionRef` + const joinedSource = joinClause.from.alias + const availableSources = [...Object.keys(sources), joinedSource] + const { mainExpr, joinedExpr } = analyzeJoinExpressions( + joinClause.left, + joinClause.right, + availableSources, + joinedSource, + rawQuery.from.type === `unionAll`, + ) + const joinedExpressionAliases = getSourceAliasesFromExpression(joinedExpr) + const joinedExpressionUsesParent = [...joinedExpressionAliases].some( + (alias) => alias !== joinedSource && !sources[alias], + ) + const routeJoinedSource = + parentKeyStream !== undefined && + (joinClause.from.type === `queryRef` || joinedExpressionUsesParent) + // Get the joined source alias and input stream const { - alias: joinedSource, + alias: processedJoinedSource, input: joinedInput, collectionId: joinedCollectionId, } = processJoinSource( @@ -174,8 +251,13 @@ function processJoin( aliasToCollectionId, aliasRemapping, sourceWhereClauses, + routeJoinedSource ? parentKeyStream : undefined, ) + if (processedJoinedSource !== joinedSource) { + throw new InvalidJoinConditionSourceMismatchError() + } + // Add the joined source to the sources map sources[joinedSource] = joinedInput if (isCollectionRef) { @@ -195,22 +277,16 @@ function processJoin( throw new JoinCollectionNotFoundError(joinedCollectionId) } - const { activeSource, lazySource } = getActiveAndLazySources( + const sourceActivity = getActiveAndLazySources( joinClause.type, mainCollection, joinedCollection, mainSourceIsParentFiltered, ) - - // Analyze which source each expression refers to and swap if necessary - const availableSources = Object.keys(sources) - const { mainExpr, joinedExpr } = analyzeJoinExpressions( - joinClause.left, - joinClause.right, - availableSources, - joinedSource, - rawQuery.from.type === `unionAll`, - ) + const activeSource = routeJoinedSource + ? undefined + : sourceActivity.activeSource + const lazySource = sourceActivity.lazySource // Pre-compile the join expressions const compiledMainExpr = compileExpression(mainExpr) @@ -220,7 +296,10 @@ function processJoin( let mainPipeline = pipeline.pipe( map(([currentKey, namespacedRow]) => { // Extract the join key from the main source expression - const mainKey = normalizeValue(compiledMainExpr(namespacedRow)) + const value = normalizeValue(compiledMainExpr(namespacedRow)) + const mainKey = routeJoinedSource + ? getRouteJoinKey(namespacedRow, mainSource, value) + : value // Return [joinKey, [originalKey, namespacedRow]] return [mainKey, [currentKey, namespacedRow]] as [ @@ -234,10 +313,13 @@ function processJoin( let joinedPipeline = joinedInput.pipe( map(([currentKey, row]) => { // Wrap the row in a namespaced structure - const namespacedRow: NamespacedRow = { [joinedSource]: row } + const namespacedRow = wrapJoinedInputRow(joinedSource, row) // Extract the join key from the joined source expression - const joinedKey = normalizeValue(compiledJoinedExpr(namespacedRow)) + const value = normalizeValue(compiledJoinedExpr(namespacedRow)) + const joinedKey = routeJoinedSource + ? getRouteJoinKey(namespacedRow, joinedSource, value) + : value // Return [joinKey, [originalKey, namespacedRow]] return [joinedKey, [currentKey, namespacedRow]] as [ @@ -472,6 +554,7 @@ function processJoinSource( aliasToCollectionId: Record, aliasRemapping: Record, sourceWhereClauses: Map>, + parentKeyStream?: KeyedStream, ): { alias: string; input: KeyedStream; collectionId: string } { switch (from.type) { case `collectionRef`: { @@ -484,7 +567,13 @@ function processJoinSource( ) } aliasToCollectionId[from.alias] = from.collection.id - return { alias: from.alias, input, collectionId: from.collection.id } + return { + alias: from.alias, + input: parentKeyStream + ? parameterizeJoinInputByParentRoutes(input, parentKeyStream) + : input, + collectionId: from.collection.id, + } } case `queryRef`: { // Find the original query for caching purposes @@ -502,6 +591,7 @@ function processJoinSource( setWindowFn, cache, queryMapping, + parentKeyStream, ) // Pull up alias mappings from subquery to parent scope. @@ -559,8 +649,18 @@ function processJoinSource( // We need to extract just the value for use in parent queries const extractedInput = subQueryInput.pipe( map((data: any) => { - const [key, [value, _orderByIndex]] = data - return [key, value] as [unknown, any] + const [key, [value, _orderByIndex, correlationKey, parentContext]] = + data + return [ + key, + value != null && typeof value === `object` + ? { + ...value, + __correlationKey: correlationKey, + __parentContext: parentContext, + } + : value, + ] as [unknown, any] }), ) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index f0a390ce6f..b84e75ff8b 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -132,6 +132,38 @@ Implementations use canonical values, interned handles, or nested maps; they do not reconstruct array or object keys and expect JavaScript `Map` identity to match. +### Route-context transport + +A parent reference is a lexical dependency, even when it appears below the +immediate child query. For every parent reference that the builder can inspect, +the compiler must: + +1. discover it across nested includes, `QueryRef` sources, union branches, and + joined sources; +2. include its evaluated value in the route identity; +3. attach that route context before the first operator that evaluates it; and +4. preserve it through each later recursive source, join, grouping, and + materialization edge. + +The third rule fixes the evaluation order. A parent-dependent filter, join key, +aggregate wrapper, order, or window must run once per parent route. It cannot +run on a shared child relation first and receive a route after the fact. + +The route-context grammar crosses these dimensions: + +```text +lexical dependency scope + x recursive source boundary (nested include, QueryRef, union) + x evaluation phase (filter, join, group, aggregate, order, window) + x join side and correlation attachment point + x materialization form + x parent or child update +``` + +Adding one dimension to the query language requires checking its product with +the others. A passing one-level filter case does not prove a nested aggregate, +joined subquery, or union branch transports the same context. + A materialization cell identifies one include field on one parent-row occurrence: @@ -529,6 +561,7 @@ create recursive Collection machinery. | Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | | Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | | Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | +| Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | | Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | | Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | diff --git a/packages/db/tests/query/includes-context-transport-oracle.test.ts b/packages/db/tests/query/includes-context-transport-oracle.test.ts new file mode 100644 index 0000000000..0d3b5b4e08 --- /dev/null +++ b/packages/db/tests/query/includes-context-transport-oracle.test.ts @@ -0,0 +1,387 @@ +import { describe, expect, test } from 'vitest' +import { + add, + count, + createLiveQueryCollection, + eq, + lt, + lte, + materialize, + multiply, + toArray, +} from '../../src/query/index.js' +import { createControlledCollection } from './includes-oracle-helpers.js' + +type Cleanable = { cleanup: () => Promise } + +async function cleanup( + live: Cleanable, + sources: Array<{ collection: Cleanable }>, +): Promise { + await live.cleanup() + await Promise.all(sources.map(({ collection }) => collection.cleanup())) +} + +function ids(rows: Iterable<{ id: number }>): Array { + return [...rows].map((row) => row.id) +} + +describe(`correlated include route-context transport oracle`, () => { + test(`keeps ancestor context through a nested include`, async () => { + const parents = createControlledCollection(`nested-context-parents`, [ + { id: 1, group: 1, threshold: 2 }, + { id: 2, group: 1, threshold: 4 }, + ]) + const children = createControlledCollection(`nested-context-children`, [ + { id: 10, parentGroup: 1, group: 10 }, + ]) + const grandchildren = createControlledCollection( + `nested-context-grandchildren`, + [ + { id: 100, parentGroup: 10, value: 1 }, + { id: 200, parentGroup: 10, value: 3 }, + ], + ) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const childRows = () => + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => { + const grandchildRows = q + .from({ grandchild: grandchildren.collection }) + .where(({ grandchild }) => + eq(grandchild.parentGroup, child.group), + ) + .where(({ grandchild }) => + lt(grandchild.value, parent.threshold), + ) + .orderBy(({ grandchild }) => grandchild.id) + .select(({ grandchild }) => ({ id: grandchild.id })) + + return { + id: child.id, + grandchildren: materialize(grandchildRows), + } + }) + + return { + id: parent.id, + facade: childRows(), + array: toArray(childRows()), + materialized: materialize(childRows()), + } + }), + ) + + const project = (parentId: number) => { + const row = live.get(parentId)! + const grandchildrenOf = ( + values: Iterable<{ grandchildren: Array<{ id: number }> }>, + ) => ids([...values][0]?.grandchildren ?? []) + return { + facade: grandchildrenOf(row.facade.values()), + array: grandchildrenOf(row.array), + materialized: grandchildrenOf(row.materialized), + } + } + + try { + await live.preload() + expect(project(1)).toEqual({ + facade: [100], + array: [100], + materialized: [100], + }) + expect(project(2)).toEqual({ + facade: [100, 200], + array: [100, 200], + materialized: [100, 200], + }) + + parents.write(`update`, { id: 1, group: 1, threshold: 4 }) + expect(project(1)).toEqual({ + facade: [100, 200], + array: [100, 200], + materialized: [100, 200], + }) + } finally { + await cleanup(live, [parents, children, grandchildren]) + } + }) + + test(`evaluates wrapped aggregates with parent context`, async () => { + const parents = createControlledCollection(`wrapped-aggregate-parents`, [ + { id: 1, group: 1, factor: 0 }, + { id: 2, group: 1, factor: -6 }, + ]) + const children = createControlledCollection(`wrapped-aggregate-children`, [ + { id: 10, parentGroup: 1 }, + { id: 20, parentGroup: 1 }, + { id: 30, parentGroup: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const explicit = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .groupBy(({ child }) => child.parentGroup) + .select(({ child }) => ({ + id: child.parentGroup, + score: add(count(child.id), parent.factor), + })) + const implicit = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => ({ + score: add(count(child.id), parent.factor), + })) + return { + id: parent.id, + explicit: materialize(explicit), + implicit: materialize(implicit), + } + }), + ) + + const project = () => + live.toArray.map((row) => ({ + id: row.id, + explicit: row.explicit.map(({ score }) => score), + implicit: row.implicit.map(({ score }) => score), + })) + + try { + await live.preload() + expect(project()).toEqual([ + { id: 1, explicit: [3], implicit: [3] }, + { id: 2, explicit: [-3], implicit: [-3] }, + ]) + + parents.write(`update`, { id: 2, group: 1, factor: 1 }) + expect(project()[1]).toEqual({ id: 2, explicit: [4], implicit: [4] }) + } finally { + await cleanup(live, [parents, children]) + } + }) + + test(`parameterizes a FROM QueryRef before its order and limit`, async () => { + const parents = createControlledCollection(`from-queryref-parents`, [ + { id: 1, group: 1, direction: 1 }, + { id: 2, group: 1, direction: -1 }, + ]) + const children = createControlledCollection(`from-queryref-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const ordered = q + .from({ child: children.collection }) + .orderBy(({ child }) => multiply(child.value, parent.direction)) + .limit(1) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + })) + const routed = q + .from({ chosen: ordered }) + .where(({ chosen }) => eq(chosen.parentGroup, parent.group)) + .select(({ chosen }) => ({ id: chosen.id })) + return { id: parent.id, children: materialize(routed) } + }), + ) + + try { + await live.preload() + expect(live.toArray.map((row) => ids(row.children))).toEqual([[10], [20]]) + } finally { + await cleanup(live, [parents, children]) + } + }) + + test(`parameterizes a joined QueryRef before its filter`, async () => { + const parents = createControlledCollection(`joined-queryref-parents`, [ + { id: 1, group: 1, maximumTag: 1 }, + { id: 2, group: 1, maximumTag: 2 }, + ]) + const children = createControlledCollection(`joined-queryref-children`, [ + { id: 10, parentGroup: 1, tagId: 1 }, + { id: 20, parentGroup: 1, tagId: 2 }, + ]) + const tags = createControlledCollection(`joined-queryref-tags`, [ + { id: 1 }, + { id: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const allowedTags = q + .from({ tag: tags.collection }) + .where(({ tag }) => lte(tag.id, parent.maximumTag)) + .select(({ tag }) => ({ id: tag.id })) + const childRows = q + .from({ child: children.collection }) + .innerJoin({ tag: allowedTags }, ({ child, tag }) => + eq(child.tagId, tag.id), + ) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ id: child.id })) + return { id: parent.id, children: materialize(childRows) } + }), + ) + + try { + await live.preload() + expect(live.toArray.map((row) => ids(row.children))).toEqual([ + [10], + [10, 20], + ]) + } finally { + await cleanup(live, [parents, children, tags]) + } + }) + + test(`keeps parent context inside joined union branches`, async () => { + const parents = createControlledCollection(`joined-union-parents`, [ + { id: 1, group: 1, maximumTag: 1 }, + { id: 2, group: 1, maximumTag: 2 }, + ]) + const children = createControlledCollection(`joined-union-children`, [ + { id: 10, parentGroup: 1, tagId: 1 }, + { id: 20, parentGroup: 1, tagId: 2 }, + ]) + const oddTags = createControlledCollection(`joined-union-odd-tags`, [ + { id: 1 }, + ]) + const evenTags = createControlledCollection(`joined-union-even-tags`, [ + { id: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const oddBranch = q + .from({ oddTag: oddTags.collection }) + .where(({ oddTag }) => lte(oddTag.id, parent.maximumTag)) + .select(({ oddTag }) => ({ id: oddTag.id })) + const evenBranch = q + .from({ evenTag: evenTags.collection }) + .where(({ evenTag }) => lte(evenTag.id, parent.maximumTag)) + .select(({ evenTag }) => ({ id: evenTag.id })) + const allowedTags = q.unionAll(oddBranch, evenBranch) + const childRows = q + .from({ child: children.collection }) + .innerJoin({ tag: allowedTags }, ({ child, tag }) => + eq(child.tagId, tag.id), + ) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ id: child.id })) + return { id: parent.id, children: materialize(childRows) } + }), + ) + + try { + await live.preload() + expect(live.toArray.map((row) => ids(row.children))).toEqual([ + [10], + [10, 20], + ]) + } finally { + await cleanup(live, [parents, children, oddTags, evenTags]) + } + }) + + test(`evaluates a joined-side key with its parent route`, async () => { + const parents = createControlledCollection(`joined-key-parents`, [ + { id: 1, group: 1, offset: 0 }, + { id: 2, group: 1, offset: 1 }, + ]) + const children = createControlledCollection(`joined-key-children`, [ + { id: 10, parentGroup: 1, tagId: 2 }, + ]) + const tags = createControlledCollection(`joined-key-tags`, [ + { id: 1, label: `one` }, + { id: 2, label: `two` }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const childRows = q + .from({ child: children.collection }) + .innerJoin({ tag: tags.collection }, ({ child, tag }) => + eq(add(tag.id, parent.offset), child.tagId), + ) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child, tag }) => ({ id: child.id, label: tag.label })) + return { id: parent.id, children: materialize(childRows) } + }), + ) + + try { + await live.preload() + expect(live.toArray.map((row) => row.children[0]?.label)).toEqual([ + `two`, + `one`, + ]) + } finally { + await cleanup(live, [parents, children, tags]) + } + }) + + test(`attaches routes before correlating through a joined source`, async () => { + const parents = createControlledCollection(`joined-correlation-parents`, [ + { id: 1, group: 1, offset: 1 }, + { id: 2, group: 2, offset: 0 }, + ]) + const children = createControlledCollection(`joined-correlation-children`, [ + { id: 10, tagId: 2 }, + ]) + const tags = createControlledCollection(`joined-correlation-tags`, [ + { id: 1, group: 1, label: `one` }, + { id: 2, group: 2, label: `two` }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const childRows = q + .from({ child: children.collection }) + .innerJoin({ tag: tags.collection }, ({ child, tag }) => + eq(add(tag.id, parent.offset), child.tagId), + ) + .where(({ tag }) => eq(tag.group, parent.group)) + .select(({ child, tag }) => ({ id: child.id, label: tag.label })) + return { id: parent.id, children: materialize(childRows) } + }), + ) + + try { + await live.preload() + expect(live.toArray.map((row) => row.children[0]?.label)).toEqual([ + `one`, + `two`, + ]) + } finally { + await cleanup(live, [parents, children, tags]) + } + }) +}) From 0bb0541abcb104bdc7f077bead730eda3dcffd38 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 22 Aug 2026 16:11:09 -0600 Subject: [PATCH 4/8] fix: preserve union route metadata --- packages/db/src/query/compiler/index.ts | 19 +++++++++++---- .../includes-context-transport-oracle.test.ts | 23 +++++++++++-------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 0f2ff39ad2..4de7ae04d1 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -159,6 +159,7 @@ function parameterizeByParentRoutes( namespaced[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? rowKey, } if (parentContext != null) Object.assign(namespaced, parentContext) + namespaced.__correlationKey = correlationKey namespaced.__parentContext = parentContext return [ serializeValue([rowKey, correlationKey, parentContext]), @@ -1461,10 +1462,20 @@ function processFromClause( : wrapInputWithAlias(input, alias) const branch = routedBranch.pipe( map(([key, row]) => { - return [`${alias}:${encodeKeyForUnionBranch(key)}`, row] as [ - string, - typeof row, - ] + const branchKey = `${alias}:${encodeKeyForUnionBranch(key)}` + const aliasRow = row[alias] as Record | undefined + const publicKey = aliasRow?.[INCLUDES_PUBLIC_KEY] ?? key + const branchPublicKey = `${alias}:${encodeKeyForUnionBranch(publicKey)}` + const branchRow = parentKeyStream + ? { + ...row, + [alias]: { + ...row[alias], + [INCLUDES_PUBLIC_KEY]: branchPublicKey, + }, + } + : row + return [branchKey, branchRow] as [string, typeof row] }), ) diff --git a/packages/db/tests/query/includes-context-transport-oracle.test.ts b/packages/db/tests/query/includes-context-transport-oracle.test.ts index 0d3b5b4e08..35a79f2c23 100644 --- a/packages/db/tests/query/includes-context-transport-oracle.test.ts +++ b/packages/db/tests/query/includes-context-transport-oracle.test.ts @@ -255,7 +255,7 @@ describe(`correlated include route-context transport oracle`, () => { } }) - test(`keeps parent context inside joined union branches`, async () => { + test(`keeps parent context inside union branches`, async () => { const parents = createControlledCollection(`joined-union-parents`, [ { id: 1, group: 1, maximumTag: 1 }, { id: 2, group: 1, maximumTag: 2 }, @@ -265,10 +265,10 @@ describe(`correlated include route-context transport oracle`, () => { { id: 20, parentGroup: 1, tagId: 2 }, ]) const oddTags = createControlledCollection(`joined-union-odd-tags`, [ - { id: 1 }, + { id: 1, parentGroup: 1 }, ]) const evenTags = createControlledCollection(`joined-union-even-tags`, [ - { id: 2 }, + { id: 2, parentGroup: 1 }, ]) const live = createLiveQueryCollection((q) => q @@ -278,16 +278,21 @@ describe(`correlated include route-context transport oracle`, () => { const oddBranch = q .from({ oddTag: oddTags.collection }) .where(({ oddTag }) => lte(oddTag.id, parent.maximumTag)) - .select(({ oddTag }) => ({ id: oddTag.id })) + .select(({ oddTag }) => ({ + id: oddTag.id, + parentGroup: oddTag.parentGroup, + })) const evenBranch = q .from({ evenTag: evenTags.collection }) .where(({ evenTag }) => lte(evenTag.id, parent.maximumTag)) - .select(({ evenTag }) => ({ id: evenTag.id })) - const allowedTags = q.unionAll(oddBranch, evenBranch) + .select(({ evenTag }) => ({ + id: evenTag.id, + parentGroup: evenTag.parentGroup, + })) const childRows = q - .from({ child: children.collection }) - .innerJoin({ tag: allowedTags }, ({ child, tag }) => - eq(child.tagId, tag.id), + .unionAll(oddBranch, evenBranch) + .innerJoin({ child: children.collection }, ({ id, child }) => + eq(id, child.tagId), ) .where(({ child }) => eq(child.parentGroup, parent.group)) .orderBy(({ child }) => child.id) From 2ff8c6e48246fd33200d7c68112961fd14f0add1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 22 Aug 2026 16:33:25 -0600 Subject: [PATCH 5/8] test: generate route context grammar --- packages/db/src/query/live/ARCHITECTURE.md | 12 + .../includes-context-transport-oracle.test.ts | 1155 ++++++++++++----- 2 files changed, 845 insertions(+), 322 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index b84e75ff8b..a79a798f13 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -164,6 +164,18 @@ Adding one dimension to the query language requires checking its product with the others. A passing one-level filter case does not prove a nested aggregate, joined subquery, or union branch transports the same context. +The executable oracle factors that product into valid compiler sub-grammars: + +- lexical scope, including nested outer and inner materialization forms; +- grouping mode by aggregate-expression placement; +- recursive source boundary by evaluation phase; and +- join-key side by correlation attachment point. + +Every valid plan is checked as a Collection, `toArray`, and `materialize` +include at initial load, after a parent-route update, and after a child update. +The grammar declarations generate the cases; individual reported defects do +not get one-off tests outside that product. + A materialization cell identifies one include field on one parent-row occurrence: diff --git a/packages/db/tests/query/includes-context-transport-oracle.test.ts b/packages/db/tests/query/includes-context-transport-oracle.test.ts index 35a79f2c23..213bfa18f3 100644 --- a/packages/db/tests/query/includes-context-transport-oracle.test.ts +++ b/packages/db/tests/query/includes-context-transport-oracle.test.ts @@ -4,15 +4,116 @@ import { count, createLiveQueryCollection, eq, + gt, lt, lte, materialize, multiply, + sum, toArray, } from '../../src/query/index.js' import { createControlledCollection } from './includes-oracle-helpers.js' +import type { Collection } from '../../src/collection/index.js' +import type { Context, QueryBuilder } from '../../src/query/builder/index.js' +import type { ControlledCollection } from './includes-oracle-helpers.js' type Cleanable = { cleanup: () => Promise } +type MaterializationForm = (typeof materializationForms)[number] + +type MaterializedForms = { + collection: { values: () => Iterable } + array: Iterable + materialized: Iterable +} + +const materializationForms = [`collection`, `array`, `materialized`] as const +const checkpoints = [`initial`, `parent-update`, `child-update`] as const + +const routeContextGrammar = { + lexicalScope: { + scopes: [`immediate-parent`, `lexical-ancestor`] as const, + }, + aggregation: { + groupings: [`implicit`, `explicit`] as const, + placements: [`inside-aggregate`, `wrapped-aggregate`] as const, + }, + recursiveSource: { + boundaries: [`from-query-ref`, `joined-query-ref`, `union-branch`] as const, + phases: [ + `filter`, + `projection`, + `aggregate`, + `having`, + `order-window`, + ] as const, + }, + join: { + keySides: [`main`, `joined`] as const, + correlationAttachments: [`main`, `joined`] as const, + }, +} as const + +type LexicalScopeCell = { + family: `lexical-scope` + scope: (typeof routeContextGrammar.lexicalScope.scopes)[number] +} + +type AggregationCell = { + family: `aggregation` + grouping: (typeof routeContextGrammar.aggregation.groupings)[number] + placement: (typeof routeContextGrammar.aggregation.placements)[number] +} + +type RecursiveSourceCell = { + family: `recursive-source` + boundary: (typeof routeContextGrammar.recursiveSource.boundaries)[number] + phase: (typeof routeContextGrammar.recursiveSource.phases)[number] +} + +type JoinCell = { + family: `join` + keySide: (typeof routeContextGrammar.join.keySides)[number] + correlationAttachment: (typeof routeContextGrammar.join.correlationAttachments)[number] +} + +type GrammarCell = + | LexicalScopeCell + | AggregationCell + | RecursiveSourceCell + | JoinCell + +const grammarCells: Array = [ + ...routeContextGrammar.lexicalScope.scopes.map( + (scope): LexicalScopeCell => ({ family: `lexical-scope`, scope }), + ), + ...routeContextGrammar.aggregation.groupings.flatMap((grouping) => + routeContextGrammar.aggregation.placements.map( + (placement): AggregationCell => ({ + family: `aggregation`, + grouping, + placement, + }), + ), + ), + ...routeContextGrammar.recursiveSource.boundaries.flatMap((boundary) => + routeContextGrammar.recursiveSource.phases.map( + (phase): RecursiveSourceCell => ({ + family: `recursive-source`, + boundary, + phase, + }), + ), + ), + ...routeContextGrammar.join.keySides.flatMap((keySide) => + routeContextGrammar.join.correlationAttachments.map( + (correlationAttachment): JoinCell => ({ + family: `join`, + keySide, + correlationAttachment, + }), + ), + ), +] async function cleanup( live: Cleanable, @@ -22,371 +123,781 @@ async function cleanup( await Promise.all(sources.map(({ collection }) => collection.cleanup())) } +function createGrammarCollection( + name: string, + rows: ReadonlyArray, +): ControlledCollection { + return createControlledCollection(name, rows, { autoIndex: `eager` }) +} + +function includeInEveryForm( + query: QueryBuilder, +) { + return { + collection: query, + array: toArray(query), + materialized: materialize(query), + } +} + +function readEveryForm( + forms: MaterializedForms, + project: (rows: Iterable) => U, +): Record { + return { + collection: project(forms.collection.values()), + array: project(forms.array), + materialized: project(forms.materialized), + } +} + +function expectEveryForm( + forms: MaterializedForms, + project: (rows: Iterable) => U, + expected: U, +): void { + expect(readEveryForm(forms, project)).toEqual( + Object.fromEntries(materializationForms.map((form) => [form, expected])), + ) +} + function ids(rows: Iterable<{ id: number }>): Array { return [...rows].map((row) => row.id) } -describe(`correlated include route-context transport oracle`, () => { - test(`keeps ancestor context through a nested include`, async () => { - const parents = createControlledCollection(`nested-context-parents`, [ - { id: 1, group: 1, threshold: 2 }, - { id: 2, group: 1, threshold: 4 }, - ]) - const children = createControlledCollection(`nested-context-children`, [ - { id: 10, parentGroup: 1, group: 10 }, - ]) - const grandchildren = createControlledCollection( - `nested-context-grandchildren`, - [ - { id: 100, parentGroup: 10, value: 1 }, - { id: 200, parentGroup: 10, value: 3 }, - ], - ) +function grammarCellName(cell: GrammarCell): string { + switch (cell.family) { + case `lexical-scope`: + return `${cell.family} / ${cell.scope}` + case `aggregation`: + return `${cell.family} / ${cell.grouping} / ${cell.placement}` + case `recursive-source`: + return `${cell.family} / ${cell.boundary} / ${cell.phase}` + case `join`: + return `${cell.family} / ${cell.keySide}-side key / ${cell.correlationAttachment}-side correlation` + } +} + +async function runLexicalScopeCell({ scope }: LexicalScopeCell): Promise { + const parents = createGrammarCollection(`scope-${scope}-parents`, [ + { id: 1, group: 1, threshold: 2 }, + { id: 2, group: 1, threshold: 4 }, + ]) + const children = createGrammarCollection(`scope-${scope}-children`, [ + { id: 10, parentGroup: 1, group: 10, value: 1 }, + { id: 20, parentGroup: 1, group: 20, value: 3 }, + ]) + const grandchildren = createGrammarCollection( + `scope-${scope}-grandchildren`, + [ + { id: 100, parentGroup: 10, value: 1 }, + { id: 200, parentGroup: 10, value: 3 }, + ], + ) + + if (scope === `immediate-parent`) { const live = createLiveQueryCollection((q) => q .from({ parent: parents.collection }) .orderBy(({ parent }) => parent.id) .select(({ parent }) => { - const childRows = () => - q - .from({ child: children.collection }) - .where(({ child }) => eq(child.parentGroup, parent.group)) - .select(({ child }) => { - const grandchildRows = q - .from({ grandchild: grandchildren.collection }) - .where(({ grandchild }) => - eq(grandchild.parentGroup, child.group), - ) - .where(({ grandchild }) => - lt(grandchild.value, parent.threshold), - ) - .orderBy(({ grandchild }) => grandchild.id) - .select(({ grandchild }) => ({ id: grandchild.id })) - - return { - id: child.id, - grandchildren: materialize(grandchildRows), - } - }) - - return { - id: parent.id, - facade: childRows(), - array: toArray(childRows()), - materialized: materialize(childRows()), - } + const childRows = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .where(({ child }) => lt(child.value, parent.threshold)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ id: child.id })) + return { id: parent.id, ...includeInEveryForm(childRows) } }), ) - const project = (parentId: number) => { - const row = live.get(parentId)! - const grandchildrenOf = ( - values: Iterable<{ grandchildren: Array<{ id: number }> }>, - ) => ids([...values][0]?.grandchildren ?? []) - return { - facade: grandchildrenOf(row.facade.values()), - array: grandchildrenOf(row.array), - materialized: grandchildrenOf(row.materialized), - } + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + return children.collection.toArray + .filter( + (child) => + child.parentGroup === parent.group && + child.value < parent.threshold, + ) + .map(({ id }) => id) + .sort((left, right) => left - right) } try { await live.preload() - expect(project(1)).toEqual({ - facade: [100], - array: [100], - materialized: [100], - }) - expect(project(2)).toEqual({ - facade: [100, 200], - array: [100, 200], - materialized: [100, 200], - }) + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, ids, expected(parent.id)) + } parents.write(`update`, { id: 1, group: 1, threshold: 4 }) - expect(project(1)).toEqual({ - facade: [100, 200], - array: [100, 200], - materialized: [100, 200], + expectEveryForm(live.get(1)!, ids, expected(1)) + + children.write(`insert`, { + id: 30, + parentGroup: 1, + group: 30, + value: 2, }) + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, ids, expected(parent.id)) + } } finally { await cleanup(live, [parents, children, grandchildren]) } - }) + return + } - test(`evaluates wrapped aggregates with parent context`, async () => { - const parents = createControlledCollection(`wrapped-aggregate-parents`, [ - { id: 1, group: 1, factor: 0 }, - { id: 2, group: 1, factor: -6 }, - ]) - const children = createControlledCollection(`wrapped-aggregate-children`, [ - { id: 10, parentGroup: 1 }, - { id: 20, parentGroup: 1 }, - { id: 30, parentGroup: 1 }, - ]) - const live = createLiveQueryCollection((q) => - q - .from({ parent: parents.collection }) - .orderBy(({ parent }) => parent.id) - .select(({ parent }) => { - const explicit = q - .from({ child: children.collection }) - .where(({ child }) => eq(child.parentGroup, parent.group)) - .groupBy(({ child }) => child.parentGroup) - .select(({ child }) => ({ - id: child.parentGroup, - score: add(count(child.id), parent.factor), - })) - const implicit = q - .from({ child: children.collection }) - .where(({ child }) => eq(child.parentGroup, parent.group)) - .select(({ child }) => ({ - score: add(count(child.id), parent.factor), - })) - return { - id: parent.id, - explicit: materialize(explicit), - implicit: materialize(implicit), - } - }), - ) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const childRows = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => { + const grandchildRows = q + .from({ grandchild: grandchildren.collection }) + .where(({ grandchild }) => + eq(grandchild.parentGroup, child.group), + ) + .where(({ grandchild }) => lt(grandchild.value, parent.threshold)) + .orderBy(({ grandchild }) => grandchild.id) + .select(({ grandchild }) => ({ id: grandchild.id })) - const project = () => - live.toArray.map((row) => ({ - id: row.id, - explicit: row.explicit.map(({ score }) => score), - implicit: row.implicit.map(({ score }) => score), - })) + return { + id: child.id, + ...includeInEveryForm(grandchildRows), + } + }) - try { - await live.preload() - expect(project()).toEqual([ - { id: 1, explicit: [3], implicit: [3] }, - { id: 2, explicit: [-3], implicit: [-3] }, - ]) + return { id: parent.id, ...includeInEveryForm(childRows) } + }), + ) - parents.write(`update`, { id: 2, group: 1, factor: 1 }) - expect(project()[1]).toEqual({ id: 2, explicit: [4], implicit: [4] }) - } finally { - await cleanup(live, [parents, children]) + const expected = (parentId: number, childGroup: number) => { + const parent = parents.collection.get(parentId)! + return grandchildren.collection.toArray + .filter( + (grandchild) => + grandchild.parentGroup === childGroup && + grandchild.value < parent.threshold, + ) + .map(({ id }) => id) + .sort((left, right) => left - right) + } + + const projectOuterForm = ( + parentId: number, + outerForm: MaterializationForm, + ) => { + const parentRow = live.get(parentId)! + const outerRows = + outerForm === `collection` + ? parentRow.collection.values() + : parentRow[outerForm] + return [...outerRows].map((child) => ({ + id: child.id, + grandchildren: readEveryForm(child, ids), + })) + } + + const assertNestedProduct = (parentId: number) => { + const expectedRows = children.collection.toArray + .filter( + (child) => + child.parentGroup === parents.collection.get(parentId)!.group, + ) + .map((child) => ({ + id: child.id, + grandchildren: Object.fromEntries( + materializationForms.map((form) => [ + form, + expected(parentId, child.group), + ]), + ), + })) + for (const outerForm of materializationForms) { + expect(projectOuterForm(parentId, outerForm)).toEqual(expectedRows) } - }) + } - test(`parameterizes a FROM QueryRef before its order and limit`, async () => { - const parents = createControlledCollection(`from-queryref-parents`, [ - { id: 1, group: 1, direction: 1 }, - { id: 2, group: 1, direction: -1 }, - ]) - const children = createControlledCollection(`from-queryref-children`, [ - { id: 10, parentGroup: 1, value: 1 }, - { id: 20, parentGroup: 1, value: 2 }, - ]) - const live = createLiveQueryCollection((q) => - q - .from({ parent: parents.collection }) - .orderBy(({ parent }) => parent.id) - .select(({ parent }) => { - const ordered = q - .from({ child: children.collection }) - .orderBy(({ child }) => multiply(child.value, parent.direction)) - .limit(1) - .select(({ child }) => ({ - id: child.id, - parentGroup: child.parentGroup, - })) - const routed = q - .from({ chosen: ordered }) - .where(({ chosen }) => eq(chosen.parentGroup, parent.group)) - .select(({ chosen }) => ({ id: chosen.id })) - return { id: parent.id, children: materialize(routed) } - }), - ) + try { + await live.preload() + assertNestedProduct(1) + assertNestedProduct(2) - try { - await live.preload() - expect(live.toArray.map((row) => ids(row.children))).toEqual([[10], [20]]) - } finally { - await cleanup(live, [parents, children]) + parents.write(`update`, { id: 1, group: 1, threshold: 4 }) + assertNestedProduct(1) + + grandchildren.write(`insert`, { id: 300, parentGroup: 10, value: 2 }) + assertNestedProduct(1) + assertNestedProduct(2) + } finally { + await cleanup(live, [parents, children, grandchildren]) + } +} + +async function runAggregationCell({ + grouping, + placement, +}: AggregationCell): Promise { + const name = `${grouping}-${placement}` + const parents = createGrammarCollection(`aggregate-${name}-parents`, [ + { id: 1, group: 1, factor: 2 }, + { id: 2, group: 1, factor: -1 }, + ]) + const children = createGrammarCollection(`aggregate-${name}-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const correlated = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + const rows = + grouping === `explicit` + ? correlated + .groupBy(({ child }) => child.parentGroup) + .select(({ child }) => ({ + score: + placement === `inside-aggregate` + ? sum(multiply(child.value, parent.factor)) + : multiply(sum(child.value), parent.factor), + })) + : correlated.select(({ child }) => ({ + score: + placement === `inside-aggregate` + ? sum(multiply(child.value, parent.factor)) + : multiply(sum(child.value), parent.factor), + })) + return { id: parent.id, ...includeInEveryForm(rows) } + }), + ) + + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + const total = children.collection.toArray + .filter((child) => child.parentGroup === parent.group) + .reduce((result, child) => result + child.value, 0) + return [total * parent.factor] + } + + const project = (rows: Iterable<{ score: number }>) => + [...rows].map(({ score }) => score) + + try { + await live.preload() + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) } - }) - test(`parameterizes a joined QueryRef before its filter`, async () => { - const parents = createControlledCollection(`joined-queryref-parents`, [ - { id: 1, group: 1, maximumTag: 1 }, - { id: 2, group: 1, maximumTag: 2 }, - ]) - const children = createControlledCollection(`joined-queryref-children`, [ - { id: 10, parentGroup: 1, tagId: 1 }, - { id: 20, parentGroup: 1, tagId: 2 }, - ]) - const tags = createControlledCollection(`joined-queryref-tags`, [ - { id: 1 }, - { id: 2 }, - ]) - const live = createLiveQueryCollection((q) => - q - .from({ parent: parents.collection }) - .orderBy(({ parent }) => parent.id) - .select(({ parent }) => { - const allowedTags = q - .from({ tag: tags.collection }) - .where(({ tag }) => lte(tag.id, parent.maximumTag)) - .select(({ tag }) => ({ id: tag.id })) - const childRows = q - .from({ child: children.collection }) - .innerJoin({ tag: allowedTags }, ({ child, tag }) => - eq(child.tagId, tag.id), - ) - .where(({ child }) => eq(child.parentGroup, parent.group)) - .orderBy(({ child }) => child.id) - .select(({ child }) => ({ id: child.id })) - return { id: parent.id, children: materialize(childRows) } - }), - ) + parents.write(`update`, { id: 2, group: 1, factor: 3 }) + expectEveryForm(live.get(2)!, project, expected(2)) - try { - await live.preload() - expect(live.toArray.map((row) => ids(row.children))).toEqual([ - [10], - [10, 20], - ]) - } finally { - await cleanup(live, [parents, children, tags]) + children.write(`insert`, { id: 30, parentGroup: 1, value: 4 }) + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) } - }) + } finally { + await cleanup(live, [parents, children]) + } +} - test(`keeps parent context inside union branches`, async () => { - const parents = createControlledCollection(`joined-union-parents`, [ - { id: 1, group: 1, maximumTag: 1 }, - { id: 2, group: 1, maximumTag: 2 }, - ]) - const children = createControlledCollection(`joined-union-children`, [ - { id: 10, parentGroup: 1, tagId: 1 }, - { id: 20, parentGroup: 1, tagId: 2 }, - ]) - const oddTags = createControlledCollection(`joined-union-odd-tags`, [ - { id: 1, parentGroup: 1 }, - ]) - const evenTags = createControlledCollection(`joined-union-even-tags`, [ - { id: 2, parentGroup: 1 }, - ]) - const live = createLiveQueryCollection((q) => - q - .from({ parent: parents.collection }) - .orderBy(({ parent }) => parent.id) - .select(({ parent }) => { - const oddBranch = q - .from({ oddTag: oddTags.collection }) - .where(({ oddTag }) => lte(oddTag.id, parent.maximumTag)) - .select(({ oddTag }) => ({ - id: oddTag.id, - parentGroup: oddTag.parentGroup, - })) - const evenBranch = q - .from({ evenTag: evenTags.collection }) - .where(({ evenTag }) => lte(evenTag.id, parent.maximumTag)) - .select(({ evenTag }) => ({ - id: evenTag.id, - parentGroup: evenTag.parentGroup, - })) - const childRows = q - .unionAll(oddBranch, evenBranch) - .innerJoin({ child: children.collection }, ({ id, child }) => - eq(id, child.tagId), +type CandidateRow = { + id: number + parentGroup: number + value: number +} + +async function runRecursiveSourceCell({ + boundary, + phase, +}: RecursiveSourceCell): Promise { + const name = `${boundary}-${phase}` + const initialParameter = + phase === `order-window` + ? [1, -1] + : phase === `projection` || phase === `aggregate` + ? [10, 20] + : phase === `having` + ? [0, 1] + : [1, 2] + const parents = createGrammarCollection(`recursive-${name}-parents`, [ + { id: 1, group: 1, parameter: initialParameter[0]! }, + { id: 2, group: 1, parameter: initialParameter[1]! }, + ]) + const initialCandidates: Array = [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 1, value: 2 }, + { id: 30, parentGroup: 1, value: 3 }, + { id: 40, parentGroup: 1, value: 4 }, + ] + const candidates = createGrammarCollection( + `recursive-${name}-candidates`, + initialCandidates, + ) + const left = createGrammarCollection( + `recursive-${name}-left`, + initialCandidates.filter(({ id }) => id % 20 === 10), + ) + const right = createGrammarCollection( + `recursive-${name}-right`, + initialCandidates.filter(({ id }) => id % 20 === 0), + ) + const anchors = createGrammarCollection(`recursive-${name}-anchors`, [ + ...initialCandidates.map(({ id, parentGroup }) => ({ id, parentGroup })), + { id: 50, parentGroup: 1 }, + ]) + + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const buildCandidates = (source: Collection) => { + const correlated = q + .from({ candidate: source }) + .where(({ candidate }) => eq(candidate.parentGroup, parent.group)) + switch (phase) { + case `filter`: + return correlated + .where(({ candidate }) => + lte(candidate.value, parent.parameter), + ) + .select(({ candidate }) => ({ + id: candidate.id, + parentGroup: candidate.parentGroup, + value: candidate.id, + })) + case `projection`: + return correlated.select(({ candidate }) => ({ + id: candidate.id, + parentGroup: candidate.parentGroup, + value: add(candidate.value, parent.parameter), + })) + case `aggregate`: + return correlated + .groupBy(({ candidate }) => [ + candidate.id, + candidate.parentGroup, + ]) + .select(({ candidate }) => ({ + id: candidate.id, + parentGroup: candidate.parentGroup, + value: add(count(candidate.id), parent.parameter), + })) + case `having`: + return correlated + .groupBy(({ candidate }) => [ + candidate.id, + candidate.parentGroup, + ]) + .having(({ candidate }) => + gt(count(candidate.id), parent.parameter), + ) + .select(({ candidate }) => ({ + id: candidate.id, + parentGroup: candidate.parentGroup, + value: count(candidate.id), + })) + case `order-window`: + return correlated + .orderBy(({ candidate }) => + multiply(candidate.value, parent.parameter), + ) + .orderBy(({ candidate }) => candidate.id) + .limit(1) + .select(({ candidate }) => ({ + id: candidate.id, + parentGroup: candidate.parentGroup, + value: candidate.id, + })) + } + } + + // unionAll branches must use distinct lexical aliases. Keep these two + // adapters explicit so the grammar exercises the public builder rules + // without erasing their types behind a cast. + const buildLeftCandidates = () => { + const correlated = q + .from({ leftCandidate: left.collection }) + .where(({ leftCandidate }) => + eq(leftCandidate.parentGroup, parent.group), ) - .where(({ child }) => eq(child.parentGroup, parent.group)) - .orderBy(({ child }) => child.id) - .select(({ child }) => ({ id: child.id })) - return { id: parent.id, children: materialize(childRows) } - }), + switch (phase) { + case `filter`: + return correlated + .where(({ leftCandidate }) => + lte(leftCandidate.value, parent.parameter), + ) + .select(({ leftCandidate }) => ({ + id: leftCandidate.id, + parentGroup: leftCandidate.parentGroup, + value: leftCandidate.id, + })) + case `projection`: + return correlated.select(({ leftCandidate }) => ({ + id: leftCandidate.id, + parentGroup: leftCandidate.parentGroup, + value: add(leftCandidate.value, parent.parameter), + })) + case `aggregate`: + return correlated + .groupBy(({ leftCandidate }) => [ + leftCandidate.id, + leftCandidate.parentGroup, + ]) + .select(({ leftCandidate }) => ({ + id: leftCandidate.id, + parentGroup: leftCandidate.parentGroup, + value: add(count(leftCandidate.id), parent.parameter), + })) + case `having`: + return correlated + .groupBy(({ leftCandidate }) => [ + leftCandidate.id, + leftCandidate.parentGroup, + ]) + .having(({ leftCandidate }) => + gt(count(leftCandidate.id), parent.parameter), + ) + .select(({ leftCandidate }) => ({ + id: leftCandidate.id, + parentGroup: leftCandidate.parentGroup, + value: count(leftCandidate.id), + })) + case `order-window`: + return correlated + .orderBy(({ leftCandidate }) => + multiply(leftCandidate.value, parent.parameter), + ) + .orderBy(({ leftCandidate }) => leftCandidate.id) + .limit(1) + .select(({ leftCandidate }) => ({ + id: leftCandidate.id, + parentGroup: leftCandidate.parentGroup, + value: leftCandidate.id, + })) + } + } + + const buildRightCandidates = () => { + const correlated = q + .from({ rightCandidate: right.collection }) + .where(({ rightCandidate }) => + eq(rightCandidate.parentGroup, parent.group), + ) + switch (phase) { + case `filter`: + return correlated + .where(({ rightCandidate }) => + lte(rightCandidate.value, parent.parameter), + ) + .select(({ rightCandidate }) => ({ + id: rightCandidate.id, + parentGroup: rightCandidate.parentGroup, + value: rightCandidate.id, + })) + case `projection`: + return correlated.select(({ rightCandidate }) => ({ + id: rightCandidate.id, + parentGroup: rightCandidate.parentGroup, + value: add(rightCandidate.value, parent.parameter), + })) + case `aggregate`: + return correlated + .groupBy(({ rightCandidate }) => [ + rightCandidate.id, + rightCandidate.parentGroup, + ]) + .select(({ rightCandidate }) => ({ + id: rightCandidate.id, + parentGroup: rightCandidate.parentGroup, + value: add(count(rightCandidate.id), parent.parameter), + })) + case `having`: + return correlated + .groupBy(({ rightCandidate }) => [ + rightCandidate.id, + rightCandidate.parentGroup, + ]) + .having(({ rightCandidate }) => + gt(count(rightCandidate.id), parent.parameter), + ) + .select(({ rightCandidate }) => ({ + id: rightCandidate.id, + parentGroup: rightCandidate.parentGroup, + value: count(rightCandidate.id), + })) + case `order-window`: + return correlated + .orderBy(({ rightCandidate }) => + multiply(rightCandidate.value, parent.parameter), + ) + .orderBy(({ rightCandidate }) => rightCandidate.id) + .limit(1) + .select(({ rightCandidate }) => ({ + id: rightCandidate.id, + parentGroup: rightCandidate.parentGroup, + value: rightCandidate.id, + })) + } + } + + switch (boundary) { + case `from-query-ref`: { + const routed = q + .from({ result: buildCandidates(candidates.collection) }) + .where(({ result }) => eq(result.parentGroup, parent.group)) + .select(({ result }) => ({ + id: result.id, + value: result.value, + })) + return { id: parent.id, ...includeInEveryForm(routed) } + } + case `joined-query-ref`: { + const routed = q + .from({ anchor: anchors.collection }) + .innerJoin( + { result: buildCandidates(candidates.collection) }, + ({ anchor, result }) => eq(anchor.id, result.id), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ result }) => ({ + id: result.id, + value: result.value, + })) + return { id: parent.id, ...includeInEveryForm(routed) } + } + case `union-branch`: { + const routed = q + .unionAll(buildLeftCandidates(), buildRightCandidates()) + .innerJoin({ anchor: anchors.collection }, ({ id, anchor }) => + eq(id, anchor.id), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ id, value }) => ({ id, value })) + return { id: parent.id, ...includeInEveryForm(routed) } + } + } + }), + ) + + const modelRows = new Map(initialCandidates.map((row) => [row.id, row])) + const leftModelIds = new Set(left.collection.toArray.map(({ id }) => id)) + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + const correlated = [...modelRows.values()].filter( + (candidate) => candidate.parentGroup === parent.group, ) + switch (phase) { + case `filter`: + return correlated + .filter((candidate) => candidate.value <= parent.parameter) + .map((candidate) => ({ id: candidate.id, value: candidate.id })) + case `projection`: + return correlated.map((candidate) => ({ + id: candidate.id, + value: candidate.value + parent.parameter, + })) + case `aggregate`: + return correlated.map((candidate) => ({ + id: candidate.id, + value: 1 + parent.parameter, + })) + case `having`: + return parent.parameter < 1 + ? correlated.map((candidate) => ({ id: candidate.id, value: 1 })) + : [] + case `order-window`: { + const partitions = + boundary === `union-branch` + ? [ + correlated.filter(({ id }) => leftModelIds.has(id)), + correlated.filter(({ id }) => !leftModelIds.has(id)), + ] + : [correlated] + return partitions.flatMap((partition) => + partition + .sort( + (leftRow, rightRow) => + leftRow.value * parent.parameter - + rightRow.value * parent.parameter || leftRow.id - rightRow.id, + ) + .slice(0, 1) + .map((candidate) => ({ id: candidate.id, value: candidate.id })), + ) + } + } + } - try { - await live.preload() - expect(live.toArray.map((row) => ids(row.children))).toEqual([ - [10], - [10, 20], - ]) - } finally { - await cleanup(live, [parents, children, oddTags, evenTags]) + const project = (rows: Iterable<{ id: number; value: number }>) => + [...rows] + .map(({ id, value }) => ({ id, value })) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + const assertParents = () => { + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) } - }) + } - test(`evaluates a joined-side key with its parent route`, async () => { - const parents = createControlledCollection(`joined-key-parents`, [ - { id: 1, group: 1, offset: 0 }, - { id: 2, group: 1, offset: 1 }, - ]) - const children = createControlledCollection(`joined-key-children`, [ - { id: 10, parentGroup: 1, tagId: 2 }, - ]) - const tags = createControlledCollection(`joined-key-tags`, [ - { id: 1, label: `one` }, - { id: 2, label: `two` }, - ]) - const live = createLiveQueryCollection((q) => - q - .from({ parent: parents.collection }) - .orderBy(({ parent }) => parent.id) - .select(({ parent }) => { - const childRows = q - .from({ child: children.collection }) - .innerJoin({ tag: tags.collection }, ({ child, tag }) => - eq(add(tag.id, parent.offset), child.tagId), - ) - .where(({ child }) => eq(child.parentGroup, parent.group)) - .select(({ child, tag }) => ({ id: child.id, label: tag.label })) - return { id: parent.id, children: materialize(childRows) } - }), - ) + try { + await live.preload() + assertParents() - try { - await live.preload() - expect(live.toArray.map((row) => row.children[0]?.label)).toEqual([ - `two`, - `one`, - ]) - } finally { - await cleanup(live, [parents, children, tags]) + const updatedParameter = + phase === `order-window` + ? -1 + : phase === `projection` || phase === `aggregate` + ? 30 + : phase === `having` + ? 0 + : 4 + parents.write(`update`, { id: 1, group: 1, parameter: updatedParameter }) + assertParents() + + const inserted = { + id: 50, + parentGroup: 1, + value: phase === `filter` ? 1 : 5, } - }) + modelRows.set(inserted.id, inserted) + if (boundary === `union-branch`) right.write(`insert`, inserted) + else candidates.write(`insert`, inserted) + assertParents() + } finally { + await cleanup(live, [parents, candidates, left, right, anchors]) + } +} - test(`attaches routes before correlating through a joined source`, async () => { - const parents = createControlledCollection(`joined-correlation-parents`, [ - { id: 1, group: 1, offset: 1 }, - { id: 2, group: 2, offset: 0 }, - ]) - const children = createControlledCollection(`joined-correlation-children`, [ - { id: 10, tagId: 2 }, - ]) - const tags = createControlledCollection(`joined-correlation-tags`, [ - { id: 1, group: 1, label: `one` }, - { id: 2, group: 2, label: `two` }, - ]) - const live = createLiveQueryCollection((q) => - q - .from({ parent: parents.collection }) - .orderBy(({ parent }) => parent.id) - .select(({ parent }) => { - const childRows = q - .from({ child: children.collection }) - .innerJoin({ tag: tags.collection }, ({ child, tag }) => - eq(add(tag.id, parent.offset), child.tagId), - ) - .where(({ tag }) => eq(tag.group, parent.group)) - .select(({ child, tag }) => ({ id: child.id, label: tag.label })) - return { id: parent.id, children: materialize(childRows) } - }), - ) +async function runJoinCell({ + keySide, + correlationAttachment, +}: JoinCell): Promise { + const name = `${keySide}-${correlationAttachment}` + const parents = createGrammarCollection(`join-${name}-parents`, [ + { id: 1, group: 1, offset: 0 }, + { id: 2, group: 2, offset: 1 }, + ]) + const children = createGrammarCollection(`join-${name}-children`, [ + { id: 10, parentGroup: 1, tagId: 2 }, + { id: 20, parentGroup: 2, tagId: 2 }, + ]) + const tags = createGrammarCollection(`join-${name}-tags`, [ + { id: 1, parentGroup: 2, label: `one` }, + { id: 2, parentGroup: 1, label: `two` }, + { id: 3, parentGroup: 2, label: `three` }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const joined = + keySide === `main` + ? q + .from({ child: children.collection }) + .innerJoin({ tag: tags.collection }, ({ child, tag }) => + eq(tag.id, add(child.tagId, parent.offset)), + ) + : q + .from({ child: children.collection }) + .innerJoin({ tag: tags.collection }, ({ child, tag }) => + eq(add(tag.id, parent.offset), child.tagId), + ) + const routed = ( + correlationAttachment === `main` + ? joined.where(({ child }) => eq(child.parentGroup, parent.group)) + : joined.where(({ tag }) => eq(tag.parentGroup, parent.group)) + ) + .orderBy(({ child }) => child.id) + .select(({ child, tag }) => ({ id: child.id, value: tag.label })) + return { id: parent.id, ...includeInEveryForm(routed) } + }), + ) - try { - await live.preload() - expect(live.toArray.map((row) => row.children[0]?.label)).toEqual([ - `one`, - `two`, - ]) - } finally { - await cleanup(live, [parents, children, tags]) + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + return children.collection.toArray + .flatMap((child) => + tags.collection.toArray + .filter((tag) => { + const keyMatches = + keySide === `main` + ? tag.id === child.tagId + parent.offset + : tag.id + parent.offset === child.tagId + const routeMatches = + correlationAttachment === `main` + ? child.parentGroup === parent.group + : tag.parentGroup === parent.group + return keyMatches && routeMatches + }) + .map((tag) => ({ id: child.id, value: tag.label })), + ) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + } + + const project = (rows: Iterable<{ id: number; value: string }>) => + [...rows].map(({ id, value }) => ({ id, value })) + const assertParents = () => { + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) } + } + + try { + await live.preload() + assertParents() + + parents.write(`update`, { id: 1, group: 1, offset: 1 }) + assertParents() + + children.write(`insert`, { id: 30, parentGroup: 1, tagId: 2 }) + assertParents() + } finally { + await cleanup(live, [parents, children, tags]) + } +} + +async function runGrammarCell(cell: GrammarCell): Promise { + switch (cell.family) { + case `lexical-scope`: + return runLexicalScopeCell(cell) + case `aggregation`: + return runAggregationCell(cell) + case `recursive-source`: + return runRecursiveSourceCell(cell) + case `join`: + return runJoinCell(cell) + } +} + +describe(`correlated include route-context transport grammar`, () => { + test(`expands every declared product without duplicate cells`, () => { + const expectedCellCount = + routeContextGrammar.lexicalScope.scopes.length + + routeContextGrammar.aggregation.groupings.length * + routeContextGrammar.aggregation.placements.length + + routeContextGrammar.recursiveSource.boundaries.length * + routeContextGrammar.recursiveSource.phases.length + + routeContextGrammar.join.keySides.length * + routeContextGrammar.join.correlationAttachments.length + const names = grammarCells.map(grammarCellName) + + expect(grammarCells).toHaveLength(expectedCellCount) + expect(new Set(names)).toHaveLength(expectedCellCount) + expect( + grammarCells.length * materializationForms.length * checkpoints.length, + ).toBe(225) }) + + for (const cell of grammarCells) { + test(`${grammarCellName(cell)} × every materialization form × parent/child updates`, () => + runGrammarCell(cell)) + } }) From 8946a9d6d405fdbc83ecb4ea937232e17949ee7c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 22 Aug 2026 17:13:28 -0600 Subject: [PATCH 6/8] fix: close correlated route coverage gaps --- packages/db/src/query/compiler/index.ts | 43 +- packages/db/src/query/compiler/joins.ts | 3 + .../includes-context-transport-oracle.test.ts | 437 +++++++++++++++++- 3 files changed, 461 insertions(+), 22 deletions(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 4de7ae04d1..e9ca8b2b28 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -95,14 +95,23 @@ type CompiledParentProjection = { function projectParentContext( nsRow: NamespacedRow, projections: Array, -): Record> { +): Record { const inherited = (nsRow as any).__parentContext - const parentContext: Record> = inherited != + const parentContext: Record = inherited != null && typeof inherited === `object` ? { ...inherited } : {} for (const projection of projections) { + if (projection.field.length === 0) { + const projectedAlias = projection.compiled(nsRow) + parentContext[projection.alias] = + projectedAlias != null && typeof projectedAlias === `object` + ? { ...projectedAlias } + : projectedAlias + continue + } + const inheritedAlias = parentContext[projection.alias] const aliasContext = inheritedAlias != null && typeof inheritedAlias === `object` @@ -1469,6 +1478,7 @@ function processFromClause( const branchRow = parentKeyStream ? { ...row, + [INCLUDES_PUBLIC_KEY]: branchPublicKey, [alias]: { ...row[alias], [INCLUDES_PUBLIC_KEY]: branchPublicKey, @@ -1570,16 +1580,19 @@ function processUnionAll( map((data: any) => { const [key, [row, _order, correlationKey, parentContext, publicKey]] = data - const routedRow = attachRouteMetadataToResult( - row, - correlationKey, - parentContext, - publicKey, - ) - return [`${index}:${encodeKeyForUnionBranch(key)}`, routedRow] as [ - string, - Record, - ] + const branchKey = `${index}:${encodeKeyForUnionBranch(key)}` + const branchPublicKey = `${index}:${encodeKeyForUnionBranch( + publicKey ?? key, + )}` + const routedRow = parentKeyStream + ? attachRouteMetadataToResult( + row, + correlationKey, + parentContext, + branchPublicKey, + ) + : row + return [branchKey, routedRow] as [string, Record] }), ) @@ -1847,7 +1860,11 @@ function getIncludesPublicKey( mainSource: string, fallback: unknown, ): unknown { - return row[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? fallback + return ( + row[mainSource]?.[INCLUDES_PUBLIC_KEY] ?? + (row as any)[INCLUDES_PUBLIC_KEY] ?? + fallback + ) } /** diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index d389b2fb2d..5d61f34b1f 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -651,6 +651,9 @@ function processJoinSource( map((data: any) => { const [key, [value, _orderByIndex, correlationKey, parentContext]] = data + if (!parentKeyStream) { + return [key, value] as [unknown, any] + } return [ key, value != null && typeof value === `object` diff --git a/packages/db/tests/query/includes-context-transport-oracle.test.ts b/packages/db/tests/query/includes-context-transport-oracle.test.ts index 213bfa18f3..75e09fd610 100644 --- a/packages/db/tests/query/includes-context-transport-oracle.test.ts +++ b/packages/db/tests/query/includes-context-transport-oracle.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from 'vitest' import { add, + coalesce, count, createLiveQueryCollection, eq, @@ -30,6 +31,12 @@ const materializationForms = [`collection`, `array`, `materialized`] as const const checkpoints = [`initial`, `parent-update`, `child-update`] as const const routeContextGrammar = { + parentProjection: { + shapes: [`field`, `whole-row`] as const, + }, + correlationDomain: { + states: [`unmatched`, `null`] as const, + }, lexicalScope: { scopes: [`immediate-parent`, `lexical-ancestor`] as const, }, @@ -51,8 +58,27 @@ const routeContextGrammar = { keySides: [`main`, `joined`] as const, correlationAttachments: [`main`, `joined`] as const, }, + unionIdentity: { + forms: [`union-from`, `union-all`] as const, + }, } as const +const queryRefMetadataGrammar = { + routeModes: [`plain`, `routed`] as const, +} as const + +type QueryRefMetadataMode = (typeof queryRefMetadataGrammar.routeModes)[number] + +type ParentProjectionCell = { + family: `parent-projection` + shape: (typeof routeContextGrammar.parentProjection.shapes)[number] +} + +type CorrelationDomainCell = { + family: `correlation-domain` + state: (typeof routeContextGrammar.correlationDomain.states)[number] +} + type LexicalScopeCell = { family: `lexical-scope` scope: (typeof routeContextGrammar.lexicalScope.scopes)[number] @@ -76,13 +102,33 @@ type JoinCell = { correlationAttachment: (typeof routeContextGrammar.join.correlationAttachments)[number] } +type UnionIdentityCell = { + family: `union-identity` + form: (typeof routeContextGrammar.unionIdentity.forms)[number] +} + type GrammarCell = + | ParentProjectionCell + | CorrelationDomainCell | LexicalScopeCell | AggregationCell | RecursiveSourceCell | JoinCell + | UnionIdentityCell const grammarCells: Array = [ + ...routeContextGrammar.parentProjection.shapes.map( + (shape): ParentProjectionCell => ({ + family: `parent-projection`, + shape, + }), + ), + ...routeContextGrammar.correlationDomain.states.map( + (state): CorrelationDomainCell => ({ + family: `correlation-domain`, + state, + }), + ), ...routeContextGrammar.lexicalScope.scopes.map( (scope): LexicalScopeCell => ({ family: `lexical-scope`, scope }), ), @@ -113,6 +159,9 @@ const grammarCells: Array = [ }), ), ), + ...routeContextGrammar.unionIdentity.forms.map( + (form): UnionIdentityCell => ({ family: `union-identity`, form }), + ), ] async function cleanup( @@ -167,6 +216,10 @@ function ids(rows: Iterable<{ id: number }>): Array { function grammarCellName(cell: GrammarCell): string { switch (cell.family) { + case `parent-projection`: + return `${cell.family} / ${cell.shape}` + case `correlation-domain`: + return `${cell.family} / ${cell.state}` case `lexical-scope`: return `${cell.family} / ${cell.scope}` case `aggregation`: @@ -175,6 +228,139 @@ function grammarCellName(cell: GrammarCell): string { return `${cell.family} / ${cell.boundary} / ${cell.phase}` case `join`: return `${cell.family} / ${cell.keySide}-side key / ${cell.correlationAttachment}-side correlation` + case `union-identity`: + return `${cell.family} / ${cell.form}` + } +} + +async function runParentProjectionCell({ + shape, +}: ParentProjectionCell): Promise { + type ParentRow = { id: number; group: number; token: string } + const parentRows: Array = [ + { id: 1, group: 1, token: `one` }, + { id: 2, group: 1, token: `two` }, + ] + const parents = createGrammarCollection( + `projection-${shape}-parents`, + parentRows, + ) + const children = createGrammarCollection(`projection-${shape}-children`, [ + { id: 10, parentGroup: 1 }, + { id: 20, parentGroup: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const correlated = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + const rows = correlated + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ + id: child.id, + parentSnapshot: + shape === `field` + ? { token: parent.token } + : coalesce(parent, null), + })) + return { id: parent.id, ...includeInEveryForm(rows) } + }), + ) + + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + return children.collection.toArray + .filter((child) => child.parentGroup === parent.group) + .map(({ id }) => ({ id, token: parent.token })) + } + const project = ( + rows: Iterable<{ id: number; parentSnapshot: unknown }>, + ) => + [...rows].map(({ id, parentSnapshot }) => { + const token = + parentSnapshot != null && + typeof parentSnapshot === `object` && + `token` in parentSnapshot && + typeof parentSnapshot.token === `string` + ? parentSnapshot.token + : undefined + return { id, token } + }) + const assertParents = () => { + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) + } + } + + try { + await live.preload() + assertParents() + + const updatedParent = { id: 1, group: 1, token: `updated` } + parents.write(`update`, updatedParent) + assertParents() + + children.write(`insert`, { id: 30, parentGroup: 1 }) + assertParents() + } finally { + await cleanup(live, [parents, children]) + } +} + +async function runCorrelationDomainCell({ + state, +}: CorrelationDomainCell): Promise { + type ParentRow = { id: number; group: number | null } + type ChildRow = { id: number; parentGroup: number | null } + const parents = createGrammarCollection( + `correlation-${state}-parents`, + [{ id: 1, group: state === `null` ? null : 999 }], + ) + const children = createGrammarCollection( + `correlation-${state}-children`, + [ + { id: 10, parentGroup: null }, + { id: 20, parentGroup: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const rows = q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ id: child.id })) + return { id: parent.id, ...includeInEveryForm(rows) } + }), + ) + + const expected = () => { + const parentGroup = parents.collection.get(1)!.group + return children.collection.toArray + .filter( + (child) => + parentGroup != null && + child.parentGroup != null && + child.parentGroup === parentGroup, + ) + .map(({ id }) => id) + } + const assertResult = () => expectEveryForm(live.get(1)!, ids, expected()) + + try { + await live.preload() + assertResult() + + parents.write(`update`, { id: 1, group: 1 }) + assertResult() + + children.write(`insert`, { id: 30, parentGroup: 1 }) + assertResult() + } finally { + await cleanup(live, [parents, children]) } } @@ -366,14 +552,14 @@ async function runAggregationCell({ .select(({ child }) => ({ score: placement === `inside-aggregate` - ? sum(multiply(child.value, parent.factor)) - : multiply(sum(child.value), parent.factor), + ? sum(add(child.value, parent.factor)) + : add(sum(child.value), parent.factor), })) : correlated.select(({ child }) => ({ score: placement === `inside-aggregate` - ? sum(multiply(child.value, parent.factor)) - : multiply(sum(child.value), parent.factor), + ? sum(add(child.value, parent.factor)) + : add(sum(child.value), parent.factor), })) return { id: parent.id, ...includeInEveryForm(rows) } }), @@ -384,7 +570,14 @@ async function runAggregationCell({ const total = children.collection.toArray .filter((child) => child.parentGroup === parent.group) .reduce((result, child) => result + child.value, 0) - return [total * parent.factor] + const childCount = children.collection.toArray.filter( + (child) => child.parentGroup === parent.group, + ).length + return [ + placement === `inside-aggregate` + ? total + childCount * parent.factor + : total + parent.factor, + ] } const project = (rows: Iterable<{ score: number }>) => @@ -689,7 +882,9 @@ async function runRecursiveSourceCell({ ) const modelRows = new Map(initialCandidates.map((row) => [row.id, row])) - const leftModelIds = new Set(left.collection.toArray.map(({ id }) => id)) + const leftModelIds = new Set( + initialCandidates.filter(({ id }) => id % 20 === 10).map(({ id }) => id), + ) const expected = (parentId: number) => { const parent = parents.collection.get(parentId)! const correlated = [...modelRows.values()].filter( @@ -756,7 +951,7 @@ async function runRecursiveSourceCell({ : phase === `projection` || phase === `aggregate` ? 30 : phase === `having` - ? 0 + ? 1 : 4 parents.write(`update`, { id: 1, group: 1, parameter: updatedParameter }) assertParents() @@ -775,6 +970,216 @@ async function runRecursiveSourceCell({ } } +async function runUnionIdentityCell({ + form, +}: UnionIdentityCell): Promise { + const parents = createGrammarCollection(`identity-${form}-parents`, [ + { id: 1, group: 1, parameter: 10 }, + { id: 2, group: 1, parameter: 20 }, + ]) + const left = createGrammarCollection(`identity-${form}-left`, [ + { id: 10, parentGroup: 1, value: 1 }, + ]) + const right = createGrammarCollection( + `identity-${form}-right`, + [{ id: 20, parentGroup: 1, value: 2 }], + ) + const anchors = createGrammarCollection(`identity-${form}-anchors`, [ + { id: 10, parentGroup: 1 }, + { id: 20, parentGroup: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + const unionAllRows = () => { + const leftRows = q + .from({ leftCandidate: left.collection }) + .select(({ leftCandidate }) => ({ + id: leftCandidate.id, + value: add(leftCandidate.value, parent.parameter), + })) + const rightRows = q + .from({ rightCandidate: right.collection }) + .select(({ rightCandidate }) => ({ + id: rightCandidate.id, + value: add(rightCandidate.value, parent.parameter), + })) + return q + .unionAll(leftRows, rightRows) + .innerJoin({ anchor: anchors.collection }, ({ id, anchor }) => + eq(id, anchor.id), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ id, value }) => ({ id, value })) + } + const unionFromRows = () => + q + .unionAll({ + leftCandidate: left.collection, + rightCandidate: right.collection, + }) + .innerJoin( + { anchor: anchors.collection }, + ({ leftCandidate, rightCandidate, anchor }) => + eq(coalesce(leftCandidate.id, rightCandidate.id), anchor.id), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ leftCandidate, rightCandidate }) => ({ + id: coalesce(leftCandidate.id, rightCandidate.id), + value: add( + coalesce(leftCandidate.value, rightCandidate.value), + parent.parameter, + ), + })) + return form === `union-all` + ? { id: parent.id, ...includeInEveryForm(unionAllRows()) } + : { id: parent.id, ...includeInEveryForm(unionFromRows()) } + }), + ) + + const project = (rows: Iterable<{ id: number; value: number }>) => + [...rows] + .map(({ id, value }) => ({ id, value })) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + return [...left.collection.toArray, ...right.collection.toArray] + .filter((candidate) => candidate.parentGroup === parent.group) + .map((candidate) => ({ + id: candidate.id, + value: candidate.value + parent.parameter, + })) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + } + const keys = (parentId: number) => { + const collection = live.get(parentId)!.collection + return new Map( + collection.toArray.map((row) => [row.id, collection.getKeyFromItem(row)]), + ) + } + const assertParents = () => { + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) + } + } + const assertRouteIndependentKeys = () => expect(keys(2)).toEqual(keys(1)) + + try { + await live.preload() + assertParents() + assertRouteIndependentKeys() + const initialKeys = keys(1) + + parents.write(`update`, { id: 1, group: 1, parameter: 30 }) + assertParents() + expect(keys(1)).toEqual(initialKeys) + assertRouteIndependentKeys() + + right.write(`update`, { id: 20, parentGroup: 1, value: 5 }) + assertParents() + expect(keys(1)).toEqual(initialKeys) + assertRouteIndependentKeys() + } finally { + await cleanup(live, [parents, left, right, anchors]) + } +} + +function expectNoRouteMetadata(row: object): void { + expect(Object.hasOwn(row, `__correlationKey`)).toBe(false) + expect(Object.hasOwn(row, `__parentContext`)).toBe(false) +} + +async function runQueryRefMetadataCell( + routeMode: QueryRefMetadataMode, +): Promise { + const anchors = createGrammarCollection(`metadata-${routeMode}-anchors`, [ + { id: 1, candidateId: 10, parentGroup: 1 }, + ]) + const candidates = createGrammarCollection( + `metadata-${routeMode}-candidates`, + [{ id: 10, label: `ten` }], + ) + + if (routeMode === `plain`) { + const live = createLiveQueryCollection((q) => { + const projectedCandidates = q + .from({ candidate: candidates.collection }) + .select(({ candidate }) => ({ + id: candidate.id, + label: candidate.label, + })) + return q + .from({ anchor: anchors.collection }) + .innerJoin( + { candidateResult: projectedCandidates }, + ({ anchor, candidateResult }) => + eq(anchor.candidateId, candidateResult.id), + ) + .select(({ candidateResult }) => candidateResult) + }) + + try { + await live.preload() + expectNoRouteMetadata(live.toArray[0]!) + + candidates.write(`update`, { id: 10, label: `updated` }) + expectNoRouteMetadata(live.toArray[0]!) + } finally { + await cleanup(live, [anchors, candidates]) + } + return + } + + const parents = createGrammarCollection(`metadata-routed-parents`, [ + { id: 1, group: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => { + const projectedCandidates = q + .from({ candidate: candidates.collection }) + .select(({ candidate }) => ({ + id: candidate.id, + label: candidate.label, + })) + const rows = q + .from({ anchor: anchors.collection }) + .innerJoin( + { candidateResult: projectedCandidates }, + ({ anchor, candidateResult }) => + eq(anchor.candidateId, candidateResult.id), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ candidateResult }) => candidateResult) + return { id: parent.id, ...includeInEveryForm(rows) } + }), + ) + const assertClean = () => { + const forms = live.get(1)! + for (const rows of [ + forms.collection.values(), + forms.array, + forms.materialized, + ]) { + for (const row of rows) expectNoRouteMetadata(row) + } + } + + try { + await live.preload() + assertClean() + + parents.write(`update`, { id: 1, group: 2 }) + assertClean() + + anchors.write(`update`, { id: 1, candidateId: 10, parentGroup: 2 }) + assertClean() + } finally { + await cleanup(live, [parents, anchors, candidates]) + } +} + async function runJoinCell({ keySide, correlationAttachment, @@ -866,6 +1271,10 @@ async function runJoinCell({ async function runGrammarCell(cell: GrammarCell): Promise { switch (cell.family) { + case `parent-projection`: + return runParentProjectionCell(cell) + case `correlation-domain`: + return runCorrelationDomainCell(cell) case `lexical-scope`: return runLexicalScopeCell(cell) case `aggregation`: @@ -874,30 +1283,40 @@ async function runGrammarCell(cell: GrammarCell): Promise { return runRecursiveSourceCell(cell) case `join`: return runJoinCell(cell) + case `union-identity`: + return runUnionIdentityCell(cell) } } describe(`correlated include route-context transport grammar`, () => { test(`expands every declared product without duplicate cells`, () => { const expectedCellCount = + routeContextGrammar.parentProjection.shapes.length + + routeContextGrammar.correlationDomain.states.length + routeContextGrammar.lexicalScope.scopes.length + routeContextGrammar.aggregation.groupings.length * routeContextGrammar.aggregation.placements.length + routeContextGrammar.recursiveSource.boundaries.length * routeContextGrammar.recursiveSource.phases.length + routeContextGrammar.join.keySides.length * - routeContextGrammar.join.correlationAttachments.length + routeContextGrammar.join.correlationAttachments.length + + routeContextGrammar.unionIdentity.forms.length const names = grammarCells.map(grammarCellName) expect(grammarCells).toHaveLength(expectedCellCount) expect(new Set(names)).toHaveLength(expectedCellCount) expect( grammarCells.length * materializationForms.length * checkpoints.length, - ).toBe(225) + ).toBe(279) }) for (const cell of grammarCells) { test(`${grammarCellName(cell)} × every materialization form × parent/child updates`, () => runGrammarCell(cell)) } + + for (const routeMode of queryRefMetadataGrammar.routeModes) { + test(`query-ref metadata / ${routeMode}`, () => + runQueryRefMetadataCell(routeMode)) + } }) From 27106f0e5acee96b2ca927c069a028e9f1ea6d34 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Sat, 22 Aug 2026 23:14:55 +0000 Subject: [PATCH 7/8] ci: apply automated fixes --- packages/db/src/query/compiler/index.ts | 6 ++---- .../tests/query/includes-context-transport-oracle.test.ts | 4 +--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index e9ca8b2b28..b45ce95ec7 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -97,10 +97,8 @@ function projectParentContext( projections: Array, ): Record { const inherited = (nsRow as any).__parentContext - const parentContext: Record = inherited != - null && typeof inherited === `object` - ? { ...inherited } - : {} + const parentContext: Record = + inherited != null && typeof inherited === `object` ? { ...inherited } : {} for (const projection of projections) { if (projection.field.length === 0) { diff --git a/packages/db/tests/query/includes-context-transport-oracle.test.ts b/packages/db/tests/query/includes-context-transport-oracle.test.ts index 75e09fd610..d202c81644 100644 --- a/packages/db/tests/query/includes-context-transport-oracle.test.ts +++ b/packages/db/tests/query/includes-context-transport-oracle.test.ts @@ -276,9 +276,7 @@ async function runParentProjectionCell({ .filter((child) => child.parentGroup === parent.group) .map(({ id }) => ({ id, token: parent.token })) } - const project = ( - rows: Iterable<{ id: number; parentSnapshot: unknown }>, - ) => + const project = (rows: Iterable<{ id: number; parentSnapshot: unknown }>) => [...rows].map(({ id, parentSnapshot }) => { const token = parentSnapshot != null && From e7c3a369df77fe15cc55e82dd366891a2929fa06 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 22 Aug 2026 18:23:36 -0600 Subject: [PATCH 8/8] fix: preserve routes for scalar derived rows --- packages/db/src/query/builder/types.ts | 4 +- packages/db/src/query/compiler/index.ts | 120 +++++----- packages/db/src/query/compiler/joins.ts | 75 +++--- .../db/src/query/compiler/parent-routes.ts | 40 ++++ .../db/src/query/compiler/route-metadata.ts | 67 ++++++ packages/db/src/query/compiler/select.ts | 10 + packages/db/src/query/live/ARCHITECTURE.md | 13 +- .../includes-context-transport-oracle.test.ts | 222 +++++++++++++++++- 8 files changed, 440 insertions(+), 111 deletions(-) create mode 100644 packages/db/src/query/compiler/parent-routes.ts create mode 100644 packages/db/src/query/compiler/route-metadata.ts diff --git a/packages/db/src/query/builder/types.ts b/packages/db/src/query/builder/types.ts index db20942b59..6286403e27 100644 --- a/packages/db/src/query/builder/types.ts +++ b/packages/db/src/query/builder/types.ts @@ -93,7 +93,7 @@ export type Source = { [alias: string]: | CollectionImpl | CollectionOptionsIdentity - | QueryBuilder + | QueryBuilder } /** @@ -168,7 +168,7 @@ export type ContextFromUnionSource = : ContextFromSource type ResultFromBranch = - TBranch extends QueryBuilder ? GetResult : never + TBranch extends QueryBuilder ? GetRawResult : never type UnionBranchResult>> = ResultFromBranch diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index b45ce95ec7..a0bba00e34 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -40,6 +40,12 @@ import { processJoins, registerLazyDemandPlan } from './joins.js' import { containsAggregate, processGroupBy } from './group-by.js' import { getLazyLoadTargets } from './lazy-targets.js' import { processOrderBy } from './order-by.js' +import { crossJoinParentRoutes } from './parent-routes.js' +import { + INCLUDES_PUBLIC_KEY, + attachRouteMetadataToResult, + getRoutedScalarMetadata, +} from './route-metadata.js' import { processSelect } from './select.js' import type { CollectionSubscription } from '../../collection/subscription.js' import type { OrderByOptimizationInfo } from './order-by.js' @@ -63,13 +69,12 @@ import type { import type { QueryCache, QueryMapping, WindowOptions } from './types.js' export type { WindowOptions } from './types.js' +export { INCLUDES_PUBLIC_KEY } from './route-metadata.js' /** Symbol used to tag parent $selected with routing metadata for includes */ export const INCLUDES_ROUTING = Symbol(`includesRouting`) -export const INCLUDES_PUBLIC_KEY = Symbol(`includesPublicKey`) export const FN_SELECT_STATE = Symbol(`fnSelectState`) const SKIP_INCLUDE = Symbol(`skipInclude`) -const PARENT_ROUTE_CROSS_KEY = `__tanstack_parent_route_cross__` type ConditionalSelectGuard = { condition: BasicExpression @@ -140,25 +145,13 @@ function parameterizeByParentRoutes( parentKeyStream: KeyedStream, mainSource: string, ): NamespacedAndKeyedStream { - const rows: any = pipeline.pipe( - map(([rowKey, row]) => [PARENT_ROUTE_CROSS_KEY, [rowKey, row]]), - ) - const routes: any = parentKeyStream.pipe( - map(([correlationKey, parentContext]) => [ - PARENT_ROUTE_CROSS_KEY, - [correlationKey, parentContext], - ]), - ) - - return rows.pipe( - joinOperator(routes, `inner`), - filter(([, [rowSide, routeSide]]: any) => - Boolean(rowSide != null && routeSide != null), - ), - map(([, [rowSide, routeSide]]: any) => { - const [rowKey, row] = rowSide! - const [correlationKey, parentContext] = routeSide! - const namespaced = { ...row } as Record + return crossJoinParentRoutes( + pipeline, + parentKeyStream, + (rowKey, row, correlationKey, parentContext) => { + const namespaced = { + ...(row as Record), + } as Record namespaced[mainSource] = { ...namespaced[mainSource], __correlationKey: correlationKey, @@ -172,7 +165,7 @@ function parameterizeByParentRoutes( serializeValue([rowKey, correlationKey, parentContext]), namespaced, ] as [string, NamespacedRow] - }), + }, ) as NamespacedAndKeyedStream } @@ -195,29 +188,6 @@ function correlationValuesEqual(left: unknown, right: unknown): boolean { ) } -function attachRouteMetadataToResult( - value: any, - correlationKey: unknown, - parentContext: unknown, - publicKey: unknown, -): any { - if ( - value == null || - typeof value !== `object` || - (correlationKey === undefined && - parentContext === undefined && - publicKey === undefined) - ) { - return value - } - return { - ...value, - __correlationKey: correlationKey, - __parentContext: parentContext, - [INCLUDES_PUBLIC_KEY]: publicKey, - } -} - /** * Result of compiling an includes subquery, including the child pipeline * and metadata needed to route child results to parent-scoped Collections. @@ -637,14 +607,14 @@ export function compileQuery( condition: compileExpression(guard.condition), expected: guard.expected, })) + const compiledProjections: Array = + subquery.parentProjection?.map((ref) => ({ + alias: ref.path[0]!, + field: ref.path.slice(1), + compiled: compileExpression(ref), + })) ?? [] let parentKeys: any - if (subquery.parentProjection && subquery.parentProjection.length > 0) { - const compiledProjections: Array = - subquery.parentProjection.map((ref) => ({ - alias: ref.path[0]!, - field: ref.path.slice(1), - compiled: compileExpression(ref), - })) + if (compiledProjections.length > 0) { parentKeys = pipeline.pipe( map(([_key, nsRow]: any) => { if (!matchesConditionalSelectGuards(compiledGuards, nsRow)) { @@ -807,13 +777,7 @@ export function compileQuery( }) // Capture routing function for INCLUDES_ROUTING tagging - if (subquery.parentProjection && subquery.parentProjection.length > 0) { - const compiledProjs: Array = - subquery.parentProjection.map((ref) => ({ - alias: ref.path[0]!, - field: ref.path.slice(1), - compiled: compileExpression(ref), - })) + if (compiledProjections.length > 0) { const compiledCorr = compiledCorrelation const compiledRoutingGuards = compiledGuards includesRoutingFns.push({ @@ -826,7 +790,10 @@ export function compileQuery( parentContext: null, } } - const parentContext = projectParentContext(nsRow, compiledProjs) + const parentContext = projectParentContext( + nsRow, + compiledProjections, + ) return { active: true, correlationKey: compiledCorr(nsRow), @@ -918,10 +885,13 @@ export function compileQuery( // If no SELECT clause, create $selected with the main table data pipeline = pipeline.pipe( map(([key, namespacedRow]) => { + const routedScalar = getRoutedScalarMetadata(namespacedRow) const selectResults = - !isUnionFrom && !query.join && !query.groupBy - ? namespacedRow[mainSource] - : namespacedRow + isUnionFrom && routedScalar + ? routedScalar.value + : !isUnionFrom && !query.join && !query.groupBy + ? namespacedRow[mainSource] + : namespacedRow return [ key, @@ -1216,7 +1186,7 @@ function canonicalizeSelectedRows( value: row.$selected, routing: row.$selected?.[INCLUDES_ROUTING], outerCorrelation: isIncludedRelation - ? row[mainSource]?.__correlationKey + ? (row[mainSource]?.__correlationKey ?? row.__correlationKey) : undefined, parentContext: isIncludedRelation ? (row.__parentContext ?? row[mainSource]?.__parentContext ?? null) @@ -1617,6 +1587,28 @@ function wrapInputWithAlias( ): NamespacedAndKeyedStream { return input.pipe( map(([key, row]) => { + const inputRow: unknown = row + const scalar = getRoutedScalarMetadata(inputRow) + if (scalar) { + const nsRow = { + [alias]: scalar.value, + __correlationKey: scalar.correlationKey, + __parentContext: scalar.parentContext, + [INCLUDES_PUBLIC_KEY]: scalar.publicKey, + } as unknown as NamespacedRow + if ( + scalar.parentContext != null && + typeof scalar.parentContext === `object` + ) { + Object.assign(nsRow, scalar.parentContext) + } + return [key, nsRow] as [unknown, NamespacedRow] + } + + if (inputRow == null || typeof inputRow !== `object`) { + return [key, { [alias]: inputRow }] as [unknown, NamespacedRow] + } + // Initialize the record with a nested structure. // If __parentContext exists (from parent-referencing includes), merge parent // aliases into the namespaced row so WHERE can resolve parent refs. diff --git a/packages/db/src/query/compiler/joins.ts b/packages/db/src/query/compiler/joins.ts index 5d61f34b1f..45ce965878 100644 --- a/packages/db/src/query/compiler/joins.ts +++ b/packages/db/src/query/compiler/joins.ts @@ -20,6 +20,12 @@ import { normalizeValue } from '../../utils/comparison.js' import { ensureIndexForField } from '../../indexes/auto-index.js' import { compileExpression } from './evaluators.js' import { getLazyLoadTargets } from './lazy-targets.js' +import { crossJoinParentRoutes } from './parent-routes.js' +import { + INCLUDES_PUBLIC_KEY, + attachRouteMetadataToResult, + getRoutedScalarMetadata, +} from './route-metadata.js' import type { CompileQueryFn } from './index.js' import type { OrderByOptimizationInfo } from './order-by.js' import type { @@ -53,43 +59,49 @@ export type LazyCollectionCallbacks = { } let nextLazyDemandPlanId = 0 -const PARENT_ROUTE_CROSS_KEY = `__tanstack_join_parent_route_cross__` function parameterizeJoinInputByParentRoutes( input: KeyedStream, parentKeyStream: KeyedStream, ): KeyedStream { - const rows: any = input.pipe( - map(([rowKey, row]) => [PARENT_ROUTE_CROSS_KEY, [rowKey, row]]), - ) - const routes: any = parentKeyStream.pipe( - map(([correlationKey, parentContext]) => [ - PARENT_ROUTE_CROSS_KEY, - [correlationKey, parentContext], - ]), - ) - - return rows.pipe( - joinOperator(routes, `inner`), - filter(([, [rowSide, routeSide]]: any) => - Boolean(rowSide != null && routeSide != null), - ), - map(([, [rowSide, routeSide]]: any) => { - const [rowKey, row] = rowSide - const [correlationKey, parentContext] = routeSide + return crossJoinParentRoutes( + input, + parentKeyStream, + (rowKey, row, correlationKey, parentContext) => { return [ serializeValue([rowKey, correlationKey, parentContext]), { - ...row, + ...(row as Record), __correlationKey: correlationKey, __parentContext: parentContext, }, ] - }), - ) as KeyedStream + }, + ) } function wrapJoinedInputRow(alias: string, row: any): NamespacedRow { + const scalar = getRoutedScalarMetadata(row) + if (scalar) { + const namespaced = { + [alias]: scalar.value, + __correlationKey: scalar.correlationKey, + __parentContext: scalar.parentContext, + [INCLUDES_PUBLIC_KEY]: scalar.publicKey, + } as unknown as NamespacedRow + if ( + scalar.parentContext != null && + typeof scalar.parentContext === `object` + ) { + Object.assign(namespaced, scalar.parentContext) + } + return namespaced + } + + if (row == null || typeof row !== `object`) { + return { [alias]: row } + } + const { __parentContext, ...cleanRow } = row const namespaced: NamespacedRow = { [alias]: cleanRow } if (__parentContext != null) { @@ -649,20 +661,21 @@ function processJoinSource( // We need to extract just the value for use in parent queries const extractedInput = subQueryInput.pipe( map((data: any) => { - const [key, [value, _orderByIndex, correlationKey, parentContext]] = - data + const [ + key, + [value, _orderByIndex, correlationKey, parentContext, publicKey], + ] = data if (!parentKeyStream) { return [key, value] as [unknown, any] } return [ key, - value != null && typeof value === `object` - ? { - ...value, - __correlationKey: correlationKey, - __parentContext: parentContext, - } - : value, + attachRouteMetadataToResult( + value, + correlationKey, + parentContext, + publicKey, + ), ] as [unknown, any] }), ) diff --git a/packages/db/src/query/compiler/parent-routes.ts b/packages/db/src/query/compiler/parent-routes.ts new file mode 100644 index 0000000000..b6eddb32f9 --- /dev/null +++ b/packages/db/src/query/compiler/parent-routes.ts @@ -0,0 +1,40 @@ +import { filter, join as joinOperator, map } from '@tanstack/db-ivm' +import type { KeyedStream } from '../../types.js' + +const PARENT_ROUTE_CROSS_KEY = `__tanstack_parent_route_cross__` + +export function crossJoinParentRoutes( + input: KeyedStream, + parentKeyStream: KeyedStream, + assemble: ( + rowKey: unknown, + row: unknown, + correlationKey: unknown, + parentContext: unknown, + ) => [unknown, unknown], +): KeyedStream { + // Recursive sources need their route before a correlation field is always + // available. The constant key intentionally creates one copy of each input + // row per active route; callers filter those copies once the field is visible. + const rows: any = input.pipe( + map(([rowKey, row]) => [PARENT_ROUTE_CROSS_KEY, [rowKey, row]]), + ) + const routes: any = parentKeyStream.pipe( + map(([correlationKey, parentContext]) => [ + PARENT_ROUTE_CROSS_KEY, + [correlationKey, parentContext], + ]), + ) + + return rows.pipe( + joinOperator(routes, `inner`), + filter(([, [rowSide, routeSide]]: any) => + Boolean(rowSide != null && routeSide != null), + ), + map(([, [rowSide, routeSide]]: any) => { + const [rowKey, row] = rowSide + const [correlationKey, parentContext] = routeSide + return assemble(rowKey, row, correlationKey, parentContext) + }), + ) as KeyedStream +} diff --git a/packages/db/src/query/compiler/route-metadata.ts b/packages/db/src/query/compiler/route-metadata.ts new file mode 100644 index 0000000000..703c7bd037 --- /dev/null +++ b/packages/db/src/query/compiler/route-metadata.ts @@ -0,0 +1,67 @@ +const ROUTED_SCALAR_VALUE = Symbol(`tanstack_db_routed_scalar_value`) +export const INCLUDES_PUBLIC_KEY = Symbol(`includesPublicKey`) + +type RoutedScalarResult = { + [ROUTED_SCALAR_VALUE]: unknown + __correlationKey: unknown + __parentContext: unknown + [INCLUDES_PUBLIC_KEY]: unknown +} + +export type RoutedScalarMetadata = { + value: unknown + correlationKey: unknown + parentContext: unknown + publicKey: unknown +} + +export function attachRouteMetadataToResult( + value: unknown, + correlationKey: unknown, + parentContext: unknown, + publicKey: unknown, +): unknown { + if ( + correlationKey === undefined && + parentContext === undefined && + publicKey === undefined + ) { + return value + } + + if (value != null && typeof value === `object`) { + return { + ...value, + __correlationKey: correlationKey, + __parentContext: parentContext, + [INCLUDES_PUBLIC_KEY]: publicKey, + } + } + + return { + [ROUTED_SCALAR_VALUE]: value, + __correlationKey: correlationKey, + __parentContext: parentContext, + [INCLUDES_PUBLIC_KEY]: publicKey, + } satisfies RoutedScalarResult +} + +export function getRoutedScalarMetadata( + value: unknown, +): RoutedScalarMetadata | undefined { + if ( + value == null || + typeof value !== `object` || + !(ROUTED_SCALAR_VALUE in value) + ) { + return undefined + } + + const routed = value as RoutedScalarResult + return { + value: routed[ROUTED_SCALAR_VALUE], + correlationKey: routed.__correlationKey, + parentContext: routed.__parentContext, + publicKey: routed[INCLUDES_PUBLIC_KEY], + } +} diff --git a/packages/db/src/query/compiler/select.ts b/packages/db/src/query/compiler/select.ts index 88ec842915..de428ff515 100644 --- a/packages/db/src/query/compiler/select.ts +++ b/packages/db/src/query/compiler/select.ts @@ -156,6 +156,16 @@ export function processSelect( select: Select, _allInputs: Record, ): NamespacedAndKeyedStream { + if (!isNestedSelectObject(select)) { + const compiled = compileSelectValue(select as SelectValueExpression) + return pipeline.pipe( + map(([key, namespacedRow]) => [ + key, + { ...namespacedRow, $selected: compiled(namespacedRow) }, + ]), + ) as NamespacedAndKeyedStream + } + // Build ordered operations to preserve authoring order (spreads and fields) const ops: Array = [] diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index a79a798f13..ad44e21ae5 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -154,6 +154,7 @@ The route-context grammar crosses these dimensions: ```text lexical dependency scope x recursive source boundary (nested include, QueryRef, union) + x recursive result shape (record, scalar, nullable scalar) x evaluation phase (filter, join, group, aggregate, order, window) x join side and correlation attachment point x materialization form @@ -166,10 +167,20 @@ joined subquery, or union branch transports the same context. The executable oracle factors that product into valid compiler sub-grammars: +- parent field projection by whole-row projection; +- unmatched correlation values by null correlation values; - lexical scope, including nested outer and inner materialization forms; - grouping mode by aggregate-expression placement; - recursive source boundary by evaluation phase; and -- join-key side by correlation attachment point. +- join-key side by correlation attachment point; +- union form and public-key identity; and +- derived-result boundary by selection mode and scalar nullability. + +Objects carry route metadata as hidden fields while the compiler moves them +through recursive sources. Scalars, including `null`, cannot carry fields, so +the compiler uses an internal envelope at those same edges. Namespacing and +join adapters unwrap the value, keep the route beside it, and never expose the +envelope in the public query result. Every valid plan is checked as a Collection, `toArray`, and `materialize` include at initial load, after a parent-route update, and after a child update. diff --git a/packages/db/tests/query/includes-context-transport-oracle.test.ts b/packages/db/tests/query/includes-context-transport-oracle.test.ts index d202c81644..40d12d1ddf 100644 --- a/packages/db/tests/query/includes-context-transport-oracle.test.ts +++ b/packages/db/tests/query/includes-context-transport-oracle.test.ts @@ -61,6 +61,11 @@ const routeContextGrammar = { unionIdentity: { forms: [`union-from`, `union-all`] as const, }, + derivedResult: { + boundaries: [`from-query-ref`, `joined-query-ref`, `union-all`] as const, + selections: [`expression`, `functional`] as const, + domains: [`non-null`, `nullable`] as const, + }, } as const const queryRefMetadataGrammar = { @@ -107,6 +112,13 @@ type UnionIdentityCell = { form: (typeof routeContextGrammar.unionIdentity.forms)[number] } +type DerivedResultCell = { + family: `derived-result` + boundary: (typeof routeContextGrammar.derivedResult.boundaries)[number] + selection: (typeof routeContextGrammar.derivedResult.selections)[number] + domain: (typeof routeContextGrammar.derivedResult.domains)[number] +} + type GrammarCell = | ParentProjectionCell | CorrelationDomainCell @@ -115,6 +127,7 @@ type GrammarCell = | RecursiveSourceCell | JoinCell | UnionIdentityCell + | DerivedResultCell const grammarCells: Array = [ ...routeContextGrammar.parentProjection.shapes.map( @@ -162,6 +175,18 @@ const grammarCells: Array = [ ...routeContextGrammar.unionIdentity.forms.map( (form): UnionIdentityCell => ({ family: `union-identity`, form }), ), + ...routeContextGrammar.derivedResult.boundaries.flatMap((boundary) => + routeContextGrammar.derivedResult.selections.flatMap((selection) => + routeContextGrammar.derivedResult.domains.map( + (domain): DerivedResultCell => ({ + family: `derived-result`, + boundary, + selection, + domain, + }), + ), + ), + ), ] async function cleanup( @@ -230,6 +255,8 @@ function grammarCellName(cell: GrammarCell): string { return `${cell.family} / ${cell.keySide}-side key / ${cell.correlationAttachment}-side correlation` case `union-identity`: return `${cell.family} / ${cell.form}` + case `derived-result`: + return `${cell.family} / ${cell.boundary} / ${cell.selection} / ${cell.domain}` } } @@ -274,19 +301,14 @@ async function runParentProjectionCell({ const parent = parents.collection.get(parentId)! return children.collection.toArray .filter((child) => child.parentGroup === parent.group) - .map(({ id }) => ({ id, token: parent.token })) + .map(({ id }) => ({ + id, + parentSnapshot: + shape === `field` ? { token: parent.token } : { ...parent }, + })) } const project = (rows: Iterable<{ id: number; parentSnapshot: unknown }>) => - [...rows].map(({ id, parentSnapshot }) => { - const token = - parentSnapshot != null && - typeof parentSnapshot === `object` && - `token` in parentSnapshot && - typeof parentSnapshot.token === `string` - ? parentSnapshot.token - : undefined - return { id, token } - }) + [...rows].map(({ id, parentSnapshot }) => ({ id, parentSnapshot })) const assertParents = () => { for (const parent of parents.collection.toArray) { expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) @@ -1084,6 +1106,175 @@ async function runUnionIdentityCell({ } } +type DerivedCandidateRow = { + id: number + value: number | null +} + +async function runDerivedResultCell({ + boundary, + selection, + domain, +}: DerivedResultCell): Promise { + const name = `${boundary}-${selection}-${domain}` + const parents = createGrammarCollection(`derived-${name}-parents`, [ + { id: 1, group: 1 }, + { id: 2, group: 2 }, + ]) + const initialCandidates: Array = [ + { id: 10, value: 10 }, + { id: 15, value: domain === `nullable` ? null : 15 }, + { id: 20, value: 20 }, + ] + const candidates = createGrammarCollection( + `derived-${name}-candidates`, + initialCandidates, + ) + const left = createGrammarCollection( + `derived-${name}-left`, + initialCandidates.filter(({ id }) => id !== 20), + ) + const right = createGrammarCollection( + `derived-${name}-right`, + initialCandidates.filter(({ id }) => id === 20), + ) + const anchors = createGrammarCollection(`derived-${name}-anchors`, [ + { id: 100, parentGroup: 1, value: 10 }, + { id: 200, parentGroup: 2, value: 20 }, + { id: 300, parentGroup: 2, value: 30 }, + ]) + + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => { + if (boundary === `union-all`) { + const leftRows = q.from({ leftCandidate: left.collection }) + const rightRows = q.from({ rightCandidate: right.collection }) + const values = + selection === `expression` + ? q.unionAll( + leftRows.select(({ leftCandidate }) => + coalesce(leftCandidate.value, null), + ), + rightRows.select(({ rightCandidate }) => + coalesce(rightCandidate.value, null), + ), + ) + : q.unionAll( + leftRows.fn.select( + ({ leftCandidate }) => leftCandidate.value, + ), + rightRows.fn.select( + ({ rightCandidate }) => rightCandidate.value, + ), + ) + const rows = q + .from({ anchor: anchors.collection }) + .innerJoin({ value: values }, ({ anchor, value }) => + eq(anchor.value, value), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ anchor, value }) => ({ id: anchor.id, value })) + return { id: parent.id, ...includeInEveryForm(rows) } + } + + if (selection === `expression`) { + const values = q + .from({ candidate: candidates.collection }) + .select(({ candidate }) => coalesce(candidate.value, null)) + if (boundary === `from-query-ref`) { + const rows = q + .from({ value: values }) + .innerJoin({ anchor: anchors.collection }, ({ value, anchor }) => + eq(value, anchor.value), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ anchor, value }) => ({ id: anchor.id, value })) + return { id: parent.id, ...includeInEveryForm(rows) } + } + + const rows = q + .from({ anchor: anchors.collection }) + .innerJoin({ value: values }, ({ anchor, value }) => + eq(anchor.value, value), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ anchor, value }) => ({ id: anchor.id, value })) + return { id: parent.id, ...includeInEveryForm(rows) } + } + + const values = q + .from({ candidate: candidates.collection }) + .fn.select(({ candidate }) => candidate.value) + if (boundary === `from-query-ref`) { + const rows = q + .from({ value: values }) + .innerJoin({ anchor: anchors.collection }, ({ value, anchor }) => + eq(value, anchor.value), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ anchor, value }) => ({ id: anchor.id, value })) + return { id: parent.id, ...includeInEveryForm(rows) } + } + + const rows = q + .from({ anchor: anchors.collection }) + .innerJoin({ value: values }, ({ anchor, value }) => + eq(anchor.value, value), + ) + .where(({ anchor }) => eq(anchor.parentGroup, parent.group)) + .select(({ anchor, value }) => ({ id: anchor.id, value })) + return { id: parent.id, ...includeInEveryForm(rows) } + }), + ) + + const modelRows = new Map(initialCandidates.map((row) => [row.id, row])) + const expected = (parentId: number) => { + const parent = parents.collection.get(parentId)! + return [...modelRows.values()] + .flatMap((candidate) => + anchors.collection.toArray + .filter( + (anchor) => + candidate.value != null && + anchor.value === candidate.value && + anchor.parentGroup === parent.group, + ) + .map((anchor) => ({ id: anchor.id, value: candidate.value })), + ) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + } + const project = ( + rows: Iterable<{ id: number; value: number | null | undefined }>, + ) => + [...rows] + .map(({ id, value }) => ({ id, value })) + .sort((leftRow, rightRow) => leftRow.id - rightRow.id) + const assertParents = () => { + for (const parent of parents.collection.toArray) { + expectEveryForm(live.get(parent.id)!, project, expected(parent.id)) + } + } + + try { + await live.preload() + assertParents() + + parents.write(`update`, { id: 1, group: 2 }) + assertParents() + + const updated = { id: 20, value: 30 } + modelRows.set(updated.id, updated) + if (boundary === `union-all`) right.write(`update`, updated) + else candidates.write(`update`, updated) + assertParents() + } finally { + await cleanup(live, [parents, candidates, left, right, anchors]) + } +} + function expectNoRouteMetadata(row: object): void { expect(Object.hasOwn(row, `__correlationKey`)).toBe(false) expect(Object.hasOwn(row, `__parentContext`)).toBe(false) @@ -1283,6 +1474,8 @@ async function runGrammarCell(cell: GrammarCell): Promise { return runJoinCell(cell) case `union-identity`: return runUnionIdentityCell(cell) + case `derived-result`: + return runDerivedResultCell(cell) } } @@ -1298,14 +1491,17 @@ describe(`correlated include route-context transport grammar`, () => { routeContextGrammar.recursiveSource.phases.length + routeContextGrammar.join.keySides.length * routeContextGrammar.join.correlationAttachments.length + - routeContextGrammar.unionIdentity.forms.length + routeContextGrammar.unionIdentity.forms.length + + routeContextGrammar.derivedResult.boundaries.length * + routeContextGrammar.derivedResult.selections.length * + routeContextGrammar.derivedResult.domains.length const names = grammarCells.map(grammarCellName) expect(grammarCells).toHaveLength(expectedCellCount) expect(new Set(names)).toHaveLength(expectedCellCount) expect( grammarCells.length * materializationForms.length * checkpoints.length, - ).toBe(279) + ).toBe(387) }) for (const cell of grammarCells) {