From 3b81de816e989b70cf3da179545e16ffb0f4d674 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 17 Oct 2026 11:03:57 -0700 Subject: [PATCH 1/4] feat: richer column value profiles in schema_inspect (ranked values, uniqueness, value shapes, MongoDB paths) - SQL profile (sqlite, duckdb, postgres) stays one query per table but now returns frequency-ranked values with counts; high-cardinality text shows the 3 most frequent values instead of arbitrary ones - mark columns whose distinct count equals their non-null count as `unique in sample` - report the top value shapes of informative text columns (`H01M4/9066` -> `A9+A9/9+`), skipping free text and plain single-shape words - show one ~160 character example for long text columns - MongoDB: the driver reports nested dotted paths (depth <= 3, `field[]` for arrays of objects) with a presence share, exposes `sampleDocuments`, and `schema_inspect` builds the same value profile from the sampled documents Co-Authored-By: Claude Opus 5.5 --- packages/drivers/src/mongodb.ts | 102 ++++++- packages/drivers/src/types.ts | 4 + packages/drivers/test/mongodb-unit.test.ts | 46 ++- .../altimate/native/connections/register.ts | 1 + .../opencode/src/altimate/native/types.ts | 2 + .../src/altimate/tools/schema-inspect.ts | 7 +- .../src/altimate/tools/schema-values.ts | 281 ++++++++++++++++-- .../tools/schema-inspect-mongodb.test.ts | 36 +++ .../tools/schema-inspect-values.test.ts | 195 +++++++++++- 9 files changed, 613 insertions(+), 61 deletions(-) diff --git a/packages/drivers/src/mongodb.ts b/packages/drivers/src/mongodb.ts index 1c0c59bde9..83b4ac52ff 100644 --- a/packages/drivers/src/mongodb.ts +++ b/packages/drivers/src/mongodb.ts @@ -245,6 +245,60 @@ function extractFields(docs: Record[]): Map return fieldTypes } +/** Deepest dotted path reported by `extractNestedPaths`, counting `field[]` as part of its segment. */ +const NESTED_PATH_MAX_DEPTH = 3 +/** Cap on nested paths reported per collection. */ +const NESTED_PATH_MAX_COUNT = 100 +/** Documents sampled to infer a collection schema. */ +const SAMPLE_DOCUMENTS = 100 + +export interface NestedPath { + types: Set + /** Sampled documents containing the path. */ + docs: number +} + +function isPlainObject(value: unknown): value is Record { + if (typeof value !== "object" || value === null) return false + const proto = Object.getPrototypeOf(value) + return proto === Object.prototype || proto === null +} + +/** + * Extract dotted paths below the top level (`a.b`, `items[].sku`) up to depth 3, with + * the types seen and the number of documents containing each path. Arrays of objects + * contribute `field[]` followed by the element's keys. + */ +export function extractNestedPaths(docs: Record[]): Map { + const paths = new Map() + for (const doc of docs) { + const seen = new Set() + const visit = (value: unknown, path: string, depth: number) => { + if (depth >= NESTED_PATH_MAX_DEPTH) return + if (isPlainObject(value)) { + for (const [key, child] of Object.entries(value)) record(child, `${path}.${key}`, depth + 1) + } else if (Array.isArray(value)) { + for (const element of value) { + if (!isPlainObject(element)) continue + for (const [key, child] of Object.entries(element)) record(child, `${path}[].${key}`, depth + 1) + } + } + } + const record = (value: unknown, path: string, depth: number) => { + const entry = paths.get(path) ?? { types: new Set(), docs: 0 } + entry.types.add(inferType(value)) + if (!seen.has(path)) { + seen.add(path) + entry.docs++ + } + paths.set(path, entry) + visit(value, path, depth) + } + for (const [key, value] of Object.entries(doc)) visit(value, key, 1) + } + return paths +} + /** Server error code for an operation stopped by `maxTimeMS`. */ const MONGO_MAX_TIME_EXPIRED = 50 @@ -355,6 +409,17 @@ export async function connect(config: ConnectionConfig): Promise { return JSON.stringify(val) } + async function sampleDocuments(schema: string, table: string, limit = SAMPLE_DOCUMENTS): Promise[]> { + // altimate_change start — format the existing cursor chain + return client + .db(schema) + .collection(table) + .find({}, withMaxTime(resolveQueryTimeout(config).ms)) + .limit(limit) + .toArray() + // altimate_change end + } + const connector: Connector & { // altimate_change start runCommand( @@ -873,17 +938,11 @@ export async function connect(config: ConnectionConfig): Promise { .sort((a: { name: string }, b: { name: string }) => a.name.localeCompare(b.name)) }, - async describeTable(schema: string, table: string): Promise { - const db = client.db(schema) - const coll = db.collection(table) + sampleDocuments, + async describeTable(schema: string, table: string): Promise { // Sample up to 100 documents to infer schema - // altimate_change start — format the existing cursor chain - const docs = await coll - .find({}, withMaxTime(resolveQueryTimeout(config).ms)) - .limit(100) - .toArray() - // altimate_change end + const docs = await sampleDocuments(schema, table) if (docs.length === 0) { return [] @@ -898,13 +957,15 @@ export async function connect(config: ConnectionConfig): Promise { } } + const joinTypes = (types: Iterable) => { + const nonNullTypes = Array.from(types).filter((t) => t !== "null") + return nonNullTypes.length === 0 ? "null" : nonNullTypes.length === 1 ? nonNullTypes[0] : nonNullTypes.join(" | ") + } + const columns: SchemaColumn[] = [] for (const [name, types] of fieldTypes) { - const typeArr = Array.from(types) - const hasNull = typeArr.includes("null") - const nonNullTypes = typeArr.filter((t) => t !== "null") - const dataType = - nonNullTypes.length === 0 ? "null" : nonNullTypes.length === 1 ? nonNullTypes[0] : nonNullTypes.join(" | ") + const hasNull = types.has("null") + const dataType = joinTypes(types) // Field is nullable if it has null values OR is missing from some documents const presentIn = fieldPresence.get(name) ?? 0 @@ -917,7 +978,18 @@ export async function connect(config: ConnectionConfig): Promise { }) } - // Sort: _id first, then alphabetical + const nested = [...extractNestedPaths(docs)].sort(([a], [b]) => a.localeCompare(b)).slice(0, NESTED_PATH_MAX_COUNT) + for (const [name, info] of nested) { + const presence = info.docs / docs.length + columns.push({ + name, + data_type: joinTypes(info.types), + nullable: info.types.has("null") || presence < 1, + presence, + }) + } + + // Sort: _id first, then alphabetical (nested paths land under their parent) columns.sort((a, b) => { if (a.name === "_id") return -1 if (b.name === "_id") return 1 diff --git a/packages/drivers/src/types.ts b/packages/drivers/src/types.ts index 31a575514e..80d01c7c62 100644 --- a/packages/drivers/src/types.ts +++ b/packages/drivers/src/types.ts @@ -18,6 +18,8 @@ export interface SchemaColumn { name: string data_type: string nullable: boolean + /** Share (0-1) of sampled documents containing this field; set on MongoDB nested paths. */ + presence?: number } export interface SchemaMetadata { @@ -135,6 +137,8 @@ export interface Connector { listSchemas(): Promise listTables(schema: string): Promise> describeTable(schema: string, table: string): Promise + /** Document stores only: the raw documents `describeTable` infers its schema from. */ + sampleDocuments?(schema: string, table: string, limit?: number): Promise[]> introspectSchema?(signal?: AbortSignal): Promise close(): Promise } diff --git a/packages/drivers/test/mongodb-unit.test.ts b/packages/drivers/test/mongodb-unit.test.ts index 5fe0b2a7d5..2594d357d4 100644 --- a/packages/drivers/test/mongodb-unit.test.ts +++ b/packages/drivers/test/mongodb-unit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, mock, test } from "bun:test" import { createRequire } from "node:module" -import { normalizeMqlCommand } from "../src/mongodb" +import { extractNestedPaths, normalizeMqlCommand } from "../src/mongodb" import { restoreModuleMocksAfterTests } from "./module-mock" await restoreModuleMocksAfterTests("mongodb") @@ -460,3 +460,47 @@ describe("MongoDB native command filters", () => { expect(counts).toEqual([{ filter: { status: "B" }, options: { maxTimeMS: 120_000 } }]) }) }) + +describe("MongoDB nested field paths", () => { + const docs = [ + { _id: 1, user: { name: "a", geo: { lat: 1.5, deep: { too: "far" } } }, items: [{ sku: "x", qty: 1 }, { sku: "y" }], tags: ["t"] }, + { _id: 2, user: { name: "b" }, items: [{ sku: "z", qty: null }], tags: [] }, + { _id: 3, user: null, items: [] }, + ] + + test("extracts dotted and array paths to depth 3 with types and per-document presence", () => { + const paths = extractNestedPaths(docs) + expect([...paths.keys()].sort()).toEqual([ + "items[].qty", + "items[].sku", + "user.geo", + "user.geo.deep", + "user.geo.lat", + "user.name", + ]) + expect(paths.get("user.geo.deep")!.types).toEqual(new Set(["object"])) + // sku appears twice in document 1 but the document is counted once + expect(paths.get("items[].sku")!.docs).toBe(2) + expect([...paths.get("items[].qty")!.types].sort()).toEqual(["int32", "null"]) + expect(paths.get("user.name")!.docs).toBe(2) + }) + + test("describeTable keeps top-level columns and adds nested paths with a presence share", async () => { + const { connector } = await fakeMongo(docs) + const columns = await connector.describeTable("test", "articles") + const byName = Object.fromEntries(columns.map((c) => [c.name, c])) + expect(byName.user).toEqual({ name: "user", data_type: "object", nullable: true }) + expect(byName.items).toEqual({ name: "items", data_type: "array", nullable: false }) + expect(byName["user.name"]).toEqual({ name: "user.name", data_type: "string", nullable: true, presence: 2 / 3 }) + expect(byName["items[].sku"]).toMatchObject({ data_type: "string", presence: 2 / 3 }) + expect(byName["items[].qty"]).toMatchObject({ data_type: "int32", nullable: true }) + expect(columns[0]!.name).toBe("_id") + const names = columns.map((c) => c.name) + expect(names.slice(names.indexOf("user"))).toEqual(["user", "user.geo", "user.geo.deep", "user.geo.lat", "user.name"]) + }) + + test("sampleDocuments returns the sampled documents", async () => { + const { connector } = await fakeMongo(docs) + expect(await connector.sampleDocuments!("test", "articles", 50)).toEqual(docs) + }) +}) diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index a763ef0036..ebd120f8a3 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -823,6 +823,7 @@ register("schema.inspect", async (params: SchemaInspectParams): Promise): string { lines.push("Column | Type | Nullable | PK") lines.push("-------|------|----------|---") for (const col of result.columns ?? []) { + // MongoDB nested paths carry the share of sampled documents that contain them. + const presence = col.presence !== undefined && col.presence < 1 ? ` (in ${Math.round(col.presence * 100)}% of docs)` : "" lines.push( - `${col.name} | ${col.data_type ?? "unknown"} | ${col.nullable ? "YES" : "NO"} | ${col.primary_key ? "YES" : ""}`, + `${col.name} | ${col.data_type ?? "unknown"}${presence} | ${col.nullable ? "YES" : "NO"} | ${col.primary_key ? "YES" : ""}`, ) } return lines.join("\n") diff --git a/packages/opencode/src/altimate/tools/schema-values.ts b/packages/opencode/src/altimate/tools/schema-values.ts index 7cf7e1a9d1..445d556f83 100644 --- a/packages/opencode/src/altimate/tools/schema-values.ts +++ b/packages/opencode/src/altimate/tools/schema-values.ts @@ -1,12 +1,15 @@ /** * Value grounding for `schema_inspect`: one bounded profiling query per table that - * returns, per column, the distinct count, null share and a few real values. The - * values are what reveal stored formats (case, whitespace, prefixes) that a column - * name and type never show. + * returns, per column, the distinct count, null share and the most frequent real + * values with their counts. The values are what reveal stored formats (case, + * whitespace, prefixes) that a column name and type never show; uniqueness and the + * value shape of text columns show keys and code formats at a glance. MongoDB + * collections get the same profile, computed in TypeScript from sampled documents. */ import type { LegacyContext } from "../tool-zod-compat" import * as Precedence from "../workspace/precedence" import { Dispatcher } from "../native" +import * as Registry from "../native/connections/registry" import { isRecord, normalizeError } from "./response-normalization" /** Rows scanned per table; keeps wide or large tables cheap. */ @@ -17,6 +20,18 @@ export const PROFILE_MAX_COLUMNS = 40 export const PROFILE_LOW_CARDINALITY = 20 const PROFILE_EXAMPLES = 3 const PROFILE_VALUE_CHARS = 40 +/** Frequency-ranked values fetched per column; feeds examples, counts and shapes. */ +const PROFILE_FETCHED_VALUES = 50 +/** Fetched values are clipped to this many characters, bounding transfer for long text. */ +const PROFILE_FETCH_CHARS = 200 +/** Text whose typical value exceeds the normal clip shows one longer example. */ +const PROFILE_LONG_EXAMPLE_CHARS = 160 +/** Above this average length, or this many distinct shapes, text is free text and gets no shape. */ +const PROFILE_SHAPE_MAX_AVG_CHARS = 30 +const PROFILE_SHAPE_MAX_VARIETY = 5 +const PROFILE_SHAPES_SHOWN = 3 +/** Documents sampled for MongoDB collections. */ +export const PROFILE_SAMPLE_DOCUMENTS = 100 const PROFILE_TIMEOUT_MS = 15_000 export type ProfileDialect = "sqlite" | "duckdb" | "postgres" @@ -33,13 +48,23 @@ export interface ProfileColumn { data_type?: string } +interface ValueCount { + value: string + count: number +} + interface ColumnProfile { name: string total: number nonNull: number distinct: number - values: string[] + /** Most frequent values; sorted by `rank` before use. */ + values: ValueCount[] text: boolean + /** Set for values that can repeat inside one row (array elements), where uniqueness is meaningless. */ + multi?: boolean + /** Column type suggests a format already (uuid), so a shape adds nothing. */ + typed?: boolean } const quote = (id: string) => `"${id.replace(/"/g, '""')}"` @@ -57,23 +82,25 @@ export function buildProfileSql(table: string, columns: ProfileColumn[], dialect const aggregate = dialect === "sqlite" ? "json_group_array" : "array_agg" const branches = columns.map((c) => { const col = `CAST(${quote(c.name)} AS TEXT)` + const clipped = `SUBSTR(${col}, 1, ${PROFILE_FETCH_CHARS})` const lit = `'${c.name.replace(/'/g, "''")}'` + // Each entry is ":"; the count is digits only, so the first colon splits it. return ( `SELECT ${lit} AS col, COUNT(*) AS total, COUNT(${col}) AS nonnull, COUNT(DISTINCT ${col}) AS ndistinct, ` + - `(SELECT ${aggregate}(v) FROM (SELECT DISTINCT ${col} AS v FROM s WHERE ${col} IS NOT NULL ` + - `LIMIT ${PROFILE_LOW_CARDINALITY + 1}) AS sampled_values) AS vals FROM s` + `(SELECT ${aggregate}(CAST(pn AS TEXT) || ':' || pv) FROM (SELECT ${clipped} AS pv, COUNT(*) AS pn FROM s ` + + `WHERE ${col} IS NOT NULL GROUP BY ${clipped} ORDER BY pn DESC, pv LIMIT ${PROFILE_FETCHED_VALUES}) AS top_values) AS vals FROM s` ) }) return `${sample}\n${branches.join("\nUNION ALL\n")}` } -function clipRaw(value: string): string { +function clipTo(value: string, max: number): string { const chars = Array.from(value) - return chars.length > PROFILE_VALUE_CHARS ? chars.slice(0, PROFILE_VALUE_CHARS).join("") + "…" : value + return chars.length > max ? chars.slice(0, max).join("") + "…" : value } function clip(value: string): string { - const text = clipRaw(value) + const text = clipTo(value, PROFILE_VALUE_CHARS) // Quote anything whose edges would otherwise be invisible. return text === "" || text !== text.trim() || /[\r\n\t]/.test(text) ? JSON.stringify(text) : text } @@ -83,13 +110,15 @@ function num(value: unknown): number { return Number.isFinite(n) ? n : 0 } -function parseValues(value: unknown): string[] { +function parseValues(value: unknown): ValueCount[] { // SQLite returns JSON text; DuckDB/Postgres return native arrays (or null when empty). - const values: unknown = typeof value === "string" ? JSON.parse(value) : value ?? [] - if (!Array.isArray(values) || !values.every((v) => typeof v === "string")) { - throw new Error("unexpected profiling values") - } - return values + const entries: unknown = typeof value === "string" ? JSON.parse(value) : value ?? [] + if (!Array.isArray(entries)) throw new Error("unexpected profiling values") + return entries.map((entry) => { + const match = typeof entry === "string" ? /^(\d+):([\s\S]*)$/.exec(entry) : null + if (!match) throw new Error("unexpected profiling values") + return { value: match[2]!, count: Number(match[1]) } + }) } function parseRows(rows: unknown[], columns: ProfileColumn[]): ColumnProfile[] { @@ -106,22 +135,178 @@ function parseRows(rows: unknown[], columns: ProfileColumn[]): ColumnProfile[] { distinct: num(row[3]), values: parseValues(row[4]), text: TEXTUAL.test(meta.data_type ?? ""), + typed: /uuid/i.test(meta.data_type ?? ""), }) } return out } -function renderProfile(p: ColumnProfile): string { - const sampled = p.total >= PROFILE_SAMPLE_ROWS +/** + * Character-class pattern of a string: upper letters A, lower letters a, digits 9; + * punctuation and spaces stay. Runs of one class collapse to a symbol plus "+", so + * `H01M4/9066` becomes `A9+A9/9+`. + */ +export function valueShape(value: string): string { + let out = "" + let prev = "" + let run = 0 + const flush = () => { + if (run > 1) out += "+" + run = 0 + prev = "" + } + for (const ch of value) { + const cls = /\p{Lu}/u.test(ch) ? "A" : /\p{L}/u.test(ch) ? "a" : /\p{Nd}/u.test(ch) ? "9" : "" + if (!cls) { + flush() + out += ch + } else if (cls === prev) { + run++ + } else { + flush() + out += cls + prev = cls + run = 1 + } + } + flush() + return out +} + +function rank(values: ValueCount[]): ValueCount[] { + return [...values].sort((a, b) => b.count - a.count || (a.value < b.value ? -1 : a.value > b.value ? 1 : 0)) +} + +function averageLength(values: ValueCount[]): number { + const weight = values.reduce((n, v) => n + v.count, 0) + if (weight === 0) return 0 + return values.reduce((n, v) => n + Array.from(v.value).length * v.count, 0) / weight +} + +/** Top shapes with their share of the fetched values, or "" when the shape is uninformative. */ +function renderShapes(values: ValueCount[], avgLength: number): string { + if (avgLength > PROFILE_SHAPE_MAX_AVG_CHARS) return "" + const shapes = new Map() + for (const v of values) { + const shape = valueShape(v.value) + shapes.set(shape, (shapes.get(shape) ?? 0) + v.count) + } + if (shapes.size > PROFILE_SHAPE_MAX_VARIETY) return "" + // A lone letters-only shape (`Aa+`) is just "words"; the type already says as much. + if (shapes.size === 1 && /^[Aa+]+$/.test([...shapes.keys()][0]!)) return "" + const weight = values.reduce((n, v) => n + v.count, 0) + return [...shapes] + .sort((a, b) => b[1] - a[1]) + .slice(0, PROFILE_SHAPES_SHOWN) + .map(([shape, n]) => (shapes.size === 1 ? `\`${shape}\`` : `\`${shape}\` ${Math.round((n / weight) * 100)}%`)) + .join(", ") +} + +function renderProfile(p: ColumnProfile, sampled: boolean): string { const nullPct = p.total > 0 ? Math.round(((p.total - p.nonNull) / p.total) * 100) : 0 const nulls = nullPct > 0 ? `, ${nullPct}% null` : p.total > p.nonNull ? `, <1% null` : "" if (p.nonNull === 0) return `${p.name}: all NULL` const approx = sampled ? "~" : "" + const unique = !p.multi && p.total > 1 && p.nonNull > 1 && p.distinct === p.nonNull + const head = `${p.name}: ${approx}${p.distinct.toLocaleString()} distinct${unique ? ", unique in sample" : ""}` + const values = rank(p.values) + // When every value occurs once the counts carry no information. + const counted = (text: string, v: ValueCount) => (unique ? text : `${text} (${v.count.toLocaleString()})`) if (p.distinct <= PROFILE_LOW_CARDINALITY) { - return `${p.name}: ${approx}${p.distinct} distinct — ${p.values.map(clip).join(", ")}${nulls}` + return `${head} — ${values.map((v) => counted(clip(v.value), v)).join(", ")}${nulls}` + } + if (!p.text || values.length === 0) return `${head}${nulls}` + const long = averageLength(values) > PROFILE_VALUE_CHARS + const examples = values + .slice(0, PROFILE_EXAMPLES) + .map((v, i) => counted(JSON.stringify(clipTo(v.value, long && i === 0 ? PROFILE_LONG_EXAMPLE_CHARS : PROFILE_VALUE_CHARS)), v)) + .join(", ") + const shapes = p.typed ? "" : renderShapes(values, averageLength(values)) + return `${head}, e.g. ${examples}${shapes ? `, shape ${shapes}` : ""}${nulls}` +} + +function formatProfiles(profiles: ColumnProfile[], header: string, skipped: number, sampled: boolean, notice?: string): string { + if (profiles.length === 0) return "" + return [ + ...(notice ? [notice] : []), + header, + ...profiles.map((p) => renderProfile(p, sampled)), + ...(skipped > 0 ? [`(${skipped} more columns not profiled)`] : []), + ].join("\n") +} + +function textOf(value: unknown): string | undefined { + if (typeof value === "string") return value + if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return String(value) + if (value instanceof Date) return Number.isNaN(value.getTime()) ? undefined : value.toISOString() + const bson = isRecord(value) ? value._bsontype : undefined + // ObjectId, Long, Decimal128 and friends print as their natural text; binary does not. + if (typeof bson === "string" && bson !== "Binary") return String(value) + return undefined +} + +/** Values at a dotted path as reported by the driver (`a.b`, `items[].sku`); missing paths yield none. */ +function valuesAt(doc: Record, path: string): unknown[] { + let current: unknown[] = [doc] + for (const segment of path.split(".")) { + const array = segment.endsWith("[]") + const key = array ? segment.slice(0, -2) : segment + const next: unknown[] = [] + for (const item of current) { + if (!isRecord(item)) continue + const value = item[key] + if (!array) next.push(value) + else if (Array.isArray(value)) next.push(...value) + } + current = next } - const examples = p.text && p.values.length ? `, e.g. ${p.values.slice(0, PROFILE_EXAMPLES).map((v) => JSON.stringify(clipRaw(v))).join(", ")}` : "" - return `${p.name}: ${approx}${p.distinct.toLocaleString()} distinct${examples}${nulls}` + return current +} + +const CONTAINER_ONLY = /^(object|array)$/ + +/** + * Value profile for sampled documents, in the same format as the SQL profile. + * Columns are the field paths the driver reported; containers and binary are skipped. + */ +export function profileDocuments(docs: Record[], columns: ProfileColumn[]): string { + const eligible = columns.filter((c) => !UNPROFILABLE.test(c.data_type ?? "") && !CONTAINER_ONLY.test(c.data_type ?? "")) + // Top-level fields come first so the column cap never crowds them out with nested paths. + const ordered = [...eligible.filter((c) => !/[.[]/.test(c.name)), ...eligible.filter((c) => /[.[]/.test(c.name))] + const cols = ordered.slice(0, PROFILE_MAX_COLUMNS) + const profiles = cols.map((c): ColumnProfile => { + const counts = new Map() + let nonNull = 0 + for (const doc of docs) { + let present = false + for (const raw of valuesAt(doc, c.name)) { + const text = textOf(raw) + if (text === undefined) continue + present = true + const key = text.slice(0, PROFILE_FETCH_CHARS) + counts.set(key, (counts.get(key) ?? 0) + 1) + } + if (present) nonNull++ + } + const values = rank([...counts].map(([value, count]) => ({ value, count }))) + return { + name: c.name, + total: docs.length, + nonNull, + distinct: counts.size, + values: values.slice(0, PROFILE_FETCHED_VALUES), + text: TEXTUAL.test(c.data_type ?? ""), + multi: c.name.includes("[]"), + typed: /uuid/i.test(c.data_type ?? ""), + } + }) + const sampled = docs.length >= PROFILE_SAMPLE_DOCUMENTS + return formatProfiles( + profiles, + `Value samples${sampled ? ` (first ${PROFILE_SAMPLE_DOCUMENTS} documents)` : ""}:`, + ordered.length - cols.length, + sampled, + ) } export interface ValueProfileInput { @@ -129,6 +314,8 @@ export interface ValueProfileInput { table: string columns: ProfileColumn[] warehouse?: string + /** Database and collection, used instead of `table` for MongoDB. */ + document?: { database: string; collection: string } } /** Never throws: a failed profile becomes a one-line note. */ @@ -144,6 +331,9 @@ export async function profileValues( }> const target = input.warehouse ? warehouses.find((w) => w.name === input.warehouse) : warehouses[0] const dialect = profileDialect(target?.type) + if (target?.name && input.document && Registry.canonicalType(target.type) === "mongodb") { + return await profileCollection(input, input.document, target.name, ctx) + } if (!target?.name || !dialect) { return `Value samples skipped: not supported for ${target?.type ?? "this"} connections.` } @@ -171,18 +361,53 @@ export async function profileValues( if (error) throw new Error(error) if (!isRecord(data) || !Array.isArray(data.rows)) throw new Error("unexpected profiling response") const profiles = parseRows(data.rows, cols) - const lines = profiles.map(renderProfile) - if (lines.length === 0) return "" const sampled = profiles.some((p) => p.total >= PROFILE_SAMPLE_ROWS) - const skipped = eligible.length - cols.length - return [ - ...(precedence.notice ? [precedence.notice] : []), + return formatProfiles( + profiles, `Value samples${sampled ? ` (first ${PROFILE_SAMPLE_ROWS.toLocaleString()} rows)` : ""}:`, - ...lines, - ...(skipped > 0 ? [`(${skipped} more columns not profiled)`] : []), - ].join("\n") + eligible.length - cols.length, + sampled, + precedence.notice, + ) } catch (e) { const msg = (e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, 160) return `Value samples unavailable: ${msg}` } } + +type ProfileContext = Pick & { abort?: AbortSignal } + +/** Read-only: samples the same documents `describeTable` infers its schema from. */ +async function profileCollection( + input: ValueProfileInput, + document: { database: string; collection: string }, + warehouse: string, + ctx: ProfileContext, +): Promise { + const connector = await Registry.get(warehouse) + if (!connector.sampleDocuments) return "Value samples skipped: not supported for mongodb connections." + await ctx.ask({ + permission: "sql_execute", + patterns: [ + JSON.stringify({ ...document, command: "find", limit: PROFILE_SAMPLE_DOCUMENTS }), + ], + always: ["*"], + metadata: { warehouse }, + }) + const precedence = await Precedence.check(ctx.sessionID, "sql_execute", warehouse) + if (precedence.redirect) return `Value samples skipped: ${precedence.redirect.output}` + let timer: ReturnType | undefined + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("sampling timed out")), PROFILE_TIMEOUT_MS) + }) + try { + const docs = await Promise.race([ + connector.sampleDocuments(document.database, document.collection, PROFILE_SAMPLE_DOCUMENTS), + deadline, + ]) + const profile = profileDocuments(docs, input.columns) + return profile && precedence.notice ? `${precedence.notice}\n${profile}` : profile + } finally { + clearTimeout(timer) + } +} diff --git a/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts b/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts index a791176f10..5a489822bf 100644 --- a/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts +++ b/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts @@ -75,4 +75,40 @@ describe("schema_inspect MongoDB collections", () => { expect(result.output).toContain('Collection "empty" has no documents to infer a schema') expect(result.output).not.toContain("was not found") }) + + test("profiles sampled documents instead of skipping, including nested paths", async () => { + const connector = configure("records") + const docs = Array.from({ length: 10 }, (_, i) => ({ + id: `e${i}`, + status: i < 7 ? "open" : "closed", + meta: { source: i % 2 ? "web" : "api" }, + })) + connector.describeTable.mockImplementation(async () => [ + { name: "id", data_type: "string", nullable: false }, + { name: "meta", data_type: "object", nullable: false }, + { name: "meta.source", data_type: "string", nullable: true, presence: 0.9 }, + { name: "status", data_type: "string", nullable: false }, + ]) + Object.assign(connector, { sampleDocuments: mock(async () => docs) }) + const tool = await initTool(SchemaInspectTool) + const result = await tool.execute({ warehouse: "documents", table: "records.events" }, ctx) + expect(result.metadata.success).toBe(true) + expect(result.output).not.toContain("skipped") + expect(result.output).toContain("meta.source | string (in 90% of docs) | YES") + expect(result.output).toContain("id: 10 distinct, unique in sample — e0, e1") + expect(result.output).toContain("status: 2 distinct — open (7), closed (3)") + expect(result.output).toContain("meta.source: 2 distinct — api (5), web (5)") + expect(result.output).not.toContain("\nmeta:") + expect((connector as any).sampleDocuments.mock.calls).toEqual([["records", "events", 100]]) + }) + + test("a failing document sample keeps the schema", async () => { + const connector = configure("records") + Object.assign(connector, { sampleDocuments: mock(async () => { throw new Error("not authorized") }) }) + const tool = await initTool(SchemaInspectTool) + const result = await tool.execute({ warehouse: "documents", table: "records.events" }, ctx) + expect(result.metadata.success).toBe(true) + expect(result.output).toContain("id | STRING") + expect(result.output).toContain("Value samples unavailable: not authorized") + }) }) diff --git a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts index 00bdbb2358..77cdecc0ee 100644 --- a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts +++ b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts @@ -5,7 +5,7 @@ import * as Dispatcher from "../../../src/altimate/native/dispatcher" import * as Registry from "../../../src/altimate/native/connections/registry" import { registerAll } from "../../../src/altimate/native/connections/register" import { SchemaInspectTool } from "../../../src/altimate/tools/schema-inspect" -import { buildProfileSql } from "../../../src/altimate/tools/schema-values" +import { buildProfileSql, profileDocuments, valueShape } from "../../../src/altimate/tools/schema-values" const ctx = { sessionID: "test-session", @@ -68,9 +68,11 @@ afterAll(() => { function expectAppendix(output: string) { expect(output).toContain("Value samples:") expect(output).toContain("id: 60 distinct") - expect(output).toMatch(/region: 4 distinct — (north|south|east|west)(, (north|south|east|west)){3}/) - expect(output).toMatch(/customer_code: 60 distinct, e\.g\. "[^"]+", "[^"]+", "[^"]+"/) - expect(output).toContain("note: 1 distinct — x, 50% null") + expect(output).toContain("id: 60 distinct, unique in sample\n") + expect(output).toMatch(/region: 4 distinct — (north|south|east|west) \(15\)(, (north|south|east|west) \(15\)){3}/) + expect(output).toMatch(/customer_code: 60 distinct, unique in sample, e\.g\. "[^"]+", "[^"]+", "[^"]+", shape /) + expect(output).toContain("shape ` A-9+` 40%, `A-9+` 40%, `a9+` 20%") + expect(output).toContain("note: 1 distinct — x (30), 50% null") } describe("schema_inspect value grounding", () => { @@ -98,7 +100,7 @@ describe("schema_inspect value grounding", () => { install("sqlite", async (sql) => small.query(sql).values() as unknown[][], []) const result = await inspect({}) small.close() - expect(result.output).toMatch(/customer_code: 3 distinct — .*" C-00019"/) + expect(result.output).toMatch(/customer_code: 3 distinct, unique in sample — .*" C-00019"/) expect(result.output).toContain("c00018") expect(result.output).toContain("note: all NULL") }) @@ -110,8 +112,8 @@ describe("schema_inspect value grounding", () => { small.run("INSERT INTO t VALUES (1, 'left␟right', '', NULL)") install("sqlite", async (sql) => small.query(sql).values() as unknown[][], []) const result = await inspect({}) - expect(result.output).toContain("region: 1 distinct — left␟right\n") - expect(result.output).toContain('customer_code: 1 distinct — ""\n') + expect(result.output).toContain("region: 1 distinct — left␟right (1)\n") + expect(result.output).toContain('customer_code: 1 distinct — "" (1)\n') } finally { small.close() } @@ -152,7 +154,7 @@ describe("schema_inspect value grounding", () => { test("unsupported engines are skipped with a note", async () => { const calls: string[] = [] install("sqlite", run, calls) - Dispatcher.register("warehouse.list" as any, async () => ({ warehouses: [{ name: "w", type: "mongodb" }] })) + Dispatcher.register("warehouse.list" as any, async () => ({ warehouses: [{ name: "w", type: "snowflake" }] })) const result = await inspect({}) expect(calls).toHaveLength(0) expect(result.output).toContain("Value samples skipped") @@ -173,13 +175,13 @@ describe("schema_inspect value grounding", () => { const result = await inspect({}) expectAppendix(result.output) expect(calls).toHaveLength(1) - expect(calls[0]).toContain("array_agg(v)") + expect(calls[0]).toContain("array_agg(CAST(pn AS TEXT)") expect(calls[0]).toContain('FROM "main"."t"') await conn.execute("DELETE FROM t", 1000) await conn.execute("INSERT INTO t VALUES (1, 'left␟right', '', NULL)", 1000) const edgeValues = await inspect({}) - expect(edgeValues.output).toContain("region: 1 distinct — left␟right\n") - expect(edgeValues.output).toContain('customer_code: 1 distinct — ""\n') + expect(edgeValues.output).toContain("region: 1 distinct — left␟right (1)\n") + expect(edgeValues.output).toContain('customer_code: 1 distinct — "" (1)\n') } finally { await conn.close() } @@ -189,16 +191,16 @@ describe("schema_inspect value grounding", () => { test("postgres uses array_agg and preserves quoted identifiers containing an aggregate name", async () => { const calls: string[] = [] const column = "group_concat(value" - install("postgres", async () => [[column, 1, 1, 1, ["left␟right"]]], calls) + install("postgres", async () => [[column, 1, 1, 1, ["1:left␟right"]]], calls) Dispatcher.register("schema.inspect" as any, async () => ({ table: "t", schema_name: "sales", columns: [{ name: column, data_type: "TEXT", nullable: false }], })) const result = await inspect({ table: "sales.t" }) - expect(calls[0]).toContain("array_agg(v)") + expect(calls[0]).toContain("array_agg(CAST(pn AS TEXT)") expect(calls[0]).toContain('CAST("group_concat(value" AS TEXT)') expect(calls[0]).toContain("'group_concat(value' AS col") expect(calls[0]).toContain('FROM "sales"."t"') - expect(result.output).toContain("group_concat(value: 1 distinct — left␟right") + expect(result.output).toContain("group_concat(value: 1 distinct — left␟right (1)") }) test.each(["postgres", "postgresql", "sqlite", "duckdb"])( @@ -211,14 +213,14 @@ describe("schema_inspect value grounding", () => { })) const execute = mock(async () => ({ columns: ["col", "total", "nonnull", "ndistinct", "vals"], - rows: [["region", 1, 1, 1, ["north"]]], + rows: [["region", 1, 1, 1, ["1:north"]]], row_count: 1, truncated: false, })) const get = spyOn(Registry, "get").mockResolvedValue({ execute } as any) try { const result = await inspect({ warehouse: "w" }) - expect(result.output).toContain("region: 1 distinct — north") + expect(result.output).toContain("region: 1 distinct — north (1)") expect(execute).toHaveBeenCalledTimes(1) const [sql, limit, binds, options] = execute.mock.calls[0] as unknown as [string, number, unknown, Record] expect(sql).toContain('FROM "public"."t"') @@ -238,3 +240,164 @@ describe("schema_inspect value grounding", () => { expect(buildProfileSql('"t"', cols.slice(0, 40)).split("UNION ALL")).toHaveLength(40) }) }) + +const RICH_COLUMNS = ["id", "status", "cpc", "tag", "abstract", "title", "name"].map((name) => ({ + name, + data_type: name === "id" ? "INTEGER" : "TEXT", + nullable: true, + primary_key: name === "id", +})) +const NAMES = Array.from({ length: 25 }, (_, i) => String.fromCharCode(65 + (i % 25)) + "ob".repeat(1 + (i % 3)) + "by") +const TITLE_FORMS = [(i: number) => `Alpha ${i}`, (i: number) => `alpha-${i}`, (i: number) => `A, b ${i}`, (i: number) => `A b c ${i}`, + (i: number) => `Q(${i})`, (i: number) => `x_${i}_y`, (i: number) => `R${i}/${i}`, (i: number) => `N.${i}`] + +/** 200 rows with skewed statuses, patent-like codes, a skewed tag, long abstracts and free-form titles. */ +function richSeed(): string[] { + const rows: string[] = [] + for (let i = 0; i < 200; i++) { + const status = i < 120 ? "open" : i < 170 ? "closed" : "pending" + const tag = i < 80 ? "alpha" : i < 120 ? "beta" : i < 140 ? "gamma" : `t${i}` + const abstract = `This abstract describes item number ${i} in a rather long and detailed manner, so the field is clearly prose. ${"It keeps going with more explanatory sentences for the reader. ".repeat(3)}` + rows.push( + `INSERT INTO t VALUES (${i}, '${status}', 'H01M${i % 9}/${9000 + i}', '${tag}', '${abstract}', '${TITLE_FORMS[i % 8]!(i % 6)}', '${NAMES[i % 25]}')`, + ) + } + return ["CREATE TABLE t (id INTEGER, status TEXT, cpc TEXT, tag TEXT, abstract TEXT, title TEXT, name TEXT)", ...rows] +} + +function expectRichProfile(output: string) { + expect(output).toContain("id: 200 distinct, unique in sample\n") + // ranked by frequency, not alphabetically + expect(output).toContain("status: 3 distinct — open (120), closed (50), pending (30)\n") + // collapsed run classes: H01M4/9066 -> A9+A9/9+ + expect(output).toContain("cpc: 200 distinct, unique in sample, e.g.") + expect(output).toContain("shape `A9+A9/9+`") + // high cardinality shows the most frequent values with counts + expect(output).toContain('tag: 63 distinct, e.g. "alpha" (80), "beta" (40), "gamma" (20)') + // long prose: one 160-character example, the rest clipped at 40, and no shape + const abstract = output.split("\n").find((l) => l.startsWith("abstract:"))! + expect(abstract).toContain("unique in sample") + expect(abstract).not.toContain("shape") + const quoted = abstract.match(/"[^"]*"/g)! + expect(quoted[0]!.length - 2).toBe(161) + expect(quoted[1]!.length - 2).toBe(41) + // free-form text with many shapes and plain single-shape words get no shape + expect(output.split("\n").find((l) => l.startsWith("title:"))).not.toContain("shape") + expect(output.split("\n").find((l) => l.startsWith("name:"))).not.toContain("shape") +} + +describe("schema_inspect richer value profiles", () => { + const useRichSchema = () => + Dispatcher.register("schema.inspect" as any, async () => ({ table: "t", schema_name: "main", columns: RICH_COLUMNS })) + + test("sqlite: frequency order, uniqueness, shapes and long-text examples", async () => { + const db = new Database(":memory:") + try { + richSeed().forEach((q) => db.run(q)) + const calls: string[] = [] + install("sqlite", async (sql) => db.query(sql).values() as unknown[][], calls) + useRichSchema() + const result = await inspect({}) + expect(calls).toHaveLength(1) + expect(calls[0]).toContain("json_group_array(CAST(pn AS TEXT) || ':' || pv)") + expect(calls[0]).toContain("ORDER BY pn DESC, pv LIMIT 50") + expectRichProfile(result.output) + } finally { + db.close() + } + }) + + test("duckdb: same profile from a real engine", async () => { + const { connect } = await import("../../../../drivers/src/duckdb") + const dir = (await import("fs")).mkdtempSync((await import("os")).tmpdir() + "/schemavals-rich-") + const conn = await connect({ type: "duckdb", path: dir + "/t.duckdb", create: true } as any) + try { + await conn.connect() + for (const q of richSeed()) await conn.execute(q, 1000) + const calls: string[] = [] + install("duckdb", async (sql) => (await conn.execute(sql, 1000)).rows as unknown[][], calls) + useRichSchema() + const result = await inspect({}) + expect(calls[0]).toContain("array_agg(CAST(pn AS TEXT) || ':' || pv)") + expectRichProfile(result.output) + } finally { + await conn.close() + } + }) + + test("postgres SQL uses standard aggregation and the same ranked subquery", () => { + const sql = buildProfileSql('"public"."t"', [{ name: "status", data_type: "text" }], "postgres") + expect(sql).toContain("array_agg(CAST(pn AS TEXT) || ':' || pv)") + expect(sql).toContain("GROUP BY SUBSTR(CAST(\"status\" AS TEXT), 1, 200) ORDER BY pn DESC, pv LIMIT 50") + expect(sql).not.toContain("json_group_array") + expect(sql).not.toMatch(/FILTER|WITHIN GROUP|STRING_AGG|GROUP_CONCAT/i) + }) + + test("a repeated key is not unique, and every-value-once is", async () => { + const db = new Database(":memory:") + try { + db.run("CREATE TABLE t (id INTEGER, status TEXT, cpc TEXT, tag TEXT, abstract TEXT, title TEXT, name TEXT)") + db.run("INSERT INTO t VALUES (1,'a','a','a','a','a','a'), (1,'b','b','b','b','b','b')") + install("sqlite", async (sql) => db.query(sql).values() as unknown[][], []) + useRichSchema() + const result = await inspect({}) + expect(result.output).toContain("id: 1 distinct — 1 (2)") + expect(result.output).toContain("status: 2 distinct, unique in sample — a, b") + } finally { + db.close() + } + }) + + test("valueShape collapses runs of one class", () => { + expect(valueShape("H01M4/9066")).toBe("A9+A9/9+") + expect(valueShape("Hello, World 42")).toBe("Aa+, Aa+ 9+") + expect(valueShape("ab-12")).toBe("a+-9+") + expect(valueShape("")).toBe("") + }) +}) + +describe("MongoDB value profile", () => { + const columns = [ + "_id", "status", "sku", "address.city", "items[].sku", "notes", "address", "tags", + ].map((name) => ({ + name, + data_type: name === "address" ? "object" : name === "tags" ? "array" : name === "_id" ? "objectId" : "string", + })) + const docs = Array.from({ length: 12 }, (_, i) => ({ + _id: { _bsontype: "ObjectId", toString: () => `id${i}` }, + status: i < 8 ? "open" : "closed", + sku: `AB${i}-${1000 + i}`, + address: { city: i % 2 ? "Oslo" : "Rome" }, + items: [{ sku: "X1" }, { sku: i % 3 ? "X2" : "X1" }], + ...(i < 4 ? { notes: null } : {}), + tags: ["a"], + })) + + test("ranked values, uniqueness, shape, nulls and nested paths, skipping containers", () => { + const out = profileDocuments(docs, columns) + expect(out.split("\n")[0]).toBe("Value samples:") + expect(out).toContain("status: 2 distinct — open (8), closed (4)\n") + expect(out).toContain("sku: 12 distinct, unique in sample — ") + expect(out).toContain("address.city: 2 distinct — Oslo (6), Rome (6)") + expect(out).toContain("items[].sku: 2 distinct — X1 (") + expect(out).toContain("notes: all NULL") + expect(out).not.toContain("\naddress:") + expect(out).not.toContain("\ntags:") + // objectId text is shown but is not a text column, so no examples or shape + expect(out).toContain("_id: 12 distinct, unique in sample") + }) + + test("high-cardinality text shows frequent examples and a shape", () => { + const many = Array.from({ length: 30 }, (_, i) => ({ code: i < 10 ? "ZZ-1" : `ab${i}-${100 + i}` })) + const out = profileDocuments(many, [{ name: "code", data_type: "string" }]) + expect(out).toContain('code: 21 distinct, e.g. "ZZ-1" (10),') + expect(out).toContain("shape `a+9+-9+` 67%, `A+-9` 33%") + }) + + test("a 100-document sample is labelled approximate", () => { + const many = Array.from({ length: 100 }, (_, i) => ({ k: i % 2 ? "a" : "b" })) + const out = profileDocuments(many, [{ name: "k", data_type: "string" }]) + expect(out).toContain("Value samples (first 100 documents):") + expect(out).toContain("k: ~2 distinct — a (50), b (50)") + }) +}) From deb2828de76d10f099c616d0614b31180b260786 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 17 Oct 2026 11:15:18 -0700 Subject: [PATCH 2/4] fix: [review] MongoDB value profile renders BSON types, dotted keys and MongoDB notation - render UUID, ObjectId, Decimal128, Long, Timestamp, regex, Code and MinKey/MaxKey as readable text; values with no text form report "values not profiled (type)" instead of "all NULL" - carry key paths as segments so keys containing "." are read correctly - name nested fields in MongoDB dot notation (`items.sku`) and mark array membership in a "Nested fields" listing - revert `describeTable` to its top-level behaviour; nested fields are derived in `schema_inspect` from the single document sample used for the value profile - bound the sample with `maxTimeMS` and the tool call's abort signal - add real-BSON unit tests, a describeTable regression test, and a real-server e2e case Co-Authored-By: Claude Opus 5.5 --- packages/drivers/src/mongodb.ts | 128 +++------ packages/drivers/src/types.ts | 11 +- packages/drivers/test/mongodb-unit.test.ts | 60 ++-- .../altimate/native/connections/register.ts | 1 - .../opencode/src/altimate/native/types.ts | 2 - .../src/altimate/tools/schema-inspect.ts | 4 +- .../src/altimate/tools/schema-values.ts | 267 +++++++++++++----- .../test/altimate/drivers-mongodb-e2e.test.ts | 36 +++ .../tools/schema-inspect-mongodb.test.ts | 24 +- .../tools/schema-inspect-values.test.ts | 92 ++++-- 10 files changed, 381 insertions(+), 244 deletions(-) diff --git a/packages/drivers/src/mongodb.ts b/packages/drivers/src/mongodb.ts index 83b4ac52ff..9211628c96 100644 --- a/packages/drivers/src/mongodb.ts +++ b/packages/drivers/src/mongodb.ts @@ -245,60 +245,6 @@ function extractFields(docs: Record[]): Map return fieldTypes } -/** Deepest dotted path reported by `extractNestedPaths`, counting `field[]` as part of its segment. */ -const NESTED_PATH_MAX_DEPTH = 3 -/** Cap on nested paths reported per collection. */ -const NESTED_PATH_MAX_COUNT = 100 -/** Documents sampled to infer a collection schema. */ -const SAMPLE_DOCUMENTS = 100 - -export interface NestedPath { - types: Set - /** Sampled documents containing the path. */ - docs: number -} - -function isPlainObject(value: unknown): value is Record { - if (typeof value !== "object" || value === null) return false - const proto = Object.getPrototypeOf(value) - return proto === Object.prototype || proto === null -} - -/** - * Extract dotted paths below the top level (`a.b`, `items[].sku`) up to depth 3, with - * the types seen and the number of documents containing each path. Arrays of objects - * contribute `field[]` followed by the element's keys. - */ -export function extractNestedPaths(docs: Record[]): Map { - const paths = new Map() - for (const doc of docs) { - const seen = new Set() - const visit = (value: unknown, path: string, depth: number) => { - if (depth >= NESTED_PATH_MAX_DEPTH) return - if (isPlainObject(value)) { - for (const [key, child] of Object.entries(value)) record(child, `${path}.${key}`, depth + 1) - } else if (Array.isArray(value)) { - for (const element of value) { - if (!isPlainObject(element)) continue - for (const [key, child] of Object.entries(element)) record(child, `${path}[].${key}`, depth + 1) - } - } - } - const record = (value: unknown, path: string, depth: number) => { - const entry = paths.get(path) ?? { types: new Set(), docs: 0 } - entry.types.add(inferType(value)) - if (!seen.has(path)) { - seen.add(path) - entry.docs++ - } - paths.set(path, entry) - visit(value, path, depth) - } - for (const [key, value] of Object.entries(doc)) visit(value, key, 1) - } - return paths -} - /** Server error code for an operation stopped by `maxTimeMS`. */ const MONGO_MAX_TIME_EXPIRED = 50 @@ -409,15 +355,36 @@ export async function connect(config: ConnectionConfig): Promise { return JSON.stringify(val) } - async function sampleDocuments(schema: string, table: string, limit = SAMPLE_DOCUMENTS): Promise[]> { - // altimate_change start — format the existing cursor chain - return client - .db(schema) - .collection(table) - .find({}, withMaxTime(resolveQueryTimeout(config).ms)) - .limit(limit) - .toArray() - // altimate_change end + /** + * Raw documents for value profiling. `maxTimeMS` is the smaller of the caller's cap and the + * connection's query timeout; `signal` abandons the read and closes the cursor. + */ + async function sampleDocuments( + schema: string, + table: string, + limit = 100, + options: { signal?: AbortSignal; maxTimeMS?: number } = {}, + ): Promise[]> { + options.signal?.throwIfAborted() + const configured = resolveQueryTimeout(config).ms + const maxTimeMS = + options.maxTimeMS === undefined ? configured : configured === undefined ? options.maxTimeMS : Math.min(configured, options.maxTimeMS) + const cursor = client.db(schema).collection(table).find({}, withMaxTime(maxTimeMS)).limit(limit) + const { signal } = options + if (!signal) return cursor.toArray() + let abort: (() => void) | undefined + const aborted = new Promise((_, reject) => { + abort = () => { + void Promise.resolve(cursor.close?.()).catch(() => {}) + reject(signal.reason) + } + signal.addEventListener("abort", abort, { once: true }) + }) + try { + return await Promise.race([cursor.toArray(), aborted]) + } finally { + signal.removeEventListener("abort", abort!) + } } const connector: Connector & { @@ -941,8 +908,16 @@ export async function connect(config: ConnectionConfig): Promise { sampleDocuments, async describeTable(schema: string, table: string): Promise { + const db = client.db(schema) + const coll = db.collection(table) + // Sample up to 100 documents to infer schema - const docs = await sampleDocuments(schema, table) + // altimate_change start — format the existing cursor chain + const docs = await coll + .find({}, withMaxTime(resolveQueryTimeout(config).ms)) + .limit(100) + .toArray() + // altimate_change end if (docs.length === 0) { return [] @@ -957,15 +932,13 @@ export async function connect(config: ConnectionConfig): Promise { } } - const joinTypes = (types: Iterable) => { - const nonNullTypes = Array.from(types).filter((t) => t !== "null") - return nonNullTypes.length === 0 ? "null" : nonNullTypes.length === 1 ? nonNullTypes[0] : nonNullTypes.join(" | ") - } - const columns: SchemaColumn[] = [] for (const [name, types] of fieldTypes) { - const hasNull = types.has("null") - const dataType = joinTypes(types) + const typeArr = Array.from(types) + const hasNull = typeArr.includes("null") + const nonNullTypes = typeArr.filter((t) => t !== "null") + const dataType = + nonNullTypes.length === 0 ? "null" : nonNullTypes.length === 1 ? nonNullTypes[0] : nonNullTypes.join(" | ") // Field is nullable if it has null values OR is missing from some documents const presentIn = fieldPresence.get(name) ?? 0 @@ -978,18 +951,7 @@ export async function connect(config: ConnectionConfig): Promise { }) } - const nested = [...extractNestedPaths(docs)].sort(([a], [b]) => a.localeCompare(b)).slice(0, NESTED_PATH_MAX_COUNT) - for (const [name, info] of nested) { - const presence = info.docs / docs.length - columns.push({ - name, - data_type: joinTypes(info.types), - nullable: info.types.has("null") || presence < 1, - presence, - }) - } - - // Sort: _id first, then alphabetical (nested paths land under their parent) + // Sort: _id first, then alphabetical columns.sort((a, b) => { if (a.name === "_id") return -1 if (b.name === "_id") return 1 diff --git a/packages/drivers/src/types.ts b/packages/drivers/src/types.ts index 80d01c7c62..7fcbac62f1 100644 --- a/packages/drivers/src/types.ts +++ b/packages/drivers/src/types.ts @@ -18,8 +18,6 @@ export interface SchemaColumn { name: string data_type: string nullable: boolean - /** Share (0-1) of sampled documents containing this field; set on MongoDB nested paths. */ - presence?: number } export interface SchemaMetadata { @@ -137,8 +135,13 @@ export interface Connector { listSchemas(): Promise listTables(schema: string): Promise> describeTable(schema: string, table: string): Promise - /** Document stores only: the raw documents `describeTable` infers its schema from. */ - sampleDocuments?(schema: string, table: string, limit?: number): Promise[]> + /** Document stores only: raw documents, bounded by `maxTimeMS` and cancellable through `signal`. */ + sampleDocuments?( + schema: string, + table: string, + limit?: number, + options?: { signal?: AbortSignal; maxTimeMS?: number }, + ): Promise[]> introspectSchema?(signal?: AbortSignal): Promise close(): Promise } diff --git a/packages/drivers/test/mongodb-unit.test.ts b/packages/drivers/test/mongodb-unit.test.ts index 2594d357d4..c810d82600 100644 --- a/packages/drivers/test/mongodb-unit.test.ts +++ b/packages/drivers/test/mongodb-unit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, mock, test } from "bun:test" import { createRequire } from "node:module" -import { extractNestedPaths, normalizeMqlCommand } from "../src/mongodb" +import { normalizeMqlCommand } from "../src/mongodb" import { restoreModuleMocksAfterTests } from "./module-mock" await restoreModuleMocksAfterTests("mongodb") @@ -13,6 +13,7 @@ async function fakeMongo( metadata: { collections?: Record; error?: Error } = {}, ) { const finds: unknown[] = [] + const findOptions: unknown[] = [] const pipelines: Record[][] = [] const aggregateOptions: unknown[] = [] const counts: { filter: unknown; options: unknown }[] = [] @@ -50,8 +51,9 @@ async function fakeMongo( collections[database].push(name) }, collection: () => ({ - find: (filter: unknown) => { + find: (filter: unknown, options: unknown) => { finds.push(filter) + findOptions.push(options) return { limit: () => ({ toArray: async () => docs, async *[Symbol.asyncIterator]() { yield* docs } }) } }, aggregate: (pipeline: Record[], options: unknown) => { @@ -81,7 +83,7 @@ async function fakeMongo( const connector = await connect({ type: "mongodb", database: "test" }) await connector.connect() const query = (command: string, extra: object = {}) => JSON.stringify({ collection: "articles", command, ...extra }) - return { connector, query, collections, finds, pipelines, aggregateOptions, counts, distincts, collectionLists, databaseLists, estimates } + return { connector, query, collections, finds, findOptions, pipelines, aggregateOptions, counts, distincts, collectionLists, databaseLists, estimates } } describe("MongoDB read command shapes", () => { @@ -461,46 +463,30 @@ describe("MongoDB native command filters", () => { }) }) -describe("MongoDB nested field paths", () => { +describe("MongoDB nested documents", () => { const docs = [ - { _id: 1, user: { name: "a", geo: { lat: 1.5, deep: { too: "far" } } }, items: [{ sku: "x", qty: 1 }, { sku: "y" }], tags: ["t"] }, - { _id: 2, user: { name: "b" }, items: [{ sku: "z", qty: null }], tags: [] }, - { _id: 3, user: null, items: [] }, + { _id: 1, user: { name: "a", geo: { lat: 1.5 } }, items: [{ sku: "x" }], "a.b": 1 }, + { _id: 2, user: null, items: [] }, ] - test("extracts dotted and array paths to depth 3 with types and per-document presence", () => { - const paths = extractNestedPaths(docs) - expect([...paths.keys()].sort()).toEqual([ - "items[].qty", - "items[].sku", - "user.geo", - "user.geo.deep", - "user.geo.lat", - "user.name", + test("describeTable stays top-level: nested objects are 'object', arrays 'array', no nested paths", async () => { + const { connector } = await fakeMongo(docs) + expect(await connector.describeTable("test", "articles")).toEqual([ + { name: "_id", data_type: "int32", nullable: false }, + { name: "a.b", data_type: "int32", nullable: true }, + { name: "items", data_type: "array", nullable: false }, + { name: "user", data_type: "object", nullable: true }, ]) - expect(paths.get("user.geo.deep")!.types).toEqual(new Set(["object"])) - // sku appears twice in document 1 but the document is counted once - expect(paths.get("items[].sku")!.docs).toBe(2) - expect([...paths.get("items[].qty")!.types].sort()).toEqual(["int32", "null"]) - expect(paths.get("user.name")!.docs).toBe(2) }) - test("describeTable keeps top-level columns and adds nested paths with a presence share", async () => { - const { connector } = await fakeMongo(docs) - const columns = await connector.describeTable("test", "articles") - const byName = Object.fromEntries(columns.map((c) => [c.name, c])) - expect(byName.user).toEqual({ name: "user", data_type: "object", nullable: true }) - expect(byName.items).toEqual({ name: "items", data_type: "array", nullable: false }) - expect(byName["user.name"]).toEqual({ name: "user.name", data_type: "string", nullable: true, presence: 2 / 3 }) - expect(byName["items[].sku"]).toMatchObject({ data_type: "string", presence: 2 / 3 }) - expect(byName["items[].qty"]).toMatchObject({ data_type: "int32", nullable: true }) - expect(columns[0]!.name).toBe("_id") - const names = columns.map((c) => c.name) - expect(names.slice(names.indexOf("user"))).toEqual(["user", "user.geo", "user.geo.deep", "user.geo.lat", "user.name"]) - }) - - test("sampleDocuments returns the sampled documents", async () => { + test("sampleDocuments returns raw documents under the smaller of the caller and connection deadlines", async () => { + const { connector, findOptions } = await fakeMongo(docs) + expect(await connector.sampleDocuments!("test", "articles", 50, { maxTimeMS: 15_000 })).toEqual(docs) + expect(findOptions).toEqual([{ maxTimeMS: 15_000 }]) + }) + + test("sampleDocuments honours an aborted signal", async () => { const { connector } = await fakeMongo(docs) - expect(await connector.sampleDocuments!("test", "articles", 50)).toEqual(docs) + await expect(connector.sampleDocuments!("test", "articles", 50, { signal: AbortSignal.abort(new Error("stop")) })).rejects.toThrow("stop") }) }) diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index ebd120f8a3..a763ef0036 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -823,7 +823,6 @@ register("schema.inspect", async (params: SchemaInspectParams): Promise): string { lines.push("Column | Type | Nullable | PK") lines.push("-------|------|----------|---") for (const col of result.columns ?? []) { - // MongoDB nested paths carry the share of sampled documents that contain them. - const presence = col.presence !== undefined && col.presence < 1 ? ` (in ${Math.round(col.presence * 100)}% of docs)` : "" lines.push( - `${col.name} | ${col.data_type ?? "unknown"}${presence} | ${col.nullable ? "YES" : "NO"} | ${col.primary_key ? "YES" : ""}`, + `${col.name} | ${col.data_type ?? "unknown"} | ${col.nullable ? "YES" : "NO"} | ${col.primary_key ? "YES" : ""}`, ) } return lines.join("\n") diff --git a/packages/opencode/src/altimate/tools/schema-values.ts b/packages/opencode/src/altimate/tools/schema-values.ts index 445d556f83..ae7769c9dd 100644 --- a/packages/opencode/src/altimate/tools/schema-values.ts +++ b/packages/opencode/src/altimate/tools/schema-values.ts @@ -61,6 +61,8 @@ interface ColumnProfile { /** Most frequent values; sorted by `rank` before use. */ values: ValueCount[] text: boolean + /** Rows or documents with no value, when that differs from `total - nonNull` (unrenderable values). */ + nulls?: number /** Set for values that can repeat inside one row (array elements), where uniqueness is meaningless. */ multi?: boolean /** Column type suggests a format already (uuid), so a shape adds nothing. */ @@ -203,8 +205,9 @@ function renderShapes(values: ValueCount[], avgLength: number): string { } function renderProfile(p: ColumnProfile, sampled: boolean): string { - const nullPct = p.total > 0 ? Math.round(((p.total - p.nonNull) / p.total) * 100) : 0 - const nulls = nullPct > 0 ? `, ${nullPct}% null` : p.total > p.nonNull ? `, <1% null` : "" + const missing = p.nulls ?? p.total - p.nonNull + const nullPct = p.total > 0 ? Math.round((missing / p.total) * 100) : 0 + const nulls = nullPct > 0 ? `, ${nullPct}% null` : missing > 0 ? `, <1% null` : "" if (p.nonNull === 0) return `${p.name}: all NULL` const approx = sampled ? "~" : "" const unique = !p.multi && p.total > 1 && p.nonNull > 1 && p.distinct === p.nonNull @@ -235,28 +238,119 @@ function formatProfiles(profiles: ColumnProfile[], header: string, skipped: numb ].join("\n") } -function textOf(value: unknown): string | undefined { - if (typeof value === "string") return value - if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") return String(value) - if (value instanceof Date) return Number.isNaN(value.getTime()) ? undefined : value.toISOString() - const bson = isRecord(value) ? value._bsontype : undefined - // ObjectId, Long, Decimal128 and friends print as their natural text; binary does not. - if (typeof bson === "string" && bson !== "Binary") return String(value) - return undefined +interface BsonValue { + _bsontype?: string + [key: string]: any } -/** Values at a dotted path as reported by the driver (`a.b`, `items[].sku`); missing paths yield none. */ -function valuesAt(doc: Record, path: string): unknown[] { +const UUID_DASHES = /^(.{8})(.{4})(.{4})(.{4})(.{12})$/ + +/** MongoDB type name and display text of a value; `text` is absent when it has no useful text form. */ +function describeValue(value: unknown): { type: string; text?: string } { + if (value === null || value === undefined) return { type: "null" } + if (typeof value === "string") return { type: "string", text: value } + if (typeof value === "boolean") return { type: "bool", text: String(value) } + if (typeof value === "number") return { type: Number.isInteger(value) ? "int32" : "double", text: String(value) } + if (typeof value === "bigint") return { type: "int64", text: String(value) } + if (Array.isArray(value)) return { type: "array" } + if (value instanceof Date) return { type: "date", text: Number.isNaN(value.getTime()) ? undefined : value.toISOString() } + const v = value as BsonValue + switch (v._bsontype) { + case undefined: + return { type: "object" } + case "ObjectId": + case "ObjectID": + return { type: "objectId", text: v.toHexString?.() ?? String(v) } + case "Decimal128": + return { type: "decimal128", text: String(v) } + case "Long": + return { type: "int64", text: String(v) } + case "Int32": + return { type: "int32", text: String(v) } + case "Double": + return { type: "double", text: String(v) } + case "Timestamp": + // `t` is seconds since the epoch and `i` the ordinal within that second. + return { type: "timestamp", text: typeof v.t === "number" ? `Timestamp(${v.t}, ${v.i})` : undefined } + case "BSONRegExp": + return { type: "regex", text: `/${v.pattern}/${v.options ?? ""}` } + case "Code": + return { type: "javascript", text: String(v.code) } + case "BSONSymbol": + return { type: "symbol", text: String(v.value) } + case "MinKey": + return { type: "minKey", text: "MinKey" } + case "MaxKey": + return { type: "maxKey", text: "MaxKey" } + case "Binary": { + if (v.sub_type !== 4 || typeof v.toUUID !== "function") return { type: "binary" } + const hex = String(v.toUUID().toHexString()).replace(/-/g, "") + return { type: "uuid", text: hex.replace(UUID_DASHES, "$1-$2-$3-$4-$5") } + } + default: + return { type: String(v._bsontype).toLowerCase() } + } +} + +/** Deepest field path reported, counting each key as one segment. */ +const NESTED_MAX_DEPTH = 3 +const NESTED_MAX_LISTED = 60 + +interface DocField { + /** Key segments from the document root; never re-split from `name`, since keys may contain ".". */ + path: string[] + /** MongoDB dot notation, as it is written in a filter or projection. */ + name: string + types: Set + /** Sampled documents containing the field. */ + docs: number + /** The path passes through an array of objects. */ + inArray: boolean +} + +function isPlainObject(value: unknown): value is Record { + if (typeof value !== "object" || value === null) return false + const proto = Object.getPrototypeOf(value) + return proto === Object.prototype || proto === null +} + +/** Fields of the sampled documents to depth 3: top-level keys plus nested keys, descending into arrays of objects. */ +function collectFields(docs: Record[]): DocField[] { + const fields = new Map() + for (const doc of docs) { + const seen = new Set() + const visit = (value: Record, parent: string[], inArray: boolean) => { + for (const [key, child] of Object.entries(value)) { + const path = [...parent, key] + const id = path.join("\0") + const field = fields.get(id) ?? { path, name: path.join("."), types: new Set(), docs: 0, inArray } + field.types.add(describeValue(child).type) + if (!seen.has(id)) { + seen.add(id) + field.docs++ + } + fields.set(id, field) + if (path.length >= NESTED_MAX_DEPTH) continue + if (isPlainObject(child)) visit(child, path, inArray) + else if (Array.isArray(child)) { + for (const element of child) if (isPlainObject(element)) visit(element, path, true) + } + } + } + visit(doc, [], false) + } + return [...fields.values()] +} + +/** Values at a key path; arrays met on the way are searched element by element. */ +function valuesAt(doc: Record, path: string[]): unknown[] { let current: unknown[] = [doc] - for (const segment of path.split(".")) { - const array = segment.endsWith("[]") - const key = array ? segment.slice(0, -2) : segment + for (const key of path) { const next: unknown[] = [] for (const item of current) { - if (!isRecord(item)) continue - const value = item[key] - if (!array) next.push(value) - else if (Array.isArray(value)) next.push(...value) + for (const candidate of Array.isArray(item) ? item : [item]) { + if (isRecord(candidate) && Object.hasOwn(candidate, key)) next.push(candidate[key]) + } } current = next } @@ -265,48 +359,78 @@ function valuesAt(doc: Record, path: string): unknown[] { const CONTAINER_ONLY = /^(object|array)$/ +function renderNested(fields: DocField[], docCount: number): string { + if (fields.length === 0) return "" + const listed = fields.slice(0, NESTED_MAX_LISTED).map((f) => { + const notes = [ + ...(f.inArray ? ["array element"] : []), + ...(f.docs < docCount ? [`in ${Math.round((f.docs / docCount) * 100)}% of docs`] : []), + ] + return `${f.name}: ${[...f.types].filter((t) => t !== "null").join(" | ") || "null"}${notes.length ? ` (${notes.join(", ")})` : ""}` + }) + const more = fields.length - listed.length + return [`Nested fields (sampled ${docCount} docs):`, ...listed, ...(more > 0 ? [`(${more} more not listed)`] : [])].join("\n") +} + +function profileField(docs: Record[], f: DocField): string | ColumnProfile { + const counts = new Map() + let present = 0 + let rendered = 0 + for (const doc of docs) { + let hasValue = false + let hasText = false + for (const raw of valuesAt(doc, f.path)) { + const { text } = describeValue(raw) + if (raw === null || raw === undefined) continue + hasValue = true + if (text === undefined) continue + hasText = true + const key = text.slice(0, PROFILE_FETCH_CHARS) + counts.set(key, (counts.get(key) ?? 0) + 1) + } + if (hasValue) present++ + if (hasText) rendered++ + } + const types = [...f.types].filter((t) => t !== "null") + // Never report "all NULL" for values that exist but have no text form. + if (present > 0 && rendered === 0) return `${f.name}: values not profiled (${types.join(" | ")})` + return { + name: f.name, + total: docs.length, + nonNull: rendered, + nulls: docs.length - present, + distinct: counts.size, + values: rank([...counts].map(([value, count]) => ({ value, count }))).slice(0, PROFILE_FETCHED_VALUES), + text: types.some((t) => t === "string" || t === "uuid"), + multi: f.inArray, + typed: types.length === 1 && types[0] === "uuid", + } +} + /** - * Value profile for sampled documents, in the same format as the SQL profile. - * Columns are the field paths the driver reported; containers and binary are skipped. + * Nested-field listing and value profile for sampled documents, in the same format as the SQL + * profile. Fields come from the same documents the values do, so presence and counts agree. */ -export function profileDocuments(docs: Record[], columns: ProfileColumn[]): string { - const eligible = columns.filter((c) => !UNPROFILABLE.test(c.data_type ?? "") && !CONTAINER_ONLY.test(c.data_type ?? "")) +export function profileDocuments(docs: Record[]): string { + if (docs.length === 0) return "" + const fields = collectFields(docs).sort((a, b) => a.name.localeCompare(b.name)) + const nested = fields.filter((f) => f.path.length > 1) + const eligible = fields.filter((f) => { + const types = [...f.types].filter((t) => t !== "null") + return types.length === 0 || !types.every((t) => CONTAINER_ONLY.test(t)) + }) // Top-level fields come first so the column cap never crowds them out with nested paths. - const ordered = [...eligible.filter((c) => !/[.[]/.test(c.name)), ...eligible.filter((c) => /[.[]/.test(c.name))] + const ordered = [...eligible.filter((f) => f.path.length === 1), ...eligible.filter((f) => f.path.length > 1)] const cols = ordered.slice(0, PROFILE_MAX_COLUMNS) - const profiles = cols.map((c): ColumnProfile => { - const counts = new Map() - let nonNull = 0 - for (const doc of docs) { - let present = false - for (const raw of valuesAt(doc, c.name)) { - const text = textOf(raw) - if (text === undefined) continue - present = true - const key = text.slice(0, PROFILE_FETCH_CHARS) - counts.set(key, (counts.get(key) ?? 0) + 1) - } - if (present) nonNull++ - } - const values = rank([...counts].map(([value, count]) => ({ value, count }))) - return { - name: c.name, - total: docs.length, - nonNull, - distinct: counts.size, - values: values.slice(0, PROFILE_FETCHED_VALUES), - text: TEXTUAL.test(c.data_type ?? ""), - multi: c.name.includes("[]"), - typed: /uuid/i.test(c.data_type ?? ""), - } - }) + const results = cols.map((f) => profileField(docs, f)) const sampled = docs.length >= PROFILE_SAMPLE_DOCUMENTS - return formatProfiles( - profiles, + const lines = results.map((r) => (typeof r === "string" ? r : renderProfile(r, sampled))) + const values = [ `Value samples${sampled ? ` (first ${PROFILE_SAMPLE_DOCUMENTS} documents)` : ""}:`, - ordered.length - cols.length, - sampled, - ) + ...lines, + ...(ordered.length > cols.length ? [`(${ordered.length - cols.length} more columns not profiled)`] : []), + ].join("\n") + return [renderNested(nested, docs.length), values].filter(Boolean).join("\n\n") } export interface ValueProfileInput { @@ -332,7 +456,7 @@ export async function profileValues( const target = input.warehouse ? warehouses.find((w) => w.name === input.warehouse) : warehouses[0] const dialect = profileDialect(target?.type) if (target?.name && input.document && Registry.canonicalType(target.type) === "mongodb") { - return await profileCollection(input, input.document, target.name, ctx) + return await profileCollection(input.document, target.name, ctx) } if (!target?.name || !dialect) { return `Value samples skipped: not supported for ${target?.type ?? "this"} connections.` @@ -377,9 +501,12 @@ export async function profileValues( type ProfileContext = Pick & { abort?: AbortSignal } -/** Read-only: samples the same documents `describeTable` infers its schema from. */ +/** + * Read-only: samples documents through the connector rather than `sql.execute`, which flattens + * each document to strings per column and would lose the nested structure the profile reads. + * The find is bounded by `maxTimeMS` and cancelled with the tool call. + */ async function profileCollection( - input: ValueProfileInput, document: { database: string; collection: string }, warehouse: string, ctx: ProfileContext, @@ -388,26 +515,18 @@ async function profileCollection( if (!connector.sampleDocuments) return "Value samples skipped: not supported for mongodb connections." await ctx.ask({ permission: "sql_execute", - patterns: [ - JSON.stringify({ ...document, command: "find", limit: PROFILE_SAMPLE_DOCUMENTS }), - ], + patterns: [JSON.stringify({ ...document, command: "find", limit: PROFILE_SAMPLE_DOCUMENTS })], always: ["*"], metadata: { warehouse }, }) const precedence = await Precedence.check(ctx.sessionID, "sql_execute", warehouse) if (precedence.redirect) return `Value samples skipped: ${precedence.redirect.output}` - let timer: ReturnType | undefined - const deadline = new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error("sampling timed out")), PROFILE_TIMEOUT_MS) + const deadline = AbortSignal.timeout(PROFILE_TIMEOUT_MS) + const signal = ctx.abort ? AbortSignal.any([ctx.abort, deadline]) : deadline + const docs = await connector.sampleDocuments(document.database, document.collection, PROFILE_SAMPLE_DOCUMENTS, { + signal, + maxTimeMS: PROFILE_TIMEOUT_MS, }) - try { - const docs = await Promise.race([ - connector.sampleDocuments(document.database, document.collection, PROFILE_SAMPLE_DOCUMENTS), - deadline, - ]) - const profile = profileDocuments(docs, input.columns) - return profile && precedence.notice ? `${precedence.notice}\n${profile}` : profile - } finally { - clearTimeout(timer) - } + const profile = profileDocuments(docs) + return profile && precedence.notice ? `${precedence.notice}\n${profile}` : profile } diff --git a/packages/opencode/test/altimate/drivers-mongodb-e2e.test.ts b/packages/opencode/test/altimate/drivers-mongodb-e2e.test.ts index 7741fdc4ab..cdf7cc36d9 100644 --- a/packages/opencode/test/altimate/drivers-mongodb-e2e.test.ts +++ b/packages/opencode/test/altimate/drivers-mongodb-e2e.test.ts @@ -871,6 +871,42 @@ describe.skipIf(!DOCKER && !MONGODB_USE_CI)("MongoDB Driver E2E", () => { ) }) + test("sampleDocuments + profileDocuments — nested fields and values; describeTable stays top-level", async () => { + const { profileDocuments } = await import("../../src/altimate/tools/schema-values") + await connector.execute(cmd({ command: "createCollection", database: "testdb", name: "nested_profile" })) + await connector.execute( + cmd({ + command: "insertMany", + database: "testdb", + collection: "nested_profile", + documents: [ + { status: "open", address: { city: "Oslo" }, items: [{ sku: "S1" }, { sku: "S2" }] }, + { status: "open", address: { city: "Rome" }, items: [{ sku: "S1" }] }, + { status: "closed", address: { city: "Oslo" }, items: [] }, + ], + }), + ) + try { + const columns = await connector.describeTable("testdb", "nested_profile") + expect(columns.map((c) => [c.name, c.data_type])).toEqual([ + ["_id", "objectId"], + ["address", "object"], + ["items", "array"], + ["status", "string"], + ]) + const docs = await connector.sampleDocuments!("testdb", "nested_profile", 100, { maxTimeMS: 15_000 }) + expect(docs).toHaveLength(3) + const out = profileDocuments(docs) + expect(out).toContain("Nested fields (sampled 3 docs):\naddress.city: string\nitems.sku: string (array element, in 67% of docs)") + expect(out).toContain("status: 2 distinct — open (2), closed (1)") + expect(out).toContain("address.city: 2 distinct — Oslo (2), Rome (1)") + expect(out).toContain("items.sku: 2 distinct — S1 (2), S2 (1)") + expect(out).toContain("_id: 3 distinct, unique in sample") + } finally { + await connector.execute(cmd({ command: "dropCollection", database: "testdb", collection: "nested_profile" })) + } + }) + test("describeTable — mixed-type fields show union type", async () => { await connector.execute( cmd({ diff --git a/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts b/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts index 5a489822bf..55dd96c7b8 100644 --- a/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts +++ b/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts @@ -76,30 +76,32 @@ describe("schema_inspect MongoDB collections", () => { expect(result.output).not.toContain("was not found") }) - test("profiles sampled documents instead of skipping, including nested paths", async () => { + test("profiles one document sample, lists nested fields, and bounds the read", async () => { const connector = configure("records") const docs = Array.from({ length: 10 }, (_, i) => ({ id: `e${i}`, status: i < 7 ? "open" : "closed", meta: { source: i % 2 ? "web" : "api" }, + items: [{ sku: "S1" }], })) - connector.describeTable.mockImplementation(async () => [ - { name: "id", data_type: "string", nullable: false }, - { name: "meta", data_type: "object", nullable: false }, - { name: "meta.source", data_type: "string", nullable: true, presence: 0.9 }, - { name: "status", data_type: "string", nullable: false }, - ]) - Object.assign(connector, { sampleDocuments: mock(async () => docs) }) + const sample = mock(async (..._args: unknown[]) => docs) + Object.assign(connector, { sampleDocuments: sample }) + const abort = new AbortController() const tool = await initTool(SchemaInspectTool) - const result = await tool.execute({ warehouse: "documents", table: "records.events" }, ctx) + const result = await tool.execute({ warehouse: "documents", table: "records.events" }, { ...ctx, abort: abort.signal }) expect(result.metadata.success).toBe(true) expect(result.output).not.toContain("skipped") - expect(result.output).toContain("meta.source | string (in 90% of docs) | YES") + expect(result.output).toContain("Nested fields (sampled 10 docs):\nitems.sku: string (array element)\nmeta.source: string") expect(result.output).toContain("id: 10 distinct, unique in sample — e0, e1") expect(result.output).toContain("status: 2 distinct — open (7), closed (3)") expect(result.output).toContain("meta.source: 2 distinct — api (5), web (5)") expect(result.output).not.toContain("\nmeta:") - expect((connector as any).sampleDocuments.mock.calls).toEqual([["records", "events", 100]]) + expect(sample.mock.calls).toHaveLength(1) + const [schema, table, limit, options] = sample.mock.calls[0] as unknown as [string, string, number, { signal: AbortSignal; maxTimeMS: number }] + expect([schema, table, limit, options.maxTimeMS]).toEqual(["records", "events", 100, 15_000]) + expect(options.signal.aborted).toBe(false) + abort.abort() + expect(options.signal.aborted).toBe(true) }) test("a failing document sample keeps the schema", async () => { diff --git a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts index 77cdecc0ee..a8d661ef04 100644 --- a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts +++ b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts @@ -5,6 +5,7 @@ import * as Dispatcher from "../../../src/altimate/native/dispatcher" import * as Registry from "../../../src/altimate/native/connections/registry" import { registerAll } from "../../../src/altimate/native/connections/register" import { SchemaInspectTool } from "../../../src/altimate/tools/schema-inspect" +import { createRequire } from "module" import { buildProfileSql, profileDocuments, valueShape } from "../../../src/altimate/tools/schema-values" const ctx = { @@ -356,47 +357,80 @@ describe("schema_inspect richer value profiles", () => { }) }) +// Real BSON values: `bson` is a dependency of `mongodb`, which the drivers package owns. +const driversRequire = createRequire(new URL("../../../../drivers/package.json", import.meta.url)) +const bson = createRequire(driversRequire.resolve("mongodb"))("bson") as any + describe("MongoDB value profile", () => { - const columns = [ - "_id", "status", "sku", "address.city", "items[].sku", "notes", "address", "tags", - ].map((name) => ({ - name, - data_type: name === "address" ? "object" : name === "tags" ? "array" : name === "_id" ? "objectId" : "string", - })) - const docs = Array.from({ length: 12 }, (_, i) => ({ - _id: { _bsontype: "ObjectId", toString: () => `id${i}` }, - status: i < 8 ? "open" : "closed", - sku: `AB${i}-${1000 + i}`, - address: { city: i % 2 ? "Oslo" : "Rome" }, - items: [{ sku: "X1" }, { sku: i % 3 ? "X2" : "X1" }], - ...(i < 4 ? { notes: null } : {}), - tags: ["a"], - })) + test("renders real BSON types, dotted keys, nested fields and unrenderable values", () => { + const uuid = new bson.UUID("0f8fad5b-d9cb-469f-a165-70867728950e") + const docs = Array.from({ length: 4 }, (_, i) => ({ + _id: new bson.ObjectId("65a1b2c3d4e5f60718293a4" + i), + ref: i === 0 ? uuid : new bson.UUID(), + price: bson.Decimal128.fromString("12.50"), + big: bson.Long.fromString(`900719925474099${i}`), + pattern: new bson.BSONRegExp("^ab+c", "i"), + script: new bson.Code("function () { return 1 }"), + stamp: new bson.Timestamp({ t: 1700000000, i: 7 }), + created: new Date("2024-03-01T10:00:00Z"), + raw: new bson.Binary(Buffer.from([1, 2, 3])), + "a.b": i % 2 ? "x" : "y", + address: { city: "Oslo", "x.y": i, geo: { lat: 1.5 } }, + items: [{ sku: "S1" }, { sku: i === 3 ? "S2" : "S1" }], + min: new bson.MinKey(), + tags: ["t"], + })) + const out = profileDocuments(docs) + const line = (prefix: string) => + out.slice(out.indexOf("Value samples")).split("\n").find((l) => l.startsWith(prefix + ":")) + expect(line("ref")).toBe( + `ref: 4 distinct, unique in sample — ${docs.map((d) => d.ref.toString()).sort().join(", ")}`, + ) + expect(line("ref")).toContain("0f8fad5b-d9cb-469f-a165-70867728950e") + expect(line("ref")).not.toContain("NULL") + expect(line("_id")).toBe("_id: 4 distinct, unique in sample — 65a1b2c3d4e5f60718293a40, 65a1b2c3d4e5f60718293a41, 65a1b2c3d4e5f60718293a42, 65a1b2c3d4e5f60718293a43") + expect(line("price")).toBe("price: 1 distinct — 12.50 (4)") + expect(line("big")).toBe(`big: 4 distinct, unique in sample — ${docs.map((d) => d.big.toString()).sort().join(", ")}`) + expect(line("big")).toContain("9007199254740990, 9007199254740991") + expect(docs[0]!.big.toString()).toMatch(/^\d{16}$/) + expect(line("pattern")).toBe("pattern: 1 distinct — /^ab+c/i (4)") + expect(line("script")).toBe("script: 1 distinct — function () { return 1 } (4)") + expect(line("stamp")).toBe("stamp: 1 distinct — Timestamp(1700000000, 7) (4)") + expect(line("created")).toBe("created: 1 distinct — 2024-03-01T10:00:00.000Z (4)") + expect(line("min")).toBe("min: 1 distinct — MinKey (4)") + expect(line("raw")).toBe("raw: values not profiled (binary)") + // a real top-level key containing "." is found, and nested keys containing "." are too + expect(line("a.b")).toBe("a.b: 2 distinct — x (2), y (2)") + expect(line("address.x.y")).toBe("address.x.y: 4 distinct, unique in sample — 0, 1, 2, 3") + expect(line("address.city")).toBe("address.city: 1 distinct — Oslo (4)") + // MongoDB dot notation for array elements, with membership in the nested listing + expect(line("items.sku")).toBe("items.sku: 2 distinct — S1 (7), S2 (1)") + expect(out).toContain("Nested fields (sampled 4 docs):") + expect(out).toContain("items.sku: string (array element)") + expect(out).toContain("address.geo.lat: double") + expect(out).not.toContain("[]") + expect(out).not.toContain("[object") + expect(line("tags")).toBeUndefined() + expect(line("address")).toBeUndefined() + }) - test("ranked values, uniqueness, shape, nulls and nested paths, skipping containers", () => { - const out = profileDocuments(docs, columns) - expect(out.split("\n")[0]).toBe("Value samples:") - expect(out).toContain("status: 2 distinct — open (8), closed (4)\n") - expect(out).toContain("sku: 12 distinct, unique in sample — ") - expect(out).toContain("address.city: 2 distinct — Oslo (6), Rome (6)") - expect(out).toContain("items[].sku: 2 distinct — X1 (") - expect(out).toContain("notes: all NULL") - expect(out).not.toContain("\naddress:") - expect(out).not.toContain("\ntags:") - // objectId text is shown but is not a text column, so no examples or shape - expect(out).toContain("_id: 12 distinct, unique in sample") + test("a field whose values are all null is NULL, one with unrenderable values is not", () => { + const docs = [{ gone: null, blob: new bson.Binary(Buffer.from([1])) }, { gone: null }] + const out = profileDocuments(docs) + expect(out).toContain("gone: all NULL") + expect(out).toContain("blob: values not profiled (binary)") }) test("high-cardinality text shows frequent examples and a shape", () => { const many = Array.from({ length: 30 }, (_, i) => ({ code: i < 10 ? "ZZ-1" : `ab${i}-${100 + i}` })) - const out = profileDocuments(many, [{ name: "code", data_type: "string" }]) + const out = profileDocuments(many) expect(out).toContain('code: 21 distinct, e.g. "ZZ-1" (10),') expect(out).toContain("shape `a+9+-9+` 67%, `A+-9` 33%") }) test("a 100-document sample is labelled approximate", () => { const many = Array.from({ length: 100 }, (_, i) => ({ k: i % 2 ? "a" : "b" })) - const out = profileDocuments(many, [{ name: "k", data_type: "string" }]) + const out = profileDocuments(many) expect(out).toContain("Value samples (first 100 documents):") expect(out).toContain("k: ~2 distinct — a (50), b (50)") }) From 40966651a5eac0f5e1a487fc97222605dbd74e7a Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 17 Oct 2026 11:49:46 -0700 Subject: [PATCH 3/4] fix: [review] handle late sampleDocuments rejection and keep array-element marking - attach a handler to the in-flight read so a rejection after abort is not unhandled, and pass the signal to find - accumulate inArray across every occurrence of a nested path so mixed-shape fields are never unique - add tests for both Co-Authored-By: Claude Opus 5.5 --- packages/drivers/src/mongodb.ts | 12 +++++++-- packages/drivers/test/mongodb-unit.test.ts | 27 +++++++++++++++++-- .../src/altimate/tools/schema-values.ts | 2 ++ .../tools/schema-inspect-values.test.ts | 8 ++++++ 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/packages/drivers/src/mongodb.ts b/packages/drivers/src/mongodb.ts index 991952eff9..bc580f00dd 100644 --- a/packages/drivers/src/mongodb.ts +++ b/packages/drivers/src/mongodb.ts @@ -403,8 +403,13 @@ export async function connect(config: ConnectionConfig): Promise { const configured = resolveQueryTimeout(config).ms const maxTimeMS = options.maxTimeMS === undefined ? configured : configured === undefined ? options.maxTimeMS : Math.min(configured, options.maxTimeMS) - const cursor = client.db(schema).collection(table).find({}, withMaxTime(maxTimeMS)).limit(limit) const { signal } = options + // The driver (6.x) cancels the operation on `signal`; the race below also covers fakes and older drivers. + const cursor = client + .db(schema) + .collection(table) + .find({}, { ...withMaxTime(maxTimeMS), ...(signal ? { signal } : {}) }) + .limit(limit) if (!signal) return cursor.toArray() let abort: (() => void) | undefined const aborted = new Promise((_, reject) => { @@ -414,8 +419,11 @@ export async function connect(config: ConnectionConfig): Promise { } signal.addEventListener("abort", abort, { once: true }) }) + // A read that rejects after the abort won the race must not surface as an unhandled rejection. + const read: Promise[]> = cursor.toArray() + read.catch(() => {}) try { - return await Promise.race([cursor.toArray(), aborted]) + return await Promise.race([read, aborted]) } finally { signal.removeEventListener("abort", abort!) } diff --git a/packages/drivers/test/mongodb-unit.test.ts b/packages/drivers/test/mongodb-unit.test.ts index 68dd03078f..78e9df1bb8 100644 --- a/packages/drivers/test/mongodb-unit.test.ts +++ b/packages/drivers/test/mongodb-unit.test.ts @@ -10,7 +10,7 @@ const { Binary } = createRequire(import.meta.resolve("mongodb"))("bson") async function fakeMongo( docs?: Record[], - metadata: { collections?: Record; error?: Error } = {}, + metadata: { collections?: Record; error?: Error; read?: () => Promise[]> } = {}, ) { const finds: unknown[] = [] const findOptions: unknown[] = [] @@ -54,7 +54,7 @@ async function fakeMongo( find: (filter: unknown, options: unknown) => { finds.push(filter) findOptions.push(options) - return { limit: () => ({ toArray: async () => docs ?? [], async *[Symbol.asyncIterator]() { yield* docs ?? [] } }) } + return { limit: () => ({ toArray: () => (metadata.read ? metadata.read() : Promise.resolve(docs ?? [])), async *[Symbol.asyncIterator]() { yield* docs ?? [] } }) } }, aggregate: (pipeline: Record[], options: unknown) => { pipelines.push(pipeline) @@ -501,6 +501,29 @@ describe("MongoDB nested documents", () => { const { connector } = await fakeMongo(docs) await expect(connector.sampleDocuments!("test", "articles", 50, { signal: AbortSignal.abort(new Error("stop")) })).rejects.toThrow("stop") }) + + test("a read rejecting after the abort is handled and the abort reason wins", async () => { + const unhandled: unknown[] = [] + const onUnhandled = (reason: unknown) => unhandled.push(reason) + process.on("unhandledRejection", onUnhandled) + try { + let rejectRead!: (e: Error) => void + const { connector, findOptions } = await fakeMongo(docs, { + read: () => new Promise((_, reject) => { rejectRead = reject }), + }) + const controller = new AbortController() + const pending = connector.sampleDocuments!("test", "articles", 50, { signal: controller.signal }) + const abortReason = new Error("cancelled") + controller.abort(abortReason) + await expect(pending).rejects.toBe(abortReason) + rejectRead(new Error("cursor closed")) + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(unhandled).toEqual([]) + expect(findOptions[0]).toMatchObject({ signal: controller.signal }) + } finally { + process.off("unhandledRejection", onUnhandled) + } + }) }) describe("MongoDB collection command shorthand", () => { diff --git a/packages/opencode/src/altimate/tools/schema-values.ts b/packages/opencode/src/altimate/tools/schema-values.ts index ae7769c9dd..6755482144 100644 --- a/packages/opencode/src/altimate/tools/schema-values.ts +++ b/packages/opencode/src/altimate/tools/schema-values.ts @@ -325,6 +325,8 @@ function collectFields(docs: Record[]): DocField[] { const id = path.join("\0") const field = fields.get(id) ?? { path, name: path.join("."), types: new Set(), docs: 0, inArray } field.types.add(describeValue(child).type) + // A path seen both inside and outside arrays stays an array element. + field.inArray ||= inArray if (!seen.has(id)) { seen.add(id) field.docs++ diff --git a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts index a8d661ef04..7f0d570cee 100644 --- a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts +++ b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts @@ -414,6 +414,14 @@ describe("MongoDB value profile", () => { expect(line("address")).toBeUndefined() }) + test("a path seen outside and then inside an array is an array element, never unique", () => { + const docs = [{ tag: { id: "a" } }, { tag: [{ id: "b" }, { id: "c" }] }] + const out = profileDocuments(docs) + expect(out).toContain("tag.id: string (array element)") + const line = out.slice(out.indexOf("Value samples")).split("\n").find((l) => l.startsWith("tag.id:"))! + expect(line).not.toContain("unique in sample") + }) + test("a field whose values are all null is NULL, one with unrenderable values is not", () => { const docs = [{ gone: null, blob: new bson.Binary(Buffer.from([1])) }, { gone: null }] const out = profileDocuments(docs) From 4edad6ca39cffe6437d5321ec5dc9b44f4b001e2 Mon Sep 17 00:00:00 2001 From: anandgupta42 Date: Sat, 17 Oct 2026 11:54:48 -0700 Subject: [PATCH 4/4] fix: [review] count and rank profile values on the full text, clip only for output SQL groups and ranks the full value and applies SUBSTR in the outer projection; the MongoDB profile keys counts by the full rendered text. Long values sharing a 200 character prefix stay distinct and unique. Co-Authored-By: Claude Opus 5.5 --- .../src/altimate/tools/schema-values.ts | 14 ++--- .../tools/schema-inspect-values.test.ts | 54 +++++++++++++++++-- 2 files changed, 58 insertions(+), 10 deletions(-) diff --git a/packages/opencode/src/altimate/tools/schema-values.ts b/packages/opencode/src/altimate/tools/schema-values.ts index 6755482144..e15e1ac8e6 100644 --- a/packages/opencode/src/altimate/tools/schema-values.ts +++ b/packages/opencode/src/altimate/tools/schema-values.ts @@ -84,13 +84,13 @@ export function buildProfileSql(table: string, columns: ProfileColumn[], dialect const aggregate = dialect === "sqlite" ? "json_group_array" : "array_agg" const branches = columns.map((c) => { const col = `CAST(${quote(c.name)} AS TEXT)` - const clipped = `SUBSTR(${col}, 1, ${PROFILE_FETCH_CHARS})` const lit = `'${c.name.replace(/'/g, "''")}'` + // Values are grouped and ranked in full; only the selected entries are clipped for transfer. // Each entry is ":"; the count is digits only, so the first colon splits it. return ( `SELECT ${lit} AS col, COUNT(*) AS total, COUNT(${col}) AS nonnull, COUNT(DISTINCT ${col}) AS ndistinct, ` + - `(SELECT ${aggregate}(CAST(pn AS TEXT) || ':' || pv) FROM (SELECT ${clipped} AS pv, COUNT(*) AS pn FROM s ` + - `WHERE ${col} IS NOT NULL GROUP BY ${clipped} ORDER BY pn DESC, pv LIMIT ${PROFILE_FETCHED_VALUES}) AS top_values) AS vals FROM s` + `(SELECT ${aggregate}(CAST(pn AS TEXT) || ':' || SUBSTR(pv, 1, ${PROFILE_FETCH_CHARS})) FROM (SELECT ${col} AS pv, COUNT(*) AS pn FROM s ` + + `WHERE ${col} IS NOT NULL GROUP BY ${col} ORDER BY pn DESC, pv LIMIT ${PROFILE_FETCHED_VALUES}) AS top_values) AS vals FROM s` ) }) return `${sample}\n${branches.join("\nUNION ALL\n")}` @@ -387,8 +387,7 @@ function profileField(docs: Record[], f: DocField): string | Co hasValue = true if (text === undefined) continue hasText = true - const key = text.slice(0, PROFILE_FETCH_CHARS) - counts.set(key, (counts.get(key) ?? 0) + 1) + counts.set(text, (counts.get(text) ?? 0) + 1) } if (hasValue) present++ if (hasText) rendered++ @@ -402,7 +401,10 @@ function profileField(docs: Record[], f: DocField): string | Co nonNull: rendered, nulls: docs.length - present, distinct: counts.size, - values: rank([...counts].map(([value, count]) => ({ value, count }))).slice(0, PROFILE_FETCHED_VALUES), + // Counted on the full text; only the kept values are clipped. + values: rank([...counts].map(([value, count]) => ({ value, count }))) + .slice(0, PROFILE_FETCHED_VALUES) + .map((v) => ({ value: v.value.slice(0, PROFILE_FETCH_CHARS), count: v.count })), text: types.some((t) => t === "string" || t === "uuid"), multi: f.inArray, typed: types.length === 1 && types[0] === "uuid", diff --git a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts index 7f0d570cee..2327b0caee 100644 --- a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts +++ b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts @@ -300,7 +300,7 @@ describe("schema_inspect richer value profiles", () => { useRichSchema() const result = await inspect({}) expect(calls).toHaveLength(1) - expect(calls[0]).toContain("json_group_array(CAST(pn AS TEXT) || ':' || pv)") + expect(calls[0]).toContain("json_group_array(CAST(pn AS TEXT) || ':' || SUBSTR(pv, 1, 200))") expect(calls[0]).toContain("ORDER BY pn DESC, pv LIMIT 50") expectRichProfile(result.output) } finally { @@ -319,17 +319,56 @@ describe("schema_inspect richer value profiles", () => { install("duckdb", async (sql) => (await conn.execute(sql, 1000)).rows as unknown[][], calls) useRichSchema() const result = await inspect({}) - expect(calls[0]).toContain("array_agg(CAST(pn AS TEXT) || ':' || pv)") + expect(calls[0]).toContain("array_agg(CAST(pn AS TEXT) || ':' || SUBSTR(pv, 1, 200))") expectRichProfile(result.output) } finally { await conn.close() } }) + const LONG_COLUMNS = [{ name: "id", data_type: "INTEGER", nullable: true, primary_key: false }, { name: "body", data_type: "TEXT", nullable: true, primary_key: false }] + const longSeed = () => [ + "CREATE TABLE t (id INTEGER, body TEXT)", + ...Array.from({ length: 30 }, (_, i) => `INSERT INTO t VALUES (${i}, '${"p".repeat(200)}-${String(i).padStart(2, "0")}')`), + ] + const expectLongDistinct = (output: string) => { + const line = output.split("\n").find((l) => l.startsWith("body:"))! + expect(line).toStartWith("body: 30 distinct, unique in sample, e.g. ") + expect(line).not.toMatch(/\(\d+\)/) + } + + test("sqlite: long values sharing a 200-char prefix stay distinct", async () => { + const db = new Database(":memory:") + try { + longSeed().forEach((q) => db.run(q)) + install("sqlite", async (sql) => db.query(sql).values() as unknown[][], []) + Dispatcher.register("schema.inspect" as any, async () => ({ table: "t", schema_name: "main", columns: LONG_COLUMNS })) + expectLongDistinct((await inspect({})).output) + } finally { + db.close() + } + }) + + test("duckdb: long values sharing a 200-char prefix stay distinct", async () => { + const { connect } = await import("../../../../drivers/src/duckdb") + const dir = (await import("fs")).mkdtempSync((await import("os")).tmpdir() + "/schemavals-long-") + const conn = await connect({ type: "duckdb", path: dir + "/t.duckdb", create: true } as any) + try { + await conn.connect() + for (const q of longSeed()) await conn.execute(q, 1000) + install("duckdb", async (sql) => (await conn.execute(sql, 1000)).rows as unknown[][], []) + Dispatcher.register("schema.inspect" as any, async () => ({ table: "t", schema_name: "main", columns: LONG_COLUMNS })) + expectLongDistinct((await inspect({})).output) + } finally { + await conn.close() + } + }) + test("postgres SQL uses standard aggregation and the same ranked subquery", () => { const sql = buildProfileSql('"public"."t"', [{ name: "status", data_type: "text" }], "postgres") - expect(sql).toContain("array_agg(CAST(pn AS TEXT) || ':' || pv)") - expect(sql).toContain("GROUP BY SUBSTR(CAST(\"status\" AS TEXT), 1, 200) ORDER BY pn DESC, pv LIMIT 50") + expect(sql).toContain("array_agg(CAST(pn AS TEXT) || ':' || SUBSTR(pv, 1, 200))") + expect(sql).toContain("GROUP BY CAST(\"status\" AS TEXT) ORDER BY pn DESC, pv LIMIT 50") + expect(sql).toContain("SUBSTR(pv, 1, 200)") expect(sql).not.toContain("json_group_array") expect(sql).not.toMatch(/FILTER|WITHIN GROUP|STRING_AGG|GROUP_CONCAT/i) }) @@ -422,6 +461,13 @@ describe("MongoDB value profile", () => { expect(line).not.toContain("unique in sample") }) + test("long values sharing a 200-char prefix stay distinct and unique", () => { + const docs = Array.from({ length: 30 }, (_, i) => ({ body: `${"p".repeat(200)}-${String(i).padStart(2, "0")}` })) + const line = profileDocuments(docs).split("\n").find((l) => l.startsWith("body:"))! + expect(line).toStartWith("body: 30 distinct, unique in sample, e.g. ") + expect(line).not.toMatch(/\(\d+\)/) + }) + test("a field whose values are all null is NULL, one with unrenderable values is not", () => { const docs = [{ gone: null, blob: new bson.Binary(Buffer.from([1])) }, { gone: null }] const out = profileDocuments(docs)