diff --git a/packages/drivers/src/mongodb.ts b/packages/drivers/src/mongodb.ts index 1d465f4f0f..bc580f00dd 100644 --- a/packages/drivers/src/mongodb.ts +++ b/packages/drivers/src/mongodb.ts @@ -389,6 +389,46 @@ export async function connect(config: ConnectionConfig): Promise { return JSON.stringify(val) } + /** + * Raw documents for value profiling. `maxTimeMS` is the smaller of the caller's cap and the + * connection's query timeout; `signal` abandons the read and closes the cursor. + */ + async function sampleDocuments( + schema: string, + table: string, + limit = 100, + options: { signal?: AbortSignal; maxTimeMS?: number } = {}, + ): Promise[]> { + options.signal?.throwIfAborted() + const configured = resolveQueryTimeout(config).ms + const maxTimeMS = + options.maxTimeMS === undefined ? configured : configured === undefined ? options.maxTimeMS : Math.min(configured, options.maxTimeMS) + const { signal } = options + // The driver (6.x) cancels the operation on `signal`; the race below also covers fakes and older drivers. + const cursor = client + .db(schema) + .collection(table) + .find({}, { ...withMaxTime(maxTimeMS), ...(signal ? { signal } : {}) }) + .limit(limit) + if (!signal) return cursor.toArray() + let abort: (() => void) | undefined + const aborted = new Promise((_, reject) => { + abort = () => { + void Promise.resolve(cursor.close?.()).catch(() => {}) + reject(signal.reason) + } + signal.addEventListener("abort", abort, { once: true }) + }) + // A read that rejects after the abort won the race must not surface as an unhandled rejection. + const read: Promise[]> = cursor.toArray() + read.catch(() => {}) + try { + return await Promise.race([read, aborted]) + } finally { + signal.removeEventListener("abort", abort!) + } + } + const connector: Connector & { // altimate_change start runCommand( @@ -910,6 +950,8 @@ export async function connect(config: ConnectionConfig): Promise { .sort((a: { name: string }, b: { name: string }) => a.name.localeCompare(b.name)) }, + sampleDocuments, + async describeTable(schema: string, table: string): Promise { const db = client.db(schema) const coll = db.collection(table) diff --git a/packages/drivers/src/types.ts b/packages/drivers/src/types.ts index d5ec39e575..16fe92a519 100644 --- a/packages/drivers/src/types.ts +++ b/packages/drivers/src/types.ts @@ -135,6 +135,13 @@ export interface Connector { listSchemas(): Promise listTables(schema: string): Promise> describeTable(schema: string, table: string): Promise + /** Document stores only: raw documents, bounded by `maxTimeMS` and cancellable through `signal`. */ + sampleDocuments?( + schema: string, + table: string, + limit?: number, + options?: { signal?: AbortSignal; maxTimeMS?: number }, + ): Promise[]> introspectSchema?(signal?: AbortSignal): Promise close(): Promise } diff --git a/packages/drivers/test/mongodb-unit.test.ts b/packages/drivers/test/mongodb-unit.test.ts index f7ef97baf2..78e9df1bb8 100644 --- a/packages/drivers/test/mongodb-unit.test.ts +++ b/packages/drivers/test/mongodb-unit.test.ts @@ -10,9 +10,10 @@ const { Binary } = createRequire(import.meta.resolve("mongodb"))("bson") async function fakeMongo( docs?: Record[], - metadata: { collections?: Record; error?: Error } = {}, + metadata: { collections?: Record; error?: Error; read?: () => Promise[]> } = {}, ) { const finds: unknown[] = [] + const findOptions: unknown[] = [] const pipelines: Record[][] = [] const aggregateOptions: unknown[] = [] const counts: { filter: unknown; options: unknown }[] = [] @@ -50,9 +51,10 @@ async function fakeMongo( collections[database].push(name) }, collection: () => ({ - find: (filter: unknown) => { + find: (filter: unknown, options: unknown) => { finds.push(filter) - return { limit: () => ({ toArray: async () => docs ?? [], async *[Symbol.asyncIterator]() { yield* docs ?? [] } }) } + findOptions.push(options) + return { limit: () => ({ toArray: () => (metadata.read ? metadata.read() : Promise.resolve(docs ?? [])), async *[Symbol.asyncIterator]() { yield* docs ?? [] } }) } }, aggregate: (pipeline: Record[], options: unknown) => { pipelines.push(pipeline) @@ -81,7 +83,7 @@ async function fakeMongo( const connector = await connect({ type: "mongodb", database: "test" }) await connector.connect() const query = (command: string, extra: object = {}) => JSON.stringify({ collection: "articles", command, ...extra }) - return { connector, query, collections, finds, pipelines, aggregateOptions, counts, distincts, collectionLists, databaseLists, estimates } + return { connector, query, collections, finds, findOptions, pipelines, aggregateOptions, counts, distincts, collectionLists, databaseLists, estimates } } describe("MongoDB read command shapes", () => { @@ -473,6 +475,57 @@ describe("MongoDB native command filters", () => { }) }) +describe("MongoDB nested documents", () => { + const docs = [ + { _id: 1, user: { name: "a", geo: { lat: 1.5 } }, items: [{ sku: "x" }], "a.b": 1 }, + { _id: 2, user: null, items: [] }, + ] + + test("describeTable stays top-level: nested objects are 'object', arrays 'array', no nested paths", async () => { + const { connector } = await fakeMongo(docs) + expect(await connector.describeTable("test", "articles")).toEqual([ + { name: "_id", data_type: "int32", nullable: false }, + { name: "a.b", data_type: "int32", nullable: true }, + { name: "items", data_type: "array", nullable: false }, + { name: "user", data_type: "object", nullable: true }, + ]) + }) + + test("sampleDocuments returns raw documents under the smaller of the caller and connection deadlines", async () => { + const { connector, findOptions } = await fakeMongo(docs) + expect(await connector.sampleDocuments!("test", "articles", 50, { maxTimeMS: 15_000 })).toEqual(docs) + expect(findOptions).toEqual([{ maxTimeMS: 15_000 }]) + }) + + test("sampleDocuments honours an aborted signal", async () => { + const { connector } = await fakeMongo(docs) + await expect(connector.sampleDocuments!("test", "articles", 50, { signal: AbortSignal.abort(new Error("stop")) })).rejects.toThrow("stop") + }) + + test("a read rejecting after the abort is handled and the abort reason wins", async () => { + const unhandled: unknown[] = [] + const onUnhandled = (reason: unknown) => unhandled.push(reason) + process.on("unhandledRejection", onUnhandled) + try { + let rejectRead!: (e: Error) => void + const { connector, findOptions } = await fakeMongo(docs, { + read: () => new Promise((_, reject) => { rejectRead = reject }), + }) + const controller = new AbortController() + const pending = connector.sampleDocuments!("test", "articles", 50, { signal: controller.signal }) + const abortReason = new Error("cancelled") + controller.abort(abortReason) + await expect(pending).rejects.toBe(abortReason) + rejectRead(new Error("cursor closed")) + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(unhandled).toEqual([]) + expect(findOptions[0]).toMatchObject({ signal: controller.signal }) + } finally { + process.off("unhandledRejection", onUnhandled) + } + }) +}) + describe("MongoDB collection command shorthand", () => { test.each([ { name: "key", aliases: { key: "status" } }, diff --git a/packages/opencode/src/altimate/tools/schema-inspect.ts b/packages/opencode/src/altimate/tools/schema-inspect.ts index 2474a76939..981efed1e8 100644 --- a/packages/opencode/src/altimate/tools/schema-inspect.ts +++ b/packages/opencode/src/altimate/tools/schema-inspect.ts @@ -128,6 +128,9 @@ export const SchemaInspectTool = Tool.define("schema_inspect", { table: ref, columns: schemaResult.columns!, warehouse: schemaResult.warehouse ?? target.warehouse, + ...(schemaResult.schema_name + ? { document: { database: schemaResult.schema_name, collection: schemaResult.table ?? args.table } } + : {}), }, ctx, ) diff --git a/packages/opencode/src/altimate/tools/schema-values.ts b/packages/opencode/src/altimate/tools/schema-values.ts index 7cf7e1a9d1..e15e1ac8e6 100644 --- a/packages/opencode/src/altimate/tools/schema-values.ts +++ b/packages/opencode/src/altimate/tools/schema-values.ts @@ -1,12 +1,15 @@ /** * Value grounding for `schema_inspect`: one bounded profiling query per table that - * returns, per column, the distinct count, null share and a few real values. The - * values are what reveal stored formats (case, whitespace, prefixes) that a column - * name and type never show. + * returns, per column, the distinct count, null share and the most frequent real + * values with their counts. The values are what reveal stored formats (case, + * whitespace, prefixes) that a column name and type never show; uniqueness and the + * value shape of text columns show keys and code formats at a glance. MongoDB + * collections get the same profile, computed in TypeScript from sampled documents. */ import type { LegacyContext } from "../tool-zod-compat" import * as Precedence from "../workspace/precedence" import { Dispatcher } from "../native" +import * as Registry from "../native/connections/registry" import { isRecord, normalizeError } from "./response-normalization" /** Rows scanned per table; keeps wide or large tables cheap. */ @@ -17,6 +20,18 @@ export const PROFILE_MAX_COLUMNS = 40 export const PROFILE_LOW_CARDINALITY = 20 const PROFILE_EXAMPLES = 3 const PROFILE_VALUE_CHARS = 40 +/** Frequency-ranked values fetched per column; feeds examples, counts and shapes. */ +const PROFILE_FETCHED_VALUES = 50 +/** Fetched values are clipped to this many characters, bounding transfer for long text. */ +const PROFILE_FETCH_CHARS = 200 +/** Text whose typical value exceeds the normal clip shows one longer example. */ +const PROFILE_LONG_EXAMPLE_CHARS = 160 +/** Above this average length, or this many distinct shapes, text is free text and gets no shape. */ +const PROFILE_SHAPE_MAX_AVG_CHARS = 30 +const PROFILE_SHAPE_MAX_VARIETY = 5 +const PROFILE_SHAPES_SHOWN = 3 +/** Documents sampled for MongoDB collections. */ +export const PROFILE_SAMPLE_DOCUMENTS = 100 const PROFILE_TIMEOUT_MS = 15_000 export type ProfileDialect = "sqlite" | "duckdb" | "postgres" @@ -33,13 +48,25 @@ export interface ProfileColumn { data_type?: string } +interface ValueCount { + value: string + count: number +} + interface ColumnProfile { name: string total: number nonNull: number distinct: number - values: string[] + /** Most frequent values; sorted by `rank` before use. */ + values: ValueCount[] text: boolean + /** Rows or documents with no value, when that differs from `total - nonNull` (unrenderable values). */ + nulls?: number + /** Set for values that can repeat inside one row (array elements), where uniqueness is meaningless. */ + multi?: boolean + /** Column type suggests a format already (uuid), so a shape adds nothing. */ + typed?: boolean } const quote = (id: string) => `"${id.replace(/"/g, '""')}"` @@ -58,22 +85,24 @@ export function buildProfileSql(table: string, columns: ProfileColumn[], dialect const branches = columns.map((c) => { const col = `CAST(${quote(c.name)} AS TEXT)` const lit = `'${c.name.replace(/'/g, "''")}'` + // Values are grouped and ranked in full; only the selected entries are clipped for transfer. + // Each entry is ":"; the count is digits only, so the first colon splits it. return ( `SELECT ${lit} AS col, COUNT(*) AS total, COUNT(${col}) AS nonnull, COUNT(DISTINCT ${col}) AS ndistinct, ` + - `(SELECT ${aggregate}(v) FROM (SELECT DISTINCT ${col} AS v FROM s WHERE ${col} IS NOT NULL ` + - `LIMIT ${PROFILE_LOW_CARDINALITY + 1}) AS sampled_values) AS vals FROM s` + `(SELECT ${aggregate}(CAST(pn AS TEXT) || ':' || SUBSTR(pv, 1, ${PROFILE_FETCH_CHARS})) FROM (SELECT ${col} AS pv, COUNT(*) AS pn FROM s ` + + `WHERE ${col} IS NOT NULL GROUP BY ${col} ORDER BY pn DESC, pv LIMIT ${PROFILE_FETCHED_VALUES}) AS top_values) AS vals FROM s` ) }) return `${sample}\n${branches.join("\nUNION ALL\n")}` } -function clipRaw(value: string): string { +function clipTo(value: string, max: number): string { const chars = Array.from(value) - return chars.length > PROFILE_VALUE_CHARS ? chars.slice(0, PROFILE_VALUE_CHARS).join("") + "…" : value + return chars.length > max ? chars.slice(0, max).join("") + "…" : value } function clip(value: string): string { - const text = clipRaw(value) + const text = clipTo(value, PROFILE_VALUE_CHARS) // Quote anything whose edges would otherwise be invisible. return text === "" || text !== text.trim() || /[\r\n\t]/.test(text) ? JSON.stringify(text) : text } @@ -83,13 +112,15 @@ function num(value: unknown): number { return Number.isFinite(n) ? n : 0 } -function parseValues(value: unknown): string[] { +function parseValues(value: unknown): ValueCount[] { // SQLite returns JSON text; DuckDB/Postgres return native arrays (or null when empty). - const values: unknown = typeof value === "string" ? JSON.parse(value) : value ?? [] - if (!Array.isArray(values) || !values.every((v) => typeof v === "string")) { - throw new Error("unexpected profiling values") - } - return values + const entries: unknown = typeof value === "string" ? JSON.parse(value) : value ?? [] + if (!Array.isArray(entries)) throw new Error("unexpected profiling values") + return entries.map((entry) => { + const match = typeof entry === "string" ? /^(\d+):([\s\S]*)$/.exec(entry) : null + if (!match) throw new Error("unexpected profiling values") + return { value: match[2]!, count: Number(match[1]) } + }) } function parseRows(rows: unknown[], columns: ProfileColumn[]): ColumnProfile[] { @@ -106,22 +137,304 @@ function parseRows(rows: unknown[], columns: ProfileColumn[]): ColumnProfile[] { distinct: num(row[3]), values: parseValues(row[4]), text: TEXTUAL.test(meta.data_type ?? ""), + typed: /uuid/i.test(meta.data_type ?? ""), }) } return out } -function renderProfile(p: ColumnProfile): string { - const sampled = p.total >= PROFILE_SAMPLE_ROWS - const nullPct = p.total > 0 ? Math.round(((p.total - p.nonNull) / p.total) * 100) : 0 - const nulls = nullPct > 0 ? `, ${nullPct}% null` : p.total > p.nonNull ? `, <1% null` : "" +/** + * Character-class pattern of a string: upper letters A, lower letters a, digits 9; + * punctuation and spaces stay. Runs of one class collapse to a symbol plus "+", so + * `H01M4/9066` becomes `A9+A9/9+`. + */ +export function valueShape(value: string): string { + let out = "" + let prev = "" + let run = 0 + const flush = () => { + if (run > 1) out += "+" + run = 0 + prev = "" + } + for (const ch of value) { + const cls = /\p{Lu}/u.test(ch) ? "A" : /\p{L}/u.test(ch) ? "a" : /\p{Nd}/u.test(ch) ? "9" : "" + if (!cls) { + flush() + out += ch + } else if (cls === prev) { + run++ + } else { + flush() + out += cls + prev = cls + run = 1 + } + } + flush() + return out +} + +function rank(values: ValueCount[]): ValueCount[] { + return [...values].sort((a, b) => b.count - a.count || (a.value < b.value ? -1 : a.value > b.value ? 1 : 0)) +} + +function averageLength(values: ValueCount[]): number { + const weight = values.reduce((n, v) => n + v.count, 0) + if (weight === 0) return 0 + return values.reduce((n, v) => n + Array.from(v.value).length * v.count, 0) / weight +} + +/** Top shapes with their share of the fetched values, or "" when the shape is uninformative. */ +function renderShapes(values: ValueCount[], avgLength: number): string { + if (avgLength > PROFILE_SHAPE_MAX_AVG_CHARS) return "" + const shapes = new Map() + for (const v of values) { + const shape = valueShape(v.value) + shapes.set(shape, (shapes.get(shape) ?? 0) + v.count) + } + if (shapes.size > PROFILE_SHAPE_MAX_VARIETY) return "" + // A lone letters-only shape (`Aa+`) is just "words"; the type already says as much. + if (shapes.size === 1 && /^[Aa+]+$/.test([...shapes.keys()][0]!)) return "" + const weight = values.reduce((n, v) => n + v.count, 0) + return [...shapes] + .sort((a, b) => b[1] - a[1]) + .slice(0, PROFILE_SHAPES_SHOWN) + .map(([shape, n]) => (shapes.size === 1 ? `\`${shape}\`` : `\`${shape}\` ${Math.round((n / weight) * 100)}%`)) + .join(", ") +} + +function renderProfile(p: ColumnProfile, sampled: boolean): string { + const missing = p.nulls ?? p.total - p.nonNull + const nullPct = p.total > 0 ? Math.round((missing / p.total) * 100) : 0 + const nulls = nullPct > 0 ? `, ${nullPct}% null` : missing > 0 ? `, <1% null` : "" if (p.nonNull === 0) return `${p.name}: all NULL` const approx = sampled ? "~" : "" + const unique = !p.multi && p.total > 1 && p.nonNull > 1 && p.distinct === p.nonNull + const head = `${p.name}: ${approx}${p.distinct.toLocaleString()} distinct${unique ? ", unique in sample" : ""}` + const values = rank(p.values) + // When every value occurs once the counts carry no information. + const counted = (text: string, v: ValueCount) => (unique ? text : `${text} (${v.count.toLocaleString()})`) if (p.distinct <= PROFILE_LOW_CARDINALITY) { - return `${p.name}: ${approx}${p.distinct} distinct — ${p.values.map(clip).join(", ")}${nulls}` + return `${head} — ${values.map((v) => counted(clip(v.value), v)).join(", ")}${nulls}` + } + if (!p.text || values.length === 0) return `${head}${nulls}` + const long = averageLength(values) > PROFILE_VALUE_CHARS + const examples = values + .slice(0, PROFILE_EXAMPLES) + .map((v, i) => counted(JSON.stringify(clipTo(v.value, long && i === 0 ? PROFILE_LONG_EXAMPLE_CHARS : PROFILE_VALUE_CHARS)), v)) + .join(", ") + const shapes = p.typed ? "" : renderShapes(values, averageLength(values)) + return `${head}, e.g. ${examples}${shapes ? `, shape ${shapes}` : ""}${nulls}` +} + +function formatProfiles(profiles: ColumnProfile[], header: string, skipped: number, sampled: boolean, notice?: string): string { + if (profiles.length === 0) return "" + return [ + ...(notice ? [notice] : []), + header, + ...profiles.map((p) => renderProfile(p, sampled)), + ...(skipped > 0 ? [`(${skipped} more columns not profiled)`] : []), + ].join("\n") +} + +interface BsonValue { + _bsontype?: string + [key: string]: any +} + +const UUID_DASHES = /^(.{8})(.{4})(.{4})(.{4})(.{12})$/ + +/** MongoDB type name and display text of a value; `text` is absent when it has no useful text form. */ +function describeValue(value: unknown): { type: string; text?: string } { + if (value === null || value === undefined) return { type: "null" } + if (typeof value === "string") return { type: "string", text: value } + if (typeof value === "boolean") return { type: "bool", text: String(value) } + if (typeof value === "number") return { type: Number.isInteger(value) ? "int32" : "double", text: String(value) } + if (typeof value === "bigint") return { type: "int64", text: String(value) } + if (Array.isArray(value)) return { type: "array" } + if (value instanceof Date) return { type: "date", text: Number.isNaN(value.getTime()) ? undefined : value.toISOString() } + const v = value as BsonValue + switch (v._bsontype) { + case undefined: + return { type: "object" } + case "ObjectId": + case "ObjectID": + return { type: "objectId", text: v.toHexString?.() ?? String(v) } + case "Decimal128": + return { type: "decimal128", text: String(v) } + case "Long": + return { type: "int64", text: String(v) } + case "Int32": + return { type: "int32", text: String(v) } + case "Double": + return { type: "double", text: String(v) } + case "Timestamp": + // `t` is seconds since the epoch and `i` the ordinal within that second. + return { type: "timestamp", text: typeof v.t === "number" ? `Timestamp(${v.t}, ${v.i})` : undefined } + case "BSONRegExp": + return { type: "regex", text: `/${v.pattern}/${v.options ?? ""}` } + case "Code": + return { type: "javascript", text: String(v.code) } + case "BSONSymbol": + return { type: "symbol", text: String(v.value) } + case "MinKey": + return { type: "minKey", text: "MinKey" } + case "MaxKey": + return { type: "maxKey", text: "MaxKey" } + case "Binary": { + if (v.sub_type !== 4 || typeof v.toUUID !== "function") return { type: "binary" } + const hex = String(v.toUUID().toHexString()).replace(/-/g, "") + return { type: "uuid", text: hex.replace(UUID_DASHES, "$1-$2-$3-$4-$5") } + } + default: + return { type: String(v._bsontype).toLowerCase() } + } +} + +/** Deepest field path reported, counting each key as one segment. */ +const NESTED_MAX_DEPTH = 3 +const NESTED_MAX_LISTED = 60 + +interface DocField { + /** Key segments from the document root; never re-split from `name`, since keys may contain ".". */ + path: string[] + /** MongoDB dot notation, as it is written in a filter or projection. */ + name: string + types: Set + /** Sampled documents containing the field. */ + docs: number + /** The path passes through an array of objects. */ + inArray: boolean +} + +function isPlainObject(value: unknown): value is Record { + if (typeof value !== "object" || value === null) return false + const proto = Object.getPrototypeOf(value) + return proto === Object.prototype || proto === null +} + +/** Fields of the sampled documents to depth 3: top-level keys plus nested keys, descending into arrays of objects. */ +function collectFields(docs: Record[]): DocField[] { + const fields = new Map() + for (const doc of docs) { + const seen = new Set() + const visit = (value: Record, parent: string[], inArray: boolean) => { + for (const [key, child] of Object.entries(value)) { + const path = [...parent, key] + const id = path.join("\0") + const field = fields.get(id) ?? { path, name: path.join("."), types: new Set(), docs: 0, inArray } + field.types.add(describeValue(child).type) + // A path seen both inside and outside arrays stays an array element. + field.inArray ||= inArray + if (!seen.has(id)) { + seen.add(id) + field.docs++ + } + fields.set(id, field) + if (path.length >= NESTED_MAX_DEPTH) continue + if (isPlainObject(child)) visit(child, path, inArray) + else if (Array.isArray(child)) { + for (const element of child) if (isPlainObject(element)) visit(element, path, true) + } + } + } + visit(doc, [], false) + } + return [...fields.values()] +} + +/** Values at a key path; arrays met on the way are searched element by element. */ +function valuesAt(doc: Record, path: string[]): unknown[] { + let current: unknown[] = [doc] + for (const key of path) { + const next: unknown[] = [] + for (const item of current) { + for (const candidate of Array.isArray(item) ? item : [item]) { + if (isRecord(candidate) && Object.hasOwn(candidate, key)) next.push(candidate[key]) + } + } + current = next } - const examples = p.text && p.values.length ? `, e.g. ${p.values.slice(0, PROFILE_EXAMPLES).map((v) => JSON.stringify(clipRaw(v))).join(", ")}` : "" - return `${p.name}: ${approx}${p.distinct.toLocaleString()} distinct${examples}${nulls}` + return current +} + +const CONTAINER_ONLY = /^(object|array)$/ + +function renderNested(fields: DocField[], docCount: number): string { + if (fields.length === 0) return "" + const listed = fields.slice(0, NESTED_MAX_LISTED).map((f) => { + const notes = [ + ...(f.inArray ? ["array element"] : []), + ...(f.docs < docCount ? [`in ${Math.round((f.docs / docCount) * 100)}% of docs`] : []), + ] + return `${f.name}: ${[...f.types].filter((t) => t !== "null").join(" | ") || "null"}${notes.length ? ` (${notes.join(", ")})` : ""}` + }) + const more = fields.length - listed.length + return [`Nested fields (sampled ${docCount} docs):`, ...listed, ...(more > 0 ? [`(${more} more not listed)`] : [])].join("\n") +} + +function profileField(docs: Record[], f: DocField): string | ColumnProfile { + const counts = new Map() + let present = 0 + let rendered = 0 + for (const doc of docs) { + let hasValue = false + let hasText = false + for (const raw of valuesAt(doc, f.path)) { + const { text } = describeValue(raw) + if (raw === null || raw === undefined) continue + hasValue = true + if (text === undefined) continue + hasText = true + counts.set(text, (counts.get(text) ?? 0) + 1) + } + if (hasValue) present++ + if (hasText) rendered++ + } + const types = [...f.types].filter((t) => t !== "null") + // Never report "all NULL" for values that exist but have no text form. + if (present > 0 && rendered === 0) return `${f.name}: values not profiled (${types.join(" | ")})` + return { + name: f.name, + total: docs.length, + nonNull: rendered, + nulls: docs.length - present, + distinct: counts.size, + // Counted on the full text; only the kept values are clipped. + values: rank([...counts].map(([value, count]) => ({ value, count }))) + .slice(0, PROFILE_FETCHED_VALUES) + .map((v) => ({ value: v.value.slice(0, PROFILE_FETCH_CHARS), count: v.count })), + text: types.some((t) => t === "string" || t === "uuid"), + multi: f.inArray, + typed: types.length === 1 && types[0] === "uuid", + } +} + +/** + * Nested-field listing and value profile for sampled documents, in the same format as the SQL + * profile. Fields come from the same documents the values do, so presence and counts agree. + */ +export function profileDocuments(docs: Record[]): string { + if (docs.length === 0) return "" + const fields = collectFields(docs).sort((a, b) => a.name.localeCompare(b.name)) + const nested = fields.filter((f) => f.path.length > 1) + const eligible = fields.filter((f) => { + const types = [...f.types].filter((t) => t !== "null") + return types.length === 0 || !types.every((t) => CONTAINER_ONLY.test(t)) + }) + // Top-level fields come first so the column cap never crowds them out with nested paths. + const ordered = [...eligible.filter((f) => f.path.length === 1), ...eligible.filter((f) => f.path.length > 1)] + const cols = ordered.slice(0, PROFILE_MAX_COLUMNS) + const results = cols.map((f) => profileField(docs, f)) + const sampled = docs.length >= PROFILE_SAMPLE_DOCUMENTS + const lines = results.map((r) => (typeof r === "string" ? r : renderProfile(r, sampled))) + const values = [ + `Value samples${sampled ? ` (first ${PROFILE_SAMPLE_DOCUMENTS} documents)` : ""}:`, + ...lines, + ...(ordered.length > cols.length ? [`(${ordered.length - cols.length} more columns not profiled)`] : []), + ].join("\n") + return [renderNested(nested, docs.length), values].filter(Boolean).join("\n\n") } export interface ValueProfileInput { @@ -129,6 +442,8 @@ export interface ValueProfileInput { table: string columns: ProfileColumn[] warehouse?: string + /** Database and collection, used instead of `table` for MongoDB. */ + document?: { database: string; collection: string } } /** Never throws: a failed profile becomes a one-line note. */ @@ -144,6 +459,9 @@ export async function profileValues( }> const target = input.warehouse ? warehouses.find((w) => w.name === input.warehouse) : warehouses[0] const dialect = profileDialect(target?.type) + if (target?.name && input.document && Registry.canonicalType(target.type) === "mongodb") { + return await profileCollection(input.document, target.name, ctx) + } if (!target?.name || !dialect) { return `Value samples skipped: not supported for ${target?.type ?? "this"} connections.` } @@ -171,18 +489,48 @@ export async function profileValues( if (error) throw new Error(error) if (!isRecord(data) || !Array.isArray(data.rows)) throw new Error("unexpected profiling response") const profiles = parseRows(data.rows, cols) - const lines = profiles.map(renderProfile) - if (lines.length === 0) return "" const sampled = profiles.some((p) => p.total >= PROFILE_SAMPLE_ROWS) - const skipped = eligible.length - cols.length - return [ - ...(precedence.notice ? [precedence.notice] : []), + return formatProfiles( + profiles, `Value samples${sampled ? ` (first ${PROFILE_SAMPLE_ROWS.toLocaleString()} rows)` : ""}:`, - ...lines, - ...(skipped > 0 ? [`(${skipped} more columns not profiled)`] : []), - ].join("\n") + eligible.length - cols.length, + sampled, + precedence.notice, + ) } catch (e) { const msg = (e instanceof Error ? e.message : String(e)).split("\n")[0]!.slice(0, 160) return `Value samples unavailable: ${msg}` } } + +type ProfileContext = Pick & { abort?: AbortSignal } + +/** + * Read-only: samples documents through the connector rather than `sql.execute`, which flattens + * each document to strings per column and would lose the nested structure the profile reads. + * The find is bounded by `maxTimeMS` and cancelled with the tool call. + */ +async function profileCollection( + document: { database: string; collection: string }, + warehouse: string, + ctx: ProfileContext, +): Promise { + const connector = await Registry.get(warehouse) + if (!connector.sampleDocuments) return "Value samples skipped: not supported for mongodb connections." + await ctx.ask({ + permission: "sql_execute", + patterns: [JSON.stringify({ ...document, command: "find", limit: PROFILE_SAMPLE_DOCUMENTS })], + always: ["*"], + metadata: { warehouse }, + }) + const precedence = await Precedence.check(ctx.sessionID, "sql_execute", warehouse) + if (precedence.redirect) return `Value samples skipped: ${precedence.redirect.output}` + const deadline = AbortSignal.timeout(PROFILE_TIMEOUT_MS) + const signal = ctx.abort ? AbortSignal.any([ctx.abort, deadline]) : deadline + const docs = await connector.sampleDocuments(document.database, document.collection, PROFILE_SAMPLE_DOCUMENTS, { + signal, + maxTimeMS: PROFILE_TIMEOUT_MS, + }) + const profile = profileDocuments(docs) + return profile && precedence.notice ? `${precedence.notice}\n${profile}` : profile +} diff --git a/packages/opencode/test/altimate/drivers-mongodb-e2e.test.ts b/packages/opencode/test/altimate/drivers-mongodb-e2e.test.ts index 7741fdc4ab..cdf7cc36d9 100644 --- a/packages/opencode/test/altimate/drivers-mongodb-e2e.test.ts +++ b/packages/opencode/test/altimate/drivers-mongodb-e2e.test.ts @@ -871,6 +871,42 @@ describe.skipIf(!DOCKER && !MONGODB_USE_CI)("MongoDB Driver E2E", () => { ) }) + test("sampleDocuments + profileDocuments — nested fields and values; describeTable stays top-level", async () => { + const { profileDocuments } = await import("../../src/altimate/tools/schema-values") + await connector.execute(cmd({ command: "createCollection", database: "testdb", name: "nested_profile" })) + await connector.execute( + cmd({ + command: "insertMany", + database: "testdb", + collection: "nested_profile", + documents: [ + { status: "open", address: { city: "Oslo" }, items: [{ sku: "S1" }, { sku: "S2" }] }, + { status: "open", address: { city: "Rome" }, items: [{ sku: "S1" }] }, + { status: "closed", address: { city: "Oslo" }, items: [] }, + ], + }), + ) + try { + const columns = await connector.describeTable("testdb", "nested_profile") + expect(columns.map((c) => [c.name, c.data_type])).toEqual([ + ["_id", "objectId"], + ["address", "object"], + ["items", "array"], + ["status", "string"], + ]) + const docs = await connector.sampleDocuments!("testdb", "nested_profile", 100, { maxTimeMS: 15_000 }) + expect(docs).toHaveLength(3) + const out = profileDocuments(docs) + expect(out).toContain("Nested fields (sampled 3 docs):\naddress.city: string\nitems.sku: string (array element, in 67% of docs)") + expect(out).toContain("status: 2 distinct — open (2), closed (1)") + expect(out).toContain("address.city: 2 distinct — Oslo (2), Rome (1)") + expect(out).toContain("items.sku: 2 distinct — S1 (2), S2 (1)") + expect(out).toContain("_id: 3 distinct, unique in sample") + } finally { + await connector.execute(cmd({ command: "dropCollection", database: "testdb", collection: "nested_profile" })) + } + }) + test("describeTable — mixed-type fields show union type", async () => { await connector.execute( cmd({ diff --git a/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts b/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts index a791176f10..55dd96c7b8 100644 --- a/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts +++ b/packages/opencode/test/altimate/tools/schema-inspect-mongodb.test.ts @@ -75,4 +75,42 @@ describe("schema_inspect MongoDB collections", () => { expect(result.output).toContain('Collection "empty" has no documents to infer a schema') expect(result.output).not.toContain("was not found") }) + + test("profiles one document sample, lists nested fields, and bounds the read", async () => { + const connector = configure("records") + const docs = Array.from({ length: 10 }, (_, i) => ({ + id: `e${i}`, + status: i < 7 ? "open" : "closed", + meta: { source: i % 2 ? "web" : "api" }, + items: [{ sku: "S1" }], + })) + const sample = mock(async (..._args: unknown[]) => docs) + Object.assign(connector, { sampleDocuments: sample }) + const abort = new AbortController() + const tool = await initTool(SchemaInspectTool) + const result = await tool.execute({ warehouse: "documents", table: "records.events" }, { ...ctx, abort: abort.signal }) + expect(result.metadata.success).toBe(true) + expect(result.output).not.toContain("skipped") + expect(result.output).toContain("Nested fields (sampled 10 docs):\nitems.sku: string (array element)\nmeta.source: string") + expect(result.output).toContain("id: 10 distinct, unique in sample — e0, e1") + expect(result.output).toContain("status: 2 distinct — open (7), closed (3)") + expect(result.output).toContain("meta.source: 2 distinct — api (5), web (5)") + expect(result.output).not.toContain("\nmeta:") + expect(sample.mock.calls).toHaveLength(1) + const [schema, table, limit, options] = sample.mock.calls[0] as unknown as [string, string, number, { signal: AbortSignal; maxTimeMS: number }] + expect([schema, table, limit, options.maxTimeMS]).toEqual(["records", "events", 100, 15_000]) + expect(options.signal.aborted).toBe(false) + abort.abort() + expect(options.signal.aborted).toBe(true) + }) + + test("a failing document sample keeps the schema", async () => { + const connector = configure("records") + Object.assign(connector, { sampleDocuments: mock(async () => { throw new Error("not authorized") }) }) + const tool = await initTool(SchemaInspectTool) + const result = await tool.execute({ warehouse: "documents", table: "records.events" }, ctx) + expect(result.metadata.success).toBe(true) + expect(result.output).toContain("id | STRING") + expect(result.output).toContain("Value samples unavailable: not authorized") + }) }) diff --git a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts index 00bdbb2358..2327b0caee 100644 --- a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts +++ b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts @@ -5,7 +5,8 @@ import * as Dispatcher from "../../../src/altimate/native/dispatcher" import * as Registry from "../../../src/altimate/native/connections/registry" import { registerAll } from "../../../src/altimate/native/connections/register" import { SchemaInspectTool } from "../../../src/altimate/tools/schema-inspect" -import { buildProfileSql } from "../../../src/altimate/tools/schema-values" +import { createRequire } from "module" +import { buildProfileSql, profileDocuments, valueShape } from "../../../src/altimate/tools/schema-values" const ctx = { sessionID: "test-session", @@ -68,9 +69,11 @@ afterAll(() => { function expectAppendix(output: string) { expect(output).toContain("Value samples:") expect(output).toContain("id: 60 distinct") - expect(output).toMatch(/region: 4 distinct — (north|south|east|west)(, (north|south|east|west)){3}/) - expect(output).toMatch(/customer_code: 60 distinct, e\.g\. "[^"]+", "[^"]+", "[^"]+"/) - expect(output).toContain("note: 1 distinct — x, 50% null") + expect(output).toContain("id: 60 distinct, unique in sample\n") + expect(output).toMatch(/region: 4 distinct — (north|south|east|west) \(15\)(, (north|south|east|west) \(15\)){3}/) + expect(output).toMatch(/customer_code: 60 distinct, unique in sample, e\.g\. "[^"]+", "[^"]+", "[^"]+", shape /) + expect(output).toContain("shape ` A-9+` 40%, `A-9+` 40%, `a9+` 20%") + expect(output).toContain("note: 1 distinct — x (30), 50% null") } describe("schema_inspect value grounding", () => { @@ -98,7 +101,7 @@ describe("schema_inspect value grounding", () => { install("sqlite", async (sql) => small.query(sql).values() as unknown[][], []) const result = await inspect({}) small.close() - expect(result.output).toMatch(/customer_code: 3 distinct — .*" C-00019"/) + expect(result.output).toMatch(/customer_code: 3 distinct, unique in sample — .*" C-00019"/) expect(result.output).toContain("c00018") expect(result.output).toContain("note: all NULL") }) @@ -110,8 +113,8 @@ describe("schema_inspect value grounding", () => { small.run("INSERT INTO t VALUES (1, 'left␟right', '', NULL)") install("sqlite", async (sql) => small.query(sql).values() as unknown[][], []) const result = await inspect({}) - expect(result.output).toContain("region: 1 distinct — left␟right\n") - expect(result.output).toContain('customer_code: 1 distinct — ""\n') + expect(result.output).toContain("region: 1 distinct — left␟right (1)\n") + expect(result.output).toContain('customer_code: 1 distinct — "" (1)\n') } finally { small.close() } @@ -152,7 +155,7 @@ describe("schema_inspect value grounding", () => { test("unsupported engines are skipped with a note", async () => { const calls: string[] = [] install("sqlite", run, calls) - Dispatcher.register("warehouse.list" as any, async () => ({ warehouses: [{ name: "w", type: "mongodb" }] })) + Dispatcher.register("warehouse.list" as any, async () => ({ warehouses: [{ name: "w", type: "snowflake" }] })) const result = await inspect({}) expect(calls).toHaveLength(0) expect(result.output).toContain("Value samples skipped") @@ -173,13 +176,13 @@ describe("schema_inspect value grounding", () => { const result = await inspect({}) expectAppendix(result.output) expect(calls).toHaveLength(1) - expect(calls[0]).toContain("array_agg(v)") + expect(calls[0]).toContain("array_agg(CAST(pn AS TEXT)") expect(calls[0]).toContain('FROM "main"."t"') await conn.execute("DELETE FROM t", 1000) await conn.execute("INSERT INTO t VALUES (1, 'left␟right', '', NULL)", 1000) const edgeValues = await inspect({}) - expect(edgeValues.output).toContain("region: 1 distinct — left␟right\n") - expect(edgeValues.output).toContain('customer_code: 1 distinct — ""\n') + expect(edgeValues.output).toContain("region: 1 distinct — left␟right (1)\n") + expect(edgeValues.output).toContain('customer_code: 1 distinct — "" (1)\n') } finally { await conn.close() } @@ -189,16 +192,16 @@ describe("schema_inspect value grounding", () => { test("postgres uses array_agg and preserves quoted identifiers containing an aggregate name", async () => { const calls: string[] = [] const column = "group_concat(value" - install("postgres", async () => [[column, 1, 1, 1, ["left␟right"]]], calls) + install("postgres", async () => [[column, 1, 1, 1, ["1:left␟right"]]], calls) Dispatcher.register("schema.inspect" as any, async () => ({ table: "t", schema_name: "sales", columns: [{ name: column, data_type: "TEXT", nullable: false }], })) const result = await inspect({ table: "sales.t" }) - expect(calls[0]).toContain("array_agg(v)") + expect(calls[0]).toContain("array_agg(CAST(pn AS TEXT)") expect(calls[0]).toContain('CAST("group_concat(value" AS TEXT)') expect(calls[0]).toContain("'group_concat(value' AS col") expect(calls[0]).toContain('FROM "sales"."t"') - expect(result.output).toContain("group_concat(value: 1 distinct — left␟right") + expect(result.output).toContain("group_concat(value: 1 distinct — left␟right (1)") }) test.each(["postgres", "postgresql", "sqlite", "duckdb"])( @@ -211,14 +214,14 @@ describe("schema_inspect value grounding", () => { })) const execute = mock(async () => ({ columns: ["col", "total", "nonnull", "ndistinct", "vals"], - rows: [["region", 1, 1, 1, ["north"]]], + rows: [["region", 1, 1, 1, ["1:north"]]], row_count: 1, truncated: false, })) const get = spyOn(Registry, "get").mockResolvedValue({ execute } as any) try { const result = await inspect({ warehouse: "w" }) - expect(result.output).toContain("region: 1 distinct — north") + expect(result.output).toContain("region: 1 distinct — north (1)") expect(execute).toHaveBeenCalledTimes(1) const [sql, limit, binds, options] = execute.mock.calls[0] as unknown as [string, number, unknown, Record] expect(sql).toContain('FROM "public"."t"') @@ -238,3 +241,251 @@ describe("schema_inspect value grounding", () => { expect(buildProfileSql('"t"', cols.slice(0, 40)).split("UNION ALL")).toHaveLength(40) }) }) + +const RICH_COLUMNS = ["id", "status", "cpc", "tag", "abstract", "title", "name"].map((name) => ({ + name, + data_type: name === "id" ? "INTEGER" : "TEXT", + nullable: true, + primary_key: name === "id", +})) +const NAMES = Array.from({ length: 25 }, (_, i) => String.fromCharCode(65 + (i % 25)) + "ob".repeat(1 + (i % 3)) + "by") +const TITLE_FORMS = [(i: number) => `Alpha ${i}`, (i: number) => `alpha-${i}`, (i: number) => `A, b ${i}`, (i: number) => `A b c ${i}`, + (i: number) => `Q(${i})`, (i: number) => `x_${i}_y`, (i: number) => `R${i}/${i}`, (i: number) => `N.${i}`] + +/** 200 rows with skewed statuses, patent-like codes, a skewed tag, long abstracts and free-form titles. */ +function richSeed(): string[] { + const rows: string[] = [] + for (let i = 0; i < 200; i++) { + const status = i < 120 ? "open" : i < 170 ? "closed" : "pending" + const tag = i < 80 ? "alpha" : i < 120 ? "beta" : i < 140 ? "gamma" : `t${i}` + const abstract = `This abstract describes item number ${i} in a rather long and detailed manner, so the field is clearly prose. ${"It keeps going with more explanatory sentences for the reader. ".repeat(3)}` + rows.push( + `INSERT INTO t VALUES (${i}, '${status}', 'H01M${i % 9}/${9000 + i}', '${tag}', '${abstract}', '${TITLE_FORMS[i % 8]!(i % 6)}', '${NAMES[i % 25]}')`, + ) + } + return ["CREATE TABLE t (id INTEGER, status TEXT, cpc TEXT, tag TEXT, abstract TEXT, title TEXT, name TEXT)", ...rows] +} + +function expectRichProfile(output: string) { + expect(output).toContain("id: 200 distinct, unique in sample\n") + // ranked by frequency, not alphabetically + expect(output).toContain("status: 3 distinct — open (120), closed (50), pending (30)\n") + // collapsed run classes: H01M4/9066 -> A9+A9/9+ + expect(output).toContain("cpc: 200 distinct, unique in sample, e.g.") + expect(output).toContain("shape `A9+A9/9+`") + // high cardinality shows the most frequent values with counts + expect(output).toContain('tag: 63 distinct, e.g. "alpha" (80), "beta" (40), "gamma" (20)') + // long prose: one 160-character example, the rest clipped at 40, and no shape + const abstract = output.split("\n").find((l) => l.startsWith("abstract:"))! + expect(abstract).toContain("unique in sample") + expect(abstract).not.toContain("shape") + const quoted = abstract.match(/"[^"]*"/g)! + expect(quoted[0]!.length - 2).toBe(161) + expect(quoted[1]!.length - 2).toBe(41) + // free-form text with many shapes and plain single-shape words get no shape + expect(output.split("\n").find((l) => l.startsWith("title:"))).not.toContain("shape") + expect(output.split("\n").find((l) => l.startsWith("name:"))).not.toContain("shape") +} + +describe("schema_inspect richer value profiles", () => { + const useRichSchema = () => + Dispatcher.register("schema.inspect" as any, async () => ({ table: "t", schema_name: "main", columns: RICH_COLUMNS })) + + test("sqlite: frequency order, uniqueness, shapes and long-text examples", async () => { + const db = new Database(":memory:") + try { + richSeed().forEach((q) => db.run(q)) + const calls: string[] = [] + install("sqlite", async (sql) => db.query(sql).values() as unknown[][], calls) + useRichSchema() + const result = await inspect({}) + expect(calls).toHaveLength(1) + expect(calls[0]).toContain("json_group_array(CAST(pn AS TEXT) || ':' || SUBSTR(pv, 1, 200))") + expect(calls[0]).toContain("ORDER BY pn DESC, pv LIMIT 50") + expectRichProfile(result.output) + } finally { + db.close() + } + }) + + test("duckdb: same profile from a real engine", async () => { + const { connect } = await import("../../../../drivers/src/duckdb") + const dir = (await import("fs")).mkdtempSync((await import("os")).tmpdir() + "/schemavals-rich-") + const conn = await connect({ type: "duckdb", path: dir + "/t.duckdb", create: true } as any) + try { + await conn.connect() + for (const q of richSeed()) await conn.execute(q, 1000) + const calls: string[] = [] + install("duckdb", async (sql) => (await conn.execute(sql, 1000)).rows as unknown[][], calls) + useRichSchema() + const result = await inspect({}) + expect(calls[0]).toContain("array_agg(CAST(pn AS TEXT) || ':' || SUBSTR(pv, 1, 200))") + expectRichProfile(result.output) + } finally { + await conn.close() + } + }) + + const LONG_COLUMNS = [{ name: "id", data_type: "INTEGER", nullable: true, primary_key: false }, { name: "body", data_type: "TEXT", nullable: true, primary_key: false }] + const longSeed = () => [ + "CREATE TABLE t (id INTEGER, body TEXT)", + ...Array.from({ length: 30 }, (_, i) => `INSERT INTO t VALUES (${i}, '${"p".repeat(200)}-${String(i).padStart(2, "0")}')`), + ] + const expectLongDistinct = (output: string) => { + const line = output.split("\n").find((l) => l.startsWith("body:"))! + expect(line).toStartWith("body: 30 distinct, unique in sample, e.g. ") + expect(line).not.toMatch(/\(\d+\)/) + } + + test("sqlite: long values sharing a 200-char prefix stay distinct", async () => { + const db = new Database(":memory:") + try { + longSeed().forEach((q) => db.run(q)) + install("sqlite", async (sql) => db.query(sql).values() as unknown[][], []) + Dispatcher.register("schema.inspect" as any, async () => ({ table: "t", schema_name: "main", columns: LONG_COLUMNS })) + expectLongDistinct((await inspect({})).output) + } finally { + db.close() + } + }) + + test("duckdb: long values sharing a 200-char prefix stay distinct", async () => { + const { connect } = await import("../../../../drivers/src/duckdb") + const dir = (await import("fs")).mkdtempSync((await import("os")).tmpdir() + "/schemavals-long-") + const conn = await connect({ type: "duckdb", path: dir + "/t.duckdb", create: true } as any) + try { + await conn.connect() + for (const q of longSeed()) await conn.execute(q, 1000) + install("duckdb", async (sql) => (await conn.execute(sql, 1000)).rows as unknown[][], []) + Dispatcher.register("schema.inspect" as any, async () => ({ table: "t", schema_name: "main", columns: LONG_COLUMNS })) + expectLongDistinct((await inspect({})).output) + } finally { + await conn.close() + } + }) + + test("postgres SQL uses standard aggregation and the same ranked subquery", () => { + const sql = buildProfileSql('"public"."t"', [{ name: "status", data_type: "text" }], "postgres") + expect(sql).toContain("array_agg(CAST(pn AS TEXT) || ':' || SUBSTR(pv, 1, 200))") + expect(sql).toContain("GROUP BY CAST(\"status\" AS TEXT) ORDER BY pn DESC, pv LIMIT 50") + expect(sql).toContain("SUBSTR(pv, 1, 200)") + expect(sql).not.toContain("json_group_array") + expect(sql).not.toMatch(/FILTER|WITHIN GROUP|STRING_AGG|GROUP_CONCAT/i) + }) + + test("a repeated key is not unique, and every-value-once is", async () => { + const db = new Database(":memory:") + try { + db.run("CREATE TABLE t (id INTEGER, status TEXT, cpc TEXT, tag TEXT, abstract TEXT, title TEXT, name TEXT)") + db.run("INSERT INTO t VALUES (1,'a','a','a','a','a','a'), (1,'b','b','b','b','b','b')") + install("sqlite", async (sql) => db.query(sql).values() as unknown[][], []) + useRichSchema() + const result = await inspect({}) + expect(result.output).toContain("id: 1 distinct — 1 (2)") + expect(result.output).toContain("status: 2 distinct, unique in sample — a, b") + } finally { + db.close() + } + }) + + test("valueShape collapses runs of one class", () => { + expect(valueShape("H01M4/9066")).toBe("A9+A9/9+") + expect(valueShape("Hello, World 42")).toBe("Aa+, Aa+ 9+") + expect(valueShape("ab-12")).toBe("a+-9+") + expect(valueShape("")).toBe("") + }) +}) + +// Real BSON values: `bson` is a dependency of `mongodb`, which the drivers package owns. +const driversRequire = createRequire(new URL("../../../../drivers/package.json", import.meta.url)) +const bson = createRequire(driversRequire.resolve("mongodb"))("bson") as any + +describe("MongoDB value profile", () => { + test("renders real BSON types, dotted keys, nested fields and unrenderable values", () => { + const uuid = new bson.UUID("0f8fad5b-d9cb-469f-a165-70867728950e") + const docs = Array.from({ length: 4 }, (_, i) => ({ + _id: new bson.ObjectId("65a1b2c3d4e5f60718293a4" + i), + ref: i === 0 ? uuid : new bson.UUID(), + price: bson.Decimal128.fromString("12.50"), + big: bson.Long.fromString(`900719925474099${i}`), + pattern: new bson.BSONRegExp("^ab+c", "i"), + script: new bson.Code("function () { return 1 }"), + stamp: new bson.Timestamp({ t: 1700000000, i: 7 }), + created: new Date("2024-03-01T10:00:00Z"), + raw: new bson.Binary(Buffer.from([1, 2, 3])), + "a.b": i % 2 ? "x" : "y", + address: { city: "Oslo", "x.y": i, geo: { lat: 1.5 } }, + items: [{ sku: "S1" }, { sku: i === 3 ? "S2" : "S1" }], + min: new bson.MinKey(), + tags: ["t"], + })) + const out = profileDocuments(docs) + const line = (prefix: string) => + out.slice(out.indexOf("Value samples")).split("\n").find((l) => l.startsWith(prefix + ":")) + expect(line("ref")).toBe( + `ref: 4 distinct, unique in sample — ${docs.map((d) => d.ref.toString()).sort().join(", ")}`, + ) + expect(line("ref")).toContain("0f8fad5b-d9cb-469f-a165-70867728950e") + expect(line("ref")).not.toContain("NULL") + expect(line("_id")).toBe("_id: 4 distinct, unique in sample — 65a1b2c3d4e5f60718293a40, 65a1b2c3d4e5f60718293a41, 65a1b2c3d4e5f60718293a42, 65a1b2c3d4e5f60718293a43") + expect(line("price")).toBe("price: 1 distinct — 12.50 (4)") + expect(line("big")).toBe(`big: 4 distinct, unique in sample — ${docs.map((d) => d.big.toString()).sort().join(", ")}`) + expect(line("big")).toContain("9007199254740990, 9007199254740991") + expect(docs[0]!.big.toString()).toMatch(/^\d{16}$/) + expect(line("pattern")).toBe("pattern: 1 distinct — /^ab+c/i (4)") + expect(line("script")).toBe("script: 1 distinct — function () { return 1 } (4)") + expect(line("stamp")).toBe("stamp: 1 distinct — Timestamp(1700000000, 7) (4)") + expect(line("created")).toBe("created: 1 distinct — 2024-03-01T10:00:00.000Z (4)") + expect(line("min")).toBe("min: 1 distinct — MinKey (4)") + expect(line("raw")).toBe("raw: values not profiled (binary)") + // a real top-level key containing "." is found, and nested keys containing "." are too + expect(line("a.b")).toBe("a.b: 2 distinct — x (2), y (2)") + expect(line("address.x.y")).toBe("address.x.y: 4 distinct, unique in sample — 0, 1, 2, 3") + expect(line("address.city")).toBe("address.city: 1 distinct — Oslo (4)") + // MongoDB dot notation for array elements, with membership in the nested listing + expect(line("items.sku")).toBe("items.sku: 2 distinct — S1 (7), S2 (1)") + expect(out).toContain("Nested fields (sampled 4 docs):") + expect(out).toContain("items.sku: string (array element)") + expect(out).toContain("address.geo.lat: double") + expect(out).not.toContain("[]") + expect(out).not.toContain("[object") + expect(line("tags")).toBeUndefined() + expect(line("address")).toBeUndefined() + }) + + test("a path seen outside and then inside an array is an array element, never unique", () => { + const docs = [{ tag: { id: "a" } }, { tag: [{ id: "b" }, { id: "c" }] }] + const out = profileDocuments(docs) + expect(out).toContain("tag.id: string (array element)") + const line = out.slice(out.indexOf("Value samples")).split("\n").find((l) => l.startsWith("tag.id:"))! + expect(line).not.toContain("unique in sample") + }) + + test("long values sharing a 200-char prefix stay distinct and unique", () => { + const docs = Array.from({ length: 30 }, (_, i) => ({ body: `${"p".repeat(200)}-${String(i).padStart(2, "0")}` })) + const line = profileDocuments(docs).split("\n").find((l) => l.startsWith("body:"))! + expect(line).toStartWith("body: 30 distinct, unique in sample, e.g. ") + expect(line).not.toMatch(/\(\d+\)/) + }) + + test("a field whose values are all null is NULL, one with unrenderable values is not", () => { + const docs = [{ gone: null, blob: new bson.Binary(Buffer.from([1])) }, { gone: null }] + const out = profileDocuments(docs) + expect(out).toContain("gone: all NULL") + expect(out).toContain("blob: values not profiled (binary)") + }) + + test("high-cardinality text shows frequent examples and a shape", () => { + const many = Array.from({ length: 30 }, (_, i) => ({ code: i < 10 ? "ZZ-1" : `ab${i}-${100 + i}` })) + const out = profileDocuments(many) + expect(out).toContain('code: 21 distinct, e.g. "ZZ-1" (10),') + expect(out).toContain("shape `a+9+-9+` 67%, `A+-9` 33%") + }) + + test("a 100-document sample is labelled approximate", () => { + const many = Array.from({ length: 100 }, (_, i) => ({ k: i % 2 ? "a" : "b" })) + const out = profileDocuments(many) + expect(out).toContain("Value samples (first 100 documents):") + expect(out).toContain("k: ~2 distinct — a (50), b (50)") + }) +})