diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 154e240ef6..ce525b2d15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,20 @@ jobs: - '.github/workflows/ci.yml' # altimate_change end drivers: + # altimate_change start — run real MCP drivers for MCP/tool-only changes + - 'packages/opencode/src/altimate/mcp-server/**' + - 'packages/opencode/test/altimate/mcp-server/**' + - 'packages/opencode/src/altimate/tools/schema-inspect.ts' + - 'packages/opencode/src/altimate/tools/schema-values.ts' + # The tools the MCP server exposes; server.test.ts is their only real-driver run. + - 'packages/opencode/src/altimate/tools/warehouse-list.ts' + - 'packages/opencode/src/altimate/tools/warehouse-add.ts' + - 'packages/opencode/src/altimate/tools/sql-execute.ts' + - 'packages/opencode/src/altimate/tools/sql-check.ts' + - 'packages/opencode/src/altimate/tools/sql-materialize.ts' + - 'packages/opencode/src/altimate/tools/answer-commit*.ts' + - 'packages/opencode/src/tool/truncate.ts' + # altimate_change end - 'packages/drivers/src/**' - 'packages/opencode/src/altimate/native/connections/**' - 'packages/opencode/test/altimate/drivers-e2e.test.ts' @@ -512,6 +526,14 @@ jobs: env: ALTIMATE_DUCKDB_E2E: "1" + # Same reason as above: the in-process MCP server tests use real SQLite/DuckDB drivers, which the + # whole-package run replaces with module mocks, so they run in their own process. + - name: Run MCP server tests (real drivers, no mocks) + run: bun test --timeout 90000 test/altimate/mcp-server/server.test.ts + working-directory: packages/opencode + env: + ALTIMATE_MCP_E2E: "1" + - name: Run Docker driver E2E (MySQL, SQL Server, Redshift) run: bun test test/altimate/drivers-docker-e2e.test.ts working-directory: packages/opencode diff --git a/docs/docs/configure/mcp-servers.md b/docs/docs/configure/mcp-servers.md index c2b860e7b1..afabe54ba1 100644 --- a/docs/docs/configure/mcp-servers.md +++ b/docs/docs/configure/mcp-servers.md @@ -6,6 +6,43 @@ description: "Configure Model Context Protocol (MCP) servers in Altimate Code to altimate supports the Model Context Protocol (MCP) for connecting to external tool servers. +## Use Altimate Code tools from another MCP client + +Run `altimate-code mcp serve` to expose data tools over stdio. Your MCP client's model handles reasoning; this server runs tools directly without calling a model. + +Add this entry to your MCP client's configuration (for example, Claude Code's `.mcp.json`): + +```json +{ + "mcpServers": { + "altimate-code": { + "command": "altimate-code", + "args": ["mcp", "serve", "--connections", "/absolute/path/to/connections.json"] + } + } +} +``` + +By default the server exposes `warehouse_list`, `warehouse_add`, `sql_execute`, `sql_materialize` and `answer_commit`. Pass `tables` to `sql_execute` to get those tables' columns and value profiles in the same call as its queries. Start with `--all-tools` to also expose `schema_inspect` and `sql_check`. Without `--connections`, the server reads the usual connection sources and re-reads them while none are configured, so a connections file written mid-session, or a connection added with `warehouse_add`, is picked up without a restart. A connections file is a JSON object keyed by connection name: + +```json +{ + "local": { "type": "sqlite", "path": "./analytics.sqlite" } +} +``` + +Relative database paths resolve beside this file. With `--connections`, this file is the only source of connections: global and project `connections.json` files and `ALTIMATE_CODE_CONN_*` overrides are ignored. Omit `--connections` to use normal connection discovery: global and project `.altimate-code/connections.json` files, followed by `ALTIMATE_CODE_CONN_*` environment overrides. Set your client's working directory to the project when using project discovery. + +At startup the server lists each connection's name and type in its instructions, read from configuration only: it does not connect to remote or server databases or run queries on them. For local SQLite and DuckDB files (not behind an SSH tunnel) it also lists table names from one catalog query (no row counts); a file with more than 40 tables is noted as such instead. Call `warehouse_list` for columns and sample rows. Listing tools use catalog row estimates where available (including DuckDB); SQLite listings omit row counts. They never count rows by scanning each table. + + + +Source writes are refused by default. Without `--allow-write`, source queries request database-enforced read-only execution on PostgreSQL, SQLite and DuckDB. For other connection types, read-only is enforced by SQL classification only; stored functions with side effects are not blocked — use a read-only database user. The server prints this caveat on stderr at startup and includes it in `warehouse_list` for each affected connection. Add `--allow-write` to enable SQL write permissions; existing hard safety blocks still apply. `sql_materialize` always requires a read-only source query and writes only into the server session's temporary `scratch` warehouse. Scratch tables disappear when the server exits. Large results may include a local spill file path that your client can open with any file reader. Spill files use a private directory and are deleted when the server exits. Local `tool_output.max_lines` and `tool_output.max_bytes` settings control output limits; remote agent configuration is not loaded. + + + +The server writes only MCP protocol messages to stdout and sends logs to stderr. It does not initialize telemetry and honors `ALTIMATE_TELEMETRY_DISABLED`. + ## Local MCP Servers Run an MCP server as a local subprocess: diff --git a/packages/drivers/src/mongodb.ts b/packages/drivers/src/mongodb.ts index bc580f00dd..2f8efe1e81 100644 --- a/packages/drivers/src/mongodb.ts +++ b/packages/drivers/src/mongodb.ts @@ -42,6 +42,9 @@ type MqlCommand = | "dropCollection" | "createIndex" | "listIndexes" + // altimate_change start — scalar database catalog counts + | "dbStats" + // altimate_change end | "listCollections" | "ping" @@ -514,7 +517,7 @@ export async function connect(config: ConnectionConfig): Promise { maxResultBytes?: number, signal?: AbortSignal, ): Promise { - async function readCursor(cursor: any): Promise[]> { + async function readCursor(cursor: any, rowLimit?: number): Promise[]> { const consume = createResultByteBudget(maxResultBytes) let stopped = false let failure: unknown @@ -539,7 +542,7 @@ export async function connect(config: ConnectionConfig): Promise { }) const read = async () => { if (stopped) throw failure - if (maxResultBytes === undefined && maxTimeMS === undefined && !signal) return cursor.toArray() + if (rowLimit === undefined && maxResultBytes === undefined && maxTimeMS === undefined && !signal) return cursor.toArray() // One document per round trip made a 127k-document read take ~10 minutes. A reply // batch is still bounded (a server reply never exceeds 16 MiB) and the byte budget // stops the read at the first document past the cap. @@ -548,6 +551,7 @@ export async function connect(config: ConnectionConfig): Promise { if (stopped) throw failure if (maxResultBytes === undefined) { docs.push(doc) + if (rowLimit !== undefined && docs.length >= rowLimit) break continue } const serialized = Object.fromEntries( @@ -605,17 +609,20 @@ export async function connect(config: ConnectionConfig): Promise { return { columns: ["ok"], rows: [[result.ok]], row_count: 1, truncated: false } } + // altimate_change start — catalog discovery consumes only a bounded cursor, never the metadata cache + if (cmd === "dbStats") { + const stats = await db.command({ dbStats: 1, ...withMaxTime(maxTimeMS) }) + return { columns: ["collections"], rows: [[stats.collections]], row_count: 1, truncated: false } + } if (cmd === "listCollections") { - const collections = await readCollections(db, maxTimeMS) - const matches = parsed.filter - ? await db.listCollections(parsed.filter, { - nameOnly: true, authorizedCollections: true, ...withMaxTime(maxTimeMS), - }).toArray() - : collections + const matches = await readCursor(db.listCollections(parsed.filter ?? {}, { + nameOnly: true, authorizedCollections: true, batchSize: Math.min(effectiveLimit + 1, 501), ...withMaxTime(maxTimeMS), + }), effectiveLimit + 1) const rows = matches.slice(0, effectiveLimit) - .map((collection: { name: string; type: string }) => [collection.name, collection.type]) + .map((collection) => [collection.name, collection.type]) return { columns: ["name", "type"], rows, row_count: rows.length, truncated: matches.length > effectiveLimit } } + // altimate_change end if (!MONGO_COLLECTION_READS.has(cmd)) clearMetadata() diff --git a/packages/drivers/test/mongodb-catalog-bounds.test.ts b/packages/drivers/test/mongodb-catalog-bounds.test.ts new file mode 100644 index 0000000000..a26daf4cd2 --- /dev/null +++ b/packages/drivers/test/mongodb-catalog-bounds.test.ts @@ -0,0 +1,74 @@ +// altimate_change start — schema discovery never materializes all collection metadata +import { expect, mock, test } from "bun:test" +import { connect } from "../src/mongodb" +import { restoreModuleMocksAfterTests } from "./module-mock" + +await restoreModuleMocksAfterTests("mongodb") + +test("two million MongoDB collections stop at 501 and close the cursor", async () => { + let fetched = 0 + let closed = 0 + const options: unknown[] = [] + const commands: unknown[] = [] + mock.module("mongodb", () => ({ + default: { + MongoClient: class { + async connect() {} + async close() {} + db() { + return { + databaseName: "large", + command: async (command: unknown) => { + commands.push(command) + return { collections: 2_000_000 } + }, + listCollections: (_filter: unknown, opts: unknown) => { + options.push(opts) + return { + async toArray() { + throw new Error("unbounded collection fetch") + }, + async close() { + closed++ + }, + async *[Symbol.asyncIterator]() { + for (let i = 0; i < 2_000_000; i++) { + fetched++ + yield { name: `c${i}`, type: "collection" } + } + }, + } + }, + } + } + }, + }, + })) + const connector = await connect({ type: "mongodb" }) + await connector.connect() + try { + const start = performance.now() + const result = await connector.execute( + JSON.stringify({ command: "listCollections", database: "large" }), + 500, + undefined, + { readOnly: true, queryTimeoutMs: 1000 }, + ) + expect(result.rows).toHaveLength(500) + expect(result.truncated).toBe(true) + expect(fetched).toBe(501) + expect(closed).toBe(1) + expect(performance.now() - start).toBeLessThan(1000) + expect(options[0]).toMatchObject({ nameOnly: true, authorizedCollections: true, batchSize: 501, maxTimeMS: 1000 }) + const counts = await connector.execute(JSON.stringify({ command: "dbStats", database: "large" }), 1, undefined, { + readOnly: true, + queryTimeoutMs: 1000, + }) + expect(counts.rows).toEqual([[2_000_000]]) + expect(fetched).toBe(501) + expect(commands).toEqual([{ dbStats: 1, maxTimeMS: 1000 }]) + } finally { + await connector.close() + } +}) +// altimate_change end diff --git a/packages/drivers/test/mongodb-unit.test.ts b/packages/drivers/test/mongodb-unit.test.ts index 78e9df1bb8..881139a8c4 100644 --- a/packages/drivers/test/mongodb-unit.test.ts +++ b/packages/drivers/test/mongodb-unit.test.ts @@ -37,15 +37,22 @@ async function fakeMongo( return { databases: Object.keys(collections).map((name) => ({ name })) } }, }), - listCollections: (filter: { name?: string }, options: unknown) => ({ - toArray: async () => { + // altimate_change start — model listCollections as a cursor, including bounded consumption + listCollections: (filter: { name?: string }, options: unknown) => { + const rows = () => { collectionLists.push({ database, options }) if (metadata.error) throw metadata.error return (collections[database] ?? []) .filter((name) => !filter.name || name === filter.name) .map((name) => ({ name, type: "collection" })) - }, - }), + } + return { + toArray: async () => rows(), + async *[Symbol.asyncIterator]() { yield* rows() }, + async close() {}, + } + }, + // altimate_change end createCollection: async (name: string) => { collections[database] ??= [] collections[database].push(name) @@ -174,7 +181,9 @@ describe("MongoDB read namespace validation", () => { expect(await connector.execute(JSON.stringify({ command: "listCollections", database: "archive" }))).toEqual({ columns: ["name", "type"], rows: [], row_count: 0, truncated: false, }) - expect(databaseLists).toEqual([{ nameOnly: true, authorizedDatabases: true, maxTimeMS: 120_000 }]) + // altimate_change start — an empty bounded cursor needs no unbounded metadata validation + expect(databaseLists).toEqual([]) + // altimate_change end }) test("metadata failures cannot become an empty read", async () => { diff --git a/packages/opencode/src/altimate/mcp-server/adapter.ts b/packages/opencode/src/altimate/mcp-server/adapter.ts new file mode 100644 index 0000000000..a485aa5440 --- /dev/null +++ b/packages/opencode/src/altimate/mcp-server/adapter.ts @@ -0,0 +1,462 @@ +import { FSUtil } from "@opencode-ai/core/fs-util" +import { NodePath } from "@effect/platform-node" +import { Effect, Layer } from "effect" +import { mkdtemp, chmod, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { randomUUID } from "node:crypto" +import { Agent } from "@/agent/agent" +import { Tool } from "@/tool/tool" +import { Truncate } from "@/tool/truncate" +import { outputLimits } from "./output-settings" +import { WarehouseListTool } from "../tools/warehouse-list" +import { WarehouseAddTool } from "../tools/warehouse-add" +import { SchemaInspectTool } from "../tools/schema-inspect" +import { SqlExecuteTool } from "../tools/sql-execute" +import { SqlCheckTool } from "../tools/sql-check" +import { SqlMaterializeTool } from "../tools/sql-materialize" +import { AnswerCommitTool, renderAnswer } from "../tools/answer-commit" +import { AnswerCommitConjunction } from "../tools/answer-commit-conjunction" +import * as Precedence from "../workspace/precedence" +import { closeScratch } from "../native/connections/scratch" +import type { SessionID, MessageID } from "@/session/schema" + +export const descriptions: Record = { + warehouse_list: "List connections; for local databases also tables, columns and a sample row.", + warehouse_add: "Add a connection when warehouse_list shows none or the one you need is missing.", + schema_inspect: "List tables when table is omitted; otherwise one table's columns and value profile.", + sql_execute: + "Run SQL and return rows. Chain queries in one call with `queries`: a later one can use IN ({{label.column}}) (an earlier result's values) or {{label}} (its rows as a scratch table, warehouse scratch). `tables` describes tables first. Source writes need --allow-write.", + sql_check: "Read-only data check: fanout, keys, grain, nulls or coverage.", + sql_materialize: "Store a read-only query's rows as a scratch table, to join across connections.", + answer_commit: + "Finish a data question; call last, then reply with exactly its FINAL ANSWER text. Use one of fields, rows or unresolved, with a field (or rows column) per asked item, named after it. Never collapse a compound request into one field: 'the repository and the most copied file' needs two. An item that only selects the answer ('the customer with the largest invoice') is not asked. Numbers may be your own computation; exact decimals as strings.", +} + +/** + * Short MCP property descriptions, keyed by property path (`a.b` nests, `a[]` is an array's items). + * Every tool definition is re-sent on each model request, so the advertised schema keeps only what a + * caller needs to choose and fill a property; `null` drops the description, `DROP` the property. + */ +const DROP = Symbol("drop") +const PROPERTY_DESCRIPTIONS: Record> = { + warehouse_add: { + name: "Connection name", + config: + 'Must include "type". postgres/mysql: host, port, database, user, password; mongodb: connection_string, ' + + "database; duckdb/sqlite: path (absolute, existing file); snowflake/bigquery/others: their usual fields", + }, + schema_inspect: { + table: "Table, optionally schema-qualified; omit to list tables", + schema_name: null, + warehouse: null, + values: "Include the value profile (default true)", + }, + sql_execute: { + query: "SQL, or a MongoDB JSON command", + queries: "Up to 8 queries, run in order, one labeled result each", + "queries[].label": null, + "queries[].query": null, + "queries[].warehouse": null, + query_from: "Run the SQL string a one-row SELECT returns; warehouse is the target", + "query_from.warehouse": null, + "query_from.sql": null, + warehouse: null, + limit: "Max rows (default 100, max 10000)", + full_text_columns: "Columns to show in full (20-row preview)", + query_timeout_ms: null, + }, + sql_check: { + sql: "SELECT for grain, nulls or coverage", + columns: "Columns for grain or nulls", + column: "Column for coverage", + pattern: "Regex (SQLite: LIKE pattern)", + left_sql: "SELECT for fanout or keys", + right_sql: "SELECT for fanout or keys", + on: "fanout: [left, right] column pairs", + join: "inner (default) or left", + left_col: "keys: left column", + right_col: "keys: right column", + left_warehouse: DROP, + right_warehouse: DROP, + query_timeout_ms: null, + }, + sql_materialize: { + source_warehouse: "Warehouse to read, or scratch", + sql: "Read-only SELECT", + target_table: "New scratch table name", + mode: "create (default), replace or append", + max_rows: null, + max_bytes: null, + query_timeout_ms: null, + }, + answer_commit: { + fields: null, + "fields[].name": null, + // Units can be part of a string value; the field stays accepted. + "fields[].units": DROP, + "fields[].precision": "Decimals", + asked: "Requested items, in order", + "rows.format": "lines: no header", + "rows.values": "N rows if N is asked, even with ties", + "rows.precision": "Decimals by column", + "rows.note": "Tie or caveat", + basis: "1-2 sentences: population, grain, source column, caveats", + answer_only: "Reply is only the value(s)/table", + readings: "If two readings fit: the chosen one; decided_by: the rejected one and why", + request: "The question, verbatim", + }, +} + +type JsonNode = Record +const isNode = (value: unknown): value is JsonNode => typeof value === "object" && value !== null && !Array.isArray(value) + +/** Strict-schema fillers the parsers accept for unused optional properties: null, "", [] and {}. */ +function isFiller(node: unknown): boolean { + if (!isNode(node)) return false + if (node.type === "null") return true + if (node.const === "" && node.type === "string") return true + if (node.type === "array" && node.maxItems === 0) return true + return node.type === "object" && isNode(node.properties) && Object.keys(node.properties).length === 0 && node.additionalProperties === false +} + +/** + * Collapses an optional property's filler union to its value schema (omitting the property means the + * same to the parsers) and drops keys that only restate what the parsers enforce. Unions elsewhere, + * such as nullable table cells, keep every member. + */ +function collapse(node: unknown, property = false): unknown { + if (Array.isArray(node)) return node.map((item) => collapse(item)) + if (!isNode(node)) return node + let copy: JsonNode = { ...node } + if (property && Array.isArray(copy.anyOf)) { + const kept = copy.anyOf.filter((member) => !isFiller(member)) + if (kept.length === 1 && isNode(kept[0])) { + const { anyOf: _, ...rest } = copy + copy = { ...(kept[0] as JsonNode), ...rest } + } else copy.anyOf = kept + } + for (const key of ["$schema", "default", "propertyNames", "minItems", "additionalProperties"]) { + if (key !== "additionalProperties" || copy[key] === false) delete copy[key] + } + if (copy.maximum === Number.MAX_SAFE_INTEGER) delete copy.maximum + for (const [key, value] of Object.entries(copy)) { + if (key === "properties" && isNode(value)) { + copy.properties = Object.fromEntries(Object.entries(value).map(([name, child]) => [name, collapse(child, true)])) + } else if (key === "items" || key === "prefixItems" || key === "anyOf" || (key === "additionalProperties" && isNode(value))) { + copy[key] = collapse(value) + } + } + return copy +} + +/** The property schema at a description path, or undefined when the path does not exist. */ +function propertyAt(schema: JsonNode, pathText: string): { owner: JsonNode; parent: JsonNode; name: string } | undefined { + let node: unknown = schema + const segments = pathText.split(".") + for (const [index, segment] of segments.entries()) { + const array = segment.endsWith("[]") + const name = array ? segment.slice(0, -2) : segment + const properties = isNode(node) ? node.properties : undefined + if (!isNode(node) || !isNode(properties) || !isNode(properties[name])) return undefined + if (index === segments.length - 1 && !array) return { owner: node, parent: properties, name } + node = array ? (properties[name] as JsonNode).items : properties[name] + } + return undefined +} + +/** The advertised MCP input schema: fillers collapsed and descriptions shortened. Parsing is unchanged. */ +export function compactSchema(name: string, schema: T): T { + const copy = collapse(schema) as JsonNode + for (const [pathText, description] of Object.entries(PROPERTY_DESCRIPTIONS[name] ?? {})) { + const target = propertyAt(copy, pathText) + if (!target) throw new Error(`MCP schema for ${name} has no property ${pathText}`) + const property = target.parent[target.name] as JsonNode + if (description === DROP) { + // Only optional properties are dropped; a required one would make every call fail. + if (Array.isArray(target.owner.required) && target.owner.required.includes(target.name)) + throw new Error(`MCP schema for ${name} cannot drop required property ${pathText}`) + delete target.parent[target.name] + } else if (description === null) delete property.description + else property.description = description + } + return copy as T +} + +// The native tool caps `basis` at 400 characters; over the MCP boundary the text is trimmed instead of rejected. +const BASIS_LIMIT = 400 +const MCP_ONLY_TOOLS = new Set(["answer_commit"]) + +/** + * Tool definitions are re-sent with every model request, so the default toolset is the five tools clients use; + * `sql_execute.tables` covers table inspection in the same call as the queries. `allTools` also advertises + * schema_inspect and sql_check. + */ +const LEAN_TOOLS = new Set(["warehouse_list", "warehouse_add", "sql_execute", "sql_materialize", "answer_commit"]) +const MAX_FUSED_TABLES = 8 + +/** Adds `tables` to the advertised sql_execute schema: tables described in the same call, before its queries. */ +function withTables(schema: T): T { + const copy = JSON.parse(JSON.stringify(schema)) as { properties?: Record } + copy.properties = { + tables: { + type: "array", + items: { type: "string" }, + maxItems: MAX_FUSED_TABLES, + description: "Tables to describe in this call (columns, value profile); query optional", + }, + ...copy.properties, + } + return copy as T +} + +function hasQuery(args: Record): boolean { + return Boolean(args.query) || Boolean(args.query_from) || (Array.isArray(args.queries) && args.queries.length > 0) +} + +/** Session-level record the native commit checks read: tool outputs and rejections of the current turn. */ +type TurnMessage = Tool.Context["messages"][number] +/** + * The FINAL ANSWER line carries only the answer, rendered as answer_only would, so the + * explanation never becomes part of the answer text a client copies. The basis is kept, + * above that line, as context for the user. + */ +export function finalAnswerText(args: Record, committed: string, request?: unknown): string { + // Answer-only only for a complete commit: rows, or fields covering every item the request names. + // Otherwise keep the committed text (basis on the line) so an item kept only in the basis is not lost. + const fields = Array.isArray(args.fields) ? args.fields.length : 0 + const complete = + Boolean(args.rows) || + (fields > 0 && fields >= AnswerCommitConjunction.requestedItems(typeof request === "string" ? request : "").count) + if (!complete) return `FINAL ANSWER: ${committed}` + let answer = committed + try { + answer = renderAnswer({ ...args, answer_only: true }) + } catch { + // Rendering already succeeded once inside the tool; keep its text if this variant fails. + } + const basis = typeof args.basis === "string" && args.answer_only !== true ? args.basis.trim() : "" + return basis ? `Basis: ${basis}\n\nFINAL ANSWER: ${answer}` : `FINAL ANSWER: ${answer}` +} + +function userMessage(): TurnMessage { + return { info: { role: "user" }, parts: [] } as unknown as TurnMessage +} +function toolMessage(tool: string, state: Record): TurnMessage { + return { info: { role: "assistant" }, parts: [{ type: "tool", tool, state }] } as unknown as TurnMessage +} + +// No agent runtime or provider initialization; only local output settings are loaded. +const analyst = { name: "analyst", mode: "primary", permission: [], options: {} } as Agent.Info +function services(directory: string, limits: Awaited>) { + return Layer.mergeAll( + Layer.succeed( + Agent.Service, + Agent.Service.of({ + get: () => Effect.succeed(analyst), + list: () => Effect.succeed([analyst]), + defaultInfo: () => Effect.succeed(analyst), + defaultAgent: () => Effect.succeed(analyst.name), + generate: () => Effect.die(new Error("MCP data tools never generate agents or call a model")), + }), + ), + Truncate.makeLayer({ + directory, + limits: () => Effect.succeed(limits), + hint: (file, outcome) => + `The tool call ${outcome === "error" ? "failed" : "succeeded"} but the output was truncated. Full output saved to: ${file}\nOpen this path with any file reader to inspect the remaining output.`, + }).pipe(Layer.provide(FSUtil.defaultLayer), Layer.provide(NodePath.layer)), + ) +} + +/** + * MCP clients do not forward the user's message, so answer_commit takes the request as an MCP-only + * argument; the adapter records it as the current turn's user text for the commit checks. + */ +function withRequest(schema: T): T { + const copy = JSON.parse(JSON.stringify(schema)) as { properties?: Record; required?: string[] } + copy.properties = { + ...copy.properties, + request: { + type: "string", + description: + "The user's question, copied verbatim: only the question itself, without database descriptions or other context.", + }, + } + copy.required = [...(copy.required ?? []), "request"] + return copy as T +} + +function recordRequest(messages: TurnMessage[], request: unknown) { + if (typeof request !== "string" || !request.trim()) return + const turn = messages.findLast((message) => message.info.role === "user") + if (turn) (turn as { parts: unknown[] }).parts = [{ type: "text", text: request.trim() }] +} + +function relaxBasis(schema: T): T { + const copy = JSON.parse(JSON.stringify(schema).replaceAll(`at most ${BASIS_LIMIT} characters`, "kept brief")) as { + properties?: { basis?: { maxLength?: number } } + } + delete copy.properties?.basis?.maxLength + return copy as T +} + +export async function createAdapter(options: { allowWrite?: boolean; allTools?: boolean } = {}) { + const limits = await outputLimits() + const directory = await mkdtemp(path.join(tmpdir(), "altimate-code-mcp-results-")) + await chmod(directory, 0o700) + const sessionID = `mcp-${randomUUID()}` as SessionID + Precedence.bindLocalSession(sessionID) + const tools = await Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const result = new Map() + for (const source of [ + WarehouseListTool, + WarehouseAddTool, + SchemaInspectTool, + SqlExecuteTool, + SqlCheckTool, + SqlMaterializeTool, + AnswerCommitTool, + ]) { + const info = yield* source + const tool = yield* Tool.init(info, { agent: analyst }) + if (tool.jsonSchema) { + tool.jsonSchema = JSON.parse( + JSON.stringify(tool.jsonSchema).replaceAll( + "use read with char_offset/max_chars to recover clipped text", + "open result_path with any file reader to recover clipped text", + ), + ) + } + if (tool.jsonSchema && info.id === "answer_commit") tool.jsonSchema = withRequest(relaxBasis(tool.jsonSchema)) + if (tool.jsonSchema) tool.jsonSchema = compactSchema(info.id, tool.jsonSchema) + if (tool.jsonSchema && info.id === "sql_execute") tool.jsonSchema = withTables(tool.jsonSchema) + result.set(info.id, tool) + } + return result + }).pipe(Effect.provide(services(directory, limits))), + ), + ).catch(async (error) => { + Precedence.releaseLocalSession(sessionID) + await rm(directory, { recursive: true, force: true }) + throw error + }) + + const messages: TurnMessage[] = [userMessage()] + // Sequential spill names keep model-visible paths stable across runs, apart from the private directory. + let spills = 0 + const writeSpill = async (text: string, extension: string) => { + const file = path.join(directory, `result-${++spills}.${extension}`) + await writeFile(file, text, { mode: 0o600, flag: "wx" }) + return file + } + const shutdown = new AbortController() + const active = new Set>() + // Feed the native commit checks: completed tool output and rejections, with a new turn after an accepted commit. + const record = (name: string, result: Tool.ExecuteResult, args: Record, request?: unknown) => { + if (name !== "answer_commit") { + if (result.status !== "error") messages.push(toolMessage(name, { status: "completed", output: result.output })) + return result + } + if (result.status === "error") { + messages.push(toolMessage(name, { status: "error", error: result.output })) + return result + } + messages.push(userMessage()) + return { ...result, output: finalAnswerText(args, result.metadata.committed_text ?? result.output, request) } + } + const advertised = new Map([...tools].filter(([name]) => options.allTools || LEAN_TOOLS.has(name))) + /** sql_execute with `tables`: describe each table with schema_inspect, then run the queries, in one call. */ + const fused = async (tables: unknown[], args: Record, ctx: Tool.Context): Promise => { + const inspect = tools.get("schema_inspect")! + const parts: string[] = [] + let inspected = 0 + for (const table of tables.slice(0, MAX_FUSED_TABLES)) { + if (typeof table !== "string" || !table.trim()) continue + ctx.abort.throwIfAborted() + const target = { table: table.trim(), ...(typeof args.warehouse === "string" ? { warehouse: args.warehouse } : {}) } + try { + const result = await Effect.runPromise(inspect.execute(target, ctx)) + if (result.status !== "error" && !result.metadata?.error) inspected++ + parts.push(result.output) + } catch (error) { + if (ctx.abort.aborted) throw error + parts.push(`Table ${table}: ${error instanceof Error ? error.message : String(error)}`) + } + } + ctx.abort.throwIfAborted() + if (!hasQuery(args)) { + const output = parts.join("\n\n") + // Nothing described and nothing queried: report a failure, not completed evidence. + if (inspected === 0) return { title: "Schema: no table described", metadata: { error: output }, output, status: "error" } + return { title: `Schema: ${inspected} table(s)`, metadata: {}, output } + } + const result = await Effect.runPromise(tools.get("sql_execute")!.execute(args, ctx)) + return { ...result, output: [...parts, result.output].join("\n\n") } + } + return { + tools: advertised, + async execute(name: string, args: Record, signal: AbortSignal) { + signal = AbortSignal.any([signal, shutdown.signal]) + signal.throwIfAborted() + const tool = advertised.get(name) + if (!tool) throw new Error(`Unknown tool: ${name}`) + const ctx: Tool.Context = { + sessionID, + messageID: `mcp-${randomUUID()}` as MessageID, + agent: "analyst", + abort: signal, + extra: { mcpClient: true, forceReadOnly: !options.allowWrite, writeSpill }, + messages, + metadata: () => Effect.void, + // Fail closed for new permission kinds. sql_execute is the validated source-read permission. + ask: ({ permission }) => + permission === "sql_execute" || + permission === "schema_inspect" || + permission === "answer_commit" || options.allowWrite + ? Effect.void + : Effect.die(new Error(`Permission denied: ${permission}. Restart with --allow-write to enable writes.`)), + } + if (MCP_ONLY_TOOLS.has(name) && typeof args.basis === "string" && args.basis.length > BASIS_LIMIT) + args = { ...args, basis: args.basis.slice(0, BASIS_LIMIT) } + let request: unknown + if (name === "answer_commit") { + const { request: text, ...rest } = args + request = text + recordRequest(messages, request) + args = rest + } + let pending: Promise + if (name === "sql_execute" && Array.isArray(args.tables) && args.tables.length > 0) { + const { tables, ...rest } = args + args = rest + pending = fused(tables, rest, ctx) + } else { + if (name === "sql_execute") args = Object.fromEntries(Object.entries(args).filter(([key]) => key !== "tables")) + pending = Effect.runPromise(tool.execute(args, ctx)) + } + active.add(pending) + try { + const result = await pending + return record(name, result, args, request) + } catch (error) { + if (name === "answer_commit") messages.push(toolMessage(name, { status: "error", error: String(error) })) + throw error + } finally { + active.delete(pending) + } + }, + async close() { + shutdown.abort(new Error("MCP server closed")) + await Promise.allSettled([...active]) + try { + await closeScratch(sessionID) + } finally { + Precedence.releaseLocalSession(sessionID) + await rm(directory, { recursive: true, force: true }) + } + }, + } +} diff --git a/packages/opencode/src/altimate/mcp-server/output-settings.ts b/packages/opencode/src/altimate/mcp-server/output-settings.ts new file mode 100644 index 0000000000..2c14e65a76 --- /dev/null +++ b/packages/opencode/src/altimate/mcp-server/output-settings.ts @@ -0,0 +1,80 @@ +import { readFile } from "node:fs/promises" +import path from "node:path" +import { Effect } from "effect" +import { parse } from "jsonc-parser" +import { Flag, env } from "@opencode-ai/core/flag/flag" +import { FSUtil } from "@opencode-ai/core/fs-util" +import { Global } from "@/global" +import { ConfigPaths } from "@/config/paths" +import { ConfigVariable } from "@/config/variable" +import { git } from "@/util/git" +import { MAX_LINES, MAX_BYTES } from "@/tool/truncate-core" + +/** The project's git worktree root, as the agent's Instance resolves it (`Project.fromDirectory`). */ +async function projectWorktree(directory: string): Promise { + const dotgit = await Effect.runPromise( + Effect.gen(function* () { + return yield* (yield* FSUtil.Service).up({ targets: [".git"], start: directory }) + }).pipe( + Effect.provide(FSUtil.defaultLayer), + Effect.orElseSucceed(() => [] as string[]), + ), + ) + if (!dotgit[0]) return "/" + const sandbox = path.dirname(dotgit[0]) + const result = await git(["rev-parse", "--path-format=absolute", "--git-common-dir"], { cwd: sandbox }) + const common = result.exitCode === 0 ? result.text().replace(/[\r\n]+$/, "") : "" + if (!common) return sandbox + const resolved = path.resolve(sandbox, common) + return resolved === sandbox ? sandbox : path.dirname(resolved) +} + +/** + * Read only output settings: loading the full agent config can install dependencies or fetch remote + * configuration, neither of which belongs in this standalone server. The files, their order and the + * `ALTIMATE_CLI_*` / `OPENCODE_*` variables are the agent loader's (`config/config.ts`), through the + * same `ConfigPaths` and `Flag` helpers, so one config resolves to the same limits in both. + */ +export async function outputLimits() { + const directory = process.cwd() + const limits = { maxLines: MAX_LINES, maxBytes: MAX_BYTES } + // `{env:VAR}`, `${VAR}` and `{file:path}` resolve as in the agent loader; text that cannot be substituted is skipped. + const apply = async (text: string, from: { path: string } | { source: string }) => { + const expanded = await ConfigVariable.substitute( + "path" in from ? { text, type: "path", path: from.path } : { text, type: "virtual", dir: directory, source: from.source }, + ).catch(() => undefined) + if (!expanded) return + const output = parse(expanded)?.tool_output + if (Number.isInteger(output?.max_lines) && output.max_lines > 0) limits.maxLines = output.max_lines + if (Number.isInteger(output?.max_bytes) && output.max_bytes > 0) limits.maxBytes = output.max_bytes + } + // A missing or unreadable file contributes nothing; output limits never stop the server from starting. + const load = async (file: string) => { + const text = await readFile(file, "utf8").catch(() => undefined) + if (text) await apply(text, { path: file }) + } + const worktree = await projectWorktree(directory) + const discover = (effect: Effect.Effect, fallback: A) => + Effect.runPromise(effect.pipe(Effect.provide(FSUtil.defaultLayer), Effect.orElseSucceed(() => fallback))) + + for (const name of ["config.json", "opencode.json", "opencode.jsonc", "altimate-code.json", "altimate-code.jsonc"]) { + await load(path.join(Global.Path.config, name)) + } + const custom = env("OPENCODE_CONFIG") + if (custom) await load(custom) + if (!Flag.OPENCODE_DISABLE_PROJECT_CONFIG) { + for (const file of await discover(ConfigPaths.files("opencode", directory, worktree), [] as string[])) { + await load(file) + } + } + const configDir = Flag.OPENCODE_CONFIG_DIR + for (const dir of await discover(ConfigPaths.directories(directory, worktree), [] as string[])) { + if (!dir.endsWith(".altimate-code") && !dir.endsWith(".opencode") && dir !== configDir) continue + for (const name of ["altimate-code.json", "altimate-code.jsonc", "opencode.json", "opencode.jsonc"]) { + await load(path.join(dir, name)) + } + } + const content = env("OPENCODE_CONFIG_CONTENT") + if (content) await apply(content, { source: "OPENCODE_CONFIG_CONTENT" }) + return limits +} diff --git a/packages/opencode/src/altimate/mcp-server/server.ts b/packages/opencode/src/altimate/mcp-server/server.ts new file mode 100644 index 0000000000..620c77a129 --- /dev/null +++ b/packages/opencode/src/altimate/mcp-server/server.ts @@ -0,0 +1,199 @@ +import { Server } from "@modelcontextprotocol/sdk/server/index.js" +import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js" +import { + CallToolRequestSchema, + GetPromptRequestSchema, + ListPromptsRequestSchema, + ListToolsRequestSchema, + type Tool as McpTool, +} from "@modelcontextprotocol/sdk/types.js" +import * as Registry from "../native/connections/registry" +import { createAdapter, descriptions } from "./adapter" +import { catalogSummary } from "../tools/warehouse-discovery" +import { readOnlyCaveat } from "../native/connections/read-only" +import path from "node:path" +import { pathToFileURL } from "node:url" +import { InstallationVersion } from "@opencode-ai/core/installation/version" + +export const SERVER_INSTRUCTIONS = [ + "Analyst method for data questions:", + "1. Connections are listed below (warehouse_list adds columns and a sample row); if none fit, warehouse_add one.", + "2. Check stored value formats before filtering or joining: sql_execute `tables` returns value profiles in the same call as your queries.", + "3. Compute in the database; chain dependent steps (any connection) in one sql_execute `queries` call.", + "4. Cross-connection joins: sql_materialize the smaller filtered side into `scratch` and join there.", + "5. End with `FINAL ANSWER: ` naming every requested item (entity and related item both); answer_commit is optional.", +].join("\n") + +/** + * The analyst method plus the connection list known at startup (names and types from config; table names only for + * small local SQLite/DuckDB files, see catalogSummary), so a client can start querying without a warehouse_list round + * trip. Remote connections are never contacted here. A failing or empty listing leaves the method alone. + */ +async function serverInstructions(): Promise { + try { + const warehouses = Registry.list().warehouses + if (!warehouses.length) return SERVER_INSTRUCTIONS + const summary = await catalogSummary(warehouses) + return summary ? `${SERVER_INSTRUCTIONS}\n\nConnections:\n${summary}` : SERVER_INSTRUCTIONS + } catch { + return SERVER_INSTRUCTIONS + } +} + +/** Spilled results: a resource_link next to the text, which keeps the preview, row count and path. */ +function resultLinks(metadata: Record) { + // A batched sql_execute call reports one saved file per spilled query. + const batch = Array.isArray(metadata.result_paths) ? metadata.result_paths : [] + const single = [metadata.result_path, metadata.outputPath].find((value) => typeof value === "string" && value) + const files = [...new Set([...batch, ...(batch.length ? [] : [single])])].filter( + (file): file is string => typeof file === "string" && !!file, + ) + return files.map((file) => ({ + type: "resource_link" as const, + uri: pathToFileURL(file).href, + name: path.basename(file), + mimeType: file.endsWith(".jsonl") ? "application/x-ndjson" : file.endsWith(".csv") ? "text/csv" : "text/plain", + })) +} + +// Surfaces as a slash command in interactive clients. +const analyzePrompt = { + name: "analyze", + description: "Answer a data question with the analyst method: inspect, compute in the database, then answer_commit.", + arguments: [{ name: "question", description: "The data question to answer", required: false }], +} + +export interface ServeOptions { + connections?: string + allowWrite?: boolean + // Also advertise schema_inspect and sql_check (default: the five-tool set, see adapter LEAN_TOOLS). + allTools?: boolean + // Bounds the reading question so an unanswered prompt cannot outlast the client's tool-call timeout. + elicitTimeoutMs?: number +} + +const ELICIT_TIMEOUT_MS = 20_000 + +export async function createServer(options: ServeOptions = {}) { + if (options.connections) await Registry.loadFile(options.connections) + if (!options.allowWrite) { + for (const connection of Registry.list().warehouses) { + const caveat = readOnlyCaveat(connection) + if (caveat) console.error(caveat) + } + } + const adapter = await createAdapter(options) + const server = new Server({ name: "altimate-code", version: InstallationVersion }, { + capabilities: { tools: {}, prompts: {} }, + instructions: await serverInstructions(), + }) + server.setRequestHandler(ListToolsRequestSchema, async () => ({ + tools: [...adapter.tools].map(([name, tool]) => ({ + name, + description: descriptions[name], + inputSchema: tool.jsonSchema as McpTool["inputSchema"], + })), + })) + // Ask the user once per session when the model weighed two readings and the client can ask; any + // failure, decline or cancel (headless clients cancel) leaves the commit as accepted. + let elicited = false + const clarifyReading = async ( + args: Record | undefined, + result: { status?: string }, + signal: AbortSignal, + ) => { + const readings = Array.isArray(args?.readings) ? (args.readings as { reading?: unknown }[]) : [] + const choices = readings.map((item) => item.reading).filter((item): item is string => typeof item === "string" && !!item) + if (elicited || result.status === "error" || choices.length < 2 || !server.getClientCapabilities()?.elicitation) return undefined + elicited = true + try { + const answer = await server.elicitInput({ + message: "The question can be read two ways. Which did you mean?", + requestedSchema: { + type: "object", + properties: { reading: { type: "string", enum: choices, description: "Intended reading" } }, + required: ["reading"], + }, + }, { signal, timeout: options.elicitTimeoutMs ?? ELICIT_TIMEOUT_MS }) + const chosen = answer.action === "accept" ? answer.content?.reading : undefined + if (typeof chosen !== "string" || chosen === choices[0]) return undefined + return `The user chose this reading: ${JSON.stringify(chosen)}. Recompute under it and call answer_commit again.` + } catch { + return undefined + } + } + server.setRequestHandler(ListPromptsRequestSchema, async () => ({ prompts: [analyzePrompt] })) + server.setRequestHandler(GetPromptRequestSchema, async (request) => { + if (request.params.name !== analyzePrompt.name) throw new Error(`Unknown prompt: ${request.params.name}`) + const question = request.params.arguments?.question?.trim() + const text = [SERVER_INSTRUCTIONS, ...(question ? ["", `Question: ${question}`] : [])].join("\n") + return { description: analyzePrompt.description, messages: [{ role: "user" as const, content: { type: "text" as const, text } }] } + }) + server.setRequestHandler(CallToolRequestSchema, async (request, extra) => { + try { + const result = await adapter.execute(request.params.name, request.params.arguments ?? {}, extra.signal) + const clarified = request.params.name === "answer_commit" ? await clarifyReading(request.params.arguments, result, extra.signal) : undefined + if (clarified) return { content: [{ type: "text" as const, text: clarified }], isError: true } + const isError = result.status === "error" || (result.metadata.success !== true && Boolean(result.metadata.error)) + return { + content: [{ type: "text" as const, text: result.output }, ...resultLinks(result.metadata)], + isError, + } + } catch (error) { + return { + content: [{ type: "text" as const, text: error instanceof Error ? error.message : String(error) }], + isError: true, + } + } + }) + let cleanup: Promise | undefined + const dispose = () => (cleanup ??= adapter.close()) + server.onclose = async () => { + await dispose().catch(console.error) + } + const close = server.close.bind(server) + server.close = async () => { + try { + await close() + } finally { + await dispose() + } + } + return server +} + +export async function serveStdio(options: ServeOptions = {}): Promise { + // Protocol writes use the transport's stdout directly; application logging must use stderr. + console.log = console.error.bind(console) + console.info = console.error.bind(console) + console.debug = console.error.bind(console) + const server = await createServer(options) + server.onerror = (error) => console.error(error) + const closed = Promise.withResolvers() + const onclose = server.onclose + server.onclose = async () => { + try { + await onclose?.() + await Registry.closeAll() + closed.resolve() + } catch (error) { + closed.reject(error) + } + } + const close = () => { + void server.close().catch(closed.reject) + } + process.stdin.once("end", close) + process.once("SIGINT", close) + process.once("SIGTERM", close) + try { + await server.connect(new StdioServerTransport()) + await closed.promise + } finally { + process.stdin.off("end", close) + process.off("SIGINT", close) + process.off("SIGTERM", close) + await server.close() + await Registry.closeAll() + } +} diff --git a/packages/opencode/src/altimate/native/connections/read-only.ts b/packages/opencode/src/altimate/native/connections/read-only.ts new file mode 100644 index 0000000000..968d27999f --- /dev/null +++ b/packages/opencode/src/altimate/native/connections/read-only.ts @@ -0,0 +1,11 @@ +/** Drivers that enforce ExecuteOptions.readOnly at the database boundary. */ +const ENFORCED = new Set(["postgres", "postgresql", "sqlite", "duckdb"]) + +/** + * The caveat for a connection whose driver does not enforce read-only. A malformed connection without a string + * type has no driver to caveat (it cannot connect), so it gets none rather than crashing the caller. + */ +export function readOnlyCaveat(connection: { name: string; type?: unknown }): string | undefined { + if (typeof connection.type !== "string" || ENFORCED.has(connection.type.toLowerCase())) return undefined + return `${connection.name} (${connection.type}): read-only is enforced by SQL classification only; stored functions with side effects are not blocked — use a read-only database user` +} diff --git a/packages/opencode/src/altimate/native/connections/register.ts b/packages/opencode/src/altimate/native/connections/register.ts index a763ef0036..e7eddfa570 100644 --- a/packages/opencode/src/altimate/native/connections/register.ts +++ b/packages/opencode/src/altimate/native/connections/register.ts @@ -7,6 +7,7 @@ */ import { register } from "../dispatcher" +import { listSchemaTables } from "./schema-list" import * as Registry from "./registry" import { defaultInspectionSchema, resolveInspectionTable, resolveInspectionTarget } from "./inspection-target" import { discoverContainers } from "./docker-discovery" @@ -525,7 +526,7 @@ register("sql.execute", async (params: SqlExecuteParams, context): Promise { @@ -745,6 +746,8 @@ register("warehouse.discover", async (): Promise => { } }) +register("schema.list", listSchemaTables) + // --- schema.inspect --- function schemaAddressingHint(): string { const warehouses = Registry.list().warehouses diff --git a/packages/opencode/src/altimate/native/connections/registry.ts b/packages/opencode/src/altimate/native/connections/registry.ts index 5154de671b..7643a00d3f 100644 --- a/packages/opencode/src/altimate/native/connections/registry.ts +++ b/packages/opencode/src/altimate/native/connections/registry.ts @@ -328,6 +328,27 @@ export function load(): void { loadInto(projectState()) } +/** + * Load only an explicit connections file (`mcp serve --connections`), resolving stores relative to that file. Like + * setConfigs(), the file becomes the one registry every project uses and is never re-read from project sources. + */ +export async function loadFile(file: string): Promise { + await closeAll() + const absolute = path.resolve(file) + const parsed: unknown = JSON.parse(await fs.promises.readFile(absolute, "utf8")) + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("Connections file must contain an object keyed by connection name") + } + for (const [name, value] of Object.entries(parsed)) { + if (!value || typeof value !== "object" || typeof value.type !== "string") { + throw new Error(`Invalid connection: ${name}`) + } + } + setConfigs(resolveStorePaths(parsed as Record, path.dirname(absolute))) + if (state().configs.has(SCRATCH_NAME)) Log.Default.warn(SCRATCH_NAME_CONFLICT) + censusSent = false +} + /** Ensure the current project's configs are loaded; returns its state. */ function ensureLoaded(): State { const s = state() @@ -380,8 +401,9 @@ const DRIVER_MAP: Record = { * different service with different credentials and endpoints, where Postgres * wire-compatibility is an implementation detail rather than an identity. */ -export function canonicalType(type: string | undefined | null): string | null { - if (!type) return null +export function canonicalType(type: unknown): string | null { + // A hand-edited connection may carry a non-string type; it maps to no driver rather than throwing. + if (typeof type !== "string" || !type) return null const driverPath = DRIVER_MAP[type.toLowerCase()] if (!driverPath) return null return driverPath.slice(driverPath.lastIndexOf("/") + 1) diff --git a/packages/opencode/src/altimate/native/connections/schema-list.ts b/packages/opencode/src/altimate/native/connections/schema-list.ts new file mode 100644 index 0000000000..60a44dd5c2 --- /dev/null +++ b/packages/opencode/src/altimate/native/connections/schema-list.ts @@ -0,0 +1,135 @@ +import { absoluteFileUriPath, isLocalFilePath } from "@altimateai/drivers/file-store" +import type { ConnectionConfig } from "@altimateai/drivers/types" +import type { SchemaListParams, SchemaListResult } from "../types" +import { executeQuery, type QueryContext } from "../query-execution" +import { sanitizeBqRegion } from "../finops/bq-utils" +import * as Registry from "./registry" + +const TABLE_LIMIT = 500 +const SCHEMA_LIMIT = 200 +const quoteIdent = (id: string) => `"${id.replace(/"/g, '""')}"` + +/** Undefined means this driver has no supported catalog SQL, never a failed catalog query. */ +function catalogQuery(type: string, config: ConnectionConfig | undefined, schema?: string): string | undefined { + if (type === "sqlite") { + const catalog = `${quoteIdent(schema ?? "main")}.sqlite_master` + const filter = "type IN ('table','view') AND name NOT LIKE 'sqlite_%'" + return schema + ? `SELECT name, type FROM ${catalog} WHERE ${filter} ORDER BY name LIMIT 501` + : `SELECT 'main', COUNT(*) FROM ${catalog} WHERE ${filter} LIMIT 201` + } + if (!["postgres", "mysql", "snowflake", "duckdb", "redshift", "bigquery"].includes(type)) return undefined + const catalog = + type === "bigquery" + ? `\`region-${sanitizeBqRegion(config?.location)}.INFORMATION_SCHEMA.TABLES\`` + : "information_schema.tables" + const filters = ["t.table_schema NOT IN ('information_schema', 'pg_catalog', 'pg_internal', 'INFORMATION_SCHEMA')"] + if (type === "duckdb") filters.push("t.table_catalog = current_database()") + if (schema) + filters.push(`t.table_schema = ${type === "postgres" || type === "redshift" || type === "duckdb" ? "$1" : "?"}`) + if (!schema) { + return `SELECT t.table_schema, COUNT(*) FROM ${catalog} t WHERE ${filters.join(" AND ")} GROUP BY t.table_schema ORDER BY t.table_schema LIMIT 201` + } + const stats = + type === "mysql" + ? ", t.table_rows" + : type === "snowflake" + ? ", t.row_count" + : type === "postgres" + ? ", c.reltuples" + : type === "duckdb" + ? ", d.estimated_size" + : "" + const join = + type === "postgres" + ? " LEFT JOIN pg_catalog.pg_namespace n ON n.nspname = t.table_schema LEFT JOIN pg_catalog.pg_class c ON c.relnamespace = n.oid AND c.relname = t.table_name" + : type === "duckdb" + ? " LEFT JOIN duckdb_tables() d ON d.database_name = current_database() AND d.schema_name = t.table_schema AND d.table_name = t.table_name AND t.table_type = 'BASE TABLE'" + : "" + return `SELECT t.table_name, t.table_type${stats} FROM ${catalog} t${join} WHERE ${filters.join(" AND ")} ORDER BY t.table_name LIMIT 501` +} + +function nonnegativeCount(value: unknown): number | undefined { + if (value === undefined || value === null) return undefined + const count = Number(value) + return Number.isSafeInteger(count) && count >= 0 ? count : undefined +} + +export async function listSchemaTables(params: SchemaListParams, context?: QueryContext): Promise { + const warehouse = params.warehouse ?? Registry.list().warehouses[0]?.name + if (!warehouse) throw new Error("No warehouse configured.") + context?.signal?.throwIfAborted() + const connector = await Registry.get(warehouse) + const config = Registry.getConfig(warehouse) + const type = Registry.canonicalType(config?.type) ?? "" + const schema = params.schema_name?.trim() || undefined + const query = catalogQuery(type, config, schema) + const store = typeof config?.path === "string" ? config.path : "" + const local = + type === "sqlite" || + (type === "duckdb" && (store === ":memory:" || isLocalFilePath(store) || absoluteFileUriPath(store) !== undefined)) + const run = (sql: string, limit: number, binds?: unknown[]) => + executeQuery( + config, + { readOnly: true, ...(local ? {} : { queryTimeoutMs: 10_000 }) }, + context, + (options) => connector.execute(sql, limit, binds, options), + ) + if (!schema) { + if (query) { + const result = await run(query, SCHEMA_LIMIT + 1) + return { + warehouse, + tables: [], + schemas: result.rows + .slice(0, SCHEMA_LIMIT) + .map((row) => ({ name: String(row[0]), table_count: nonnegativeCount(row[1]) })), + truncated: result.rows.length > SCHEMA_LIMIT || result.truncated, + } + } + const names = await connector.listSchemas() + const schemas: NonNullable = [] + for (const name of names.slice(0, SCHEMA_LIMIT).sort()) { + context?.signal?.throwIfAborted() + let table_count: number | undefined + if (type === "mongodb") { + // dbStats returns a scalar catalog count without transferring collection names. + try { + const result = await run(JSON.stringify({ command: "dbStats", database: name }), 1) + table_count = nonnegativeCount(result.rows[0]?.[0]) + } catch { + context?.signal?.throwIfAborted() + // Listing authorized databases may be allowed even when dbStats is denied. + } + } + schemas.push({ name, table_count }) + } + return { warehouse, tables: [], schemas, truncated: names.length > SCHEMA_LIMIT } + } + + let tables: SchemaListResult["tables"] + let truncated: boolean + const fallback = !query && type !== "mongodb" + if (query || type === "mongodb") { + const result = await run( + query ?? JSON.stringify({ command: "listCollections", database: schema }), + query ? TABLE_LIMIT + 1 : TABLE_LIMIT, + query && type !== "sqlite" ? [schema] : undefined, + ) + truncated = result.rows.length > TABLE_LIMIT || result.truncated + tables = result.rows.slice(0, TABLE_LIMIT).map((row) => ({ + schema, + name: String(row[0]), + type: String(row[1]).toLowerCase().includes("view") ? "view" : type === "mongodb" ? "collection" : "table", + ...(row.length > 2 ? { row_count: nonnegativeCount(row[2]), row_count_estimated: true } : {}), + })) + } else { + const listed = await connector.listTables(schema) + truncated = listed.length > TABLE_LIMIT + tables = listed + .sort((a, b) => a.name.localeCompare(b.name)) + .slice(0, TABLE_LIMIT) + .map((table) => ({ schema, ...table })) + } + return { warehouse, tables, truncated, ...(fallback ? { fallback: true } : {}) } +} diff --git a/packages/opencode/src/altimate/native/query-execution.ts b/packages/opencode/src/altimate/native/query-execution.ts index 84e3817c14..701e14b21e 100644 --- a/packages/opencode/src/altimate/native/query-execution.ts +++ b/packages/opencode/src/altimate/native/query-execution.ts @@ -6,6 +6,8 @@ import { RunBudget } from "../run-budget" export interface QueryContext { signal?: AbortSignal sessionID?: string + /** MCP source reads request database protection even when SQL classifies as a read. */ + forceReadOnly?: boolean } /** Share the session cutoff across probes, extraction and scratch writes. */ diff --git a/packages/opencode/src/altimate/native/types.ts b/packages/opencode/src/altimate/native/types.ts index ed995589e5..431d1f41a5 100644 --- a/packages/opencode/src/altimate/native/types.ts +++ b/packages/opencode/src/altimate/native/types.ts @@ -106,6 +106,25 @@ export interface SqlOptimizeResult { // --- Schema --- +export interface SchemaListParams { + schema_name?: string + warehouse?: string +} + +export interface SchemaListResult { + warehouse: string + tables: Array<{ + schema: string + name: string + type: string + row_count?: number + row_count_estimated?: boolean + }> + schemas?: Array<{ name: string; table_count?: number }> + fallback?: boolean + truncated: boolean +} + export interface SchemaInspectParams { table: string schema_name?: string @@ -1208,6 +1227,7 @@ export const BridgeMethods = { "sql.format": {} as { params: SqlFormatParams; result: SqlFormatResult }, "sql.fix": {} as { params: SqlFixParams; result: SqlFixResult }, "sql.autocomplete": {} as { params: SqlAutocompleteParams; result: SqlAutocompleteResult }, + "schema.list": {} as { params: SchemaListParams; result: SchemaListResult }, "schema.inspect": {} as { params: SchemaInspectParams; result: SchemaInspectResult }, "schema.index": {} as { params: SchemaIndexParams; result: SchemaIndexResult }, "schema.search": {} as { params: SchemaSearchParams; result: SchemaSearchResult }, diff --git a/packages/opencode/src/altimate/prompts/analyst.txt b/packages/opencode/src/altimate/prompts/analyst.txt index f0460ec7e9..6b3830b341 100644 --- a/packages/opencode/src/altimate/prompts/analyst.txt +++ b/packages/opencode/src/altimate/prompts/analyst.txt @@ -1,6 +1,6 @@ You are altimate-code: a read-only analyst. Do not modify source data or files. When available, `sql_materialize` may write session-local `scratch` tables. -Inspect connections, schemas and actual rows before joining or parsing; key formats may differ across databases. Establish population, grain, denominator and assumptions. Every clause of the question must bite: a filter that removes nothing usually means the wrong column. When a phrase admits two readings (which population, denominator, transform or owner column), compute both, then choose by the question's own definitions and the data's shape; keep that reasoning out of the final message. When a rule or policy decides the answer, read its full text (not a preview), enumerate each competing clause, and evaluate every condition as satisfied, violated or unknown against the row's own fields and dates before choosing. Use `sql_check` for keys, fan-out, grain, nulls or extraction coverage when they affect the answer; trivial queries need no ritual. `sql_analyze` is optional. +Inspect connections, schemas and actual rows before joining or parsing; key formats may differ across databases. Establish population, grain, denominator and assumptions. Every clause of the question must bite: a filter that removes nothing usually means the wrong column. When a phrase admits two readings (which population, denominator, transform or owner column), compute both, then choose by the question's own definitions and the data's shape; keep that reasoning out of the final message, and record the chosen and rejected reading as one `answer_commit` `readings` entry. Never reject a reading only because it yields few rows; reject it on the question's own wording or the data's shape. When a sentence lists a related item alongside the entity as something to identify ("the repository and the most copied file"), commit both; an item that only selects or filters the answer is not asked. When a rule or policy decides the answer, read its full text (not a preview), enumerate each competing clause, and evaluate every condition as satisfied, violated or unknown against the row's own fields and dates before choosing, testing each rule against every line with SQL (never by eye); when a rule no row can violate under its literal reading, check the text for a unit, scope or date-column reading that gives it force before dismissing it, while supplied definitions stay authoritative. Use `sql_check` for keys, fan-out, grain, nulls or extraction coverage when they affect the answer; trivial queries need no ritual. `sql_analyze` is optional. Derive large intermediate sets in-engine. For a cross-engine join, move the smaller side: a few hundred keys go into an IN list or VALUES against the big table; materialize into `scratch` only the side that is small after filtering, never a multi-million-row table when a filter or aggregate could shrink it first. Preserve requested weighting: group means weighted by contributing counts can be exact; unweighted averages change the weighting. For many similar tables, prefer catalog-driven SQL (information_schema, DuckDB query_table/UNION ALL BY NAME) after confirming dialect support. @@ -23,4 +23,4 @@ Use `training_list` to review learned knowledge. Skills: /teach, /train, /traini When the request supplies an output JSON schema (the `StructuredOutput` tool is available), finish with `StructuredOutput` instead of `answer_commit` or a plain-text reply; follow that schema. The text finalization rules below apply when no output schema is supplied. -For a data question, finish by calling `answer_commit` with the single chosen value(s) with one field per item the request asks for (every named item, such as an entity and its related item or count, must be its own field: when the request asks for only the answer, anything left in the basis or prose is dropped), a one-or-two-sentence basis (population, grain, source object/column), or `unresolved` with the reason, then reply with exactly the text it returns and nothing else. Always supply `basis`, even when unresolved; it is never shown when the request asks for only the answer. Never list excluded candidates or alternative answers; a top-N with a tie at the boundary still returns N concrete rows (note the tie briefly in `rows.note`); when the request asks for only the answer or a specific format, set `answer_only`. If `answer_commit` is unavailable, finish with one or two sentences of basis and a final line `**Answer:** `, or `**Answer:** unable to determine — `; when the request asks for only the answer or a specific format, the final message is exactly the requested value(s) or table in the requested format, with no label and no basis (or `unable to determine — ` if unresolved); never guess. +For a data question, finish by calling `answer_commit` with the single chosen value(s) with one field per item the request asks for (every named item, such as an entity and its related item or count, must be its own field: when the request asks for only the answer, anything left in the basis or prose is dropped), `asked` listing those items from the question in order, a one-or-two-sentence basis (population, grain, source object/column), or `unresolved` with the reason, then reply with exactly the text it returns and nothing else. Always supply `basis`, even when unresolved; it is never shown when the request asks for only the answer. Never list excluded candidates or alternative answers; a top-N with a tie at the boundary still returns N concrete rows (note the tie briefly in `rows.note`); when the request asks for only the answer or a specific format, set `answer_only`. If `answer_commit` is unavailable, finish with one or two sentences of basis and a final line `**Answer:** `, or `**Answer:** unable to determine — `; when the request asks for only the answer or a specific format, the final message is exactly the requested value(s) or table in the requested format, with no label and no basis (or `unable to determine — ` if unresolved); never guess. diff --git a/packages/opencode/src/altimate/session/analyst-effort.ts b/packages/opencode/src/altimate/session/analyst-effort.ts new file mode 100644 index 0000000000..400f766383 --- /dev/null +++ b/packages/opencode/src/altimate/session/analyst-effort.ts @@ -0,0 +1,36 @@ +import type { Provider } from "@/provider/provider" +import { ProviderTransform } from "@/provider/transform" + +/** Reasoning effort the analyst requests when neither the user nor the agent config chooses one. */ +export const ANALYST_DEFAULT_EFFORT = "high" +/** Overrides the analyst default effort (low, medium, high, xhigh, max); "off" sends no effort option. */ +export const ANALYST_EFFORT_ENV = "ALTIMATE_ANALYST_EFFORT" + +const DISABLED = new Set(["", "off", "none", "default"]) + +function adaptive(options: Record | undefined): boolean { + return [options?.thinking, options?.reasoningConfig, options?.modelParams?.thinking].some( + (value) => value?.type === "adaptive", + ) +} + +/** + * Default request options that raise the analyst's reasoning effort on adaptive-thinking models. + * Without them the provider applies its own default effort, which can be lower than the analyst needs. + * An explicit variant (from the request or the agent's `variant` config), or `effort`/`thinking` in the + * agent's `options`, takes precedence; the run-budget effort downgrade still applies afterwards. + */ +export function analystEffortOptions(input: { + agent: { name: string; options?: Record } + model: Provider.Model + variant?: string + small?: boolean +}): Record { + if (input.small || input.variant || input.agent.name !== "analyst") return {} + const configured = input.agent.options ?? {} + if ("effort" in configured || "thinking" in configured || "reasoningConfig" in configured) return {} + const effort = (process.env[ANALYST_EFFORT_ENV] ?? ANALYST_DEFAULT_EFFORT).trim().toLowerCase() + if (DISABLED.has(effort) || !input.model.capabilities.reasoning) return {} + const options = ProviderTransform.variants(input.model)[effort] + return adaptive(options) ? options! : {} +} diff --git a/packages/opencode/src/altimate/tools/answer-commit-checks.ts b/packages/opencode/src/altimate/tools/answer-commit-checks.ts new file mode 100644 index 0000000000..ec52aa3ff4 --- /dev/null +++ b/packages/opencode/src/altimate/tools/answer-commit-checks.ts @@ -0,0 +1,238 @@ +import { Decimal } from "decimal.js" +import type { Tool } from "../../tool/tool" + +/** Markers that identify each check's rejections in this turn's history. */ +export const COVERAGE_REJECTION_MARKER = "answer_commit requested-items check:" +export const NUMBER_REJECTION_MARKER = "answer_commit number check:" +export const READING_REJECTION_MARKER = "answer_commit reading check:" +/** Coverage rejections per user turn before a commit is accepted as is. */ +export const MAX_COVERAGE_REJECTIONS = 2 +/** Number-provenance warnings per user turn before a commit is accepted as is. */ +export const MAX_NUMBER_REJECTIONS = 1 +/** Reading-challenge warnings per user turn before a commit is accepted as is. */ +export const MAX_READING_REJECTIONS = 1 +/** Integers up to this magnitude are commonly counted from listed rows; they are not checked. */ +const SMALL_INTEGER_LIMIT = 10 +/** Bound the scan over prior tool output so a huge session cannot stall the commit. */ +const MAX_SCANNED_CHARACTERS = 4_000_000 +/** Significant digits that strip binary float noise (0.1 + 0.2) without losing real precision. */ +const FLOAT_NOISE_PRECISION = 15 + +type Cell = string | number | null +type Committed = { + fields?: { name: string; value: string | number }[] + rows?: { columns: string[]; values: Cell[][] } + asked?: string[] +} + +type LoosePart = { + type?: string + tool?: string + text?: string + state?: { status?: string; output?: unknown; error?: unknown } +} + +/** Messages of the current user turn: everything after the latest user message, plus that message. */ +function currentTurn(messages: Tool.Context["messages"]) { + const start = messages.findLastIndex((message) => message.info.role === "user") + return start < 0 ? messages : messages.slice(start) +} + +/** Earlier answer_commit rejections in this turn carrying the given marker. */ +export function priorRejections(messages: Tool.Context["messages"], marker: string): number { + return currentTurn(messages) + .flatMap((message) => message.parts as LoosePart[]) + .filter( + (part) => + part.type === "tool" && + part.tool === "answer_commit" && + part.state?.status === "error" && + String(part.state.error ?? "").includes(marker), + ).length +} + +const STOPWORDS = new Set([ + "a", "an", "the", "of", "and", "or", "for", "with", "its", "their", "his", "her", "each", "per", "in", "on", + "by", "to", "from", "at", "is", "are", "which", "what", "name", "names", +]) + +function words(label: string): string[] { + return label + .toLowerCase() + .split(/[^\p{L}\p{N}]+/u) + .filter((word) => word && !STOPWORDS.has(word)) + .map((word) => (word.length > 3 && word.endsWith("s") ? word.slice(0, -1) : word)) +} + +/** 3 = same label, 2 = one label contains the other, 1 = a shared content word, 0 = unrelated. */ +function labelScore(asked: string, label: string): number { + const a = words(asked).join(" ") + const b = words(label).join(" ") + if (!a || !b) return 0 + if (a === b) return 3 + if (a.includes(b) || b.includes(a)) return 2 + const shared = new Set(words(asked)) + return words(label).some((word) => shared.has(word)) ? 1 : 0 +} + +/** Pair each asked item with at most one committed label, strongest matches first. */ +function matchAsked(asked: string[], labels: string[]): Map { + const pairs = asked + .flatMap((item, a) => labels.map((label, l) => ({ a, l, score: labelScore(item, label) }))) + .filter((pair) => pair.score > 0) + .sort((x, y) => y.score - x.score || x.a - y.a || x.l - y.l) + const matched = new Map() + const used = new Set() + for (const pair of pairs) { + if (matched.has(pair.a) || used.has(pair.l)) continue + matched.set(pair.a, pair.l) + used.add(pair.l) + } + return matched +} + +const blank = (cell: Cell) => cell === null || (typeof cell === "string" && !cell.trim()) + +/** Asked items that no committed field or column supplies with a non-empty value. */ +export function missingAskedItems(args: Committed): string[] { + const asked = (args.asked ?? []).map((item) => item.trim()).filter(Boolean) + if (asked.length <= 1) return [] + if (args.fields) { + const matched = matchAsked( + asked, + args.fields.map((field) => field.name), + ) + return asked.filter((_, index) => !matched.has(index)) + } + if (args.rows) { + const { columns, values } = args.rows + const matched = matchAsked(asked, columns) + return asked.filter((_, index) => { + const column = matched.get(index) + return column === undefined || values.every((row) => blank(row[column] ?? null)) + }) + } + return [] +} + +export function coverageRejection(missing: string[], asked: string[]): string { + return `${COVERAGE_REJECTION_MARKER} the request asks for ${asked.length} items (${asked.map((item) => JSON.stringify(item)).join(", ")}), but no committed field or column supplies ${missing.map((item) => JSON.stringify(item)).join(", ")}. Call answer_commit again with one non-empty field (or rows column) per asked item, named after that item, in the order asked; a value kept only in basis is dropped from answer-only output.` +} + +/** A committed value as a plain decimal, or undefined for text that is not purely numeric. */ +function numericValue(value: Cell): Decimal | undefined { + if (typeof value === "number") { + return Number.isFinite(value) ? new Decimal(value.toPrecision(FLOAT_NOISE_PRECISION)) : undefined + } + if (typeof value !== "string") return undefined + const stripped = value.trim().replace(/^[$€£¥]\s*/, "").replace(/\s*%$/, "") + if (!/^[-+]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?$/.test(stripped)) return undefined + return new Decimal(stripped.replaceAll(",", "")) +} + +function committedNumbers(args: Committed): Decimal[] { + const cells: Cell[] = [ + ...(args.fields ?? []).map((field) => field.value), + ...(args.rows?.values.flat() ?? []), + ] + return cells + .map(numericValue) + .filter((value): value is Decimal => value !== undefined) + .map((value) => value.abs()) + .filter((value) => !(value.isInteger() && value.lte(SMALL_INTEGER_LIMIT))) +} + +const NUMBER_TOKENS = /\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?(?:[eE][-+]?\d+)?/g + +function textNumbers(text: string, into: Set) { + for (const [token] of text.matchAll(NUMBER_TOKENS)) { + // "12,345" may be one grouped number or two delimited cells; keep both readings. + for (const candidate of token.includes(",") ? [token.replaceAll(",", ""), ...token.split(",")] : [token]) { + try { + into.add(new Decimal(candidate).toString()) + } catch { + // Malformed tokens are not evidence. + } + } + } +} + +const isSqlTool = (tool: string | undefined) => !!tool?.startsWith("sql_") + +/** Numbers seen in this session's tool output and user requests, or undefined when no SQL tool has produced output. */ +export function sessionNumbers(messages: Tool.Context["messages"]): Set | undefined { + const parts = messages.flatMap((message) => + (message.parts as LoosePart[]).map((part) => ({ role: message.info.role, part })), + ) + const sql = parts.some( + ({ part }) => part.type === "tool" && isSqlTool(part.tool) && part.state?.status === "completed", + ) + if (!sql) return undefined + const seen = new Set() + let budget = MAX_SCANNED_CHARACTERS + for (const { role, part } of parts.toReversed()) { + const text = + part.type === "tool" && part.tool !== "answer_commit" && part.state?.status === "completed" + ? String(part.state.output ?? "") + : role === "user" && part.type === "text" + ? (part.text ?? "") + : "" + if (!text) continue + textNumbers(text.slice(0, budget), seen) + budget -= text.length + if (budget <= 0) break + } + return seen +} + +/** Whether any sorted value lies in [low, high). */ +function anyWithin(sorted: number[], low: number, high: number): boolean { + let start = 0 + let end = sorted.length + while (start < end) { + const middle = (start + end) >> 1 + if (sorted[middle]! < low) start = middle + 1 + else end = middle + } + return start < sorted.length && sorted[start]! < high +} + +/** + * Whether the committed number is a seen number rounded (half up) or truncated to the committed + * decimal places, directly or as a ratio/percentage conversion (x100 or /100). + */ +function derivable(value: Decimal, sorted: number[]): boolean { + const unit = new Decimal(10).pow(-value.decimalPlaces()) + // Half-up rounding reaches value from [value - unit/2, value + unit/2); truncation from [value, value + unit). + const low = value.minus(unit.dividedBy(2)) + const high = value.plus(unit) + return [new Decimal(1), new Decimal(100), new Decimal(0.01)].some((scale) => + anyWithin(sorted, low.dividedBy(scale).toNumber(), high.dividedBy(scale).toNumber()), + ) +} + +/** Committed numbers that no prior tool output (or the request) contains, even after simple rounding. */ +export function unsourcedNumbers(args: Committed, seen: Set): string[] { + const sorted = [...seen].map(Number).sort((a, b) => a - b) + const unsourced = committedNumbers(args).filter((value) => !seen.has(value.toString()) && !derivable(value, sorted)) + return [...new Set(unsourced.map((value) => value.toString()))] +} + +export function numberRejection(unsourced: string[]): string { + return `${NUMBER_REJECTION_MARKER} ${unsourced.join(", ")} did not appear in any query result this session and is not a rounding of one. Compute it in SQL and commit the queried value, or, if it is correct as is (for example derived from the question), call answer_commit again unchanged and say how it was derived in basis. This warning is shown once.` +} + +/** A rejection reason that rests on how few rows a reading returns rather than on what the request says. */ +const RESULT_SIZE_REASON = + /\b(?:only (?:one|1|a single)\b(?![\d,.])|just (?:one|1)\b(?![\d,.])|(?:returns?|yields?|gives?|leaves?) (?:just |only )?(?:one|1|a single|no|zero|0)\b(?![\d,.])|single (?:row|record|result|match|value|candidate|opportunit\w*|entity|group)|no comparison|nothing to compare|too few|too small|trivial(?:ly)?|degenerate)/i +/** decided_by that cites the request's own wording is a reasoned choice, not a size-based one. */ +const CITES_REQUEST = /\b(?:asks?|requests?|requested|explicit(?:ly)?|states?|stated|says?|defines?|defined)\b/i + +/** Entries whose decided_by rejects an alternative reading because it yields a small or trivial result. */ +export function sizeBasedRejections(readings: { clause: string; decided_by: string }[] | undefined): string[] { + return (readings ?? []).filter((entry) => RESULT_SIZE_REASON.test(entry.decided_by) && !CITES_REQUEST.test(entry.decided_by)).map((entry) => entry.clause) +} + +export function readingRejection(clauses: string[]): string { + return `${READING_REJECTION_MARKER} your reading of ${clauses.map((clause) => JSON.stringify(clause)).join(", ")} rejected an alternative because it returns few or trivial rows. A result being small is not evidence against a reading: when the request, a policy or a definition it supplies states the rule, apply that text literally and accept a one-row answer. Choose the alternative only if the request's own wording supports it. Call answer_commit again with the reading the supplied text dictates, or unchanged with decided_by naming the wording that supports your choice. This warning is shown once.` +} diff --git a/packages/opencode/src/altimate/tools/answer-commit-conjunction.ts b/packages/opencode/src/altimate/tools/answer-commit-conjunction.ts index 730ed8aa72..ad22eae3e2 100644 --- a/packages/opencode/src/altimate/tools/answer-commit-conjunction.ts +++ b/packages/opencode/src/altimate/tools/answer-commit-conjunction.ts @@ -94,19 +94,24 @@ function budgetEnding(sessionID: string): boolean { } /** - * Rejects a fields commit with fewer fields than the items the latest user request joins. Fires at most - * once per session and request, so the next commit, changed or not, is accepted, and never once the run - * nears its time budget. Purely lexical: no queries or model calls. + * Rejects a commit naming fewer items than the latest user request joins. A commit's items are its + * non-blank `asked` labels when given (so a rows commit is checked too), otherwise its field names. + * Fires at most once per session and request, so the next commit, changed or not, is accepted, and + * never for an unresolved answer or once the run nears its time budget. Purely lexical: no queries or + * model calls. */ export function conjunctionRejection( - args: { fields?: { name: string }[] }, + args: { fields?: { name: string }[]; unresolved?: unknown; asked?: string[] }, messages: Tool.Context["messages"], sessionID: string, ): string | undefined { - if (!args.fields || budgetEnding(sessionID)) return undefined + if (args.unresolved || budgetEnding(sessionID)) return undefined + const asked = args.asked?.map((item) => item.trim()).filter(Boolean) + const names = asked?.length ? asked : args.fields?.map((field) => field.name) + if (!names) return undefined const text = latestRequest(messages) const { count, joins } = requestedItems(text) - if (count <= args.fields.length) return undefined + if (count <= names.length) return undefined const key = `${sessionID}\u0000${text}` if (advised.has(key)) return undefined const latest = messages.findLastIndex((message) => message.info.role === "user") @@ -127,9 +132,10 @@ export function conjunctionRejection( return undefined advised.add(key) if (advised.size > MAX_ADVISED_REQUESTS) advised.delete(advised.values().next().value!) - const names = args.fields.map((field) => field.name) - const fields = `${names.length} field${names.length === 1 ? "" : "s"}` - return `${CONJUNCTION_REJECTION_MARKER} the request appears to ask for ${count} items, but this commit has ${fields} (${JSON.stringify(names)}). The request also asks for ${joins.map((join) => `"${join}"`).join(", ")}. Call answer_commit again with each requested item as its own field, named after it. If the request asks for fewer items, recommit unchanged; the next commit is accepted.` + const items = asked?.length + ? `${names.length} asked item${names.length === 1 ? "" : "s"}` + : `${names.length} field${names.length === 1 ? "" : "s"}` + return `${CONJUNCTION_REJECTION_MARKER} the request appears to ask for ${count} items, but this commit has ${items} (${JSON.stringify(names)}). The request also asks for ${joins.map((join) => `"${join}"`).join(", ")}. Call answer_commit again with each requested item as its own field (or rows column), named after it, and list them all in \`asked\`. If the request asks for fewer items, recommit unchanged; the next commit is accepted.` } export * as AnswerCommitConjunction from "./answer-commit-conjunction" diff --git a/packages/opencode/src/altimate/tools/answer-commit.ts b/packages/opencode/src/altimate/tools/answer-commit.ts index c9e8fee0fe..6542f3dffb 100644 --- a/packages/opencode/src/altimate/tools/answer-commit.ts +++ b/packages/opencode/src/altimate/tools/answer-commit.ts @@ -3,6 +3,22 @@ import { Decimal } from "decimal.js" import { Effect, Option, Schema, SchemaIssue } from "effect" import type { JSONSchema7 } from "@ai-sdk/provider" import { Tool } from "../../tool/tool" +import { + COVERAGE_REJECTION_MARKER, + MAX_COVERAGE_REJECTIONS, + MAX_NUMBER_REJECTIONS, + MAX_READING_REJECTIONS, + NUMBER_REJECTION_MARKER, + READING_REJECTION_MARKER, + coverageRejection, + missingAskedItems, + numberRejection, + priorRejections, + readingRejection, + sessionNumbers, + sizeBasedRejections, + unsourcedNumbers, +} from "./answer-commit-checks" // altimate_change start — one-time conjunction review import { AnswerCommitConjunction } from "./answer-commit-conjunction" // altimate_change end @@ -117,6 +133,9 @@ const parameters = z }) }), ).describe("One entry per item the request asks for, in the order asked."), + asked: optional(z.array(z.string())).describe( + "The items the user's request asks for, in order, as short labels taken from the request (e.g. ['repository', 'most copied file']); name each field or rows column after its item. Never rendered.", + ), rows: optional(rows), unresolved: optional( z @@ -127,8 +146,14 @@ const parameters = z ), basis: z .string({ error: basisRequired }) - .max(MAX_BASIS_CHARACTERS, `basis must contain at most ${MAX_BASIS_CHARACTERS} characters`) .refine((value) => value.trim().length > 0, basisRequired) + // altimate_change start - trim an over-long basis instead of rejecting the commit (a rejected commit costs a retry turn) + .transform((value) => { + if (value.length <= MAX_BASIS_CHARACTERS) return value + const text = value.trim() + return text.length <= MAX_BASIS_CHARACTERS ? text : `${text.slice(0, MAX_BASIS_CHARACTERS - 1).trimEnd()}…` + }) + // altimate_change end .describe( `basis is always required: one or two sentences, at most ${MAX_BASIS_CHARACTERS} characters, naming population, grain, the source object/column that supplies the requested entity or value, and caveats; never rendered with answer_only`, ), @@ -144,7 +169,9 @@ const parameters = z }) .strict(), ), - ).describe("Advisory readings recorded only in metadata, never in the answer text"), + ).describe( + "Required when the request's wording admits more than one reading (population, threshold unit, which status or date column) and you weighed alternatives: one entry whose reading names the chosen interpretation and whose decided_by names the rejected one and why. Recorded only in metadata, never in the answer text", + ), }) .strict() .refine( @@ -282,6 +309,40 @@ export function renderAnswer(payload: unknown): string { return limitRenderedAnswer(args.answer_only ? unresolved : `${basis}**Answer:** ${unresolved}`) } +type CommitMetadata = { + readings: NonNullable["readings"]> + asked: string[] + truncated: boolean + committed_text?: string + failure_kind?: string +} + +/** + * Commit-time checks, each bounded per user turn so a commit is always accepted eventually: + * every asked item needs its own non-empty field or column (two rejections), and numbers should + * come from query output (one warning). Earlier rejections are counted from this turn's history. + */ +function commitRejection( + args: z.output, + messages: Tool.Context["messages"], + checkNumbers: boolean, +): string | undefined { + if (args.unresolved) return undefined + const missing = missingAskedItems(args) + if (missing.length && priorRejections(messages, COVERAGE_REJECTION_MARKER) < MAX_COVERAGE_REJECTIONS) + return coverageRejection(missing, args.asked!.map((item) => item.trim()).filter(Boolean)) + // altimate_change start - challenge a reading rejected only because it yields a small result + const sizeBased = sizeBasedRejections(args.readings) + if (sizeBased.length && priorRejections(messages, READING_REJECTION_MARKER) < MAX_READING_REJECTIONS) + return readingRejection(sizeBased) + // altimate_change end + if (!checkNumbers) return undefined + if (priorRejections(messages, NUMBER_REJECTION_MARKER) >= MAX_NUMBER_REJECTIONS) return undefined + const seen = sessionNumbers(messages) + const unsourced = seen ? unsourcedNumbers(args, seen) : [] + return unsourced.length ? numberRejection(unsourced) : undefined +} + // Keep the renderer and native Effect tool on the same parser, including filler normalization. const Parameters = Schema.declareConstructor, AnswerCommitInput>()([], () => (input) => { const parsed = parser.safeParse(input) @@ -294,7 +355,7 @@ export const AnswerCommitTool = Tool.define( "answer_commit", Effect.succeed({ // altimate_change start - expose format selection to the model before the terminal tool call - description: `Deterministically finalize a data-question answer. Call this as the LAST tool call, once the answer is decided and all queries are finished. After it returns, reply with exactly the returned text and nothing else. Supply exactly one of fields, rows, or unresolved. Supply one field per item the request asks for, in the order asked: a request for 'the repository and the most copied file' needs two fields (one named for the repository, one for the file); never collapse a compound request into one field, and never add fields the request did not ask for. Every named item (an entity AND a related item or count) must be its own field, because basis, notes and prose are not shown when the request asks for only the answer; an element found but kept only there is lost. Numbers use JavaScript precision; integers must be safe. Pass exact decimal values, trailing zeros, or currency strings as strings. For fields.precision and each rows.precision value, use an integer from 0 to 10 (inclusive). Set rows.precision by column name to display numeric cells with fixed decimal places, including integers; strings are never changed. Without explicit precision, each column uses the maximum decimal scale of its numeric cells. When the request names a tabular format, set rows.format accordingly: markdown (default), csv, tsv, or lines (one row per line with cells separated by a single space, without headers, for single-column or two-column plain lists). CSV/TSV include headers and quote cells containing the delimiter, quotes, or newlines, doubling embedded quotes; null cells are empty. Always supply \`basis\`; it is never shown when the request asks for only the answer. Supply one or two sentences, at most ${MAX_BASIS_CHARACTERS} characters, naming population, grain, the source object/column that supplies the requested entity or value, and caveats. Resolve policy eligibility separately from metric and entity attribution: a rule about a related object does not change which object supplies the requested entity. Say in \`basis\` which source object and column supply it. When the request names N, rows.values must contain exactly N concrete rows, even with a tie at the top-N boundary; note the tie briefly in rows.note. Use answer_only when the request asks for only the answer or a specific format: the reply is exactly the value(s) or table, with no basis, field names, note, or answer label. An unresolved answer retains its unable-to-determine status. Omit unused optional properties. ${renderedLengthGuidance} This tool is not available for JSON-schema requests; use StructuredOutput instead.`, + description: `Deterministically finalize a data-question answer. Call this as the LAST tool call, once the answer is decided and all queries are finished. After it returns, reply with exactly the returned text and nothing else. Supply exactly one of fields, rows, or unresolved. Supply one field per item the request asks for, in the order asked: a request for 'the repository and the most copied file' needs two fields (one named for the repository, one for the file); never collapse a compound request into one field, and never add fields the request did not ask for. Always set \`asked\` to the requested items, in order, as short labels from the request, and name each field or rows column after its item; a commit that leaves an asked item without its own non-empty value is rejected. When the wording admits more than one reading, add one \`readings\` entry naming the chosen reading and the rejected one; never reject a reading just because it returns few rows. When a sentence lists a related item alongside the entity as something to identify (for example 'the repository and the most copied file'), treat it as asked and commit both; an item that only selects or filters the answer ('the customer with the largest invoice') is not asked. Committed numbers should appear in query output (rounding is fine); compute derived numbers in SQL.Every named item (an entity AND a related item or count) must be its own field, because basis, notes and prose are not shown when the request asks for only the answer; an element found but kept only there is lost. Numbers use JavaScript precision; integers must be safe. Pass exact decimal values, trailing zeros, or currency strings as strings. For fields.precision and each rows.precision value, use an integer from 0 to 10 (inclusive). Set rows.precision by column name to display numeric cells with fixed decimal places, including integers; strings are never changed. Without explicit precision, each column uses the maximum decimal scale of its numeric cells. When the request names a tabular format, set rows.format accordingly: markdown (default), csv, tsv, or lines (one row per line with cells separated by a single space, without headers, for single-column or two-column plain lists). CSV/TSV include headers and quote cells containing the delimiter, quotes, or newlines, doubling embedded quotes; null cells are empty. Always supply \`basis\`; it is never shown when the request asks for only the answer. Supply one or two sentences, at most ${MAX_BASIS_CHARACTERS} characters, naming population, grain, the source object/column that supplies the requested entity or value, and caveats. Resolve policy eligibility separately from metric and entity attribution: a rule about a related object does not change which object supplies the requested entity. Say in \`basis\` which source object and column supply it. When the request names N, rows.values must contain exactly N concrete rows, even with a tie at the top-N boundary; note the tie briefly in rows.note. Use answer_only when the request asks for only the answer or a specific format: the reply is exactly the value(s) or table, with no basis, field names, note, or answer label. An unresolved answer retains its unable-to-determine status. Omit unused optional properties. ${renderedLengthGuidance} This tool is not available for JSON-schema requests; use StructuredOutput instead.`, // altimate_change end parameters: Parameters, formatValidationError: (error: unknown) => (Schema.isSchemaError(error) ? error.message : String(error)), @@ -308,12 +369,22 @@ export const AnswerCommitTool = Tool.define( yield* ctx.ask({ permission: "answer_commit", patterns: ["*"], always: ["*"], metadata: {} }) const output = renderAnswer(args) // altimate_change start — one-time conjunction review of under-asked commits - const rejection = AnswerCommitConjunction.conjunctionRejection(args, ctx.messages ?? [], ctx.sessionID) + // An MCP server sees only its own tool output, never the numbers a client computed in its own + // shell or code, so the number-provenance check would reject correct derived values there. + const checkNumbers = ctx.extra?.mcpClient !== true + const commitCheck = commitRejection(args, ctx.messages ?? [], checkNumbers) + const rejection = + commitCheck ?? AnswerCommitConjunction.conjunctionRejection(args, ctx.messages ?? [], ctx.sessionID) if (rejection) { - const rejected: Tool.ExecuteResult = { + const rejected: Tool.ExecuteResult = { title: "Answer not committed", - // failure_kind separates this advisory from real commit failures in telemetry. - metadata: { readings: args.readings ?? [], truncated: false, failure_kind: "conjunction_check" }, + // failure_kind separates these advisories from real commit failures in telemetry. + metadata: { + readings: args.readings ?? [], + asked: args.asked ?? [], + truncated: false, + failure_kind: commitCheck ? "commit_check" : "conjunction_check", + }, output: rejection, status: "error", } @@ -323,7 +394,7 @@ export const AnswerCommitTool = Tool.define( return { title: "Answer committed", // The processor caps model-facing output but preserves metadata for the final echo. - metadata: { readings: args.readings ?? [], truncated: false, committed_text: output }, + metadata: { readings: args.readings ?? [], asked: args.asked ?? [], truncated: false, committed_text: output }, output, } }), diff --git a/packages/opencode/src/altimate/tools/schema-inspect.ts b/packages/opencode/src/altimate/tools/schema-inspect.ts index 981efed1e8..e5d1858404 100644 --- a/packages/opencode/src/altimate/tools/schema-inspect.ts +++ b/packages/opencode/src/altimate/tools/schema-inspect.ts @@ -10,15 +10,18 @@ import * as Registry from "../native/connections/registry" import { profileValues } from "./schema-values" import { isRecord, normalizeError } from "./response-normalization" import { validateTableName, validateWarehouseName } from "./input-validation" +import { firstTime } from "./session-hints" // altimate_change start — workspace precedence import * as Precedence from "../workspace/precedence" // altimate_change end export const SchemaInspectTool = Tool.define("schema_inspect", { - description: "Inspect database schema — list columns, types, and constraints for a table.", + description: + "List schemas with table/collection counts when table and schema_name are omitted; list up to 500 tables/collections with available catalog row estimates (no table scans) when schema_name is given; otherwise inspect columns, types, and constraints for one table.", parameters: z.object({ table: z .string() + .optional() .describe( "Table name (optionally schema-qualified, e.g. public.orders); a warehouse qualifier such as scratch.orders resolves to that warehouse's default schema when no such schema exists", ), @@ -35,7 +38,8 @@ export const SchemaInspectTool = Tool.define("schema_inspect", { // Pre-flight validation outranks the redirect, as it does for `sql_explain`: bad // input is answered here rather than forwarded to the engine tool, and an empty // warehouse string is never read as "the default" by the routing decision. - const inputError = validateTableName(args.table) ?? validateWarehouseName(args.warehouse) + const inputError = + (args.table?.trim() ? validateTableName(args.table) : null) ?? validateWarehouseName(args.warehouse) if (inputError) { return { title: "Schema: INVALID INPUT", @@ -60,6 +64,76 @@ export const SchemaInspectTool = Tool.define("schema_inspect", { // Strict-schema callers send schema_name: "" for "not specified"; a blank qualifier // must mean the engine default, not a schema literally named "". const schemaName = args.schema_name?.trim() || undefined + if (!args.table?.trim()) { + precedence = await Precedence.check(ctx.sessionID, "schema_inspect", args.warehouse) + if (precedence.redirect) return precedence.redirect + await ctx.ask({ + permission: "schema_inspect", + patterns: [args.warehouse ?? "*"], + always: ["*"], + metadata: { warehouse: args.warehouse, schema_name: schemaName }, + }) + const result = await Dispatcher.call( + "schema.list", + { warehouse: args.warehouse, schema_name: schemaName }, + { signal: ctx.abort, sessionID: ctx.sessionID, forceReadOnly: true }, + ) + const schemas = result.schemas + // altimate_change start — MCP: compact tables, usage hints once per session + const mcp = ctx.extra?.mcpClient === true + const rule = (text: string) => (mcp ? [] : [text]) + const hint = (key: string, text: string) => (!mcp || firstTime(ctx.sessionID, key) ? ["", text] : []) + const lines = schemas + ? [ + `Schemas on warehouse "${result.warehouse}":`, + "", + "Schema | Tables/collections", + ...rule("-------|-------------------"), + ...schemas.map((schema) => `${schema.name} | ${schema.table_count ?? "unavailable"}`), + ] + : [ + `Tables on warehouse "${result.warehouse}":`, + "", + "Schema | Name | Type | Rows", + ...rule("-------|------|------|-----"), + ...result.tables.map( + (table) => + `${table.schema} | ${table.name} | ${table.type} | ${table.row_count === undefined ? "" : `${table.row_count}${table.row_count_estimated ? " (estimate)" : ""}`}`, + ), + ] + if (schemas ? !schemas.length : !result.tables.length) + lines.push(schemas ? "No schemas found." : "No tables or collections found.") + if (result.truncated) + lines.push( + "", + schemas + ? "Showing first 200 schemas of ≥201; pass schema_name to narrow the listing." + : "Showing first 500 of ≥501 — pass a narrower schema or inspect a table by name.", + ) + if (result.fallback) lines.push("", "Driver fallback: table list may be truncated by the driver.") + if (schemas) + lines.push( + ...hint("list-tables", `List tables with schema_inspect(warehouse="${result.warehouse}", schema_name="").`), + ) + lines.push( + ...hint( + "inspect-table", + `Inspect one table with schema_inspect(warehouse="${result.warehouse}", schema_name="", table="").`, + ), + ) + // altimate_change end + return Precedence.annotate(precedence, { + title: `${schemas ? "Schemas" : "Tables"}: ${result.warehouse}`, + metadata: { + success: true, + columnCount: 0, + rowCount: undefined, + ...(schemas ? { schemaCount: schemas.length } : { tableCount: result.tables.length }), + truncated: result.truncated, + }, + output: lines.join("\n"), + }) + } const target = await resolveInspectionTarget({ table: args.table, schema_name: schemaName, @@ -114,7 +188,7 @@ export const SchemaInspectTool = Tool.define("schema_inspect", { } // altimate_change start — progressive disclosure suggestions - let output = formatSchema(schemaResult) + let output = formatSchema(schemaResult, ctx.extra?.mcpClient !== true) if (schemaResult.resolution) output = `${schemaResult.resolution}\n\n${output}` if (args.values !== false) { const resolvedTable = schemaResult.table ?? args.table @@ -136,7 +210,7 @@ export const SchemaInspectTool = Tool.define("schema_inspect", { ) if (profile) output += "\n\n" + profile } - const suggestion = PostConnectSuggestions.getProgressiveSuggestion("schema_inspect") + const suggestion = ctx.extra?.mcpClient ? null : PostConnectSuggestions.getProgressiveSuggestion("schema_inspect") if (suggestion) { output += "\n\n" + suggestion PostConnectSuggestions.trackSuggestions({ @@ -169,7 +243,7 @@ function schemaError(msg: string) { } } -function formatSchema(result: Partial): string { +function formatSchema(result: Partial, rule = true): string { const lines: string[] = [] const table = result.table ?? "unknown" const qualified = result.schema_name ? `${result.schema_name}.${table}` : table @@ -179,7 +253,7 @@ function formatSchema(result: Partial): string { } lines.push("") lines.push("Column | Type | Nullable | PK") - lines.push("-------|------|----------|---") + if (rule) lines.push("-------|------|----------|---") for (const col of result.columns ?? []) { lines.push( `${col.name} | ${col.data_type ?? "unknown"} | ${col.nullable ? "YES" : "NO"} | ${col.primary_key ? "YES" : ""}`, diff --git a/packages/opencode/src/altimate/tools/schema-values.ts b/packages/opencode/src/altimate/tools/schema-values.ts index e15e1ac8e6..e8f3929626 100644 --- a/packages/opencode/src/altimate/tools/schema-values.ts +++ b/packages/opencode/src/altimate/tools/schema-values.ts @@ -11,6 +11,7 @@ import * as Precedence from "../workspace/precedence" import { Dispatcher } from "../native" import * as Registry from "../native/connections/registry" import { isRecord, normalizeError } from "./response-normalization" +import { isLocalConnection } from "./warehouse-discovery" /** Rows scanned per table; keeps wide or large tables cheap. */ export const PROFILE_SAMPLE_ROWS = 10_000 @@ -449,7 +450,7 @@ export interface ValueProfileInput { /** Never throws: a failed profile becomes a one-line note. */ export async function profileValues( input: ValueProfileInput, - ctx: Pick & { abort?: AbortSignal }, + ctx: Pick & { abort?: AbortSignal }, ): Promise { try { const list = (await Dispatcher.call("warehouse.list", {})) as unknown @@ -481,9 +482,14 @@ export async function profileValues( sql, warehouse: target.name, limit: PROFILE_MAX_COLUMNS + 1, - query_timeout_ms: PROFILE_TIMEOUT_MS, + // A shorter deadline would restart a local engine and drop session temp tables; the sample is + // already bounded by its LIMIT. SQLite always runs in the local helper (including `file:` URIs); + // remote DuckDB stores (e.g. MotherDuck) keep the profiling cap. + ...(dialect === "sqlite" || (dialect === "duckdb" && isLocalConnection(Registry.getConfig(target.name))) + ? {} + : { query_timeout_ms: PROFILE_TIMEOUT_MS }), ...(dialect === "postgres" ? { read_only: true } : {}), - }, { signal: ctx.abort, sessionID: ctx.sessionID })) as unknown + }, { signal: ctx.abort, sessionID: ctx.sessionID, ...(ctx.extra?.forceReadOnly === true ? { forceReadOnly: true } : {}) })) as unknown const data = isRecord(res) && isRecord(res.data) ? res.data : res const error = normalizeError(isRecord(res) ? res.error : undefined) ?? normalizeError(isRecord(data) ? data.error : undefined) if (error) throw new Error(error) diff --git a/packages/opencode/src/altimate/tools/session-hints.ts b/packages/opencode/src/altimate/tools/session-hints.ts new file mode 100644 index 0000000000..383583295f --- /dev/null +++ b/packages/opencode/src/altimate/tools/session-hints.ts @@ -0,0 +1,20 @@ +/** + * Usage hints shown once per session. Tool output stays in the conversation and is re-read on every + * later model request, so a tutorial repeated on each call costs tokens without adding information. + */ +const shown = new Map>() +/** Bounds the in-process record; the oldest sessions are dropped first. */ +const MAX_SESSIONS = 1_000 + +/** True the first time `key` is asked for in this session, false afterwards. */ +export function firstTime(sessionID: string, key: string): boolean { + let keys = shown.get(sessionID) + if (!keys) { + keys = new Set() + shown.set(sessionID, keys) + if (shown.size > MAX_SESSIONS) shown.delete(shown.keys().next().value!) + } + if (keys.has(key)) return false + keys.add(key) + return true +} diff --git a/packages/opencode/src/altimate/tools/sql-execute.ts b/packages/opencode/src/altimate/tools/sql-execute.ts index 2af69625b3..4839d4bed5 100644 --- a/packages/opencode/src/altimate/tools/sql-execute.ts +++ b/packages/opencode/src/altimate/tools/sql-execute.ts @@ -2,7 +2,7 @@ import z from "zod" import { Effect } from "effect" import { EffectBridge } from "../../effect/bridge" import { Truncate } from "../../tool/truncate" -import { legacyToInit, type LegacyToolDef } from "../tool-zod-compat" +import { legacyToInit, type LegacyContext, type LegacyResult, type LegacyToolDef } from "../tool-zod-compat" import { formatSqlResult, SQL_MAX_PREVIEW_BYTES } from "./sql-result-format" import { lintSqlResult } from "./sql-result-lint" import { Tool } from "../../tool/tool" @@ -35,12 +35,30 @@ import * as Precedence from "../workspace/precedence" import { normalizeError } from "./response-normalization" // altimate_change end import { readSql } from "./sql-check" +import { MaterializeError, storeRows } from "./sql-materialize" +import { firstTime } from "./session-hints" const MAX_SQL_ROWS = 10_000 const LIMIT_ERROR = "limit must be a finite positive integer" const MAX_GENERATED_SQL_BYTES = 1024 * 1024 // Below the default tool-output cap, so a redirect never spills the SQL into a truncated file. const MAX_REDIRECT_SQL_BYTES = 40 * 1024 +// A batch shares one output budget; each query gets an equal share of what is left, so small results donate the rest. +const MAX_BATCH_QUERIES = 8 +const BATCH_MAX_PREVIEW_BYTES = 24 * 1024 + +/** Per-query overrides a batch applies to one item. */ +interface ItemBudget { + maxBytes: number + maxLines: number + // A later batch step references this one: fetch at least this many rows and hand back the raw result. + minRows?: number + capture?: (result: SqlExecuteResult) => void +} + +// Batch steps reference earlier steps: {{label.column}} (literal value list) or {{label}} (scratch table). +const STEP_REF = /\{\{([A-Za-z_]\w*)(?:\.([A-Za-z_]\w*))?\}\}/g +const MAX_BOUND_ROWS = 1000 function generatedRead(sql: string, warehouse: string, stage: "source" | "result"): string { try { @@ -52,7 +70,20 @@ function generatedRead(sql: string, warehouse: string, stage: "source" | "result const parameters = z .object({ - query: z.string().nullish().describe("SQL query to execute; omit when using query_from"), + query: z.string().nullish().describe("SQL query to execute; omit when using query_from or queries"), + queries: z + .array( + z.object({ + label: z.string().nullish().describe("Short name for this result (default q1, q2, ...)"), + query: z.string().describe("Read-only SQL (or MongoDB JSON command)"), + warehouse: z.string().nullish().describe("Warehouse for this query (default: warehouse)"), + }), + ) + .max(MAX_BATCH_QUERIES, `queries accepts at most ${MAX_BATCH_QUERIES} items`) + .nullish() + .describe( + `Up to ${MAX_BATCH_QUERIES} read-only queries run in order in one call, each returning its own labeled result; one failing query does not stop the others. A later query can use an earlier one's rows: {{label.column}} becomes its values as comma-separated literals, e.g. IN ({{ids.id}}) (at most ${MAX_BOUND_ROWS} rows) and {{label}} a scratch table (warehouse scratch).`, + ), query_from: z .object({ warehouse: z.string().describe("Warehouse containing the SQL generator or saved SQL"), @@ -95,14 +126,19 @@ export const SqlExecuteTool = Tool.define( return () => Effect.gen(function* () { const bridge = yield* EffectBridge.make() - return yield* legacyToInit({ + const def = { description: "Execute SQL against a connected data warehouse. Returns all selected columns as a formatted table. full_text_columns prioritizes named cells within the ordinary output budget and always saves returned rows to result_path; read with char_offset/max_chars recovers clipped text. [] or null, or omitting the field, turns the mode off. A non-empty list caps preview rows at 20: omit limit to use the default; larger limits clamp to 20 with a result note. Missing column names produce a bounded note listing available columns while still returning rows. " + - "Provide exactly one of query or query_from. query_from runs a read-only generator SELECT and executes its single SQL string result (up to 1 MiB) as a read-only SELECT on warehouse, without previewing or retyping the generated SQL. " + + "Provide exactly one of query, query_from or queries. query_from runs a read-only generator SELECT and executes its single SQL string result (up to 1 MiB) as a read-only SELECT on warehouse, without previewing or retyping the generated SQL. " + + `queries runs up to ${MAX_BATCH_QUERIES} read-only queries in order in one call and returns one labeled result per query; a later query can reference an earlier one with IN ({{label.column}}) (its values) or {{label}} (scratch table). ` + "For DuckDB table sets, generate query_table([...], true) with escaped table-name string literals. For thousands of tables, UNION ALL BY NAME batches of at most 128 names to stay within the default expression depth.", parameters, - formatValidationError: (error) => error.issues?.map((issue) => issue.message).join("; ") || LIMIT_ERROR, - async execute(args, ctx) { + formatValidationError: (error: z.ZodError) => error.issues?.map((issue) => issue.message).join("; ") || LIMIT_ERROR, + async execute(args: z.infer, ctx: LegacyContext, budget?: ItemBudget): Promise { + if (!budget && args.queries?.length) { + const limits = await bridge.promise(truncate.limits()) + return executeBatch(args, ctx, limits, (item, itemBudget) => def.execute(item, ctx, itemBudget)) + } const fullText = Boolean(args.full_text_columns?.length) const limit = fullText ? Math.min(args.limit ?? 20, 20) : Math.min(args.limit ?? 100, MAX_SQL_ROWS) const limitNote = @@ -140,7 +176,7 @@ export const SqlExecuteTool = Tool.define( ...(Registry.canonicalType(Registry.getConfig(source.warehouse)?.type) === "postgres" ? { read_only: true } : {}), - }, { signal: ctx.abort, sessionID: ctx.sessionID }) + }, { signal: ctx.abort, sessionID: ctx.sessionID, ...(ctx.extra?.forceReadOnly === true ? { forceReadOnly: true } : {}) }) const error = normalizeError(result.error) if (error !== undefined) { throw new Error(`query_from source failed: ${error.trim() || "SQL execution failed."}`) @@ -200,6 +236,9 @@ export const SqlExecuteTool = Tool.define( "DROP DATABASE, DROP SCHEMA, TRUNCATE, and executable SQL comments are blocked for safety. This cannot be overridden.", ) } + if (budget && queryType !== "read") { + throw new Error("Batched queries must be read-only; run a write as its own sql_execute call.") + } const parseError = "parseError" in classification ? classification.parseError : undefined if (!mongo && queryType === "read") { // classifyAndCheck verifies complete lexical/parser agreement and every @@ -278,10 +317,18 @@ export const SqlExecuteTool = Tool.define( const result = await Dispatcher.call("sql.execute", { sql: query, warehouse: args.warehouse, - limit, + limit: Math.max(limit, budget?.minRows ?? 0), query_timeout_ms: args.query_timeout_ms, ...(readOnly ? { read_only: true } : {}), - }, { signal: ctx.abort, sessionID: ctx.sessionID }) + }, { + signal: ctx.abort, + sessionID: ctx.sessionID, + // forceReadOnly protects source connections; the session's own scratch tables take the + // same table writes here that need no write permission above. + ...(ctx.extra?.forceReadOnly === true && !(queryType === "write" && isScratch(args.warehouse) && isScratchSafeWrite(sql)) + ? { forceReadOnly: true } + : {}), + }) // altimate_change start — a failure must not be rendered as "(0 rows)". // sql.execute never throws: it catches every connection and query error @@ -309,6 +356,7 @@ export const SqlExecuteTool = Tool.define( }) } // altimate_change end + budget?.capture?.(result) // altimate_change start — emit SQL structure fingerprint telemetry on the // success path, BEFORE formatting the result. A query that reached this point @@ -337,7 +385,10 @@ export const SqlExecuteTool = Tool.define( } // altimate_change end - const limits = await bridge.promise(truncate.limits()) + const configured = await bridge.promise(truncate.limits()) + const limits = budget + ? { maxBytes: Math.min(configured.maxBytes, budget.maxBytes), maxLines: Math.min(configured.maxLines, budget.maxLines) } + : configured const maxBytes = Math.min(limits.maxBytes, SQL_MAX_PREVIEW_BYTES) // Reserve the routing explanation before the formatter budgets optional notes. const routingNotice = fullText ? Precedence.annotate(precedence, { output: "" }).output : "" @@ -360,7 +411,8 @@ export const SqlExecuteTool = Tool.define( const suggestion = ctx.agent === "analyst" ? null : PostConnectSuggestions.getProgressiveSuggestion("sql_execute") const lint = await lintSqlResult(sql, args.warehouse, ctx.sessionID) const formatted = await formatSqlResult(result, { - limit, + mcpClient: ctx.extra?.mcpClient === true, + limit: Math.max(limit, budget?.minRows ?? 0), fullTextColumns: args.full_text_columns, // The statement text lets the formatter add the empty/near-empty // result hint when a string-literal filter is present. @@ -373,7 +425,16 @@ export const SqlExecuteTool = Tool.define( ].filter(Boolean), maxBytes: routingClipped ? 0 : maxBytes - routingBytes, maxLines: limits.maxLines - routingLines, - write: (text) => bridge.promise(truncate.write(text)), + // The MCP adapter names its spill files itself (sequential, private directory). + write: (text) => + typeof ctx.extra?.writeSpill === "function" + ? ctx.extra.writeSpill(text, "csv") + : bridge.promise(truncate.write(text)), + store: + ctx.extra?.mcpClient === true + ? () => storeSpill(result, { signal: ctx.abort, sessionID: ctx.sessionID }) + : undefined, + firstHint: ctx.extra?.mcpClient === true ? (key) => firstTime(ctx.sessionID, key) : undefined, }) // altimate_change start — progressive disclosure suggestions if (suggestion && formatted.output.includes(suggestion)) { @@ -412,11 +473,203 @@ export const SqlExecuteTool = Tool.define( }) } }, - } satisfies LegacyToolDef) + } satisfies LegacyToolDef + return yield* legacyToInit(def) }) }), ) +/** Next spill table number per session; names are sequential, so outputs stay reproducible. */ +const spillCounters = new Map() +const MAX_SPILL_SESSIONS = 1_000 +const SPILL_NAME_ATTEMPTS = 5 + +/** + * Stores a spilled result's returned rows as scratch table result_N, so the client queries them + * instead of reading the file. Undefined when storing fails; the file remains the fallback. + */ +async function storeSpill(result: SqlExecuteResult, context: { signal: AbortSignal; sessionID: string }) { + for (let attempt = 0; attempt < SPILL_NAME_ATTEMPTS; attempt++) { + const next = (spillCounters.get(context.sessionID) ?? 0) + 1 + spillCounters.delete(context.sessionID) + spillCounters.set(context.sessionID, next) + if (spillCounters.size > MAX_SPILL_SESSIONS) spillCounters.delete(spillCounters.keys().next().value!) + const table = `result_${next}` + try { + await storeRows(table, result, context) + return table + } catch (error) { + context.signal.throwIfAborted() + // A table the client created under this name: try the next number. + if (!(error instanceof MaterializeError && error.message.includes("already exists"))) return undefined + } + } + return undefined +} + +type BatchItem = { label?: string | null; query: string; warehouse?: string | null } + +/** + * Runs independent read queries one after another (one connection per warehouse) and renders each + * result under its label. A failing query becomes its own error block; the others still run. + * Every block, error or not, is clipped to its share of one shared byte/line budget. + */ +async function executeBatch( + args: z.infer, + ctx: LegacyContext, + configured: { maxBytes: number; maxLines: number }, + run: (item: z.infer, budget: ItemBudget) => Promise, +): Promise { + const items = args.queries as BatchItem[] + if (args.query?.trim() || args.query_from?.warehouse?.trim() || args.query_from?.sql?.trim()) { + throw new Error("Provide either queries or query/query_from, not both.") + } + let bytesLeft = Math.min(configured.maxBytes, BATCH_MAX_PREVIEW_BYTES) + let linesLeft = configured.maxLines + const blocks: string[] = [] + const summary: { label: string; rowCount: number; error?: string; result_path?: string }[] = [] + const labels = items.map((item, index) => (item.label ?? "").replace(/\s+/g, " ").trim().slice(0, 60) || `q${index + 1}`) + // Steps a later step references, as a value list ({{label.column}}) or a scratch table ({{label}}). + const listRefs = new Set() + const tableRefs = new Set() + for (const item of items.slice(1)) + for (const [, ref, column] of item.query.matchAll(STEP_REF)) (column ? listRefs : tableRefs).add(ref) + const captured = new Map() + const stored = new Map() + for (const [index, item] of items.entries()) { + ctx.abort.throwIfAborted() + const share = Math.floor(bytesLeft / (items.length - index)) + const lineShare = Math.max(1, Math.floor(linesLeft / (items.length - index))) + const label = labels[index] + const header = `[${label}]\n` + const separator = blocks.length ? "\n\n" : "" + const reserved = Buffer.byteLength(separator + header, "utf8") + Buffer.byteLength(BATCH_CLIPPED, "utf8") + const referenced = listRefs.has(label) || tableRefs.has(label) + const budget: ItemBudget = { + maxBytes: Math.max(0, share - reserved), + maxLines: Math.max(1, lineShare - 2), + ...(referenced ? { minRows: MAX_BOUND_ROWS, capture: (raw: SqlExecuteResult) => captured.set(label, raw) } : {}), + } + const warehouse = item.warehouse?.trim() || args.warehouse + let result: LegacyResult + try { + const query = resolveStepRefs(item.query, labels.slice(0, index), captured, stored, warehouse) + result = await run( + { + query, + warehouse, + limit: args.limit, + full_text_columns: args.full_text_columns, + query_timeout_ms: args.query_timeout_ms, + }, + budget, + ) + } catch (error) { + if (ctx.abort.aborted) throw error + const message = error instanceof Error ? error.message : String(error) + result = { title: "SQL: ERROR", metadata: { error: message }, output: `Failed to execute SQL: ${message}` } + } + const failedStep = typeof result.metadata?.error === "string" && result.metadata?.success !== true + if (failedStep) captured.delete(label) + // A step referenced as {{label}} is stored in scratch (result_N), like a spilled result. + const raw = captured.get(label) + if (raw && tableRefs.has(label) && !raw.truncated) { + const table = await storeSpill(raw, { signal: ctx.abort, sessionID: ctx.sessionID }) + if (table) stored.set(label, table) + } + const body = clipToBudget(result.output, budget.maxBytes, budget.maxLines) + const block = separator + header + body + blocks.push(block) + bytesLeft -= Buffer.byteLength(block, "utf8") + linesLeft -= block.split("\n").length - (separator ? 1 : 0) + const metadata = (result.metadata ?? {}) as Record + summary.push({ + label, + rowCount: Number(metadata.rowCount ?? 0), + ...(typeof metadata.error === "string" && metadata.success !== true ? { error: metadata.error } : {}), + ...(typeof metadata.result_path === "string" ? { result_path: metadata.result_path } : {}), + }) + } + const failed = summary.filter((item) => item.error !== undefined) + return { + title: `SQL: ${items.length} queries${failed.length ? `, ${failed.length} failed` : ""}`, + metadata: { + rowCount: summary.reduce((total, item) => total + item.rowCount, 0), + queries: summary, + result_paths: summary.flatMap((item) => (item.result_path ? [item.result_path] : [])), + // This tool owns clipping; every block is already bounded. + truncated: false, + ...(failed.length === items.length ? { error: failed.map((item) => `${item.label}: ${item.error}`).join("\n") } : {}), + }, + output: blocks.join(""), + } +} + +/** + * Replaces {{label.column}} with that earlier step's distinct column values as comma-separated literals (no brackets, + * so `IN ({{x.id}})`, a DuckDB list `[{{x.id}}]` and a MongoDB `[{{x.id}}]` all work; NULL when empty) and {{label}} + * with the scratch table holding its rows. Throws, failing only this step, when the + * reference names no earlier successful step, a missing column, a step with more than MAX_BOUND_ROWS rows, or a + * table reference outside scratch. + */ +function resolveStepRefs( + query: string, + earlier: string[], + captured: Map, + stored: Map, + warehouse: string | undefined, +): string { + return query.replace(STEP_REF, (match, ref: string, column: string | undefined) => { + if (!earlier.includes(ref)) throw new Error(`${match} names no earlier step in this call.`) + const source = captured.get(ref) + if (!source) throw new Error(`${match}: step "${ref}" failed, so this step did not run.`) + if (source.truncated) + throw new Error(`${match}: step "${ref}" has more than ${MAX_BOUND_ROWS} rows; filter or aggregate it first.`) + if (!column) { + if (!isScratch(warehouse)) + throw new Error(`${match} is a scratch table: run this step on warehouse "scratch", or use {{${ref}.column}}.`) + const table = stored.get(ref) + if (!table) throw new Error(`${match}: storing step "${ref}" in scratch failed.`) + return table + } + const index = source.columns.findIndex((name) => name.toLowerCase() === column.toLowerCase()) + // Some drivers return no column names for an empty result; an empty step is an empty list either way. + if (index < 0 && !source.rows.length && !source.columns.length) return isMongoWarehouse(warehouse) ? "null" : "NULL" + if (index < 0) throw new Error(`${match}: step "${ref}" has no column ${column} (columns: ${source.columns.join(", ")}).`) + const values = [...new Set(source.rows.map((row) => row[index]).filter((value) => value !== null && value !== undefined))] + if (isMongoWarehouse(warehouse)) return values.map((value) => JSON.stringify(value)).join(", ") || "null" + const mysql = ["mysql", "mariadb"].includes(Registry.canonicalType(warehouse ? Registry.getConfig(warehouse)?.type : undefined) ?? "") + return values.length ? values.map((value) => sqlLiteral(value, mysql)).join(", ") : "NULL" + }) +} + +function sqlLiteral(value: unknown, mysql: boolean): string { + if (typeof value === "number" && Number.isFinite(value)) return String(value) + if (typeof value === "bigint") return value.toString() + if (typeof value === "boolean") return value ? "TRUE" : "FALSE" + const text = typeof value === "string" ? value : value instanceof Date ? value.toISOString() : JSON.stringify(value) + const escaped = text.replaceAll("'", "''") + return `'${mysql ? escaped.replaceAll("\\", "\\\\") : escaped}'` +} + +const BATCH_CLIPPED = "…[output clipped]" + +/** Clip text to a byte and line budget on code-point boundaries, marking any clip. */ +function clipToBudget(text: string, maxBytes: number, maxLines: number): string { + if (Buffer.byteLength(text, "utf8") <= maxBytes && text.split("\n").length <= maxLines) return text + let bytes = 0 + let lines = 1 + let end = 0 + for (const char of text) { + const size = Buffer.byteLength(char, "utf8") + if (bytes + size > maxBytes || (char === "\n" && lines >= maxLines)) break + bytes += size + if (char === "\n") lines++ + end += char.length + } + return text.slice(0, end) + BATCH_CLIPPED +} + // altimate_change start — pre-execution SQL validation via cached schema const CACHE_TTL_MS = 24 * 60 * 60 * 1000 // 24 hours // High ceiling so large warehouses aren't arbitrarily truncated; we emit diff --git a/packages/opencode/src/altimate/tools/sql-materialize.ts b/packages/opencode/src/altimate/tools/sql-materialize.ts index 298c924269..9a5c79b01a 100644 --- a/packages/opencode/src/altimate/tools/sql-materialize.ts +++ b/packages/opencode/src/altimate/tools/sql-materialize.ts @@ -11,6 +11,7 @@ import { readSql } from "./sql-check" import { lintSqlResult } from "./sql-result-lint" import * as Precedence from "../workspace/precedence" import { executeQuery, type QueryContext } from "../native/query-execution" +import { firstTime } from "./session-hints" /** Default and hard ceilings, see the tool description. */ export const DEFAULT_MAX_ROWS = 2_000_000 @@ -242,7 +243,7 @@ async function executeSource(source: Connector, input: MaterializeInput, sql: st const options = { queryTimeoutMs: input.query_timeout_ms, maxResultBytes: maxBytes, - ...(type === "postgres" ? { readOnly: true } : {}), + ...(type === "postgres" || context?.forceReadOnly ? { readOnly: true } : {}), } return executeQuery(config, options, context, (bounded) => source.execute(sql, limit, undefined, bounded)) } @@ -315,34 +316,73 @@ async function materializeWithinDeadline(input: MaterializeInput, context: Query } } - // Type inference (the drivers expose no column metadata, only names, so the type comes from the values: - // integers stay BIGINT, any fractional value widens to DOUBLE, mixed kinds fall back to VARCHAR, an - // all-NULL column is VARCHAR). - const types: ScratchType[] = [] - for (let c = 0; c < names.length; c++) { - let t: ScratchType | null = null - for (const row of result.rows) { - const v = row[c] - if (t !== "VARCHAR") { - const k = kindOf(v) - if (k !== null) t = widen(t, k === "JSON" ? "VARCHAR" : k) - } - } - types.push(t ?? "VARCHAR") - } - return { result, names, types } + return { result, names, types: inferTypes(names.length, result.rows) } } // Remote sources are read and buffered before taking the destination's transaction lock. A scratch // source is read inside the lock so a derivation sees and replaces consistent state. // The buffer reservation spans the destination write too: the buffered rows stay live until then. - const writeToScratch = async (prepared?: Awaited>) => { + const writeToScratch = (prepared?: Prepared) => + writeRows({ table, mode, query_timeout_ms: input.query_timeout_ms }, context, async (scratch) => prepared ?? prepare(scratch)) + if (isScratch(input.source_warehouse)) return writeToScratch() + return withBufferBudget(maxBytes, async () => writeToScratch(await prepare(source))) +} + +/** + * Type inference: the drivers expose no column metadata, only names, so the type comes from the values: + * integers stay BIGINT, any fractional value widens to DOUBLE, mixed kinds fall back to VARCHAR, an + * all-NULL column is VARCHAR. + */ +function inferTypes(width: number, rows: unknown[][]): ScratchType[] { + const types: ScratchType[] = [] + for (let c = 0; c < width; c++) { + let t: ScratchType | null = null + for (const row of rows) { + const v = row[c] + if (t !== "VARCHAR") { + const k = kindOf(v) + if (k !== null) t = widen(t, k === "JSON" ? "VARCHAR" : k) + } + } + types.push(t ?? "VARCHAR") + } + return types +} + +type Prepared = { result: { rows: unknown[][] }; names: string[]; types: ScratchType[] } + +/** + * Store rows already in memory (a query result) as a new scratch table, typed the same way as + * sql_materialize. Fails if the table exists. + */ +export async function storeRows( + table: string, + result: { columns: string[]; rows: unknown[][] }, + context: QueryContext, +): Promise { + if (!IDENTIFIER.test(table)) throw new MaterializeError(`Invalid scratch table name "${table}".`) + await Registry.get(SCRATCH_NAME) + const names = uniqueColumnNames(result.columns) + return writeRows({ table, mode: "create" }, context, async () => ({ + result, + names, + types: inferTypes(names.length, result.rows), + })) +} + +async function writeRows( + target: { table: string; mode: "create" | "replace" | "append"; query_timeout_ms?: number }, + context: QueryContext, + load: (scratch: Connector) => Promise, +): Promise { + const { table, mode } = target + { const warnings: string[] = [] return withScratchConnection(async (scratch, directory) => { - const { result, names, types } = prepared ?? (await prepare(scratch)) + const { result, names, types } = await load(scratch) const run = (sql: string) => executeQuery( Registry.getConfig(SCRATCH_NAME), - { noLimit: true, queryTimeoutMs: input.query_timeout_ms }, + { noLimit: true, queryTimeoutMs: target.query_timeout_ms }, context, (options) => scratch.execute(sql, undefined, undefined, options), ) @@ -439,8 +479,6 @@ async function materializeWithinDeadline(input: MaterializeInput, context: Query } }) } - if (isScratch(input.source_warehouse)) return writeToScratch() - return withBufferBudget(maxBytes, async () => writeToScratch(await prepare(source))) } export const SqlMaterializeTool = Tool.define("sql_materialize", { @@ -503,7 +541,7 @@ export const SqlMaterializeTool = Tool.define("sql_materialize", { if (verdict.redirect) return verdict.redirect // altimate_change end try { - const r = await materialize(args, { signal: ctx.abort, sessionID: ctx.sessionID }) + const r = await materialize(args, { signal: ctx.abort, sessionID: ctx.sessionID, ...(ctx.extra?.forceReadOnly === true ? { forceReadOnly: true } : {}) }) const lint = await lintSqlResult(args.sql, args.source_warehouse, ctx.sessionID) const checks = lint ? "\n\n" + lint.output : "" const lintMetadata = lint ? { lint_rules: lint.rules } : {} @@ -524,6 +562,19 @@ export const SqlMaterializeTool = Tool.define("sql_materialize", { } const cols = r.columns.map((c) => `${c.name} ${c.type}`).join(", ") const rowsLine = r.mode === "append" ? `${r.rows} rows appended (${r.total_rows} total)` : `${r.rows} rows` + // altimate_change start — MCP: the client already has its SQL; the join hint is shown once per session + const mcp = ctx.extra?.mcpClient === true + const joinHint = `Join it in scratch with sql_execute(warehouse: "${SCRATCH_NAME}", query: "SELECT ... FROM ${r.table} ...") ` + const source = isScratch(args.source_warehouse) ? SCRATCH_NAME : args.source_warehouse + const receipt = mcp + ? `${label}: ${rowsLine}, ${r.columns.length} columns (${r.mode}) from ${source}.\nColumns: ${cols}` + + (firstTime(ctx.sessionID, "materialize-join") ? `\n\n${joinHint.trimEnd()}.` : "") + : `${label}: ${rowsLine}, ${r.columns.length} columns (${r.mode === "append" ? "append" : r.mode}).\n` + + `Columns: ${cols}\n` + + `Source: ${source}. SQL run:\n${args.sql}\n\n` + + joinHint + + `(the table is also addressable as ${label}).` + // altimate_change end return Precedence.annotate(verdict, { title: `Materialize ${label}: ${r.rows} rows`, metadata: { @@ -534,14 +585,7 @@ export const SqlMaterializeTool = Tool.define("sql_materialize", { mode: r.mode, ...lintMetadata, }, - output: - warning + - `${label}: ${rowsLine}, ${r.columns.length} columns (${r.mode === "append" ? "append" : r.mode}).\n` + - `Columns: ${cols}\n` + - `Source: ${isScratch(args.source_warehouse) ? SCRATCH_NAME : args.source_warehouse}. SQL run:\n${args.sql}\n\n` + - `Join it in scratch with sql_execute(warehouse: "${SCRATCH_NAME}", query: "SELECT ... FROM ${r.table} ...") ` + - `(the table is also addressable as ${label}).` + - checks, + output: warning + receipt + checks, }) } catch (e) { const msg = e instanceof Error ? e.message : String(e) diff --git a/packages/opencode/src/altimate/tools/sql-result-format.ts b/packages/opencode/src/altimate/tools/sql-result-format.ts index 89ca714961..b050d4937f 100644 --- a/packages/opencode/src/altimate/tools/sql-result-format.ts +++ b/packages/opencode/src/altimate/tools/sql-result-format.ts @@ -6,6 +6,7 @@ export const SQL_MAX_PREVIEW_BYTES = 16 * 1024 export const SQL_MAX_SPILL_BYTES = 32 * 1024 * 1024 interface FormatOptions { + mcpClient?: boolean limit: number maxBytes?: number maxLines?: number @@ -14,8 +15,16 @@ interface FormatOptions { /** The executed statement; enables the empty-result hint. */ sql?: string write: (text: string) => Promise + /** MCP: called when the result spills; stores every returned row as a scratch table and returns its name. */ + store?: () => Promise + /** MCP: true the first time a usage hint is due in this session; later outputs use the short form. */ + firstHint?: (key: string) => boolean } +/** MCP spills show a short preview; the full rows are queryable in scratch. */ +const MCP_SPILL_PREVIEW_ROWS = 10 +const MCP_SPILL_CELL_BYTES = 160 + function lineCount(text: string): number { let lines = 1 for (let i = 0; i < text.length; i++) if (text.charCodeAt(i) === 10) lines++ @@ -46,7 +55,13 @@ function clipCell(text: string, maxBytes: number, maxLines: number): string | un return clipText(text, maxBytes, maxLines, `…[cell clipped, ${chars} chars]`) } -function previewSchema(columns: string[], maxBytes: number, maxLines: number, priority?: Set) { +function previewSchema( + columns: string[], + maxBytes: number, + maxLines: number, + priority?: Set, + separator = true, +) { const order = columns.map((_, index) => index) if (priority) order.sort((a, b) => Number(priority.has(columns[b]!)) - Number(priority.has(columns[a]!))) const render = (name: string) => (/[\n\r]| \| /.test(name) ? JSON.stringify(name) : name) @@ -63,8 +78,8 @@ function previewSchema(columns: string[], maxBytes: number, maxLines: number, pr const aliases = rendered.map((name) => clipText(name, aliasBytes, 1, "…[alias clipped]")) if (aliases.some((name) => name === undefined)) continue const names = aliases as string[] - const text = - names.join(" | ") + "\n" + names.map((name) => "-".repeat(Math.max(name.length, 4))).join("-+-") + "\n" + omitted + const rule = separator ? names.map((name) => "-".repeat(Math.max(name.length, 4))).join("-+-") + "\n" : "" + const text = names.join(" | ") + "\n" + rule + omitted if (Buffer.byteLength(text, "utf8") > maxBytes || lineCount(text) > maxLines) continue return { text, @@ -157,17 +172,25 @@ function displayCell(value: unknown, ancestors = new Set()): { text: st return { text: value === null ? "NULL" : String(value), clipped: false } } -function previewRows(rows: unknown[][], maxBytes: number, maxLines: number, columns: number) { +function previewRows( + rows: unknown[][], + maxBytes: number, + maxLines: number, + columns: number, + maxRows = SQL_MAX_PREVIEW_ROWS, + cellCap?: number, +) { if (!columns) return { body: "", rows: 0, clipped: rows.length > 0 } const shown: string[] = [] let bytes = 0 let lines = 0 let clipped = false - for (const row of rows.slice(0, SQL_MAX_PREVIEW_ROWS)) { + for (const row of rows.slice(0, maxRows)) { const values = row.slice(0, columns).map((v) => { const cell = displayCell(v) - clipped ||= cell.clipped - return cell.text + const text = cellCap === undefined ? cell.text : clipCell(cell.text, cellCap, 1)! + clipped ||= cell.clipped || text !== cell.text + return text }) const remainingBytes = maxBytes - bytes - (shown.length ? 1 : 0) const remainingLines = maxLines - lines @@ -408,6 +431,42 @@ export function serializeRows(result: SqlExecuteResult, emit: (chunk: string) => } } +/** One CSV field (RFC 4180). NULL is an empty field and an empty string is `""`, so the two stay distinct. */ +function csvCell(value: unknown): string { + if (value === null || value === undefined) return "" + if (value === "") return '""' + const text = + typeof value === "string" + ? value + : value instanceof Uint8Array + ? "base64:" + Buffer.from(value).toString("base64") + : value instanceof Date + ? Number.isNaN(value.getTime()) + ? "Invalid Date" + : value.toISOString() + : typeof value === "object" + ? [...spillValue(value)].join("") + : String(value) + return /[",\r\n]/.test(text) || text !== text.trim() ? `"${text.replaceAll('"', '""')}"` : text +} + +/** + * CSV spill for MCP clients: column names once in a header row, then one line per row. Nested values + * are tagged JSON text, binary is `base64:`, dates are ISO 8601. + */ +export function serializeCsv(result: SqlExecuteResult, emit: (chunk: string) => void): void { + let bytes = 0 + const line = (cells: unknown[]) => { + checkSpillSize(cells, { left: SQL_MAX_SPILL_BYTES - bytes }) + const text = cells.map(csvCell).join(",") + "\n" + bytes += Buffer.byteLength(text, "utf8") + if (bytes > SQL_MAX_SPILL_BYTES) throw new Error("result exceeds the SQL spill size limit") + emit(text) + } + line(result.columns) + for (const values of result.rows) line(result.columns.map((_, index) => (index in values ? values[index] : null))) +} + const EMPTY_RESULT_HINT = "Hint: 0 rows. If a literal filter was used, verify the stored values first, e.g. SELECT DISTINCT FROM LIMIT 50 or a LIKE '%…%' probe — stored formats often differ (case, whitespace, prefixes)." @@ -437,6 +496,8 @@ function isScalar(result: SqlExecuteResult): boolean { } export async function formatSqlResult(result: SqlExecuteResult, options: FormatOptions) { + // MCP output is compact: no table rule line, CSV spills, short previews with a scratch table. + const mcp = options.mcpClient === true const fullTextColumns = new Set(options.fullTextColumns ?? []) const fullText = fullTextColumns.size > 0 const limit = fullText ? Math.min(options.limit, 20) : options.limit @@ -466,6 +527,7 @@ export async function formatSqlResult(result: SqlExecuteResult, options: FormatO row_limited: result.truncated || excess, output_truncated: false, result_path: undefined as string | undefined, + scratch_table: undefined as string | undefined, // This tool owns clipping; the generic truncator must not clip away the envelope. truncated: false, } @@ -480,7 +542,7 @@ export async function formatSqlResult(result: SqlExecuteResult, options: FormatO return { output: bounded(withHint(footer + note)), metadata } } - const rowsFor = (bytes: number, lines: number, indices: number[]) => + const rowsFor = (bytes: number, lines: number, indices: number[], maxRows?: number, cellBytes?: number) => fullText ? fullTextRows( rows.map((row) => indices.map((index) => (index in row ? row[index] : null))), @@ -489,7 +551,7 @@ export async function formatSqlResult(result: SqlExecuteResult, options: FormatO bytes, lines, ) - : previewRows(rows, bytes, lines, indices.length) + : previewRows(rows, bytes, lines, indices.length, maxRows, cellBytes) // Ordinary, complete results retain their compact presentation. Full-text mode // always takes the spill path, even for empty results or unknown selected names. if (!fullText) { @@ -497,6 +559,8 @@ export async function formatSqlResult(result: SqlExecuteResult, options: FormatO result.columns, maxBytes - Buffer.byteLength(footer + note), maxLines - lineCount(footer + note) + 1, + undefined, + !mcp, ) const preview = rowsFor( Math.max(0, maxBytes - Buffer.byteLength(schema.text + footer + note)), @@ -516,31 +580,60 @@ export async function formatSqlResult(result: SqlExecuteResult, options: FormatO } let spill: string + // MCP: a short preview, with every returned row queryable as a scratch table instead of a file to read. + const previewLimit = mcp && !fullText ? MCP_SPILL_PREVIEW_ROWS : SQL_MAX_PREVIEW_ROWS + let saved = false try { const chunks: string[] = [] - serializeRows(result, (chunk) => chunks.push(chunk)) + if (mcp) serializeCsv(result, (chunk) => chunks.push(chunk)) + else serializeRows(result, (chunk) => chunks.push(chunk)) metadata.result_path = await options.write(chunks.join("")) - spill = `All ${result.rows.length} returned rows saved to ${metadata.result_path}. JSONL text: columns/values per row; $bigint, $date (ISO), $bytes (base64) tags; strings unchanged. Use read with char_offset/max_chars to recover any clipped text.` + saved = true } catch { spill = `Spill failed (write/serialization error or ${SQL_MAX_SPILL_BYTES / 1024 / 1024} MiB size limit); returned rows were not saved.` } + if (saved) { + // Outside the write's try: a cancelled store must surface as cancellation, not as a spill failure. + const table = mcp && result.rows.length ? await options.store?.() : undefined + metadata.scratch_table = table + const first = (key: string) => options.firstHint?.(key) ?? true + if (table) { + spill = first("spill-table") + ? `All ${result.rows.length} returned rows are in scratch table ${table} (typed like sql_materialize): query it with sql_execute(warehouse: "scratch", query: "SELECT ... FROM ${table}") instead of reading the file. CSV copy: ${metadata.result_path}.` + : `All ${result.rows.length} returned rows are in scratch table ${table} (CSV copy: ${metadata.result_path}).` + } else if (mcp) { + spill = first("spill-csv") + ? `All ${result.rows.length} returned rows saved to ${metadata.result_path}: CSV with a header row; an empty field is NULL, "" is an empty string. Open this path with any file reader to recover clipped text.` + : `All ${result.rows.length} returned rows saved to ${metadata.result_path} (CSV).` + } else { + spill = `All ${result.rows.length} returned rows saved to ${metadata.result_path}. JSONL text: columns/values per row; $bigint, $date (ISO), $bytes (base64) tags; strings unchanged. Use read with char_offset/max_chars to recover any clipped text.` + } + } const envelope = (shown: number, clipped: boolean) => - `Preview: ${shown} of ${result.rows.length} returned rows shown. ` + - (excess - ? `Note: connector returned ${result.rows.length} rows; showing at most ${limit} (requested row limit). ` - : result.truncated - ? "Result may be row-limited (connector reported a limit; total unknown). More rows may exist. " - : "Row-limited: no. ") + - `Output truncated: ${clipped ? "yes" : "no"}. ${spill}` + - (clipped && !fullText ? " Prioritize specific cells with full_text_columns; omit limit to use the default." : "") + - "\n" + mcp + ? `Preview: ${shown} of ${result.rows.length} returned rows. ` + + (excess + ? `The connector returned more rows than the limit (${limit}). ` + : result.truncated + ? "Row limit reached; more rows may exist. " + : "") + + `${spill}\n` + : `Preview: ${shown} of ${result.rows.length} returned rows shown. ` + + (excess + ? `Note: connector returned ${result.rows.length} rows; showing at most ${limit} (requested row limit). ` + : result.truncated + ? "Result may be row-limited (connector reported a limit; total unknown). More rows may exist. " + : "Row-limited: no. ") + + `Output truncated: ${clipped ? "yes" : "no"}. ${spill}` + + (clipped && !fullText ? " Prioritize specific cells with full_text_columns; omit limit to use the default." : "") + + "\n" // Notes share the output budget, including missing names and tool annotations. // Keep room for the result contract even when a note is much larger than the cap. const shownNote = clipText(note, Math.floor(maxBytes / 4), Math.max(1, Math.floor(maxLines / 4)), "…[note clipped]") ?? "" const notesClipped = shownNote !== note - const essentials = envelope(Math.min(rows.length, SQL_MAX_PREVIEW_ROWS), true) + footer + shownNote + const essentials = envelope(Math.min(rows.length, previewLimit), true) + footer + shownNote if (!fits(essentials)) { metadata.schema_truncated = result.columns.length > 0 metadata.output_truncated = metadata.truncated = true @@ -552,13 +645,16 @@ export async function formatSqlResult(result: SqlExecuteResult, options: FormatO maxBytes - Buffer.byteLength(essentials), maxLines - lineCount(essentials) + 1, fullText ? fullTextColumns : undefined, + !mcp, ) : { text: "", indices: [], columns: 0, clipped: false } - const reserved = envelope(Math.min(rows.length, SQL_MAX_PREVIEW_ROWS), true) + schema.text + footer + shownNote + const reserved = envelope(Math.min(rows.length, previewLimit), true) + schema.text + footer + shownNote const preview = rowsFor( Math.max(0, maxBytes - Buffer.byteLength(reserved)), Math.max(0, maxLines - lineCount(reserved) + 1), schema.indices, + previewLimit, + mcp ? MCP_SPILL_CELL_BYTES : undefined, ) metadata.preview_rows = preview.rows metadata.preview_columns = schema.columns diff --git a/packages/opencode/src/altimate/tools/warehouse-add.ts b/packages/opencode/src/altimate/tools/warehouse-add.ts index 54adc86657..a7a5398e6b 100644 --- a/packages/opencode/src/altimate/tools/warehouse-add.ts +++ b/packages/opencode/src/altimate/tools/warehouse-add.ts @@ -1,4 +1,6 @@ import z from "zod" +import fs from "node:fs" +import path from "node:path" import { Tool } from "../../tool/tool" import { Dispatcher } from "../native" // altimate_change start — post-connect feature suggestions @@ -49,12 +51,35 @@ IMPORTANT: For private key file paths, always use "private_key_path" (not "priva } } + // altimate_change start — a read-only MCP session must not create a database file through `create` + if (ctx.extra?.forceReadOnly === true && args.config.create === true) { + return { + title: `Add '${args.name}': FAILED`, + metadata: { success: false, name: args.name, type: String(args.config.type) }, + output: `This session is read-only, so "create": true is not allowed. Point "path" at an existing database file.`, + } + } + // altimate_change end + // altimate_change start — a store this process cannot write (file or directory) is opened read-only; opening it + // read-write failed every query with "attempt to write a readonly database". A property of the file, so it is + // correct to persist; writable stores are unchanged. + const config = unwritableFileStore(args.config) ? { ...args.config, readonly: true } : args.config + // altimate_change end try { const result = await Dispatcher.call("warehouse.add", { name: args.name, - config: args.config, + config, }) + // altimate_change start — MCP clients have only the toolkit: no warehouse_test or suggested follow-up tools + if (result.success && ctx.extra?.mcpClient === true) { + let output = `Added warehouse '${result.name}' (type: ${result.type}). Call warehouse_list to see its tables.` + output += driverReadinessNote(result.type) + if (result.warnings?.length) + output += `\n\nWarning — not every credential was saved:\n${result.warnings.map((w) => `- ${w}`).join("\n")}` + return { title: `Add '${args.name}': OK`, metadata: { success: true, name: result.name, type: result.type }, output } + } + // altimate_change end if (result.success) { // altimate_change start — append post-connect feature suggestions (async, non-blocking) let output = `Successfully added warehouse '${result.name}' (type: ${result.type}).\n\nUse warehouse_test to verify connectivity.` @@ -157,6 +182,22 @@ IMPORTANT: For private key file paths, always use "private_key_path" (not "priva }, }) +// altimate_change start — read-only detection for existing SQLite/DuckDB store files +function unwritableFileStore(config: Record): boolean { + const type = String(config.type).toLowerCase() + const store = config.path + if ((type !== "sqlite" && type !== "duckdb") || typeof store !== "string" || !path.isAbsolute(store)) return false + if (config.readonly !== undefined || !fs.existsSync(store)) return false + try { + fs.accessSync(store, fs.constants.W_OK) + fs.accessSync(path.dirname(store), fs.constants.W_OK) + return false + } catch { + return true + } +} +// altimate_change end + // altimate_change start — driver readiness note for newly added warehouses /** * Note appended to a successful add when the warehouse's driver is missing. diff --git a/packages/opencode/src/altimate/tools/warehouse-discovery.ts b/packages/opencode/src/altimate/tools/warehouse-discovery.ts new file mode 100644 index 0000000000..43828b04a5 --- /dev/null +++ b/packages/opencode/src/altimate/tools/warehouse-discovery.ts @@ -0,0 +1,292 @@ +import { absoluteFileUriPath, isLocalFilePath } from "@altimateai/drivers/file-store" +import type { ConnectionConfig } from "@altimateai/drivers/types" +import { Dispatcher } from "../native" +import * as Registry from "../native/connections/registry" +import { isScratch } from "../native/connections/scratch" +import { executeQuery } from "../native/query-execution" + +/** + * One-call catalog discovery for warehouse_list: tables with columns and a one-row sample, so a + * client can start querying without separate schema_inspect round trips. + * + * Bounded by the data before any work: local engines only (remote warehouses run no queries here), + * table/column/sample budgets shared across connections, one overall deadline checked before every + * call (queries use the connection's own timeout), and a byte cap on the rendered text. Samples run + * only on SQLite and local DuckDB base tables, whose drivers enforce read-only at the database. + */ +const MAX_CONNECTIONS = 8 +const MAX_SCHEMAS = 10 +const TABLE_BUDGET = 40 +const COLUMN_BUDGET = 30 +const SAMPLE_BUDGET = 15 +const MAX_COLUMNS_SHOWN = 40 +const SAMPLE_CELL_CHARS = 40 +const OUTPUT_BYTES = 8 * 1024 +const NOTICE_RESERVE_BYTES = 256 +const DEADLINE_MS = 4_000 +const MONGO_SYSTEM_DATABASES = new Set(["admin", "config", "local"]) +const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "::1", "[::1]"]) +const SAMPLED_ENGINES = new Set(["sqlite", "duckdb"]) + +interface DiscoveryContext { + signal: AbortSignal + sessionID: string +} + +type Table = { schema: string; name: string; type: string; row_count?: number; row_count_estimated?: boolean } + +/** Data on this machine: a local SQLite/DuckDB file (or memory), or a server on a loopback host or socket. */ +export function isLocalConnection(config: ConnectionConfig | undefined): boolean { + // An SSH tunnel reaches a remote host even when the configured host is loopback. + if (!config || config.ssh_host) return false + const type = Registry.canonicalType(config.type) ?? "" + if (type === "sqlite" || type === "duckdb") { + const store = typeof config.path === "string" ? config.path : "" + return store === ":memory:" || (store !== "" && (isLocalFilePath(store) || absoluteFileUriPath(store) !== undefined)) + } + if (!["postgres", "mysql", "mongodb"].includes(type)) return false + const url = [config.connection_string, config.uri, config.url].find((value) => typeof value === "string" && value) + if (typeof url === "string") { + try { + const host = new URL(url).hostname + return LOCAL_HOSTS.has(host.toLowerCase()) + } catch { + return false + } + } + const host = typeof config.host === "string" ? config.host.trim().toLowerCase() : "" + // An absolute host path is a Unix socket on this machine. + return LOCAL_HOSTS.has(host) || host.startsWith("/") +} + +function rowsLabel(table: Table): string { + if (table.row_count === undefined) return "" + return ` (${table.row_count_estimated ? "~" : ""}${table.row_count} rows)` +} + +function sampleCell(value: unknown): string { + const text = + value === null || value === undefined + ? "NULL" + : value instanceof Date + ? value.toISOString() + : typeof value === "object" + ? (JSON.stringify(value, (_key, item) => (typeof item === "bigint" ? item.toString() : item)) ?? String(value)) + : String(value) + const flat = text.replace(/\s+/g, " ") + return flat.length > SAMPLE_CELL_CHARS ? `${flat.slice(0, SAMPLE_CELL_CHARS)}…` : flat +} + +const quoteIdent = (id: string) => `"${id.replace(/"/g, '""')}"` + +/** Renders local connections' tables; returns "" when nothing is discoverable. */ +export async function discoverCatalog(warehouses: { name: string; type: string }[], context: DiscoveryContext): Promise { + const deadline = Date.now() + DEADLINE_MS + const budget = { tables: TABLE_BUDGET, columns: COLUMN_BUDGET, samples: SAMPLE_BUDGET } + const lines: string[] = [] + let bytes = 0 + let full = false + let skipped = 0 + const push = (line: string) => { + const size = Buffer.byteLength(line, "utf8") + 1 + // The two trailing notices are pushed unconditionally; keep room for them inside the cap. + if (full || bytes + size > OUTPUT_BYTES - NOTICE_RESERVE_BYTES) { + full = true + return false + } + lines.push(line) + bytes += size + return true + } + const timeLeft = () => deadline - Date.now() + let remote = 0 + for (const warehouse of warehouses.slice(0, MAX_CONNECTIONS)) { + if (isScratch(warehouse.name)) continue + const config = Registry.getConfig(warehouse.name) + if (!isLocalConnection(config)) { + remote++ + continue + } + if (full || timeLeft() <= 0) { + skipped++ + continue + } + context.signal.throwIfAborted() + const type = Registry.canonicalType(config?.type) ?? "" + try { + const tables = await listTables(warehouse.name, type, budget, context, timeLeft) + if (!tables) { + push(`${warehouse.name}: catalog too large to list here; use schema_inspect.`) + continue + } + if (!tables.length) { + push(`${warehouse.name}: no tables.`) + continue + } + for (const table of tables) { + if (full) break + let line = `${warehouse.name}: ${table.schema}.${table.name}${rowsLabel(table)}` + let names: string[] | undefined + if (budget.columns > 0 && timeLeft() > 0) { + budget.columns-- + context.signal.throwIfAborted() + const inspected = await Dispatcher.call("schema.inspect", { + table: table.name, + schema_name: table.schema, + warehouse: warehouse.name, + }).catch(() => { + context.signal.throwIfAborted() + return undefined + }) + const columns = inspected?.columns ?? [] + if (columns.length) { + names = columns.map((column) => column.name) + const shown = columns.slice(0, MAX_COLUMNS_SHOWN).map((column) => `${column.name} ${column.data_type}`) + line += `: ${shown.join(", ")}${columns.length > shown.length ? `, +${columns.length - shown.length} more` : ""}` + } + } + if (!push(line)) break + if (names && SAMPLED_ENGINES.has(type) && table.type === "table" && budget.samples > 0 && timeLeft() > 0) { + budget.samples-- + const sample = await sampleRow(warehouse.name, config!, table, names.length, context).catch(() => { + context.signal.throwIfAborted() + return undefined + }) + if (sample) push(` e.g. ${sample}`) + } + } + } catch { + context.signal.throwIfAborted() + push(`${warehouse.name}: catalog unavailable here; use schema_inspect.`) + } + } + const omitted = Math.max(0, warehouses.length - MAX_CONNECTIONS) + if (full || skipped || omitted || timeLeft() <= 0) { + lines.push("More tables or connections exist than shown; use schema_inspect for the rest.") + } + if (remote && lines.length) lines.push("Remote connections are not listed here; use schema_inspect.") + return lines.join("\n") +} + +const SUMMARY_BYTES = 1_200 +// Catalog-size gate: one metadata query fetches at most this many names plus one. +const SUMMARY_TABLE_GATE = 40 + +/** Catalog metadata SQL for a local SQLite/DuckDB file; names only, never table data or row counts. */ +function summaryCatalogQuery(type: string): string | undefined { + const limit = SUMMARY_TABLE_GATE + 1 + if (type === "sqlite") + return `SELECT 'main', name FROM main.sqlite_master WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%' ORDER BY name LIMIT ${limit}` + if (type === "duckdb") + return ( + "SELECT table_schema, table_name FROM information_schema.tables WHERE table_catalog = current_database() " + + `AND table_schema NOT IN ('information_schema', 'pg_catalog') ORDER BY table_schema, table_name LIMIT ${limit}` + ) + return undefined +} + +/** Table names of a local SQLite/DuckDB file within the gate, "over" past it, or undefined when unavailable. */ +async function summaryTables(warehouse: string, config: ConnectionConfig): Promise { + const type = Registry.canonicalType(config.type) ?? "" + const sql = summaryCatalogQuery(type) + const store = typeof config.path === "string" ? config.path : "" + // File stores only: a memory store is empty at startup and other paths may name a remote store. An SSH tunnel + // setting would open a network connection before the file. + if (!sql || config.ssh_host || !(isLocalFilePath(store) || absoluteFileUriPath(store) !== undefined)) return undefined + try { + const connector = await Registry.get(warehouse) + // No signal or added deadline: the query is bounded by its LIMIT. Only the connection's own query timeout + // applies, and at startup the engine holds no session scratch tables a restart could drop. + const result = await executeQuery(config, { readOnly: true }, undefined, (options) => + connector.execute(sql, SUMMARY_TABLE_GATE + 1, undefined, options), + ) + if (result.rows.length > SUMMARY_TABLE_GATE || result.truncated) return "over" + return result.rows.map((row) => (row[0] && row[0] !== "main" ? `${row[0]}.${row[1]}` : String(row[1]))) + } catch { + return undefined + } +} + +/** + * A compact connection list for the MCP server instructions: one line per connection with its name and type from + * config. Only local SQLite/DuckDB files are opened, for one catalog-metadata query each (table names, no row counts), + * gated by SUMMARY_TABLE_GATE; remote and server connections run no network call or query. Bounded by SUMMARY_BYTES; + * a connection whose tables do not fit is listed by name only. + */ +export async function catalogSummary(warehouses: { name: string; type: unknown }[]): Promise { + const lines: string[] = [] + let bytes = 0 + const named = warehouses.filter((warehouse) => !isScratch(warehouse.name)) + const shown = named.slice(0, MAX_CONNECTIONS) + const typeOf = (warehouse: { type: unknown }) => (typeof warehouse.type === "string" ? warehouse.type : "unknown") + for (const [index, warehouse] of shown.entries()) { + const head = `- ${warehouse.name} (${typeOf(warehouse)})` + // Keep room for a name-only line for every connection still to come. + const reserve = shown.slice(index + 1).reduce((total, next) => total + next.name.length + typeOf(next).length + 8, 0) + let line = head + const config = Registry.getConfig(warehouse.name) + const tables = config ? await summaryTables(warehouse.name, config) : undefined + const detail = + tables === "over" ? `${head}: over ${SUMMARY_TABLE_GATE} tables` : tables?.length ? `${head}: ${tables.join(", ")}` : head + if (bytes + Buffer.byteLength(detail, "utf8") + 1 + reserve <= SUMMARY_BYTES) line = detail + if (bytes + Buffer.byteLength(line, "utf8") + 1 > SUMMARY_BYTES) break + lines.push(line) + bytes += Buffer.byteLength(line, "utf8") + 1 + } + if (lines.length < named.length) lines.push("- (more: warehouse_list)") + return lines.join("\n") +} + +/** The connection's tables in name order, or undefined when the catalog exceeds the remaining table budget. */ +async function listTables( + warehouse: string, + type: string, + budget: { tables: number }, + context: DiscoveryContext, + timeLeft: () => number, +): Promise { + const options = { signal: context.signal, sessionID: context.sessionID, forceReadOnly: true } + let schemas: string[] + if (type === "mongodb") { + // Database names come from one call; per-database counts would cost a command each. + const connector = await Registry.get(warehouse) + schemas = (await connector.listSchemas()).filter((name) => !MONGO_SYSTEM_DATABASES.has(name)).sort() + } else { + const listing = await Dispatcher.call("schema.list", { warehouse }, options) + const counted = listing.schemas ?? [] + if (listing.truncated || counted.some((schema) => schema.table_count === undefined)) return undefined + if (counted.reduce((total, schema) => total + schema.table_count!, 0) > budget.tables) return undefined + schemas = counted.filter((schema) => schema.table_count! > 0).map((schema) => schema.name) + } + if (schemas.length > MAX_SCHEMAS) return undefined + const tables: Table[] = [] + for (const schema of schemas) { + if (timeLeft() <= 0) return undefined + context.signal.throwIfAborted() + const listing = await Dispatcher.call("schema.list", { warehouse, schema_name: schema }, options) + if (listing.truncated || tables.length + listing.tables.length > budget.tables) return undefined + tables.push(...listing.tables) + } + budget.tables -= tables.length + return tables +} + +async function sampleRow( + warehouse: string, + config: ConnectionConfig, + table: Table, + width: number, + context: DiscoveryContext, +): Promise { + const connector = await Registry.get(warehouse) + const result = await executeQuery( + config, + { readOnly: true }, + { signal: context.signal, sessionID: context.sessionID, forceReadOnly: true }, + (options) => + connector.execute(`SELECT * FROM ${quoteIdent(table.schema)}.${quoteIdent(table.name)} LIMIT 1`, 1, undefined, options), + ) + const row = result.rows[0] + if (!row) return undefined + return row.slice(0, Math.min(width, MAX_COLUMNS_SHOWN)).map(sampleCell).join(" | ") +} diff --git a/packages/opencode/src/altimate/tools/warehouse-list.ts b/packages/opencode/src/altimate/tools/warehouse-list.ts index 7c6f75f405..52aa75572e 100644 --- a/packages/opencode/src/altimate/tools/warehouse-list.ts +++ b/packages/opencode/src/altimate/tools/warehouse-list.ts @@ -1,9 +1,11 @@ import z from "zod" +import { readOnlyCaveat } from "../native/connections/read-only" import { Tool } from "../../tool/tool" import { Dispatcher } from "../native" // altimate_change start — workspace precedence import * as Precedence from "../workspace/precedence" // altimate_change end +import { discoverCatalog } from "./warehouse-discovery" // altimate_change start — point at the locations the registry actually reads import { CONN_ENV_PREFIX, globalConfigPath, localConfigPath } from "../native/connections/registry" @@ -31,8 +33,30 @@ export function noConnectionsMessage(): string { } // altimate_change end +// altimate_change start — MCP: connection candidates from the existing project-scan detection (names and types only, +// never values); every probe is local and fail-safe. +const MAX_CANDIDATES = 10 +async function discoveredCandidates(): Promise { + const scan = await import("./project-scan") + const [env, dbt] = await Promise.all([ + scan.detectEnvVars().catch(() => []), + scan + .detectDbtProject(process.cwd()) + .then((project) => Dispatcher.call("dbt.profiles", { projectDir: project.found ? project.path : undefined })) + .then((result) => result.connections ?? []) + .catch(() => []), + ]) + const lines = [ + ...env.map((c) => `- ${c.type} from environment variable ${c.signal}`), + ...dbt.map((c) => `- ${c.type} from dbt profile ${c.name}`), + ].slice(0, MAX_CANDIDATES) + if (!lines.length) return "" + return ["", "", "Detected but not configured (read the values from their source, then call warehouse_add):", ...lines].join("\n") +} +// altimate_change end + export const WarehouseListTool = Tool.define("warehouse_list", { - description: "List all configured warehouse connections. Shows connection name, type, and database.", + description: "List all configured warehouse connections. Shows connection name, type, and database, with local catalog details and samples when available; row counts are catalog estimates only.", parameters: z.object({}), async execute(args, ctx) { try { @@ -40,10 +64,13 @@ export const WarehouseListTool = Tool.define("warehouse_list", { const warehouses = result.warehouses ?? [] if (warehouses.length === 0) { + // altimate_change start — MCP: name connections the existing env-var and dbt-profile detection can see + const candidates = ctx.extra?.mcpClient === true ? await discoveredCandidates() : "" + // altimate_change end return { title: "Warehouses: none configured", metadata: { count: 0 }, - output: noConnectionsMessage(), + output: noConnectionsMessage() + candidates, } } @@ -55,15 +82,31 @@ export const WarehouseListTool = Tool.define("warehouse_list", { // re-linked project is not told its rows are served by the old workspace. const notes = await Precedence.warehouseListNotes(ctx.sessionID, warehouses) const shadowedCount = notes.size + // altimate_change start — MCP: compact table and one-call catalog discovery + const mcp = ctx.extra?.mcpClient === true const lines: string[] = shadowedCount ? ["Name | Type | Database | Served by", "-----|------|----------|----------"] : ["Name | Type | Database", "-----|------|--------"] + // The rule line only decorates the header. + if (mcp) lines.pop() for (const wh of warehouses) { const row = `${wh.name} | ${wh.type} | ${wh.database ?? "-"}` lines.push(shadowedCount ? `${row} | ${notes.get(wh.name) ?? "local"}` : row) } + if (ctx.extra?.forceReadOnly) { + const caveats = warehouses.map(readOnlyCaveat).filter((note): note is string => !!note) + if (caveats.length) lines.push("", ...caveats) + } + + // Workspace-routed connections are served elsewhere; their local catalog is not the one queried. + if (mcp && !shadowedCount) { + const catalog = await discoverCatalog(warehouses, { signal: ctx.abort, sessionID: ctx.sessionID }) + if (catalog) lines.push("", "Tables (name: columns; e.g. = one sample row):", catalog) + } + // altimate_change end + return { title: `Warehouses: ${warehouses.length} configured`, metadata: { count: warehouses.length, shadowed: shadowedCount }, diff --git a/packages/opencode/src/altimate/workspace/precedence.ts b/packages/opencode/src/altimate/workspace/precedence.ts index ff070b578e..ff706be23b 100644 --- a/packages/opencode/src/altimate/workspace/precedence.ts +++ b/packages/opencode/src/altimate/workspace/precedence.ts @@ -190,6 +190,16 @@ const EMPTY = (reason: Precedence["disabledReason"], workspaceName = "", workspa /** Per-session precedence, refreshed once per turn by the tool resolver and read * (never recomputed) by tool bodies mid-turn. */ const bySession = new Map() + +/** A standalone native-tool server deliberately uses local drivers, without attaching an engine. */ +export function bindLocalSession(sessionID: string): void { + remember(sessionID, EMPTY("escape-hatch")) +} + +export function releaseLocalSession(sessionID: string): void { + bySession.delete(sessionID) + incarnations.delete(sessionID) +} /** One token per session incarnation: set when a session is first remembered or * recreated after eviction, dropped with it. A delivery records against the * incarnation it was published for, so ordinary refreshes (which replace the diff --git a/packages/opencode/src/cli/cmd/mcp.ts b/packages/opencode/src/cli/cmd/mcp.ts index 2fd3c56eaa..e39128c991 100644 --- a/packages/opencode/src/cli/cmd/mcp.ts +++ b/packages/opencode/src/cli/cmd/mcp.ts @@ -108,6 +108,9 @@ export const McpCommand = cmd({ yargs .command(McpAddCommand) .command(McpListCommand) + // altimate_change start — expose native data tools over stdio without an agent + .command(McpServeCommand) + // altimate_change end // altimate_change start — upstream_fix (#790): `status` is the name people reach for when a // server will not connect, and it was the one name that did not exist. It shares the list // handler rather than duplicating a view that already probes live and already prints the @@ -124,6 +127,21 @@ export const McpCommand = cmd({ async handler() {}, }) +// altimate_change start — standalone native-tool MCP server +export const McpServeCommand = cmd({ + command: "serve", + describe: "serve data tools over MCP stdio (no model required)", + builder: (yargs) => yargs + .option("connections", { type: "string", describe: "Connections JSON file" }) + .option("allow-write", { type: "boolean", default: false, describe: "Allow SQL write permissions" }) + .option("all-tools", { type: "boolean", default: false, describe: "Also expose schema_inspect and sql_check" }), + async handler(args) { + const { serveStdio } = await import("../../altimate/mcp-server/server") + await serveStdio({ connections: args.connections, allowWrite: args.allowWrite, allTools: args.allTools }) + }, +}) +// altimate_change end + // altimate_change start — upstream_fix (#878/#701): config-level diagnostics, shared by every // exit of `mcp list` / `mcp status` so a config with nothing listable still reports them. function reportConfigDiagnostics() { diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts index a9adcc16bb..1c8cd37c0e 100644 --- a/packages/opencode/src/index.ts +++ b/packages/opencode/src/index.ts @@ -137,13 +137,19 @@ let cli = yargs(args) .middleware(async (opts) => { if (opts.printLogs) process.env.OPENCODE_PRINT_LOGS = "1" if (opts.logLevel) process.env.OPENCODE_LOG_LEVEL = opts.logLevel + // altimate_change start — the altimate log shim read its level when it was imported, before this ran + if (opts.logLevel) AltimateLog.setLevel(opts.logLevel as AltimateLog.Level) + // altimate_change end + // altimate_change start - workspace precedence escape hatch. Process-wide, not + // per session: an env var inherited by child processes, and under `serve` (and + // `mcp serve`) it covers every session that process hosts. + if (opts.integrations) process.env.ALTIMATE_INTEGRATIONS = String(opts.integrations) + // altimate_change end if (opts.pure) { process.env.OPENCODE_PURE = "1" } - // altimate_change start - workspace precedence escape hatch. Process-wide, not - // per session: an env var inherited by child processes, and under `serve` it - // covers every session that process hosts. - if (opts.integrations) process.env.ALTIMATE_INTEGRATIONS = String(opts.integrations) + // altimate_change start — MCP stdio needs no banner, telemetry startup or agent initialization + if (opts._[0] === "mcp" && opts._[1] === "serve") return // altimate_change end Heap.start() diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index 1bf82c7e32..757c871066 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -51,6 +51,7 @@ import { LLMRequestEvent } from "@/altimate/observability/tracing" // altimate_change start — run deadlines override reasoning effort for each remaining request import { applyBudgetEffort, budgetMessages } from "@/altimate/session/budget-effort" import { activeRun, effortOverride, hasActiveRuns, toolsDisabled } from "@/altimate/session/run-budget-effort" +import { analystEffortOptions } from "@/altimate/session/analyst-effort" import { Session } from "@/session" import { SessionID } from "@/session/schema" // altimate_change end @@ -194,6 +195,16 @@ export namespace LLM { const options: Record = pipe( base, mergeDeep(input.model.options), + // altimate_change start — analyst default reasoning effort; agent options and variants still win + mergeDeep( + analystEffortOptions({ + agent: input.agent, + model: input.model, + variant: input.user.variant, + small: input.small, + }), + ), + // altimate_change end mergeDeep(input.agent.options), mergeDeep(variant), ) diff --git a/packages/opencode/src/tool/truncate.ts b/packages/opencode/src/tool/truncate.ts index d3a8a3b0b4..018db5a498 100644 --- a/packages/opencode/src/tool/truncate.ts +++ b/packages/opencode/src/tool/truncate.ts @@ -54,98 +54,117 @@ export interface Interface { export class Service extends Context.Service()("@opencode/Truncate") {} -export const layer = Layer.effect( - Service, - Effect.gen(function* () { - const fs = yield* FSUtil.Service - - const cleanup = Effect.fn("Truncate.cleanup")(function* () { - // altimate_change start — upstream_fix: age files by mtime, not decoded ID timestamps. - // Identifier packs `timestamp * 4096` into 48 bits, which wraps every - // 2^36 ms (~795 days) — the 26th wrap landed 2026-08-14T11:19:55Z, after - // which every new ID decoded as "ancient" and cleanup deleted files the - // moment they were written. File mtime has no wrap. - const cutoffMs = Date.now() - Duration.toMillis(RETENTION) - const entries = yield* fs.readDirectory(TRUNCATION_DIR).pipe( - Effect.map((all) => all.filter((name) => name.startsWith("tool_"))), - Effect.catch(() => Effect.succeed([])), +// altimate_change start — standalone MCP sessions share output handling with private storage and neutral hints (inner notes are plain comments: markers do not nest); carries upstream_fix: age files by mtime, not decoded ID timestamps. +export function makeLayer( + options: { + directory?: string + limits?: Interface["limits"] + hint?: (file: string, outcome: "success" | "error") => string + } = {}, +) { + const directory = options.directory ?? TRUNCATION_DIR + return Layer.effect( + Service, + Effect.gen(function* () { + const fs = yield* FSUtil.Service + + const cleanup = Effect.fn("Truncate.cleanup")(function* () { + // Age files by mtime, not decoded ID timestamps (upstream fix). + // Identifier packs `timestamp * 4096` into 48 bits, which wraps every + // 2^36 ms (~795 days) — the 26th wrap landed 2026-08-14T11:19:55Z, after + // which every new ID decoded as "ancient" and cleanup deleted files the + // moment they were written. File mtime has no wrap. + const cutoffMs = Date.now() - Duration.toMillis(RETENTION) + const entries = yield* fs.readDirectory(directory).pipe( + Effect.map((all) => all.filter((name) => name.startsWith("tool_"))), + Effect.catch(() => Effect.succeed([])), + ) + for (const entry of entries) { + const file = path.join(directory, entry) + // Stat through the INJECTED filesystem (FSUtil extends FileSystem) so + // custom/in-memory providers behave identically to the host FS. + const info = yield* fs.stat(file).pipe(Effect.catch(() => Effect.succeed(undefined))) + // Unstat-able file or absent mtime: keep it — deletion must fail safe. + const mtimeMs = info && Option.isSome(info.mtime) ? info.mtime.value.getTime() : Number.POSITIVE_INFINITY + if (mtimeMs >= cutoffMs) continue + yield* fs.remove(file).pipe(Effect.catch(() => Effect.void)) + } + }) + + const write = Effect.fn("Truncate.write")(function* (text: string) { + const file = path.join(directory, ToolID.ascending()) + yield* fs.ensureDir(directory).pipe(Effect.orDie) + yield* fs.writeFileString(file, text).pipe(Effect.orDie) + return file + }) + + const limits = Effect.fn("Truncate.limits")(function* () { + if (options.limits) return yield* options.limits() + const configSvc = yield* Effect.serviceOption(Config.Service) + if (Option.isNone(configSvc)) return { maxLines: MAX_LINES, maxBytes: MAX_BYTES } + const cfg = yield* configSvc.value.get().pipe(Effect.catch(() => Effect.succeed(undefined))) + return { + maxLines: cfg?.tool_output?.max_lines ?? MAX_LINES, + maxBytes: cfg?.tool_output?.max_bytes ?? MAX_BYTES, + } + }) + + // Default direction "middle" (head+tail, + // tail-weighted elision) via the shared truncate-core.ts algorithm. + const output = Effect.fn("Truncate.output")(function* ( + text: string, + outputOptions: Options = {}, + agent?: Agent.Info, + ) { + const resolved = yield* limits() + const maxLines = outputOptions.maxLines ?? resolved.maxLines + const maxBytes = outputOptions.maxBytes ?? resolved.maxBytes + const direction = outputOptions.direction ?? TruncateCore.DEFAULT_DIRECTION + const headRatio = outputOptions.headRatio ?? TruncateCore.DEFAULT_HEAD_RATIO + const lines = text.split("\n") + const totalBytes = Buffer.byteLength(text, "utf-8") + + if (TruncateCore.fits(lines, totalBytes, maxLines, maxBytes)) { + return { content: text, truncated: false } as const + } + + const preview = TruncateCore.preview(lines, totalBytes, { maxLines, maxBytes, direction, headRatio }) + const file = yield* write(text) + + const subagent = options.hint ? undefined : yield* Effect.promise(() => taskSubagent(agent)) + const outcome = outputOptions.outcome === "error" ? "failed" : "succeeded" + const hint = options.hint + ? options.hint(file, outputOptions.outcome ?? "success") + : subagent + ? `The tool call ${outcome} but the output was truncated. Full output saved to: ${file}\nUse the Task tool to have ${subagent} agent process this file with Grep and Read (with offset/limit). Do NOT read the full file yourself - delegate to save context.` + : `The tool call ${outcome} but the output was truncated. Full output saved to: ${file}\nUse Grep to search the full content or Read with offset/limit to view specific sections.` + + return { + content: TruncateCore.assemble(preview, hint, direction), + truncated: true, + outputPath: file, + } as const + }) + + yield* cleanup().pipe( + Effect.catchCause((cause) => Effect.logError("truncation cleanup failed", { cause: Cause.pretty(cause) })), + Effect.repeat(Schedule.spaced(Duration.hours(1))), + Effect.delay(Duration.minutes(1)), + Effect.forkScoped, ) - for (const entry of entries) { - const file = path.join(TRUNCATION_DIR, entry) - // Stat through the INJECTED filesystem (FSUtil extends FileSystem) so - // custom/in-memory providers behave identically to the host FS. - const info = yield* fs.stat(file).pipe(Effect.catch(() => Effect.succeed(undefined))) - // Unstat-able file or absent mtime: keep it — deletion must fail safe. - const mtimeMs = - info && Option.isSome(info.mtime) ? info.mtime.value.getTime() : Number.POSITIVE_INFINITY - if (mtimeMs >= cutoffMs) continue - yield* fs.remove(file).pipe(Effect.catch(() => Effect.void)) - } - // altimate_change end - }) - - const write = Effect.fn("Truncate.write")(function* (text: string) { - const file = path.join(TRUNCATION_DIR, ToolID.ascending()) - yield* fs.ensureDir(TRUNCATION_DIR).pipe(Effect.orDie) - yield* fs.writeFileString(file, text).pipe(Effect.orDie) - return file - }) - - const limits = Effect.fn("Truncate.limits")(function* () { - const configSvc = yield* Effect.serviceOption(Config.Service) - if (Option.isNone(configSvc)) return { maxLines: MAX_LINES, maxBytes: MAX_BYTES } - const cfg = yield* configSvc.value.get().pipe(Effect.catch(() => Effect.succeed(undefined))) - return { - maxLines: cfg?.tool_output?.max_lines ?? MAX_LINES, - maxBytes: cfg?.tool_output?.max_bytes ?? MAX_BYTES, - } - }) - - // altimate_change start — default direction "middle" (head+tail, - // tail-weighted elision) via the shared truncate-core.ts algorithm. - const output = Effect.fn("Truncate.output")(function* (text: string, options: Options = {}, agent?: Agent.Info) { - const resolved = yield* limits() - const maxLines = options.maxLines ?? resolved.maxLines - const maxBytes = options.maxBytes ?? resolved.maxBytes - const direction = options.direction ?? TruncateCore.DEFAULT_DIRECTION - const headRatio = options.headRatio ?? TruncateCore.DEFAULT_HEAD_RATIO - const lines = text.split("\n") - const totalBytes = Buffer.byteLength(text, "utf-8") - - if (TruncateCore.fits(lines, totalBytes, maxLines, maxBytes)) { - return { content: text, truncated: false } as const - } - - const preview = TruncateCore.preview(lines, totalBytes, { maxLines, maxBytes, direction, headRatio }) - const file = yield* write(text) - - const subagent = yield* Effect.promise(() => taskSubagent(agent)) - const outcome = options.outcome === "error" ? "failed" : "succeeded" - const hint = subagent - ? `The tool call ${outcome} but the output was truncated. Full output saved to: ${file}\nUse the Task tool to have ${subagent} agent process this file with Grep and Read (with offset/limit). Do NOT read the full file yourself - delegate to save context.` - : `The tool call ${outcome} but the output was truncated. Full output saved to: ${file}\nUse Grep to search the full content or Read with offset/limit to view specific sections.` - - return { - content: TruncateCore.assemble(preview, hint, direction), - truncated: true, - outputPath: file, - } as const - }) - // altimate_change end - - yield* cleanup().pipe( - Effect.catchCause((cause) => Effect.logError("truncation cleanup failed", { cause: Cause.pretty(cause) })), - Effect.repeat(Schedule.spaced(Duration.hours(1))), - Effect.delay(Duration.minutes(1)), - Effect.forkScoped, - ) - - return Service.of({ cleanup, write, output, limits }) - }), -) + + return Service.of({ cleanup, write, output, limits }) + }), + ) +} + +export const layer = makeLayer() +// altimate_change end // altimate_change start — Layer.suspend defers facade refs past circular module-init -export const defaultLayer = Layer.suspend(() => layer.pipe(Layer.provide(FSUtil.defaultLayer), Layer.provide(NodePath.layer))) +export const defaultLayer = Layer.suspend(() => + layer.pipe(Layer.provide(FSUtil.defaultLayer), Layer.provide(NodePath.layer)), +) // altimate_change end // altimate_change start — thunk LayerNode deps defers facade refs past circular module-init diff --git a/packages/opencode/test/altimate/analyst-effort.test.ts b/packages/opencode/test/altimate/analyst-effort.test.ts new file mode 100644 index 0000000000..7f54325d6e --- /dev/null +++ b/packages/opencode/test/altimate/analyst-effort.test.ts @@ -0,0 +1,174 @@ +import { afterEach, describe, expect, spyOn, test } from "bun:test" +import { createAnthropic } from "@ai-sdk/anthropic" +import { ANALYST_EFFORT_ENV, analystEffortOptions } from "@/altimate/session/analyst-effort" +import { begin, downgradeEffort } from "@/altimate/session/run-budget-effort" +import { Auth } from "@/auth" +import { Config } from "@/config/config" +import { Plugin } from "@/plugin" +import { Provider } from "@/provider/provider" +import { ModelID, ProviderID } from "@/provider/schema" +import { ProviderTransform } from "@/provider/transform" +import { Session } from "@/session" +import { LLM } from "@/session/llm" +import { MessageID, SessionID } from "@/session/schema" + +function model(npm: string, id: string): Provider.Model { + const result: Provider.Model = { + id: ModelID.make(id), + providerID: ProviderID.make("test"), + api: { npm, id, url: "https://example.invalid" }, + name: id, + capabilities: { + reasoning: true, + temperature: true, + attachment: false, + toolcall: true, + input: { text: true, audio: false, image: false, video: false, pdf: false }, + output: { text: true, audio: false, image: false, video: false, pdf: false }, + interleaved: false, + }, + cost: { input: 0, output: 0, cache: { read: 0, write: 0 } }, + limit: { context: 200_000, output: 32_000 }, + status: "active", + release_date: "2026-01-01", + options: {}, + headers: {}, + variants: {}, + } + result.variants = ProviderTransform.variants(result) + return result +} + +const analyst = { name: "analyst", options: {} } +const opus = model("@ai-sdk/anthropic", "claude-opus-5-5") + +afterEach(() => { + delete process.env[ANALYST_EFFORT_ENV] +}) + +describe("analyst default reasoning effort", () => { + test("adaptive Claude models get high effort with summarized adaptive thinking", () => { + expect(analystEffortOptions({ agent: analyst, model: opus })).toEqual({ + thinking: { type: "adaptive", display: "summarized" }, + effort: "high", + }) + expect( + analystEffortOptions({ agent: analyst, model: model("@ai-sdk/amazon-bedrock", "anthropic.claude-opus-5-5") }), + ).toMatchObject({ reasoningConfig: { type: "adaptive", maxReasoningEffort: "high" } }) + }) + + test.each([ + ["an older Claude model", model("@ai-sdk/anthropic", "claude-sonnet-4-20250514")], + ["an OpenAI model", model("@ai-sdk/openai", "gpt-5.2")], + ["a Gemini model", model("@ai-sdk/google", "gemini-3-pro")], + ])("adds nothing for %s", (_, other) => { + expect(analystEffortOptions({ agent: analyst, model: other })).toEqual({}) + }) + + test("other agents, small requests, explicit variants and agent options keep their settings", () => { + expect(analystEffortOptions({ agent: { name: "builder", options: {} }, model: opus })).toEqual({}) + expect(analystEffortOptions({ agent: analyst, model: opus, small: true })).toEqual({}) + expect(analystEffortOptions({ agent: analyst, model: opus, variant: "low" })).toEqual({}) + expect(analystEffortOptions({ agent: { name: "analyst", options: { effort: "max" } }, model: opus })).toEqual({}) + }) + + test("the environment selects another effort or disables the default", () => { + process.env[ANALYST_EFFORT_ENV] = "XHIGH" + expect(analystEffortOptions({ agent: analyst, model: opus })).toMatchObject({ effort: "xhigh" }) + process.env[ANALYST_EFFORT_ENV] = "off" + expect(analystEffortOptions({ agent: analyst, model: opus })).toEqual({}) + process.env[ANALYST_EFFORT_ENV] = "unknown" + expect(analystEffortOptions({ agent: analyst, model: opus })).toEqual({}) + }) +}) + +describe("analyst effort in the outgoing request", () => { + async function request(input: { agent: string; model: Provider.Model; downgrade?: boolean }) { + const requests: Record[] = [] + const language = createAnthropic({ + apiKey: "test-key", + fetch: (async (_url, init) => { + requests.push(JSON.parse(String(init?.body))) + const chunks = [ + { + type: "message_start", + message: { + id: "msg_answer", + type: "message", + role: "assistant", + model: input.model.api.id, + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 0 }, + }, + }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "ok" } }, + { type: "content_block_stop", index: 0 }, + { type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 1 } }, + { type: "message_stop" }, + ] + return new Response(chunks.map((chunk) => `event: ${chunk.type}\ndata: ${JSON.stringify(chunk)}\n\n`).join(""), { + headers: { "Content-Type": "text/event-stream" }, + }) + }) as typeof fetch, + })(input.model.api.id) + const spies = [ + spyOn(Provider, "getLanguage").mockResolvedValue(language), + spyOn(Provider, "getProvider").mockResolvedValue({ id: input.model.providerID, options: {} } as Provider.Info), + spyOn(Config, "get").mockResolvedValue({ tracing: { enabled: false } }), + spyOn(Auth, "get").mockResolvedValue(undefined), + spyOn(Session, "get").mockImplementation((async (id) => ({ id }) as Session.Info) as typeof Session.get), + spyOn(Plugin, "trigger").mockImplementation(async (_name, _input, output) => output), + ] + const sessionID = SessionID.make(`ses_analyst_effort_${input.agent}_${input.downgrade ? "downgrade" : "default"}`) + const end = input.downgrade ? begin(sessionID) : undefined + if (input.downgrade) downgradeEffort(sessionID) + try { + const stream = await LLM.stream({ + sessionID, + user: { + id: MessageID.make("msg_analyst_effort_test"), + sessionID, + role: "user", + time: { created: Date.now() }, + agent: input.agent, + model: { providerID: input.model.providerID, modelID: input.model.id }, + }, + model: input.model, + agent: { name: input.agent, mode: "primary", options: {}, permission: [] }, + system: [], + abort: new AbortController().signal, + messages: [{ role: "user", content: "How many?" }], + tools: {}, + }) + for await (const _event of stream.fullStream) { + /* drain the SDK stream */ + } + expect(requests).toHaveLength(1) + return requests[0]! + } finally { + end?.() + for (const spy of spies) spy.mockRestore() + } + } + + test("an Opus-5.5-style analyst request carries high effort and adaptive thinking", async () => { + const body = await request({ agent: "analyst", model: opus }) + expect(body.output_config?.effort).toBe("high") + expect(body.thinking).toEqual({ type: "adaptive", display: "summarized" }) + }) + + test("other agents and older models send no effort", async () => { + expect((await request({ agent: "builder", model: opus })).output_config?.effort).toBeUndefined() + const older = await request({ agent: "analyst", model: model("@ai-sdk/anthropic", "claude-sonnet-4-20250514") }) + expect(older.output_config?.effort).toBeUndefined() + expect(older.thinking).toBeUndefined() + }) + + test("the time-budget downgrade still lowers the analyst effort", async () => { + const body = await request({ agent: "analyst", model: opus, downgrade: true }) + expect(body.output_config?.effort).toBe("low") + }) +}) diff --git a/packages/opencode/test/altimate/mcp-server/compact-schema.test.ts b/packages/opencode/test/altimate/mcp-server/compact-schema.test.ts new file mode 100644 index 0000000000..001c92f31c --- /dev/null +++ b/packages/opencode/test/altimate/mcp-server/compact-schema.test.ts @@ -0,0 +1,44 @@ +import { expect, test } from "bun:test" +import { compactSchema } from "../../../src/altimate/mcp-server/adapter" + +const fillers = [ + { type: "null" }, + { type: "string", const: "" }, + { minItems: 0, maxItems: 0, type: "array", items: { not: {} } }, + { type: "object", properties: {}, additionalProperties: false }, +] + +test("collapses optional-property filler unions but keeps nullable array members", () => { + const schema = { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: { + note: { description: "kept", anyOf: [{ type: "string" }, ...fillers] }, + cells: { type: "array", items: { anyOf: [{ type: "string" }, { type: "number" }, { type: "null" }] } }, + size: { type: "integer", minimum: 0, maximum: Number.MAX_SAFE_INTEGER }, + }, + required: ["cells"], + additionalProperties: false, + } + expect(compactSchema("none", schema) as unknown).toEqual({ + type: "object", + properties: { + note: { type: "string", description: "kept" }, + cells: { type: "array", items: { anyOf: [{ type: "string" }, { type: "number" }, { type: "null" }] } }, + size: { type: "integer", minimum: 0 }, + }, + required: ["cells"], + }) + // The input is not mutated. + expect(schema.properties.note.anyOf).toHaveLength(5) +}) + +test("a description table naming a missing property or a required one to drop fails loudly", () => { + const schema = { + type: "object", + properties: { warehouse: { type: "string" }, check: { type: "string" } }, + required: ["warehouse", "check"], + } + // sql_check's table lists properties this schema lacks. + expect(() => compactSchema("sql_check", schema)).toThrow(/has no property/) +}) diff --git a/packages/opencode/test/altimate/mcp-server/final-answer.test.ts b/packages/opencode/test/altimate/mcp-server/final-answer.test.ts new file mode 100644 index 0000000000..eaaa79f7b5 --- /dev/null +++ b/packages/opencode/test/altimate/mcp-server/final-answer.test.ts @@ -0,0 +1,73 @@ +// altimate_change start — the MCP FINAL ANSWER line carries only the answer +import { describe, expect, test } from "bun:test" +import { finalAnswerText } from "../../../src/altimate/mcp-server/adapter" + +const basis = "Counted distinct orders in the sales table for 2024; refunds excluded." + +describe("finalAnswerText", () => { + test("puts the basis above an answer-only FINAL ANSWER line for fields", () => { + const out = finalAnswerText( + { fields: [{ name: "orders", value: 42 }], asked: ["orders"], basis }, + `${basis}\n\n**Answer:** 42`, + ) + expect(out).toBe(`Basis: ${basis}\n\nFINAL ANSWER: 42`) + }) + + test("renders rows without the basis or answer label on the FINAL ANSWER line", () => { + const out = finalAnswerText( + { + rows: { columns: ["city", "n"], values: [["Oslo", 3], ["Lima", 2]] }, + asked: ["city", "n"], + basis, + }, + "unused", + ) + const answer = out.slice(out.indexOf("FINAL ANSWER: ")) + expect(out.startsWith(`Basis: ${basis}\n\n`)).toBe(true) + expect(answer).toContain("Oslo") + expect(answer).not.toContain("**Answer:**") + expect(answer).not.toContain("refunds excluded") + }) + + test("omits the basis block when the caller already asked for answer_only", () => { + const out = finalAnswerText( + { fields: [{ name: "orders", value: 42 }], asked: ["orders"], basis, answer_only: true }, + "42", + ) + expect(out).toBe("FINAL ANSWER: 42") + }) + + test("a complete fields commit for a two-item request gets the answer-only line", () => { + const args = { + fields: [ + { name: "repository", value: "repo-a" }, + { name: "most copied file", value: "main.swift" }, + ], + asked: ["repository", "most copied file"], + basis, + } + const out = finalAnswerText(args, "unused", "Which repository and its most copied file?") + expect(out.startsWith(`Basis: ${basis}\n\nFINAL ANSWER: `)).toBe(true) + expect(out.slice(out.indexOf("FINAL ANSWER: "))).not.toContain("**Answer:**") + }) + + test("an under-asked fields commit for a two-item request keeps the committed text on the line", () => { + const committed = `${basis} The top file is main.swift.\n\n**Answer:** repository: repo-a` + const out = finalAnswerText( + { fields: [{ name: "repository", value: "repo-a" }], asked: ["repository"], basis }, + committed, + "Which repository and its most copied file?", + ) + expect(out).toBe(`FINAL ANSWER: ${committed}`) + }) + + test("unresolved commits keep the committed text on the line", () => { + const out = finalAnswerText({ unresolved: { reason: "No rows." }, basis }, "committed", "How many orders?") + expect(out).toBe("FINAL ANSWER: committed") + }) + + test("falls back to the committed text when the args cannot be re-rendered", () => { + expect(finalAnswerText({ basis: "" }, "committed")).toBe("FINAL ANSWER: committed") + }) +}) +// altimate_change end diff --git a/packages/opencode/test/altimate/mcp-server/output-settings.test.ts b/packages/opencode/test/altimate/mcp-server/output-settings.test.ts new file mode 100644 index 0000000000..aea90d44f6 --- /dev/null +++ b/packages/opencode/test/altimate/mcp-server/output-settings.test.ts @@ -0,0 +1,99 @@ +// altimate_change start — MCP output limits resolve like the agent's config loader +import { afterEach, beforeEach, expect, test } from "bun:test" +import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { outputLimits } from "../../../src/altimate/mcp-server/output-settings" +import { MAX_BYTES, MAX_LINES } from "../../../src/tool/truncate-core" + +const VARS = [ + "OPENCODE_CONFIG", + "OPENCODE_CONFIG_CONTENT", + "OPENCODE_CONFIG_DIR", + "OPENCODE_DISABLE_PROJECT_CONFIG", + "ALTIMATE_CLI_CONFIG", + "ALTIMATE_CLI_CONFIG_CONTENT", + "ALTIMATE_CLI_CONFIG_DIR", + "ALTIMATE_CLI_DISABLE_PROJECT_CONFIG", +] +let saved: Record +let root: string +let repo: string +let cwd: string + +const output = (tool_output: Record) => JSON.stringify({ tool_output }) + +beforeEach(async () => { + cwd = process.cwd() + saved = Object.fromEntries(VARS.map((name) => [name, process.env[name]])) + for (const name of VARS) delete process.env[name] + root = await realpath(await mkdtemp(path.join(os.tmpdir(), "mcp-output-settings-"))) + repo = path.join(root, "repo") + await mkdir(path.join(repo, "sub"), { recursive: true }) + const init = Bun.spawnSync(["git", "init", "-q", repo]) + expect(init.exitCode).toBe(0) + process.chdir(path.join(repo, "sub")) +}) + +afterEach(async () => { + process.chdir(cwd) + for (const [name, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[name] + else process.env[name] = value + } + await rm(root, { recursive: true, force: true }) +}) + +test("reads inline config through the documented ALTIMATE_CLI_CONFIG_CONTENT name", async () => { + process.env.ALTIMATE_CLI_CONFIG_CONTENT = output({ max_lines: 3, max_bytes: 300 }) + expect(await outputLimits()).toEqual({ maxLines: 3, maxBytes: 300 }) +}) + +test("reads ALTIMATE_CLI_CONFIG and ALTIMATE_CLI_CONFIG_DIR", async () => { + const custom = path.join(root, "custom.json") + await writeFile(custom, output({ max_lines: 4 })) + const dir = path.join(root, "config-dir") + await mkdir(dir) + await writeFile(path.join(dir, "opencode.json"), output({ max_bytes: 400 })) + process.env.ALTIMATE_CLI_CONFIG = custom + process.env.ALTIMATE_CLI_CONFIG_DIR = dir + expect(await outputLimits()).toEqual({ maxLines: 4, maxBytes: 400 }) +}) + +test("stops at the git worktree and ignores altimate-code.* at the project root, as the agent does", async () => { + await writeFile(path.join(root, "opencode.json"), output({ max_bytes: 999 })) + await writeFile(path.join(repo, "altimate-code.json"), output({ max_lines: 9 })) + await writeFile(path.join(repo, "opencode.json"), output({ max_lines: 5 })) + expect(await outputLimits()).toEqual({ maxLines: 5, maxBytes: MAX_BYTES }) +}) + +test("opencode.jsonc overrides opencode.json in the same directory; .opencode/ applies after", async () => { + await writeFile(path.join(repo, "opencode.json"), output({ max_lines: 5, max_bytes: 500 })) + await writeFile(path.join(repo, "opencode.jsonc"), output({ max_lines: 6, max_bytes: 600 })) + await mkdir(path.join(repo, ".opencode")) + await writeFile(path.join(repo, ".opencode", "opencode.json"), output({ max_bytes: 700 })) + expect(await outputLimits()).toEqual({ maxLines: 6, maxBytes: 700 }) +}) + +test("ALTIMATE_CLI_DISABLE_PROJECT_CONFIG skips project files", async () => { + await writeFile(path.join(repo, "opencode.json"), output({ max_lines: 5 })) + process.env.ALTIMATE_CLI_DISABLE_PROJECT_CONFIG = "1" + expect(await outputLimits()).toEqual({ maxLines: MAX_LINES, maxBytes: MAX_BYTES }) +}) + +test("substitutes {env:VAR} as the agent loader does", async () => { + process.env.MCP_OUTPUT_TEST_LINES = "11" + try { + await writeFile(path.join(repo, "opencode.json"), '{ "tool_output": { "max_lines": {env:MCP_OUTPUT_TEST_LINES} } }') + expect(await outputLimits()).toEqual({ maxLines: 11, maxBytes: MAX_BYTES }) + } finally { + delete process.env.MCP_OUTPUT_TEST_LINES + } +}) + +test("an unreadable config path does not stop startup", async () => { + await mkdir(path.join(repo, "sub", "opencode.json")) + await writeFile(path.join(repo, "opencode.jsonc"), output({ max_lines: 8 })) + expect(await outputLimits()).toEqual({ maxLines: 8, maxBytes: MAX_BYTES }) +}) +// altimate_change end diff --git a/packages/opencode/test/altimate/mcp-server/server.test.ts b/packages/opencode/test/altimate/mcp-server/server.test.ts new file mode 100644 index 0000000000..290f2d3f79 --- /dev/null +++ b/packages/opencode/test/altimate/mcp-server/server.test.ts @@ -0,0 +1,1438 @@ +// altimate_change start — standalone MCP enforcement and lifecycle regressions +import { afterEach, expect, spyOn, test } from "bun:test" +import { Database } from "bun:sqlite" +import { chmod, mkdir, mkdtemp, rm, stat, readFile } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { Client } from "@modelcontextprotocol/sdk/client/index.js" +import { ElicitRequestSchema } from "@modelcontextprotocol/sdk/types.js" +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js" +import { createServer, SERVER_INSTRUCTIONS } from "../../../src/altimate/mcp-server/server" +import * as Registry from "../../../src/altimate/native/connections/registry" +import { Dispatcher } from "../../../src/altimate/native" +import type { Connector } from "@altimateai/drivers/types" +import { closeScratch } from "../../../src/altimate/native/connections/scratch" + +// These tests need real SQLite/DuckDB drivers in their own `bun test` process: files elsewhere in +// test/altimate install top-level mock.module("@altimateai/drivers/duckdb", …), which a whole-package +// run evaluates first. CI runs them in a dedicated step with ALTIMATE_MCP_E2E=1. +const mcpTest = process.env["ALTIMATE_MCP_E2E"] === "1" ? test : test.skip + +const cleanups: (() => Promise)[] = [] +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup() + await closeScratch() + await Registry.closeAll() + Registry.reset() +}) + +async function fixture( + allowWrite = false, + capabilities: Record = {}, + elicitTimeoutMs?: number, + setup?: (db: Database) => void, + allTools = true, +) { + const dir = await mkdtemp(path.join(os.tmpdir(), "mcp-tools-test-")) + cleanups.push(() => rm(dir, { recursive: true, force: true })) + const dbPath = path.join(dir, "source.sqlite") + const db = new Database(dbPath) + db.run("CREATE TABLE items (id INTEGER, name TEXT)") + db.run("INSERT INTO items VALUES (1, 'apple'), (2, 'pear')") + setup?.(db) + db.close() + const config = path.join(dir, "connections.json") + await Bun.write(config, JSON.stringify({ source: { type: "sqlite", path: "source.sqlite" } })) + const server = await createServer({ allowWrite, connections: config, elicitTimeoutMs, allTools }) + const client = new Client({ name: "test", version: "1" }, { capabilities }) + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair() + await server.connect(serverTransport) + await client.connect(clientTransport) + cleanups.push(async () => { + await client.close() + await server.close() + await Registry.closeAll() + }) + const call = (name: string, args: Record) => client.callTool({ name, arguments: args }) + return { client, call, server } +} + +function text(result: Awaited>) { + return JSON.stringify(result.content) +} + +mcpTest("initialize result carries short analyst-method instructions", async () => { + const { client } = await fixture() + const instructions = client.getInstructions() + expect(instructions!.startsWith(SERVER_INSTRUCTIONS)).toBe(true) + // The connection list known at startup follows the method, so no warehouse_list call is needed to start. + expect(instructions).toContain("Connections:\n- source (sqlite): items") + expect(SERVER_INSTRUCTIONS.split(/\s+/).length).toBeLessThanOrEqual(120) + for (const tool of ["warehouse_list", "warehouse_add", "sql_execute", "sql_materialize", "answer_commit"]) { + expect(instructions).toContain(tool) + } + // Instructions name only advertised tools. + expect(instructions).not.toMatch(/schema_inspect|sql_check/) +}) + +mcpTest("the default toolset is five tools; sql_execute `tables` describes tables and runs queries in one call", async () => { + const { client, call } = await fixture(false, {}, undefined, undefined, false) + const listed = await client.listTools() + expect(listed.tools.map((t) => t.name).sort()).toEqual([ + "answer_commit", + "sql_execute", + "sql_materialize", + "warehouse_add", + "warehouse_list", + ]) + const declared = listed.tools.reduce( + (total, t) => total + JSON.stringify({ name: t.name, description: t.description, inputSchema: t.inputSchema }).length, + 0, + ) + expect(declared).toBeLessThan(5_200) + expect(SERVER_INSTRUCTIONS.length).toBeLessThanOrEqual(620) + const sqlExecute = listed.tools.find((t) => t.name === "sql_execute")! + expect(Object.keys(sqlExecute.inputSchema.properties ?? {})).toContain("tables") + const fused = await call("sql_execute", { + warehouse: "source", + tables: ["items"], + queries: [{ label: "fruit", query: "SELECT name FROM items ORDER BY id" }], + }) + expect(fused.isError).toBe(false) + const body = text(fused) + // Schema first (column list with its value profile), then the query result, in one response. + expect(body).toContain("name") + expect(body.indexOf("items")).toBeLessThan(body.indexOf("apple")) + expect(body).toContain("pear") + const schemaOnly = await call("sql_execute", { warehouse: "source", tables: ["items"] }) + expect(schemaOnly.isError).toBe(false) + expect(text(schemaOnly)).toContain("items") + expect(text(schemaOnly)).not.toContain("apple | ") + const schemaMiss = await call("sql_execute", { warehouse: "source", tables: ["no_such_table"] }) + expect(schemaMiss.isError).toBe(true) + const missing = await call("sql_execute", { warehouse: "source", tables: ["no_such_table"], query: "SELECT 1 AS one" }) + expect(text(missing)).toContain("no_such_table") + expect(text(missing)).toContain("one") + // Unadvertised tools are not callable. + const hidden = await call("sql_check", { warehouse: "source", check: "nulls", sql: "SELECT * FROM items", columns: ["id"] }) + expect(hidden.isError).toBe(true) +}) + +mcpTest("allTools lists exactly seven tools and executes warehouse, schema, SQL and checks without routing notices", async () => { + const { client, call } = await fixture() + const listed = await client.listTools() + expect(listed.tools.map((t) => t.name).sort()).toEqual([ + "answer_commit", + "schema_inspect", + "sql_check", + "sql_execute", + "sql_materialize", + "warehouse_add", + "warehouse_list", + ]) + expect(listed.tools.every((t) => t.inputSchema.type === "object")).toBe(true) + expect(JSON.stringify(listed.tools)).not.toMatch(/char_offset|lineage_check|tool_lookup|Task tool/) + for (const [name, args, expected] of [ + ["warehouse_list", {}, "source"], + ["schema_inspect", { warehouse: "source", table: "items", values: false }, "name"], + ["sql_execute", { warehouse: "source", query: "SELECT * FROM items" }, "apple"], + ["sql_check", { warehouse: "source", check: "nulls", sql: "SELECT * FROM items", columns: ["id"] }, "null"], + ] as const) { + const result = await call(name, args) + expect(result.isError).toBe(false) + expect(text(result)).toContain(expected) + expect(text(result)).not.toMatch(/lineage_check|tool_lookup|char_offset|Task tool/) + expect(text(result)).not.toContain("Not routed through the bound workspace") + } +}) + +// altimate_change start — warehouse discovery accepts strict-schema empty table fillers +mcpTest("schema_inspect lists schemas, then SQLite tables without scanning for row counts", async () => { + const { call } = await fixture() + for (const [index, args] of [ + { warehouse: "source" }, + { warehouse: "source", table: "", schema_name: "", values: false }, + ].entries()) { + const result = await call("schema_inspect", args) + expect(result.isError).toBe(false) + // altimate_change start — discovery starts with schema counts + expect(body(result)).toContain("main | 1") + expect(body(result)).not.toContain("main | items") + const tables = await call("schema_inspect", { warehouse: "source", schema_name: "main" }) + expect(tables.isError).toBe(false) + // SQLite has no catalog row statistic, so the Rows cell stays empty rather than counting. + expect(body(tables)).toMatch(/^main \| items \| table \| *$/m) + // altimate_change end + // The usage hint is shown once per session; the table rule line is gone over MCP. + if (index === 0) expect(body(result)).toContain("Inspect one table with schema_inspect") + else expect(body(result)).not.toContain("Inspect one table with schema_inspect") + expect(body(result)).not.toContain("-------|") + expect(body(tables)).not.toContain("-------|") + } +}) +// altimate_change end + +mcpTest("source writes are denied by default and cannot modify the source", async () => { + const { call } = await fixture() + const denied = await call("sql_execute", { warehouse: "source", query: "INSERT INTO items VALUES (3, 'plum')" }) + expect(denied.isError).toBe(true) + expect(text(denied)).toContain("--allow-write") + const rows = await call("sql_execute", { warehouse: "source", query: "SELECT COUNT(*) AS total FROM items" }) + expect(rows.isError).toBe(false) + expect(text(rows)).toContain("2") +}) + +mcpTest("allow-write permits SQL mutations", async () => { + const { call } = await fixture(true) + expect( + (await call("sql_execute", { warehouse: "source", query: "INSERT INTO items VALUES (3, 'plum')" })).isError, + ).toBe(false) + expect( + text(await call("sql_execute", { warehouse: "source", query: "SELECT name FROM items WHERE id=3" })), + ).toContain("plum") +}) + +mcpTest("materializes read-only sources into scratch, refuses source mutations", async () => { + const { call } = await fixture() + const result = await call("sql_materialize", { + source_warehouse: "source", + sql: "SELECT * FROM items", + target_table: "copied", + }) + expect(result.isError).toBe(false) + expect(text(result)).not.toContain("Not routed through the bound workspace") + const rows = await call("sql_execute", { warehouse: "scratch", query: "SELECT * FROM copied" }) + expect(rows.isError).toBe(false) + expect(text(rows)).toContain("apple") + const denied = await call("sql_materialize", { + source_warehouse: "source", + sql: "DELETE FROM items", + target_table: "bad", + }) + expect(denied.isError).toBe(true) +}) + +mcpTest("scratch table writes run without --allow-write; source writes stay refused", async () => { + const { call } = await fixture() + for (const query of ["CREATE TABLE notes (n INTEGER)", "INSERT INTO notes VALUES (41)", "UPDATE notes SET n = n + 1"]) { + const result = await call("sql_execute", { warehouse: "scratch", query }) + expect(result.isError, `${query}: ${text(result)}`).toBe(false) + } + expect(text(await call("sql_execute", { warehouse: "scratch", query: "SELECT n FROM notes" }))).toContain("42") + expect((await call("sql_execute", { warehouse: "source", query: "INSERT INTO items VALUES (3, 'fig')" })).isError).toBe(true) +}) + +mcpTest("a discovered connection without a string type stops neither startup nor warehouse_list", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "mcp-malformed-type-")) + cleanups.push(() => rm(dir, { recursive: true, force: true })) + const db = new Database(path.join(dir, "ok.sqlite")) + db.run("CREATE TABLE items (id INTEGER)") + db.close() + Registry.setConfigs({ missing: {} as never, numeric: { type: 42 } as never, ok: { type: "sqlite", path: path.join(dir, "ok.sqlite") } }) + const server = await createServer({}) + const client = new Client({ name: "test", version: "1" }, { capabilities: {} }) + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair() + await server.connect(serverTransport) + await client.connect(clientTransport) + cleanups.push(async () => { + await client.close() + await server.close() + }) + const listed = await client.callTool({ name: "warehouse_list", arguments: {} }) + expect(listed.isError, text(listed)).toBe(false) + expect(text(listed)).toContain("ok") +}) + +mcpTest("MCP cancellation reaches the active tool abort signal", async () => { + const { client } = await fixture() + const started = Promise.withResolvers() + const stopped = Promise.withResolvers() + const original = Registry.get + const execute: Connector["execute"] = (_sql, _limit, _binds, options) => { + started.resolve(options!.signal!) + return new Promise((_resolve, reject) => + options!.signal!.addEventListener( + "abort", + () => { + stopped.resolve() + reject(new Error("Cancelled")) + }, + { once: true }, + ), + ) + } + const dispatch = spyOn(Registry, "get").mockImplementation(async (name) => + name === "source" ? ({ execute, supportsResultByteLimit: true } as Connector) : original(name), + ) + try { + const abort = new AbortController() + const request = client.callTool( + { name: "sql_execute", arguments: { warehouse: "source", query: "SELECT * FROM items" } }, + undefined, + { signal: abort.signal }, + ) + const rejected = request.then( + () => { + throw new Error("Expected cancellation rejection") + }, + (error) => error, + ) + const signal = await started.promise + abort.abort(new Error("Client cancelled")) + expect(await rejected).toBeInstanceOf(Error) + await stopped.promise + expect(signal.aborted).toBe(true) + } finally { + dispatch.mockRestore() + } +}) + +mcpTest("spill output gives MCP-compatible file-reader instructions", async () => { + const { call } = await fixture() + const result = await call("sql_execute", { + warehouse: "source", + query: "SELECT name FROM items", + full_text_columns: ["name"], + }) + expect(result.isError).toBe(false) + expect(text(result)).toContain("scratch table result_1") + expect(text(result)).not.toContain("char_offset") + expect(text(result)).not.toContain("Task tool") + // When the scratch table cannot be stored, the file stays the documented fallback. + const original = Registry.get + const get = spyOn(Registry, "get").mockImplementation(async (name) => { + if (name === "scratch") throw new Error("scratch unavailable") + return original(name) + }) + try { + const fallback = await call("sql_execute", { warehouse: "source", query: "SELECT name FROM items", full_text_columns: ["name"] }) + expect(text(fallback)).toContain("any file reader") + expect(text(fallback)).not.toContain("scratch table") + } finally { + get.mockRestore() + } +}) + +mcpTest("generic truncation saves output with client-neutral hints", async () => { + const { call } = await fixture() + const listing = spyOn(Registry, "list").mockReturnValue({ + warehouses: Array.from({ length: 2500 }, (_, i) => ({ name: `warehouse_${i}`, type: "sqlite" })), + }) + try { + const result = await call("warehouse_list", {}) + expect(result.isError).toBe(false) + expect(text(result)).toContain("Full output saved to:") + expect(text(result)).toContain("any file reader") + expect(text(result)).not.toContain("Task tool") + expect(text(result)).not.toContain("offset/limit") + } finally { + listing.mockRestore() + } +}) + +mcpTest("server startup and tool listing make no network calls", async () => { + const fetch = spyOn(globalThis, "fetch").mockImplementation( + Object.assign( + () => { + throw new Error("Unexpected network call") + }, + { preconnect: globalThis.fetch.preconnect }, + ), + ) + try { + const { client } = await fixture() + expect((await client.listTools()).tools).toHaveLength(7) + expect(fetch).not.toHaveBeenCalled() + } finally { + fetch.mockRestore() + } +}) + +mcpTest("malformed arguments and unknown tools return MCP errors", async () => { + const { call } = await fixture() + expect((await call("unknown", {})).isError).toBe(true) + expect((await call("sql_execute", { warehouse: "source", query: 42 })).isError).toBe(true) +}) + +// altimate_change end + +// altimate_change start — database protection, disclosures, spill lifecycle +function body(result: Awaited>) { + return (result.content as { text: string }[]).map((part) => part.text).join("\n") +} + +mcpTest("SQLite MCP SELECT runs with database query_only enabled", async () => { + const { call } = await fixture() + const result = await call("sql_execute", { + warehouse: "source", + query: + "SELECT CASE WHEN query_only = 1 THEN 'database protected' ELSE 'unprotected' END AS protection FROM pragma_query_only()", + }) + expect(result.isError).toBe(false) + expect(text(result)).toContain("database protected") +}) + +mcpTest("MCP forces database protection for every SQLite source query path", async () => { + const { call } = await fixture() + const original = Registry.get + const options: unknown[] = [] + const get = spyOn(Registry, "get").mockImplementation(async (name) => { + const connector = await original(name) + if (name !== "source") return connector + return { + ...connector, + execute: (sql, limit, binds, opts) => { + options.push(opts) + return connector.execute(sql, limit, binds, opts) + }, + } as Connector + }) + try { + for (const [tool, args] of [ + ["sql_execute", { warehouse: "source", query: "SELECT * FROM items" }], + [ + "sql_execute", + { warehouse: "source", query_from: { warehouse: "source", sql: "SELECT 'SELECT * FROM items' AS sql" } }, + ], + ["schema_inspect", { warehouse: "source", table: "items", values: true }], + // altimate_change start — inventory counts also enforce database read-only + ["schema_inspect", { warehouse: "source" }], + // altimate_change end + // Discovery's counts and sample rows go through the same protection. + ["warehouse_list", {}], + ["sql_execute", { warehouse: "source", queries: [{ query: "SELECT * FROM items" }, { query: "SELECT 1 AS one" }] }], + ["sql_check", { warehouse: "source", check: "nulls", sql: "SELECT * FROM items", columns: ["id"] }], + ["sql_materialize", { source_warehouse: "source", sql: "SELECT * FROM items", target_table: "protected_copy" }], + ] as const) { + options.length = 0 + const result = await call(tool, args) + expect(result.isError).toBe(false) + expect(options.length).toBeGreaterThan(0) + for (const option of options) expect(option).toMatchObject({ readOnly: true }) + } + } finally { + get.mockRestore() + } +}) + +mcpTest("Postgres MCP sql_execute requests dispatcher read_only", async () => { + const { call } = await fixture() + Registry.setConfigs({ source: { type: "postgres" } }) + const original = Dispatcher.call + const dispatch = spyOn(Dispatcher, "call").mockImplementation(async (method, args, context) => { + if (method === "sql.execute") { + expect(args).toMatchObject({ read_only: true }) + expect(context).toMatchObject({ forceReadOnly: true }) + return { columns: ["value"], rows: [[42]], row_count: 1, truncated: false } as never + } + return original(method, args, context) + }) + try { + const result = await call("sql_execute", { warehouse: "source", query: "SELECT app.mutate_source()" }) + expect(result.isError).toBe(false) + expect(dispatch.mock.calls.some(([method]) => method === "sql.execute")).toBe(true) + } finally { + dispatch.mockRestore() + } +}) + +mcpTest("classification-only connections warn at startup and in warehouse_list", async () => { + const notice = spyOn(console, "error").mockImplementation(() => {}) + Registry.setConfigs({ mysql_source: { type: "mysql" }, pg_source: { type: "postgres" } }) + const server = await createServer() + const client = new Client({ name: "test", version: "1" }) + const [left, right] = InMemoryTransport.createLinkedPair() + try { + expect(notice.mock.calls).toEqual([ + [ + "mysql_source (mysql): read-only is enforced by SQL classification only; stored functions with side effects are not blocked — use a read-only database user", + ], + ]) + await server.connect(right) + await client.connect(left) + const result = await client.callTool({ name: "warehouse_list", arguments: {} }) + expect(text(result)).toContain("mysql_source (mysql): read-only is enforced by SQL classification only") + expect(text(result)).toContain("use a read-only database user") + expect(text(result)).not.toContain("pg_source (postgres):") + } finally { + notice.mockRestore() + await client.close() + await server.close() + } +}) + +mcpTest("private spill directory and its files are removed on server close", async () => { + const { call, server } = await fixture() + const result = await call("sql_execute", { + warehouse: "source", + query: "SELECT name FROM items", + full_text_columns: ["name"], + }) + const saved = body(result).match(/(\/[^\s`()]+\/result-\d+\.csv)/)?.[1] + expect(saved).toBeDefined() + expect((await stat(path.dirname(saved!))).mode & 0o777).toBe(0o700) + expect(await readFile(saved!, "utf8")).toContain("apple") + await server.close() + expect(await Bun.file(saved!).exists()).toBe(false) + await expect(stat(path.dirname(saved!))).rejects.toThrow() +}) + +mcpTest("shared truncation honors configured limits and preserves failure wording", async () => { + const previous = process.env.OPENCODE_CONFIG_CONTENT + process.env.OPENCODE_CONFIG_CONTENT = JSON.stringify({ tool_output: { max_lines: 2, max_bytes: 40 } }) + try { + const { call } = await fixture() + const result = await call("warehouse_list", {}) + expect(text(result)).toContain("Full output saved to:") + expect(text(result)).toContain("succeeded but the output was truncated") + const listing = spyOn(Registry, "list").mockImplementation(() => { + throw new Error("A long warehouse listing failure that exceeds the configured output byte limit") + }) + try { + const failure = await call("warehouse_list", {}) + expect(failure.isError).toBe(true) + expect(text(failure)).toContain("failed but the output was truncated") + } finally { + listing.mockRestore() + } + } finally { + if (previous === undefined) delete process.env.OPENCODE_CONFIG_CONTENT + else process.env.OPENCODE_CONFIG_CONTENT = previous + } +}) +// altimate_change end + +// altimate_change start — SELECT over a macro hides a write from SQL classification +for (const allowWrite of [false, true]) { + mcpTest(`DuckDB hidden function writes reach the database read-only guard (allow-write=${allowWrite})`, async () => { + const { call } = await fixture(allowWrite) + // The server opened the fixture's SQLite connector to list its tables at startup; start from a clean registry. + await Registry.closeAll() + Registry.reset() + Registry.setConfigs({ source: { type: "duckdb", path: ":memory:" } }) + const connector = await Registry.get("source") + await connector.execute("CREATE SEQUENCE source_sequence") + await connector.execute("CREATE MACRO mutate_source() AS nextval('source_sequence')") + const result = await call("sql_execute", { warehouse: "source", query: "SELECT mutate_source() AS id" }) + expect(result.isError).toBe(!allowWrite) + if (!allowWrite) expect(text(result)).toMatch(/read.only/i) + const state = await connector.execute( + "SELECT last_value FROM duckdb_sequences() WHERE sequence_name = 'source_sequence'", + ) + expect(state.rows).toEqual([[allowWrite ? 1n : null]]) + }) +} +// altimate_change end + +// altimate_change start — the explicit production loader owns validation and path resolution +for (const configs of [[], null, { bad: {} }, { bad: { type: 1 } }]) { + mcpTest(`explicit connections loader rejects invalid input ${JSON.stringify(configs)}`, async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "mcp-invalid-config-")) + cleanups.push(() => rm(dir, { recursive: true, force: true })) + const file = path.join(dir, "connections.json") + await Bun.write(file, JSON.stringify(configs)) + await expect(createServer({ connections: file })).rejects.toThrow(/connection/i) + }) +} + +mcpTest("explicit connections loader preserves reserved scratch conflict", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "mcp-scratch-conflict-")) + cleanups.push(() => rm(dir, { recursive: true, force: true })) + const file = path.join(dir, "connections.json") + await Bun.write(file, JSON.stringify({ scratch: { type: "sqlite", path: "scratch.sqlite" } })) + await Registry.loadFile(file) + expect(Registry.getConfig("scratch")?.path).toBe(path.join(dir, "scratch.sqlite")) + expect(Registry.list().warehouses[0]?.conflict).toContain("reserved built-in scratch") +}) +// altimate_change end + +// altimate_change start — closing aborts in-flight work before deleting private output +mcpTest("server close aborts active source queries before cleanup", async () => { + const { client, server } = await fixture() + const started = Promise.withResolvers() + const execute: Connector["execute"] = (_sql, _limit, _binds, options) => { + const signal = options!.signal! + started.resolve(signal) + return new Promise((_resolve, reject) => signal.addEventListener("abort", () => reject(signal.reason), { once: true })) + } + const original = Registry.get + const get = spyOn(Registry, "get").mockImplementation(async (name) => + name === "source" ? ({ execute, supportsResultByteLimit: true } as Connector) : original(name), + ) + try { + const pending = client.callTool({ name: "sql_execute", arguments: { warehouse: "source", query: "SELECT * FROM items" } }) + .catch((error) => error) + const signal = await started.promise + await server.close() + expect(signal.aborted).toBe(true) + await pending + } finally { get.mockRestore() } +}) +// altimate_change end + +// altimate_change start — answer_commit over MCP: requested-item coverage, number provenance, basis trimming +const commit = (fields: { name: string; value: string | number }[], asked: string[], basis = "Counted items.") => ({ + fields, + asked, + basis, +}) + +mcpTest("answer_commit returns the canonical FINAL ANSWER text and trims a long basis", async () => { + const { call } = await fixture() + const result = await call("answer_commit", commit([{ name: "", value: "apple" }], ["item"], "x".repeat(900))) + expect(result.isError).toBeFalsy() + const out = (result.content as { text: string }[])[0]!.text + // altimate_change start — the FINAL ANSWER line is answer-only; the trimmed basis sits above it + expect(out).toMatch(/(^|\n\n)FINAL ANSWER: /) + const answer = out.slice(out.indexOf("FINAL ANSWER: ")) + expect(answer).toContain("apple") + expect(answer).not.toContain("xxx") + // altimate_change end +}) + +mcpTest("answer_commit rejects a missing asked item at most twice, then accepts", async () => { + const { call } = await fixture() + const args = commit([{ name: "item", value: "apple" }], ["item", "count"]) + for (let i = 0; i < 2; i++) { + const rejected = await call("answer_commit", args) + expect(rejected.isError).toBe(true) + expect(text(rejected)).toContain("requested-items check") + } + expect((await call("answer_commit", args)).isError).toBeFalsy() +}) + +// The server cannot see numbers the client computed in its own shell or code, so over MCP a number +// absent from this server's query output is accepted on the first commit (native sessions still check). +mcpTest("answer_commit accepts a client-computed number absent from query output on the first commit", async () => { + const { call } = await fixture() + await call("sql_execute", { warehouse: "source", query: "SELECT COUNT(*) AS n FROM items" }) + const accepted = await call("answer_commit", commit([{ name: "total", value: 98765 }], ["total"])) + expect(accepted.isError).toBeFalsy() + expect(text(accepted)).not.toContain("number check") + expect(text(accepted)).toContain("98765") +}) + +mcpTest("answer_commit over MCP still runs the requested-items check before accepting a computed number", async () => { + const { call } = await fixture() + await call("sql_execute", { warehouse: "source", query: "SELECT COUNT(*) AS n FROM items" }) + const rejected = await call("answer_commit", commit([{ name: "total", value: 98765 }], ["total", "item"])) + expect(rejected.isError).toBe(true) + expect(text(rejected)).toContain("requested-items check") +}) + +mcpTest("answer_commit accepts a number that appeared in query output", async () => { + const { call } = await fixture() + await call("sql_execute", { warehouse: "source", query: "SELECT 4321 AS n" }) + expect((await call("answer_commit", commit([{ name: "n", value: 4321 }], ["n"]))).isError).toBeFalsy() +}) +// altimate_change end + +// altimate_change start — conjunction check over MCP: the request arrives as an MCP-only argument +mcpTest("answer_commit exposes a required request argument over MCP", async () => { + const { client } = await fixture() + const tool = (await client.listTools()).tools.find((item) => item.name === "answer_commit") + expect(Object.keys(tool!.inputSchema.properties ?? {})).toContain("request") + expect(tool!.inputSchema.required).toContain("request") +}) + +mcpTest("answer_commit rejects an under-asked commit on a compound request once, then accepts", async () => { + const { call } = await fixture() + const args = { ...commit([{ name: "item", value: "apple" }], ["item"]), request: "Which item and its id?" } + const rejected = await call("answer_commit", args) + expect(rejected.isError).toBe(true) + expect(text(rejected)).toContain("answer_commit conjunction check:") + const accepted = await call("answer_commit", args) + expect(accepted.isError).toBeFalsy() + // An under-asked commit keeps the basis on the FINAL ANSWER line. + const out = (accepted.content as { text: string }[])[0]!.text + expect(out.startsWith("FINAL ANSWER: Counted items.")).toBe(true) + expect(out).toContain("**Answer:**") + expect(out).toContain("apple") +}) + +mcpTest("answer_commit accepts a single-item request and a commit without request", async () => { + const { call } = await fixture() + const single = { ...commit([{ name: "item", value: "apple" }], ["item"]), request: "Which item sorts first?" } + expect((await call("answer_commit", single)).isError).toBeFalsy() + expect((await call("answer_commit", commit([{ name: "item", value: "apple" }], ["item"]))).isError).toBeFalsy() +}) + +mcpTest("answer_commit accepts a compound request committed with both items", async () => { + const { call } = await fixture() + const both = { + ...commit( + [ + { name: "item", value: "apple" }, + { name: "id", value: "one" }, + ], + ["item", "id"], + ), + request: "Which item and its id?", + } + expect((await call("answer_commit", both)).isError).toBeFalsy() +}) +// altimate_change end + +// altimate_change start — spilled sql_execute results carry a resource_link beside the text fallback +mcpTest("sql_execute spill returns text with the path plus a resource_link to the saved rows", async () => { + const { call } = await fixture() + const result = await call("sql_execute", { + warehouse: "source", + query: "SELECT id, name FROM items", + full_text_columns: ["name"], + }) + const content = result.content as { type: string; text?: string; uri?: string }[] + const link = content.find((item) => item.type === "resource_link") + expect(link?.uri?.startsWith("file://")).toBe(true) + expect(content[0]!.type).toBe("text") + expect(content[0]!.text).toContain(`CSV copy: ${fileURLToPath(link!.uri!)}`) +}) + +mcpTest("sql_execute without a spill returns text only", async () => { + const { call } = await fixture() + const result = await call("sql_execute", { warehouse: "source", query: "SELECT id FROM items" }) + expect((result.content as { type: string }[]).map((item) => item.type)).toEqual(["text"]) +}) +// altimate_change end + +// altimate_change start — answer_commit returns only the FINAL ANSWER text: Claude Code shows structuredContent in place of the text +mcpTest("answer_commit returns the FINAL ANSWER text without structured output", async () => { + const { client, call } = await fixture() + const tool = (await client.listTools()).tools.find((item) => item.name === "answer_commit") + expect(tool?.outputSchema).toBeUndefined() + const result = await call("answer_commit", commit([{ name: "", value: "apple" }], ["item"])) + expect(result.structuredContent).toBeUndefined() + expect((result.content as { text: string }[])[0]!.text).toMatch(/(^|\n\n)FINAL ANSWER: /) +}) +// altimate_change end + +// altimate_change start — the analyze prompt carries the analyst method for interactive clients +mcpTest("analyze prompt returns the analyst method and the question", async () => { + const { client } = await fixture() + expect((await client.listPrompts()).prompts.map((item) => item.name)).toEqual(["analyze"]) + const prompt = await client.getPrompt({ name: "analyze", arguments: { question: "How many items?" } }) + const body = (prompt.messages[0]!.content as { text: string }).text + expect(body).toContain(SERVER_INSTRUCTIONS) + expect(body).toContain("Question: How many items?") + await expect(client.getPrompt({ name: "nope" })).rejects.toThrow() +}) +// altimate_change end + +// altimate_change start — answer_commit asks which reading was meant only when two exist and the client can ask +const twoReadings = { + ...commit([{ name: "", value: "apple" }], ["item"]), + readings: [ + { clause: "c", reading: "by name", result_summary: "apple", decided_by: "x" }, + { clause: "c", reading: "by id", result_summary: "pear", decided_by: "y" }, + ], +} + +mcpTest("answer_commit elicits a reading and asks for a recompute when another is chosen", async () => { + const { client, call } = await fixture(false, { elicitation: {} }) + let asked = 0 + client.setRequestHandler(ElicitRequestSchema, async () => { + asked++ + return { action: "accept", content: { reading: "by id" } } + }) + const first = await call("answer_commit", twoReadings) + expect(first.isError).toBe(true) + expect(text(first)).toContain("by id") + // Asked once per session: the follow-up commit is accepted. + expect((await call("answer_commit", twoReadings)).isError).toBeFalsy() + expect(asked).toBe(1) +}) + +mcpTest("answer_commit degrades silently when the client cancels or lacks elicitation", async () => { + const cancelled = await fixture(false, { elicitation: {} }) + cancelled.client.setRequestHandler(ElicitRequestSchema, async () => ({ action: "cancel" })) + expect((await cancelled.call("answer_commit", twoReadings)).isError).toBeFalsy() +}) + +mcpTest("answer_commit accepts the commit when the client never answers the reading question", async () => { + const silent = await fixture(false, { elicitation: {} }, 200) + silent.client.setRequestHandler(ElicitRequestSchema, () => new Promise(() => {})) + const started = Date.now() + const result = await silent.call("answer_commit", twoReadings) + expect(result.isError).toBeFalsy() + expect(Date.now() - started).toBeLessThan(5_000) +}) + +mcpTest("answer_commit without elicitation support or with one reading never asks", async () => { + const plain = await fixture() + expect((await plain.call("answer_commit", twoReadings)).isError).toBeFalsy() +}) +// altimate_change end + +// altimate_change start — batched sql_execute: one call, one labeled block per independent query +mcpTest("sql_execute queries runs every item and keeps going past a failing one", async () => { + const { call } = await fixture() + const result = await call("sql_execute", { + warehouse: "source", + queries: [ + { label: "count", query: "SELECT COUNT(*) AS n FROM items" }, + { label: "broken", query: "SELECT nope FROM missing_table" }, + { query: "SELECT name FROM items WHERE id = 2" }, + ], + }) + expect(result.isError).toBe(false) + const out = body(result) + const count = out.indexOf("[count]\n") + const broken = out.indexOf("[broken]\n") + const third = out.indexOf("[q3]\n") + expect(count).toBe(0) + expect(broken).toBeGreaterThan(count) + expect(third).toBeGreaterThan(broken) + expect(out.slice(count, broken)).toContain("2") + expect(out.slice(broken, third)).toContain("Failed to execute SQL") + expect(out.slice(third)).toContain("pear") +}) + +mcpTest("sql_execute queries refuses writes per item, even with --allow-write", async () => { + const { call } = await fixture(true) + const result = await call("sql_execute", { + warehouse: "source", + queries: [{ query: "INSERT INTO items VALUES (3, 'plum')" }, { query: "SELECT COUNT(*) AS n FROM items" }], + }) + expect(result.isError).toBe(false) + expect(body(result)).toContain("Batched queries must be read-only") + const rows = await call("sql_execute", { warehouse: "source", query: "SELECT COUNT(*) AS n FROM items" }) + expect(body(rows)).toMatch(/\b2\b/) + expect(body(rows)).not.toContain("plum") +}) + +mcpTest("sql_execute queries is an error only when every item fails", async () => { + const { call } = await fixture() + const result = await call("sql_execute", { + warehouse: "source", + queries: [{ query: "SELECT a FROM nowhere" }, { query: "SELECT b FROM nowhere_else" }], + }) + expect(result.isError).toBe(true) + expect(body(result)).toContain("[q1]") + expect(body(result)).toContain("[q2]") +}) + +mcpTest("sql_execute queries is exclusive with query and capped at eight items", async () => { + const { call } = await fixture() + const both = await call("sql_execute", { + warehouse: "source", + query: "SELECT 1", + queries: [{ query: "SELECT 2" }], + }) + expect(both.isError).toBe(true) + expect(body(both)).toContain("not both") + const many = await call("sql_execute", { + warehouse: "source", + queries: Array.from({ length: 9 }, (_, i) => ({ query: `SELECT ${i}` })), + }) + expect(many.isError).toBe(true) + expect(body(many)).toContain("at most 8") +}) + +mcpTest("sql_execute queries keeps the whole envelope inside one shared budget", async () => { + const { call } = await fixture(false, {}, undefined, (db) => { + db.run("CREATE TABLE wide (id INTEGER, payload TEXT)") + const insert = db.prepare("INSERT INTO wide VALUES (?, ?)") + for (let i = 0; i < 400; i++) insert.run(i, `${"é".repeat(60)}-${i}`) + }) + const label = "L".repeat(500) + const result = await call("sql_execute", { + warehouse: "source", + limit: 400, + queries: [ + { label, query: "SELECT * FROM wide" }, + { query: "SELECT * FROM wide ORDER BY id DESC" }, + { query: `SELECT '${"x".repeat(30_000)}' AS huge FROM nowhere` }, + ], + }) + const out = body(result) + expect(Buffer.byteLength((result.content as { type: string; text?: string }[])[0]!.text!, "utf8")).toBeLessThanOrEqual( + 24 * 1024, + ) + expect(out).toContain(`[${"L".repeat(60)}]`) + expect(out).not.toContain("L".repeat(61)) + expect(out).toContain("[q2]") + expect(out).toContain("[q3]") + // Both large results were saved; each gets its own resource link. + const links = (result.content as { type: string; uri?: string }[]).filter((item) => item.type === "resource_link") + expect(links).toHaveLength(2) + expect(new Set(links.map((link) => link.uri)).size).toBe(2) +}) +// altimate_change end + +// altimate_change start — spilled results become a scratch table to query, with a short preview +const spillRows = (db: Database) => { + db.run("CREATE TABLE events (id INTEGER, code TEXT, note TEXT, ratio REAL)") + const insert = db.prepare("INSERT INTO events VALUES (?, ?, ?, ?)") + for (let i = 1; i <= 200; i++) { + insert.run(i, `00${i}`, i % 3 === 0 ? null : i % 3 === 1 ? "" : `a,"b"\nc ${"x".repeat(300)}`, i / 8) + } +} + +mcpTest("a spilled result is queryable as a scratch table with exact values", async () => { + const { call } = await fixture(false, {}, undefined, spillRows) + const result = await call("sql_execute", { warehouse: "source", query: "SELECT * FROM events ORDER BY id", limit: 500 }) + expect(result.isError).toBe(false) + const out = body(result) + expect(out).toContain("Preview: 10 of 200 returned rows.") + expect(out).toContain('scratch table result_1 (typed like sql_materialize): query it with sql_execute(warehouse: "scratch"') + // Cells in the preview are clipped; the table keeps them whole. + expect(out).toContain("[cell clipped") + expect(out).not.toContain("x".repeat(300)) + const check = await call("sql_execute", { + warehouse: "scratch", + query: + "SELECT COUNT(*) AS n, SUM(id) AS total, COUNT(note) AS non_null, SUM(CASE WHEN note = '' THEN 1 ELSE 0 END) AS empty, " + + "MIN(code) AS first_code, MAX(length(note)) AS longest, SUM(ratio) AS ratios, typeof(ANY_VALUE(code)) AS code_type FROM result_1", + }) + // Text codes keep their leading zeros, NULL stays distinct from '', and long cells are whole. + expect(body(check)).toContain("200 | 20100 | 134 | 67 | 001 | 308 | 2512.5 | VARCHAR") + const longest = await call("sql_execute", { + warehouse: "scratch", + query: "SELECT note FROM result_1 WHERE id = 2", + }) + expect(body(longest)).toContain('a,\\"b\\"\\nc ') +}) + +mcpTest("spill tables are numbered per session and skip a name the client already used", async () => { + const { call } = await fixture(false, {}, undefined, spillRows) + const created = await call("sql_materialize", { + source_warehouse: "source", + sql: "SELECT 1 AS taken", + target_table: "result_2", + }) + expect(created.isError).toBe(false) + const first = body(await call("sql_execute", { warehouse: "source", query: "SELECT * FROM events", limit: 500 })) + const second = body(await call("sql_execute", { warehouse: "source", query: "SELECT id FROM events", limit: 500 })) + expect(first).toContain("scratch table result_1") + expect(second).toContain("scratch table result_3") + expect(body(await call("sql_execute", { warehouse: "scratch", query: "SELECT taken FROM result_2" }))).toContain("1") +}) +// altimate_change end + +// altimate_change start — compact tool definitions: re-sent with every model request +mcpTest("tool definitions stay compact and keep their required properties", async () => { + const { client } = await fixture() + const { tools } = await client.listTools() + const size = (tool: (typeof tools)[number]) => + JSON.stringify({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema }).length + const byName = Object.fromEntries(tools.map((tool) => [tool.name, tool])) + expect(size(byName.answer_commit!)).toBeLessThanOrEqual(2_250) + expect(byName.answer_commit!.description).toContain("'the repository and the most copied file' needs two") + expect(tools.reduce((total, tool) => total + size(tool), 0)).toBeLessThan(7_000) + expect(client.getInstructions()!.length).toBeLessThanOrEqual(723) + const declared = JSON.stringify(tools) + expect(declared).not.toContain("$schema") + expect(declared).not.toContain('"const":""') + expect(byName.answer_commit!.inputSchema.required).toEqual(["basis", "request"]) + expect(byName.sql_check!.inputSchema.required).toEqual(["warehouse", "check"]) + expect(byName.sql_materialize!.inputSchema.required).toEqual(["source_warehouse", "sql", "target_table"]) + // A nullable table cell keeps its null member; only optional-property fillers collapse. + expect(JSON.stringify(byName.answer_commit!.inputSchema.properties!.rows)).toContain('{"type":"null"}') +}) + +mcpTest("strict-schema filler values are still accepted after the schema is compacted", async () => { + const { call } = await fixture() + const executed = await call("sql_execute", { + warehouse: "source", + query: "SELECT name FROM items WHERE id = 1", + queries: null, + query_from: null, + full_text_columns: [], + }) + expect(executed.isError).toBe(false) + expect(body(executed)).toContain("apple") + const committed = await call("answer_commit", { + fields: [{ name: "", value: "apple", units: "", precision: null }], + rows: null, + unresolved: {}, + asked: [], + readings: [], + answer_only: "", + basis: "Read items.name for id 1.", + request: "Which item has id 1?", + }) + expect(committed.isError).toBeFalsy() + expect(body(committed)).toContain("FINAL ANSWER: apple") +}) +// altimate_change end + +// altimate_change start — warehouse_list discovery: local tables with columns and a sample in one call +mcpTest("warehouse_list lists local tables with columns and one sample row", async () => { + const { call } = await fixture(false, {}, undefined, (db) => { + db.run("CREATE VIEW item_names AS SELECT name FROM items") + }) + const out = body(await call("warehouse_list", {})) + expect(out).toContain("source: main.items: id INTEGER, name TEXT") + expect(out).toMatch(/source: main\.items[^\n]*\n {2}e\.g\. 1 \| apple/) + // Views are listed but never sampled: a view may run arbitrary work. + expect(out).toMatch(/source: main\.item_names: name/) + expect(out).not.toMatch(/item_names[^\n]*\n {2}e\.g\./) +}) + +mcpTest("warehouse_list runs no discovery queries on remote connections", async () => { + const { call } = await fixture() + Registry.setConfigs({ remote: { type: "postgres", host: "db.example.com", database: "app" } }) + const original = Dispatcher.call + const methods: string[] = [] + const dispatch = spyOn(Dispatcher, "call").mockImplementation(async (method, args, context) => { + methods.push(method) + return original(method, args, context) + }) + const get = spyOn(Registry, "get") + try { + const out = body(await call("warehouse_list", {})) + expect(out).toContain("remote | postgres") + expect(out).not.toContain("remote:") + expect(methods.filter((method) => method !== "warehouse.list")).toEqual([]) + expect(get).not.toHaveBeenCalled() + } finally { + dispatch.mockRestore() + get.mockRestore() + } +}) + +mcpTest("warehouse_list skips table detail for a catalog over the table budget", async () => { + const { call } = await fixture(false, {}, undefined, (db) => { + for (let i = 0; i < 45; i++) db.run(`CREATE TABLE t${i} (id INTEGER)`) + }) + const out = body(await call("warehouse_list", {})) + expect(out).toContain("source: catalog too large to list here; use schema_inspect.") + expect(out).not.toContain("main.t1") +}) + +mcpTest("warehouse_list discovery stops starting calls once its deadline passes", async () => { + const { call } = await fixture(false, {}, undefined, (db) => { + for (let i = 0; i < 6; i++) db.run(`CREATE TABLE slow${i} (id INTEGER)`) + }) + const original = Dispatcher.call + let inspections = 0 + const dispatch = spyOn(Dispatcher, "call").mockImplementation(async (method, args, context) => { + if (method === "schema.inspect") { + inspections++ + await Bun.sleep(1_500) + } + return original(method, args, context) + }) + try { + const started = Date.now() + const out = body(await call("warehouse_list", {})) + expect(Date.now() - started).toBeLessThan(7_000) + expect(inspections).toBeLessThanOrEqual(3) + expect(out).toContain("More tables or connections exist than shown") + } finally { + dispatch.mockRestore() + } +}, 15_000) +// altimate_change end + +// altimate_change start — compact MCP output: CSV spills, no rule lines, hints once, short receipts +mcpTest("an MCP spill file is CSV with the column names once and NULL distinct from an empty string", async () => { + const { call } = await fixture(false, {}, undefined, spillRows) + const result = await call("sql_execute", { + warehouse: "source", + query: "SELECT id, code, note FROM events WHERE id <= 3 ORDER BY id", + full_text_columns: ["note"], + }) + const file = body(result).match(/(\/[^\s`()]+\/result-1\.csv)/)?.[1] + expect(file).toBeDefined() + const csv = await readFile(file!, "utf8") + const lines = csv.split("\n") + expect(lines[0]).toBe("id,code,note") + expect(lines[1]).toBe('1,001,""') + expect(lines[2]).toBe(`2,002,"a,""b""`) + expect(csv).toContain(`\n3,003,\n`) + expect(csv.match(/id,code,note/g)).toHaveLength(1) +}) + +mcpTest("MCP inline results, listings and receipts drop the rule line and repeat no tutorial", async () => { + const { call } = await fixture(false, {}, undefined, spillRows) + const inline = body(await call("sql_execute", { warehouse: "source", query: "SELECT id, name FROM items ORDER BY id" })) + expect(inline).toBe("id | name\n1 | apple\n2 | pear\n\n(2 rows)") + expect(body(await call("warehouse_list", {}))).not.toMatch(/^-+\|/m) + const first = body(await call("sql_execute", { warehouse: "source", query: "SELECT * FROM events", limit: 500 })) + const second = body(await call("sql_execute", { warehouse: "source", query: "SELECT id FROM events", limit: 500 })) + expect(first).toContain("instead of reading the file") + expect(second).not.toContain("instead of reading the file") + expect(second).toContain("All 200 returned rows are in scratch table result_2 (CSV copy:") + const sql = "SELECT id, code FROM events WHERE id < 5" + const made = body(await call("sql_materialize", { source_warehouse: "source", sql, target_table: "small" })) + expect(made).toStartWith("scratch.small: 4 rows, 2 columns (create) from source.\nColumns: id BIGINT, code VARCHAR") + expect(made).not.toContain(sql) + expect(made).toContain("Join it in scratch") + const again = body( + await call("sql_materialize", { source_warehouse: "source", sql, target_table: "small", mode: "replace" }), + ) + expect(again).not.toContain("Join it in scratch") +}) +// altimate_change end + +// altimate_change start — setup from zero: no connections file, then a file or warehouse_add +async function emptyProject(env: Record = {}) { + const project = await mkdtemp(path.join(os.tmpdir(), "mcp-empty-project-")) + const previous = process.cwd() + const saved: Record = {} + // Ambient connection variables would make the empty-registry assertions depend on the caller's shell. + for (const key of Object.keys(process.env)) { + if (key.startsWith(Registry.CONN_ENV_PREFIX) || key in env) saved[key] = process.env[key] + if (key.startsWith(Registry.CONN_ENV_PREFIX)) delete process.env[key] + } + for (const [key, value] of Object.entries(env)) { + saved[key] ??= process.env[key] + process.env[key] = value + } + process.chdir(project) + Registry.reset() + const db = new Database(path.join(project, "store.sqlite")) + db.run("CREATE TABLE fruit (id INTEGER, name TEXT)") + db.run("INSERT INTO fruit VALUES (1, 'fig')") + db.close() + const server = await createServer({}) + const client = new Client({ name: "test", version: "1" }) + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair() + await server.connect(serverTransport) + await client.connect(clientTransport) + cleanups.push(async () => { + await client.close() + await server.close() + await Registry.closeAll() + process.chdir(previous) + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value + } + // warehouse_add writes the (test-isolated) global file; later tests must not see it. + await rm(Registry.globalConfigPath(), { force: true }) + await rm(project, { recursive: true, force: true }) + }) + const call = async (name: string, args: Record) => { + const result = await client.callTool({ name, arguments: args }) + return { result, body: (result.content as { text?: string }[]).map((part) => part.text ?? "").join("\n") } + } + return { project, call } +} + +mcpTest("an empty registry names the real config paths and sees a connections file written afterwards", async () => { + const { project, call } = await emptyProject() + const first = await call("warehouse_list", {}) + expect(first.body).toContain("No warehouse connections configured.") + expect(first.body).toContain(Registry.localConfigPath()) + expect(first.body).toContain(Registry.globalConfigPath()) + expect(first.body).toContain(Registry.CONN_ENV_PREFIX) + expect(first.body).not.toContain(".opencode/connections.json") + await Bun.write( + path.join(project, ".altimate-code", "connections.json"), + JSON.stringify({ local: { type: "sqlite", path: path.join(project, "store.sqlite") } }), + ) + const second = await call("warehouse_list", {}) + expect(second.body).toContain("local | sqlite") + const rows = await call("sql_execute", { warehouse: "local", query: "SELECT name FROM fruit" }) + expect(rows.result.isError).toBe(false) + expect(rows.body).toContain("fig") +}) + +mcpTest("warehouse_add from an empty registry makes the connection queryable", async () => { + const { project, call } = await emptyProject() + const added = await call("warehouse_add", { + name: "added", + config: { type: "sqlite", path: path.join(project, "store.sqlite") }, + }) + expect(added.result.isError).toBe(false) + expect(added.body).toContain("Call warehouse_list") + expect(added.body).not.toMatch(/warehouse_test|sql_analyze|lineage_check/) + expect((await call("warehouse_list", {})).body).toContain("added | sqlite") + // A writable store keeps its configured mode. + expect(Registry.getConfig("added")?.readonly).toBeUndefined() + const rows = await call("sql_execute", { warehouse: "added", query: "SELECT name FROM fruit" }) + expect(rows.result.isError).toBe(false) + expect(rows.body).toContain("fig") +}) + +mcpTest("an empty registry lists detected connection candidates by type and source, never their values", async () => { + const { call } = await emptyProject({ SNOWFLAKE_ACCOUNT: "acct-value-xyz", SNOWFLAKE_PASSWORD: "secret-value-xyz" }) + const { body } = await call("warehouse_list", {}) + expect(body).toContain("snowflake from environment variable SNOWFLAKE_ACCOUNT") + expect(body).not.toContain("acct-value-xyz") + expect(body).not.toContain("secret-value-xyz") +}) +// altimate_change end + +// altimate_change start — read-only MCP sessions cannot create a store through warehouse_add +mcpTest("warehouse_add refuses create:true in a read-only session and creates no file", async () => { + const { project, call } = await emptyProject() + const target = path.join(project, "new.duckdb") + const added = await call("warehouse_add", { name: "fresh", config: { type: "duckdb", path: target, create: true } }) + expect(added.body).toContain("read-only") + expect((await call("warehouse_list", {})).body).not.toContain("fresh") + expect(await Bun.file(target).exists()).toBe(false) +}) +// altimate_change end + +// altimate_change start — a read-only session opens added file stores read-only (read-only files and directories work) +mcpTest("warehouse_add of a SQLite file this process cannot write opens it read-only so it can be queried", async () => { + const { project, call } = await emptyProject() + const locked = path.join(project, "locked") + await mkdir(locked) + const store = path.join(locked, "ro.sqlite") + const db = new Database(store) + db.run("CREATE TABLE fruit (id INTEGER, name TEXT)") + db.run("INSERT INTO fruit VALUES (1, 'quince')") + db.close() + await chmod(store, 0o444) + await chmod(locked, 0o555) + cleanups.push(() => chmod(locked, 0o755)) + const added = await call("warehouse_add", { name: "ro", config: { type: "sqlite", path: store } }) + expect(added.result.isError).toBe(false) + const rows = await call("sql_execute", { warehouse: "ro", tables: ["fruit"], query: "SELECT name FROM fruit" }) + expect(rows.body).not.toContain("readonly database") + expect(rows.result.isError).toBe(false) + expect(rows.body).toContain("quince") +}) +// altimate_change end + +// altimate_change start — chained batch steps: {{label.column}} value lists and {{label}} scratch tables +async function chainFixture() { + const dir = await mkdtemp(path.join(os.tmpdir(), "mcp-chain-test-")) + cleanups.push(() => rm(dir, { recursive: true, force: true })) + const a = new Database(path.join(dir, "a.sqlite")) + a.run("CREATE TABLE items (id INTEGER, name TEXT)") + a.run("INSERT INTO items VALUES (1, 'apple'), (2, 'pear'), (3, 'o''brien')") + a.run("CREATE TABLE big (id INTEGER)") + a.run(`INSERT INTO big VALUES ${Array.from({ length: 1500 }, (_, i) => `(${i})`).join(",")}`) + a.close() + const b = new Database(path.join(dir, "b.sqlite")) + b.run("CREATE TABLE prices (item_id INTEGER, label TEXT, price REAL)") + b.run("INSERT INTO prices VALUES (1, 'apple', 3.5), (2, 'pear', 1.25), (3, 'o''brien', 9.75)") + b.close() + const config = path.join(dir, "connections.json") + await Bun.write(config, JSON.stringify({ a: { type: "sqlite", path: "a.sqlite" }, b: { type: "sqlite", path: "b.sqlite" } })) + const server = await createServer({ connections: config }) + const client = new Client({ name: "test", version: "1" }) + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair() + await server.connect(serverTransport) + await client.connect(clientTransport) + cleanups.push(async () => { + await client.close() + await server.close() + await Registry.closeAll() + }) + return async (queries: Record[], warehouse = "a") => { + const result = await client.callTool({ name: "sql_execute", arguments: { warehouse, queries } }) + return { result, body: (result.content as { text?: string }[]).map((part) => part.text ?? "").join("\n") } + } +} + +function block(body: string, label: string): string { + const start = body.indexOf(`[${label}]`) + if (start < 0) return "" + const next = body.indexOf("\n\n[", start + 1) + return body.slice(start, next < 0 ? undefined : next) +} + +mcpTest("a later batch step filters on an earlier step's column on the same connection", async () => { + const batch = await chainFixture() + const { result, body } = await batch([ + { label: "pick", query: "SELECT id FROM items WHERE name IN ('apple', 'o''brien')" }, + { label: "names", query: "SELECT name FROM items WHERE id IN ({{pick.id}}) ORDER BY id" }, + ]) + expect(result.isError).toBe(false) + expect(block(body, "names")).toContain("apple") + expect(block(body, "names")).toContain("o'brien") + expect(block(body, "names")).not.toContain("pear") + // The values also fit a DuckDB list, and an empty earlier result matches nothing instead of failing. + const more = await batch([ + { label: "pick", warehouse: "a", query: "SELECT id FROM items WHERE name = 'pear'" }, + { label: "none", warehouse: "a", query: "SELECT id FROM items WHERE name = 'kiwi'" }, + { label: "list", warehouse: "scratch", query: "SELECT list_contains([{{pick.id}}], 2) AS has_pear" }, + { label: "empty", warehouse: "a", query: "SELECT count(*) AS n FROM items WHERE id IN ({{none.id}})" }, + ]) + expect(block(more.body, "list")).toContain("true") + expect(block(more.body, "empty")).not.toContain("Failed") + expect(block(more.body, "empty")).toMatch(/\b0\b/) +}) + +mcpTest("a batch step feeds another connection's IN list, and a scratch step joins stored step rows", async () => { + const batch = await chainFixture() + const { result, body } = await batch([ + { label: "ids", warehouse: "a", query: "SELECT id, name FROM items WHERE name <> 'pear'" }, + { label: "price", warehouse: "b", query: "SELECT item_id, label, price FROM prices WHERE label IN ({{ids.name}})" }, + { + label: "joined", + warehouse: "scratch", + query: "SELECT i.name, p.price FROM {{ids}} i JOIN {{price}} p ON p.item_id = i.id ORDER BY p.price DESC", + }, + ]) + expect(result.isError).toBe(false) + expect(block(body, "price")).toContain("3.5") + expect(block(body, "price")).not.toContain("1.25") + expect(block(body, "joined")).toContain("9.75") + expect(block(body, "joined")).toContain("o'brien") + // A table reference outside scratch fails only that step. + const outside = await batch([ + { label: "ids", query: "SELECT id FROM items" }, + { label: "bad", query: "SELECT * FROM {{ids}}" }, + ]) + expect(block(outside.body, "bad")).toContain('warehouse "scratch"') +}) + +mcpTest("a value list from a step with more than the bound row cap fails that step only", async () => { + const batch = await chainFixture() + const { body } = await batch([ + { label: "all", query: "SELECT id FROM big" }, + { label: "use", query: "SELECT count(*) AS n FROM big WHERE id IN ({{all.id}})" }, + { label: "few", query: "SELECT id FROM big WHERE id < 3" }, + { label: "small", query: "SELECT count(*) AS n FROM big WHERE id IN ({{few.id}})" }, + ]) + expect(block(body, "use")).toContain("more than 1000 rows") + expect(block(body, "small")).toMatch(/\b3\b/) +}) + +mcpTest("a failing batch step fails its dependents and the remaining steps still run", async () => { + const batch = await chainFixture() + const { result, body } = await batch([ + { label: "bad", query: "SELEC id FROM items" }, + { label: "dep", query: "SELECT * FROM items WHERE id IN ({{bad.id}})" }, + { label: "missing", query: "SELECT * FROM items WHERE id IN ({{nope.id}})" }, + { label: "ok", query: "SELECT 42 AS answer" }, + ]) + expect(result.isError).toBe(false) + expect(block(body, "bad")).toContain("Failed") + expect(block(body, "dep")).toContain('step "bad" failed') + expect(block(body, "missing")).toContain("names no earlier step") + expect(block(body, "ok")).toContain("42") +}) +// altimate_change end + +// altimate_change start — startup connection list in the instructions stays bounded +mcpTest("a large catalog is listed by connection name only and the instructions stay within the cap", async () => { + const { client } = await fixture(false, {}, undefined, (db) => { + for (let i = 0; i < 60; i++) db.run(`CREATE TABLE wide_table_with_a_long_name_${i} (id INTEGER)`) + }) + const instructions = client.getInstructions()! + expect(instructions).toContain("Connections:\n- source (sqlite)") + expect(instructions).not.toContain("wide_table_with_a_long_name_0") + expect(instructions.length - SERVER_INSTRUCTIONS.length).toBeLessThanOrEqual(1_250) +}) + +/** Starts a server on a connections file and records every connector opened and SQL run during startup. */ +async function startupFixture(connections: Record, setup?: (dir: string) => Promise) { + const dir = await mkdtemp(path.join(os.tmpdir(), "mcp-startup-test-")) + cleanups.push(() => rm(dir, { recursive: true, force: true })) + await setup?.(dir) + const file = path.join(dir, "connections.json") + await Bun.write(file, JSON.stringify(connections)) + const opened: string[] = [] + const executed: { warehouse: string; sql: string }[] = [] + const original = Registry.get + const get = spyOn(Registry, "get").mockImplementation(async (name) => { + opened.push(name) + const config = Registry.getConfig(name) + if (config?.type !== "sqlite" && config?.type !== "duckdb") { + // A remote or server connection double: it records calls and never answers, like an unreachable host. + const hang = (sql: string) => { + executed.push({ warehouse: name, sql }) + return new Promise(() => {}) + } + return { + connect: () => hang(""), + execute: (sql: string) => hang(sql), + listSchemas: () => hang(""), + listTables: () => hang(""), + describeTable: () => hang(""), + close: async () => {}, + } as unknown as Connector + } + const connector = await original(name) + return new Proxy(connector, { + get(target, key, receiver) { + if (key !== "execute") return Reflect.get(target, key, receiver) + return (sql: string, ...rest: unknown[]) => { + executed.push({ warehouse: name, sql }) + return (target.execute as (...args: unknown[]) => unknown).call(target, sql, ...rest) + } + }, + }) + }) + try { + const started = Date.now() + const server = await createServer({ connections: file }) + const elapsed = Date.now() - started + const client = new Client({ name: "test", version: "1" }, { capabilities: {} }) + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair() + await server.connect(serverTransport) + await client.connect(clientTransport) + cleanups.push(async () => { + await client.close() + await server.close() + }) + return { instructions: client.getInstructions()!, opened: [...opened], executed: [...executed], elapsed } + } finally { + get.mockRestore() + } +} + +const REMOTE_CONNECTIONS = { + pg_local: { type: "postgres", host: "localhost", database: "app", user: "u" }, + pg_remote: { type: "postgres", host: "db.example.com", database: "app", user: "u" }, + mongo_local: { type: "mongodb", connection_string: "mongodb://localhost:27017" }, + warehouse: { type: "snowflake", account: "acct", user: "u", password: "p" }, + lake: { type: "bigquery", project: "proj" }, + // A local file behind an SSH tunnel would open a network connection first. + tunneled: { type: "sqlite", path: "/tmp/never-opened.sqlite", ssh_host: "bastion.example.com", ssh_user: "u" }, +} + +mcpTest("startup lists remote and server connections by name and type without opening them", async () => { + const { instructions, opened, executed, elapsed } = await startupFixture(REMOTE_CONNECTIONS) + expect(opened).toEqual([]) + expect(executed).toEqual([]) + // No deadline is raced: nothing slow ran, so startup returns at once. + expect(elapsed).toBeLessThan(2_000) + expect(instructions).toContain( + "Connections:\n- pg_local (postgres)\n- pg_remote (postgres)\n- mongo_local (mongodb)\n- warehouse (snowflake)\n- lake (bigquery)\n- tunneled (sqlite)", + ) +}) + +mcpTest("startup lists a local SQLite file's tables with one catalog query and no row counts", async () => { + const { instructions, executed } = await startupFixture( + { source: { type: "sqlite", path: "source.sqlite" }, ...REMOTE_CONNECTIONS }, + async (dir) => { + const db = new Database(path.join(dir, "source.sqlite")) + db.run("CREATE TABLE items (id INTEGER)") + db.run("INSERT INTO items VALUES (1), (2)") + db.run("CREATE VIEW item_ids AS SELECT id FROM items") + db.close() + }, + ) + expect(instructions).toContain("Connections:\n- source (sqlite): item_ids, items\n- pg_local (postgres)") + expect(instructions).not.toMatch(/rows/) + expect(executed.map((call) => call.warehouse)).toEqual(["source"]) + expect(executed.some((call) => /count\s*\(/i.test(call.sql))).toBe(false) + expect(executed[0]!.sql).toContain("sqlite_master") +}) + +mcpTest("startup lists a local DuckDB file's tables with their schema", async () => { + const { instructions, executed } = await startupFixture({ lake: { type: "duckdb", path: "lake.duckdb" } }, async (dir) => { + Registry.setConfigs({ lake: { type: "duckdb", path: path.join(dir, "lake.duckdb"), create: true } }) + const connector = await Registry.get("lake") + await connector.execute("CREATE SCHEMA analytics") + await connector.execute("CREATE TABLE analytics.orders (id INTEGER)") + await connector.execute("CREATE TABLE customers (id INTEGER)") + await Registry.closeAll() + Registry.reset() + }) + expect(instructions).toContain("Connections:\n- lake (duckdb): analytics.orders, customers") + expect(executed.some((call) => /count\s*\(/i.test(call.sql))).toBe(false) +}) + +mcpTest("startup skips a local catalog over the table gate with a short note", async () => { + const { instructions, executed } = await startupFixture({ source: { type: "sqlite", path: "source.sqlite" } }, async (dir) => { + const db = new Database(path.join(dir, "source.sqlite")) + for (let i = 0; i < 45; i++) db.run(`CREATE TABLE t${i} (id INTEGER)`) + db.close() + }) + expect(instructions).toContain("Connections:\n- source (sqlite): over 40 tables") + expect(instructions).not.toContain("t1") + expect(executed).toHaveLength(1) + expect(executed[0]!.sql).toContain("LIMIT 41") +}) +// altimate_change end diff --git a/packages/opencode/test/altimate/mcp-server/stdio.test.ts b/packages/opencode/test/altimate/mcp-server/stdio.test.ts new file mode 100644 index 0000000000..7e9f53f61d --- /dev/null +++ b/packages/opencode/test/altimate/mcp-server/stdio.test.ts @@ -0,0 +1,214 @@ +// altimate_change start — standalone MCP policy and middleware regressions +import { expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import path from "node:path" +import os from "node:os" +import { Database } from "bun:sqlite" + +const CLI_ENTRY = path.resolve(import.meta.dir, "../../../src/index.ts") + +// Minimal newline-delimited JSON-RPC client over the child's stdio. The full suite shares one Bun process and +// test/mcp/* mock the MCP SDK client modules (never restored), so using the SDK here would talk to those mocks. +function rpcClient(child: ReturnType) { + const pending = new Map void>() + let id = 0 + void (async () => { + let buffer = "" + for await (const chunk of child.stdout as unknown as AsyncIterable) { + buffer += Buffer.from(chunk).toString() + for (let nl = buffer.indexOf("\n"); nl >= 0; nl = buffer.indexOf("\n")) { + const line = buffer.slice(0, nl).trim() + buffer = buffer.slice(nl + 1) + if (!line) continue + const message = JSON.parse(line) + pending.get(message.id)?.(message) + } + } + })() + const write = (message: object) => (child.stdin as import("bun").FileSink).write(JSON.stringify(message) + "\n") + const request = (method: string, params: object = {}) => + new Promise((resolve, reject) => { + const n = ++id + pending.set(n, (message) => (message.error ? reject(new Error(JSON.stringify(message.error))) : resolve(message.result))) + write({ jsonrpc: "2.0", id: n, method, params }) + }) + return { + async connect() { + await request("initialize", { protocolVersion: "2025-03-26", capabilities: {}, clientInfo: { name: "stdio-test", version: "1" } }) + write({ jsonrpc: "2.0", method: "notifications/initialized" }) + }, + listTools: () => request("tools/list"), + callTool: (params: { name: string; arguments: object }) => request("tools/call", params), + async close() { + ;(child.stdin as import("bun").FileSink).end() + await child.exited + }, + } +} + +for (const allowWrite of [false, true]) { + test(`CLI stdio handshake, read and write policy (allow-write=${allowWrite})`, async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "mcp-stdio-test-")) + const db = new Database(path.join(dir, "db.sqlite")) + db.run("CREATE TABLE items (id INTEGER)") + db.close() + const connections = path.join(dir, "connections.json") + await Bun.write(connections, JSON.stringify({ source: { type: "sqlite", path: "db.sqlite" } })) + // Any network attempt, including module initialization, fails this test. stdout must contain only MCP frames. + // The guard is an entry wrapper, not `--preload`: the package bunfig.toml `preload` replaces a CLI one. + const guard = path.join(dir, "no-network.ts") + await Bun.write( + guard, + 'globalThis.fetch = (() => { process.stderr.write("UNEXPECTED_NETWORK_CALL\\n"); throw new Error("No network allowed") }) as typeof fetch\n' + + `process.stderr.write("NETWORK_GUARD_ACTIVE\\n")\nawait import(${JSON.stringify(CLI_ENTRY)})\n`, + ) + const child = Bun.spawn( + [ + process.execPath, + "run", + "--conditions=browser", + guard, + "mcp", + "serve", + "--connections", + connections, + ...(allowWrite ? ["--allow-write"] : []), + ], + { + cwd: path.resolve(import.meta.dir, "../../.."), + env: { ...process.env, ALTIMATE_TELEMETRY_DISABLED: "1" }, + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }, + ) + let stderr = "" + void (async () => { + for await (const chunk of child.stderr as unknown as AsyncIterable) stderr += Buffer.from(chunk).toString() + })() + const client = rpcClient(child) + let spill: string | undefined + try { + await client.connect() + expect((await client.listTools()).tools).toHaveLength(5) + const write = await client.callTool({ + name: "sql_execute", + arguments: { warehouse: "source", query: "INSERT INTO items VALUES (7)" }, + }) + expect(write.isError).toBe(!allowWrite) + const read = await client.callTool({ + name: "sql_execute", + arguments: { + warehouse: "source", + query: "SELECT CASE WHEN COUNT(*) = 1 THEN 'one inserted item' ELSE 'empty source' END AS value FROM items", + }, + }) + expect(read.isError).toBe(false) + expect(JSON.stringify(read.content)).toContain(allowWrite ? "one inserted item" : "empty source") + const output = await client.callTool({ + name: "sql_execute", + arguments: { warehouse: "source", query: "SELECT 'private server output' AS value", full_text_columns: ["value"] }, + }) + spill = JSON.stringify(output.content).match(/(\/[^\s`"\\()]+\/result-\d+\.csv)/)?.[1] + expect(spill).toBeDefined() + expect(await Bun.file(spill!).exists()).toBe(true) + } finally { + await client.close() + if (spill) expect(await Bun.file(spill).exists()).toBe(false) + await rm(dir, { recursive: true, force: true }) + } + expect(stderr).toContain("NETWORK_GUARD_ACTIVE") + expect(stderr).not.toContain("UNEXPECTED_NETWORK_CALL") + }, 15_000) +} + +test("CLI exits cleanly when the client closes stdin", async () => { + const process = Bun.spawn([Bun.which("bun")!, "run", "--conditions=browser", "src/index.ts", "mcp", "serve"], { + cwd: path.resolve(import.meta.dir, "../../.."), + env: { ...Bun.env, ALTIMATE_TELEMETRY_DISABLED: "1" }, + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }) + process.stdin.end() + try { + expect(await process.exited).toBe(0) + expect(await new Response(process.stdout).text()).toBe("") + expect(await new Response(process.stderr).text()).toBe("") + } finally { + process.kill() + } +}, 15_000) + +test("global --log-level and --integrations apply to mcp serve", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "mcp-stdio-flags-")) + // Runs at exit, after the CLI middleware: logs through the same shim instance and reports the integrations mode. + const probe = path.join(dir, "probe.ts") + const shim = path.resolve(import.meta.dir, "../../../src/altimate/util/log.ts") + await Bun.write( + probe, + `import { Log } from ${JSON.stringify(shim)}\n` + + `process.on("exit", () => { Log.Default.info("probe-info-line"); Log.Default.error("probe-error-line"); ` + + `require("node:fs").writeSync(2, "integrations=" + process.env.ALTIMATE_INTEGRATIONS + "\\n") })\n` + + `await import(${JSON.stringify(CLI_ENTRY)})\n`, + ) + const child = Bun.spawn( + [ + process.execPath, + "run", + "--conditions=browser", + probe, + "mcp", + "serve", + "--print-logs", + "--log-level", + "ERROR", + "--integrations", + "local", + ], + { + cwd: path.resolve(import.meta.dir, "../../.."), + env: { ...process.env, ALTIMATE_TELEMETRY_DISABLED: "1", ALTIMATE_LOG_LEVEL: "", OPENCODE_LOG_LEVEL: "" }, + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }, + ) + child.stdin.end() + try { + expect(await child.exited).toBe(0) + const stderr = await new Response(child.stderr).text() + expect(stderr).toContain("integrations=local") + expect(stderr).toContain("probe-error-line") + expect(stderr).not.toContain("probe-info-line") + } finally { + child.kill() + await rm(dir, { recursive: true, force: true }) + } +}, 15_000) + +// altimate_change end + +// altimate_change start — execute the entrypoint's actual middleware without launching agent services +// Exercise the guard directly too, so deleting it fails deterministically even +// when the stdio child exits while the CLI is still unwinding. +test("mcp serve skips CLI middleware before starting the agent heap service", async () => { + const source = await Bun.file(path.resolve(import.meta.dir, "../../../src/index.ts")).text() + const opening = ".middleware(async (opts) => {" + const start = source.indexOf(opening) + const end = source.indexOf("\n })\n .usage", start) + expect(start).toBeGreaterThan(0) + expect(end).toBeGreaterThan(start) + const body = source.slice(start + opening.length, end) + const code = new Bun.Transpiler({ loader: "ts" }).transformSync(`async function middleware(opts: any) {${body}\n}`) + let starts = 0 + const middleware = new Function("Heap", `${code}; return middleware`)({ + start() { + starts++ + throw new Error("Agent heap initialization reached by MCP server") + }, + }) + await middleware({ _: ["mcp", "serve"] }) + expect(starts).toBe(0) +}) +// altimate_change end diff --git a/packages/opencode/test/altimate/precedence-guard-order.test.ts b/packages/opencode/test/altimate/precedence-guard-order.test.ts index b39118a5c5..7386e79d46 100644 --- a/packages/opencode/test/altimate/precedence-guard-order.test.ts +++ b/packages/opencode/test/altimate/precedence-guard-order.test.ts @@ -201,13 +201,14 @@ describe("sql_explain — input validation outranks the redirect", () => { }) describe("schema_inspect — input validation outranks the redirect", () => { - test("an empty table reports invalid input rather than redirecting", async () => { + // altimate_change start — omitted/blank tables discover inventory through the same routing guard + test("an empty table listing still respects warehouse precedence", async () => { const tool = await initTool(SchemaInspectTool) const result: any = await tool.execute({ table: " ", warehouse: "shadowed_snow" }, ctx) - expect(result.metadata.error_class).toBe("input_validation") - expect(result.metadata.redirected).toBeUndefined() + expect(result.metadata.redirected).toBe(true) }) + // altimate_change end test("an empty warehouse string is invalid input, not the default target", async () => { const tool = await initTool(SchemaInspectTool) const result: any = await tool.execute({ table: "orders", warehouse: " " }, ctx) diff --git a/packages/opencode/test/altimate/read-only-caveat.test.ts b/packages/opencode/test/altimate/read-only-caveat.test.ts new file mode 100644 index 0000000000..5313255ec9 --- /dev/null +++ b/packages/opencode/test/altimate/read-only-caveat.test.ts @@ -0,0 +1,12 @@ +import { expect, test } from "bun:test" +import { readOnlyCaveat } from "../../src/altimate/native/connections/read-only" + +test("a malformed connection without a string type gets no caveat instead of throwing", () => { + expect(readOnlyCaveat({ name: "broken" })).toBeUndefined() + expect(readOnlyCaveat({ name: "broken", type: 42 })).toBeUndefined() +}) + +test("enforcing drivers get no caveat; others do", () => { + expect(readOnlyCaveat({ name: "pg", type: "PostgreSQL" })).toBeUndefined() + expect(readOnlyCaveat({ name: "wh", type: "snowflake" })).toContain("wh (snowflake): read-only is enforced by SQL classification only") +}) diff --git a/packages/opencode/test/altimate/sql-validation-e2e.test.ts b/packages/opencode/test/altimate/sql-validation-e2e.test.ts index 83e867dbd2..d7d7de3459 100644 --- a/packages/opencode/test/altimate/sql-validation-e2e.test.ts +++ b/packages/opencode/test/altimate/sql-validation-e2e.test.ts @@ -68,7 +68,10 @@ describe("Tool name consistency in prompts", () => { expect(analyst).toBeDefined() expect(analyst!.prompt).not.toContain("sql_validate") expect(analyst!.prompt).toContain( - "keep that reasoning out of the final message. When a rule or policy decides the answer, read its full text (not a preview), enumerate each competing clause, and evaluate every condition as satisfied, violated or unknown against the row's own fields and dates before choosing.", + "keep that reasoning out of the final message, and record the chosen and rejected reading as one `answer_commit` `readings` entry. Never reject a reading only because it yields few rows; reject it on the question's own wording or the data's shape.", + ) + expect(analyst!.prompt).toContain( + "When a rule or policy decides the answer, read its full text (not a preview), enumerate each competing clause, and evaluate every condition as satisfied, violated or unknown against the row's own fields and dates before choosing", ) }, }) @@ -109,7 +112,7 @@ describe("Tool name consistency in prompts", () => { const paragraph = prompt.trim().split("\n\n").at(-1) // altimate_change start - require one field for each requested answer item expect(paragraph).toContain( - "For a data question, finish by calling `answer_commit` with the single chosen value(s) with one field per item the request asks for (every named item, such as an entity and its related item or count, must be its own field: when the request asks for only the answer, anything left in the basis or prose is dropped), a one-or-two-sentence basis (population, grain, source object/column), or `unresolved` with the reason, then reply with exactly the text it returns and nothing else.", + "For a data question, finish by calling `answer_commit` with the single chosen value(s) with one field per item the request asks for (every named item, such as an entity and its related item or count, must be its own field: when the request asks for only the answer, anything left in the basis or prose is dropped), `asked` listing those items from the question in order, a one-or-two-sentence basis (population, grain, source object/column), or `unresolved` with the reason, then reply with exactly the text it returns and nothing else.", ) // altimate_change end expect(paragraph).toContain("Always supply `basis`, even when unresolved") diff --git a/packages/opencode/test/altimate/tools/answer-commit-conjunction.test.ts b/packages/opencode/test/altimate/tools/answer-commit-conjunction.test.ts index 0fd8708f34..433b8392bf 100644 --- a/packages/opencode/test/altimate/tools/answer-commit-conjunction.test.ts +++ b/packages/opencode/test/altimate/tools/answer-commit-conjunction.test.ts @@ -126,6 +126,23 @@ describe("answer_commit conjunction check", () => { expect((await tool.execute(unresolved, context([user(compound)]))).output).toBe(renderAnswer(unresolved)) expect((await tool.execute(payload, context())).output).toBe(renderAnswer(payload)) }) + test("counts non-blank asked labels, so a rows commit is checked too; blank labels fall back to fields", async () => { + const tool = await initTool(AnswerCommitTool) + const rows = { basis, rows: { columns: ["repository"], values: [["repo-a"]] } } + const underAsked = await tool.execute({ ...rows, asked: ["repository"] }, context([user(compound)])) + expect(underAsked.status).toBe("error") + expect(underAsked.output).toContain('1 asked item (["repository"])') + expect((await tool.execute({ ...payload, asked: [" "] }, context([user(compound)]))).status).toBe("error") + const both = { ...complete, asked: ["repository", "most copied file"] } + const accepted = await tool.execute(both, context([user(compound)])) + expect(accepted.metadata.asked).toEqual(both.asked) + expect(accepted.output).toBe(renderAnswer({ ...both, asked: undefined })) + }) + test("an unresolved commit with asked labels is accepted", async () => { + const tool = await initTool(AnswerCommitTool) + const unresolved = { basis, unresolved: { reason: "No matching records." }, asked: ["repository"] } + expect((await tool.execute(unresolved, context([user(compound)]))).output).toBe(renderAnswer(unresolved)) + }) }) describe("near the time budget a commit is never rejected", () => { diff --git a/packages/opencode/test/altimate/tools/answer-commit.test.ts b/packages/opencode/test/altimate/tools/answer-commit.test.ts index 23a43fe90e..6bcffbeaa8 100644 --- a/packages/opencode/test/altimate/tools/answer-commit.test.ts +++ b/packages/opencode/test/altimate/tools/answer-commit.test.ts @@ -1012,17 +1012,35 @@ describe("renderAnswer basis length bound", () => { expect((await tool.execute(payload, context)).output).toBe(expected) }) - test.each(["x".repeat(401), ` ${"x".repeat(399)} `])( - "rejects 401 raw basis characters with the named bound", + test.each(["x".repeat(401), "x".repeat(1200), ` ${"x".repeat(400)}`])( + "trims an over-long basis to the bound instead of rejecting the commit", async (basis) => { const tool = await initTool(AnswerCommit.AnswerCommitTool) for (const answer_only of [false, true]) { const payload = { basis, answer_only, fields: [{ name: "", value: 2 }] } - expect(() => AnswerCommit.renderAnswer(payload)).toThrow(/basis.*400 characters/i) - await rejects(tool.execute(payload, context), /basis.*400 characters/i) + const result = await tool.execute(payload, context) + expect(result.status).toBeUndefined() + const shown = String(result.output) + if (answer_only) expect(shown).toBe("2") + else { + const trimmed = shown.split("\n\n**Answer:** 2")[0]! + expect(trimmed.length).toBeLessThanOrEqual(400) + expect(trimmed.endsWith("…")).toBe(basis.trim().length > 400) + } } }, ) + + test("keeps substantive basis text that follows leading whitespace", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const payload = { basis: `${" ".repeat(450)}real caveat`, fields: [{ name: "", value: 2 }] } + expect(String((await tool.execute(payload, context)).output)).toContain("real caveat") + }) + + test("still requires a non-blank basis", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + await rejects(tool.execute({ basis: " ", fields: [{ name: "", value: 2 }] }, context), /basis is always required/i) + }) }) describe("renderAnswer structural validation", () => { @@ -1130,6 +1148,7 @@ test("answer_commit exposes a serializable object input schema", async () => { expect(tool.jsonSchema?.type).toBe("object") expect(Object.keys(tool.jsonSchema?.properties ?? {})).toEqual([ "fields", + "asked", "rows", "unresolved", "basis", @@ -1240,3 +1259,174 @@ test("answer_commit guidance requires every named item as its own field because }), ).toBe("repo-a; src/a.swift; 23") }) + +describe("answer_commit commit-time checks", () => { + type Part = Record + const user = (text: string) => ({ info: { role: "user" }, parts: [{ type: "text", text }] }) + const assistant = (...parts: Part[]) => ({ info: { role: "assistant" }, parts }) + const sql = (output: string) => ({ type: "tool", tool: "sql_execute", state: { status: "completed", output } }) + const failedCommit = (error: string) => ({ type: "tool", tool: "answer_commit", state: { status: "error", error } }) + const run = async (payload: Record, messages: unknown[]) => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + return tool.execute({ basis: recordBasis, ...payload }, { ...context, messages }) + } + const compound = { + answer_only: true, + asked: ["repository", "most copied file"], + fields: [{ name: "repository", value: "repo-a" }], + } + + test("rejects a commit that leaves the second asked item in basis only", async () => { + const result = await run(compound, [user("Which repository, and its most copied file?")]) + expect(result.status).toBe("error") + expect(result.output).toContain("answer_commit requested-items check:") + expect(result.output).toContain('"most copied file"') + expect(result.output).not.toContain('supplies "repository"') + expect(result.metadata.committed_text).toBeUndefined() + }) + + test("accepts full coverage and keeps asked out of the answer", async () => { + const result = await run( + { ...compound, fields: [...compound.fields, { name: "file", value: "src/a.swift" }] }, + [user("Which repository, and its most copied file?")], + ) + expect(result.status).toBeUndefined() + expect(result.output).toBe("repo-a; src/a.swift") + expect(result.metadata.asked).toEqual(compound.asked) + }) + + test("rows columns cover asked items and an all-empty column does not", async () => { + const rows = { columns: ["package", "version"], values: [["a", null], ["b", ""]] } + const asked = ["package name", "version"] + expect((await run({ asked, rows }, [])).output).toContain('supplies "version"') + const filled = { ...rows, values: [["a", "1.0"], ["b", null]] } + expect((await run({ asked, rows: filled }, [])).status).toBeUndefined() + }) + + test.each([[[]], [""], [null], [["only item"]], [["", " "]]])("ignores asked filler %j", async (asked) => { + const result = await run({ asked, fields: [{ name: "", value: "repo-a" }], answer_only: true }, []) + expect(result.status).toBeUndefined() + expect(result.output).toBe("repo-a") + }) + + test("accepts after two coverage rejections in the same turn", async () => { + const rejection = (await run(compound, [])).output + const once = [user("q"), assistant(failedCommit(rejection))] + expect((await run(compound, once)).status).toBe("error") + const twice = [user("q"), assistant(failedCommit(rejection)), assistant(failedCommit(rejection))] + const accepted = await run(compound, twice) + expect(accepted.status).toBeUndefined() + expect(accepted.output).toBe("repo-a") + // A new user turn starts a fresh allowance. + expect((await run(compound, [...twice, user("next question")])).status).toBe("error") + }) + + test("legacy calls without asked behave as before", async () => { + const payload = { answer_only: true, fields: [{ name: "repository", value: "repo-a" }] } + const result = await run(payload, [user("Which repository and file?")]) + expect(result).toEqual({ + title: "Answer committed", + metadata: { readings: [], asked: [], truncated: false, committed_text: "repo-a" }, + output: "repo-a", + }) + }) + + test("readings stay in metadata and out of answer-only text", async () => { + const readings = [ + { + clause: "over $5", + reading: "quantity over 5 units", + result_summary: "line 3 violates", + decided_by: "rejected the dollar reading: every line satisfies it, so the clause would not bite", + }, + ] + const result = await run({ answer_only: true, fields: [{ name: "", value: "Volume tiers" }], readings }, []) + expect(result.output).toBe("Volume tiers") + expect(result.metadata.readings).toEqual(readings) + for (const reading of Object.values(readings[0]!)) expect(result.output).not.toContain(reading) + }) + + test("warns once about a number that no query returned, then accepts", async () => { + const history = [user("Fastest average cycle?"), assistant(sql("owner | days\nA | 79\nB | 84\nC | 95"))] + const payload = { answer_only: true, fields: [{ name: "days", value: 81 }] } + const warned = await run(payload, history) + expect(warned.status).toBe("error") + expect(warned.output).toContain("answer_commit number check: 81") + const accepted = await run(payload, [...history, assistant(failedCommit(warned.output))]) + expect(accepted.status).toBeUndefined() + expect(accepted.output).toBe("81") + }) + + test("an MCP client's computed number is accepted without the number check", async () => { + const history = [user("Fastest average cycle?"), assistant(sql("owner | days\nA | 79\nB | 84\nC | 95"))] + const payload = { basis: recordBasis, answer_only: true, fields: [{ name: "days", value: 81 }] } + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const accepted = await tool.execute(payload, { ...context, messages: history, extra: { mcpClient: true } }) + expect(accepted.status).toBeUndefined() + expect(accepted.output).toBe("81") + // The requested-items check still applies over MCP. + const missing = await tool.execute( + { ...payload, asked: ["days", "owner"] }, + { ...context, messages: history, extra: { mcpClient: true } }, + ) + expect(missing.status).toBe("error") + expect(missing.output).toContain("answer_commit requested-items check:") + }) + + test.each([ + ["rounded", 0.16, "ratio\n0.1621"], + ["truncated", 12.3, "avg 12.39"], + ["percentage of a ratio", "16.2%", "share 0.1621"], + ["grouped", "$1,234.50", '{"total": 1234.5}'], + ["float noise", 0.1 + 0.2, "0.3"], + ["from the question", 2015, ""], + ])("accepts a %s number", async (_, value, output) => { + const history = [user("Articles in 2015?"), assistant(sql(output || "count\n42"))] + const result = await run({ answer_only: true, fields: [{ name: "", value }] }, history) + expect(result.status).toBeUndefined() + }) + + test("skips the number check without prior SQL output and for small counts", async () => { + expect((await run({ fields: [{ name: "", value: 81 }] }, [user("q")])).status).toBeUndefined() + const history = [user("q"), assistant(sql("a\nb\nc"))] + expect((await run({ fields: [{ name: "", value: 3 }] }, history)).status).toBeUndefined() + }) + + const sized = (decided_by: string) => ({ + answer_only: true, + fields: [{ name: "agent", value: "A1" }], + readings: [{ clause: "closed in April", reading: "close date", result_summary: "3 agents", decided_by }], + }) + + test("challenges a reading rejected only for returning a single row, once", async () => { + const payload = sized("Rejected the signed-date reading because it yields only one opportunity, no comparison") + const warned = await run(payload, [user("q")]) + expect(warned.status).toBe("error") + expect(warned.output).toContain("answer_commit reading check:") + expect(warned.output).toContain('"closed in April"') + const accepted = await run(payload, [user("q"), assistant(failedCommit(warned.output))]) + expect(accepted.status).toBeUndefined() + expect(accepted.output).toBe("A1") + }) + + test("challenges a reading rejected because it returns 1 row", async () => { + const result = await run(sized("Rejected the second reading since it returns 1 row"), [user("q")]) + expect(result.output).toContain("answer_commit reading check:") + }) + + test.each([ + "Rejected the signed-date reading because the policy defines closing as the close date", + "Rejected summing orders: the question asks for distinct customers", + "The request explicitly asks for one row", + "Rejected the 100-customer subset: only 100 customers matched the stated filter", + ])("does not challenge a reading rejected on the request's wording: %s", async (decided_by) => { + expect((await run(sized(decided_by), [user("q")])).status).toBeUndefined() + }) + + test("checks numeric row cells", async () => { + const history = [user("q"), assistant(sql("name,score\nx,10.5\ny,9.25"))] + const rows = { columns: ["name", "score"], values: [["x", 10.5], ["y", 47.75]] } + const result = await run({ rows }, history) + expect(result.output).toContain("number check: 47.75") + }) +}) diff --git a/packages/opencode/test/altimate/tools/schema-inspect-list.test.ts b/packages/opencode/test/altimate/tools/schema-inspect-list.test.ts new file mode 100644 index 0000000000..6bba66cdbd --- /dev/null +++ b/packages/opencode/test/altimate/tools/schema-inspect-list.test.ts @@ -0,0 +1,285 @@ +// altimate_change start — warehouse inventory discovery regressions +import { afterEach, expect, mock, spyOn, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import type { Connector } from "@altimateai/drivers" +import { resolveQueryTimeout } from "@altimateai/drivers/query-timeout" +import { initTool } from "../tool-fixture" +import { discoverCatalog } from "../../../src/altimate/tools/warehouse-discovery" +import { SchemaInspectTool } from "../../../src/altimate/tools/schema-inspect" +import * as Registry from "../../../src/altimate/native/connections/registry" +import { registerAll } from "../../../src/altimate/native/connections/register" +import * as Precedence from "../../../src/altimate/workspace/precedence" + +const ctx = { + sessionID: "schema-list-test", + messageID: "message", + callID: "call", + agent: "test", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + extra: { mcpClient: true, forceReadOnly: true }, +} +let restore: (() => void) | undefined +let directory: string | undefined +// MCP usage hints are shown once per session; each call gets its own session so outputs compare equal. +let calls = 0 +const sessions: string[] = [] + +afterEach(async () => { + restore?.() + restore = undefined + await Registry.closeAll() + Registry.reset() + for (const session of sessions.splice(0)) Precedence.releaseLocalSession(session) + if (directory) await rm(directory, { recursive: true, force: true }) + directory = undefined +}) + +async function inspect(args: Record = {}, ask = async (_input: unknown) => {}) { + const sessionID = `${ctx.sessionID}-${++calls}` + sessions.push(sessionID) + Precedence.bindLocalSession(sessionID) + const tool = await initTool(SchemaInspectTool) + return tool.execute(args, { ...ctx, sessionID, ask }) +} + +for (const type of ["sqlite", "duckdb"] as const) { + test(`${type}: real local tables list with catalog metadata; blank table and schema mean absent`, async () => { + registerAll() + directory = await mkdtemp(path.join(os.tmpdir(), "schema-list-")) + Registry.setConfigs({ local: { type, path: path.join(directory, `source.${type}`), create: true } }) + const connector = await Registry.get("local") + await connector.execute('CREATE TABLE "odd""name" (id INTEGER)') + await connector.execute('INSERT INTO "odd""name" VALUES (1), (2)') + await connector.execute("CREATE TABLE empty (id INTEGER)") + await connector.execute('CREATE VIEW sample AS SELECT * FROM "odd""name"') + const overview = await inspect({ warehouse: "local" }) + expect(overview.output).toContain("main | 3") + expect(overview.output).not.toContain('odd"name') + const result = await inspect({ warehouse: "local", schema_name: "main" }) + expect(result.metadata.success).toBe(true) + expect(result.output).toContain('main | odd"name | table | ' + (type === "duckdb" ? "2 (estimate)" : "")) + expect(result.output).toContain("main | empty | table | " + (type === "duckdb" ? "0 (estimate)" : "")) + expect(result.output).toContain("main | sample | view | \n") + expect(result.output).toContain('schema_inspect(warehouse="local", schema_name="", table="")') + expect(await inspect({ warehouse: "local", table: "", schema_name: "" })).toEqual(overview) + expect(await inspect({ warehouse: "local", table: " " })).toEqual(overview) + if (type === "duckdb") { + await connector.execute("CREATE SCHEMA analytics") + await connector.execute("CREATE TABLE analytics.other (id INTEGER)") + expect((await inspect({ warehouse: "local" })).output).toContain("analytics | 1") + const filtered = await inspect({ warehouse: "local", schema_name: "analytics" }) + expect(filtered.output).toContain("analytics | other | table | 0 (estimate)") + expect(filtered.output).not.toContain("main |") + } + await connector.execute("CREATE TEMP TABLE scratch_probe (id INTEGER)") + await connector.execute("INSERT INTO scratch_probe VALUES (42)") + // Both listing paths must actually run against this engine before the temp table is checked. + expect((await inspect({ warehouse: "local", schema_name: "main" })).output).toContain("main | empty | table") + const discovered = await discoverCatalog([{ name: "local", type }], { signal: ctx.abort, sessionID: ctx.sessionID }) + expect(discovered).toContain("local: main.empty") + expect((await connector.execute("SELECT id FROM scratch_probe")).rows).toEqual([[42]]) + const single = await inspect({ warehouse: "local", table: "empty", values: false }) + expect(single).toEqual({ + title: "Schema: empty", + metadata: { success: true, columnCount: 1, rowCount: undefined, truncated: false }, + output: + // MCP output drops the table rule line. + "Table: main.empty\n\nColumn | Type | Nullable | PK\nid | INTEGER | YES | ", + }) + }) +} + +function fake(type: string, count = 1, store?: string) { + registerAll() + Registry.setConfigs({ remote: { type, ...(store ? { path: store } : {}) } }) + const connector = { + listSchemas: mock(async () => ["one", "two"]), + listTables: mock(async (_schema: string) => + Array.from({ length: count }, (_, i) => ({ name: `t${i}`, type: "table" })), + ), + execute: mock(async (sql: string, limit?: number, _binds?: unknown[], _options?: unknown) => { + let rows: unknown[][] + if (sql.includes("GROUP BY") || sql.includes("SELECT 'main'")) + rows = [ + ["one", count], + ["two", count], + ] + else if (sql.includes("dbStats")) rows = [[count]] + else if (/^SELECT COUNT/.test(sql)) rows = [[3]] + else + rows = Array.from({ length: Math.min(count, limit ?? 501) }, (_, i) => [ + `t${String(i).padStart(7, "0")}`, + type === "mongodb" ? "collection" : "BASE TABLE", + ]) + return { rows, columns: [], row_count: rows.length, truncated: type === "mongodb" && count > (limit ?? 501) } + }), + } + const get = spyOn(Registry, "get").mockResolvedValue(connector as unknown as Connector) + restore = () => get.mockRestore() + return connector +} + +test.each(["postgres", "mysql", "snowflake", "duckdb", "redshift", "bigquery"])( + "%s uses one bounded aggregate, then an ordered bounded table catalog", + async (type) => { + const connector = fake(type, 2_000_000, type === "duckdb" ? "md:remote" : undefined) + const result = await inspect({ warehouse: "remote" }) + expect(result.metadata).toMatchObject({ success: true, schemaCount: 2 }) + expect(result.output).toContain("one | 2000000") + expect(result.output).not.toContain("t0000000") + expect(connector.execute).toHaveBeenCalledTimes(1) + expect(connector.execute.mock.calls[0][0]).toMatch(/GROUP BY.*ORDER BY.*LIMIT 201/) + expect(connector.execute.mock.calls[0][3]).toMatchObject({ readOnly: true, queryTimeoutMs: 10_000, signal: expect.any(AbortSignal) }) + const start = performance.now() + const filtered = await inspect({ warehouse: "remote", schema_name: "two" }) + expect(performance.now() - start).toBeLessThan(2000) + expect(filtered.metadata).toMatchObject({ tableCount: 500, truncated: true }) + expect(filtered.output).toContain("Showing first 500 of ≥501") + expect(connector.execute.mock.calls[1]).toMatchObject([ + expect.stringMatching(/ORDER BY t.table_name LIMIT 501$/), + 501, + ["two"], + { readOnly: true }, + ]) + // The fake models a two-million-table catalog but allocates only the requested 501 rows. + expect( + ((await connector.execute.mock.results[1].value) as Awaited>).rows, + ).toHaveLength(501) + expect(connector.listTables).not.toHaveBeenCalled() + expect(connector.listSchemas).not.toHaveBeenCalled() + expect(connector.execute.mock.calls.some(([sql]) => /^SELECT COUNT/.test(sql))).toBe(false) + }, +) + +test("fallback lists schemas without tables and warns when listing tables", async () => { + const connector = fake("trino") + expect((await inspect({ warehouse: "remote" })).output).toContain("one | unavailable") + expect(connector.listTables).not.toHaveBeenCalled() + const result = await inspect({ warehouse: "remote", schema_name: "two" }) + expect(result.output).toContain("Driver fallback: table list may be truncated") + expect(connector.execute).not.toHaveBeenCalled() + expect(connector.listTables.mock.calls).toEqual([["two"]]) +}) + +test("MongoDB database counts use scalar metadata and collections use a bounded command", async () => { + const connector = fake("mongodb", 2_000_000) + const overview = await inspect({ warehouse: "remote" }) + expect(overview.output).toContain("one | 2000000") + expect(connector.listTables).not.toHaveBeenCalled() + const result = await inspect({ warehouse: "remote", schema_name: "two" }) + expect(result.metadata).toMatchObject({ tableCount: 500, truncated: true }) + expect(connector.execute.mock.calls[2]).toMatchObject([ + JSON.stringify({ command: "listCollections", database: "two" }), + 500, + undefined, + { readOnly: true }, + ]) +}) + +test("schema output caps at 200 using one aggregate", async () => { + const connector = fake("postgres") + connector.execute.mockResolvedValueOnce({ + rows: Array.from({ length: 201 }, (_, i) => [`s${i}`, 2_000_000]), + columns: [], + row_count: 201, + truncated: false, + }) + const result = await inspect({ warehouse: "remote" }) + expect(result.metadata).toMatchObject({ schemaCount: 200, truncated: true }) + expect(result.output).toContain("Showing first 200 schemas of ≥201") + expect(connector.execute).toHaveBeenCalledTimes(1) +}) + +test("exactly 500 tables is not truncated", async () => { + fake("postgres", 500) + expect((await inspect({ warehouse: "remote", schema_name: "one" })).metadata.truncated).toBe(false) +}) + +test("denied schema permission stops inventory access", async () => { + const connector = fake("postgres") + const result = await inspect({ warehouse: "remote" }, async () => { + throw new Error("permission denied") + }) + expect(result.metadata.success).toBe(false) + expect(connector.execute).not.toHaveBeenCalled() + expect(connector.listSchemas).not.toHaveBeenCalled() +}) + +test("catalog failure never falls back to an unbounded list", async () => { + const connector = fake("postgres") + connector.execute.mockRejectedValue(new Error("catalog denied")) + expect((await inspect({ warehouse: "remote", schema_name: "one" })).metadata.success).toBe(false) + expect(connector.listTables).not.toHaveBeenCalled() +}) + +for (const type of ["sqlite", "duckdb"]) { + for (const store of [":memory:", "/tmp/catalog-test.db"]) { + for (const count of [1, 501]) { + test(`${type} ${store}: catalog-only listing with ${count} tables adds no deadline`, async () => { + const connector = fake(type, count, store) + if (type === "duckdb" && count === 1) + connector.execute.mockResolvedValueOnce({ + rows: [["orders", "BASE TABLE", 23], ["view", "VIEW", null]], + columns: [], row_count: 2, truncated: false, + }) + const timeout = spyOn(AbortSignal, "timeout") + try { + const result = await inspect({ warehouse: "remote", schema_name: "one" }) + expect(result.metadata.success).toBe(true) + expect(connector.execute).toHaveBeenCalledTimes(1) + const [sql, limit, , options] = connector.execute.mock.calls[0] + expect(sql).not.toMatch(/COUNT\s*\(/i) + expect(sql).toContain("LIMIT 501") + expect(limit).toBe(501) + expect(options).toMatchObject({ readOnly: true }) + // Only the connection's own timeout applies; the listing adds no shorter deadline. + expect((options as { queryTimeoutMs?: number }).queryTimeoutMs).toBe( + resolveQueryTimeout(Registry.getConfig("remote")).ms, + ) + expect(timeout).not.toHaveBeenCalled() + if (type === "duckdb") expect(sql).toContain("d.estimated_size") + if (count > 500) { + expect(result.output).toContain("Showing first 500 of ≥501") + expect(result.metadata.truncated).toBe(true) + } else if (type === "duckdb") { + expect(result.output).toContain("orders | table | 23 (estimate)") + expect(result.output).toContain("view | view | ") + expect(result.output).not.toContain("view | view | 0") + } else { + expect(result.output).toContain("t0000000 | table | ") + expect(result.output).not.toContain("(estimate)") + } + } finally { + timeout.mockRestore() + } + }) + } + } +} + +test("remote catalog statistics are displayed as estimates without table scans", async () => { + const connector = fake("postgres") + connector.execute.mockResolvedValueOnce({ + rows: [["orders", "BASE TABLE", 2000000]], + columns: [], + row_count: 1, + truncated: false, + }) + const result = await inspect({ warehouse: "remote", schema_name: "one" }) + expect(result.output).toContain("orders | table | 2000000 (estimate)") + expect(connector.execute).toHaveBeenCalledTimes(1) +}) + +test("empty inventory gives next-step instructions", async () => { + const connector = fake("postgres", 0) + connector.execute.mockResolvedValueOnce({ rows: [], columns: [], row_count: 0, truncated: false }) + const result = await inspect({}) + expect(result.metadata.success).toBe(true) + expect(result.output).toContain("No schemas found.") + expect(result.output).toContain("schema_name=") +}) +// altimate_change end diff --git a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts index 2327b0caee..af8956dbbc 100644 --- a/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts +++ b/packages/opencode/test/altimate/tools/schema-inspect-values.test.ts @@ -1,5 +1,6 @@ import { afterAll, beforeAll, beforeEach, describe, expect, mock, spyOn, test } from "bun:test" import { Database } from "bun:sqlite" +import { resolveQueryTimeout } from "@altimateai/drivers/query-timeout" import { initTool } from "../tool-fixture" import * as Dispatcher from "../../../src/altimate/native/dispatcher" import * as Registry from "../../../src/altimate/native/connections/registry" @@ -204,10 +205,17 @@ describe("schema_inspect value grounding", () => { expect(result.output).toContain("group_concat(value: 1 distinct — left␟right (1)") }) - test.each(["postgres", "postgresql", "sqlite", "duckdb"])( - "profiling protects PostgreSQL reads and preserves other connector options (%s)", - async (type) => { - Registry.setConfigs({ w: { type } }) + test.each([ + ["postgres", undefined], + ["postgresql", undefined], + ["sqlite", ":memory:"], + ["sqlite", "file::memory:"], + ["duckdb", ":memory:"], + ["duckdb", "md:remote_db"], + ])( + "profiling protects PostgreSQL reads and preserves other connector options (%s %s)", + async (type, path) => { + Registry.setConfigs({ w: path ? { type, path } : { type } }) registerAll() Dispatcher.register("schema.inspect" as any, async () => ({ table: "t", schema_name: "public", warehouse: "w", columns: COLUMNS, @@ -227,7 +235,11 @@ describe("schema_inspect value grounding", () => { expect(sql).toContain('FROM "public"."t"') expect(limit).toBe(41) expect(binds).toBeUndefined() - expect(options.queryTimeoutMs).toBe(15_000) + // Local engines keep the connection's own timeout: a shorter deadline would restart the helper. + // Remote stores, including MotherDuck, keep the profiling cap. + expect(options.queryTimeoutMs).toBe( + type === "sqlite" || path === ":memory:" ? resolveQueryTimeout(Registry.getConfig("w")).ms : 15_000, + ) if (type === "postgres" || type === "postgresql") expect(options.readOnly).toBe(true) else expect(options).not.toHaveProperty("readOnly") } finally { diff --git a/packages/opencode/test/altimate/tools/schema-inspect-warehouse.test.ts b/packages/opencode/test/altimate/tools/schema-inspect-warehouse.test.ts index d471ed191b..d29841df31 100644 --- a/packages/opencode/test/altimate/tools/schema-inspect-warehouse.test.ts +++ b/packages/opencode/test/altimate/tools/schema-inspect-warehouse.test.ts @@ -168,11 +168,12 @@ describe("schema_inspect warehouse qualifiers", () => { expect(result.output).toContain("warehouse=reporting, schema_name=main") }) - test("requires table and does not expose a table_name alias", async () => { + test("allows table discovery and does not expose a table_name alias", async () => { const tool = await initTool(SchemaInspectTool) - expect(tool.jsonSchema?.required).toContain("table") + expect(tool.jsonSchema?.required ?? []).not.toContain("table") expect(tool.jsonSchema?.properties).not.toHaveProperty("table_name") - await expect(inspect({ table_name: "t" })).rejects.toThrow("table") + expect((await inspect({ warehouse: "primary" })).output).toContain("main | 1") + expect((await inspect({ warehouse: "primary", schema_name: "main" })).output).toContain("main | t | table | 1 (estimate)") }) }) // altimate_change end diff --git a/packages/opencode/test/altimate/tools/sql-result-format.test.ts b/packages/opencode/test/altimate/tools/sql-result-format.test.ts index da5349aadd..eb012e0cb7 100644 --- a/packages/opencode/test/altimate/tools/sql-result-format.test.ts +++ b/packages/opencode/test/altimate/tools/sql-result-format.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test" import type { SqlExecuteResult } from "../../../src/altimate/native/types" import { formatSqlResult, + serializeCsv, serializeRows, SQL_MAX_PREVIEW_BYTES, SQL_MAX_PREVIEW_COLUMNS, @@ -1105,3 +1106,55 @@ describe("SQL full text retrieval", () => { ]) }) }) + +describe("MCP compact output", () => { + test("serializeCsv writes the header once and keeps NULL, empty strings and special values apart", () => { + const chunks: string[] = [] + serializeCsv( + result( + [ + [null, "", " padded", 'say "hi", ok'], + [12345678901234567890n, new Date("2024-01-02T03:04:05.006Z"), new Uint8Array([1, 2, 3]), { a: [1, 2n] }], + ], + ["a", "", "c", "d"], + ), + (chunk) => chunks.push(chunk), + ) + expect(chunks.join("")).toBe( + 'a,"",c,d\n' + + ',""," padded","say ""hi"", ok"\n' + + '12345678901234567890,2024-01-02T03:04:05.006Z,base64:AQID,"{""a"":[1,{""$bigint"":""2""}]}"\n', + ) + }) + + test("serializeCsv enforces the spill size limit", () => { + expect(() => serializeCsv(result([["x".repeat(SQL_MAX_SPILL_BYTES)]]), () => {})).toThrow(/spill size limit/) + }) + + test("an MCP inline result has no rule line; the native one keeps it", async () => { + const rows = result([[1, "a"]], ["id", "name"]) + const write = async () => resultPath + expect((await formatSqlResult(rows, { limit: 100, write, mcpClient: true })).output).toBe("id | name\n1 | a\n\n(1 rows)") + expect((await formatSqlResult(rows, { limit: 100, write })).output).toBe("id | name\n-----+-----\n1 | a\n\n(1 rows)") + }) + + test("an MCP spill previews ten rows, names the scratch table once in full, then briefly", async () => { + const rows = result(Array.from({ length: 30 }, (_, i) => [i, "v".repeat(400)]), ["id", "text"], true) + const seen = new Set() + const options = { + limit: 100, + mcpClient: true, + write: async () => resultPath, + store: async () => "result_1", + firstHint: (key: string) => !seen.has(key) && Boolean(seen.add(key)), + } + const first = await formatSqlResult(rows, options) + expect(first.output).toStartWith("Preview: 10 of 30 returned rows. Row limit reached; more rows may exist.") + expect(first.output).toContain("instead of reading the file") + expect(first.output).not.toContain("v".repeat(161)) + expect(first.metadata.scratch_table).toBe("result_1") + const second = await formatSqlResult(rows, options) + expect(second.output).toContain(`scratch table result_1 (CSV copy: ${resultPath}).`) + expect(second.output).not.toContain("instead of reading the file") + }) +}) diff --git a/packages/opencode/test/altimate/tools/warehouse-discovery.test.ts b/packages/opencode/test/altimate/tools/warehouse-discovery.test.ts new file mode 100644 index 0000000000..f2d102e384 --- /dev/null +++ b/packages/opencode/test/altimate/tools/warehouse-discovery.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, mock, spyOn, test } from "bun:test" +import type { Connector } from "@altimateai/drivers" +import type { ExecuteOptions } from "@altimateai/drivers/types" +import { resolveQueryTimeout } from "@altimateai/drivers/query-timeout" +import * as Registry from "../../../src/altimate/native/connections/registry" +import { Dispatcher } from "../../../src/altimate/native" +import { discoverCatalog, isLocalConnection } from "../../../src/altimate/tools/warehouse-discovery" + +describe("isLocalConnection", () => { + test("local SQLite/DuckDB stores are local; remote store schemes are not", () => { + expect(isLocalConnection({ type: "sqlite", path: "/data/app.sqlite" })).toBe(true) + expect(isLocalConnection({ type: "duckdb", path: ":memory:" })).toBe(true) + expect(isLocalConnection({ type: "duckdb", path: "file:///data/w.duckdb" })).toBe(true) + expect(isLocalConnection({ type: "duckdb", path: "md:my_db" })).toBe(false) + expect(isLocalConnection({ type: "duckdb", path: "s3://bucket/w.duckdb" })).toBe(false) + expect(isLocalConnection({ type: "sqlite" })).toBe(false) + }) + + test("servers count as local only on a loopback host or a socket", () => { + expect(isLocalConnection({ type: "postgres", host: "localhost" })).toBe(true) + expect(isLocalConnection({ type: "postgresql", host: "127.0.0.1" })).toBe(true) + expect(isLocalConnection({ type: "postgres", host: "/var/run/postgresql" })).toBe(true) + expect(isLocalConnection({ type: "mongodb", connection_string: "mongodb://localhost:27017" })).toBe(true) + expect(isLocalConnection({ type: "postgres", host: "db.example.com" })).toBe(false) + expect(isLocalConnection({ type: "mongodb", connection_string: "mongodb+srv://cluster.example.net" })).toBe(false) + expect(isLocalConnection({ type: "postgres" })).toBe(false) + }) + + test("an SSH-tunneled connection is never local, even with a loopback host", () => { + expect(isLocalConnection({ type: "postgres", host: "localhost", ssh_host: "bastion.example.com" })).toBe(false) + expect(isLocalConnection({ type: "sqlite", path: "/data/app.sqlite", ssh_host: "bastion.example.com" })).toBe(false) + }) + + test("other engines are never local", () => { + expect(isLocalConnection({ type: "snowflake", host: "localhost" })).toBe(false) + expect(isLocalConnection({ type: "bigquery" })).toBe(false) + expect(isLocalConnection(undefined)).toBe(false) + }) +}) + +for (const type of ["sqlite", "duckdb"]) { + test(`${type}: bounded sample has no added timeout and retains caller cancellation`, async () => { + Registry.setConfigs({ local: { type, path: ":memory:" } }) + const caller = new AbortController() + let relayedCancellation = false + const execute = mock(async (_sql: string, _limit?: number, _binds?: unknown[], options?: ExecuteOptions) => { + caller.abort() + relayedCancellation = options?.signal?.aborted === true + return { rows: [[7]], columns: ["id"], row_count: 1, truncated: false } + }) + const get = spyOn(Registry, "get").mockResolvedValue({ execute } as unknown as Connector) + const call = spyOn(Dispatcher, "call").mockImplementation(async (method: string, params: any) => { + if (method === "schema.inspect") return { columns: [{ name: "id", data_type: "INTEGER" }] } as any + return params.schema_name + ? { tables: [{ schema: "main", name: "items", type: "table" }] } as any + : { schemas: [{ name: "main", table_count: 1 }] } as any + }) + const timeout = spyOn(AbortSignal, "timeout") + try { + await discoverCatalog([{ name: "local", type }], { signal: caller.signal, sessionID: "sample-test" }).catch(() => {}) + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0].slice(0, 3)).toEqual(['SELECT * FROM "main"."items" LIMIT 1', 1, undefined]) + expect(relayedCancellation).toBe(true) + expect(execute.mock.calls[0][3]?.queryTimeoutMs).toBe(resolveQueryTimeout(Registry.getConfig("local")).ms) + expect(execute.mock.calls[0][3]?.signal?.aborted).toBe(true) + expect(timeout).not.toHaveBeenCalled() + } finally { + timeout.mockRestore() + call.mockRestore() + get.mockRestore() + Registry.reset() + } + }) +} + +for (const type of ["sqlite", "duckdb"]) { + for (const mode of ["table", "view", "over-limit"]) { + test(`${type}: discovery ${mode} samples only base tables within the catalog gate`, async () => { + Registry.setConfigs({ local: { type, path: ":memory:" } }) + const execute = mock(async (_sql: string, _limit?: number, _binds?: unknown[], _options?: ExecuteOptions) => ({ + rows: [[7]], columns: ["id"], row_count: 1, truncated: false, + })) + const get = spyOn(Registry, "get").mockResolvedValue({ execute } as unknown as Connector) + const call = spyOn(Dispatcher, "call").mockImplementation(async (method: string, params: any) => { + if (method === "schema.inspect") return { columns: [{ name: "id", data_type: "INTEGER" }] } as any + return params.schema_name + ? { tables: [{ schema: "main", name: "items", type: mode }] } as any + : { schemas: [{ name: "main", table_count: mode === "over-limit" ? 41 : 1 }] } as any + }) + const timeout = spyOn(AbortSignal, "timeout") + try { + const result = await discoverCatalog([{ name: "local", type }], { + signal: new AbortController().signal, sessionID: "sample-test", + }) + expect(timeout).not.toHaveBeenCalled() + if (mode === "table") { + expect(result).toContain("local: main.items: id INTEGER") + expect(result).toContain("e.g. 7") + expect(execute).toHaveBeenCalledTimes(1) + expect(execute.mock.calls[0][3]?.queryTimeoutMs).toBe(resolveQueryTimeout(Registry.getConfig("local")).ms) + expect(execute.mock.calls[0][3]?.signal?.aborted).toBe(false) + } else { + expect(execute).not.toHaveBeenCalled() + expect(result).not.toContain("e.g.") + if (mode === "over-limit") { + expect(result).toContain("catalog too large") + expect(call).toHaveBeenCalledTimes(1) + } + } + } finally { + timeout.mockRestore() + call.mockRestore() + get.mockRestore() + Registry.reset() + } + }) + } +} diff --git a/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap b/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap index 6c6eecf454..7cef008fca 100644 --- a/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap +++ b/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap @@ -34,6 +34,7 @@ manage MCP (Model Context Protocol) servers Commands: altimate-code mcp add add an MCP server altimate-code mcp list list MCP servers and their status [aliases: ls] + altimate-code mcp serve serve data tools over MCP stdio (no model required) altimate-code mcp status show MCP server health altimate-code mcp auth [name] authenticate with an OAuth-enabled MCP server altimate-code mcp logout [name] remove OAuth credentials for an MCP server