diff --git a/packages/opencode/src/altimate/tools/answer-commit-evidence.ts b/packages/opencode/src/altimate/tools/answer-commit-evidence.ts new file mode 100644 index 0000000000..616c8ecff9 --- /dev/null +++ b/packages/opencode/src/altimate/tools/answer-commit-evidence.ts @@ -0,0 +1,445 @@ +// altimate_change - new file +import { isLocalFilePath, type Connector } from "@altimateai/drivers" +import type { Tool } from "../../tool/tool" +import * as Registry from "../native/connections/registry" +import { isScratch, withScratchSession } from "../native/connections/scratch" + +/** + * Evidence that answer_commit gathers before accepting an answer: requested filters that removed + * nothing, and alternative readings whose queries disagree. Every check re-runs bounded queries on + * local DuckDB only (the session scratch database or a local DuckDB file) and never calls a model. + */ + +/** Whole budget for each check. Past it no further query is sent and the commit proceeds unchanged. */ +export const EVIDENCE_BUDGET_MS = 250 +/** + * Every base table a re-run reads must be a local DuckDB table with at most this many rows + * (catalog estimate). Local queries are never interrupted (that restarts the engine and drops the + * session's scratch tables), so the ceiling and the row cap below keep each query short. + */ +export const LOCAL_ROW_CEILING = 500_000 +/** Row counts stop at this many joined rows; a larger count cannot be compared and skips the check. */ +export const COUNT_CAP = 1_000_000 +/** More top-level filters than this skips the vacuous-filter check. */ +export const MAX_CONJUNCTS = 6 +/** Readings are compared on at most this many result rows; larger results are not compared. */ +export const MAX_READING_ROWS = 200 +/** Marker that identifies this check's one-time rejection in the current turn. */ +export const REVIEW_MARKER = "answer_commit review:" + +/** `ALTIMATE_SQL_AUTO_CHECK=off` disables automatic checks; they otherwise run on local DuckDB only. */ +export function autoCheckEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + const value = env["ALTIMATE_SQL_AUTO_CHECK"]?.trim().toLowerCase() + return value !== "off" && value !== "0" && value !== "false" +} + +type Json = any +type Reading = { clause: string; reading: string; sql?: string; warehouse?: string } + +export type VacuousFilter = { filter: string; rows: number } +export type ReadingComparison = { compared: number; agree: boolean } +export type Evidence = { + vacuous: VacuousFilter[] + readings?: ReadingComparison + alternatives: string[] +} + +type LoosePart = { type?: string; tool?: string; state?: { status?: string; input?: Json; error?: unknown } } + +/** Messages of the current user turn: everything after the latest user message, plus that message. */ +function currentTurn(messages: Tool.Context["messages"]) { + const start = messages.findLastIndex((message) => message.info.role === "user") + return start < 0 ? messages : messages.slice(start) +} + +function turnParts(messages: Tool.Context["messages"]): LoosePart[] { + return currentTurn(messages).flatMap((message) => message.parts as LoosePart[]) +} + +/** Earlier review rejections from this check in the current turn. */ +export function priorReviews(messages: Tool.Context["messages"]): number { + return turnParts(messages).filter( + (part) => + part.type === "tool" && + part.tool === "answer_commit" && + part.state?.status === "error" && + String(part.state.error ?? "").includes(REVIEW_MARKER), + ).length +} + +/** The latest completed plain `sql_execute` query of the current turn, with its warehouse. */ +export function lastQuery(messages: Tool.Context["messages"]): { sql: string; warehouse: string } | undefined { + const part = turnParts(messages).findLast( + (part) => + part.type === "tool" && + part.tool === "sql_execute" && + part.state?.status === "completed" && + typeof part.state.input?.query === "string" && + typeof part.state.input?.warehouse === "string", + ) + return part ? { sql: part.state!.input.query, warehouse: part.state!.input.warehouse } : undefined +} + +const literal = (text: string) => `'${text.replace(/'/g, "''")}'` +/** A DuckDB path that names remote or extension storage; in-memory, empty and `file:` paths are local. */ +const isRemotePath = (path: unknown) => + typeof path === "string" && path !== "" && path !== ":memory:" && !/^file:/i.test(path) && !isLocalFilePath(path) + +/** True when the warehouse is the session scratch database or a local DuckDB file. */ +function localDuckdb(warehouse: string): boolean { + if (isScratch(warehouse)) return true + const config = Registry.getConfig(warehouse) + return ( + Registry.canonicalType(config?.type) === "duckdb" && !isRemotePath((config as { path?: unknown } | undefined)?.path) + ) +} + +/** + * A deadline-aware query runner. Once the budget has passed it refuses to send another query; + * queries already running are never interrupted (that would restart the local engine). + */ +type Run = (sql: string) => Promise[]> +function runner(connector: Connector, deadline: number): Run { + return async (sql) => { + if (Date.now() >= deadline) throw new Error("evidence budget spent") + const response = await connector.execute(sql, undefined, undefined, { noLimit: true, readOnly: true }) + return response.rows.map((row: unknown[]) => + Object.fromEntries(response.columns.map((col: string, i: number) => [col, row[i]])), + ) + } +} + +/** Runs the work within the budget; any error, refusal or overrun yields undefined. */ +async function withinBudget(work: (deadline: number) => Promise): Promise { + const deadline = Date.now() + EVIDENCE_BUDGET_MS + let timer: ReturnType | undefined + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), EVIDENCE_BUDGET_MS) + }) + try { + return await Promise.race([work(deadline).catch(() => undefined), timeout]) + } finally { + clearTimeout(timer) + } +} + +async function serialize(run: Run, sql: string): Promise { + const [parsed] = await run(`SELECT json_serialize_sql(${literal(sql)}) AS j`) + // uint64 locations lose precision in JSON.parse; they are not needed. + const statement = JSON.parse(String(parsed?.j ?? "").replace(/"query_location":\s*\d+/g, '"query_location":0')) + if (statement?.error || statement?.statements?.length !== 1) return undefined + return statement +} + +async function deserialize(run: Run, statement: Json): Promise { + const [row] = await run(`SELECT json_deserialize_sql(${literal(JSON.stringify(statement))}) AS s`) + return typeof row?.s === "string" && row.s ? row.s : undefined +} + +/** Every node of a parsed statement, depth first. */ +function* nodes(value: Json): Generator { + if (!value || typeof value !== "object") return + yield value + for (const child of Array.isArray(value) ? value : Object.values(value)) yield* nodes(child) +} + +const isEquality = (n: Json) => + n?.type === "COMPARE_EQUAL" && n.left?.class === "COLUMN_REF" && n.right?.class === "COLUMN_REF" +/** A join condition with a column equality, so DuckDB hash-joins it and the row cap stops the work. */ +const equiJoin = (condition: Json) => + isEquality(condition) || (condition?.type === "CONJUNCTION_AND" && condition.children.some(isEquality)) + +type PlainSelect = { node: Json; tables: Json[]; functions: Set } +/** + * The single SELECT of a statement whose cost the checks can bound, with its base tables and called + * functions; undefined for anything else: set operations, CTEs, subqueries, table functions (which + * can read files) and joins without a column equality (a nested-loop join is not bounded by the cap). + */ +function plainSelect(statement: Json): PlainSelect | undefined { + const node = statement?.statements?.[0]?.node + if (node?.type !== "SELECT_NODE") return undefined + const tables: Json[] = [] + const functions = new Set() + for (const n of nodes(node)) { + if (n?.cte_map?.map?.length) return undefined + if (n?.class === "SUBQUERY" || n?.type === "SUBQUERY" || n?.type === "TABLE_FUNCTION") return undefined + if (n?.type === "JOIN" && (n.ref_type !== "REGULAR" || !equiJoin(n.condition))) return undefined + if (n?.type === "BASE_TABLE") tables.push(n) + if (n?.class === "FUNCTION" && !n.is_operator) functions.add(String(n.function_name).toLowerCase()) + } + return tables.length ? { node, tables, functions } : undefined +} + +/** The relation resolves to exactly one local DuckDB base table under the ceiling, and no view shares its name. */ +async function smallLocalTable(run: Run, side: Json): Promise { + const name = literal(String(side.table_name)) + const scope = [ + side.schema_name ? `AND lower(schema_name) = lower(${literal(side.schema_name)})` : "", + side.catalog_name ? `AND lower(database_name) = lower(${literal(side.catalog_name)})` : "", + ].join(" ") + const [views] = await run( + `SELECT COUNT(*) AS n FROM duckdb_views() WHERE NOT internal AND lower(view_name) = lower(${name})`, + ) + if (Number(views?.n) !== 0) return false + const tables = await run( + `SELECT d.type AS type, d.path AS path, t.estimated_size AS size + FROM duckdb_tables() t JOIN duckdb_databases() d USING (database_name) + WHERE lower(t.table_name) = lower(${name}) ${scope}`, + ) + const [table] = tables + return ( + tables.length === 1 && + String(table.type).toLowerCase() === "duckdb" && + !isRemotePath(table.path) && + Number(table.size) <= LOCAL_ROW_CEILING + ) +} + +/** + * Every table is small and local, and every function is built in: a user macro can hide a large + * read or an attached remote warehouse behind a function call. + */ +async function bounded(run: Run, select: PlainSelect): Promise { + if (select.functions.size) { + const names = [...select.functions].map(literal).join(", ") + const [user] = await run( + `SELECT COUNT(*) AS n FROM duckdb_functions() WHERE NOT internal AND lower(function_name) IN (${names})`, + ) + if (Number(user?.n) !== 0) return false + } + for (const table of select.tables) if (!(await smallLocalTable(run, table))) return false + return true +} + +const and = (filters: Json[]) => + !filters.length + ? null + : filters.length === 1 + ? filters[0] + : { class: "CONJUNCTION", type: "CONJUNCTION_AND", alias: "", query_location: 0, children: filters } + +/** + * Counts the rows of the query's FROM under the given filters, stopping at COUNT_CAP + 1 rows so a + * large join cannot run long. Grouping, HAVING, DISTINCT, ORDER and LIMIT are dropped: the count is + * of the rows the filters keep. Undefined at or above the cap. + */ +async function cappedCount(run: Run, statement: Json, node: Json, filters: Json[]): Promise { + const [one] = (await serialize(run, "SELECT 1 AS __one"))?.statements[0]?.node?.select_list ?? [] + if (!one) return undefined + const inner = { + ...statement, + statements: [ + { + ...statement.statements[0], + node: { + ...node, + select_list: [one], + where_clause: and(filters), + group_expressions: [], + group_sets: [], + having: null, + qualify: null, + modifiers: [], + }, + }, + ], + } + const text = await deserialize(run, inner) + if (!text) return undefined + const [row] = await run(`SELECT COUNT(*) AS __n FROM (${text} LIMIT ${COUNT_CAP + 1}) __capped`) + const n = Number(row?.__n) + return Number.isFinite(n) && n <= COUNT_CAP ? n : undefined +} + +const isNotNull = (conjunct: Json) => conjunct?.class === "OPERATOR" && conjunct.type === "OPERATOR_IS_NOT_NULL" + +/** + * Re-counts the final query's rows with each top-level WHERE filter removed in turn and returns + * the filters whose removal keeps the same rows: they excluded nothing, which usually means they + * target a different column than the question means. `IS NOT NULL` guards are skipped. + */ +export async function vacuousFilters(sql: string, warehouse: string): Promise { + if (!localDuckdb(warehouse) || !/\bwhere\b/i.test(sql)) return undefined + return withinBudget(async (deadline) => { + const run = runner(await Registry.get(warehouse), deadline) + const statement = await serialize(run, sql) + const select = plainSelect(statement) + if (!select?.node.where_clause) return undefined + const where = select.node.where_clause + const conjuncts: Json[] = where.type === "CONJUNCTION_AND" ? where.children : [where] + if (conjuncts.length > MAX_CONJUNCTS || !(await bounded(run, select))) return undefined + const kept = await cappedCount(run, statement, select.node, conjuncts) + if (kept === undefined || kept === 0) return undefined + const found: VacuousFilter[] = [] + for (const [index, conjunct] of conjuncts.entries()) { + if (isNotNull(conjunct)) continue + const without = await cappedCount( + run, + statement, + select.node, + conjuncts.filter((_, i) => i !== index), + ) + if (without !== kept) continue + const text = await deserialize(run, { + ...statement, + statements: [{ ...statement.statements[0], node: { ...select.node, where_clause: conjunct, modifiers: [] } }], + }) + const at = text?.search(/\bWHERE\b/i) ?? -1 + const filter = at >= 0 ? text!.slice(at + "WHERE".length).trim() : "" + if (filter) found.push({ filter, rows: kept }) + } + return found + }) +} + +/** A cell with its type kept, so NULL, `'null'`, `1` and `'1'` stay distinct. */ +function canonical(value: unknown): string { + if (value === null || value === undefined) return "null" + if (typeof value === "bigint") return `n:${value.toString()}` + if (typeof value === "number") return `n:${value}` + if (typeof value === "string") return `s:${value}` + if (typeof value === "boolean") return `b:${value}` + if (value instanceof Date) return `d:${value.toISOString()}` + return `o:${JSON.stringify(value, (_, v) => (typeof v === "bigint" ? v.toString() : v))}` +} + +/** + * Executes each reading's SQL (bounded: plain SELECT on local DuckDB, small base tables, at most + * COUNT_CAP joined rows and MAX_READING_ROWS result rows) and compares the results as unordered row + * sets. Undefined when fewer than two readings carry SQL or any of them cannot run under these limits. + */ +export async function compareReadings( + readings: Reading[] | undefined, + fallbackWarehouse: string | undefined, +): Promise { + const runnable = (readings ?? []).filter((r) => typeof r.sql === "string" && r.sql.trim()) + if (runnable.length < 2) return undefined + return withinBudget(async (deadline) => { + const results: string[] = [] + for (const reading of runnable) { + const warehouse = reading.warehouse || fallbackWarehouse + if (!warehouse || !localDuckdb(warehouse)) return undefined + const run = runner(await Registry.get(warehouse), deadline) + const statement = await serialize(run, reading.sql!) + const select = plainSelect(statement) + if (!select || !(await bounded(run, select))) return undefined + const where = select.node.where_clause + const filters: Json[] = !where ? [] : where.type === "CONJUNCTION_AND" ? where.children : [where] + // Bound the rows the reading aggregates before running it. + if ((await cappedCount(run, statement, select.node, filters)) === undefined) return undefined + const text = await deserialize(run, statement) + if (!text) return undefined + const result = await run(`SELECT * FROM (${text}) __reading LIMIT ${MAX_READING_ROWS + 1}`) + if (result.length > MAX_READING_ROWS) return undefined + results.push(JSON.stringify(result.map((row) => Object.values(row).map(canonical).join("\u0001")).sort())) + } + return { compared: results.length, agree: results.every((result) => result === results[0]) } + }) +} + +const DATE_LIKE = /(^|_)(date|time|timestamp|day|month|year|period|week|quarter)s?($|_)|_(at|on)$/i +const STATUS_LIKE = /(^|_)(status|state|stage|phase|type|category|flag|active|is)($|_)/i +const kind = (name: string, type: string) => + /^(date|timestamp|time)/i.test(type) || DATE_LIKE.test(name) ? "date" : STATUS_LIKE.test(name) ? "status" : undefined + +/** + * Columns of the same tables that could be meant instead of the date-like or status-like columns the + * final query filters or groups by. Deterministic from the catalog; no data is read. + */ +export async function alternativeColumns(sql: string, warehouse: string): Promise { + if (!localDuckdb(warehouse)) return undefined + return withinBudget(async (deadline) => { + const run = runner(await Registry.get(warehouse), deadline) + const select = plainSelect(await serialize(run, sql)) + if (!select || select.tables.length > 4) return undefined + const used = new Set() + for (const n of nodes([select.node.where_clause, select.node.group_expressions])) + if (n?.class === "COLUMN_REF") used.add(String(n.column_names.at(-1)).toLowerCase()) + if (!used.size) return [] + const names = select.tables.map((t) => literal(String(t.table_name).toLowerCase())).join(", ") + const columns = await run( + `SELECT table_name AS t, column_name AS c, data_type AS d FROM duckdb_columns() + WHERE lower(table_name) IN (${names}) ORDER BY table_name, column_index`, + ) + const lines: string[] = [] + for (const table of new Set(columns.map((c) => String(c.t)))) { + const own = columns.filter((c) => String(c.t) === table) + for (const group of ["date", "status"] as const) { + const members = own.filter((c) => kind(String(c.c), String(c.d)) === group).map((c) => String(c.c)) + const chosen = members.filter((c) => used.has(c.toLowerCase())) + const others = members.filter((c) => !used.has(c.toLowerCase())) + if (chosen.length && others.length) + lines.push(`${table}: uses ${chosen.join(", ")}; other ${group}-like columns: ${others.join(", ")}`) + } + } + return lines + }) +} + +/** + * Synchronous gate: true only when the checks are on, no review ran yet in this turn, and there is + * something to re-run (a final local query, or at least two readings with SQL). Commits that fail + * the gate stay fully synchronous. + */ +export function needsEvidence( + messages: Tool.Context["messages"], + readings: Reading[] | undefined, + env: NodeJS.ProcessEnv = process.env, +): boolean { + if (!autoCheckEnabled(env) || priorReviews(messages) > 0) return false + const query = lastQuery(messages) + if (query && localDuckdb(query.warehouse)) return true + return (readings ?? []).filter((r) => typeof r.sql === "string" && r.sql.trim()).length >= 2 +} + +/** + * Gathers the evidence for a commit, with the scratch warehouse bound to the committing session. + * Undefined when the checks are off or nothing could be checked. + */ +export function gatherEvidence( + sessionID: string, + messages: Tool.Context["messages"], + readings: Reading[] | undefined, + env: NodeJS.ProcessEnv = process.env, +): Promise { + return withScratchSession(sessionID, () => evidenceFor(messages, readings, env)) +} + +async function evidenceFor( + messages: Tool.Context["messages"], + readings: Reading[] | undefined, + env: NodeJS.ProcessEnv, +): Promise { + if (!autoCheckEnabled(env)) return undefined + const query = lastQuery(messages) + const vacuous = query ? ((await vacuousFilters(query.sql, query.warehouse)) ?? []) : [] + const comparison = await compareReadings(readings, query?.warehouse) + if (!vacuous.length && !comparison) return undefined + const findings = vacuous.length > 0 || comparison?.agree === false + const alternatives = findings && query ? ((await alternativeColumns(query.sql, query.warehouse)) ?? []) : [] + return { vacuous, readings: comparison, alternatives } +} + +/** The one-time review text, or undefined when the evidence needs no second look. */ +export function reviewMessage(evidence: Evidence | undefined, readingEvidence: string | undefined): string | undefined { + if (!evidence) return undefined + const disagree = evidence.readings?.agree === false && !readingEvidence?.trim() + if (!evidence.vacuous.length && !disagree) return undefined + const lines = [`${REVIEW_MARKER} not committed yet; check this evidence once.`] + for (const v of evidence.vacuous) + lines.push( + `- Filter ${v.filter} removed 0 of the ${v.rows} rows the other filters keep. Confirm it targets the column the question means; a filter that excludes nothing often sits on the wrong column.`, + ) + if (disagree) + lines.push( + `- The ${evidence.readings!.compared} readings you listed return different results. Choose with data, not wording, and state the deciding fact in reading_evidence.`, + ) + if (evidence.alternatives.length) { + lines.push("- Columns that could be meant instead:") + for (const alternative of evidence.alternatives) lines.push(` ${alternative}`) + } + lines.push( + "If the answer stands, call answer_commit again with the same answer (add reading_evidence when readings disagree); otherwise revise the query first.", + ) + return lines.join("\n") +} diff --git a/packages/opencode/src/altimate/tools/answer-commit.ts b/packages/opencode/src/altimate/tools/answer-commit.ts index baf76f3059..dd466256c6 100644 --- a/packages/opencode/src/altimate/tools/answer-commit.ts +++ b/packages/opencode/src/altimate/tools/answer-commit.ts @@ -3,6 +3,9 @@ import { Decimal } from "decimal.js" import { Effect, Option, Schema, SchemaIssue } from "effect" import type { JSONSchema7 } from "@ai-sdk/provider" import { Tool } from "../../tool/tool" +// altimate_change start — bounded re-run evidence for commits +import { gatherEvidence, needsEvidence, reviewMessage } from "./answer-commit-evidence" +// altimate_change end /** Maximum rendered answer length, including Markdown and basis, before the model echoes it. */ export const MAX_RENDERED_ANSWER_CHARACTERS = 60_000 @@ -138,10 +141,21 @@ const parameters = z reading: z.string(), result_summary: z.string(), decided_by: z.string(), + // altimate_change start — a reading may carry the query that implements it, so readings can be compared on data + sql: optional(z.string()).describe( + "Optional read-only SELECT that implements this reading; readings with SQL on local DuckDB are executed and compared", + ), + warehouse: optional(z.string()).describe("Warehouse for sql; omit to use the warehouse of the last query"), + // altimate_change end }) .strict(), ), ).describe("Advisory readings recorded only in metadata, never in the answer text"), + // altimate_change start — the data fact that decides between readings whose results differ + reading_evidence: optional(z.string().max(MAX_BASIS_CHARACTERS)).describe( + `When listed readings return different results, the data fact that decides between them, at most ${MAX_BASIS_CHARACTERS} characters; never rendered`, + ), + // altimate_change end }) .strict() .refine( @@ -304,10 +318,26 @@ export const AnswerCommitTool = Tool.define( return yield* Effect.die(new Error("answer_commit is only available to the analyst agent")) yield* ctx.ask({ permission: "answer_commit", patterns: ["*"], always: ["*"], metadata: {} }) const output = renderAnswer(args) + // altimate_change start — one-time review when re-run evidence contradicts the answer (local DuckDB only) + const messages = ctx.messages ?? [] + const evidence = needsEvidence(messages, args.readings) + ? yield* Effect.promise(() => gatherEvidence(ctx.sessionID, messages, args.readings)) + : undefined + const review = reviewMessage(evidence, args.reading_evidence) + if (review) { + const rejected: Tool.ExecuteResult = { + title: "Answer not committed", + metadata: { readings: args.readings ?? [], truncated: false, evidence }, + output: review, + status: "error", + } + return rejected + } + // altimate_change end return { title: "Answer committed", // The processor caps model-facing output but preserves metadata for the final echo. - metadata: { readings: args.readings ?? [], truncated: false, committed_text: output }, + metadata: { readings: args.readings ?? [], truncated: false, committed_text: output, evidence }, output, } }), diff --git a/packages/opencode/test/altimate/tools/answer-commit-evidence.test.ts b/packages/opencode/test/altimate/tools/answer-commit-evidence.test.ts new file mode 100644 index 0000000000..fc2aa37a58 --- /dev/null +++ b/packages/opencode/test/altimate/tools/answer-commit-evidence.test.ts @@ -0,0 +1,346 @@ +// altimate_change start — bounded re-run evidence for answer_commit, against real scratch DuckDB and SQLite +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test" +import { Database } from "bun:sqlite" +import path from "path" +import { tmpdir } from "../../fixture/fixture" +import { initTool } from "../tool-fixture" +import * as Registry from "../../../src/altimate/native/connections/registry" +import { closeScratch, getScratch, withScratchSession } from "../../../src/altimate/native/connections/scratch" +import { AnswerCommit } from "../../../src/altimate/tools/answer-commit" +import { + alternativeColumns, + compareReadings, + gatherEvidence, + REVIEW_MARKER, + vacuousFilters, +} from "../../../src/altimate/tools/answer-commit-evidence" + +const setup = [ + "CREATE TABLE orders (id INTEGER, status TEXT, region TEXT, note TEXT, created_at DATE, shipped_at DATE)", + `INSERT INTO orders VALUES + (1, 'open', 'EU', 'a', DATE '2023-04-02', DATE '2023-05-01'), + (2, 'open', 'EU', 'b', DATE '2023-04-15', DATE '2023-04-20'), + (3, 'closed', 'EU', 'c', DATE '2023-03-30', DATE '2023-04-03'), + (4, 'open', 'EU', NULL, DATE '2023-05-10', DATE '2023-05-12')`, + "CREATE TABLE customers (id INTEGER, name TEXT)", + "INSERT INTO customers VALUES (1, 'x'), (2, 'y'), (3, 'z'), (4, 'w')", + "CREATE TABLE big AS SELECT range AS id FROM range(600000)", + "CREATE TABLE wide AS SELECT range AS id FROM range(2000)", +] + +let fixture: Awaited> +let sqlitePath: string +let previousFlag: string | undefined + +beforeAll(async () => { + fixture = await tmpdir() + sqlitePath = path.join(fixture.path, "orders.sqlite") + const sqlite = new Database(sqlitePath) + sqlite.run("CREATE TABLE orders (id INTEGER, region TEXT)") + sqlite.run("INSERT INTO orders VALUES (1, 'EU')") + sqlite.close() +}) + +beforeEach(async () => { + previousFlag = process.env.ALTIMATE_SQL_AUTO_CHECK + delete process.env.ALTIMATE_SQL_AUTO_CHECK + Registry.setConfigs({ sqlite_orders: { type: "sqlite", path: sqlitePath } }) + await closeScratch() + // Only the committing session's scratch store holds the tables, so a check that reads another store finds nothing. + const duckdb = await getScratch("test-session") + for (const sql of setup) await duckdb.execute(sql, undefined, undefined, { noLimit: true }) +}) + +afterEach(async () => { + if (previousFlag === undefined) delete process.env.ALTIMATE_SQL_AUTO_CHECK + else process.env.ALTIMATE_SQL_AUTO_CHECK = previousFlag + await Registry.closeAll() +}) + +afterAll(async () => { + await closeScratch() + await fixture?.[Symbol.asyncDispose]() +}) + +const queryPart = (query: string, warehouse = "scratch") => ({ + type: "tool", + tool: "sql_execute", + state: { status: "completed", input: { query, warehouse }, output: "ok" }, +}) +const turn = (...parts: unknown[]) => + [ + { info: { role: "user" }, parts: [] }, + { info: { role: "assistant" }, parts }, + ] as never + +/** Direct helper calls run inside the session, as answer_commit runs them. */ +const inSession = (name: string, body: () => Promise) => + test(name, () => withScratchSession("test-session", body)) + +const basis = "Counted orders at order grain using orders.id." +const context = (messages: unknown) => ({ + sessionID: "test-session", + messageID: "test-message", + agent: "analyst", + abort: new AbortController().signal, + messages, +}) + +describe("vacuous filters", () => { + inSession("flags a filter that removes no rows and keeps the effective one", async () => { + const found = await vacuousFilters("SELECT COUNT(*) FROM orders WHERE status = 'open' AND region = 'EU'", "scratch") + expect(found).toHaveLength(1) + expect(found![0]!.filter).toContain("region") + expect(found![0]!.rows).toBe(3) + }) + + inSession("reports nothing when every filter removes rows", async () => { + expect(await vacuousFilters("SELECT id FROM orders WHERE status = 'open' AND id < 4", "scratch")).toEqual([]) + }) + + inSession("skips IS NOT NULL guards and counts before GROUP BY", async () => { + expect( + await vacuousFilters( + "SELECT status, COUNT(*) FROM orders WHERE id IS NOT NULL AND id < 3 GROUP BY status", + "scratch", + ), + ).toEqual([]) + }) + + inSession("skips SQLite, large tables, subqueries and joins without a condition", async () => { + expect(await vacuousFilters("SELECT * FROM orders WHERE region = 'EU'", "sqlite_orders")).toBeUndefined() + expect(await vacuousFilters("SELECT COUNT(*) FROM big WHERE id >= 0", "scratch")).toBeUndefined() + expect( + await vacuousFilters("SELECT * FROM orders WHERE id IN (SELECT id FROM customers) AND region = 'EU'", "scratch"), + ).toBeUndefined() + expect( + await vacuousFilters("SELECT * FROM orders o, customers c WHERE o.id = c.id AND o.region = 'EU'", "scratch"), + ).toBeUndefined() + expect( + await vacuousFilters( + "WITH huge AS (SELECT * FROM orders) SELECT COUNT(*) FROM huge WHERE status = 'open' AND region = 'EU'", + "scratch", + ), + ).toBeUndefined() + }) + + inSession("skips joins the row cap cannot bound", async () => { + // A join without a column equality runs as a nested loop that the cap does not stop. + expect( + await vacuousFilters("SELECT COUNT(*) FROM wide a JOIN wide b ON a.id <= b.id WHERE a.id >= 0", "scratch"), + ).toBeUndefined() + // An equi-join that passes the cap: 2000 x 2000 rows on the constant key. + expect( + await vacuousFilters( + "SELECT COUNT(*) FROM wide a JOIN wide b ON a.id % 1 = b.id % 1 AND a.id = a.id WHERE a.id >= 0", + "scratch", + ), + ).toBeUndefined() + }) + + inSession("skips queries that call a user macro", async () => { + const duckdb = await getScratch("test-session") + await duckdb.execute("CREATE MACRO hidden(x) AS x", undefined, undefined, { noLimit: true }) + expect( + await vacuousFilters("SELECT COUNT(*) FROM orders WHERE hidden(status) = 'open' AND region = 'EU'", "scratch"), + ).toBeUndefined() + expect( + await compareReadings( + [ + { clause: "a", reading: "b", sql: "SELECT hidden(id) FROM orders WHERE id = 1" }, + { clause: "a", reading: "c", sql: "SELECT id FROM orders WHERE id = 1" }, + ], + "scratch", + ), + ).toBeUndefined() + expect( + await vacuousFilters("SELECT COUNT(*) FROM orders WHERE upper(status) = 'OPEN' AND region = 'EU'", "scratch"), + ).toHaveLength(1) + }) + + inSession("checks an inner join with a condition", async () => { + const found = await vacuousFilters( + "SELECT c.name FROM orders o JOIN customers c ON o.id = c.id WHERE o.region = 'EU' AND o.status = 'open'", + "scratch", + ) + expect(found?.map((f) => f.filter.includes("region"))).toEqual([true]) + }) +}) + +describe("readings", () => { + inSession("compares readings with SQL as unordered row sets", async () => { + const agree = await compareReadings( + [ + { clause: "open", reading: "status", sql: "SELECT id FROM orders WHERE status = 'open' ORDER BY id" }, + { clause: "open", reading: "status desc", sql: "SELECT id FROM orders WHERE status = 'open' ORDER BY id DESC" }, + ], + "scratch", + ) + expect(agree).toEqual({ compared: 2, agree: true }) + const disagree = await compareReadings( + [ + { clause: "April", reading: "created", sql: "SELECT id FROM orders WHERE month(created_at) = 4" }, + { clause: "April", reading: "shipped", sql: "SELECT id FROM orders WHERE month(shipped_at) = 4" }, + ], + "scratch", + ) + expect(disagree).toEqual({ compared: 2, agree: false }) + }) + + inSession("keeps NULL, text and numbers distinct", async () => { + const pair = (a: string, b: string) => + compareReadings( + [ + { clause: "x", reading: "a", sql: `SELECT ${a} AS v FROM orders WHERE id = 1` }, + { clause: "x", reading: "b", sql: `SELECT ${b} AS v FROM orders WHERE id = 1` }, + ], + "scratch", + ) + expect(await pair("CAST(NULL AS TEXT)", "'null'")).toEqual({ compared: 2, agree: false }) + expect(await pair("1", "'1'")).toEqual({ compared: 2, agree: false }) + }) + + inSession("does not compare fewer than two runnable readings or unsafe ones", async () => { + expect(await compareReadings([{ clause: "a", reading: "b", sql: "SELECT 1" }], "scratch")).toBeUndefined() + expect( + await compareReadings( + [ + { clause: "a", reading: "b", sql: "SELECT COUNT(*) FROM big" }, + { clause: "a", reading: "c", sql: "SELECT COUNT(*) FROM orders" }, + ], + "scratch", + ), + ).toBeUndefined() + expect( + await compareReadings( + [ + { clause: "a", reading: "b", sql: "SELECT * FROM orders", warehouse: "sqlite_orders" }, + { clause: "a", reading: "c", sql: "SELECT * FROM orders", warehouse: "sqlite_orders" }, + ], + "scratch", + ), + ).toBeUndefined() + }) + + inSession("lists other date-like columns of the tables the query filters", async () => { + const lines = await alternativeColumns("SELECT COUNT(*) FROM orders WHERE month(created_at) = 4", "scratch") + expect(lines).toEqual(["orders: uses created_at; other date-like columns: shipped_at"]) + }) +}) + +describe("answer_commit review", () => { + const commit = { fields: [{ name: "", value: 3 }], basis } + + test("rejects once with the vacuous filter, then accepts the same answer", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const sql = "SELECT COUNT(*) FROM orders WHERE status = 'open' AND region = 'EU'" + const first = await tool.execute(commit, context(turn(queryPart(sql)))) + expect(first.status).toBe("error") + expect(first.output).toContain(REVIEW_MARKER) + expect(first.output).toContain("removed 0 of the 3 rows") + const rejected = { + type: "tool", + tool: "answer_commit", + state: { status: "error", input: commit, error: first.output }, + } + const second = await tool.execute(commit, context(turn(queryPart(sql), rejected))) + expect(second.status).not.toBe("error") + expect(second.output).toContain("**Answer:** 3") + }) + + test("commits directly when the filters all remove rows", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const result = await tool.execute( + { fields: [{ name: "", value: 2 }], basis }, + context(turn(queryPart("SELECT COUNT(*) FROM orders WHERE status = 'open' AND id < 4"))), + ) + expect(result.status).not.toBe("error") + }) + + test("asks for reading evidence when readings disagree, and accepts it", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const readings = [ + { + clause: "in April", + reading: "created in April", + result_summary: "2", + decided_by: "creation is when an order exists", + sql: "SELECT id FROM orders WHERE month(created_at) = 4", + }, + { + clause: "in April", + reading: "shipped in April", + result_summary: "2", + decided_by: "rejected", + sql: "SELECT id FROM orders WHERE month(shipped_at) = 4", + }, + ] + const messages = turn(queryPart("SELECT COUNT(*) FROM orders WHERE month(created_at) = 4")) + const asked = await tool.execute({ ...commit, readings }, context(messages)) + expect(asked.status).toBe("error") + expect(asked.output).toContain("reading_evidence") + expect(asked.output).toContain("other date-like columns: shipped_at") + const accepted = await tool.execute( + { ...commit, readings, reading_evidence: "Orders are counted by the month they were created." }, + context(messages), + ) + expect(accepted.status).not.toBe("error") + expect(accepted.metadata.evidence).toMatchObject({ readings: { compared: 2, agree: false } }) + }) + + test("records agreeing readings without a review", async () => { + const evidence = await gatherEvidence("test-session", turn(queryPart("SELECT id FROM orders WHERE id < 3")), [ + { clause: "a", reading: "b", sql: "SELECT id FROM orders WHERE id < 3" }, + { clause: "a", reading: "c", sql: "SELECT id FROM orders WHERE id IN (1, 2)" }, + ]) + expect(evidence?.readings).toEqual({ compared: 2, agree: true }) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const result = await tool.execute( + { + fields: [{ name: "", value: 2 }], + basis, + readings: [ + { + clause: "a", + reading: "b", + result_summary: "2", + decided_by: "x", + sql: "SELECT id FROM orders WHERE id < 3", + }, + { + clause: "a", + reading: "c", + result_summary: "2", + decided_by: "y", + sql: "SELECT id FROM orders WHERE id IN (1, 2)", + }, + ], + }, + context(turn(queryPart("SELECT id FROM orders WHERE id < 3"))), + ) + expect(result.status).not.toBe("error") + expect(result.metadata.evidence).toMatchObject({ readings: { agree: true } }) + }) + + test("reads the committing session's scratch store", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const sql = "SELECT COUNT(*) FROM orders WHERE status = 'open' AND region = 'EU'" + expect(await gatherEvidence("other-session", turn(queryPart(sql)), undefined)).toBeUndefined() + const other = await tool.execute(commit, { ...context(turn(queryPart(sql))), sessionID: "other-session" }) + expect(other.status).not.toBe("error") + }) + + test("does nothing when ALTIMATE_SQL_AUTO_CHECK=off or the query ran on SQLite", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const sql = "SELECT COUNT(*) FROM orders WHERE status = 'open' AND region = 'EU'" + process.env.ALTIMATE_SQL_AUTO_CHECK = "off" + expect((await tool.execute(commit, context(turn(queryPart(sql))))).status).not.toBe("error") + delete process.env.ALTIMATE_SQL_AUTO_CHECK + const sqlite = await tool.execute( + commit, + context(turn(queryPart("SELECT COUNT(*) FROM orders WHERE region = 'EU'", "sqlite_orders"))), + ) + expect(sqlite.status).not.toBe("error") + }) +}) +// altimate_change end diff --git a/packages/opencode/test/altimate/tools/answer-commit.test.ts b/packages/opencode/test/altimate/tools/answer-commit.test.ts index 23a43fe90e..17ee20399c 100644 --- a/packages/opencode/test/altimate/tools/answer-commit.test.ts +++ b/packages/opencode/test/altimate/tools/answer-commit.test.ts @@ -1135,6 +1135,9 @@ test("answer_commit exposes a serializable object input schema", async () => { "basis", "answer_only", "readings", + // altimate_change start — the data fact that decides between disagreeing readings + "reading_evidence", + // altimate_change end ]) })