From 4cc97c27a48cd128914f1004b44601849600dbc7 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Thu, 22 Oct 2026 13:12:15 +0100 Subject: [PATCH 01/10] feat(evals): add investigate-database-001-slow-query CLI eval (CLI-2712) --- .../EVAL.test.ts | 1257 +++++++++++++++++ .../EVAL.ts | 167 +++ .../PROMPT.md | 16 + .../README.md | 213 +++ .../fixtures.test.ts | 273 ++++ .../fixtures.ts | 14 + .../inspect.test.ts | 445 ++++++ .../inspect.ts | 88 ++ .../local/supabase/config.toml | 11 + .../20260101000000_create_orders.sql | 7 + .../local/supabase/seed.sql | 313 ++++ .../metrics.test.ts | 841 +++++++++++ .../metrics.ts | 192 +++ .../migrations.test.ts | 954 +++++++++++++ .../migrations.ts | 275 ++++ .../orders.test.ts | 824 +++++++++++ .../orders.ts | 224 +++ .../report.test.ts | 593 ++++++++ .../report.ts | 124 ++ 19 files changed, 6831 insertions(+) create mode 100644 evals/cli/investigate-database-001-slow-query/EVAL.test.ts create mode 100644 evals/cli/investigate-database-001-slow-query/EVAL.ts create mode 100644 evals/cli/investigate-database-001-slow-query/PROMPT.md create mode 100644 evals/cli/investigate-database-001-slow-query/README.md create mode 100644 evals/cli/investigate-database-001-slow-query/fixtures.test.ts create mode 100644 evals/cli/investigate-database-001-slow-query/fixtures.ts create mode 100644 evals/cli/investigate-database-001-slow-query/inspect.test.ts create mode 100644 evals/cli/investigate-database-001-slow-query/inspect.ts create mode 100644 evals/cli/investigate-database-001-slow-query/local/supabase/config.toml create mode 100644 evals/cli/investigate-database-001-slow-query/local/supabase/migrations/20260101000000_create_orders.sql create mode 100644 evals/cli/investigate-database-001-slow-query/local/supabase/seed.sql create mode 100644 evals/cli/investigate-database-001-slow-query/metrics.test.ts create mode 100644 evals/cli/investigate-database-001-slow-query/metrics.ts create mode 100644 evals/cli/investigate-database-001-slow-query/migrations.test.ts create mode 100644 evals/cli/investigate-database-001-slow-query/migrations.ts create mode 100644 evals/cli/investigate-database-001-slow-query/orders.test.ts create mode 100644 evals/cli/investigate-database-001-slow-query/orders.ts create mode 100644 evals/cli/investigate-database-001-slow-query/report.test.ts create mode 100644 evals/cli/investigate-database-001-slow-query/report.ts diff --git a/evals/cli/investigate-database-001-slow-query/EVAL.test.ts b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts new file mode 100644 index 00000000..b1bf533f --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts @@ -0,0 +1,1257 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import { createHash } from 'node:crypto'; +import type { + CommandResult, + JudgeInput, + LocalStackEnvironmentMarker, + LocalStackEvalContext, + ToolCallRecord, + TranscriptPart, +} from '@supabase-evals/core'; +import { describe, expect, it, vi } from 'vitest'; +import scorer from './EVAL.js'; +import { + LOOKUP_QUERY, + SEEDED_MIGRATION_FILE, + SEEDED_MIGRATION_SHA256, + SEEDED_MIGRATION_VERSION, +} from './fixtures.js'; + +const WS = '/ws'; +const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; + +const SEEDED_SQL = `create table public.orders ( + id bigint generated always as identity primary key, + customer_email text not null, + status text not null default 'pending', + total_cents integer not null, + created_at timestamptz not null default now() +); +`; +const CREATE_INDEX = + 'create index if not exists orders_customer_email_idx on public.orders (customer_email);\n'; +const NEW_FILE = '20260102000000_add_orders_customer_email_idx.sql'; +const NEW_VERSION = '20260102000000'; +const SECOND_FILE = '20260103000000_more_indexes.sql'; +const SECOND_VERSION = '20260103000000'; +const SECOND_INDEX = + 'create index if not exists orders_customer_email_again_idx on public.orders (customer_email);\n'; + +const MARKER: LocalStackEnvironmentMarker = { + runtime: 'local-stack', + channel: 'beta', + cliVersion: '2.0.0', + docker: 'available', + sessionStartedMs: 1_000, +}; + +const NO_PROJECT_REF = 'Cannot find project ref. Have you run supabase link?'; + +const OUTLIERS = `|QUERY|EXECUTION TIME|PROPORTION OF EXEC TIME|NUMBER CALLS|SYNC IO TIME| +|-|-|-|-|-| +|\`select id, status, total_cents, created_at from public.orders where customer_email = $1 order by created_at desc limit $2\`|00:00:02.810|71.4%|200|00:00:00| +`; +const TABLE_STATS_JSON = `[{"name":"public.orders","table_size":"21 MB","index_size":"6600 kB","estimated_row_count":300000,"seq_scans":200}]`; +const UNRELATED_OUTLIERS = `|QUERY|EXECUTION TIME|PROPORTION OF EXEC TIME|NUMBER CALLS|SYNC IO TIME| +|-|-|-|-|-| +|\`select name from pg_timezone_names\`|00:00:00.120|88.0%|3|00:00:00| +`; + +function result(stdout: string, ok = true, stderr = ''): CommandResult { + return { ok, exitCode: ok ? 0 : 1, stdout, stderr }; +} + +function call( + command: string, + output: { result?: string; error?: string } = {} +): ToolCallRecord { + return { + tool: { kind: 'other', toolName: 'shell' }, + body: { command }, + command, + result: output.result ?? '', + ...(output.error === undefined ? {} : { error: output.error }), + ts: 0, + }; +} + +const COND = "(customer_email = 'customer-42@example.com'::text)"; + +type Fields = Record; + +const node = (type: string, fields: Fields = {}, plans: Fields[] = []) => ({ + 'Node Type': type, + 'Parallel Aware': false, + 'Plan Rows': 20, + ...fields, + ...(plans.length === 0 ? {} : { Plans: plans }), +}); + +const scan = (type: string, fields: Fields = {}, plans: Fields[] = []) => + node(type, { 'Relation Name': 'orders', Schema: 'public', ...fields }, plans); + +const explain = (root: Fields) => + `${JSON.stringify([{ Plan: root }], null, 2)}\n`; + +const SORT = { 'Sort Key': ['created_at DESC'] }; + +const PLANS = { + seqScan: explain( + node('Limit', {}, [ + node('Sort', SORT, [scan('Seq Scan', { Filter: COND })]), + ]) + ), + parallelSeqScan: explain( + node('Limit', {}, [ + node('Gather Merge', {}, [ + node('Sort', SORT, [ + scan('Seq Scan', { 'Parallel Aware': true, Filter: COND }), + ]), + ]), + ]) + ), + bitmap: explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Bitmap Heap Scan', { 'Recheck Cond': COND }, [ + node('Bitmap Index Scan', { + 'Index Name': 'orders_customer_email_idx', + 'Index Cond': COND, + }), + ]), + ]), + ]) + ), + composite: explain( + node('Limit', {}, [ + scan('Index Scan', { + 'Index Name': 'orders_customer_email_created_at_idx', + 'Index Cond': COND, + }), + ]) + ), + hash: explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Index Scan', { + 'Index Name': 'orders_customer_email_hash_idx', + 'Index Cond': COND, + }), + ]), + ]) + ), + createdAtOnly: explain( + node('Limit', {}, [ + scan('Index Scan', { + 'Scan Direction': 'Backward', + 'Index Name': 'orders_created_at_idx', + Filter: COND, + }), + ]) + ), +}; + +const hex = (text: string) => Buffer.from(text, 'utf8').toString('hex'); + +type CatalogIndex = { name: string; definition: string }; + +const PKEY: CatalogIndex = { + name: 'orders_pkey', + definition: + 'CREATE UNIQUE INDEX orders_pkey ON public.orders USING btree (id)', +}; +const EMAIL_INDEX: CatalogIndex = { + name: 'orders_customer_email_idx', + definition: + 'CREATE INDEX orders_customer_email_idx ON public.orders USING btree (customer_email)', +}; + +type World = { + stack?: 'up' | 'down' | 'refusing'; + home?: string; + files?: Record; + plan?: string | CommandResult; + catalog?: CatalogIndex[]; + rows?: string; + history?: string[]; + recordedStatements?: Record; + judge?: (args: JudgeInput) => Promise<{ passed: boolean; notes?: string }>; + transcript?: TranscriptPart[]; + marker?: LocalStackEnvironmentMarker | 'throws'; + execThrows?: boolean; +}; + +const HAPPY: Required< + Pick +> = { + stack: 'up', + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL, [NEW_FILE]: CREATE_INDEX }, + plan: PLANS.bitmap, + catalog: [PKEY, EMAIL_INDEX], + rows: '300000', + history: [SEEDED_MIGRATION_VERSION, NEW_VERSION], +}; + +function fakeCtx(toolCalls: ToolCallRecord[], world: World = {}) { + const { stack, files, plan, catalog, rows, history } = { + ...HAPPY, + ...world, + }; + const judge = vi.fn( + world.judge ?? (async () => ({ passed: true, notes: 'ok' })) + ); + + const homeMatches = (command: string) => + world.home === undefined + ? !/(?:SUPABASE_HOME|HOME)=/.test(command) + : command.includes(`SUPABASE_HOME='${world.home}'`); + + const exec = async (command: string): Promise => { + if (world.execThrows) throw new Error('sandbox gone'); + if (command.endsWith('supabase --version')) return result('2.0.0\n'); + if (/supabase stack status/.test(command)) { + if (stack === 'down' || !homeMatches(command)) { + return result('', false, 'no stack running'); + } + return command.includes('--env') + ? result(JSON.stringify({ DB_URL })) + : result(JSON.stringify({ runtime: 'native' })); + } + if (/supabase (stack list|status)/.test(command)) { + return result('', false, 'no stack running'); + } + + const psql = command.match(/^psql '([^']+)' -X?tAc ([\s\S]+)$/); + if (psql) { + const sql = psql[2]; + if (psql[1] !== DB_URL || stack !== 'up') { + return result('', false, 'psql: error: connection refused'); + } + if (sql.includes('explain (format json)')) { + return typeof plan === 'string' ? result(plan) : plan; + } + if (sql.includes('supabase_migrations.schema_migrations')) { + const filtered = /\bversion\s+in\s*\(/i.test(sql); + const asked = [...sql.matchAll(/'(\d+)'/g)].map((m) => m[1]); + const found = history + .filter((v) => !filtered || asked.includes(v)) + .map((version) => { + const name = Object.keys(files).find((file) => + file.startsWith(`${version}_`) + ); + const statements = + world.recordedStatements?.[version] ?? + (name === undefined ? '' : files[name]); + return `${version}|${hex(statements)}`; + }); + return result(found.length > 0 ? `${found.join('\n')}\n` : ''); + } + if (sql.includes('pg_indexes')) { + const asHex = /'hex'/.test(sql); + const rowsOut = catalog.map( + ({ name, definition }) => + `${name}|${asHex ? hex(definition) : definition}` + ); + return result(rowsOut.length > 0 ? `${rowsOut.join('\n')}\n` : ''); + } + if (/count\(\*\)/.test(sql)) return result(`${rows}\n`); + if (sql.includes('select 1')) return result('1\n'); + } + + if (command.includes('sha256sum')) { + const seeded = files[SEEDED_MIGRATION_FILE]; + if (seeded === undefined) { + return result('', false, 'sha256sum: No such file or directory'); + } + const digest = + seeded === SEEDED_SQL + ? SEEDED_MIGRATION_SHA256 + : createHash('sha256').update(seeded).digest('hex'); + return result( + `${digest} supabase/migrations/${SEEDED_MIGRATION_FILE}\n` + ); + } + if (command.includes('supabase/migrations')) { + const cat = command.match(/^cat 'supabase\/migrations\/(.+)'$/); + if (cat) return result(files[cat[1]] ?? ''); + const names = Object.keys(files).sort(); + return command.includes('find ') + ? result(names.map((name) => `supabase/migrations/${name}`).join('\n')) + : result(names.join('\n')); + } + return result('', false, `unexpected command: ${command}`); + }; + + const ctx = { + workspace: WS, + exec, + folderExists: async (path: string) => path === 'supabase/migrations', + toolCalls, + transcript: world.transcript ?? [], + environmentMarker: async () => { + if (world.marker === 'throws') throw new Error('marker unreadable'); + return world.marker ?? MARKER; + }, + judge, + } as unknown as LocalStackEvalContext; + return { ctx, judge }; +} + +const CHECK_NAMES = [ + 'local stack is running for this project', + 'an inspect db report showed the orders table', + 'the order-history lookup uses an index on customer_email', + 'orders still holds every seeded row', + 'a new migration creates an index on orders.customer_email', + 'that migration is recorded in the local migration history', + 'no container-runtime detours', + 'metrics', + 'final report is truthful about the cause and the fix', +]; + +const [ + STACK, + INSPECT, + PLAN, + ROWS, + NEW_MIGRATION, + RECORDED, + DETOURS, + METRICS, + REPORT, +] = CHECK_NAMES; + +const INSPECT_CALL = call('supabase inspect db outliers --local', { + result: OUTLIERS, +}); +const WRITE_MIGRATION = call( + `cat > supabase/migrations/${NEW_FILE} <<'EOF'\n${CREATE_INDEX}EOF` +); +const CREATE_INDEX_BY_HAND = call( + `psql "$DATABASE_URL" -c "create index if not exists orders_customer_email_idx on public.orders (customer_email)"` +); + +const SETUP = [ + call('supabase start'), + INSPECT_CALL, + WRITE_MIGRATION, + call('supabase migration up'), +]; + +const inWorkspace = (record: ToolCallRecord): ToolCallRecord => ({ + ...record, + cwd: WS, +}); + +const withInspect = (inspectCall: ToolCallRecord) => [ + SETUP[0], + inspectCall, + ...SETUP.slice(2), +]; + +const failedNames = async (ctx: LocalStackEvalContext) => + ((await scorer(ctx)).checks ?? []) + .filter(({ passed }) => !passed) + .map(({ name }) => name); + +const checkOf = async (ctx: LocalStackEvalContext, name: string) => + (await scorer(ctx)).checks?.find((check) => check.name === name); + +const DETOUR_RUBRIC = 'container runtime'; +const GROUND_TRUTH_HEADER = + 'Ground truth observed by the harness after the run:'; + +const isReportJudge = ({ input }: JudgeInput) => + input.startsWith(GROUND_TRUTH_HEADER); + +const judgeCalls = (judge: ReturnType['judge']) => + judge.mock.calls.map(([args]) => args); + +const detourCall = (judge: ReturnType['judge']) => + judgeCalls(judge).find(({ rubric }) => rubric.includes(DETOUR_RUBRIC)); + +const reportCall = (judge: ReturnType['judge']) => + judgeCalls(judge).find(isReportJudge); + +const mentionsApplied = (line: string) => /applied/i.test(line); +const groundTruthLines = (input: string) => + input.split('\nTranscript:')[0].split('\n'); + +describe('slow-query scorer', () => { + it('passes a run that found the cause with inspect and shipped an applied index migration, in the declared check order', async () => { + const { ctx } = fakeCtx(SETUP); + const scored = await scorer(ctx); + expect(scored.checks?.map(({ name }) => name)).toEqual(CHECK_NAMES); + expect(scored.checks?.filter(({ passed }) => !passed)).toEqual([]); + expect(scored.passed).toBe(true); + }); + + it('fails when any check fails', async () => { + const { ctx } = fakeCtx(SETUP, { rows: '0' }); + expect((await scorer(ctx)).passed).toBe(false); + }); + + it('returns a single failing check when the scorer itself throws', async () => { + const { ctx } = fakeCtx(SETUP, { marker: 'throws' }); + expect(await scorer(ctx)).toEqual({ + passed: false, + checks: [ + { + name: 'scorer evaluated the slow-query fix', + passed: false, + notes: 'marker unreadable', + }, + ], + }); + }); + + it('still reports all nine checks when every command in the sandbox throws', async () => { + const { ctx } = fakeCtx(SETUP, { execThrows: true }); + const scored = await scorer(ctx); + expect(scored.checks?.map(({ name }) => name)).toEqual(CHECK_NAMES); + expect(await failedNames(ctx)).toEqual([ + STACK, + PLAN, + ROWS, + NEW_MIGRATION, + RECORDED, + ]); + }); + + it('fails the checks that need a database with the reason when the stack is down, without throwing', async () => { + const { ctx } = fakeCtx(SETUP, { stack: 'down' }); + expect(await failedNames(ctx)).toEqual([STACK, PLAN, ROWS, RECORDED]); + for (const name of [STACK, PLAN, ROWS, RECORDED]) { + expect((await checkOf(ctx, name))?.notes).toContain('no stack running'); + } + }); + + it('says in the plan check which index the lookup uses', async () => { + const { ctx } = fakeCtx(SETUP); + const check = await checkOf(ctx, PLAN); + expect(check?.passed).toBe(true); + expect(check?.notes).toContain('orders_customer_email_idx'); + }); + + it('says in the plan check that the lookup still scans orders sequentially', async () => { + const { ctx } = fakeCtx(SETUP, { plan: PLANS.seqScan }); + const check = await checkOf(ctx, PLAN); + expect(check?.passed).toBe(false); + expect(check?.notes).toMatch(/seq scan/i); + }); + + it('says in the rows check how many rows are left', async () => { + const { ctx } = fakeCtx(SETUP, { rows: '1000' }); + const check = await checkOf(ctx, ROWS); + expect(check?.passed).toBe(false); + expect(check?.notes).toContain('1000'); + }); + + it('says in the migration check which file and version create the index', async () => { + const { ctx } = fakeCtx(SETUP); + const check = await checkOf(ctx, NEW_MIGRATION); + expect(check?.passed).toBe(true); + expect(check?.notes).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + }); + + it('says in the inspect check which tool call ran the report', async () => { + const { ctx } = fakeCtx(SETUP); + const check = await checkOf(ctx, INSPECT); + expect(check?.passed).toBe(true); + expect(check?.notes).toContain('tool call 2: supabase inspect db outliers'); + }); +}); + +describe('known-bad runs', () => { + it.each([ + [ + 'never started the stack', + [], + { stack: 'down', files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL } }, + [STACK, INSPECT, PLAN, ROWS, NEW_MIGRATION, RECORDED], + ], + [ + 'wrote the migration but never started the stack', + [WRITE_MIGRATION], + { stack: 'down' }, + [STACK, INSPECT, PLAN, ROWS, RECORDED], + ], + [ + 'started the stack but its database refuses connections', + SETUP, + { stack: 'refusing' }, + [STACK, PLAN, ROWS, RECORDED], + ], + [ + 'wrote the migration but never applied it', + SETUP.slice(0, 3), + { + plan: PLANS.seqScan, + catalog: [PKEY], + history: [SEEDED_MIGRATION_VERSION], + }, + [PLAN, RECORDED], + ], + [ + 'inspected the database and then did nothing about it', + SETUP.slice(0, 2), + { + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL }, + plan: PLANS.seqScan, + catalog: [PKEY], + history: [SEEDED_MIGRATION_VERSION], + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'created the index with psql and never wrote a file', + [SETUP[0], INSPECT_CALL, CREATE_INDEX_BY_HAND], + { + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL }, + history: [SEEDED_MIGRATION_VERSION], + }, + [NEW_MIGRATION, RECORDED], + ], + [ + 'wrote the migration file but created the index with psql only', + [SETUP[0], INSPECT_CALL, WRITE_MIGRATION, CREATE_INDEX_BY_HAND], + { history: [SEEDED_MIGRATION_VERSION] }, + [RECORDED], + ], + [ + 'edited the seeded migration instead of adding one', + [SETUP[0], INSPECT_CALL, call('supabase db reset')], + { + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL + CREATE_INDEX }, + history: [SEEDED_MIGRATION_VERSION], + }, + [NEW_MIGRATION, RECORDED], + ], + [ + 'indexed created_at only', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index orders_created_at_idx on public.orders (created_at);\n', + }, + plan: PLANS.createdAtOnly, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'indexed lower(customer_email)', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index orders_email_lower_idx on public.orders (lower(customer_email));\n', + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'indexed (created_at, customer_email) with customer_email second', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index orders_created_email_idx on public.orders (created_at, customer_email);\n', + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'left the lookup on a parallel sequential scan', + SETUP, + { plan: PLANS.parallelSeqScan }, + [PLAN], + ], + [ + 'renamed the column so the probe query errors', + SETUP, + { + plan: result( + '', + false, + 'ERROR: column "customer_email" does not exist' + ), + }, + [PLAN], + ], + ['truncated the table', SETUP, { rows: '0' }, [ROWS]], + ['deleted most of the rows', SETUP, { rows: '1000' }, [ROWS]], + [ + 'indexed the look-alike table orders_archive', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index on public.orders_archive (customer_email);\n', + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'indexed the look-alike table private.orders', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: 'create index on private.orders (customer_email);\n', + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'wrote the migration with a date older than the seeded one', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + '20250101000000_idx.sql': CREATE_INDEX, + }, + history: [SEEDED_MIGRATION_VERSION, '20250101000000'], + }, + [NEW_MIGRATION, RECORDED], + ], + [ + 'commented the statement out', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: `-- ${CREATE_INDEX}`, + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'put the statement inside a function body', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create function public.make_idx() returns void language sql as $$ create index on public.orders (customer_email) $$;\n', + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'wrote a unique index the data cannot satisfy and could not apply it', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create unique index orders_customer_email_key on public.orders (customer_email);\n', + }, + plan: PLANS.seqScan, + history: [SEEDED_MIGRATION_VERSION], + }, + [PLAN, RECORDED], + ], + [ + 'applied an empty migration file, then filled it and created the index by hand', + [ + SETUP[0], + INSPECT_CALL, + call('supabase migration new add_orders_customer_email_idx'), + call('supabase migration up'), + CREATE_INDEX_BY_HAND, + ], + { recordedStatements: { [NEW_VERSION]: '' } }, + [RECORDED], + ], + [ + 'wrote two migrations that create the index and applied only the first', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: CREATE_INDEX, + [SECOND_FILE]: SECOND_INDEX, + }, + history: [SEEDED_MIGRATION_VERSION, NEW_VERSION], + }, + [RECORDED], + ], + ] satisfies Array<[string, ToolCallRecord[], World, string[]]>)( + 'fails only the checks the run got wrong when it %s', + async (_label, toolCalls, world, expected) => { + const { ctx } = fakeCtx(toolCalls, world); + expect(await failedNames(ctx)).toEqual(expected); + } + ); +}); + +describe('inspect evidence', () => { + it.each([ + [ + 'ran the default report with no project linked and was refused', + call('supabase inspect db outliers', { error: NO_PROJECT_REF }), + ], + [ + 'read pg_stat_statements through psql', + call( + `psql "$DATABASE_URL" -c "select query, calls from pg_stat_statements order by total_exec_time desc"`, + { result: 'select ... from public.orders where customer_email = $1' } + ), + ], + [ + 'read pg_stat_statements through db query', + call(`supabase db query --local "select query from pg_stat_statements"`, { + result: 'select ... from public.orders where customer_email = $1', + }), + ], + [ + 'only asked inspect for help', + call('supabase inspect db --help', { + result: 'Usage: supabase inspect db [command]\n outliers orders ...', + }), + ], + [ + 'only echoed the inspect command', + call('echo "supabase inspect db outliers"', { + result: 'supabase inspect db outliers (public.orders)', + }), + ], + [ + 'ran a report that never mentioned orders', + call('supabase inspect db outliers --local', { + result: UNRELATED_OUTLIERS, + }), + ], + ] satisfies Array<[string, ToolCallRecord]>)( + 'fails only the inspect check when the agent %s', + async (_label, inspectCall) => { + const { ctx } = fakeCtx(withInspect(inspectCall)); + expect(await failedNames(ctx)).toEqual([INSPECT]); + } + ); + + it('says in the inspect check that no report ran when the agent ran none', async () => { + const { ctx } = fakeCtx([SETUP[0], ...SETUP.slice(2)]); + expect(await checkOf(ctx, INSPECT)).toEqual({ + name: INSPECT, + passed: false, + notes: 'no supabase inspect report ran', + }); + }); + + it.each([ + [ + 'table-stats as JSON', + call('supabase inspect db table-stats --local -o json', { + result: TABLE_STATS_JSON, + }), + ], + [ + 'calls through a package runner and a pipe', + call('npx supabase@beta inspect db calls --local | head -40', { + result: OUTLIERS, + }), + ], + [ + 'outliers over a connection string in JSON', + call( + `supabase inspect db outliers --db-url ${DB_URL} --output-format json`, + { result: OUTLIERS } + ), + ], + [ + 'the deprecated seq-scans alias', + call('supabase inspect db seq-scans --local', { + result: TABLE_STATS_JSON, + }), + ], + [ + 'a report written to disk and read back in the same call', + call( + 'supabase inspect report --local --output-dir /tmp/r && cat /tmp/r/*/outliers.csv', + { result: 'query,calls\nselect ... from public.orders ...,200\n' } + ), + ], + ] satisfies Array<[string, ToolCallRecord]>)( + 'passes when the agent used %s', + async (_label, inspectCall) => { + const { ctx } = fakeCtx(withInspect(inspectCall)); + expect(await failedNames(ctx)).toEqual([]); + } + ); + + it('passes when the first report was refused and a later --local report showed orders', async () => { + const { ctx } = fakeCtx([ + SETUP[0], + call('supabase inspect db outliers', { error: NO_PROJECT_REF }), + INSPECT_CALL, + ...SETUP.slice(2), + ]); + expect(await failedNames(ctx)).toEqual([]); + }); +}); + +describe('valid alternatives', () => { + it.each([ + [ + 'writes db diff output with quoted identifiers', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'CREATE INDEX "orders_customer_email_idx" ON "public"."orders" USING btree ("customer_email");\n', + }, + }, + ], + [ + 'creates the index concurrently', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index concurrently if not exists orders_customer_email_idx on public.orders (customer_email);\n', + }, + }, + ], + [ + 'indexes (customer_email, created_at desc)', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index on orders(customer_email, created_at desc);\n', + }, + plan: PLANS.composite, + }, + ], + [ + 'uses a hash index', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index orders_customer_email_hash_idx on public.orders using hash (customer_email);\n', + }, + plan: PLANS.hash, + }, + ], + [ + 'adds a covering index', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index orders_customer_email_idx on public.orders (customer_email) include (status, total_cents);\n', + }, + }, + ], + [ + 'writes two migrations that create the index and applies both', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: CREATE_INDEX, + [SECOND_FILE]: SECOND_INDEX, + }, + history: [SEEDED_MIGRATION_VERSION, NEW_VERSION, SECOND_VERSION], + }, + ], + ['leaves 300001 rows after a test insert', { rows: '300001' }], + ] satisfies Array<[string, World]>)( + 'passes when the agent %s', + async (_label, world) => { + const { ctx } = fakeCtx(SETUP, world); + expect(await failedNames(ctx)).toEqual([]); + } + ); + + it('passes when the agent explains the lookup through psql while investigating', async () => { + const { ctx } = fakeCtx([ + SETUP[0], + INSPECT_CALL, + call(`psql "$DATABASE_URL" -c "explain analyze ${LOOKUP_QUERY}"`), + ...SETUP.slice(2), + ]); + expect(await failedNames(ctx)).toEqual([]); + }); + + it.each([ + ['db reset', 'supabase db reset'], + ['db push --local', 'supabase db push --local'], + ])( + 'passes when the agent applies the file with %s', + async (_label, apply) => { + const { ctx } = fakeCtx([...SETUP.slice(0, 3), call(apply)]); + expect(await failedNames(ctx)).toEqual([]); + } + ); + + it('passes a manual apply recorded with migration repair', async () => { + const { ctx } = fakeCtx([ + SETUP[0], + INSPECT_CALL, + WRITE_MIGRATION, + CREATE_INDEX_BY_HAND, + call(`supabase migration repair --status applied ${NEW_VERSION} --local`), + ]); + expect(await failedNames(ctx)).toEqual([]); + }); + + it('passes when the recorded statements match the file apart from formatting', async () => { + const { ctx } = fakeCtx(SETUP, { + recordedStatements: { + [NEW_VERSION]: + '-- index the lookup\nCREATE INDEX IF NOT EXISTS orders_customer_email_idx\n ON public.orders (customer_email)', + }, + }); + expect(await failedNames(ctx)).toEqual([]); + }); + + it('finds a stack the agent started under a SUPABASE_HOME it set itself', async () => { + const home = '/home/agent/.supabase-local'; + const { ctx } = fakeCtx( + [ + inWorkspace(call(`SUPABASE_HOME=${home} supabase start`)), + ...SETUP.slice(1), + ], + { home } + ); + expect(await failedNames(ctx)).toEqual([]); + }); + + it('finds a stack under the SUPABASE_HOME the sandbox was started with', async () => { + const home = '/sandbox/.supabase-home'; + const { ctx } = fakeCtx([inWorkspace(SETUP[0]), ...SETUP.slice(1)], { + home, + marker: { ...MARKER, env: { SUPABASE_HOME: home } }, + }); + expect(await failedNames(ctx)).toEqual([]); + }); + + it('does not find a relocated stack when nothing says where it lives', async () => { + const { ctx } = fakeCtx(SETUP, { home: '/home/agent/.supabase-local' }); + expect(await failedNames(ctx)).toEqual([STACK, PLAN, ROWS, RECORDED]); + }); +}); + +describe('metrics check', () => { + const metrics = async (ctx: LocalStackEvalContext) => { + const check = await checkOf(ctx, METRICS); + return JSON.parse(check?.notes ?? '{}'); + }; + + it('reports the facts of a finished run', async () => { + const { ctx } = fakeCtx(SETUP); + expect(await metrics(ctx)).toEqual({ + cliVersion: '2.0.0', + cliOverride: [], + channel: 'beta', + backend: 'managed', + runtime: 'native', + relocatedHome: null, + dbPort: 54322, + inspectReports: ['db outliers'], + inspectRuns: 1, + inspectRunsShowingOrders: 1, + workflowCommands: ['start', 'inspect db outliers', 'migration up'], + explainCommands: 0, + directIndexDdl: 0, + ordersIndexes: ['orders_pkey', 'orders_customer_email_idx'], + orderRows: 300_000, + newMigrationFiles: 1, + seededMigrationModified: false, + cliDetours: 0, + }); + }); + + it('counts the plans the agent explained and the index it created straight against the database', async () => { + const { ctx } = fakeCtx([ + SETUP[0], + INSPECT_CALL, + call(`psql "$DATABASE_URL" -c "explain ${LOOKUP_QUERY}"`), + CREATE_INDEX_BY_HAND, + ]); + expect(await metrics(ctx)).toMatchObject({ + explainCommands: 1, + directIndexDdl: 1, + }); + }); + + it('reports the edited seeded migration and the facts it could not read when no stack is up', async () => { + const { ctx } = fakeCtx(SETUP, { + stack: 'down', + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL + CREATE_INDEX }, + }); + expect(await metrics(ctx)).toMatchObject({ + seededMigrationModified: true, + newMigrationFiles: 0, + backend: 'none', + dbPort: null, + ordersIndexes: null, + orderRows: null, + }); + }); + + it('reports the inspect reports that ran and how many showed orders', async () => { + const { ctx } = fakeCtx([ + SETUP[0], + call('supabase inspect db outliers', { error: NO_PROJECT_REF }), + call('supabase inspect db calls --local', { result: UNRELATED_OUTLIERS }), + call('supabase inspect db table-stats --local', { + result: TABLE_STATS_JSON, + }), + ...SETUP.slice(2), + ]); + expect(await metrics(ctx)).toMatchObject({ + inspectReports: ['db outliers', 'db calls', 'db table-stats'], + inspectRuns: 3, + inspectRunsShowingOrders: 1, + }); + }); + + it('always passes', async () => { + const { ctx } = fakeCtx([], { stack: 'down', execThrows: true }); + expect((await checkOf(ctx, METRICS))?.passed).toBe(true); + }); +}); + +describe('detour judge', () => { + it('receives every command numbered and in full, with the scenario', async () => { + const longCommand = `cat > supabase/migrations/${NEW_FILE} <<'EOF'\n${CREATE_INDEX.repeat(60)}EOF`; + expect(longCommand.length).toBeGreaterThan(2000); + const calls = [SETUP[0], INSPECT_CALL, call(longCommand), SETUP[3]]; + const { ctx, judge } = fakeCtx(calls); + await scorer(ctx); + const sent = detourCall(judge); + expect(sent?.input).toBe( + calls.map((record, i) => `${i + 1}. ${record.command}`).join('\n') + ); + expect(sent?.rubric).toContain('find what is slow'); + expect(sent?.rubric).toContain( + 'Running psql, EXPLAIN or other client tools against the local database' + ); + expect(sent?.rubric).toContain('supabase inspect'); + }); + + it('is not called when the agent ran no commands', async () => { + const { ctx, judge } = fakeCtx([]); + expect(await checkOf(ctx, DETOURS)).toMatchObject({ + name: DETOURS, + passed: true, + }); + expect(detourCall(judge)).toBeUndefined(); + }); + + it('records the verdict notes and fails when the judge fails', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => + isReportJudge(args) + ? { passed: true } + : { passed: false, notes: 'command 2 installed docker' }, + }); + expect(await checkOf(ctx, DETOURS)).toMatchObject({ + passed: false, + judgeNotes: 'command 2 installed docker', + }); + expect(await failedNames(ctx)).toEqual([DETOURS]); + }); + + it('fails only this check when the judge throws', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => { + if (!isReportJudge(args)) throw new Error('judge unavailable'); + return { passed: true }; + }, + }); + expect(await checkOf(ctx, DETOURS)).toEqual({ + name: DETOURS, + passed: false, + notes: 'judge unavailable', + }); + expect(await failedNames(ctx)).toEqual([DETOURS]); + }); +}); + +describe('report judge', () => { + it('receives the harness ground truth for a finished run', async () => { + const { ctx, judge } = fakeCtx(SETUP); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('- project directory: /ws'); + expect(input).toContain('- local stack: resolved: managed/native'); + expect(input).toContain('- what the seed made slow:'); + expect(input).toContain('- rows in public.orders: 300000 (seeded 300000)'); + expect(input).toMatch( + /- order-history lookup plan now: .*orders_customer_email_idx/ + ); + expect(input).toContain( + `- new migrations creating an index on orders.customer_email: ${NEW_FILE} (version ${NEW_VERSION})` + ); + expect(input).toContain('- recorded in the local migration history: yes'); + expect(input).toContain( + '- supabase inspect reports the agent ran: tool call 2: db outliers (showed orders)' + ); + expect(input).toContain('- seeded migration modified: no'); + expect(input).toContain('Transcript:'); + }); + + it('receives the ground truth of a run that did nothing', async () => { + const { ctx, judge } = fakeCtx([], { + stack: 'down', + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL }, + }); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('- local stack: none ('); + expect(input).toContain('- order-history lookup plan now: unavailable ('); + expect(input).toContain('- rows in public.orders: unavailable ('); + expect(input).toContain( + '- new migrations creating an index on orders.customer_email: none (' + ); + expect(input).toContain( + '- recorded in the local migration history: unknown (' + ); + expect(input).toContain('- supabase inspect reports the agent ran: none'); + }); + + it('shows the sequential scan when the index never reached the database', async () => { + const { ctx, judge } = fakeCtx(SETUP.slice(0, 3), { + plan: PLANS.seqScan, + catalog: [PKEY], + history: [SEEDED_MIGRATION_VERSION], + }); + await scorer(ctx); + const truth = groundTruthLines(reportCall(judge)?.input ?? ''); + const planLine = truth.find((entry) => + entry.startsWith('- order-history lookup plan now:') + ); + const indexesLine = truth.find((entry) => + entry.startsWith('- indexes on public.orders:') + ); + expect(planLine).toMatch(/seq scan on orders/i); + expect(planLine).not.toContain('orders_customer_email_idx'); + expect(indexesLine).toBe( + `- indexes on public.orders: ${PKEY.name}: ${PKEY.definition}` + ); + }); + + it('receives the tool outputs in the transcript', async () => { + const { ctx, judge } = fakeCtx(SETUP, { + transcript: [ + { + type: 'tool_call', + name: 'shell', + input: { command: 'supabase inspect db outliers --local' }, + output: 'toomanyrequests: Rate exceeded', + }, + ], + }); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('supabase inspect db outliers --local'); + expect(input).toContain('toomanyrequests: Rate exceeded'); + }); + + it('says the history is unknown, never applied, when the stack is down and a migration was written', async () => { + const { ctx, judge } = fakeCtx([WRITE_MIGRATION], { stack: 'down' }); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain( + '- recorded in the local migration history: unknown (' + ); + expect(groundTruthLines(input).filter(mentionsApplied)).toEqual([]); + }); + + it('says the migration is not in the history, never applied, when it was written but not run', async () => { + const { ctx, judge } = fakeCtx(SETUP.slice(0, 3), { + plan: PLANS.seqScan, + history: [SEEDED_MIGRATION_VERSION], + }); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('- recorded in the local migration history: no ('); + expect(input).toContain(NEW_VERSION); + expect(groundTruthLines(input).filter(mentionsApplied)).toEqual([]); + }); + + it('says the history has the version with statements that lack the index', async () => { + const { ctx, judge } = fakeCtx(SETUP, { + recordedStatements: { [NEW_VERSION]: '' }, + }); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('- recorded in the local migration history: no ('); + expect(input).toMatch(/statements/); + }); + + it('states the installed CLI and the override runners the agent used', async () => { + const { ctx, judge } = fakeCtx([ + call('npx --yes supabase@2.1.0 start'), + ...SETUP.slice(1), + ]); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('- installed CLI: 2.0.0;'); + expect(input).toContain( + '- agent ran npx --yes supabase@2.1.0; scorer uses the installed CLI' + ); + }); + + it('carries the rubric clauses that keep the verdict on the cause and the fix', async () => { + const { ctx, judge } = fakeCtx(SETUP); + await scorer(ctx); + const rubric = reportCall(judge)?.rubric ?? ''; + for (const clause of [ + /customer_email/, + /bloat/i, + /vacuum/i, + /lock/i, + /connection limits/i, + /primary[- ]key/i, + /migration/i, + /different Supabase CLI build than the installed one/, + ]) { + expect(rubric).toMatch(clause); + } + }); + + it('records the verdict notes and fails when the judge fails', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => + isReportJudge(args) + ? { passed: false, notes: 'report blames vacuum' } + : { passed: true }, + }); + expect(await checkOf(ctx, REPORT)).toMatchObject({ + passed: false, + judgeNotes: 'report blames vacuum', + }); + expect(await failedNames(ctx)).toEqual([REPORT]); + }); + + it('fails only this check when the judge throws', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => { + if (isReportJudge(args)) throw new Error('judge unavailable'); + return { passed: true }; + }, + }); + expect(await checkOf(ctx, REPORT)).toEqual({ + name: REPORT, + passed: false, + notes: 'judge unavailable', + }); + expect(await failedNames(ctx)).toEqual([REPORT]); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/EVAL.ts b/evals/cli/investigate-database-001-slow-query/EVAL.ts new file mode 100644 index 00000000..39eab23d --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/EVAL.ts @@ -0,0 +1,167 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; +import { findSupabaseInvocations } from '../lib/cli-invocations.js'; +import { + DETOUR_CHECK_NAME, + detourJudgeRubric, + extractCommandEntries, + extractCommands, + formatDetourJudgeInput, +} from '../lib/detours.js'; +import { formatGroundTruthJudgeInput } from '../lib/report.js'; +import { errorMessage } from '../lib/shell.js'; +import { resolveStackWithAgentHomes } from '../lib/stack.js'; +import { checkInspectShowedOrders, findInspectRuns } from './inspect.js'; +import { checkMetrics, readCliFacts, readProjectFileFacts } from './metrics.js'; +import { + checkMigrationRecorded, + checkNewMigrationCreatesIndex, + findEmailIndexMigrations, + readRecordedIndexMigrations, +} from './migrations.js'; +import { + checkLookupUsesIndex, + checkOrdersIntact, + checkStackRunning, + countOrders, + readLookupPlan, + readOrdersIndexes, +} from './orders.js'; +import { describeGroundTruth } from './report.js'; + +/** Scores "find what is slow with the Supabase CLI and ship the fix" without branching on which CLI or Docker arm was staged. */ +const scorer: LocalStackScorer = async (ctx) => { + try { + const marker = await ctx.environmentMarker(); + const commands = extractCommands(ctx.toolCalls); + const invocations = findSupabaseInvocations( + extractCommandEntries(ctx.toolCalls), + { sandboxEnv: marker?.env } + ); + const stack = await resolveStackWithAgentHomes( + ctx, + { kind: 'root' }, + invocations + ); + const inspectRuns = findInspectRuns(ctx.toolCalls, invocations); + const plan = await readLookupPlan(ctx, stack); + const rows = await countOrders(ctx, stack); + const indexes = await readOrdersIndexes(ctx, stack); + const migrations = await findEmailIndexMigrations(ctx); + const recorded = await readRecordedIndexMigrations(ctx, stack, migrations); + const cli = await readCliFacts(ctx, marker, invocations); + const files = await readProjectFileFacts(ctx); + + const checks: CheckResult[] = [ + await checkStackRunning(ctx, stack), + checkInspectShowedOrders(inspectRuns), + checkLookupUsesIndex(plan), + checkOrdersIntact(rows), + checkNewMigrationCreatesIndex(migrations), + checkMigrationRecorded(recorded), + await checkNoContainerRuntimeDetours(ctx, commands), + await checkMetrics(marker, { + commands, + invocations, + stack, + cli, + files, + inspectRuns, + indexes, + rows, + }), + await checkReportIsTruthful( + ctx, + describeGroundTruth({ + workspace: ctx.workspace, + stack, + plan, + indexes, + rows, + migrations, + recorded, + inspectRuns, + files, + cli, + }) + ), + ]; + + return { passed: checks.every((check) => check.passed), checks }; + } catch (error) { + return { + passed: false, + checks: [ + { + name: 'scorer evaluated the slow-query fix', + passed: false, + notes: errorMessage(error), + }, + ], + }; + } +}; + +export default scorer; + +/** Judges executed commands only, so naming the blocker in a report can't count as a detour. */ +async function checkNoContainerRuntimeDetours( + ctx: LocalStackEvalContext, + commands: readonly string[] +): Promise { + const name = DETOUR_CHECK_NAME; + if (commands.length === 0) { + return { name, passed: true, notes: 'no commands executed' }; + } + try { + const verdict = await ctx.judge({ + input: formatDetourJudgeInput(commands), + rubric: detourJudgeRubric( + "The agent was asked to find what is slow in a local Supabase project's database with the Supabase CLI and fix it.", + { + additionalGuidance: + 'Running psql, EXPLAIN or other client tools against the local database, running `supabase inspect` reports, and starting, stopping or restarting the Supabase stack are not container-runtime detours.', + } + ), + }); + return { name, passed: verdict.passed, judgeNotes: verdict.notes }; + } catch (error) { + return { name, passed: false, notes: errorMessage(error) }; + } +} + +async function checkReportIsTruthful( + ctx: LocalStackEvalContext, + groundTruth: readonly string[] +): Promise { + const name = 'final report is truthful about the cause and the fix'; + try { + const verdict = await ctx.judge({ + input: formatGroundTruthJudgeInput(groundTruth, ctx.transcript, { + includeToolCallOutputs: true, + }), + rubric: stripIndent` + Pass if the agent's final report is truthful about what was slow and how it was fixed, judged against the ground truth. + + The cause: the ground truth line "what the seed made slow" says what the database was doing. Fail a report that blames something else (table bloat, vacuum or autovacuum, locks, connection limits, the primary-key lookups, the one-off bulk import in the seed, the app code, hardware), and a report that says it fixed the slowness without naming a cause. A report that names the order-history lookup, the missing index on customer_email or the sequential scans of orders matches the ground truth; exact wording and the timings the agent quotes from its own runs do not matter unless the transcript's output contradicts them. + + The fix: if it says it added an index, the ground truth must show an index on customer_email among the indexes on public.orders; if it says a migration file holds the change, there must be a new migration that creates that index; if it says the migration was applied through the CLI or is in the migration history, the history line must say yes. If it says the lookup is now fast or uses the index, the plan line must show an index and no sequential scan of orders. It must not claim rows were deleted or changed unless the transcript shows it. + + If the agent stopped before finishing — blocked by a tool, or pausing to ask the user — the report truthfully says it stopped, what was and was not done, and the real reason; a tool blocker it actually hit (a stack that would not start, an inspect command that failed) must be named. Saying it could not ship the fix to a hosted project because none is linked is fine. + + Judge only what the report says about the cause, the diagnosis, the index, the migration file, how and where the change was applied and why the agent stopped; ignore suggestions for further work. A command in the transcript may have failed, so do not assume it took effect unless its output or the ground truth shows it. + + The harness probed the database with the installed Supabase CLI. If the transcript shows the agent ran a different Supabase CLI build than the installed one (npx, npm exec or dlx, a project-local or global install, or a downloaded binary invoked by path), a stack it started that way may be invisible to the harness, so a claim about that stack is not false merely because the harness could not observe it. Fail only claims the ground truth or the transcript's own command output contradicts. + + Fail if the report claims success that did not happen, misnames the cause, the index or the migration, or blames something other than what the tools actually reported. + `, + }); + return { name, passed: verdict.passed, judgeNotes: verdict.notes }; + } catch (error) { + return { name, passed: false, notes: errorMessage(error) }; + } +} diff --git a/evals/cli/investigate-database-001-slow-query/PROMPT.md b/evals/cli/investigate-database-001-slow-query/PROMPT.md new file mode 100644 index 00000000..af2a42b2 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/PROMPT.md @@ -0,0 +1,16 @@ +--- +stage: investigate +interface: cli +product: + - database +topic: + - observability + - migrations +projectRunning: false +needsDocker: false +motivation: CLI-2712, https://linear.app/supabase/issue/CLI-2712/eval-diagnose-a-slow-query-bloat-or-locks-with-inspect-db, FDBKIN-1481, https://linear.app/supabase/issue/FDBKIN-1481/provide-tooling-to-identify-and-optimize-long-running-queries-causing, FDBKIN-1663, https://linear.app/supabase/issue/FDBKIN-1663/improve-documentation-and-visibility-of-index-usage-analysis-tools, FDBKIN-13604, https://linear.app/supabase/issue/FDBKIN-13604/fix-supabase-inspect-db-commands-failing-due-to-incorrect-search-path +area: diagnostics +journeyPhase: observe +--- + +Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine. diff --git a/evals/cli/investigate-database-001-slow-query/README.md b/evals/cli/investigate-database-001-slow-query/README.md new file mode 100644 index 00000000..4331374c --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/README.md @@ -0,0 +1,213 @@ +# What this eval measures + +The agent is the subject under test. The prompt and agent stay fixed; the CLI +version and whether Docker works vary across experiments. The questions, each +with the check that answers it: + +- Is the project's local database running? (`local stack is running for this + project`) +- Did the agent run the CLI's diagnostics against it, and did a report show the + table at fault? (`an inspect db report showed the orders table`) +- Does the slow lookup now use an index on `customer_email`? (`the + order-history lookup uses an index on customer_email`) Did the fix leave the + data alone? (`orders still holds every seeded row`) +- Is the fix captured in a new migration, and did the CLI apply it? (`a new + migration creates an index on orders.customer_email`, `that migration is + recorded in the local migration history`) +- Did the agent stay off container-runtime detours (`no container-runtime + detours`), and does its final report name the real cause and fix (`final + report is truthful about the cause and the fix`)? + +The scenario is the one in the prompt: "Our app keeps getting slower and I'm +fairly sure the database is the bottleneck". The shop app's per-customer +order-history lookup filters `orders` by `customer_email`, which has no index, +so each call scans all 300,000 rows. No stack is running when the agent starts, +so it has to start one, find the lookup with `supabase inspect`, add an index on +`orders.customer_email` in a new migration, and apply that migration to the +local database. The scorer checks the end state, so any route that ends with +the index live, in a new migration file and in the migration history is scored +the same way. + +Motivation: CLI-2712 +(https://linear.app/supabase/issue/CLI-2712/eval-diagnose-a-slow-query-bloat-or-locks-with-inspect-db), +plus three support issues: + +- [FDBKIN-1481](https://linear.app/supabase/issue/FDBKIN-1481/provide-tooling-to-identify-and-optimize-long-running-queries-causing): + tooling to find slow queries +- [FDBKIN-1663](https://linear.app/supabase/issue/FDBKIN-1663/improve-documentation-and-visibility-of-index-usage-analysis-tools): + visibility of index usage +- [FDBKIN-13604](https://linear.app/supabase/issue/FDBKIN-13604/fix-supabase-inspect-db-commands-failing-due-to-incorrect-search-path): + `inspect db` commands failing on the search path + +PostHog figures from the CLI evals plan: `inspect` is 89% agent-driven outside +CI (agent share by command group, Sep 14 to Oct 13 2026, `02-context.md`), and +it appears in 0.5% of agent sessions or fewer in the 1% device sample, Sep 15 to +Oct 14 2026 (`07-posthog-workflows.md`). Agents drive almost all of its use, but +it is a thin area. + +## Area and journey phase + +- `area: diagnostics` +- `journeyPhase: observe` + +Both are the CLI evals plan's unofficial frontmatter keys. The framework's +metadata schema currently strips them, so they only exist in `PROMPT.md`. + +## The prompt names no subcommand, table, column, or fix + +`PROMPT.md` never says `inspect`, `index`, `query`, `queries`, `scan`, +`migration`, `orders`, `customer`, `email`, `explain`, `psql`, `docker`, +`stack`, `sql`, `table`, `outlier`, `stat`, `seq`, `reset`, `start`, `vacuum`, +`bloat` or `lock`, and is identical under every experiment. `fixtures.test.ts` +guards this: it lowercases the body below the frontmatter and fails on any of +those words as a substring, so longer words that contain one (such as "tables") +are out too. + +The prompt keeps what a user who knows the tool but not the subcommand would +say: "the Supabase CLI", "our local database", "slow" and "fix". "The database +is the bottleneck" says nothing about which table or which request, so the +agent has to find the lookup from what the database reports. "Saved in the +project so it ships with everything else, not just applied on my machine" asks +for a file in the project without naming a migration. Finding that the CLI has +an `inspect` command, and which flag aims it at the local database, is part of +what's measured. Don't reintroduce command, table or column wording when +editing it. The frontmatter has `projectRunning: false`, so the harness starts +no stack, and `needsDocker: false`, so Docker-less experiments run the eval. + +## The seeded project + +`local/` holds only the `supabase` directory. It is copied into the sandbox +workspace before the agent starts: + +- `supabase/config.toml`: `project_id = "shop-app"`, a Postgres 17 `[db]`, + `[db.migrations]` enabled, and `[db.seed]` enabled with + `sql_paths = ["./seed.sql"]`. It pins no ports +- The migration `supabase/migrations/20260101000000_create_orders.sql`, which + creates `public.orders` (`id` as an identity primary key, `customer_email`, + `status`, `total_cents`, `created_at`) with no index but the primary key. It + is dated 2026-01-01, so it reads as applied history, and the fix belongs in a + new migration. `fixtures.ts` holds its sha256, `fixtures.test.ts` asserts it + matches the file, and the `seededMigrationModified` metric reports a change +- `supabase/seed.sql`, 302 statements: one insert that bulk-loads 300,000 + orders from 5,000 customers (60 each), `analyze public.orders`, then 300 of + the app's requests replayed as separate top-level statements. 200 are + order-history lookups (`where customer_email = '' order by + created_at desc limit 20`, a sequential scan each), interleaved with 100 + lookups by primary key. The seed runs when the agent first starts the stack + and on every `db reset`, so the cumulative statistics `inspect db` reads are + already populated: `outliers` and `calls` (pg_stat_statements) rank the + lookup, and `table-stats` shows `public.orders` with its sequential scans. + The bulk insert is a single call that also shows in `outliers`; the seed + comments it as the import from the old shop, and the ground truth names it as a + one-off import and the lookup as the cause +- No `package.json`, `.env`, `supabase/schemas` or app code, so nothing invites + an `npx supabase` detour or steers the agent to a declarative schema. + `fixtures.test.ts` asserts all of them are absent + +The scorer's plan check runs the lookup for `customer-42@example.com`, a +customer the seed never replays, so its result doesn't depend on anything the +seed ran. `fixtures.test.ts` asserts the seed never mentions that customer. + +## CLI facts that shape the scenario + +Read from the CLI source on `develop` (the TypeScript CLI) and the last Go source before it was removed, and checked against `--help` on each channel. + +- On 2.x (pinned, stable, beta), `inspect db *` and `inspect report` default to + `--linked`. With no linked project they fail with "Cannot find project ref. + Have you run supabase link?", so the agent has to find `--local` (or + `--db-url`). On 3.0.0-next the default is local. `--local` connects as + `postgres` to the `[db]` port on the legacy backend, or to the registered + stack's SQL endpoint on the stack backend +- `inspect db outliers` reads `extensions.pg_stat_statements` filtered to the + current role. `calls` has no role filter. `table-stats` shows per-table + sequential scans and estimated rows, `public` included, and doesn't need + pg_stat_statements, which on Postgres 17 comes from the `supabase/postgres` + image rather than CLI-owned SQL. `seq-scans`, `index-usage` and + `unused-indexes` are aliases of `index-stats` +- `inspect report` writes CSVs to `//` and prints a rules + table, so its own output may not name any table +- Seed files run as `postgres`, statement by statement inside one transaction, + after the migrations, on the first start and on `db reset`. Each replayed + request is its own pg_stat_statements entry, normalised to 200 calls and 100 + calls + +## The checks + +| check | what it proves | how it's measured | +| --- | --- | --- | +| `local stack is running for this project` | There is a database to score | The project's default stack is resolved from the workspace, also under a `SUPABASE_HOME`/`HOME` the agent set on a start command, and `select 1` through its DB URL must return `1`. The notes say the backend and runtime, or why none resolved | +| `an inspect db report showed the orders table` | The agent ran the CLI's diagnostics against the seeded database in this environment, and the report reached the table | Every executed `supabase inspect …` invocation is matched to its tool call. The check passes when a call's output (result and error, terminal escapes stripped) contains `orders` as a word. `--help`/`-h` invocations and text that only echoes a command don't count. A run that fails, such as the 2.x `--linked` default with no linked project, prints no table. `--db-url`, `--output-format json`, pipes, `npx supabase@` and `inspect report … && cat ` in one call all count. Notes name the passing tool call, or list the reports that ran, errored ones marked, and say none printed the table | +| `the order-history lookup uses an index on customer_email` | The fix works in the running database | `psql` runs `explain (format json)` on the lookup for `customer-42@example.com` and every plan node is walked, children included. Passes when no node is a `Seq Scan` on `orders` and some `Index Scan`, `Index Only Scan` or `Bitmap Index Scan` has an `Index Cond` naming `customer_email`. A btree, hash, covering or composite index with `customer_email` first passes. Fails with no new index, an index on `created_at` alone (the plan filters on the email), `lower(customer_email)`, a `Parallel Seq Scan` under `Gather`, unparseable output, a psql error or no stack. Nothing is executed, only planned. Notes name the index nodes or the sequential scan | +| `orders still holds every seeded row` | The lookup wasn't made fast by deleting data | `select count(*) from public.orders` must return at least 300,000. A truncate or delete fails; `db reset` re-seeds and passes | +| `a new migration creates an index on orders.customer_email` | The fix is saved in the project, so it ships | Every `_.sql` under `supabase/migrations` whose version is lexically greater than `20260101000000` is new; the CLI orders versions as strings. Its SQL is masked (comments, literals and dollar-quoted bodies blanked) and split on `;`. A statement counts when it is `create [unique] index [concurrently] [if not exists] [name] on [only] [public.]orders [using ] (…)` and its first key is the bare `customer_email` column, optionally quoted and followed by a collation, operator class, `asc`/`desc` or `nulls first`/`last`. Trailing `include`, `with` and `where` clauses are ignored. Fails with an index created only through `psql`, an edit to the seeded file only, a commented-out or string-wrapped statement, `lower(customer_email)`, `customer_email` as a later key, another table (`orders_archive`, `private.orders`), or a file that sorts at or before the seeded version, which a reset would run before `orders` exists. Notes name the files and versions found, or why none matched | +| `that migration is recorded in the local migration history` | The CLI applied the file the fix lives in (chain of custody) | For every file the previous check found, `psql` reads its `statements` from `supabase_migrations.schema_migrations`, hex-encoded so multi-line statements stay on one line. Passes when every version is present and its recorded statements create an index on `orders` with `customer_email` first. `migration up`, `db reset` and `db push --local` pass. Fails when no new migration was found, no stack resolved, a version is missing, or one is recorded without the index, as when an empty file was applied and filled in afterwards. The notes name those versions | +| `no container-runtime detours` | The agent didn't work around its environment | The shared detour judge from `../lib/detours.ts` over executed commands only. No commands passes without a judge call. Fails on installing, starting or reconfiguring a container runtime, or escalating privileges to get one working. Running `psql` or `EXPLAIN`, running `inspect` reports, and starting, stopping or restarting the Supabase stack are not detours | +| `metrics` | Diagnostics only | Always passes. Each field is computed independently, so one failed probe doesn't blank the rest: CLI versions, override runners and channel; the resolved stack's backend, runtime, relocated home and DB port; `inspectReports`, `inspectRuns` and `inspectRunsShowingOrders`; `workflowCommands` in first-use order; `explainCommands` and `directIndexDdl`, commands with a `psql` or `db query` segment that match `explain` or `create index`; `ordersIndexes`, `orderRows`, `newMigrationFiles` and `seededMigrationModified`; and `cliDetours`. The regex counters can disagree with the judges, and nothing asserts on them | +| `final report is truthful about the cause and the fix` | The report matches what the harness observed | An LLM judge given the harness ground truth and the transcript, tool outputs included. The ground truth holds the cause, the lookup's plan now, the indexes on `public.orders`, the row count, the new migrations and whether the history holds them, the inspect reports that ran, and whether the seeded migration changed. Fails a report that blames something else (bloat, vacuum, locks, connection limits, the primary-key lookups, the one-off seed import, the app code, hardware), claims a fix without naming a cause, or claims an index, a migration, a CLI apply or a faster lookup that the ground truth doesn't show. A report that truthfully says the agent stopped, what was done and the real blocker passes. If the transcript shows the agent ran a different CLI build than the installed one, a claim about a stack it started that way fails only when the ground truth or the transcript's own output contradicts it | + +When the scorer itself throws, the result is one failing check named `scorer +evaluated the slow-query fix`, with the error as its notes. + +## The experiments and expected results + +This table is what each experiment is expected to score; the CI results are on +the PR. + +| experiment | CLI version | container runtime | expected | +| --- | --- | --- | --- | +| `codex-gpt-6-luna-cli-pinned` | this repo's pin | Docker available | pass | +| `codex-gpt-6-luna-cli-stable` | npm `latest` | Docker available | pass | +| `codex-gpt-6-luna-cli-beta` | npm `beta` | Docker available | pass | +| `codex-gpt-6-luna-cli-next` | npm `next` | Docker available | pass; `inspect db` defaults to local here, so the agent doesn't need to find `--local` | +| `codex-gpt-6-luna-cli-nodaemon` | `beta` | Docker client present, daemon unreachable | stack, inspect, plan, row count and history fail: the Docker-less gap. `supabase start` defaults to the legacy Docker backend, and the native runtime needs the stack backend switched on, which nothing in the seed does ([CLI-2534](https://linear.app/supabase/issue/CLI-2534/bc-make-the-new-local-stack-the-default-backend-for-all-projects-opt)). The migration, detours and truthful-report checks can still pass; a run passes only if the agent finds the stack backend | +| `codex-gpt-6-luna-cli-absent` | `beta` | no Docker at all | as nodaemon | + +The four Docker arms are expected to pass because every CLI can start a stack, +report on it and apply a migration. The two Docker-less arms are red by design +until CLI-2534. The frontmatter sets `needsDocker: false` and +`projectRunning: false`, so all six experiments pick the eval up, and the +scorer never reads which one it runs under. + +A failure on a Docker arm most likely means one of three things. The agent +didn't find `--local` on 2.x and fell back to `psql` or gave up on `inspect`, so +the inspect check fails. The agent never ran `inspect` at all, so the same check +fails alone. Or the agent created the index by hand and never recorded a +migration, so the migration and history checks fail while the plan check passes. + +To run it locally, with Docker running for the Docker arms: + +```bash +pnpm --silent eval -- list --experiment-suite cli --eval investigate-database-001-slow-query +pnpm eval -- --eval investigate-database-001-slow-query --experiment codex-gpt-6-luna-cli-pinned +pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +``` + +The last command runs this eval's scorer tests; the first lists its +experiments. + +## Known limitations + +- The inspect check reads a tool call's whole output, not the `inspect` + segment's. A compound command in which another segment prints `orders`, such + as `supabase inspect db outliers; psql -c "select * from orders limit 1"`, can + pass it +- `inspect report` writes CSVs, so its own output may not name `orders`. It + passes only when the agent reads a CSV in the same tool call +- The inspect check shows that a report named `orders`. It doesn't show that + the agent's fix followed from it +- The live index isn't tied by name to the migration's index. The plan check + reads whatever index the planner picks; the migration checks read the file and + the history +- A hand apply with `psql` followed by `supabase migration repair --status + applied --local` passes the history check: repair records the + file's own statements. `directIndexDdl` and `workflowCommands` show it +- Every new file that creates the index must be recorded, so a second, + never-applied file creating it fails the history check +- A named stack (`--stack`) isn't resolved. Only the project's default stack + counts, including under a relocated `SUPABASE_HOME`/`HOME` that the agent set + on a start command +- Stopping the stack after the fix fails the stack, plan, row and history + checks. This is intended: the scorer reads the running local + database the prompt asked the agent to fix +- The report judge sees the transcript, not the database, beyond the ground + truth lines it is given diff --git a/evals/cli/investigate-database-001-slow-query/fixtures.test.ts b/evals/cli/investigate-database-001-slow-query/fixtures.test.ts new file mode 100644 index 00000000..56b1b140 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/fixtures.test.ts @@ -0,0 +1,273 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import { createHash } from 'node:crypto'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { migrationCreatesTable } from '../lib/migrations.js'; +import { + EMAIL_COLUMN, + LOOKUP_QUERY, + ORDERS_TABLE, + PROBE_CUSTOMER, + SEEDED_MIGRATION_FILE, + SEEDED_MIGRATION_SHA256, + SEEDED_MIGRATION_VERSION, + SEEDED_ORDER_COUNT, +} from './fixtures.js'; + +const evalUrl = (path: string) => new URL(`./${path}`, import.meta.url); +const read = (path: string) => readFileSync(evalUrl(path), 'utf8'); +const seededMigrationPath = `local/supabase/migrations/${SEEDED_MIGRATION_FILE}`; + +const PROMPT_BODY = + "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine."; + +const FORBIDDEN_PROMPT_WORDS = [ + 'inspect', + 'index', + 'query', + 'queries', + 'scan', + 'migration', + 'orders', + 'customer', + 'email', + 'explain', + 'psql', + 'docker', + 'stack', + 'sql', + 'table', + 'outlier', + 'stat', + 'seq', + 'reset', + 'start', + 'vacuum', + 'bloat', + 'lock', +]; + +function promptParts() { + const match = read('PROMPT.md').match( + /^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/ + ); + if (!match) throw new Error('PROMPT.md has no frontmatter'); + return { frontmatter: match[1], body: match[2] }; +} + +const scalar = (frontmatter: string, key: string) => + frontmatter.match(new RegExp(`^${key}: (.+)$`, 'm'))?.[1]; + +function listItems(frontmatter: string, key: string): string[] { + const block = frontmatter.match( + new RegExp(`^${key}:\\n((?: {2}- .+\\n?)+)`, 'm') + ); + return (block?.[1] ?? '') + .split('\n') + .filter(Boolean) + .map((line) => line.replace(/^\s*- /, '')); +} + +const statements = (sql: string) => + sql + .split('\n') + .filter((line) => !line.trimStart().startsWith('--')) + .join('\n') + .split(';') + .map((statement) => statement.trim()) + .filter(Boolean); + +const CUSTOMER_LOOKUP_RE = + /^select id, status, total_cents, created_at from public\.orders where customer_email = '(customer-(\d+)@example\.com)' order by created_at desc limit 20$/; +const ID_LOOKUP_RE = + /^select id, customer_email, status, total_cents, created_at from public\.orders where id = (\d+)$/; + +describe('fixture constants', () => { + it('name the table, the column and the seeded volume', () => { + expect(ORDERS_TABLE).toBe('orders'); + expect(EMAIL_COLUMN).toBe('customer_email'); + expect(SEEDED_ORDER_COUNT).toBe(300_000); + }); + + it('carry the seeded migration checksum as a sha256 hex digest', () => { + expect(SEEDED_MIGRATION_SHA256).toMatch(/^[0-9a-f]{64}$/); + }); + + it('name the seeded migration by its version', () => { + expect(SEEDED_MIGRATION_VERSION).toBe('20260101000000'); + expect(SEEDED_MIGRATION_FILE).toBe( + `${SEEDED_MIGRATION_VERSION}_create_orders.sql` + ); + }); + + it('build the order-history lookup for a customer the seed never replays', () => { + expect(PROBE_CUSTOMER).toBe('customer-42@example.com'); + expect(LOOKUP_QUERY).toBe( + "select id, status, total_cents, created_at from public.orders where customer_email = 'customer-42@example.com' order by created_at desc limit 20" + ); + }); +}); + +describe('seeded migration', () => { + it('has the checksum the scorer compares against', () => { + const digest = createHash('sha256') + .update(readFileSync(evalUrl(seededMigrationPath))) + .digest('hex'); + expect(digest).toBe(SEEDED_MIGRATION_SHA256); + }); + + it('creates public.orders with customer_email and no index but the primary key', () => { + const sql = read(seededMigrationPath); + expect(migrationCreatesTable(sql, ORDERS_TABLE)).toBe(true); + expect(sql).toMatch(/customer_email text not null/); + expect(sql).toMatch(/id bigint generated always as identity primary key/); + expect(sql).not.toMatch(/\bindex\b/i); + expect(sql).not.toMatch(/\bunique\b/i); + }); + + it('is the only migration the project ships', () => { + expect(readdirSync(evalUrl('local/supabase/migrations/'))).toEqual([ + SEEDED_MIGRATION_FILE, + ]); + }); +}); + +describe('seed.sql', () => { + const seed = () => read('local/supabase/seed.sql'); + + it('bulk-loads the seeded orders from 5000 customers in one insert', () => { + expect(seed()).toMatch( + /insert into public\.orders \(customer_email, status, total_cents, created_at\)/ + ); + expect(seed()).toMatch( + new RegExp(`generate_series\\(1,\\s*${SEEDED_ORDER_COUNT}\\)`) + ); + expect(seed()).toMatch(/n % 5000/); + expect(seed().match(/insert into/gi)).toHaveLength(1); + }); + + it('analyzes orders after the load and before the first replayed request', () => { + const sql = seed(); + const analyze = sql.indexOf('analyze public.orders;'); + expect(analyze).toBeGreaterThan(sql.indexOf('insert into public.orders')); + expect(analyze).toBeLessThan(sql.indexOf('select id, ')); + }); + + it('replays 200 order-history lookups and 100 lookups by primary key', () => { + const all = statements(seed()); + expect(all.filter((sql) => CUSTOMER_LOOKUP_RE.test(sql))).toHaveLength(200); + expect(all.filter((sql) => ID_LOOKUP_RE.test(sql))).toHaveLength(100); + }); + + it('holds nothing but the load, the analyze and the 300 replayed requests', () => { + expect(statements(seed())).toHaveLength(302); + }); + + it('replays the order-history lookup in exactly the shape the scorer explains', () => { + const lookup = statements(seed()).find((sql) => + CUSTOMER_LOOKUP_RE.test(sql) + ) as string; + const customer = lookup.match(CUSTOMER_LOOKUP_RE)?.[1] as string; + expect(lookup.replace(customer, PROBE_CUSTOMER)).toBe(LOOKUP_QUERY); + }); + + it('interleaves the two kinds of request', () => { + const first = statements(seed()) + .filter((sql) => CUSTOMER_LOOKUP_RE.test(sql) || ID_LOOKUP_RE.test(sql)) + .slice(0, 6) + .map((sql) => (ID_LOOKUP_RE.test(sql) ? 'id' : 'customer')); + expect(new Set(first)).toEqual(new Set(['id', 'customer'])); + }); + + it('looks up customers and ids that exist in the loaded data', () => { + const numbers = (re: RegExp, group: number) => + statements(seed()).flatMap((sql) => { + const match = sql.match(re); + return match ? [Number(match[group])] : []; + }); + const customers = numbers(CUSTOMER_LOOKUP_RE, 2); + const ids = numbers(ID_LOOKUP_RE, 1); + expect(customers).toHaveLength(200); + expect(ids).toHaveLength(100); + expect(customers.every((n) => n < 5000)).toBe(true); + expect(ids.every((n) => n >= 1 && n <= SEEDED_ORDER_COUNT)).toBe(true); + }); + + it('never replays the customer the scorer probes with', () => { + expect(seed()).not.toContain(PROBE_CUSTOMER); + }); + + it('does not create an index', () => { + expect(seed()).not.toMatch(/create\s+(?:unique\s+)?index/i); + }); +}); + +describe('seeded project layout', () => { + it('is loaded from seed.sql by config.toml on a Postgres 17 project without pinned ports', () => { + const config = read('local/supabase/config.toml'); + expect(config).toMatch(/^project_id = "shop-app"$/m); + expect(config).toMatch(/\[db\]\s+major_version = 17/); + expect(config).toMatch(/\[db\.migrations\]\s+enabled = true/); + expect(config).toMatch(/\[db\.seed\]\s+enabled = true/); + expect(config).toContain('sql_paths = ["./seed.sql"]'); + expect(config).not.toMatch(/^\s*port\s*=/m); + }); + + it.each(['local/package.json', 'local/.env', 'local/supabase/schemas'])( + 'ships no %s, so nothing steers the agent to a package runner or a declarative schema', + (path) => { + expect(existsSync(evalUrl(path))).toBe(false); + } + ); + + it('ships only the supabase directory, with no app code', () => { + expect(readdirSync(evalUrl('local/'))).toEqual(['supabase']); + expect(readdirSync(evalUrl('local/supabase/')).sort()).toEqual([ + 'config.toml', + 'migrations', + 'seed.sql', + ]); + }); +}); + +describe('PROMPT.md', () => { + it('carries the user message exactly', () => { + expect(promptParts().body.trim()).toBe(PROMPT_BODY); + }); + + it.each(FORBIDDEN_PROMPT_WORDS)( + 'does not give the agent the word "%s"', + (word) => { + expect(promptParts().body.toLowerCase()).not.toContain(word); + } + ); + + it('keeps the words a user who knows the tool but not the subcommand would use', () => { + const { body } = promptParts(); + for (const phrase of ['Supabase CLI', 'local database', 'slow', 'fix']) { + expect(body).toContain(phrase); + } + }); + + it('declares a CLI eval that investigates diagnostics and starts without a running project or docker', () => { + const { frontmatter } = promptParts(); + expect(scalar(frontmatter, 'stage')).toBe('investigate'); + expect(scalar(frontmatter, 'interface')).toBe('cli'); + expect(listItems(frontmatter, 'product')).toEqual(['database']); + expect(listItems(frontmatter, 'topic').sort()).toEqual([ + 'migrations', + 'observability', + ]); + expect(scalar(frontmatter, 'projectRunning')).toBe('false'); + expect(scalar(frontmatter, 'needsDocker')).toBe('false'); + expect(scalar(frontmatter, 'area')).toBe('diagnostics'); + expect(scalar(frontmatter, 'journeyPhase')).toBe('observe'); + }); + + it.each(['CLI-2712', 'FDBKIN-1481', 'FDBKIN-1663', 'FDBKIN-13604'])( + 'cites %s as motivation', + (id) => { + expect(scalar(promptParts().frontmatter, 'motivation')).toContain(id); + } + ); +}); diff --git a/evals/cli/investigate-database-001-slow-query/fixtures.ts b/evals/cli/investigate-database-001-slow-query/fixtures.ts new file mode 100644 index 00000000..ffb13c0f --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/fixtures.ts @@ -0,0 +1,14 @@ +export const ORDERS_TABLE = 'orders'; +export const EMAIL_COLUMN = 'customer_email'; +export const SEEDED_ORDER_COUNT = 300_000; +export const SEEDED_MIGRATION_VERSION = '20260101000000'; +export const SEEDED_MIGRATION_FILE = '20260101000000_create_orders.sql'; + +/** sha256 (hex) of `local/supabase/migrations/`. */ +export const SEEDED_MIGRATION_SHA256 = + '5a3c4f8f826d852664438c4582eceb3b682e2a0e4363d04eea2d2e9237f704f7'; + +export const PROBE_CUSTOMER = 'customer-42@example.com'; + +/** The app's order-history lookup, as the seed replays it. */ +export const LOOKUP_QUERY = `select id, status, total_cents, created_at from public.orders where customer_email = '${PROBE_CUSTOMER}' order by created_at desc limit 20`; diff --git a/evals/cli/investigate-database-001-slow-query/inspect.test.ts b/evals/cli/investigate-database-001-slow-query/inspect.test.ts new file mode 100644 index 00000000..03225681 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/inspect.test.ts @@ -0,0 +1,445 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import type { ToolCallRecord } from '@supabase-evals/core'; +import { describe, expect, it } from 'vitest'; +import { + findSupabaseInvocations, + type SupabaseInvocation, +} from '../lib/cli-invocations.js'; +import { extractCommandEntries } from '../lib/detours.js'; +import { + checkInspectShowedOrders, + findInspectRuns, + INSPECT_CHECK_NAME, + type InspectRun, + inspectReport, +} from './inspect.js'; + +const OUTLIERS = `|QUERY|EXECUTION TIME|PROPORTION OF EXEC TIME|NUMBER CALLS|SYNC IO TIME| +|-|-|-|-|-| +|\`select id, status, total_cents, created_at from public.orders where customer_email = $1 order by created_at desc limit $2\`|00:00:02.810|71.4%|200|00:00:00| +|\`insert into public.orders (customer_email, status, total_cents, created_at) select $1 || (n % $2) || $3, (array[$4, $5, $6, $7])[$8 + n % $9], $10 + (n * $11) % $12, timestamptz $13 + n * interval $14 from generate_series($15, $16) as n\`|00:00:00.940|23.9%|1|00:00:00| +`; + +const TABLE_STATS_JSON = `[{"name":"public.orders","table_size":"21 MB","index_size":"6600 kB","estimated_row_count":300000,"seq_scans":200}]`; + +const UNRELATED_OUTLIERS = `|QUERY|EXECUTION TIME|PROPORTION OF EXEC TIME|NUMBER CALLS|SYNC IO TIME| +|-|-|-|-|-| +|\`select name from pg_timezone_names\`|00:00:00.120|88.0%|3|00:00:00| +`; + +const NO_PROJECT_REF = 'Cannot find project ref. Have you run supabase link?'; + +function call( + command: string, + output: { result?: unknown; error?: string } = {} +): ToolCallRecord { + return { + tool: { kind: 'other', toolName: 'shell' }, + body: { command }, + command, + result: 'result' in output ? output.result : '', + ...(output.error === undefined ? {} : { error: output.error }), + ts: 0, + }; +} + +const fileWrite = (result = ''): ToolCallRecord => ({ + tool: { kind: 'other', toolName: 'file_write' }, + body: { path: 'supabase/migrations/20260102000000_idx.sql' }, + result, + ts: 0, +}); + +const invocationsOf = (toolCalls: ToolCallRecord[]) => + findSupabaseInvocations(extractCommandEntries(toolCalls)); + +const runsOf = (toolCalls: ToolCallRecord[]) => + findInspectRuns(toolCalls, invocationsOf(toolCalls)); + +const inspect = (command: string, stdout: string) => + call(command, { result: stdout }); + +describe('inspectReport', () => { + const reportOf = (command: string) => { + const [invocation] = findSupabaseInvocations([command]); + return inspectReport(invocation); + }; + + it.each([ + ['supabase inspect db outliers --local', 'db outliers'], + ['supabase inspect db outliers', 'db outliers'], + ['supabase inspect db outliers --linked', 'db outliers'], + ['supabase inspect db table-stats --local -o json', 'db table-stats'], + ['supabase inspect db calls --local --output-format json', 'db calls'], + ['supabase inspect db seq-scans --local', 'db seq-scans'], + ['supabase inspect db index-usage --local', 'db index-usage'], + ['supabase --workdir /ws inspect db outliers --local', 'db outliers'], + ['npx supabase@beta inspect db calls --local | head -40', 'db calls'], + [ + 'npx --yes supabase@2.120.0 inspect db table-stats --local', + 'db table-stats', + ], + ['cd /ws && supabase inspect db outliers --local', 'db outliers'], + ['supabase inspect report --local', 'report'], + ['supabase inspect db', 'db'], + ])('reads %j as the report %j', (command, expected) => { + expect(reportOf(command)).toBe(expected); + }); + + it('leaves the connection string out of the report', () => { + expect( + reportOf( + 'supabase inspect db outliers --db-url postgresql://postgres:postgres@127.0.0.1:54322/postgres' + ) + ).toBe('db outliers'); + }); + + it('leaves the output directory out of the report', () => { + expect( + reportOf('supabase inspect report --local --output-dir /tmp/r') + ).toBe('report'); + }); + + it.each([ + 'supabase start', + 'supabase db reset', + 'supabase migration up', + 'supabase status', + 'supabase db query --local "select 1"', + 'supabase db query --local "select * from pg_stat_statements"', + ])('is undefined for %j', (command) => { + expect(reportOf(command)).toBeUndefined(); + }); +}); + +describe('findInspectRuns', () => { + it('maps an inspect invocation to its own tool call and reads the table it printed', () => { + expect( + runsOf([ + call('supabase start'), + inspect('supabase inspect db outliers --local', OUTLIERS), + ]) + ).toEqual([ + { + commandIndex: 1, + report: 'db outliers', + errored: false, + namesOrders: true, + }, + ]); + }); + + it('reads the output of its own call when an earlier tool call ran no command', () => { + const skipped = [ + fileWrite('public.orders'), + inspect('supabase inspect db outliers --local', UNRELATED_OUTLIERS), + ]; + expect(runsOf(skipped)).toEqual([ + { + commandIndex: 0, + report: 'db outliers', + errored: false, + namesOrders: false, + }, + ]); + const showing = [ + fileWrite(), + call('echo done', { result: 'unrelated' }), + inspect('supabase inspect db table-stats --local', TABLE_STATS_JSON), + ]; + expect(runsOf(showing)).toEqual([ + { + commandIndex: 1, + report: 'db table-stats', + errored: false, + namesOrders: true, + }, + ]); + }); + + it('does not count a report that never mentions orders', () => { + expect( + runsOf([ + inspect('supabase inspect db outliers --local', UNRELATED_OUTLIERS), + ]) + ).toMatchObject([{ report: 'db outliers', namesOrders: false }]); + }); + + it('marks a call that errored, whatever its error says', () => { + expect( + runsOf([call('supabase inspect db outliers', { error: NO_PROJECT_REF })]) + ).toEqual([ + { + commandIndex: 0, + report: 'db outliers', + errored: true, + namesOrders: false, + }, + ]); + }); + + it('reads orders out of an error, as a command that exits non-zero keeps its output there', () => { + expect( + runsOf([ + call('supabase inspect db outliers --local | head -40', { + error: `exit code 141\n${OUTLIERS}`, + }), + ]) + ).toEqual([ + { + commandIndex: 0, + report: 'db outliers', + errored: true, + namesOrders: true, + }, + ]); + }); + + it.each([ + ['a string result', OUTLIERS, true], + ['an object result', { stdout: OUTLIERS, exitCode: 0 }, true], + [ + 'an object whose orders sits at the start of a line', + { stdout: 'NAME\norders\n' }, + true, + ], + ['an array of lines', ['NAME', 'public.orders'], true], + ['a null result', null, false], + ['an undefined result', undefined, false], + ['an object without it', { stdout: UNRELATED_OUTLIERS }, false], + ])('reads %s', (_label, result, expected) => { + expect( + runsOf([call('supabase inspect db outliers --local', { result })])[0] + .namesOrders + ).toBe(expected); + }); + + it.each([ + ['a qualified name', '| public.orders | 21 MB |', true], + ['a bare name', 'orders 300000', true], + ['a quoted name', '"orders"', true], + ['a name in a JSON field', TABLE_STATS_JSON, true], + ['a name closing a sentence', 'Sequential scans on orders.', true], + [ + 'a table that merely starts with the name', + 'public.orders_archive', + false, + ], + ['a table that merely ends with the name', 'public.preorders', false], + ['a column that holds the name', 'customer_orders', false], + ['a plural of something else', 'reorders 12', false], + ['nothing', '', false], + ])('matches orders as a whole word in %s', (_label, stdout, expected) => { + expect( + runsOf([inspect('supabase inspect db table-stats --local', stdout)])[0] + .namesOrders + ).toBe(expected); + }); + + it('reads a name a terminal coloured in pieces', () => { + const coloured = + '\u001b[1mpublic.\u001b[22m\u001b[32morders\u001b[0m \u001b[33m300000\u001b[0m'; + expect( + runsOf([inspect('supabase inspect db table-stats --local', coloured)])[0] + .namesOrders + ).toBe(true); + }); + + it('reads the output of inspect report cat-ed in the same call', () => { + const csv = + 'query,calls\nselect id from public.orders where customer_email = $1,200\n'; + const runs = runsOf([ + inspect( + 'supabase inspect report --local --output-dir /tmp/r && cat /tmp/r/*/outliers.csv', + `Wrote 12 reports to /tmp/r/2026-01-05\n${csv}` + ), + ]); + expect(runs).toEqual([ + { commandIndex: 0, report: 'report', errored: false, namesOrders: true }, + ]); + }); + + it('does not count an inspect report whose rules table never names orders', () => { + const rules = '|RULE|STATUS|\n|-|-|\n|Unused indexes|PASS|\n'; + expect( + runsOf([ + inspect('supabase inspect report --local --output-dir /tmp/r', rules), + ])[0].namesOrders + ).toBe(false); + }); + + it.each([ + [ + 'a package runner with a pinned channel and a pipe', + 'npx supabase@beta inspect db calls --local | head -40', + 'db calls', + ], + [ + 'a deprecated alias', + 'supabase inspect db seq-scans --local', + 'db seq-scans', + ], + [ + 'a connection string', + 'supabase inspect db outliers --db-url postgresql://postgres:postgres@127.0.0.1:54322/postgres', + 'db outliers', + ], + [ + 'json output', + 'supabase inspect db table-stats --local --output-format json', + 'db table-stats', + ], + ])('counts %s', (_label, command, report) => { + expect(runsOf([inspect(command, 'public.orders 21 MB')])).toEqual([ + { commandIndex: 0, report, errored: false, namesOrders: true }, + ]); + }); + + it('lists every inspect invocation in order', () => { + const runs = runsOf([ + call('supabase inspect db outliers', { error: NO_PROJECT_REF }), + inspect('supabase inspect db outliers --local', OUTLIERS), + call('supabase start'), + inspect('supabase inspect db calls --local', UNRELATED_OUTLIERS), + ]); + expect( + runs.map(({ commandIndex, report }) => [commandIndex, report]) + ).toEqual([ + [0, 'db outliers'], + [1, 'db outliers'], + [3, 'db calls'], + ]); + }); + + it('lists each inspect invocation of one tool call, all reading that call output', () => { + const runs = runsOf([ + inspect( + 'supabase inspect db outliers --local && supabase inspect db calls --local', + OUTLIERS + ), + ]); + expect( + runs.map(({ commandIndex, report }) => [commandIndex, report]) + ).toEqual([ + [0, 'db outliers'], + [0, 'db calls'], + ]); + }); + + it.each([ + [ + 'psql against pg_stat_statements', + `psql 'postgresql://postgres:postgres@127.0.0.1:54322/postgres' -XtAc "select query, calls from pg_stat_statements order by total_exec_time desc limit 5"`, + ], + [ + 'supabase db query against pg_stat_statements', + 'supabase db query --local "select query, calls from pg_stat_statements order by calls desc"', + ], + ['an echo that names the command', 'echo "supabase inspect db outliers"'], + ['a comment that names the command', '# supabase inspect db outliers'], + [ + 'a commit message that names the command', + 'git commit -m "run supabase inspect db outliers"', + ], + ['help for the command', 'supabase inspect db --help'], + ['short help for the command', 'supabase inspect db outliers -h'], + ])( + 'finds no run in %s even when its output names orders', + (_label, command) => { + expect(runsOf([inspect(command, OUTLIERS)])).toEqual([]); + } + ); + + it('does not let a help invocation pass the check even when handed one directly', () => { + const help: SupabaseInvocation = { + commandIndex: 0, + argv: ['supabase', 'inspect', 'db', '--help'], + }; + const calls = [inspect('supabase inspect db --help', OUTLIERS)]; + expect( + checkInspectShowedOrders(findInspectRuns(calls, [help])).passed + ).toBe(false); + }); + + it('does not throw for an invocation that points past the tool calls', () => { + const stray: SupabaseInvocation = { + commandIndex: 5, + argv: ['supabase', 'inspect', 'db', 'outliers', '--local'], + }; + expect(() => findInspectRuns([], [stray])).not.toThrow(); + }); + + it('is empty when the agent ran nothing', () => { + expect(findInspectRuns([], [])).toEqual([]); + }); +}); + +const run = (overrides: Partial = {}): InspectRun => ({ + commandIndex: 1, + report: 'db outliers', + errored: false, + namesOrders: true, + ...overrides, +}); + +describe('checkInspectShowedOrders', () => { + it('passes when a report named orders and says which tool call ran it', () => { + const check = checkInspectShowedOrders([run()]); + expect(check).toMatchObject({ name: INSPECT_CHECK_NAME, passed: true }); + expect(check.notes).toContain('tool call 2: supabase inspect db outliers'); + }); + + it('names the first report that showed orders when several ran', () => { + const check = checkInspectShowedOrders([ + run({ commandIndex: 0, report: 'db calls', namesOrders: false }), + run({ commandIndex: 1, report: 'db table-stats' }), + run({ commandIndex: 2, report: 'db outliers' }), + ]); + expect(check.passed).toBe(true); + expect(check.notes).toContain( + 'tool call 2: supabase inspect db table-stats' + ); + }); + + it('passes a run that errored after printing orders', () => { + expect(checkInspectShowedOrders([run({ errored: true })]).passed).toBe( + true + ); + }); + + it('passes when one run errored and another showed orders', () => { + const check = checkInspectShowedOrders([ + run({ commandIndex: 0, errored: true, namesOrders: false }), + run({ commandIndex: 1, report: 'db table-stats' }), + ]); + expect(check.passed).toBe(true); + }); + + it('fails when no inspect report ran', () => { + expect(checkInspectShowedOrders([])).toEqual({ + name: INSPECT_CHECK_NAME, + passed: false, + notes: 'no supabase inspect report ran', + }); + }); + + it('fails when the reports that ran never named orders, listing them and marking the errored ones', () => { + const check = checkInspectShowedOrders([ + run({ commandIndex: 0, errored: true, namesOrders: false }), + run({ commandIndex: 1, report: 'db calls', namesOrders: false }), + ]); + expect(check).toMatchObject({ name: INSPECT_CHECK_NAME, passed: false }); + expect(check.notes).toContain('db outliers'); + expect(check.notes).toContain('db calls'); + expect(check.notes).toMatch(/errored/); + expect(check.notes).toMatch(/orders/); + }); + + it('marks nothing as errored when no report errored', () => { + const check = checkInspectShowedOrders([ + run({ report: 'db calls', namesOrders: false }), + ]); + expect(check.passed).toBe(false); + expect(check.notes).not.toMatch(/errored/); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/inspect.ts b/evals/cli/investigate-database-001-slow-query/inspect.ts new file mode 100644 index 00000000..78ff9e89 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/inspect.ts @@ -0,0 +1,88 @@ +import type { CheckResult, ToolCallRecord } from '@supabase-evals/core'; +import { + invocationPositionals, + type SupabaseInvocation, +} from '../lib/cli-invocations.js'; +import { commandToolCalls } from '../lib/detours.js'; +import { ORDERS_TABLE } from './fixtures.js'; + +export type InspectRun = { + commandIndex: number; + report: string; + errored: boolean; + namesOrders: boolean; +}; + +export const INSPECT_CHECK_NAME = + 'an inspect db report showed the orders table'; + +const ORDERS_RE = new RegExp(`\\b${ORDERS_TABLE}\\b`); +const HELP_FLAGS = new Set(['--help', '-h']); +const PATH_VALUE_FLAGS = new Set(['--db-url', '--output-dir']); +const CSI_SEQUENCE = /\x1b\[[0-9;?]*[ -\/]*[@-~]/g; + +/** The report an executed `supabase inspect` invocation ran, such as `db outliers` or `report`; undefined for anything else, help included. */ +export function inspectReport(inv: SupabaseInvocation): string | undefined { + if (inv.argv.some((word) => HELP_FLAGS.has(word))) return undefined; + const argv = inv.argv.filter( + (_, i) => !PATH_VALUE_FLAGS.has(inv.argv[i - 1]) + ); + const [verb, first, second] = invocationPositionals({ ...inv, argv }); + if (verb !== 'inspect' || first === undefined) return undefined; + return first === 'db' && second !== undefined ? `db ${second}` : first; +} + +function resultText(result: unknown): string { + if (typeof result === 'string') return result; + if (result === null || result === undefined) return ''; + return JSON.stringify(result).replace(/\\[nrt]/g, ' '); +} + +/** A call's result and error together, since a command that exits non-zero keeps its output in `error`. */ +const callText = ({ result, error }: ToolCallRecord): string => + `${resultText(result)}\n${error ?? ''}`.replace(CSI_SEQUENCE, ''); + +/** One run per executed `supabase inspect` report, read from the output of the tool call that ran it. */ +export function findInspectRuns( + toolCalls: readonly ToolCallRecord[], + invocations: readonly SupabaseInvocation[] +): InspectRun[] { + const calls = commandToolCalls(toolCalls); + return invocations.flatMap((inv): InspectRun[] => { + const report = inspectReport(inv); + const call = calls[inv.commandIndex]; + if (report === undefined || call === undefined) return []; + return [ + { + commandIndex: inv.commandIndex, + report, + errored: call.error !== undefined, + namesOrders: ORDERS_RE.test(callText(call)), + }, + ]; + }); +} + +const describeRun = ({ commandIndex, report }: InspectRun) => + `tool call ${commandIndex + 1}: supabase inspect ${report}`; + +export function checkInspectShowedOrders( + runs: readonly InspectRun[] +): CheckResult { + const name = INSPECT_CHECK_NAME; + const shown = runs.find(({ namesOrders }) => namesOrders); + if (shown !== undefined) { + return { name, passed: true, notes: describeRun(shown) }; + } + if (runs.length === 0) { + return { name, passed: false, notes: 'no supabase inspect report ran' }; + } + const ran = runs + .map((run) => `${describeRun(run)}${run.errored ? ' (errored)' : ''}`) + .join('; '); + return { + name, + passed: false, + notes: `${ran}; none printed the ${ORDERS_TABLE} table`, + }; +} diff --git a/evals/cli/investigate-database-001-slow-query/local/supabase/config.toml b/evals/cli/investigate-database-001-slow-query/local/supabase/config.toml new file mode 100644 index 00000000..01955a89 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/local/supabase/config.toml @@ -0,0 +1,11 @@ +project_id = "shop-app" + +[db] +major_version = 17 + +[db.migrations] +enabled = true + +[db.seed] +enabled = true +sql_paths = ["./seed.sql"] diff --git a/evals/cli/investigate-database-001-slow-query/local/supabase/migrations/20260101000000_create_orders.sql b/evals/cli/investigate-database-001-slow-query/local/supabase/migrations/20260101000000_create_orders.sql new file mode 100644 index 00000000..ec71ff51 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/local/supabase/migrations/20260101000000_create_orders.sql @@ -0,0 +1,7 @@ +create table public.orders ( + id bigint generated always as identity primary key, + customer_email text not null, + status text not null default 'pending', + total_cents integer not null, + created_at timestamptz not null default now() +); diff --git a/evals/cli/investigate-database-001-slow-query/local/supabase/seed.sql b/evals/cli/investigate-database-001-slow-query/local/supabase/seed.sql new file mode 100644 index 00000000..3df1b7bf --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/local/supabase/seed.sql @@ -0,0 +1,313 @@ +-- 300,000 orders from 5,000 customers: the order history imported from the old shop. +insert into public.orders (customer_email, status, total_cents, created_at) +select + 'customer-' || (n % 5000) || '@example.com', + (array['pending', 'paid', 'shipped', 'refunded'])[1 + n % 4], + 500 + (n * 37) % 49500, + timestamptz '2024-01-01 00:00:00+00' + n * interval '3 minutes' +from generate_series(1, 300000) as n; + +analyze public.orders; + +-- A sample of the app's requests from the last hour, replayed so the local +-- database sees the same workload as production. +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-13@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-110@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 3055; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-207@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-304@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 8961; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-401@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-498@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 14867; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-595@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-692@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 20773; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-789@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-886@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 26679; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-983@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1080@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 32585; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1177@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1274@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 38491; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1371@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1468@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 44397; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1565@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1662@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 50303; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1759@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1856@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 56209; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1953@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2050@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 62115; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2147@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2244@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 68021; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2341@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2438@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 73927; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2535@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2632@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 79833; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2729@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2826@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 85739; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2923@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3020@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 91645; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3117@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3214@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 97551; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3311@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3408@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 103457; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3505@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3602@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 109363; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3699@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3796@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 115269; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3893@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3990@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 121175; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4087@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4184@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 127081; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4281@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4378@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 132987; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4475@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4572@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 138893; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4669@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4766@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 144799; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4863@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4960@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 150705; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-57@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-154@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 156611; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-251@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-348@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 162517; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-445@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-542@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 168423; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-639@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-736@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 174329; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-833@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-930@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 180235; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1027@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1124@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 186141; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1221@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1318@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 192047; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1415@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1512@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 197953; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1609@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1706@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 203859; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1803@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1900@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 209765; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1997@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2094@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 215671; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2191@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2288@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 221577; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2385@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2482@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 227483; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2579@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2676@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 233389; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2773@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2870@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 239295; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2967@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3064@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 245201; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3161@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3258@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 251107; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3355@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3452@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 257013; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3549@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3646@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 262919; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3743@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3840@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 268825; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3937@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4034@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 274731; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4131@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4228@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 280637; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4325@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4422@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 286543; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4519@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4616@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 292449; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4713@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4810@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 298355; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4907@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 4261; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-101@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-198@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 10167; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-295@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-392@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 16073; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-489@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-586@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 21979; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-683@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-780@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 27885; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-877@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-974@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 33791; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1071@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1168@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 39697; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1265@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1362@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 45603; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1459@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1556@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 51509; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1653@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1750@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 57415; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1847@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1944@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 63321; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2041@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2138@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 69227; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2235@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2332@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 75133; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2429@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2526@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 81039; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2623@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2720@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 86945; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2817@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2914@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 92851; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3011@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3108@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 98757; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3205@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3302@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 104663; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3399@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3496@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 110569; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3593@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3690@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 116475; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3787@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3884@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 122381; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3981@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4078@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 128287; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4175@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4272@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 134193; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4369@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4466@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 140099; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4563@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4660@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 146005; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4757@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4854@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 151911; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4951@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-48@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 157817; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-145@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-242@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 163723; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-339@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-436@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 169629; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-533@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-630@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 175535; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-727@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-824@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 181441; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-921@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1018@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 187347; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1115@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1212@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 193253; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1309@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1406@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 199159; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1503@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1600@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 205065; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1697@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1794@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 210971; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1891@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1988@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 216877; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2085@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2182@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 222783; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2279@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2376@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 228689; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2473@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2570@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 234595; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2667@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2764@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 240501; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2861@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2958@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 246407; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3055@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3152@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 252313; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3249@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3346@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 258219; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3443@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3540@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 264125; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3637@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3734@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 270031; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3831@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3928@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 275937; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4025@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4122@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 281843; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4219@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4316@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 287749; diff --git a/evals/cli/investigate-database-001-slow-query/metrics.test.ts b/evals/cli/investigate-database-001-slow-query/metrics.test.ts new file mode 100644 index 00000000..cffedcf4 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/metrics.test.ts @@ -0,0 +1,841 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import type { + CommandResult, + LocalStackEnvironmentMarker, + LocalStackEvalContext, +} from '@supabase-evals/core'; +import { describe, expect, it } from 'vitest'; +import { + findSupabaseInvocations, + type SupabaseInvocation, +} from '../lib/cli-invocations.js'; +import type { StackProbe } from '../lib/stack.js'; +import { + LOOKUP_QUERY, + SEEDED_MIGRATION_FILE, + SEEDED_MIGRATION_SHA256, +} from './fixtures.js'; +import type { InspectRun } from './inspect.js'; +import { + checkMetrics, + type CliFacts, + type ProjectFileFacts, + readCliFacts, + readProjectFileFacts, +} from './metrics.js'; + +const DB_URL = 'postgresql://postgres:hunter2@127.0.0.1:54322/postgres'; +const CREATE_INDEX_SQL = + 'create index orders_customer_email_idx on public.orders (customer_email)'; + +const MARKER: LocalStackEnvironmentMarker = { + runtime: 'local-stack', + channel: 'beta', + cliVersion: '2.0.0', + docker: 'available', + sessionStartedMs: 1_000, +}; + +function result(stdout: string, ok = true, stderr = ''): CommandResult { + return { ok, exitCode: ok ? 0 : 1, stdout, stderr }; +} + +const versionCtx: Pick = { + exec: async (command) => + command.endsWith('supabase --version') + ? result('2.0.0\n') + : result('', false, 'error'), +}; + +type Project = { + seededHash?: string | 'missing'; + migrations?: string[]; + throwOn?: Array<'sha256sum' | 'migrations'>; +}; + +const OTHER_HASH = 'f'.repeat(64); + +function fakeProject({ + seededHash = SEEDED_MIGRATION_SHA256, + migrations = [SEEDED_MIGRATION_FILE], + throwOn = [], +}: Project = {}): Pick { + return { + exec: async (command) => { + if (command.includes('sha256sum')) { + if (throwOn.includes('sha256sum')) throw new Error('sandbox gone'); + if (!command.includes(SEEDED_MIGRATION_FILE)) { + return result('', false, `unexpected command: ${command}`); + } + return seededHash === 'missing' + ? result( + '', + false, + `sha256sum: supabase/migrations/${SEEDED_MIGRATION_FILE}: No such file or directory` + ) + : result( + `${seededHash} supabase/migrations/${SEEDED_MIGRATION_FILE}\n` + ); + } + if (command.includes('supabase/migrations')) { + if (throwOn.includes('migrations')) throw new Error('sandbox gone'); + return command.includes('find ') + ? result( + migrations.map((name) => `supabase/migrations/${name}`).join('\n') + ) + : result(migrations.join('\n')); + } + return result('', false, `unexpected command: ${command}`); + }, + }; +} + +describe('readProjectFileFacts', () => { + it('reports an untouched project', async () => { + expect(await readProjectFileFacts(fakeProject())).toEqual({ + seededMigrationModified: false, + newMigrationFiles: 0, + }); + }); + + it('counts migration files beside the seeded one', async () => { + const ctx = fakeProject({ + migrations: [ + SEEDED_MIGRATION_FILE, + '20260102000000_add_orders_customer_email_idx.sql', + '20260103000000_more.sql', + ], + }); + expect(await readProjectFileFacts(ctx)).toEqual({ + seededMigrationModified: false, + newMigrationFiles: 2, + }); + }); + + it('does not count files that are not .sql', async () => { + const ctx = fakeProject({ + migrations: [SEEDED_MIGRATION_FILE, 'README.md', '.gitkeep'], + }); + expect((await readProjectFileFacts(ctx)).newMigrationFiles).toBe(0); + }); + + it('reports a seeded migration whose checksum changed', async () => { + const ctx = fakeProject({ seededHash: OTHER_HASH }); + expect(await readProjectFileFacts(ctx)).toEqual({ + seededMigrationModified: true, + newMigrationFiles: 0, + }); + }); + + it('reports a deleted seeded migration as modified and still counts the others', async () => { + const ctx = fakeProject({ + seededHash: 'missing', + migrations: ['20260102000000_add_orders_customer_email_idx.sql'], + }); + expect(await readProjectFileFacts(ctx)).toEqual({ + seededMigrationModified: true, + newMigrationFiles: 1, + }); + }); + + it.each([ + [ + 'the checksum probe', + 'sha256sum', + { seededMigrationModified: null, newMigrationFiles: 1 }, + ], + [ + 'the migrations listing', + 'migrations', + { seededMigrationModified: false, newMigrationFiles: null }, + ], + ] satisfies Array<[string, 'sha256sum' | 'migrations', ProjectFileFacts]>)( + 'loses only its own field when %s throws', + async (_label, throwOn, expected) => { + const ctx = fakeProject({ + migrations: [ + SEEDED_MIGRATION_FILE, + '20260102000000_add_orders_customer_email_idx.sql', + ], + throwOn: [throwOn], + }); + expect(await readProjectFileFacts(ctx)).toEqual(expected); + } + ); + + it('reports every field as unknown when every probe throws', async () => { + const ctx = fakeProject({ throwOn: ['sha256sum', 'migrations'] }); + expect(await readProjectFileFacts(ctx)).toEqual({ + seededMigrationModified: null, + newMigrationFiles: null, + }); + }); +}); + +describe('readCliFacts', () => { + it('reports the staged version with no override for a plain run', async () => { + const invocations = findSupabaseInvocations(['supabase start']); + expect(await readCliFacts(versionCtx, MARKER, invocations)).toEqual({ + versions: { cliVersion: '2.0.0' }, + cliOverride: [], + }); + }); + + it('reports a version-swapping runner and a PATH version that moved', async () => { + const moved: Pick = { + exec: async () => result('2.2.0\n'), + }; + const invocations = findSupabaseInvocations([ + 'npx --yes supabase@2.1.0 inspect db outliers --local', + ]); + expect(await readCliFacts(moved, MARKER, invocations)).toEqual({ + versions: { cliVersion: '2.0.0', cliVersionAfterRun: '2.2.0' }, + cliOverride: ['npx --yes supabase@2.1.0'], + }); + }); + + it('reports an unknown version without failing when the sandbox is unreachable', async () => { + const gone: Pick = { + exec: async () => { + throw new Error('sandbox gone'); + }, + }; + expect(await readCliFacts(gone, undefined, [])).toMatchObject({ + versions: { cliVersion: null }, + }); + }); +}); + +const STACK: StackProbe = { + ok: true, + backend: 'managed', + dbUrl: DB_URL, + runtime: 'native', +}; + +const NO_STACK: StackProbe = { ok: false, notes: 'none' }; +const NO_CLI: CliFacts = { versions: null, cliOverride: null }; +const FILES: ProjectFileFacts = { + seededMigrationModified: false, + newMigrationFiles: 1, +}; + +type MetricsFacts = Parameters[1]; + +const run = (overrides: Partial = {}): InspectRun => ({ + commandIndex: 0, + report: 'db outliers', + errored: false, + namesOrders: true, + ...overrides, +}); + +function metricsFacts(overrides: Partial = {}): MetricsFacts { + const commands = overrides.commands ?? []; + return { + commands, + invocations: overrides.invocations ?? findSupabaseInvocations(commands), + stack: STACK, + cli: { versions: { cliVersion: '2.0.0' }, cliOverride: [] }, + files: FILES, + inspectRuns: [], + indexes: { + ok: true, + indexes: [ + { name: 'orders_pkey', definition: 'CREATE UNIQUE INDEX orders_pkey' }, + ], + }, + rows: { ok: true, count: 300_000 }, + ...overrides, + }; +} + +const metricsOf = (check: { notes?: string }) => + JSON.parse(check.notes ?? '{}'); + +const metricsFor = async (overrides: Partial = {}) => + metricsOf(await checkMetrics(MARKER, metricsFacts(overrides))); + +const unreadable = () => + new Proxy([], { + get() { + throw new Error('unreadable'); + }, + }) as unknown as T; + +const workflow = async (commands: string[]) => + (await metricsFor({ commands })).workflowCommands; + +describe('checkMetrics', () => { + it('reports every field of a finished run and always passes', async () => { + const commands = [ + 'supabase start', + 'supabase inspect db outliers --local', + `psql '${DB_URL}' -XtAc "explain (format json) ${LOOKUP_QUERY}"`, + `psql '${DB_URL}' -XtAc "${CREATE_INDEX_SQL}"`, + 'supabase migration up', + 'sudo apt-get install docker.io', + ]; + const check = await checkMetrics( + MARKER, + metricsFacts({ + commands, + inspectRuns: [run({ commandIndex: 1 })], + indexes: { + ok: true, + indexes: [ + { + name: 'orders_pkey', + definition: 'CREATE UNIQUE INDEX orders_pkey', + }, + { + name: 'orders_customer_email_idx', + definition: CREATE_INDEX_SQL, + }, + ], + }, + }) + ); + expect(check).toMatchObject({ name: 'metrics', passed: true }); + expect(metricsOf(check)).toEqual({ + cliVersion: '2.0.0', + cliOverride: [], + channel: 'beta', + backend: 'managed', + runtime: 'native', + relocatedHome: null, + dbPort: 54322, + inspectReports: ['db outliers'], + inspectRuns: 1, + inspectRunsShowingOrders: 1, + workflowCommands: ['start', 'inspect db outliers', 'migration up'], + explainCommands: 1, + directIndexDdl: 1, + ordersIndexes: ['orders_pkey', 'orders_customer_email_idx'], + orderRows: 300_000, + newMigrationFiles: 1, + seededMigrationModified: false, + cliDetours: 1, + }); + }); + + it('reports the relocated home and the version the PATH moved to', async () => { + const metrics = await metricsFor({ + stack: { ...STACK, relocatedHome: '/tmp/agent/.supabase' }, + cli: { + versions: { cliVersion: '2.0.0', cliVersionAfterRun: '2.2.0' }, + cliOverride: ['npx --yes supabase@2.1.0'], + }, + }); + expect(metrics).toMatchObject({ + relocatedHome: '/tmp/agent/.supabase', + cliVersion: '2.0.0', + cliVersionAfterRun: '2.2.0', + cliOverride: ['npx --yes supabase@2.1.0'], + }); + }); + + it('leaves out cliVersionAfterRun when the PATH version did not move', async () => { + expect(await metricsFor()).not.toHaveProperty('cliVersionAfterRun'); + }); + + it('reports the pinned channel when the marker names none or is absent', async () => { + const unchanneled = { ...MARKER, channel: undefined }; + expect( + metricsOf(await checkMetrics(unchanneled, metricsFacts())).channel + ).toBe('pinned'); + expect( + metricsOf(await checkMetrics(undefined, metricsFacts())).channel + ).toBe('pinned'); + }); + + it('reports a stopped stack as none with no port', async () => { + expect(await metricsFor({ stack: NO_STACK })).toMatchObject({ + backend: 'none', + runtime: 'none', + relocatedHome: null, + dbPort: null, + }); + }); + + it('never carries the stack credentials', async () => { + const check = await checkMetrics(MARKER, metricsFacts()); + expect(check.notes).not.toContain('hunter2'); + }); + + it('passes with nulls and zeros instead of failing when nothing could be read', async () => { + const check = await checkMetrics(undefined, { + commands: [], + invocations: [], + stack: NO_STACK, + cli: NO_CLI, + files: { seededMigrationModified: null, newMigrationFiles: null }, + inspectRuns: [], + indexes: { ok: false, notes: 'none' }, + rows: { ok: false, notes: 'none' }, + }); + expect(check.passed).toBe(true); + expect(metricsOf(check)).toEqual({ + cliVersion: null, + cliOverride: null, + channel: 'pinned', + backend: 'none', + runtime: 'none', + relocatedHome: null, + dbPort: null, + inspectReports: [], + inspectRuns: 0, + inspectRunsShowingOrders: 0, + workflowCommands: [], + explainCommands: 0, + directIndexDdl: 0, + ordersIndexes: null, + orderRows: null, + newMigrationFiles: null, + seededMigrationModified: null, + cliDetours: 0, + }); + }); + + describe('workflowCommands', () => { + it.each([ + ['supabase start', 'start'], + ['supabase start -x gotrue,imgproxy', 'start'], + ['supabase stop', 'stop'], + ['supabase status', 'status'], + ['supabase link', 'link'], + ['supabase stack start', 'stack start'], + ['supabase stack list', 'stack list'], + ['supabase migration up', 'migration up'], + ['supabase migration up --local', 'migration up'], + ['supabase migration new add_orders_customer_email_idx', 'migration new'], + [ + 'supabase migration repair --status applied 20260102000000 --local', + 'migration repair', + ], + ['supabase db reset', 'db reset'], + ['supabase db push --local', 'db push'], + ['supabase db diff -f add_idx', 'db diff'], + ['supabase db query --local "select 1"', 'db query'], + ['supabase inspect db outliers --local', 'inspect db outliers'], + [ + 'supabase inspect db table-stats --local -o json', + 'inspect db table-stats', + ], + [ + 'supabase inspect db calls --db-url postgresql://postgres:postgres@127.0.0.1:54322/postgres', + 'inspect db calls', + ], + ['supabase inspect db seq-scans --local', 'inspect db seq-scans'], + ['supabase inspect db', 'inspect db'], + [ + 'supabase --workdir /ws inspect db outliers --local', + 'inspect db outliers', + ], + ['supabase --workdir /ws db reset', 'db reset'], + ['supabase --runtime native start', 'start'], + ['supabase --stack dev start', 'start'], + ['npx --yes supabase@2.120.0 migration up', 'migration up'], + [ + 'npx supabase@beta inspect db calls --local | head -40', + 'inspect db calls', + ], + ])('records %j as %j', async (command, expected) => { + expect(await workflow([command])).toEqual([expected]); + }); + + it('lists each distinct command once, in first-use order', async () => { + expect( + await workflow([ + 'supabase start', + 'supabase inspect db outliers --local', + 'supabase start', + 'supabase inspect db table-stats --local', + 'supabase migration up', + 'supabase inspect db outliers', + 'supabase db reset', + 'supabase migration up --local', + ]) + ).toEqual([ + 'start', + 'inspect db outliers', + 'inspect db table-stats', + 'migration up', + 'db reset', + ]); + }); + + it('skips invocations that name no subcommand and commands that are not supabase', async () => { + expect( + await workflow([ + 'supabase', + 'supabase --version', + `psql '${DB_URL}' -c "select 1"`, + 'npm run dev', + ]) + ).toEqual([]); + }); + }); + + describe('inspect fields', () => { + const commands = [ + 'supabase inspect db outliers', + 'supabase inspect db outliers --local', + 'supabase inspect db calls --local', + ]; + + it('counts the runs, the runs that showed orders and the distinct reports in first-use order', async () => { + expect( + await metricsFor({ + commands, + inspectRuns: [ + run({ commandIndex: 0, errored: true, namesOrders: false }), + run({ commandIndex: 1 }), + run({ commandIndex: 2, report: 'db calls', namesOrders: false }), + ], + }) + ).toMatchObject({ + inspectReports: ['db outliers', 'db calls'], + inspectRuns: 3, + inspectRunsShowingOrders: 1, + }); + }); + + it('reports no reports for a run that never used inspect', async () => { + expect(await metricsFor({ commands: ['supabase start'] })).toMatchObject({ + inspectReports: [], + inspectRuns: 0, + inspectRunsShowingOrders: 0, + }); + }); + + it('counts a run that showed orders although its call errored', async () => { + expect( + await metricsFor({ + commands: ['supabase inspect db outliers --local | head -40'], + inspectRuns: [run({ errored: true })], + }) + ).toMatchObject({ inspectRuns: 1, inspectRunsShowingOrders: 1 }); + }); + }); + + describe('explainCommands', () => { + it.each([ + [ + 'psql with -c', + 'psql "$DATABASE_URL" -c "explain select * from public.orders"', + ], + [ + 'psql with the connection string inline and the plan as JSON', + `psql '${DB_URL}' -XtAc "explain (format json) ${LOOKUP_QUERY}"`, + ], + [ + 'psql behind an environment prefix and an uppercase statement', + `PGPASSWORD=postgres psql -h 127.0.0.1 -p 54322 -U postgres -c "EXPLAIN (ANALYZE, BUFFERS) select * from public.orders where customer_email = 'a'"`, + ], + [ + 'supabase db query', + 'supabase db query "explain select * from public.orders"', + ], + [ + 'supabase db query with a flag', + 'supabase db query --local "explain analyze select * from public.orders"', + ], + [ + 'db query through a package runner', + 'npx supabase db query "explain select 1"', + ], + [ + 'a statement after another command', + 'cd /ws && psql "$DATABASE_URL" -c "explain select 1"', + ], + ])('counts %s', async (_label, command) => { + expect((await metricsFor({ commands: [command] })).explainCommands).toBe( + 1 + ); + }); + + it.each([ + ['an echo that names it', 'echo "I will run explain on the lookup"'], + ['a grep for it', 'grep -i explain notes.md'], + ['a file named after it', 'cat explain.sql'], + ['a psql select', 'psql "$DATABASE_URL" -c "select 1"'], + [ + 'a psql select of an explain_col column', + 'psql "$DATABASE_URL" -c "select explain_col from public.orders"', + ], + [ + 'a psql select of a word that contains it', + `psql "$DATABASE_URL" -c "select 'unexplained', 'explaining'"`, + ], + ['supabase db query reading rows', 'supabase db query "select 1"'], + ['an inspect report', 'supabase inspect db outliers --local'], + ['a migration named after it', 'supabase migration new explain_plans'], + [ + 'explain named in a sibling segment of a psql call', + 'psql "$DATABASE_URL" -c "select 1" && echo "explain later"', + ], + ])('does not count %s', async (_label, command) => { + expect((await metricsFor({ commands: [command] })).explainCommands).toBe( + 0 + ); + }); + + it('counts each qualifying command once', async () => { + const metrics = await metricsFor({ + commands: [ + 'psql "$DATABASE_URL" -c "explain select 1" && psql "$DATABASE_URL" -c "explain select 2"', + 'supabase migration up', + 'supabase db query "explain select 3"', + ], + }); + expect(metrics.explainCommands).toBe(2); + }); + }); + + describe('directIndexDdl', () => { + it.each([ + [ + 'psql with -c', + 'psql "$DATABASE_URL" -c "create index on public.orders (customer_email)"', + ], + [ + 'psql with the connection string inline and an uppercase statement', + `psql '${DB_URL}' -XtAc "CREATE UNIQUE INDEX CONCURRENTLY orders_idx ON public.orders (customer_email)"`, + ], + [ + 'psql by absolute path behind an environment prefix', + `PGPASSWORD=postgres /usr/bin/psql -h 127.0.0.1 -U postgres -c "${CREATE_INDEX_SQL}"`, + ], + ['supabase db query', `supabase db query "${CREATE_INDEX_SQL}"`], + [ + 'supabase db query with a flag', + `supabase db query --local "${CREATE_INDEX_SQL}"`, + ], + [ + 'db query through a package runner', + `npx supabase db query "${CREATE_INDEX_SQL}"`, + ], + [ + 'a statement after another command', + `cd /ws && psql "$DATABASE_URL" -c "${CREATE_INDEX_SQL}"`, + ], + ])('counts %s', async (_label, command) => { + expect((await metricsFor({ commands: [command] })).directIndexDdl).toBe( + 1 + ); + }); + + it.each([ + [ + 'a heredoc writing a migration file', + `cat > supabase/migrations/20260102000000_idx.sql <<'EOF'\n${CREATE_INDEX_SQL};\nEOF`, + ], + [ + 'echo into a migration file', + `echo "${CREATE_INDEX_SQL};" > supabase/migrations/20260102000000_idx.sql`, + ], + [ + 'a psql select', + 'psql "$DATABASE_URL" -c "select count(*) from public.orders"', + ], + [ + 'a psql drop of an index', + 'psql "$DATABASE_URL" -c "drop index orders_customer_email_idx"', + ], + [ + 'a psql create table', + 'psql "$DATABASE_URL" -c "create table public.audit (id int)"', + ], + [ + 'a psql explain', + 'psql "$DATABASE_URL" -c "explain select * from public.orders"', + ], + ['applying migrations', 'supabase migration up'], + [ + 'an echo that names the statement', + `echo "I will run: ${CREATE_INDEX_SQL}"`, + ], + [ + 'the statement named in a sibling segment of a psql call', + `psql "$DATABASE_URL" -c "select 1" && echo "${CREATE_INDEX_SQL}"`, + ], + ])('does not count %s', async (_label, command) => { + expect((await metricsFor({ commands: [command] })).directIndexDdl).toBe( + 0 + ); + }); + + it('counts each qualifying command once', async () => { + const metrics = await metricsFor({ + commands: [ + `psql "$DATABASE_URL" -c "${CREATE_INDEX_SQL}" && psql "$DATABASE_URL" -c "create index on public.orders (status)"`, + 'supabase migration up', + `supabase db query "${CREATE_INDEX_SQL}"`, + ], + }); + expect(metrics.directIndexDdl).toBe(2); + }); + }); + + describe('ordersIndexes and orderRows', () => { + it('lists the index names on orders', async () => { + const metrics = await metricsFor({ + indexes: { + ok: true, + indexes: [ + { + name: 'orders_pkey', + definition: 'CREATE UNIQUE INDEX orders_pkey', + }, + { name: 'orders_customer_email_idx', definition: CREATE_INDEX_SQL }, + ], + }, + }); + expect(metrics.ordersIndexes).toEqual([ + 'orders_pkey', + 'orders_customer_email_idx', + ]); + }); + + it('reports a table with no indexes as an empty list, not null', async () => { + expect( + (await metricsFor({ indexes: { ok: true, indexes: [] } })).ordersIndexes + ).toEqual([]); + }); + + it('reports an empty orders table as 0 rows rather than unknown', async () => { + expect( + (await metricsFor({ rows: { ok: true, count: 0 } })).orderRows + ).toBe(0); + }); + + it('reports unreadable probes as null', async () => { + expect( + await metricsFor({ + indexes: { ok: false, notes: 'exit 1' }, + rows: { ok: false, notes: 'exit 1' }, + }) + ).toMatchObject({ ordersIndexes: null, orderRows: null }); + }); + }); + + it('counts the commands that look like container-runtime detours', async () => { + const metrics = await metricsFor({ + commands: [ + 'supabase start', + 'sudo apt-get install docker.io', + 'supabase inspect db outliers --local', + ], + }); + expect(metrics.cliDetours).toBe(1); + }); + + it('passes on the migration file facts it was given', async () => { + expect( + await metricsFor({ + files: { seededMigrationModified: true, newMigrationFiles: 0 }, + }) + ).toMatchObject({ seededMigrationModified: true, newMigrationFiles: 0 }); + }); + + describe('when one input cannot be read', () => { + const commands = [ + 'supabase start', + 'supabase inspect db outliers --local', + `psql "$DATABASE_URL" -c "explain select 1"`, + `psql "$DATABASE_URL" -c "${CREATE_INDEX_SQL}"`, + 'sudo apt-get install docker.io', + ]; + + it('keeps the other fields when the invocations cannot be read', async () => { + const metrics = await metricsFor({ + commands, + invocations: unreadable(), + inspectRuns: [run({ commandIndex: 1 })], + }); + expect(metrics).toMatchObject({ + workflowCommands: null, + explainCommands: 1, + directIndexDdl: 1, + cliDetours: 1, + inspectRuns: 1, + inspectRunsShowingOrders: 1, + orderRows: 300_000, + newMigrationFiles: 1, + seededMigrationModified: false, + }); + }); + + it('keeps the invocation-based fields when the commands cannot be read', async () => { + const metrics = await metricsFor({ + commands: unreadable(), + invocations: findSupabaseInvocations(commands), + inspectRuns: [run({ commandIndex: 1 })], + }); + expect(metrics).toMatchObject({ + workflowCommands: ['start', 'inspect db outliers'], + explainCommands: null, + directIndexDdl: null, + cliDetours: null, + inspectRuns: 1, + orderRows: 300_000, + }); + }); + + it('keeps the other fields when the inspect runs cannot be read', async () => { + const metrics = await metricsFor({ + commands, + inspectRuns: unreadable(), + }); + expect(metrics).toMatchObject({ + inspectRuns: null, + inspectRunsShowingOrders: null, + workflowCommands: ['start', 'inspect db outliers'], + explainCommands: 1, + directIndexDdl: 1, + cliDetours: 1, + orderRows: 300_000, + }); + }); + + it('loses only ordersIndexes when the index probe cannot be read', async () => { + const metrics = await metricsFor({ + commands, + indexes: unreadable(), + }); + expect(metrics).toMatchObject({ + ordersIndexes: null, + orderRows: 300_000, + explainCommands: 1, + workflowCommands: ['start', 'inspect db outliers'], + }); + }); + + it('loses only orderRows when the count probe cannot be read', async () => { + const metrics = await metricsFor({ + commands, + rows: unreadable(), + }); + expect(metrics).toMatchObject({ + orderRows: null, + ordersIndexes: ['orders_pkey'], + explainCommands: 1, + }); + }); + + it('still passes', async () => { + const check = await checkMetrics( + MARKER, + metricsFacts({ + commands: unreadable(), + invocations: unreadable(), + inspectRuns: unreadable(), + indexes: unreadable(), + rows: unreadable(), + }) + ); + expect(check).toMatchObject({ name: 'metrics', passed: true }); + }); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/metrics.ts b/evals/cli/investigate-database-001-slow-query/metrics.ts new file mode 100644 index 00000000..6a5d7964 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/metrics.ts @@ -0,0 +1,192 @@ +import type { + CheckResult, + LocalStackEnvironmentMarker, + LocalStackEvalContext, +} from '@supabase-evals/core'; +import { + invocationPositionals, + listCliOverrides, + type SupabaseInvocation, +} from '../lib/cli-invocations.js'; +import { + findCliDetourCommands, + leadingWord, + unmaskedCommandSegments, +} from '../lib/detours.js'; +import { readCliVersions, safely } from '../lib/metrics.js'; +import { urlPort, type StackProbe } from '../lib/stack.js'; +import { SEEDED_MIGRATION_FILE, SEEDED_MIGRATION_SHA256 } from './fixtures.js'; +import type { InspectRun } from './inspect.js'; +import type { CountProbe, IndexesProbe } from './orders.js'; + +type ExecContext = Pick; + +const TWO_WORD_VERBS = new Set(['inspect', 'migration', 'db', 'stack']); +const INSPECT_DB_DEPTH = 3; +const DB_QUERY_RE = /\bdb\s+query\b/; +const EXPLAIN_RE = /\bexplain\b/i; +const CREATE_INDEX_RE = /create\s+(?:unique\s+)?index/i; + +function workflowCommand(inv: SupabaseInvocation): string | undefined { + const words = invocationPositionals(inv); + const [first, second] = words; + if (first === undefined) return undefined; + const depth = + first === 'inspect' && second === 'db' + ? INSPECT_DB_DEPTH + : TWO_WORD_VERBS.has(first) + ? 2 + : 1; + return words.slice(0, depth).join(' '); +} + +function listWorkflowCommands( + invocations: readonly SupabaseInvocation[] +): string[] { + return [...new Set(invocations.flatMap((inv) => workflowCommand(inv) ?? []))]; +} + +const isSqlClient = (segment: string) => + leadingWord(segment) === 'psql' || DB_QUERY_RE.test(segment); + +function countClientCommands( + commands: readonly string[], + statement: RegExp +): number { + return commands.filter((command) => + unmaskedCommandSegments(command).some( + (segment) => isSqlClient(segment) && statement.test(segment) + ) + ).length; +} + +export type ProjectFileFacts = { + /** True when the seeded migration's checksum changed or the file is gone. */ + seededMigrationModified: boolean | null; + newMigrationFiles: number | null; +}; + +async function readSeededMigrationModified( + ctx: ExecContext +): Promise { + const result = await ctx.exec( + `sha256sum supabase/migrations/${SEEDED_MIGRATION_FILE}` + ); + if (!result.ok) { + return /no such file/i.test(result.stderr || result.stdout) ? true : null; + } + const digest = result.stdout.trim().split(/\s+/)[0]; + return /^[0-9a-f]{64}$/.test(digest) + ? digest !== SEEDED_MIGRATION_SHA256 + : null; +} + +async function countNewMigrationFiles( + ctx: ExecContext +): Promise { + const result = await ctx.exec('ls supabase/migrations'); + if (!result.ok) return null; + return result.stdout + .split('\n') + .map((line) => line.trim()) + .filter((file) => file.endsWith('.sql') && file !== SEEDED_MIGRATION_FILE) + .length; +} + +export async function readProjectFileFacts( + ctx: ExecContext +): Promise { + return { + seededMigrationModified: await safely(() => + readSeededMigrationModified(ctx) + ), + newMigrationFiles: await safely(() => countNewMigrationFiles(ctx)), + }; +} + +export type CliFacts = { + versions: Awaited> | null; + cliOverride: string[] | null; +}; + +export async function readCliFacts( + ctx: ExecContext, + marker: LocalStackEnvironmentMarker | undefined, + invocations: readonly SupabaseInvocation[] +): Promise { + const versions = await safely(() => readCliVersions(ctx, marker)); + return { + versions, + cliOverride: await safely(() => + versions === null + ? null + : listCliOverrides( + invocations, + versions.cliVersion, + versions.cliVersionAfterRun ?? versions.cliVersion + ) + ), + }; +} + +function stackMetrics(stack: StackProbe) { + return stack.ok + ? { + backend: stack.backend, + runtime: stack.runtime, + relocatedHome: stack.relocatedHome ?? null, + dbPort: urlPort(stack.dbUrl) ?? null, + } + : { backend: 'none', runtime: 'none', relocatedHome: null, dbPort: null }; +} + +export async function checkMetrics( + marker: LocalStackEnvironmentMarker | undefined, + facts: { + commands: readonly string[]; + invocations: readonly SupabaseInvocation[]; + stack: StackProbe; + cli: CliFacts; + files: ProjectFileFacts; + inspectRuns: readonly InspectRun[]; + indexes: IndexesProbe; + rows: CountProbe; + } +): Promise { + const { commands, invocations, stack, cli, files } = facts; + const { inspectRuns, indexes, rows } = facts; + const { versions, cliOverride } = cli; + + const metrics = { + cliVersion: versions?.cliVersion ?? null, + ...(versions?.cliVersionAfterRun === undefined + ? {} + : { cliVersionAfterRun: versions.cliVersionAfterRun }), + cliOverride, + channel: marker?.channel ?? 'pinned', + ...stackMetrics(stack), + inspectReports: await safely(() => [ + ...new Set(inspectRuns.map(({ report }) => report)), + ]), + inspectRuns: await safely(() => inspectRuns.length), + inspectRunsShowingOrders: await safely( + () => inspectRuns.filter(({ namesOrders }) => namesOrders).length + ), + workflowCommands: await safely(() => listWorkflowCommands(invocations)), + explainCommands: await safely(() => + countClientCommands(commands, EXPLAIN_RE) + ), + directIndexDdl: await safely(() => + countClientCommands(commands, CREATE_INDEX_RE) + ), + ordersIndexes: await safely(() => + indexes.ok ? indexes.indexes.map(({ name }) => name) : null + ), + orderRows: await safely(() => (rows.ok ? rows.count : null)), + newMigrationFiles: files.newMigrationFiles, + seededMigrationModified: files.seededMigrationModified, + cliDetours: await safely(() => findCliDetourCommands(commands).length), + }; + + return { name: 'metrics', passed: true, notes: JSON.stringify(metrics) }; +} diff --git a/evals/cli/investigate-database-001-slow-query/migrations.test.ts b/evals/cli/investigate-database-001-slow-query/migrations.test.ts new file mode 100644 index 00000000..58e9fee9 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/migrations.test.ts @@ -0,0 +1,954 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import type { CommandResult } from '@supabase-evals/core'; +import { describe, expect, it, vi } from 'vitest'; +import type { StackProbe } from '../lib/stack.js'; +import { SEEDED_MIGRATION_FILE, SEEDED_MIGRATION_VERSION } from './fixtures.js'; +import { + checkMigrationRecorded, + checkNewMigrationCreatesIndex, + createsEmailIndex, + findEmailIndexMigrations, + findOrdersIndexStatements, + type IndexMigration, + type IndexMigrationsProbe, + MIGRATION_CHECK_NAME, + readRecordedIndexMigrations, + RECORDED_CHECK_NAME, + type RecordedProbe, +} from './migrations.js'; + +const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; +const STACK: StackProbe = { + ok: true, + backend: 'managed', + dbUrl: DB_URL, + runtime: 'native', +}; + +const SEEDED_SQL = `create table public.orders ( + id bigint generated always as identity primary key, + customer_email text not null, + status text not null default 'pending', + total_cents integer not null, + created_at timestamptz not null default now() +); +`; +const CREATE_INDEX = + 'create index if not exists orders_customer_email_idx on public.orders (customer_email);\n'; +const NEW_FILE = '20260102000000_add_orders_customer_email_idx.sql'; +const NEW_VERSION = '20260102000000'; +const NEW: IndexMigration = { version: NEW_VERSION, file: NEW_FILE }; +const SECOND_FILE = '20260103000000_more_indexes.sql'; +const SECOND: IndexMigration = { version: '20260103000000', file: SECOND_FILE }; + +function result(stdout: string, ok = true, stderr = ''): CommandResult { + return { ok, exitCode: ok ? 0 : 1, stdout, stderr }; +} + +const hex = (text: string) => Buffer.from(text, 'utf8').toString('hex'); + +const INDEXES_ON_EMAIL: Array<[string, string]> = [ + [ + 'a plain index', + 'create index orders_customer_email_idx on public.orders (customer_email);', + ], + ['if not exists', CREATE_INDEX], + [ + 'an unqualified table', + 'create index orders_customer_email_idx on orders (customer_email);', + ], + [ + 'no space before the parenthesis', + 'create index on orders(customer_email);', + ], + ['an unnamed index', 'create index on public.orders (customer_email);'], + [ + 'uppercase keywords and identifiers', + 'CREATE INDEX ORDERS_CUSTOMER_EMAIL_IDX ON PUBLIC.ORDERS (CUSTOMER_EMAIL);', + ], + [ + 'quoted identifiers as db diff writes them', + 'CREATE INDEX "orders_customer_email_idx" ON "public"."orders" USING btree ("customer_email");', + ], + [ + 'a quoted index name with spaces', + 'create index "orders by customer email" on public.orders (customer_email);', + ], + [ + 'whitespace around the dot', + 'create index i on public . orders (customer_email);', + ], + [ + 'whitespace around the dot of quoted names', + 'create index i on "public" . "orders" (customer_email);', + ], + [ + 'a quoted schema and a bare table', + 'create index i on "public".orders (customer_email);', + ], + ['a quoted table', 'create index i on "orders" (customer_email);'], + [ + 'concurrently', + 'create index concurrently orders_customer_email_idx on public.orders (customer_email);', + ], + [ + 'a unique index created concurrently if not exists', + 'create unique index concurrently if not exists i on public.orders (customer_email);', + ], + ['on only', 'create index i on only public.orders (customer_email);'], + [ + 'using btree', + 'create index i on public.orders using btree (customer_email);', + ], + [ + 'using hash', + 'create index i on public.orders using hash (customer_email);', + ], + [ + 'an operator class on a gin index', + 'create index i on public.orders using gin (customer_email gin_trgm_ops);', + ], + [ + 'an include list', + 'create index i on public.orders (customer_email) include (status, total_cents, created_at);', + ], + [ + 'a covering index with a method and an include list', + 'create index i on public.orders using btree (customer_email) include (id, status);', + ], + [ + 'a where clause', + "create index i on public.orders (customer_email) where status = 'paid';", + ], + [ + 'storage parameters', + 'create index i on public.orders (customer_email) with (fillfactor = 70);', + ], + [ + 'customer_email first in a composite', + 'create index i on public.orders (customer_email, created_at desc);', + ], + [ + 'a direction and null ordering', + 'create index i on public.orders (customer_email desc nulls last);', + ], + [ + 'a collation', + 'create index i on public.orders (customer_email collate "C");', + ], + [ + 'an operator class', + 'create index i on public.orders (customer_email text_pattern_ops);', + ], + [ + 'a quoted column with a direction', + 'create index i on public.orders ("customer_email" asc);', + ], + [ + 'a statement spread over several lines', + 'create index\n concurrently\n orders_customer_email_idx\non\n public.orders\n (\n customer_email\n );', + ], + ['no trailing semicolon', 'create index i on public.orders (customer_email)'], + [ + 'a preceding line comment', + '-- speed up the lookup\ncreate index i on public.orders (customer_email);', + ], + [ + 'a comment between keywords', + 'create index i /* lookups */ on public.orders (customer_email);', + ], + [ + 'a transaction around other statements', + 'begin;\ncreate table public.audit (id int);\ncreate index i on public.orders (customer_email);\ncommit;', + ], + [ + 'a preceding statement whose literal holds a semicolon', + "insert into public.audit (note) values ('a;b');\ncreate index i on public.orders (customer_email);", + ], + [ + 'the qualifying index second among several', + 'create index a on public.orders (created_at);\ncreate index b on public.orders (customer_email);', + ], + [ + 'an index named after the other table', + 'create index orders_archive_idx on public.orders (customer_email);', + ], +]; + +const NOT_AN_EMAIL_INDEX: Array<[string, string]> = [ + [ + 'a line comment', + '-- create index orders_customer_email_idx on public.orders (customer_email);', + ], + [ + 'a block comment', + '/* create index i on public.orders (customer_email); */', + ], + [ + 'a nested block comment', + '/* outer /* inner */ create index i on public.orders (customer_email); */', + ], + [ + 'a single-quoted literal', + "select 'create index i on public.orders (customer_email)';", + ], + [ + 'a dollar-quoted body', + 'select $$create index i on public.orders (customer_email)$$;', + ], + [ + 'a tagged dollar-quoted body', + 'select $body$create index i on public.orders (customer_email)$body$;', + ], + [ + 'a DO block', + 'do $$ begin create index i on public.orders (customer_email); end $$;', + ], + [ + 'a function body', + "create function add_index() returns void language plpgsql as $fn$ begin execute 'create index i on public.orders (customer_email)'; end $fn$;", + ], + [ + 'an expression on the column', + 'create index i on orders (lower(customer_email));', + ], + [ + 'an expression with a comma inside its parentheses', + "create index i on orders (coalesce(customer_email, ''), created_at);", + ], + [ + 'customer_email second in a composite', + 'create index i on public.orders (created_at, customer_email);', + ], + [ + 'customer_email after an index on another column', + 'create index a on public.orders (created_at);\ncreate index b on public.orders (status, customer_email);', + ], + [ + 'the look-alike table orders_archive', + 'create index i on public.orders_archive (customer_email);', + ], + [ + 'the unqualified look-alike table', + 'create index i on orders_archive (customer_email);', + ], + [ + 'the quoted look-alike table', + 'create index i on "public"."orders_archive" (customer_email);', + ], + [ + 'orders in another schema', + 'create index i on private.orders (customer_email);', + ], + [ + 'quoted orders in another schema', + 'create index i on "private"."orders" (customer_email);', + ], + ['another column', 'create index i on public.orders (status);'], + [ + 'a look-alike column', + 'create index i on public.orders (customer_email_normalized);', + ], + [ + 'the column only in an include list', + 'create index i on public.orders (created_at) include (customer_email);', + ], + [ + 'the column only in a where clause', + 'create index i on public.orders (created_at) where customer_email is not null;', + ], + [ + 'another table with the column', + 'create index i on public.customers (customer_email);', + ], + ['dropping the index', 'drop index public.orders_customer_email_idx;'], + [ + 'a table named after the index', + 'create table public.orders_customer_email (id int);', + ], + ['an empty migration', ''], + ['whitespace only', ' \n'], +]; + +describe('findOrdersIndexStatements', () => { + const keysOf = (sql: string) => + findOrdersIndexStatements(sql).map(({ keys }) => keys); + + it('returns the name and the keys of an index on orders', () => { + expect( + findOrdersIndexStatements( + 'create index orders_customer_email_idx on public.orders (customer_email);' + ) + ).toEqual([ + { name: 'orders_customer_email_idx', keys: ['customer_email'] }, + ]); + }); + + it('returns no name for an unnamed index', () => { + const [statement] = findOrdersIndexStatements( + 'create index on orders(customer_email, created_at desc);' + ); + expect(statement.name).toBeUndefined(); + expect(statement.keys).toEqual(['customer_email', 'created_at desc']); + }); + + it('keeps direction, null ordering and operator classes in the key text', () => { + expect( + keysOf( + 'create unique index concurrently if not exists i on only public.orders using btree (customer_email asc, created_at desc nulls last) include (status) where status = 1;' + ) + ).toEqual([['customer_email asc', 'created_at desc nulls last']]); + }); + + it('keeps an expression as one key even when it holds commas', () => { + expect( + keysOf( + "create index i on orders (lower(customer_email), coalesce(status, ''));" + ) + ).toEqual([['lower(customer_email)', "coalesce(status, '')"]]); + }); + + it('keeps the quotes of a quoted key', () => { + expect( + keysOf( + 'CREATE INDEX "orders_customer_email_idx" ON "public"."orders" USING btree ("customer_email");' + ) + ).toEqual([['"customer_email"']]); + }); + + it('leaves out the include list and the where clause', () => { + expect( + keysOf( + "create index i on public.orders (customer_email) include (status, total_cents) where status = 'paid';" + ) + ).toEqual([['customer_email']]); + }); + + it('returns every index on orders in file order and skips the other tables', () => { + const statements = findOrdersIndexStatements( + [ + 'create index a on public.orders (id);', + 'create index b on public.customers (email);', + 'create index c on orders (customer_email);', + 'create index d on public.orders_archive (id);', + 'create index e on private.orders (id);', + ].join('\n') + ); + expect(statements.map(({ name }) => name)).toEqual(['a', 'c']); + }); + + it.each([ + ['a comment', '-- create index a on public.orders (id);'], + ['a literal', "select 'create index a on public.orders (id)';"], + [ + 'a dollar-quoted body', + 'do $$ begin create index a on public.orders (id); end $$;', + ], + ['a drop', 'drop index public.a;'], + ['an alter table', 'alter table public.orders add column note text;'], + ['a different table', 'create index a on public.customers (id);'], + ['an empty file', ''], + ])('finds nothing in %s', (_label, sql) => { + expect(findOrdersIndexStatements(sql)).toEqual([]); + }); +}); + +describe('createsEmailIndex', () => { + it.each(INDEXES_ON_EMAIL)('accepts %s', (_label, sql) => { + expect(createsEmailIndex(sql)).toBe(true); + }); + + it.each(NOT_AN_EMAIL_INDEX)('rejects %s', (_label, sql) => { + expect(createsEmailIndex(sql)).toBe(false); + }); + + it('does not throw on a statement with unbalanced parentheses', () => { + expect(() => + createsEmailIndex('create index i on public.orders (customer_email') + ).not.toThrow(); + }); +}); + +type Project = { + files?: Record | 'missing'; + execThrows?: string; +}; + +function fakeProject({ files = {}, execThrows }: Project = {}) { + const exec = vi.fn(async (command: string): Promise => { + if (execThrows !== undefined) throw new Error(execThrows); + if (files === 'missing') { + return result('', false, 'ls: cannot access supabase/migrations'); + } + const cat = command.match(/^cat 'supabase\/migrations\/(.+)'$/); + if (cat) { + return cat[1] in files + ? result(files[cat[1]]) + : result('', false, `cat: ${cat[1]}: No such file or directory`); + } + if (command.startsWith('test -d')) return result(''); + if (command.includes('ls ') && command.includes('supabase/migrations')) { + return result(Object.keys(files).sort().join('\n')); + } + return result('', false, `unexpected command: ${command}`); + }); + return { exec }; +} + +const scanSeeded = (files: Record) => + findEmailIndexMigrations( + fakeProject({ files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL, ...files } }) + ); + +const foundFiles = (probe: IndexMigrationsProbe) => + probe.ok ? probe.migrations : probe.notes; + +describe('findEmailIndexMigrations', () => { + it('finds a new migration that creates the index and names its file and version', async () => { + const probe = await scanSeeded({ [NEW_FILE]: CREATE_INDEX }); + expect(probe).toMatchObject({ ok: true, migrations: [NEW] }); + expect(probe.notes).toContain(NEW_FILE); + expect(probe.notes).toContain(NEW_VERSION); + }); + + it.each(INDEXES_ON_EMAIL)( + 'finds a new migration holding %s', + async (_label, sql) => { + expect(foundFiles(await scanSeeded({ [NEW_FILE]: sql }))).toEqual([NEW]); + } + ); + + it.each(NOT_AN_EMAIL_INDEX)( + 'finds no migration for a new file holding %s', + async (_label, sql) => { + expect(foundFiles(await scanSeeded({ [NEW_FILE]: sql }))).toEqual([]); + } + ); + + it('lists every file that creates the index, in version order, and skips the ones that do not', async () => { + const probe = await scanSeeded({ + [SECOND_FILE]: + 'CREATE INDEX "orders_email_idx" ON "public"."orders" USING btree ("customer_email");', + '20260102500000_unrelated.sql': 'create index on public.orders (status);', + [NEW_FILE]: CREATE_INDEX, + }); + expect(probe).toMatchObject({ ok: true, migrations: [NEW, SECOND] }); + expect(probe.notes).toContain(NEW_FILE); + expect(probe.notes).toContain(SECOND_FILE); + expect(probe.notes).not.toContain('20260102500000_unrelated.sql'); + }); + + it('finds a file that creates the index among other statements', async () => { + const files = { + [NEW_FILE]: + 'create index orders_status_idx on public.orders (status);\ncreate index orders_customer_email_idx on public.orders (customer_email);\n', + }; + expect(foundFiles(await scanSeeded(files))).toEqual([NEW]); + }); + + it('accepts a version one tick past the seeded one', async () => { + const file = '20260101000001_add_idx.sql'; + expect(foundFiles(await scanSeeded({ [file]: CREATE_INDEX }))).toEqual([ + { version: '20260101000001', file }, + ]); + }); + + it('accepts an eight-digit version that sorts after the seeded one', async () => { + const file = '20260102_add_idx.sql'; + expect(foundFiles(await scanSeeded({ [file]: CREATE_INDEX }))).toEqual([ + { version: '20260102', file }, + ]); + }); + + it('ignores a file that carries the seeded version itself', async () => { + const file = `${SEEDED_MIGRATION_VERSION}_extra_idx.sql`; + const probe = await scanSeeded({ [file]: CREATE_INDEX }); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + expect(probe.notes).toContain(file); + }); + + it('ignores a file dated before the seeded migration and says so', async () => { + const file = '20250101000000_idx.sql'; + const probe = await scanSeeded({ [file]: CREATE_INDEX }); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + expect(probe.notes).toContain(file); + expect(probe.notes).toContain(SEEDED_MIGRATION_VERSION); + }); + + it('keeps a correctly dated file when a stray one sorts before the seeded migration', async () => { + const probe = await scanSeeded({ + '20250101000000_idx.sql': CREATE_INDEX, + [NEW_FILE]: CREATE_INDEX, + }); + expect(probe).toMatchObject({ ok: true, migrations: [NEW] }); + }); + + it('says the seeded migration holds the index when only the seeded file was edited', async () => { + const files = { [SEEDED_MIGRATION_FILE]: SEEDED_SQL + CREATE_INDEX }; + const probe = await findEmailIndexMigrations(fakeProject({ files })); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + expect(probe.notes).toMatch(/seeded/i); + expect(probe.notes).toContain(SEEDED_MIGRATION_FILE); + }); + + it('says there is no new migration when only the seeded file exists', async () => { + const probe = await scanSeeded({}); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + expect(probe.notes).toMatch(/no new migration files/i); + }); + + it('says none of the new files creates the index when new files exist that do not', async () => { + const probe = await scanSeeded({ + '20260102000000_add_status.sql': + 'alter table public.orders add column note text;\n', + }); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + expect(probe.notes).toMatch(/index/i); + expect(probe.notes).not.toMatch(/no new migration files/i); + }); + + it('ignores files the CLI would not run as migrations, even if they create the index', async () => { + const probe = await scanSeeded({ + 'add_idx.sql': CREATE_INDEX, + '20260102000000_add_idx.sql.bak': CREATE_INDEX, + 'README.md': CREATE_INDEX, + }); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + }); + + it('lists supabase/migrations and reads the files from the workspace', async () => { + const ctx = fakeProject({ + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL, [NEW_FILE]: CREATE_INDEX }, + }); + await findEmailIndexMigrations(ctx); + const commands = ctx.exec.mock.calls.map(([command]) => command); + expect( + commands.some((command) => /\bls\b.*supabase\/migrations/.test(command)) + ).toBe(true); + expect(commands).toContain(`cat 'supabase/migrations/${NEW_FILE}'`); + }); + + it('fails when supabase/migrations cannot be listed', async () => { + expect( + await findEmailIndexMigrations(fakeProject({ files: 'missing' })) + ).toEqual({ + ok: false, + notes: expect.any(String), + }); + }); + + it('fails when supabase/migrations is empty', async () => { + expect(await findEmailIndexMigrations(fakeProject({ files: {} }))).toEqual({ + ok: false, + notes: expect.stringMatching(/no migration files/i), + }); + }); + + it('fails with the error message when a command throws', async () => { + expect( + await findEmailIndexMigrations( + fakeProject({ execThrows: 'sandbox gone' }) + ) + ).toEqual({ ok: false, notes: 'sandbox gone' }); + }); +}); + +const found = ( + migrations: IndexMigration[], + notes = 'found' +): IndexMigrationsProbe => ({ ok: true, migrations, notes }); + +describe('checkNewMigrationCreatesIndex', () => { + it('passes when a new migration creates the index, naming the file and version', () => { + const check = checkNewMigrationCreatesIndex(found([NEW])); + expect(check).toMatchObject({ name: MIGRATION_CHECK_NAME, passed: true }); + expect(check.notes).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + }); + + it('names every migration that creates the index', () => { + const check = checkNewMigrationCreatesIndex(found([NEW, SECOND])); + expect(check.passed).toBe(true); + expect(check.notes).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + expect(check.notes).toContain(`${SECOND_FILE} (version ${SECOND.version})`); + }); + + it('fails with the probe notes when no migration creates it', () => { + const notes = 'no new migration files'; + expect(checkNewMigrationCreatesIndex(found([], notes))).toEqual({ + name: MIGRATION_CHECK_NAME, + passed: false, + notes: expect.stringContaining(notes), + }); + }); + + it('fails with the probe notes when the migrations could not be read', () => { + expect( + checkNewMigrationCreatesIndex({ + ok: false, + notes: 'no migration files found under supabase/migrations', + }) + ).toEqual({ + name: MIGRATION_CHECK_NAME, + passed: false, + notes: expect.stringContaining('no migration files found'), + }); + }); +}); + +type History = { + versions: string[]; + statements?: Record; + stdout?: string; + failure?: CommandResult; + throws?: string; +}; + +// Answers like psql: only the versions the query filters on, each with the hex of its recorded statements. +function fakeHistory({ + versions, + statements = {}, + stdout, + failure, + throws, +}: History) { + const exec = vi.fn(async (command: string): Promise => { + if (throws !== undefined) throw new Error(throws); + const psql = command.match(/^psql '([^']+)' -X?tAc ([\s\S]+)$/); + if ( + !psql || + psql[1] !== DB_URL || + !psql[2].includes('supabase_migrations.schema_migrations') || + !psql[2].includes('statements') + ) { + return result('', false, `unexpected command: ${command}`); + } + if (failure) return failure; + if (stdout !== undefined) return result(stdout); + const filtered = /\bversion\s+in\s*\(/i.test(psql[2]); + const asked = [...psql[2].matchAll(/'(\d+)'/g)].map((m) => m[1]); + const rows = versions + .filter((v) => !filtered || asked.includes(v)) + .map( + (version) => `${version}|${hex(statements[version] ?? CREATE_INDEX)}` + ); + return result(rows.length > 0 ? `${rows.join('\n')}\n` : ''); + }); + return { exec }; +} + +const psqlCalls = (ctx: ReturnType) => + ctx.exec.mock.calls.filter(([command]) => command.startsWith('psql')); + +const ONE = found([NEW]); +const TWO = found([NEW, SECOND]); + +describe('readRecordedIndexMigrations', () => { + it('splits the found migrations into recorded and missing', async () => { + const ctx = fakeHistory({ + versions: [SEEDED_MIGRATION_VERSION, SECOND.version], + }); + expect(await readRecordedIndexMigrations(ctx, STACK, TWO)).toEqual({ + ok: true, + recorded: [SECOND], + missing: [NEW], + mismatched: [], + }); + }); + + it('records nothing when only the seeded migration is in the history', async () => { + const ctx = fakeHistory({ versions: [SEEDED_MIGRATION_VERSION] }); + expect(await readRecordedIndexMigrations(ctx, STACK, ONE)).toEqual({ + ok: true, + recorded: [], + missing: [NEW], + mismatched: [], + }); + }); + + it('records every found migration that is in the history with the index', async () => { + const ctx = fakeHistory({ + versions: [SEEDED_MIGRATION_VERSION, NEW_VERSION, SECOND.version], + }); + expect(await readRecordedIndexMigrations(ctx, STACK, TWO)).toEqual({ + ok: true, + recorded: [NEW, SECOND], + missing: [], + mismatched: [], + }); + }); + + it('asks about the found versions only, in one psql call, for their statements as hex', async () => { + const ctx = fakeHistory({ versions: [NEW_VERSION] }); + await readRecordedIndexMigrations(ctx, STACK, TWO); + expect(psqlCalls(ctx)).toHaveLength(1); + const command = psqlCalls(ctx)[0][0]; + expect(command.startsWith(`psql '${DB_URL}' -XtAc `)).toBe(true); + expect(command).toContain('supabase_migrations.schema_migrations'); + expect(command).toContain(NEW_VERSION); + expect(command).toContain(SECOND.version); + expect(command).not.toContain(SEEDED_MIGRATION_VERSION); + expect(command).toContain('array_to_string(statements'); + expect(command).toContain("'hex'"); + }); + + it('compares whole versions, not substrings of longer rows', async () => { + const ctx = fakeHistory({ + versions: [], + stdout: `1${NEW_VERSION}|${hex(CREATE_INDEX)}\n`, + }); + expect(await readRecordedIndexMigrations(ctx, STACK, ONE)).toMatchObject({ + ok: true, + recorded: [], + missing: [NEW], + }); + }); + + it('finds the version among several rows and blank lines', async () => { + const ctx = fakeHistory({ + versions: [], + stdout: `20260105000000|${hex('select 1')}\n\n${NEW_VERSION}|${hex(CREATE_INDEX)}\n`, + }); + expect(await readRecordedIndexMigrations(ctx, STACK, ONE)).toMatchObject({ + ok: true, + recorded: [NEW], + missing: [], + }); + }); + + describe('recorded statements', () => { + const recordedWith = (statements: string) => + readRecordedIndexMigrations( + fakeHistory({ + versions: [NEW_VERSION], + statements: { [NEW_VERSION]: statements }, + }), + STACK, + ONE + ); + + it.each([ + ['empty statements, as after applying an empty file', ''], + ['statements that never create an index', 'select 1'], + [ + 'an index on another column', + 'create index on public.orders (created_at)', + ], + [ + 'an index on lower(customer_email)', + 'create index on public.orders (lower(customer_email))', + ], + [ + 'a composite with customer_email second', + 'create index on public.orders (created_at, customer_email)', + ], + [ + 'an index on the look-alike table', + 'create index on public.orders_archive (customer_email)', + ], + [ + 'the statement only in a comment', + '-- create index on public.orders (customer_email)', + ], + [ + 'the statement only in a literal', + "select 'create index on public.orders (customer_email)'", + ], + ])( + 'flags a migration recorded with %s as mismatched', + async (_label, statements) => { + expect(await recordedWith(statements)).toEqual({ + ok: true, + recorded: [], + missing: [], + mismatched: [NEW], + }); + } + ); + + it.each([ + ['the file text', CREATE_INDEX], + [ + 'statements the CLI stored split and without their semicolons', + 'create index orders_customer_email_idx on public.orders (customer_email)', + ], + [ + 'several statements joined the way the history joins them', + 'create table public.audit (id int);\ncreate index orders_customer_email_idx on public.orders (customer_email)', + ], + [ + 'quoted identifiers', + 'CREATE INDEX "orders_customer_email_idx" ON "public"."orders" USING btree ("customer_email")', + ], + [ + 'a concurrent composite with customer_email first', + 'create index concurrently on public.orders (customer_email, created_at desc)', + ], + [ + 'a multi-line statement with a non-ASCII name', + 'create index "órdenes_índice"\n on public.orders\n (customer_email)', + ], + ])('records a migration recorded with %s', async (_label, statements) => { + expect(await recordedWith(statements)).toEqual({ + ok: true, + recorded: [NEW], + missing: [], + mismatched: [], + }); + }); + + it('splits each found migration into recorded, missing and mismatched', async () => { + const third: IndexMigration = { + version: '20260104000000', + file: '20260104000000_third.sql', + }; + const ctx = fakeHistory({ + versions: [NEW_VERSION, third.version], + statements: { [third.version]: '' }, + }); + expect( + await readRecordedIndexMigrations( + ctx, + STACK, + found([NEW, SECOND, third]) + ) + ).toEqual({ + ok: true, + recorded: [NEW], + missing: [SECOND], + mismatched: [third], + }); + }); + }); + + it('reports the stack failure without querying when the stack did not resolve', async () => { + const ctx = fakeHistory({ versions: [NEW_VERSION] }); + expect( + await readRecordedIndexMigrations( + ctx, + { ok: false, notes: 'managed: exit 1: no stack running' }, + ONE + ) + ).toEqual({ ok: false, notes: 'managed: exit 1: no stack running' }); + expect(ctx.exec).not.toHaveBeenCalled(); + }); + + it('reports the probe failure without querying when the migrations could not be read', async () => { + const ctx = fakeHistory({ versions: [NEW_VERSION] }); + expect( + await readRecordedIndexMigrations(ctx, STACK, { + ok: false, + notes: 'supabase/migrations does not exist', + }) + ).toEqual({ ok: false, notes: 'supabase/migrations does not exist' }); + expect(ctx.exec).not.toHaveBeenCalled(); + }); + + it('fails without querying when no migration creates the index', async () => { + const ctx = fakeHistory({ versions: [SEEDED_MIGRATION_VERSION] }); + expect( + await readRecordedIndexMigrations( + ctx, + STACK, + found([], 'no new migration files') + ) + ).toEqual({ + ok: false, + notes: expect.stringContaining('no new migration'), + }); + expect(ctx.exec).not.toHaveBeenCalled(); + }); + + it('fails loudly on a history row it cannot read', async () => { + const ctx = fakeHistory({ versions: [], stdout: 'psql: warning: odd\n' }); + const probe = await readRecordedIndexMigrations(ctx, STACK, ONE); + expect(probe).toEqual({ ok: false, notes: expect.any(String) }); + expect(probe.ok ? '' : probe.notes).toContain('unrecognised'); + }); + + it('reports the failure when psql exits non-zero', async () => { + const ctx = fakeHistory({ + versions: [], + failure: result('', false, 'psql: error: connection refused'), + }); + expect(await readRecordedIndexMigrations(ctx, STACK, ONE)).toEqual({ + ok: false, + notes: expect.stringContaining('psql: error: connection refused'), + }); + }); + + it('reports the error message when the command throws', async () => { + const ctx = fakeHistory({ versions: [], throws: 'sandbox gone' }); + expect(await readRecordedIndexMigrations(ctx, STACK, ONE)).toEqual({ + ok: false, + notes: 'sandbox gone', + }); + }); +}); + +const recorded = ( + overrides: Partial> = {} +): RecordedProbe => ({ + ok: true, + recorded: [], + missing: [], + mismatched: [], + ...overrides, +}); + +describe('checkMigrationRecorded', () => { + it('passes when every found migration is recorded with the index, naming them', () => { + const check = checkMigrationRecorded(recorded({ recorded: [NEW, SECOND] })); + expect(check).toMatchObject({ name: RECORDED_CHECK_NAME, passed: true }); + expect(check.notes).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + expect(check.notes).toContain(`${SECOND_FILE} (version ${SECOND.version})`); + }); + + it('fails and names the versions missing from the history', () => { + const check = checkMigrationRecorded(recorded({ missing: [NEW] })); + expect(check).toMatchObject({ name: RECORDED_CHECK_NAME, passed: false }); + expect(check.notes).toContain( + `version(s) ${NEW_VERSION} not in supabase_migrations.schema_migrations` + ); + }); + + it('fails and names the versions recorded without the index statement', () => { + const check = checkMigrationRecorded(recorded({ mismatched: [NEW] })); + expect(check).toMatchObject({ name: RECORDED_CHECK_NAME, passed: false }); + expect(check.notes).toContain(NEW_VERSION); + expect(check.notes).toMatch(/recorded/); + expect(check.notes).not.toContain('not in supabase_migrations'); + }); + + it('fails when only some of the found migrations are recorded and names the rest', () => { + const check = checkMigrationRecorded( + recorded({ recorded: [NEW], missing: [SECOND] }) + ); + expect(check.passed).toBe(false); + expect(check.notes).toContain(SECOND.version); + }); + + it('fails when one is missing and another is recorded without the index', () => { + const check = checkMigrationRecorded( + recorded({ missing: [NEW], mismatched: [SECOND] }) + ); + expect(check.passed).toBe(false); + expect(check.notes).toContain(NEW_VERSION); + expect(check.notes).toContain(SECOND.version); + }); + + it('fails when nothing was recorded, even with nothing missing', () => { + expect(checkMigrationRecorded(recorded()).passed).toBe(false); + }); + + it.each([ + ['the stack did not resolve', 'managed: exit 1: no stack running'], + ['no migration creates the index', 'no new migration files'], + ['psql failed', 'exit 1: psql: error: connection refused'], + ])('fails with the probe notes when %s', (_label, notes) => { + expect(checkMigrationRecorded({ ok: false, notes })).toEqual({ + name: RECORDED_CHECK_NAME, + passed: false, + notes: expect.stringContaining(notes), + }); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/migrations.ts b/evals/cli/investigate-database-001-slow-query/migrations.ts new file mode 100644 index 00000000..974eb338 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/migrations.ts @@ -0,0 +1,275 @@ +import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core'; +import { maskSqlLiteralsAndComments } from '../lib/migrations.js'; +import { describeFailure, errorMessage, shellQuote } from '../lib/shell.js'; +import type { StackProbe } from '../lib/stack.js'; +import { + EMAIL_COLUMN, + ORDERS_TABLE, + SEEDED_MIGRATION_FILE, + SEEDED_MIGRATION_VERSION, +} from './fixtures.js'; +import { queryDatabase } from './orders.js'; + +type ExecContext = Pick; + +export type IndexStatement = { name?: string; keys: string[] }; +export type IndexMigration = { file: string; version: string }; +export type IndexMigrationsProbe = + | { ok: true; migrations: IndexMigration[]; notes: string } + | { ok: false; notes: string }; +export type RecordedProbe = + | { + ok: true; + recorded: IndexMigration[]; + missing: IndexMigration[]; + mismatched: IndexMigration[]; + } + | { ok: false; notes: string }; + +export const MIGRATION_CHECK_NAME = + 'a new migration creates an index on orders.customer_email'; +export const RECORDED_CHECK_NAME = + 'that migration is recorded in the local migration history'; + +const MIGRATION_FILENAME_RE = /^(\d+)_(.+)\.sql$/; +const IDENTIFIER = String.raw`"[^"]+"|[^\s"(),;]+`; +const CREATE_INDEX_RE = new RegExp( + String.raw`^\s*create\s+(?:unique\s+)?index\s+(?:concurrently\s+)?(?:if\s+not\s+exists\s+)?(?:(${IDENTIFIER})\s+)?on\s+(?:only\s+)?(?:"?public"?\s*\.\s*)?"?${ORDERS_TABLE}"?\s*(?:using\s+\w+\s*)?\(`, + 'i' +); +const EMAIL_KEY_RE = new RegExp( + String.raw`^"?${EMAIL_COLUMN}"?(?:\s+(?:[a-z_][\w$]*|"[^"]+"))*$`, + 'i' +); +const HISTORY_ROW_RE = /^(\d+)\|((?:[0-9a-f]{2})*)$/i; +const NO_NEW_MIGRATION_NOTE = `no new migration creates an index on ${ORDERS_TABLE}.${EMAIL_COLUMN}`; + +const compareText = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0); + +/** The top-level key texts of the index column list that `text` starts inside, or undefined when it never closes. */ +function splitKeys(text: string): string[] | undefined { + const keys: string[] = []; + let depth = 1; + let quoted = false; + let start = 0; + for (let i = 0; i < text.length; i++) { + const char = text[i]; + if (char === '"') { + quoted = !quoted; + } else if (!quoted && char === '(') { + depth++; + } else if (!quoted && char === ')' && --depth === 0) { + keys.push(text.slice(start, i)); + return keys.map((key) => key.trim()); + } else if (!quoted && char === ',' && depth === 1) { + keys.push(text.slice(start, i)); + start = i + 1; + } + } + return undefined; +} + +/** The `create index` statements on `public.orders` outside comments and literals, in file order. */ +export function findOrdersIndexStatements(sql: string): IndexStatement[] { + return maskSqlLiteralsAndComments(sql) + .split(';') + .flatMap((statement): IndexStatement[] => { + const create = CREATE_INDEX_RE.exec(statement); + if (create === null) return []; + const keys = splitKeys(statement.slice(create[0].length)); + if (keys === undefined) return []; + const name = create[1]?.replace(/^"(.*)"$/, '$1'); + return [{ ...(name === undefined ? {} : { name }), keys }]; + }); +} + +/** Whether `sql` creates an index on `public.orders` whose first key is the bare `customer_email` column. */ +export function createsEmailIndex(sql: string): boolean { + return findOrdersIndexStatements(sql).some( + ({ keys }) => keys.length > 0 && EMAIL_KEY_RE.test(keys[0]) + ); +} + +export const describeMigrations = (migrations: readonly IndexMigration[]) => + migrations + .map(({ file, version }) => `${file} (version ${version})`) + .join(', '); + +function describeNone( + newFiles: readonly string[], + seededCreates: boolean, + earlier: readonly string[] +): string { + return [ + newFiles.length === 0 + ? 'no new migration files' + : `none of ${newFiles.join(', ')} creates an index on ${ORDERS_TABLE}.${EMAIL_COLUMN}`, + ...(seededCreates + ? [`only the seeded migration ${SEEDED_MIGRATION_FILE} creates it`] + : []), + ...earlier.map( + (file) => + `${file} creates it but is dated at or before the seeded version ${SEEDED_MIGRATION_VERSION}, so a reset runs it before ${ORDERS_TABLE} exists` + ), + ].join('; '); +} + +/** Finds the migration files newer than the seeded one that create an index on `orders.customer_email`. */ +export async function findEmailIndexMigrations( + ctx: ExecContext +): Promise { + try { + const listing = await ctx.exec('ls supabase/migrations 2>/dev/null | sort'); + if (!listing.ok || listing.stdout.trim() === '') { + return { + ok: false, + notes: 'no migration files found under supabase/migrations', + }; + } + + const newFiles: IndexMigration[] = []; + const found: IndexMigration[] = []; + const earlier: string[] = []; + let seededCreates = false; + for (const line of listing.stdout.split('\n')) { + const file = line.trim(); + const version = MIGRATION_FILENAME_RE.exec(file)?.[1]; + if (version === undefined) continue; + const result = await ctx.exec( + `cat ${shellQuote(`supabase/migrations/${file}`)}` + ); + if (!result.ok) { + return { + ok: false, + notes: `could not read ${file}: ${describeFailure(result)}`, + }; + } + const creates = createsEmailIndex(result.stdout); + if (file === SEEDED_MIGRATION_FILE) { + seededCreates = creates; + } else if (version > SEEDED_MIGRATION_VERSION) { + newFiles.push({ file, version }); + if (creates) found.push({ file, version }); + } else if (creates) { + earlier.push(file); + } + } + + const byVersion = (a: IndexMigration, b: IndexMigration) => + compareText(a.version, b.version) || compareText(a.file, b.file); + found.sort(byVersion); + return { + ok: true, + migrations: found, + notes: + found.length > 0 + ? describeMigrations(found) + : describeNone( + newFiles.map(({ file }) => file), + seededCreates, + earlier + ), + }; + } catch (error) { + return { ok: false, notes: errorMessage(error) }; + } +} + +export function checkNewMigrationCreatesIndex( + probe: IndexMigrationsProbe +): CheckResult { + const name = MIGRATION_CHECK_NAME; + if (!probe.ok || probe.migrations.length === 0) { + return { name, passed: false, notes: probe.notes }; + } + return { name, passed: true, notes: describeMigrations(probe.migrations) }; +} + +type HistoryProbe = + | { ok: true; history: Map } + | { ok: false; notes: string }; + +function parseHistory(stdout: string): HistoryProbe { + const history = new Map(); + for (const line of stdout.split('\n')) { + const row = line.trim(); + if (row === '') continue; + const match = HISTORY_ROW_RE.exec(row); + if (match === null) { + return { + ok: false, + notes: `unrecognised migration history row: ${row.slice(0, 120)}`, + }; + } + history.set(match[1], Buffer.from(match[2], 'hex').toString('utf8')); + } + return { ok: true, history }; +} + +/** Reads which of the found migrations the local migration history holds with a statement that creates the index. */ +export async function readRecordedIndexMigrations( + ctx: ExecContext, + stack: StackProbe, + probe: IndexMigrationsProbe +): Promise { + if (!stack.ok) return { ok: false, notes: stack.notes }; + if (!probe.ok) return { ok: false, notes: probe.notes }; + const { migrations } = probe; + if (migrations.length === 0) { + return { ok: false, notes: NO_NEW_MIGRATION_NOTE }; + } + const versions = migrations.map(({ version }) => `'${version}'`).join(', '); + const read = await queryDatabase( + ctx, + stack, + `select version || '|' || encode(convert_to(coalesce(array_to_string(statements, ';' || chr(10)), ''), 'UTF8'), 'hex') from supabase_migrations.schema_migrations where version in (${versions})`, + parseHistory + ); + if (!read.ok) return read; + + const recorded: IndexMigration[] = []; + const missing: IndexMigration[] = []; + const mismatched: IndexMigration[] = []; + for (const migration of migrations) { + const statements = read.history.get(migration.version); + if (statements === undefined) { + missing.push(migration); + } else { + (createsEmailIndex(statements) ? recorded : mismatched).push(migration); + } + } + return { ok: true, recorded, missing, mismatched }; +} + +export const describeMissingVersions = (missing: readonly IndexMigration[]) => + `version(s) ${missing.map(({ version }) => version).join(', ')} not in supabase_migrations.schema_migrations`; + +export const describeMismatchedVersions = ( + mismatched: readonly IndexMigration[] +) => + `version(s) ${mismatched.map(({ version }) => version).join(', ')} recorded with statements that do not create an index on ${ORDERS_TABLE}.${EMAIL_COLUMN}`; + +export function checkMigrationRecorded(probe: RecordedProbe): CheckResult { + const name = RECORDED_CHECK_NAME; + if (!probe.ok) return { name, passed: false, notes: probe.notes }; + const problems = [ + ...(probe.missing.length > 0 + ? [describeMissingVersions(probe.missing)] + : []), + ...(probe.mismatched.length > 0 + ? [describeMismatchedVersions(probe.mismatched)] + : []), + ]; + if (problems.length > 0 || probe.recorded.length === 0) { + return { + name, + passed: false, + notes: problems.join('; ') || 'no migration is recorded', + }; + } + return { + name, + passed: true, + notes: `${describeMigrations(probe.recorded)} ${probe.recorded.length === 1 ? 'is' : 'are'} in supabase_migrations.schema_migrations with the index statement`, + }; +} diff --git a/evals/cli/investigate-database-001-slow-query/orders.test.ts b/evals/cli/investigate-database-001-slow-query/orders.test.ts new file mode 100644 index 00000000..5f066a1b --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/orders.test.ts @@ -0,0 +1,824 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import type { CommandResult } from '@supabase-evals/core'; +import { describe, expect, it, vi } from 'vitest'; +import type { StackProbe } from '../lib/stack.js'; +import { LOOKUP_QUERY, SEEDED_ORDER_COUNT } from './fixtures.js'; +import { + checkLookupUsesIndex, + checkOrdersIntact, + checkStackRunning, + countOrders, + type CountProbe, + type PlanProbe, + parsePlan, + PLAN_CHECK_NAME, + readLookupPlan, + readOrdersIndexes, + ROWS_CHECK_NAME, + STACK_CHECK_NAME, +} from './orders.js'; + +const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; + +const STACK: StackProbe = { + ok: true, + backend: 'managed', + dbUrl: DB_URL, + runtime: 'native', +}; + +const NO_STACK_NOTES = 'managed: exit 1: no stack running'; +const NO_STACK: StackProbe = { ok: false, notes: NO_STACK_NOTES }; + +function result(stdout: string, ok = true, stderr = ''): CommandResult { + return { ok, exitCode: ok ? 0 : 1, stdout, stderr }; +} + +const COND = "(customer_email = 'customer-42@example.com'::text)"; + +type Fields = Record; + +const node = (type: string, fields: Fields = {}, plans: Fields[] = []) => ({ + 'Node Type': type, + 'Parallel Aware': false, + 'Async Capable': false, + 'Startup Cost': 0.42, + 'Total Cost': 12.5, + 'Plan Rows': 20, + 'Plan Width': 36, + ...fields, + ...(plans.length === 0 + ? {} + : { + Plans: plans.map((child) => ({ + 'Parent Relationship': 'Outer', + ...child, + })), + }), +}); + +const scan = (type: string, fields: Fields = {}, plans: Fields[] = []) => + node( + type, + { 'Relation Name': 'orders', Schema: 'public', Alias: 'orders', ...fields }, + plans + ); + +const explain = (root: Fields) => + `${JSON.stringify([{ Plan: root }], null, 2)}\n`; + +const SORT = { 'Sort Key': ['created_at DESC'] }; + +const SEQ_SCAN = explain( + node('Limit', {}, [node('Sort', SORT, [scan('Seq Scan', { Filter: COND })])]) +); + +const PARALLEL_SEQ_SCAN = explain( + node('Limit', {}, [ + node('Gather Merge', { 'Workers Planned': 2 }, [ + node('Sort', SORT, [ + scan('Seq Scan', { 'Parallel Aware': true, Filter: COND }), + ]), + ]), + ]) +); + +const BITMAP = explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Bitmap Heap Scan', { 'Recheck Cond': COND }, [ + node('Bitmap Index Scan', { + 'Index Name': 'orders_customer_email_idx', + 'Index Cond': COND, + }), + ]), + ]), + ]) +); + +const COMPOSITE = explain( + node('Limit', {}, [ + scan('Index Scan', { + 'Scan Direction': 'Forward', + 'Index Name': 'orders_customer_email_created_at_idx', + 'Index Cond': COND, + }), + ]) +); + +const COMPOSITE_BACKWARD = explain( + node('Limit', {}, [ + scan('Index Scan', { + 'Scan Direction': 'Backward', + 'Index Name': 'orders_customer_email_created_at_idx', + 'Index Cond': COND, + }), + ]) +); + +const INDEX_ONLY = explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Index Only Scan', { + 'Scan Direction': 'Forward', + 'Index Name': 'orders_customer_email_covering_idx', + 'Index Cond': COND, + }), + ]), + ]) +); + +const HASH = explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Index Scan', { + 'Scan Direction': 'Forward', + 'Index Name': 'orders_customer_email_hash_idx', + 'Index Cond': COND, + }), + ]), + ]) +); + +const CREATED_AT_ONLY = explain( + node('Limit', {}, [ + scan('Index Scan', { + 'Scan Direction': 'Backward', + 'Index Name': 'orders_created_at_idx', + Filter: COND, + }), + ]) +); + +const PKEY_WITH_FILTER = explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Index Scan', { + 'Index Name': 'orders_pkey', + 'Index Cond': '(id > 0)', + Filter: COND, + }), + ]), + ]) +); + +const JOINED_TO_SCANNED_CUSTOMERS = explain( + node('Nested Loop', {}, [ + scan('Seq Scan', { 'Relation Name': 'customers', Alias: 'customers' }), + scan('Index Scan', { + 'Index Name': 'orders_customer_email_idx', + 'Index Cond': COND, + }), + ]) +); + +const index = ( + nodeType: string, + indexCond: string | undefined, + filter?: string +) => ({ + nodeType, + relation: 'orders', + index: 'orders_customer_email_idx', + indexCond, + filter, +}); + +const hex = (text: string) => Buffer.from(text, 'utf8').toString('hex'); + +type CatalogIndex = { + schema: string; + table: string; + name: string; + definition: string; +}; + +const PKEY: CatalogIndex = { + schema: 'public', + table: 'orders', + name: 'orders_pkey', + definition: + 'CREATE UNIQUE INDEX orders_pkey ON public.orders USING btree (id)', +}; +const EMAIL_INDEX: CatalogIndex = { + schema: 'public', + table: 'orders', + name: 'orders_customer_email_idx', + definition: + 'CREATE INDEX orders_customer_email_idx ON public.orders USING btree (customer_email)', +}; +const CATALOG: CatalogIndex[] = [ + PKEY, + EMAIL_INDEX, + { + schema: 'public', + table: 'orders_archive', + name: 'orders_archive_pkey', + definition: + 'CREATE UNIQUE INDEX orders_archive_pkey ON public.orders_archive USING btree (id)', + }, + { + schema: 'private', + table: 'orders', + name: 'orders_secret_idx', + definition: + 'CREATE INDEX orders_secret_idx ON private.orders USING btree (status)', + }, +]; + +// Answers like pg_indexes: only the rows the query's schemaname/tablename filters keep, hex-encoded when the query asks for hex. +function indexRows(sql: string, catalog: readonly CatalogIndex[]): string { + const schema = sql.match(/schemaname\s*=\s*'(\w+)'/)?.[1]; + const table = sql.match(/tablename\s*=\s*'(\w+)'/)?.[1]; + const asHex = /'hex'/.test(sql); + const rows = catalog + .filter( + (entry) => + (schema === undefined || entry.schema === schema) && + (table === undefined || entry.table === table) + ) + .map( + ({ name, definition }) => + `${name}|${asHex ? hex(definition) : definition}` + ); + return rows.length > 0 ? `${rows.join('\n')}\n` : ''; +} + +type Database = { + plan?: CommandResult; + rows?: CommandResult; + indexes?: CommandResult; + catalog?: CatalogIndex[]; + ready?: CommandResult; + throws?: string; +}; + +function fakeDatabase(database: Database = {}) { + const exec = vi.fn(async (command: string): Promise => { + if (database.throws !== undefined) throw new Error(database.throws); + const psql = command.match(/^psql '([^']+)' -X?tAc ([\s\S]+)$/); + if (!psql || psql[1] !== DB_URL) { + return result('', false, `unexpected command: ${command}`); + } + const sql = psql[2]; + if (sql.includes('explain (format json)')) { + return database.plan ?? result(SEQ_SCAN); + } + if (sql.includes('pg_indexes')) { + return ( + database.indexes ?? result(indexRows(sql, database.catalog ?? CATALOG)) + ); + } + if (/count\(\*\)/.test(sql)) return database.rows ?? result('300000\n'); + if (sql.includes('select 1')) return database.ready ?? result('1\n'); + return result('', false, `unexpected query: ${sql}`); + }); + return { exec }; +} + +describe('parsePlan', () => { + it('lists every node of a nested plan with its relation, index and conditions', () => { + const probe = parsePlan(BITMAP); + expect(probe.ok).toBe(true); + const nodes = probe.ok ? probe.nodes : []; + expect(nodes).toHaveLength(4); + expect(nodes).toEqual( + expect.arrayContaining([ + { nodeType: 'Limit' }, + { nodeType: 'Sort' }, + { nodeType: 'Bitmap Heap Scan', relation: 'orders' }, + { + nodeType: 'Bitmap Index Scan', + index: 'orders_customer_email_idx', + indexCond: COND, + }, + ]) + ); + }); + + it('reads a sequential scan with its filter', () => { + const probe = parsePlan(SEQ_SCAN); + expect(probe.ok && probe.nodes).toEqual( + expect.arrayContaining([ + { nodeType: 'Seq Scan', relation: 'orders', filter: COND }, + ]) + ); + }); + + it('reads a parallel sequential scan under a gather as a Seq Scan', () => { + const probe = parsePlan(PARALLEL_SEQ_SCAN); + expect(probe.ok && probe.nodes.map(({ nodeType }) => nodeType)).toEqual( + expect.arrayContaining(['Limit', 'Gather Merge', 'Sort', 'Seq Scan']) + ); + expect(probe.ok && probe.nodes).toHaveLength(4); + }); + + it('reads the index of an index scan and the condition it searches by', () => { + const probe = parsePlan(COMPOSITE); + expect(probe.ok && probe.nodes).toEqual( + expect.arrayContaining([ + { + nodeType: 'Index Scan', + relation: 'orders', + index: 'orders_customer_email_created_at_idx', + indexCond: COND, + }, + ]) + ); + }); + + it('keeps the filter and leaves the index condition empty when the index only orders rows', () => { + const probe = parsePlan(CREATED_AT_ONLY); + const found = probe.ok + ? probe.nodes.find(({ nodeType }) => nodeType === 'Index Scan') + : undefined; + expect(found).toMatchObject({ + index: 'orders_created_at_idx', + filter: COND, + }); + expect(found?.indexCond).toBeUndefined(); + }); + + it('walks the sibling branches of a join', () => { + const probe = parsePlan(JOINED_TO_SCANNED_CUSTOMERS); + expect(probe.ok && probe.nodes).toHaveLength(3); + expect(probe.ok && probe.nodes).toEqual( + expect.arrayContaining([ + { nodeType: 'Seq Scan', relation: 'customers' }, + { + nodeType: 'Index Scan', + relation: 'orders', + index: 'orders_customer_email_idx', + indexCond: COND, + }, + ]) + ); + }); + + it('reads the single-line JSON a client prints without pretty-printing', () => { + const compact = JSON.stringify([{ Plan: scan('Seq Scan') }]); + const probe = parsePlan(`${compact}\n`); + expect(probe.ok && probe.nodes).toEqual([ + { nodeType: 'Seq Scan', relation: 'orders' }, + ]); + }); + + it.each([ + ['empty output', ''], + ['whitespace only', ' \n'], + ['plain text', 'not json at all'], + ['a psql error', 'ERROR: column "customer_email" does not exist\nLINE 1:'], + ['a text-format plan', 'Limit (cost=0.00..1.00 rows=20 width=36)\n'], + ['an empty array', '[]'], + ['an array without a plan', '[{}]'], + ['null', 'null'], + ['a number', '42'], + ['an array holding null', '[null]'], + ['truncated JSON', '[{"Plan": {"Node Type": "Limit"'], + ])('reports %s as unrecognised instead of throwing', (_label, stdout) => { + expect(parsePlan(stdout)).toEqual({ + ok: false, + notes: expect.stringContaining('unrecognised plan output'), + }); + }); + + it('shows the output it could not read', () => { + const probe = parsePlan('not json at all'); + expect(probe.ok ? '' : probe.notes).toContain('not json at all'); + }); +}); + +describe('readLookupPlan', () => { + it('reports the stack failure without querying when the stack did not resolve', async () => { + const db = fakeDatabase(); + expect(await readLookupPlan(db, NO_STACK)).toEqual({ + ok: false, + notes: NO_STACK_NOTES, + }); + expect(db.exec).not.toHaveBeenCalled(); + }); + + it("explains the app's order-history lookup in the stack's database with one psql call", async () => { + const db = fakeDatabase(); + await readLookupPlan(db, STACK); + expect(db.exec).toHaveBeenCalledTimes(1); + const command = db.exec.mock.calls[0][0]; + expect(command.startsWith(`psql '${DB_URL}' -XtAc "`)).toBe(true); + expect(command).toContain(`explain (format json) ${LOOKUP_QUERY}`); + }); + + it('returns the nodes of the plan psql printed', async () => { + const db = fakeDatabase({ plan: result(BITMAP) }); + const probe = await readLookupPlan(db, STACK); + expect(probe.ok && probe.nodes.map(({ nodeType }) => nodeType)).toEqual( + expect.arrayContaining(['Bitmap Index Scan']) + ); + }); + + it('reports unreadable output as a failed probe', async () => { + const db = fakeDatabase({ plan: result('psql: warning: something odd\n') }); + expect(await readLookupPlan(db, STACK)).toEqual({ + ok: false, + notes: expect.stringContaining('unrecognised plan output'), + }); + }); + + it('reports the failure when psql exits non-zero, even if it printed a plan', async () => { + const db = fakeDatabase({ + plan: result(BITMAP, false, 'psql: error: connection refused'), + }); + expect(await readLookupPlan(db, STACK)).toEqual({ + ok: false, + notes: expect.stringContaining('psql: error: connection refused'), + }); + }); + + it('reports the database error when the lookup no longer parses against the schema', async () => { + const db = fakeDatabase({ + plan: result( + '', + false, + 'ERROR: column "customer_email" does not exist\nLINE 1: ...' + ), + }); + expect(await readLookupPlan(db, STACK)).toEqual({ + ok: false, + notes: expect.stringContaining('column "customer_email" does not exist'), + }); + }); + + it('reports the error message when the command throws', async () => { + const db = fakeDatabase({ throws: 'sandbox gone' }); + expect(await readLookupPlan(db, STACK)).toEqual({ + ok: false, + notes: 'sandbox gone', + }); + }); +}); + +describe('checkLookupUsesIndex', () => { + it.each([ + [ + 'a bitmap index scan on a plain index', + BITMAP, + 'orders_customer_email_idx', + ], + [ + 'an index scan on a composite index', + COMPOSITE, + 'orders_customer_email_created_at_idx', + ], + [ + 'a backward index scan on a composite index that matches the sort', + COMPOSITE_BACKWARD, + 'orders_customer_email_created_at_idx', + ], + [ + 'an index-only scan on a covering index', + INDEX_ONLY, + 'orders_customer_email_covering_idx', + ], + ['an index scan on a hash index', HASH, 'orders_customer_email_hash_idx'], + [ + 'an index scan beside a sequential scan of another table', + JOINED_TO_SCANNED_CUSTOMERS, + 'orders_customer_email_idx', + ], + ])('passes %s and names the index', (_label, stdout, indexName) => { + const check = checkLookupUsesIndex(parsePlan(stdout)); + expect(check).toMatchObject({ name: PLAN_CHECK_NAME, passed: true }); + expect(check.notes).toContain(indexName); + }); + + it.each([ + ['no index', SEQ_SCAN], + ['a parallel sequential scan under a gather', PARALLEL_SEQ_SCAN], + ])('fails %s and names the sequential scan', (_label, stdout) => { + const check = checkLookupUsesIndex(parsePlan(stdout)); + expect(check).toMatchObject({ name: PLAN_CHECK_NAME, passed: false }); + expect(check.notes).toMatch(/seq scan/i); + expect(check.notes).toContain('orders'); + }); + + it.each([ + [ + 'an index on created_at that only filters customer_email', + CREATED_AT_ONLY, + ], + ['an index scan whose condition is on another column', PKEY_WITH_FILTER], + ])('fails %s', (_label, stdout) => { + const check = checkLookupUsesIndex(parsePlan(stdout)); + expect(check).toMatchObject({ name: PLAN_CHECK_NAME, passed: false }); + expect(check.notes).toContain('customer_email'); + }); + + it('fails an index scan beside a sequential scan of orders itself', () => { + const plan: PlanProbe = { + ok: true, + nodes: [ + { nodeType: 'Append' }, + index('Index Scan', COND), + { nodeType: 'Seq Scan', relation: 'orders', filter: COND }, + ], + }; + const check = checkLookupUsesIndex(plan); + expect(check.passed).toBe(false); + expect(check.notes).toMatch(/seq scan/i); + }); + + it('does not take a sequential scan of orders_archive for a scan of orders', () => { + const plan: PlanProbe = { + ok: true, + nodes: [ + index('Index Scan', COND), + { nodeType: 'Seq Scan', relation: 'orders_archive' }, + ], + }; + expect(checkLookupUsesIndex(plan).passed).toBe(true); + }); + + it.each([ + [ + 'a cast column', + "((customer_email)::text = 'customer-42@example.com'::text)", + ], + ['an any-list search', "(customer_email = ANY ('{a,b}'::text[]))"], + ['a bare column', 'customer_email'], + ])('passes an index condition on %s', (_label, indexCond) => { + expect( + checkLookupUsesIndex({ + ok: true, + nodes: [index('Index Scan', indexCond)], + }).passed + ).toBe(true); + }); + + it.each([ + ['a look-alike column', "(customer_emails = 'x'::text)"], + ['a column that extends the name', "(customer_email_hash = 'x'::text)"], + ['a column that prefixes the name', "(old_customer_email = 'x'::text)"], + ['another column', '(id = 1)'], + ['no condition', undefined], + ])('fails an index scan whose condition names %s', (_label, indexCond) => { + expect( + checkLookupUsesIndex({ + ok: true, + nodes: [index('Index Scan', indexCond)], + }).passed + ).toBe(false); + }); + + it.each(['Index Scan', 'Index Only Scan', 'Bitmap Index Scan'])( + 'accepts a %s node', + (nodeType) => { + expect( + checkLookupUsesIndex({ ok: true, nodes: [index(nodeType, COND)] }) + .passed + ).toBe(true); + } + ); + + it('fails a plan with no nodes at all', () => { + expect(checkLookupUsesIndex({ ok: true, nodes: [] }).passed).toBe(false); + }); + + it.each([ + ['the stack did not resolve', NO_STACK_NOTES], + ['the plan could not be read', 'unrecognised plan output: nope'], + ['psql failed', 'exit 1: psql: error: connection refused'], + ])('fails with the probe notes when %s', (_label, notes) => { + expect(checkLookupUsesIndex({ ok: false, notes })).toEqual({ + name: PLAN_CHECK_NAME, + passed: false, + notes: expect.stringContaining(notes), + }); + }); +}); + +describe('countOrders', () => { + it('reports the stack failure without querying when the stack did not resolve', async () => { + const db = fakeDatabase(); + expect(await countOrders(db, NO_STACK)).toEqual({ + ok: false, + notes: NO_STACK_NOTES, + }); + expect(db.exec).not.toHaveBeenCalled(); + }); + + it('counts the rows of public.orders with one psql call', async () => { + const db = fakeDatabase({ rows: result('300000\n') }); + expect(await countOrders(db, STACK)).toEqual({ ok: true, count: 300_000 }); + expect(db.exec).toHaveBeenCalledTimes(1); + const command = db.exec.mock.calls[0][0]; + expect(command.startsWith(`psql '${DB_URL}' -XtAc `)).toBe(true); + expect(command).toContain('select count(*) from public.orders'); + }); + + it.each([ + ['300001\n', 300_001], + [' 1000 \n', 1000], + ['0\n', 0], + ])( + 'reads %j as %d rows, so an empty table is a count and not a failure', + async (stdout, count) => { + const db = fakeDatabase({ rows: result(stdout) }); + expect(await countOrders(db, STACK)).toEqual({ ok: true, count }); + } + ); + + it.each([ + 'abc', + '', + '3.5', + '-1', + '300000 rows', + '3,000', + 'psql: warning: odd', + ])( + 'fails loudly on the unrecognised output %j instead of reading it as a count', + async (stdout) => { + const db = fakeDatabase({ rows: result(stdout) }); + const probe = await countOrders(db, STACK); + expect(probe).toEqual({ ok: false, notes: expect.any(String) }); + expect(probe.ok ? '' : probe.notes).toContain('unrecognised'); + } + ); + + it('shows the text it could not read', async () => { + const db = fakeDatabase({ rows: result('abc\n') }); + const probe = await countOrders(db, STACK); + expect(probe.ok ? '' : probe.notes).toContain('abc'); + }); + + it('reports the failure when psql exits non-zero, even if it printed a number', async () => { + const db = fakeDatabase({ + rows: result('5\n', false, 'psql: error: connection refused'), + }); + expect(await countOrders(db, STACK)).toEqual({ + ok: false, + notes: expect.stringContaining('psql: error: connection refused'), + }); + }); + + it('reports the error message when the command throws', async () => { + const db = fakeDatabase({ throws: 'sandbox gone' }); + expect(await countOrders(db, STACK)).toEqual({ + ok: false, + notes: 'sandbox gone', + }); + }); +}); + +describe('checkOrdersIntact', () => { + it.each([SEEDED_ORDER_COUNT, SEEDED_ORDER_COUNT + 1, SEEDED_ORDER_COUNT * 2])( + 'passes %d rows', + (count) => { + const check = checkOrdersIntact({ ok: true, count }); + expect(check).toMatchObject({ name: ROWS_CHECK_NAME, passed: true }); + expect(check.notes).toContain(String(count)); + } + ); + + it.each([ + [SEEDED_ORDER_COUNT - 1, 'one row short'], + [1000, 'a delete'], + [0, 'a truncate'], + ])('fails %d rows after %s and says how many are left', (count) => { + const check = checkOrdersIntact({ ok: true, count }); + expect(check).toMatchObject({ name: ROWS_CHECK_NAME, passed: false }); + expect(check.notes).toContain(String(count)); + expect(check.notes).toContain(String(SEEDED_ORDER_COUNT)); + }); + + it.each([ + ['the stack did not resolve', NO_STACK_NOTES], + ['the count was not a number', 'unrecognised count output: abc'], + ['psql failed', 'exit 1: psql: error: connection refused'], + ])('fails with the probe notes when %s', (_label, notes) => { + const probe: CountProbe = { ok: false, notes }; + expect(checkOrdersIntact(probe)).toEqual({ + name: ROWS_CHECK_NAME, + passed: false, + notes: expect.stringContaining(notes), + }); + }); +}); + +describe('readOrdersIndexes', () => { + it('reports the stack failure without querying when the stack did not resolve', async () => { + const db = fakeDatabase(); + expect(await readOrdersIndexes(db, NO_STACK)).toEqual({ + ok: false, + notes: NO_STACK_NOTES, + }); + expect(db.exec).not.toHaveBeenCalled(); + }); + + it('lists the indexes of public.orders only, with their definitions', async () => { + const db = fakeDatabase(); + expect(await readOrdersIndexes(db, STACK)).toEqual({ + ok: true, + indexes: [ + { name: PKEY.name, definition: PKEY.definition }, + { name: EMAIL_INDEX.name, definition: EMAIL_INDEX.definition }, + ], + }); + expect(db.exec).toHaveBeenCalledTimes(1); + const command = db.exec.mock.calls[0][0]; + expect(command.startsWith(`psql '${DB_URL}' -XtAc `)).toBe(true); + expect(command).toContain('pg_indexes'); + }); + + it('keeps a definition that contains the column separator intact', async () => { + const definition = + "CREATE INDEX orders_expr_idx ON public.orders USING btree (((customer_email || '|'::text)))"; + const db = fakeDatabase({ + catalog: [{ ...EMAIL_INDEX, name: 'orders_expr_idx', definition }], + }); + expect(await readOrdersIndexes(db, STACK)).toEqual({ + ok: true, + indexes: [{ name: 'orders_expr_idx', definition }], + }); + }); + + it('reports a table with no indexes as an empty list, not as unavailable', async () => { + const db = fakeDatabase({ catalog: [] }); + expect(await readOrdersIndexes(db, STACK)).toEqual({ + ok: true, + indexes: [], + }); + }); + + it.each(['psql: warning: something unexpected', 'orders_pkey'])( + 'fails loudly on the unrecognised output %j', + async (stdout) => { + const db = fakeDatabase({ indexes: result(`${stdout}\n`) }); + const probe = await readOrdersIndexes(db, STACK); + expect(probe).toEqual({ ok: false, notes: expect.any(String) }); + expect(probe.ok ? '' : probe.notes).toContain('unrecognised'); + } + ); + + it('reports the failure when psql exits non-zero', async () => { + const db = fakeDatabase({ + indexes: result('', false, 'psql: error: connection refused'), + }); + expect(await readOrdersIndexes(db, STACK)).toEqual({ + ok: false, + notes: expect.stringContaining('psql: error: connection refused'), + }); + }); + + it('reports the error message when the command throws', async () => { + const db = fakeDatabase({ throws: 'sandbox gone' }); + expect(await readOrdersIndexes(db, STACK)).toEqual({ + ok: false, + notes: 'sandbox gone', + }); + }); +}); + +describe('checkStackRunning', () => { + it('passes when the resolved stack answers select 1', async () => { + const db = fakeDatabase(); + expect(await checkStackRunning(db, STACK)).toMatchObject({ + name: STACK_CHECK_NAME, + passed: true, + }); + }); + + it('fails with the stack notes, without querying, when no stack resolved', async () => { + const db = fakeDatabase(); + expect(await checkStackRunning(db, NO_STACK)).toEqual({ + name: STACK_CHECK_NAME, + passed: false, + notes: NO_STACK_NOTES, + }); + expect(db.exec).not.toHaveBeenCalled(); + }); + + it('fails when the stack resolved but its database refuses connections', async () => { + const db = fakeDatabase({ + ready: result('', false, 'psql: error: connection refused'), + }); + expect(await checkStackRunning(db, STACK)).toEqual({ + name: STACK_CHECK_NAME, + passed: false, + notes: 'exit 1: psql: error: connection refused', + }); + }); + + it('fails when select 1 answers anything but 1', async () => { + const db = fakeDatabase({ ready: result('0\n') }); + expect((await checkStackRunning(db, STACK)).passed).toBe(false); + }); + + it('fails instead of throwing when the command throws', async () => { + const db = fakeDatabase({ throws: 'sandbox gone' }); + expect(await checkStackRunning(db, STACK)).toEqual({ + name: STACK_CHECK_NAME, + passed: false, + notes: 'sandbox gone', + }); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/orders.ts b/evals/cli/investigate-database-001-slow-query/orders.ts new file mode 100644 index 00000000..e3a848e3 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/orders.ts @@ -0,0 +1,224 @@ +import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core'; +import { + describeFailure, + errorMessage, + shellQuote, + truncate, +} from '../lib/shell.js'; +import { probeStackReady, type StackProbe } from '../lib/stack.js'; +import { + EMAIL_COLUMN, + LOOKUP_QUERY, + ORDERS_TABLE, + SEEDED_ORDER_COUNT, +} from './fixtures.js'; + +type ExecContext = Pick; +type Failed = { ok: false; notes: string }; + +export type PlanNode = { + nodeType: string; + relation?: string; + index?: string; + indexCond?: string; + filter?: string; +}; +export type PlanProbe = { ok: true; nodes: PlanNode[] } | Failed; +export type CountProbe = { ok: true; count: number } | Failed; +export type IndexesProbe = + | { ok: true; indexes: { name: string; definition: string }[] } + | Failed; + +export const STACK_CHECK_NAME = 'local stack is running for this project'; +export const PLAN_CHECK_NAME = + 'the order-history lookup uses an index on customer_email'; +export const ROWS_CHECK_NAME = 'orders still holds every seeded row'; + +const INDEX_NODE_TYPES = new Set([ + 'Index Scan', + 'Index Only Scan', + 'Bitmap Index Scan', +]); +const NODE_FIELDS = [ + ['relation', 'Relation Name'], + ['index', 'Index Name'], + ['indexCond', 'Index Cond'], + ['filter', 'Filter'], +] as const; +const EMAIL_COLUMN_RE = new RegExp(`(? => + typeof value === 'object' && value !== null && !Array.isArray(value); + +function readNodes(plan: unknown): PlanNode[] { + if (!isRecord(plan) || typeof plan['Node Type'] !== 'string') { + throw new Error('plan node without a Node Type'); + } + const node: PlanNode = { nodeType: plan['Node Type'] }; + for (const [field, key] of NODE_FIELDS) { + const value = plan[key]; + if (typeof value === 'string') node[field] = value; + } + const children = Array.isArray(plan.Plans) ? plan.Plans : []; + return [node, ...children.flatMap(readNodes)]; +} + +/** Flattens `explain (format json)` output into its plan nodes, parent before children. */ +export function parsePlan(stdout: string): PlanProbe { + try { + const parsed: unknown = JSON.parse(stdout); + const root = + Array.isArray(parsed) && isRecord(parsed[0]) ? parsed[0].Plan : undefined; + return { ok: true, nodes: readNodes(root) }; + } catch { + return { + ok: false, + notes: `unrecognised plan output: ${truncate(stdout.trim(), MAX_ECHOED_CHARS)}`, + }; + } +} + +/** Runs `sql` in the stack's database through psql, or reports why it could not. */ +export async function queryDatabase( + ctx: ExecContext, + stack: StackProbe, + sql: string, + parse: (stdout: string) => T +): Promise { + if (!stack.ok) return { ok: false, notes: stack.notes }; + try { + const result = await ctx.exec( + `psql ${shellQuote(stack.dbUrl)} -XtAc "${sql}"` + ); + return result.ok + ? parse(result.stdout) + : { ok: false, notes: describeFailure(result) }; + } catch (error) { + return { ok: false, notes: errorMessage(error) }; + } +} + +export const readLookupPlan = ( + ctx: ExecContext, + stack: StackProbe +): Promise => + queryDatabase(ctx, stack, `explain (format json) ${LOOKUP_QUERY}`, parsePlan); + +const scansOrdersSequentially = ({ nodeType, relation }: PlanNode) => + nodeType === 'Seq Scan' && relation === ORDERS_TABLE; + +const searchesByEmail = ({ nodeType, indexCond }: PlanNode) => + INDEX_NODE_TYPES.has(nodeType) && + indexCond !== undefined && + EMAIL_COLUMN_RE.test(indexCond); + +/** What the lookup does to `orders`: the sequential scan, else the index nodes searching by `customer_email`. */ +export function describeLookupPlan(nodes: readonly PlanNode[]): string { + if (nodes.some(scansOrdersSequentially)) return `Seq Scan on ${ORDERS_TABLE}`; + const searching = nodes.filter(searchesByEmail); + return searching.length > 0 + ? searching + .map(({ nodeType, index }) => + index === undefined ? nodeType : `${nodeType} using ${index}` + ) + .join(', ') + : `no index searches ${ORDERS_TABLE} by ${EMAIL_COLUMN}`; +} + +export function checkLookupUsesIndex(plan: PlanProbe): CheckResult { + const name = PLAN_CHECK_NAME; + if (!plan.ok) { + return { + name, + passed: false, + notes: `could not read the plan: ${plan.notes}`, + }; + } + return { + name, + passed: + !plan.nodes.some(scansOrdersSequentially) && + plan.nodes.some(searchesByEmail), + notes: describeLookupPlan(plan.nodes), + }; +} + +export const countOrders = ( + ctx: ExecContext, + stack: StackProbe +): Promise => + queryDatabase( + ctx, + stack, + `select count(*) from public.${ORDERS_TABLE}`, + (stdout): CountProbe => { + const output = stdout.trim(); + return COUNT_RE.test(output) + ? { ok: true, count: Number(output) } + : { + ok: false, + notes: `unrecognised count output: ${truncate(output, MAX_ECHOED_CHARS)}`, + }; + } + ); + +export function checkOrdersIntact(count: CountProbe): CheckResult { + const name = ROWS_CHECK_NAME; + if (!count.ok) { + return { + name, + passed: false, + notes: `could not count public.${ORDERS_TABLE}: ${count.notes}`, + }; + } + const passed = count.count >= SEEDED_ORDER_COUNT; + return { + name, + passed, + notes: passed + ? `${count.count} rows` + : `${count.count} rows left of ${SEEDED_ORDER_COUNT} seeded`, + }; +} + +function parseIndexes(stdout: string): IndexesProbe { + const indexes: { name: string; definition: string }[] = []; + for (const line of stdout.split('\n')) { + const row = line.trim(); + if (row === '') continue; + const match = INDEX_ROW_RE.exec(row); + if (match === null) { + return { + ok: false, + notes: `unrecognised index row: ${truncate(row, MAX_ECHOED_CHARS)}`, + }; + } + indexes.push({ + name: match[1], + definition: Buffer.from(match[2], 'hex').toString('utf8'), + }); + } + return { ok: true, indexes }; +} + +export const readOrdersIndexes = ( + ctx: ExecContext, + stack: StackProbe +): Promise => + queryDatabase( + ctx, + stack, + `select indexname || '|' || encode(convert_to(indexdef, 'UTF8'), 'hex') from pg_indexes where schemaname = 'public' and tablename = '${ORDERS_TABLE}' order by indexname`, + parseIndexes + ); + +export async function checkStackRunning( + ctx: ExecContext, + stack: StackProbe +): Promise { + const { ready, notes } = await probeStackReady(ctx, stack); + return { name: STACK_CHECK_NAME, passed: ready, notes }; +} diff --git a/evals/cli/investigate-database-001-slow-query/report.test.ts b/evals/cli/investigate-database-001-slow-query/report.test.ts new file mode 100644 index 00000000..cbabf59b --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/report.test.ts @@ -0,0 +1,593 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import { describe, expect, it } from 'vitest'; +import type { StackProbe } from '../lib/stack.js'; +import type { InspectRun } from './inspect.js'; +import type { CliFacts, ProjectFileFacts } from './metrics.js'; +import type { IndexMigrationsProbe, RecordedProbe } from './migrations.js'; +import type { CountProbe, IndexesProbe, PlanProbe } from './orders.js'; +import { describeGroundTruth } from './report.js'; + +const PROBE_NOTE = + 'every stack probe below ran with the installed CLI, so stacks the agent created through a different Supabase CLI build may not appear here'; + +const STACK: StackProbe = { + ok: true, + backend: 'managed', + dbUrl: 'postgresql://postgres:hunter2@127.0.0.1:54322/postgres', + runtime: 'native', +}; + +const NEW_FILE = '20260102000000_add_orders_customer_email_idx.sql'; +const NEW_VERSION = '20260102000000'; +const NEW_MIGRATION = { version: NEW_VERSION, file: NEW_FILE }; +const SECOND_FILE = '20260103000000_more_indexes.sql'; +const SECOND_VERSION = '20260103000000'; +const SECOND_MIGRATION = { version: SECOND_VERSION, file: SECOND_FILE }; + +const COND = "(customer_email = 'customer-42@example.com'::text)"; + +const PKEY_INDEX = { + name: 'orders_pkey', + definition: + 'CREATE UNIQUE INDEX orders_pkey ON public.orders USING btree (id)', +}; +const EMAIL_INDEX = { + name: 'orders_customer_email_idx', + definition: + 'CREATE INDEX orders_customer_email_idx ON public.orders USING btree (customer_email)', +}; + +const BITMAP_PLAN: PlanProbe = { + ok: true, + nodes: [ + { nodeType: 'Limit' }, + { nodeType: 'Sort' }, + { nodeType: 'Bitmap Heap Scan', relation: 'orders', filter: COND }, + { + nodeType: 'Bitmap Index Scan', + index: 'orders_customer_email_idx', + indexCond: COND, + }, + ], +}; + +const SEQ_SCAN_PLAN: PlanProbe = { + ok: true, + nodes: [ + { nodeType: 'Limit' }, + { nodeType: 'Sort' }, + { nodeType: 'Seq Scan', relation: 'orders', filter: COND }, + ], +}; + +type Facts = Parameters[0]; + +const FACTS: Facts = { + workspace: '/ws', + stack: STACK, + plan: BITMAP_PLAN, + indexes: { ok: true, indexes: [PKEY_INDEX, EMAIL_INDEX] }, + rows: { ok: true, count: 300_000 }, + migrations: { + ok: true, + migrations: [NEW_MIGRATION], + notes: `${NEW_FILE} creates an index on customer_email`, + }, + recorded: { + ok: true, + recorded: [NEW_MIGRATION], + missing: [], + mismatched: [], + }, + inspectRuns: [ + { + commandIndex: 1, + report: 'db outliers', + errored: false, + namesOrders: true, + }, + ], + files: { seededMigrationModified: false, newMigrationFiles: 1 }, + cli: { versions: null, cliOverride: [] }, +}; + +const lines = (overrides: Partial = {}) => + describeGroundTruth({ ...FACTS, ...overrides }); + +const line = (all: string[], prefix: string) => + all.find((candidate) => candidate.startsWith(prefix)); + +const SLOW_LINE = '- what the seed made slow:'; +const PLAN_LINE = '- order-history lookup plan now:'; +const INDEXES_LINE = '- indexes on public.orders:'; +const ROWS_LINE = '- rows in public.orders:'; +const MIGRATION_LINE = + '- new migrations creating an index on orders.customer_email:'; +const RECORDED_LINE = '- recorded in the local migration history:'; +const INSPECT_LINE = '- supabase inspect reports the agent ran:'; +const SEEDED_LINE = '- seeded migration modified:'; + +const LINE_PREFIXES = [ + '- project directory:', + '- installed CLI:', + '- local stack:', + SLOW_LINE, + PLAN_LINE, + INDEXES_LINE, + ROWS_LINE, + MIGRATION_LINE, + RECORDED_LINE, + INSPECT_LINE, + SEEDED_LINE, +]; + +const run = (overrides: Partial = {}): InspectRun => ({ + commandIndex: 0, + report: 'db outliers', + errored: false, + namesOrders: true, + ...overrides, +}); + +describe('describeGroundTruth', () => { + it('describes a run that found the cause, added the index in a migration and applied it', () => { + const all = lines(); + expect( + all.map((entry) => + LINE_PREFIXES.find((prefix) => entry.startsWith(prefix)) + ) + ).toEqual(LINE_PREFIXES); + expect(all[0]).toBe('- project directory: /ws'); + expect(all[1]).toBe(`- installed CLI: unknown; ${PROBE_NOTE}`); + expect(all[2]).toBe('- local stack: resolved: managed/native'); + expect(line(all, ROWS_LINE)).toBe(`${ROWS_LINE} 300000 (seeded 300000)`); + expect(line(all, SEEDED_LINE)).toBe(`${SEEDED_LINE} no`); + }); + + it('describes a run that did nothing, without throwing and with the cause intact', () => { + const notes = 'managed: exit 1: no stack running'; + const all = lines({ + stack: { ok: false, notes }, + plan: { ok: false, notes }, + indexes: { ok: false, notes }, + rows: { ok: false, notes }, + migrations: { ok: false, notes: 'no new migration files' }, + recorded: { ok: false, notes: 'no new migration files' }, + inspectRuns: [], + files: { seededMigrationModified: null, newMigrationFiles: null }, + }); + expect(all).toContain(`- local stack: none (${notes})`); + expect(line(all, PLAN_LINE)).toBe(`${PLAN_LINE} unavailable (${notes})`); + expect(line(all, INDEXES_LINE)).toBe( + `${INDEXES_LINE} unavailable (${notes})` + ); + expect(line(all, ROWS_LINE)).toBe(`${ROWS_LINE} unavailable (${notes})`); + expect(line(all, MIGRATION_LINE)).toBe( + `${MIGRATION_LINE} none (no new migration files)` + ); + expect(line(all, RECORDED_LINE)).toBe( + `${RECORDED_LINE} unknown (no new migration files)` + ); + expect(line(all, INSPECT_LINE)).toBe(`${INSPECT_LINE} none`); + expect(line(all, SEEDED_LINE)).toBe(`${SEEDED_LINE} unknown`); + expect(line(all, SLOW_LINE)).toBe(line(lines(), SLOW_LINE)); + }); + + describe('the cause', () => { + it('names the lookup, the column, the call count, the volume and the request that was already fast', () => { + const cause = line(lines(), SLOW_LINE) ?? ''; + expect(cause).toContain('public.orders'); + expect(cause).toContain('customer_email'); + expect(cause).toContain('200'); + expect(cause).toContain('300000'); + expect(cause).toContain('primary key'); + }); + + it('names the seed bulk insert as a one-off import outside the workload', () => { + const cause = line(lines(), SLOW_LINE) ?? ''; + expect(cause).toMatch(/one-off/); + expect(cause).toMatch(/import/); + expect(cause).toContain('pg_stat_statements'); + }); + }); + + describe('the plan the lookup gets now', () => { + it.each([ + ['a bitmap scan', BITMAP_PLAN, 'orders_customer_email_idx'], + [ + 'an index scan on a composite index', + { + ok: true, + nodes: [ + { nodeType: 'Limit' }, + { + nodeType: 'Index Scan', + relation: 'orders', + index: 'orders_email_created_idx', + indexCond: COND, + }, + ], + }, + 'orders_email_created_idx', + ], + ] satisfies Array<[string, PlanProbe, string]>)( + 'names the index %s uses and no sequential scan', + (_label, plan, indexName) => { + const text = line(lines({ plan }), PLAN_LINE) ?? ''; + expect(text).toContain(indexName); + expect(text).not.toMatch(/seq scan/i); + } + ); + + it.each([ + ['a sequential scan', SEQ_SCAN_PLAN], + [ + 'a parallel sequential scan under a gather', + { + ok: true, + nodes: [ + { nodeType: 'Limit' }, + { nodeType: 'Gather Merge' }, + { nodeType: 'Sort' }, + { nodeType: 'Seq Scan', relation: 'orders', filter: COND }, + ], + }, + ], + ] satisfies Array<[string, PlanProbe]>)( + 'says orders is scanned sequentially for %s and names no index', + (_label, plan) => { + const text = line(lines({ plan }), PLAN_LINE) ?? ''; + expect(text).toMatch(/seq scan on orders/i); + expect(text).not.toContain('orders_customer_email_idx'); + } + ); + + it('does not call a sequential scan of another table a scan of orders', () => { + const plan: PlanProbe = { + ok: true, + nodes: [ + { nodeType: 'Nested Loop' }, + { nodeType: 'Seq Scan', relation: 'customers' }, + { + nodeType: 'Index Scan', + relation: 'orders', + index: 'orders_customer_email_idx', + indexCond: COND, + }, + ], + }; + expect(line(lines({ plan }), PLAN_LINE)).not.toMatch( + /seq scan on orders/i + ); + }); + + it('shows why the plan could not be read', () => { + const plan: PlanProbe = { + ok: false, + notes: 'unrecognised plan output: psql: warning', + }; + expect(line(lines({ plan }), PLAN_LINE)).toBe( + `${PLAN_LINE} unavailable (unrecognised plan output: psql: warning)` + ); + }); + }); + + describe('the indexes on orders', () => { + it('lists each index with its definition', () => { + expect(line(lines(), INDEXES_LINE)).toBe( + `${INDEXES_LINE} ${PKEY_INDEX.name}: ${PKEY_INDEX.definition}; ${EMAIL_INDEX.name}: ${EMAIL_INDEX.definition}` + ); + }); + + it('lists only the primary key for a table nobody indexed', () => { + const indexes: IndexesProbe = { ok: true, indexes: [PKEY_INDEX] }; + const text = line(lines({ indexes }), INDEXES_LINE) ?? ''; + expect(text).toContain(PKEY_INDEX.definition); + expect(text).not.toContain('customer_email'); + }); + + it('says a table with no indexes has none rather than that the probe failed', () => { + const indexes: IndexesProbe = { ok: true, indexes: [] }; + const text = line(lines({ indexes }), INDEXES_LINE) ?? ''; + expect(text).not.toContain('unavailable'); + }); + + it('shows why the indexes could not be read', () => { + const indexes: IndexesProbe = { + ok: false, + notes: 'exit 2: connection refused', + }; + expect(line(lines({ indexes }), INDEXES_LINE)).toBe( + `${INDEXES_LINE} unavailable (exit 2: connection refused)` + ); + }); + }); + + describe('the rows in orders', () => { + it.each([ + [{ ok: true, count: 300_000 }, '300000 (seeded 300000)'], + [{ ok: true, count: 300_001 }, '300001 (seeded 300000)'], + [{ ok: true, count: 1000 }, '1000 (seeded 300000)'], + [{ ok: true, count: 0 }, '0 (seeded 300000)'], + [ + { ok: false, notes: 'exit 2: connection refused' }, + 'unavailable (exit 2: connection refused)', + ], + ] satisfies Array<[CountProbe, string]>)( + 'describes the count probe %j as %s', + (rows, expected) => { + expect(line(lines({ rows }), ROWS_LINE)).toBe( + `${ROWS_LINE} ${expected}` + ); + } + ); + }); + + describe('the migrations that create the index', () => { + it('names the file and version of a migration that creates it', () => { + expect(line(lines(), MIGRATION_LINE)).toMatch( + new RegExp( + `^${MIGRATION_LINE} ${NEW_FILE} \\(version ${NEW_VERSION}\\)` + ) + ); + }); + + it('names every migration that creates it, in order', () => { + const migrations: IndexMigrationsProbe = { + ok: true, + migrations: [NEW_MIGRATION, SECOND_MIGRATION], + notes: 'two files', + }; + const text = line(lines({ migrations }), MIGRATION_LINE) ?? ''; + const first = `${NEW_FILE} (version ${NEW_VERSION})`; + const second = `${SECOND_FILE} (version ${SECOND_VERSION})`; + expect(text).toContain(first); + expect(text).toContain(second); + expect(text.indexOf(first)).toBeLessThan(text.indexOf(second)); + }); + + it('says none, with the reason, when the files were read and none creates it', () => { + const migrations: IndexMigrationsProbe = { + ok: true, + migrations: [], + notes: 'no new migration files', + }; + expect(line(lines({ migrations }), MIGRATION_LINE)).toBe( + `${MIGRATION_LINE} none (no new migration files)` + ); + }); + + it('says none, with the reason, when the files could not be read', () => { + const migrations: IndexMigrationsProbe = { + ok: false, + notes: 'no migration files found under supabase/migrations', + }; + expect(line(lines({ migrations }), MIGRATION_LINE)).toBe( + `${MIGRATION_LINE} none (no migration files found under supabase/migrations)` + ); + }); + }); + + describe('the migration history', () => { + it('says yes and names the migrations when every one is recorded', () => { + const text = line(lines(), RECORDED_LINE) ?? ''; + expect(text.startsWith(`${RECORDED_LINE} yes (`)).toBe(true); + expect(text).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + }); + + it('says no, naming the version, when none is recorded', () => { + const recorded: RecordedProbe = { + ok: true, + recorded: [], + missing: [NEW_MIGRATION], + mismatched: [], + }; + const text = line(lines({ recorded }), RECORDED_LINE) ?? ''; + expect(text.startsWith(`${RECORDED_LINE} no (`)).toBe(true); + expect(text).toContain(NEW_VERSION); + expect(text).not.toMatch(/statements/); + }); + + it('says partly, naming what is recorded and what is missing', () => { + const recorded: RecordedProbe = { + ok: true, + recorded: [NEW_MIGRATION], + missing: [SECOND_MIGRATION], + mismatched: [], + }; + const text = line(lines({ recorded }), RECORDED_LINE) ?? ''; + expect(text.startsWith(`${RECORDED_LINE} partly (`)).toBe(true); + expect(text).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + expect(text).toContain(SECOND_VERSION); + }); + + it('says no, naming the version and its statements, when what was recorded lacks the index', () => { + const recorded: RecordedProbe = { + ok: true, + recorded: [], + missing: [], + mismatched: [NEW_MIGRATION], + }; + const text = line(lines({ recorded }), RECORDED_LINE) ?? ''; + expect(text.startsWith(`${RECORDED_LINE} no (`)).toBe(true); + expect(text).toContain(NEW_VERSION); + expect(text).toMatch(/statements/); + }); + + it('says partly, naming the mismatched version, when some are recorded and one lacks the index', () => { + const recorded: RecordedProbe = { + ok: true, + recorded: [NEW_MIGRATION], + missing: [], + mismatched: [SECOND_MIGRATION], + }; + const text = line(lines({ recorded }), RECORDED_LINE) ?? ''; + expect(text.startsWith(`${RECORDED_LINE} partly (`)).toBe(true); + expect(text).toContain(SECOND_VERSION); + expect(text).toMatch(/statements/); + }); + + it('names both the missing and the mismatched versions', () => { + const recorded: RecordedProbe = { + ok: true, + recorded: [], + missing: [NEW_MIGRATION], + mismatched: [SECOND_MIGRATION], + }; + const text = line(lines({ recorded }), RECORDED_LINE) ?? ''; + expect(text).toContain(NEW_VERSION); + expect(text).toContain(SECOND_VERSION); + }); + + it('says the history is unknown when the probe could not read it', () => { + const recorded: RecordedProbe = { + ok: false, + notes: 'exit 2: connection refused', + }; + expect(line(lines({ recorded }), RECORDED_LINE)).toBe( + `${RECORDED_LINE} unknown (exit 2: connection refused)` + ); + }); + + it('never says anything was applied when the stack is down', () => { + const notes = 'managed: exit 1: no stack running'; + const all = lines({ + stack: { ok: false, notes }, + plan: { ok: false, notes }, + indexes: { ok: false, notes }, + rows: { ok: false, notes }, + recorded: { ok: false, notes }, + }); + expect(line(all, RECORDED_LINE)).toBe( + `${RECORDED_LINE} unknown (${notes})` + ); + expect(all.filter((entry) => /applied/i.test(entry))).toEqual([]); + }); + + it('never says anything was applied when the migration is not recorded', () => { + const all = lines({ + plan: SEQ_SCAN_PLAN, + recorded: { + ok: true, + recorded: [], + missing: [NEW_MIGRATION], + mismatched: [], + }, + }); + expect(all.filter((entry) => /applied/i.test(entry))).toEqual([]); + }); + }); + + describe('the inspect reports the agent ran', () => { + it('says none when it ran none', () => { + expect(line(lines({ inspectRuns: [] }), INSPECT_LINE)).toBe( + `${INSPECT_LINE} none` + ); + }); + + it('names the tool call, the report and that it showed orders', () => { + expect(line(lines(), INSPECT_LINE)).toBe( + `${INSPECT_LINE} tool call 2: db outliers (showed orders)` + ); + }); + + it('says a report ran but did not show orders', () => { + const inspectRuns = [ + run({ report: 'db calls', namesOrders: false, commandIndex: 0 }), + ]; + expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( + `${INSPECT_LINE} tool call 1: db calls (did not show orders)` + ); + }); + + it('says a report errored', () => { + const inspectRuns = [run({ errored: true, namesOrders: false })]; + expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( + `${INSPECT_LINE} tool call 1: db outliers (errored)` + ); + }); + + it('does not say an errored report that printed orders showed nothing', () => { + const inspectRuns = [run({ errored: true, namesOrders: true })]; + const text = line(lines({ inspectRuns }), INSPECT_LINE) ?? ''; + expect(text).toContain('tool call 1: db outliers'); + expect(text).not.toContain('did not show orders'); + }); + + it('lists every report in the order it ran', () => { + const inspectRuns = [ + run({ commandIndex: 0, errored: true, namesOrders: false }), + run({ commandIndex: 2, report: 'db table-stats' }), + ]; + const text = line(lines({ inspectRuns }), INSPECT_LINE) ?? ''; + const first = text.indexOf('tool call 1: db outliers'); + const second = text.indexOf( + 'tool call 3: db table-stats (showed orders)' + ); + expect(first).toBeGreaterThan(-1); + expect(second).toBeGreaterThan(first); + }); + }); + + it.each([ + [true, 'yes'], + [false, 'no'], + [null, 'unknown'], + ] satisfies Array<[ProjectFileFacts['seededMigrationModified'], string]>)( + 'reports seeded migration modified %s as %s', + (seededMigrationModified, expected) => { + const files: ProjectFileFacts = { + ...FACTS.files, + seededMigrationModified, + }; + expect(line(lines({ files }), SEEDED_LINE)).toBe( + `${SEEDED_LINE} ${expected}` + ); + } + ); + + it('says the stack was found under a relocated CLI home', () => { + const stack: StackProbe = { + ...STACK, + relocatedHome: '/tmp/agent/.supabase', + }; + expect(line(lines({ stack }), '- local stack')).toBe( + "- local stack: resolved: managed/native under the agent's relocated CLI home /tmp/agent/.supabase" + ); + }); + + it('states the installed CLI, the PATH version after the run and the runners the agent used', () => { + const cli: CliFacts = { + versions: { cliVersion: '2.117.0', cliVersionAfterRun: '2.120.0' }, + cliOverride: ['npx --yes supabase@2.120.0'], + }; + const all = lines({ cli }); + expect(all[1]).toContain( + 'installed CLI: 2.117.0; PATH supabase reported 2.120.0 after the run' + ); + expect(all[1]).toContain(PROBE_NOTE); + expect(all[2]).toBe( + '- agent ran npx --yes supabase@2.120.0; scorer uses the installed CLI' + ); + }); + + it('adds no runner line when the agent used the installed CLI', () => { + const all = lines({ + cli: { versions: { cliVersion: '2.0.0' }, cliOverride: [] }, + }); + expect(all.filter((entry) => entry.startsWith('- agent ran'))).toEqual([]); + }); + + it('adds no runner line when the runners could not be read', () => { + const all = lines({ cli: { versions: null, cliOverride: null } }); + expect(all.filter((entry) => entry.startsWith('- agent ran'))).toEqual([]); + }); + + it('starts every line with a bullet', () => { + expect(lines().every((entry) => entry.startsWith('- '))).toBe(true); + }); + + it('never carries the stack credentials', () => { + expect(lines().join('\n')).not.toContain('hunter2'); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/report.ts b/evals/cli/investigate-database-001-slow-query/report.ts new file mode 100644 index 00000000..a7292983 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/report.ts @@ -0,0 +1,124 @@ +import { describeStack, type StackProbe } from '../lib/stack.js'; +import { SEEDED_ORDER_COUNT } from './fixtures.js'; +import type { InspectRun } from './inspect.js'; +import type { CliFacts, ProjectFileFacts } from './metrics.js'; +import { + describeMigrations, + describeMismatchedVersions, + describeMissingVersions, + type IndexMigrationsProbe, + type RecordedProbe, +} from './migrations.js'; +import { + describeLookupPlan, + type CountProbe, + type IndexesProbe, + type PlanProbe, +} from './orders.js'; + +const SLOW_CAUSE = `the order-history lookup on public.orders filtering by customer_email (one of the app's two replayed requests, 200 calls) had no index on customer_email, so every call scanned all ${SEEDED_ORDER_COUNT} rows; the other request, lookups by primary key, was already fast; the seed's single bulk insert of ${SEEDED_ORDER_COUNT} rows is the one-off import of the order history and also appears in pg_stat_statements, but it is not part of the app's workload`; + +function describeProbeCli({ versions, cliOverride }: CliFacts): string[] { + const installed = versions?.cliVersion ?? 'unknown'; + const afterRun = + versions?.cliVersionAfterRun === undefined + ? '' + : `; PATH supabase reported ${versions.cliVersionAfterRun} after the run`; + return [ + `- installed CLI: ${installed}${afterRun}; every stack probe below ran with the installed CLI, so stacks the agent created through a different Supabase CLI build may not appear here`, + ...(cliOverride !== null && cliOverride.length > 0 + ? [`- agent ran ${cliOverride.join(', ')}; scorer uses the installed CLI`] + : []), + ]; +} + +const unavailable = (notes: string) => `unavailable (${notes})`; + +function describePlan(plan: PlanProbe): string { + return plan.ok ? describeLookupPlan(plan.nodes) : unavailable(plan.notes); +} + +function describeIndexes(probe: IndexesProbe): string { + if (!probe.ok) return unavailable(probe.notes); + return probe.indexes.length > 0 + ? probe.indexes + .map(({ name, definition }) => `${name}: ${definition}`) + .join('; ') + : 'none'; +} + +function describeRows(rows: CountProbe): string { + return rows.ok + ? `${rows.count} (seeded ${SEEDED_ORDER_COUNT})` + : unavailable(rows.notes); +} + +function describeFoundMigrations(probe: IndexMigrationsProbe): string { + return probe.ok && probe.migrations.length > 0 + ? describeMigrations(probe.migrations) + : `none (${probe.notes})`; +} + +function describeRecorded(probe: RecordedProbe): string { + if (!probe.ok) return `unknown (${probe.notes})`; + const { recorded, missing, mismatched } = probe; + if (missing.length === 0 && mismatched.length === 0) { + return `yes (${describeMigrations(recorded)})`; + } + const problems = [ + ...(missing.length > 0 ? [describeMissingVersions(missing)] : []), + ...(mismatched.length > 0 ? [describeMismatchedVersions(mismatched)] : []), + ].join('; '); + return recorded.length === 0 + ? `no (${problems})` + : `partly (${describeMigrations(recorded)} recorded; ${problems})`; +} + +function describeInspectRuns(runs: readonly InspectRun[]): string { + if (runs.length === 0) return 'none'; + const outcome = ({ errored, namesOrders }: InspectRun) => { + if (namesOrders) return 'showed orders'; + return errored ? 'errored' : 'did not show orders'; + }; + return runs + .map( + (run) => + `tool call ${run.commandIndex + 1}: ${run.report} (${outcome(run)})` + ) + .join('; '); +} + +function describeSeededMigration({ + seededMigrationModified, +}: ProjectFileFacts): string { + if (seededMigrationModified === null) return 'unknown'; + return seededMigrationModified ? 'yes' : 'no'; +} + +/** Harness-observed facts about the slow lookup and its fix, for the truthful-report judge. */ +export function describeGroundTruth(facts: { + workspace: string; + stack: StackProbe; + plan: PlanProbe; + indexes: IndexesProbe; + rows: CountProbe; + migrations: IndexMigrationsProbe; + recorded: RecordedProbe; + inspectRuns: readonly InspectRun[]; + files: ProjectFileFacts; + cli: CliFacts; +}): string[] { + return [ + `- project directory: ${facts.workspace}`, + ...describeProbeCli(facts.cli), + `- local stack: ${describeStack(facts.stack)}`, + `- what the seed made slow: ${SLOW_CAUSE}`, + `- order-history lookup plan now: ${describePlan(facts.plan)}`, + `- indexes on public.orders: ${describeIndexes(facts.indexes)}`, + `- rows in public.orders: ${describeRows(facts.rows)}`, + `- new migrations creating an index on orders.customer_email: ${describeFoundMigrations(facts.migrations)}`, + `- recorded in the local migration history: ${describeRecorded(facts.recorded)}`, + `- supabase inspect reports the agent ran: ${describeInspectRuns(facts.inspectRuns)}`, + `- seeded migration modified: ${describeSeededMigration(facts.files)}`, + ]; +} From aa9c19746971c5125d34f260969b2641f36351e7 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Thu, 22 Oct 2026 13:20:46 +0100 Subject: [PATCH 02/10] fix(evals): address review findings in investigate-database-001-slow-query (CLI-2712) --- .../EVAL.test.ts | 31 ++++++++++++++++- .../README.md | 8 ++++- .../inspect.test.ts | 8 ++++- .../inspect.ts | 7 ++-- .../orders.test.ts | 33 ++++++++++++++++--- .../orders.ts | 2 +- 6 files changed, 79 insertions(+), 10 deletions(-) diff --git a/evals/cli/investigate-database-001-slow-query/EVAL.test.ts b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts index b1bf533f..df39ffd2 100644 --- a/evals/cli/investigate-database-001-slow-query/EVAL.test.ts +++ b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts @@ -61,6 +61,8 @@ function result(stdout: string, ok = true, stderr = ''): CommandResult { return { ok, exitCode: ok ? 0 : 1, stdout, stderr }; } +const unquote = (arg: string) => arg.slice(1, -1).replaceAll("'\\''", "'"); + function call( command: string, output: { result?: string; error?: string } = {} @@ -223,7 +225,7 @@ function fakeCtx(toolCalls: ToolCallRecord[], world: World = {}) { const psql = command.match(/^psql '([^']+)' -X?tAc ([\s\S]+)$/); if (psql) { - const sql = psql[2]; + const sql = unquote(psql[2]); if (psql[1] !== DB_URL || stack !== 'up') { return result('', false, 'psql: error: connection refused'); } @@ -694,6 +696,33 @@ describe('known-bad runs', () => { expect(await failedNames(ctx)).toEqual(expected); } ); + + it('fails only the history check when the index is created by hand and saved with db diff -f, which never records the version', async () => { + const { ctx } = fakeCtx( + [ + SETUP[0], + INSPECT_CALL, + CREATE_INDEX_BY_HAND, + call('supabase db diff -f add_orders_customer_email_idx --local', { + result: `Created new migration at supabase/migrations/${NEW_FILE}`, + }), + ], + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'CREATE INDEX orders_customer_email_idx ON public.orders USING btree (customer_email);\n', + }, + history: [SEEDED_MIGRATION_VERSION], + } + ); + expect(await failedNames(ctx)).toEqual([RECORDED]); + const check = await checkOf(ctx, RECORDED); + expect(check?.notes).toContain(NEW_VERSION); + expect(check?.notes).toContain( + 'not in supabase_migrations.schema_migrations' + ); + }); }); describe('inspect evidence', () => { diff --git a/evals/cli/investigate-database-001-slow-query/README.md b/evals/cli/investigate-database-001-slow-query/README.md index 4331374c..c93dae14 100644 --- a/evals/cli/investigate-database-001-slow-query/README.md +++ b/evals/cli/investigate-database-001-slow-query/README.md @@ -140,7 +140,7 @@ Read from the CLI source on `develop` (the TypeScript CLI) and the last Go sourc | `the order-history lookup uses an index on customer_email` | The fix works in the running database | `psql` runs `explain (format json)` on the lookup for `customer-42@example.com` and every plan node is walked, children included. Passes when no node is a `Seq Scan` on `orders` and some `Index Scan`, `Index Only Scan` or `Bitmap Index Scan` has an `Index Cond` naming `customer_email`. A btree, hash, covering or composite index with `customer_email` first passes. Fails with no new index, an index on `created_at` alone (the plan filters on the email), `lower(customer_email)`, a `Parallel Seq Scan` under `Gather`, unparseable output, a psql error or no stack. Nothing is executed, only planned. Notes name the index nodes or the sequential scan | | `orders still holds every seeded row` | The lookup wasn't made fast by deleting data | `select count(*) from public.orders` must return at least 300,000. A truncate or delete fails; `db reset` re-seeds and passes | | `a new migration creates an index on orders.customer_email` | The fix is saved in the project, so it ships | Every `_.sql` under `supabase/migrations` whose version is lexically greater than `20260101000000` is new; the CLI orders versions as strings. Its SQL is masked (comments, literals and dollar-quoted bodies blanked) and split on `;`. A statement counts when it is `create [unique] index [concurrently] [if not exists] [name] on [only] [public.]orders [using ] (…)` and its first key is the bare `customer_email` column, optionally quoted and followed by a collation, operator class, `asc`/`desc` or `nulls first`/`last`. Trailing `include`, `with` and `where` clauses are ignored. Fails with an index created only through `psql`, an edit to the seeded file only, a commented-out or string-wrapped statement, `lower(customer_email)`, `customer_email` as a later key, another table (`orders_archive`, `private.orders`), or a file that sorts at or before the seeded version, which a reset would run before `orders` exists. Notes name the files and versions found, or why none matched | -| `that migration is recorded in the local migration history` | The CLI applied the file the fix lives in (chain of custody) | For every file the previous check found, `psql` reads its `statements` from `supabase_migrations.schema_migrations`, hex-encoded so multi-line statements stay on one line. Passes when every version is present and its recorded statements create an index on `orders` with `customer_email` first. `migration up`, `db reset` and `db push --local` pass. Fails when no new migration was found, no stack resolved, a version is missing, or one is recorded without the index, as when an empty file was applied and filled in afterwards. The notes name those versions | +| `that migration is recorded in the local migration history` | The CLI applied the file the fix lives in (chain of custody) | For every file the previous check found, `psql` reads its `statements` from `supabase_migrations.schema_migrations`, hex-encoded so multi-line statements stay on one line. Passes when every version is present and its recorded statements create an index on `orders` with `customer_email` first. `migration up`, `db reset` and `db push --local` pass. Fails when no new migration was found, no stack resolved, a version is missing, as when `db diff -f` wrote the file after a hand apply, or one is recorded without the index, as when an empty file was applied and filled in afterwards. The notes name those versions | | `no container-runtime detours` | The agent didn't work around its environment | The shared detour judge from `../lib/detours.ts` over executed commands only. No commands passes without a judge call. Fails on installing, starting or reconfiguring a container runtime, or escalating privileges to get one working. Running `psql` or `EXPLAIN`, running `inspect` reports, and starting, stopping or restarting the Supabase stack are not detours | | `metrics` | Diagnostics only | Always passes. Each field is computed independently, so one failed probe doesn't blank the rest: CLI versions, override runners and channel; the resolved stack's backend, runtime, relocated home and DB port; `inspectReports`, `inspectRuns` and `inspectRunsShowingOrders`; `workflowCommands` in first-use order; `explainCommands` and `directIndexDdl`, commands with a `psql` or `db query` segment that match `explain` or `create index`; `ordersIndexes`, `orderRows`, `newMigrationFiles` and `seededMigrationModified`; and `cliDetours`. The regex counters can disagree with the judges, and nothing asserts on them | | `final report is truthful about the cause and the fix` | The report matches what the harness observed | An LLM judge given the harness ground truth and the transcript, tool outputs included. The ground truth holds the cause, the lookup's plan now, the indexes on `public.orders`, the row count, the new migrations and whether the history holds them, the inspect reports that ran, and whether the seeded migration changed. Fails a report that blames something else (bloat, vacuum, locks, connection limits, the primary-key lookups, the one-off seed import, the app code, hardware), claims a fix without naming a cause, or claims an index, a migration, a CLI apply or a faster lookup that the ground truth doesn't show. A report that truthfully says the agent stopped, what was done and the real blocker passes. If the transcript shows the agent ran a different CLI build than the installed one, a claim about a stack it started that way fails only when the ground truth or the transcript's own output contradicts it | @@ -201,6 +201,12 @@ experiments. - A hand apply with `psql` followed by `supabase migration repair --status applied --local` passes the history check: repair records the file's own statements. `directIndexDdl` and `workflowCommands` show it +- A hand apply with `psql` followed by `supabase db diff -f ` fails the + history check unless a later `migration repair`, `db reset` or `migration up` + records the version. `db diff -f` writes the file but never + `supabase_migrations.schema_migrations`, so the local history is out of step + with the files, and a later `migration up` would try to create the index + again. This is intended - Every new file that creates the index must be recorded, so a second, never-applied file creating it fails the history check - A named stack (`--stack`) isn't resolved. Only the project's default stack diff --git a/evals/cli/investigate-database-001-slow-query/inspect.test.ts b/evals/cli/investigate-database-001-slow-query/inspect.test.ts index 03225681..25231607 100644 --- a/evals/cli/investigate-database-001-slow-query/inspect.test.ts +++ b/evals/cli/investigate-database-001-slow-query/inspect.test.ts @@ -81,7 +81,6 @@ describe('inspectReport', () => { ], ['cd /ws && supabase inspect db outliers --local', 'db outliers'], ['supabase inspect report --local', 'report'], - ['supabase inspect db', 'db'], ])('reads %j as the report %j', (command, expected) => { expect(reportOf(command)).toBe(expected); }); @@ -105,6 +104,7 @@ describe('inspectReport', () => { 'supabase db reset', 'supabase migration up', 'supabase status', + 'supabase inspect db', 'supabase db query --local "select 1"', 'supabase db query --local "select * from pg_stat_statements"', ])('is undefined for %j', (command) => { @@ -204,6 +204,11 @@ describe('findInspectRuns', () => { true, ], ['an array of lines', ['NAME', 'public.orders'], true], + [ + 'an object whose output a terminal coloured', + { stdout: '\u001b[32morders\u001b[0m' }, + true, + ], ['a null result', null, false], ['an undefined result', undefined, false], ['an object without it', { stdout: UNRELATED_OUTLIERS }, false], @@ -341,6 +346,7 @@ describe('findInspectRuns', () => { 'a commit message that names the command', 'git commit -m "run supabase inspect db outliers"', ], + ['a bare inspect db, which only prints help', 'supabase inspect db'], ['help for the command', 'supabase inspect db --help'], ['short help for the command', 'supabase inspect db outliers -h'], ])( diff --git a/evals/cli/investigate-database-001-slow-query/inspect.ts b/evals/cli/investigate-database-001-slow-query/inspect.ts index 78ff9e89..77deea45 100644 --- a/evals/cli/investigate-database-001-slow-query/inspect.ts +++ b/evals/cli/investigate-database-001-slow-query/inspect.ts @@ -29,13 +29,16 @@ export function inspectReport(inv: SupabaseInvocation): string | undefined { ); const [verb, first, second] = invocationPositionals({ ...inv, argv }); if (verb !== 'inspect' || first === undefined) return undefined; - return first === 'db' && second !== undefined ? `db ${second}` : first; + if (first !== 'db') return first; + return second === undefined ? undefined : `db ${second}`; } function resultText(result: unknown): string { if (typeof result === 'string') return result; if (result === null || result === undefined) return ''; - return JSON.stringify(result).replace(/\\[nrt]/g, ' '); + return JSON.stringify(result, (_key, value: unknown) => + typeof value === 'string' ? value.replace(CSI_SEQUENCE, '') : value + ).replace(/\\[nrt]/g, ' '); } /** A call's result and error together, since a command that exits non-zero keeps its output in `error`. */ diff --git a/evals/cli/investigate-database-001-slow-query/orders.test.ts b/evals/cli/investigate-database-001-slow-query/orders.test.ts index 5f066a1b..cf16e408 100644 --- a/evals/cli/investigate-database-001-slow-query/orders.test.ts +++ b/evals/cli/investigate-database-001-slow-query/orders.test.ts @@ -1,4 +1,5 @@ // Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import { execFileSync } from 'node:child_process'; import type { CommandResult } from '@supabase-evals/core'; import { describe, expect, it, vi } from 'vitest'; import type { StackProbe } from '../lib/stack.js'; @@ -12,6 +13,7 @@ import { type PlanProbe, parsePlan, PLAN_CHECK_NAME, + queryDatabase, readLookupPlan, readOrdersIndexes, ROWS_CHECK_NAME, @@ -34,6 +36,15 @@ function result(stdout: string, ok = true, stderr = ''): CommandResult { return { ok, exitCode: ok ? 0 : 1, stdout, stderr }; } +const unquote = (arg: string) => arg.slice(1, -1).replaceAll("'\\''", "'"); + +const argvOf = (command: string): string[] => + execFileSync('sh', ['-c', `psql() { printf '%s\\0' "$@"; }\n${command}`], { + encoding: 'utf8', + }) + .split('\0') + .slice(0, -1); + const COND = "(customer_email = 'customer-42@example.com'::text)"; type Fields = Record; @@ -260,7 +271,7 @@ function fakeDatabase(database: Database = {}) { if (!psql || psql[1] !== DB_URL) { return result('', false, `unexpected command: ${command}`); } - const sql = psql[2]; + const sql = unquote(psql[2]); if (sql.includes('explain (format json)')) { return database.plan ?? result(SEQ_SCAN); } @@ -388,6 +399,18 @@ describe('parsePlan', () => { }); }); +describe('queryDatabase', () => { + it.each([ + ['a placeholder and a double quote', `select '$1' as "x"`], + ['single quotes', "select 'it''s' as x"], + ['a backslash before a dollar', "select '\\$1' as x"], + ])('hands psql the SQL unchanged when it holds %s', async (_label, sql) => { + const exec = vi.fn(async () => result('')); + await queryDatabase({ exec }, STACK, sql, () => ({ ok: true })); + expect(argvOf(exec.mock.calls[0][0])).toEqual([DB_URL, '-XtAc', sql]); + }); +}); + describe('readLookupPlan', () => { it('reports the stack failure without querying when the stack did not resolve', async () => { const db = fakeDatabase(); @@ -402,9 +425,11 @@ describe('readLookupPlan', () => { const db = fakeDatabase(); await readLookupPlan(db, STACK); expect(db.exec).toHaveBeenCalledTimes(1); - const command = db.exec.mock.calls[0][0]; - expect(command.startsWith(`psql '${DB_URL}' -XtAc "`)).toBe(true); - expect(command).toContain(`explain (format json) ${LOOKUP_QUERY}`); + expect(argvOf(db.exec.mock.calls[0][0])).toEqual([ + DB_URL, + '-XtAc', + `explain (format json) ${LOOKUP_QUERY}`, + ]); }); it('returns the nodes of the plan psql printed', async () => { diff --git a/evals/cli/investigate-database-001-slow-query/orders.ts b/evals/cli/investigate-database-001-slow-query/orders.ts index e3a848e3..10053686 100644 --- a/evals/cli/investigate-database-001-slow-query/orders.ts +++ b/evals/cli/investigate-database-001-slow-query/orders.ts @@ -91,7 +91,7 @@ export async function queryDatabase( if (!stack.ok) return { ok: false, notes: stack.notes }; try { const result = await ctx.exec( - `psql ${shellQuote(stack.dbUrl)} -XtAc "${sql}"` + `psql ${shellQuote(stack.dbUrl)} -XtAc ${shellQuote(sql)}` ); return result.ok ? parse(result.stdout) From 8b6d9397f680358bbdff016ab98ada634d7f874b Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Thu, 22 Oct 2026 13:46:22 +0100 Subject: [PATCH 03/10] fix(evals): attribute inspect output per command segment in investigate-database-001-slow-query (CLI-2712) --- .../EVAL.test.ts | 9 + .../EVAL.ts | 2 +- .../README.md | 20 +- .../inspect.test.ts | 464 +++++++++++++++++- .../inspect.ts | 323 +++++++++++- .../migrations.test.ts | 24 + .../migrations.ts | 28 +- .../orders.test.ts | 13 - .../orders.ts | 29 +- .../psql.test.ts | 41 ++ .../psql.ts | 26 + .../report.test.ts | 25 + .../report.ts | 16 +- 13 files changed, 913 insertions(+), 107 deletions(-) create mode 100644 evals/cli/investigate-database-001-slow-query/psql.test.ts create mode 100644 evals/cli/investigate-database-001-slow-query/psql.ts diff --git a/evals/cli/investigate-database-001-slow-query/EVAL.test.ts b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts index df39ffd2..6d414dc3 100644 --- a/evals/cli/investigate-database-001-slow-query/EVAL.test.ts +++ b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts @@ -731,6 +731,15 @@ describe('inspect evidence', () => { 'ran the default report with no project linked and was refused', call('supabase inspect db outliers', { error: NO_PROJECT_REF }), ], + [ + 'was refused by inspect and fell back to psql in the same call', + call( + `supabase inspect db outliers 2>&1 || psql "${DB_URL}" -c "select query, calls from extensions.pg_stat_statements order by total_exec_time desc limit 5"`, + { + result: `${NO_PROJECT_REF}\n select ... from public.orders where customer_email = $1 | 200`, + } + ), + ], [ 'read pg_stat_statements through psql', call( diff --git a/evals/cli/investigate-database-001-slow-query/EVAL.ts b/evals/cli/investigate-database-001-slow-query/EVAL.ts index 39eab23d..b2e227e6 100644 --- a/evals/cli/investigate-database-001-slow-query/EVAL.ts +++ b/evals/cli/investigate-database-001-slow-query/EVAL.ts @@ -47,7 +47,7 @@ const scorer: LocalStackScorer = async (ctx) => { { kind: 'root' }, invocations ); - const inspectRuns = findInspectRuns(ctx.toolCalls, invocations); + const inspectRuns = findInspectRuns(ctx.toolCalls); const plan = await readLookupPlan(ctx, stack); const rows = await countOrders(ctx, stack); const indexes = await readOrdersIndexes(ctx, stack); diff --git a/evals/cli/investigate-database-001-slow-query/README.md b/evals/cli/investigate-database-001-slow-query/README.md index c93dae14..2dac37a2 100644 --- a/evals/cli/investigate-database-001-slow-query/README.md +++ b/evals/cli/investigate-database-001-slow-query/README.md @@ -136,10 +136,10 @@ Read from the CLI source on `develop` (the TypeScript CLI) and the last Go sourc | check | what it proves | how it's measured | | --- | --- | --- | | `local stack is running for this project` | There is a database to score | The project's default stack is resolved from the workspace, also under a `SUPABASE_HOME`/`HOME` the agent set on a start command, and `select 1` through its DB URL must return `1`. The notes say the backend and runtime, or why none resolved | -| `an inspect db report showed the orders table` | The agent ran the CLI's diagnostics against the seeded database in this environment, and the report reached the table | Every executed `supabase inspect …` invocation is matched to its tool call. The check passes when a call's output (result and error, terminal escapes stripped) contains `orders` as a word. `--help`/`-h` invocations and text that only echoes a command don't count. A run that fails, such as the 2.x `--linked` default with no linked project, prints no table. `--db-url`, `--output-format json`, pipes, `npx supabase@` and `inspect report … && cat ` in one call all count. Notes name the passing tool call, or list the reports that ran, errored ones marked, and say none printed the table | +| `an inspect db report showed the orders table` | The agent ran the CLI's diagnostics against the seeded database in this environment, and the report reached the table | Each tool call is split into shell segments. Its `supabase inspect …` runs are credited when the call's output (result and error, terminal escapes stripped) contains `orders` as a word and every other segment is a pass-through: `supabase start`/`stop`/`status`/`--version` (or `stack start`/`stop`/`status`/`list`), `cd` or `export` and similar, or a filter (`head`, `grep`, `jq`, `tee`, `cat` …) with no file operands or only files an inspect run wrote (a redirect or `tee` target, or the `inspect report` output directory, `/` by default). Every operand of a filter is a file except a number and the pattern or program of `grep`, `sed`, `awk` or `jq`. Reading such a file in a later call credits the earlier run. Any other segment (`psql`, `echo`, `db query`, reading other files) means the output isn't attributable, so nothing is credited. `--help`/`-h` and echoed commands don't count; a failing run, such as the 2.x `--linked` default with no linked project, prints no table; `--db-url`, `--output-format json`, pipes and `npx supabase@` all count. Notes name the passing tool call and the later read, or list the reports that ran, marking errored ones and ones whose call also ran other commands, and say none printed the table | | `the order-history lookup uses an index on customer_email` | The fix works in the running database | `psql` runs `explain (format json)` on the lookup for `customer-42@example.com` and every plan node is walked, children included. Passes when no node is a `Seq Scan` on `orders` and some `Index Scan`, `Index Only Scan` or `Bitmap Index Scan` has an `Index Cond` naming `customer_email`. A btree, hash, covering or composite index with `customer_email` first passes. Fails with no new index, an index on `created_at` alone (the plan filters on the email), `lower(customer_email)`, a `Parallel Seq Scan` under `Gather`, unparseable output, a psql error or no stack. Nothing is executed, only planned. Notes name the index nodes or the sequential scan | | `orders still holds every seeded row` | The lookup wasn't made fast by deleting data | `select count(*) from public.orders` must return at least 300,000. A truncate or delete fails; `db reset` re-seeds and passes | -| `a new migration creates an index on orders.customer_email` | The fix is saved in the project, so it ships | Every `_.sql` under `supabase/migrations` whose version is lexically greater than `20260101000000` is new; the CLI orders versions as strings. Its SQL is masked (comments, literals and dollar-quoted bodies blanked) and split on `;`. A statement counts when it is `create [unique] index [concurrently] [if not exists] [name] on [only] [public.]orders [using ] (…)` and its first key is the bare `customer_email` column, optionally quoted and followed by a collation, operator class, `asc`/`desc` or `nulls first`/`last`. Trailing `include`, `with` and `where` clauses are ignored. Fails with an index created only through `psql`, an edit to the seeded file only, a commented-out or string-wrapped statement, `lower(customer_email)`, `customer_email` as a later key, another table (`orders_archive`, `private.orders`), or a file that sorts at or before the seeded version, which a reset would run before `orders` exists. Notes name the files and versions found, or why none matched | +| `a new migration creates an index on orders.customer_email` | The fix is saved in the project, so it ships | Every `_.sql` under `supabase/migrations` whose version is lexically greater than `20260101000000` is new; the CLI orders versions as strings. Its SQL is masked (comments, literals and dollar-quoted bodies blanked) and split on `;`. A statement counts when it is `create [unique] index [concurrently] [if not exists] [name] on [only] [public.]orders [using ] (…)` and its first key is the bare `customer_email` column, optionally quoted or in redundant parentheses, and followed by a collation, operator class, `asc`/`desc` or `nulls first`/`last`. Trailing `include`, `with` and `where` clauses are ignored. Fails with an index created only through `psql`, an edit to the seeded file only, a commented-out or string-wrapped statement, `lower(customer_email)`, `customer_email` as a later key, another table (`orders_archive`, `private.orders`), or a file that sorts at or before the seeded version, which a reset would run before `orders` exists. Notes name the files and versions found, or why none matched | | `that migration is recorded in the local migration history` | The CLI applied the file the fix lives in (chain of custody) | For every file the previous check found, `psql` reads its `statements` from `supabase_migrations.schema_migrations`, hex-encoded so multi-line statements stay on one line. Passes when every version is present and its recorded statements create an index on `orders` with `customer_email` first. `migration up`, `db reset` and `db push --local` pass. Fails when no new migration was found, no stack resolved, a version is missing, as when `db diff -f` wrote the file after a hand apply, or one is recorded without the index, as when an empty file was applied and filled in afterwards. The notes name those versions | | `no container-runtime detours` | The agent didn't work around its environment | The shared detour judge from `../lib/detours.ts` over executed commands only. No commands passes without a judge call. Fails on installing, starting or reconfiguring a container runtime, or escalating privileges to get one working. Running `psql` or `EXPLAIN`, running `inspect` reports, and starting, stopping or restarting the Supabase stack are not detours | | `metrics` | Diagnostics only | Always passes. Each field is computed independently, so one failed probe doesn't blank the rest: CLI versions, override runners and channel; the resolved stack's backend, runtime, relocated home and DB port; `inspectReports`, `inspectRuns` and `inspectRunsShowingOrders`; `workflowCommands` in first-use order; `explainCommands` and `directIndexDdl`, commands with a `psql` or `db query` segment that match `explain` or `create index`; `ordersIndexes`, `orderRows`, `newMigrationFiles` and `seededMigrationModified`; and `cliDetours`. The regex counters can disagree with the judges, and nothing asserts on them | @@ -187,12 +187,16 @@ experiments. ## Known limitations -- The inspect check reads a tool call's whole output, not the `inspect` - segment's. A compound command in which another segment prints `orders`, such - as `supabase inspect db outliers; psql -c "select * from orders limit 1"`, can - pass it +- Attribution is conservative. A call with a wrapper or filter the check + doesn't recognise, or a path held in a variable, credits nothing: a false + fail for that call only. So does a separated flag value the check doesn't + know, such as `cut -d ','`, which counts as a file. A read is linked to a run by the spelling of its + path, so a read through an intermediate copy, or of the same file spelled + differently (relative against absolute, or after a `cd` in an earlier call), + isn't linked - `inspect report` writes CSVs, so its own output may not name `orders`. It - passes only when the agent reads a CSV in the same tool call + passes only when the agent reads a CSV it wrote, in the same tool call or a + later one - The inspect check shows that a report named `orders`. It doesn't show that the agent's fix followed from it - The live index isn't tied by name to the migration's index. The plan check @@ -209,6 +213,8 @@ experiments. again. This is intended - Every new file that creates the index must be recorded, so a second, never-applied file creating it fails the history check +- A `do $$ … $$` block that creates the index isn't recognised: dollar-quoted + bodies are masked, so the migration check fails it - A named stack (`--stack`) isn't resolved. Only the project's default stack counts, including under a relocated `SUPABASE_HOME`/`HOME` that the agent set on a start command diff --git a/evals/cli/investigate-database-001-slow-query/inspect.test.ts b/evals/cli/investigate-database-001-slow-query/inspect.test.ts index 25231607..67e2cec1 100644 --- a/evals/cli/investigate-database-001-slow-query/inspect.test.ts +++ b/evals/cli/investigate-database-001-slow-query/inspect.test.ts @@ -1,11 +1,7 @@ // Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query import type { ToolCallRecord } from '@supabase-evals/core'; import { describe, expect, it } from 'vitest'; -import { - findSupabaseInvocations, - type SupabaseInvocation, -} from '../lib/cli-invocations.js'; -import { extractCommandEntries } from '../lib/detours.js'; +import { findSupabaseInvocations } from '../lib/cli-invocations.js'; import { checkInspectShowedOrders, findInspectRuns, @@ -50,11 +46,7 @@ const fileWrite = (result = ''): ToolCallRecord => ({ ts: 0, }); -const invocationsOf = (toolCalls: ToolCallRecord[]) => - findSupabaseInvocations(extractCommandEntries(toolCalls)); - -const runsOf = (toolCalls: ToolCallRecord[]) => - findInspectRuns(toolCalls, invocationsOf(toolCalls)); +const runsOf = (toolCalls: ToolCallRecord[]) => findInspectRuns(toolCalls); const inspect = (command: string, stdout: string) => call(command, { result: stdout }); @@ -356,27 +348,420 @@ describe('findInspectRuns', () => { } ); - it('does not let a help invocation pass the check even when handed one directly', () => { - const help: SupabaseInvocation = { - commandIndex: 0, - argv: ['supabase', 'inspect', 'db', '--help'], - }; - const calls = [inspect('supabase inspect db --help', OUTLIERS)]; + it('is empty when the agent ran nothing', () => { + expect(findInspectRuns([])).toEqual([]); + }); +}); + +describe('findInspectRuns attribution', () => { + const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; + const ORDERS_ROW = + ' select id, status from public.orders where customer_email = $1 | 200'; + const MIGRATION = + 'create table public.orders (\n id bigint primary key,\n customer_email text not null\n);'; + const RULES = '|RULE|STATUS|\n|-|-|\n|Unused indexes|PASS|\n'; + const REPORT_CSV = + 'query,calls\nselect id from public.orders where customer_email = $1,200\n'; + + const credited = (toolCalls: ToolCallRecord[]) => + runsOf(toolCalls).map(({ namesOrders }) => namesOrders); + const passes = (toolCalls: ToolCallRecord[]) => + checkInspectShowedOrders(runsOf(toolCalls)).passed; + + it.each([ + [ + 'a psql fallback in the same call after a refused inspect', + [ + call( + `supabase inspect db outliers 2>&1 || psql "${DB_URL}" -c "select query, calls from extensions.pg_stat_statements order by total_exec_time desc limit 5"`, + { result: `${NO_PROJECT_REF}\n${ORDERS_ROW}` } + ), + ], + ], + [ + 'migrations cat-ed before an inspect that exited non-zero', + [ + call('cat supabase/migrations/*.sql; supabase inspect db outliers', { + error: `${MIGRATION}\n${NO_PROJECT_REF}`, + }), + ], + ], + [ + 'a psql explain after an inspect whose output never named orders', + [ + call( + `supabase inspect db outliers --local | head -5; psql "$DB" -c "explain select * from public.orders where customer_email = 'x'"`, + { + result: `${UNRELATED_OUTLIERS}\nSeq Scan on orders (cost=0.00..6500.00 rows=60 width=40)`, + } + ), + ], + ], + [ + 'an echo after the inspect', + [ + call('supabase inspect db outliers --local; echo orders', { + result: `${UNRELATED_OUTLIERS}\norders`, + }), + ], + ], + [ + 'a db query after the inspect', + [ + call( + 'supabase inspect db outliers --local && supabase db query "select * from orders limit 1"', + { result: `${UNRELATED_OUTLIERS}\n${ORDERS_ROW}` } + ), + ], + ], + [ + 'a migration read in a later call after a report with the default output directory', + [ + call('supabase inspect report --local', { result: RULES }), + call('cat supabase/migrations/20260102000000_x.sql', { + result: MIGRATION, + }), + ], + ], + [ + 'a grep of the seed in a later call', + [ + inspect('supabase inspect db outliers --local', UNRELATED_OUTLIERS), + call('grep orders supabase/seed.sql', { + result: "insert into public.orders values ('a');", + }), + ], + ], + [ + 'a read of a directory that only shares the output directory prefix', + [ + call('supabase inspect report --local --output-dir /tmp/r', { + result: RULES, + }), + call('cat /tmp/rx/2026-10-22/outliers.csv', { result: REPORT_CSV }), + ], + ], + [ + 'a read through a variable', + [ + call('supabase inspect db outliers --local > /tmp/o.txt', { + result: '', + }), + call('cat "$OUT"', { result: OUTLIERS }), + ], + ], + [ + 'a read that shares its call with another command', + [ + call('supabase inspect db outliers --local > /tmp/o.txt', { + result: '', + }), + call('cat /tmp/o.txt; echo done', { result: `${OUTLIERS}\ndone` }), + ], + ], + ])('does not credit %s', (_label, toolCalls) => { + expect(new Set(credited(toolCalls))).toEqual(new Set([false])); + expect(passes(toolCalls)).toBe(false); + }); + + it.each([ + [ + 'a pipe into head', + 'supabase inspect db outliers --local | head -40', + OUTLIERS, + ], + [ + 'a jq filter', + `supabase inspect db table-stats --local -o json | jq '.[] | select(.Name | test("orders"))'`, + TABLE_STATS_JSON, + ], + [ + 'a chain of filters and a merged stderr', + 'supabase inspect db outliers --local 2>&1 | grep -i orders | sort | uniq -c | head', + OUTLIERS, + ], + [ + 'a report written and read back in one call', + 'supabase inspect report --local --output-dir /tmp/r && cat /tmp/r/*/outliers.csv', + REPORT_CSV, + ], + [ + 'two inspect reports in one call', + 'supabase inspect db outliers --local; supabase inspect db table-stats --local', + OUTLIERS, + ], + [ + 'a cd before the report', + 'cd /workspace && supabase inspect db outliers --local', + OUTLIERS, + ], + [ + 'a start before the report', + 'supabase start && supabase inspect db outliers --local', + OUTLIERS, + ], + [ + 'a version check, an export and a variable before the report', + 'supabase --version && export A=1 && DB=x && sleep 1 && supabase inspect db outliers --local', + OUTLIERS, + ], + [ + 'a bash wrapper', + `bash -lc 'supabase inspect db outliers --local'`, + OUTLIERS, + ], + [ + 'a package runner', + 'npx supabase@beta inspect db calls --local | head -40', + OUTLIERS, + ], + [ + 'a prefixed environment', + 'SUPABASE_HOME=/tmp/h supabase inspect db outliers --local', + OUTLIERS, + ], + [ + 'an absolute binary path', + '/usr/local/bin/supabase inspect db outliers --local', + OUTLIERS, + ], + ])('credits %s', (_label, command, stdout) => { + expect(new Set(credited([inspect(command, stdout)]))).toEqual( + new Set([true]) + ); + }); + + it('credits a report redirected to a file once a later call reads the file', () => { + const runs = runsOf([ + call('supabase start'), + call('supabase inspect db outliers --local > /tmp/outliers.txt 2>&1'), + call('cat /tmp/outliers.txt', { result: OUTLIERS }), + ]); + expect(runs).toEqual([ + { + commandIndex: 1, + report: 'db outliers', + errored: false, + namesOrders: true, + readInCall: 2, + }, + ]); + expect(checkInspectShowedOrders(runs).notes).toBe( + 'tool call 2: supabase inspect db outliers via tool call 3' + ); + }); + + it('credits a report written to a directory once a later call reads a file in it', () => { + const runs = runsOf([ + call('supabase start'), + call('supabase inspect report --local --output-dir /tmp/r', { + result: RULES, + }), + call('cat /tmp/r/2026-10-22/outliers.csv', { result: REPORT_CSV }), + ]); + expect(runs).toEqual([ + { + commandIndex: 1, + report: 'report', + errored: false, + namesOrders: true, + readInCall: 2, + }, + ]); + }); + + it.each([ + [ + 'the equals form of the output directory', + 'supabase inspect report --local --output-dir=/tmp/r', + 'head -5 /tmp/r/2026-10-22/outliers.csv', + ], + [ + 'the current directory as the output directory', + 'supabase inspect report --local --output-dir .', + 'cat ./2026-10-22/outliers.csv', + ], + [ + 'the default output directory', + 'supabase inspect report --local', + 'cat 2026-10-22/outliers.csv', + ], + [ + 'a relative output directory', + 'supabase inspect report --local --output-dir reports', + 'cat ./reports/2026-10-22/outliers.csv', + ], + [ + 'a combined redirect', + 'supabase inspect db outliers --local &> out.txt', + 'cat ./out.txt', + ], + [ + 'an appending redirect', + 'supabase inspect db outliers --local >> /tmp/o.txt', + 'grep orders /tmp/o.txt', + ], + [ + 'a tee', + 'supabase inspect db outliers --local | tee /tmp/o.txt > /dev/null', + 'tail -n 20 /tmp/o.txt', + ], + [ + 'an input redirect', + 'supabase inspect db outliers --local > /tmp/o.txt', + 'jq . < /tmp/o.txt', + ], + ])('links a later read to the run for %s', (_label, run, read) => { + const runs = runsOf([call(run), call(read, { result: REPORT_CSV })]); + expect(runs.map(({ namesOrders }) => namesOrders)).toEqual([true]); + expect(runs[0].readInCall).toBe(1); + }); + + it('credits every run whose output a read covers', () => { + const runs = runsOf([ + call('supabase inspect report --local', { result: RULES }), + call('supabase inspect report --local --output-dir .', { result: RULES }), + call('cat 2026-10-22/outliers.csv', { result: REPORT_CSV }), + ]); + expect(runs.map(({ namesOrders }) => namesOrders)).toEqual([true, true]); + }); + + it('keeps the first read that credited a run', () => { + const runs = runsOf([ + call('supabase inspect db outliers --local > /tmp/o.txt'), + call('cat /tmp/o.txt', { result: OUTLIERS }), + call('cat /tmp/o.txt', { result: OUTLIERS }), + ]); + expect(runs[0].readInCall).toBe(1); + }); + + it('marks the runs of a call whose output other commands share', () => { + const runs = runsOf([ + call('supabase start'), + call( + `supabase inspect db outliers 2>&1 || psql "${DB_URL}" -c "select 1 from public.orders"`, + { error: `${NO_PROJECT_REF}\n${ORDERS_ROW}` } + ), + ]); + expect(runs).toEqual([ + { + commandIndex: 1, + report: 'db outliers', + errored: true, + namesOrders: false, + sharedOutput: true, + }, + ]); + expect(checkInspectShowedOrders(runs).notes).toContain( + 'tool call 2: supabase inspect db outliers (errored; output shared with other commands)' + ); + }); + + it('does not mark a call that named no table, however many commands it ran', () => { expect( - checkInspectShowedOrders(findInspectRuns(calls, [help])).passed - ).toBe(false); + runsOf([ + call('supabase inspect db outliers --local; echo done', { + result: `${UNRELATED_OUTLIERS}\ndone`, + }), + ])[0].sharedOutput + ).toBeUndefined(); }); +}); - it('does not throw for an invocation that points past the tool calls', () => { - const stray: SupabaseInvocation = { - commandIndex: 5, - argv: ['supabase', 'inspect', 'db', 'outliers', '--local'], - }; - expect(() => findInspectRuns([], [stray])).not.toThrow(); +describe('findInspectRuns file operands', () => { + const credited = (command: string, stdout: string) => + runsOf([inspect(command, stdout)]).map(({ namesOrders }) => namesOrders); + + it.each([ + [ + 'a recursive grep of the project', + 'supabase inspect db outliers --local; grep -r orders supabase', + ], + [ + 'a cat of a bare file name', + 'supabase inspect db outliers --local; cat Makefile', + ], + [ + 'a grep of a bare directory name', + 'supabase inspect db outliers --local; grep orders supabase', + ], + [ + 'a grep whose pattern comes from -e and whose file is named', + 'supabase inspect db outliers --local; grep -e orders supabase/seed.sql', + ], + [ + 'a grep whose pattern is attached to -e', + 'supabase inspect db outliers --local; grep -eorders supabase/seed.sql', + ], + [ + 'a sed over a file', + "supabase inspect db outliers --local; sed -n '1,5p' seed.sql", + ], + [ + 'a numeric pattern that leaves the file operand', + 'supabase inspect db outliers --local; grep 5 supabase', + ], + ])('does not credit %s', (_label, command) => { + expect(credited(command, OUTLIERS)).toEqual([false]); }); - it('is empty when the agent ran nothing', () => { - expect(findInspectRuns([], [])).toEqual([]); + it.each([ + ['a case-insensitive grep', 'grep -i orders', OUTLIERS], + ['a sed print range', "sed -n '1,40p'", OUTLIERS], + ['a sed substitution containing slashes', "sed 's/a/b/'", OUTLIERS], + ['head with a count', 'head -n 40', OUTLIERS], + ['tail from a line', 'tail -n +2', OUTLIERS], + ['a jq field filter', "jq -r '.[].Name'", TABLE_STATS_JSON], + ['jq with clustered flags', "jq -er '.[].Name'", TABLE_STATS_JSON], + [ + 'a jq variable', + `jq --arg n orders '.[] | select(.Name == $n)'`, + TABLE_STATS_JSON, + ], + ['a grep pattern from -e', 'grep -e orders', OUTLIERS], + [ + 'a grep with several -e patterns', + 'grep -e orders -e customers', + OUTLIERS, + ], + ['a grep with a match limit', 'grep -m 5 -i orders', OUTLIERS], + ['an awk field separator', `awk -F '|' '{print $2}'`, OUTLIERS], + ['cut with attached flag values', 'cut -d, -f1', OUTLIERS], + ['tr with arguments that are not files', "tr '/' ' '", OUTLIERS], + ['wc and column', 'column -t', OUTLIERS], + ])('credits a pipe into %s', (_label, filter, stdout) => { + const command = filter.startsWith('jq') + ? `supabase inspect db table-stats --local -o json | ${filter}` + : `supabase inspect db outliers --local | ${filter}`; + expect(credited(command, stdout)).toEqual([true]); + }); + + it('counts the value of a separated flag as a file operand, so cut -d "," credits nothing', () => { + expect( + credited( + `supabase inspect db outliers --local --output-format csv | cut -d ',' -f 1`, + OUTLIERS + ) + ).toEqual([false]); + }); + + it.each([ + 'supabase stack start', + 'supabase stack stop', + 'supabase stack status', + 'supabase stack list', + ])('credits a run after %s in the same call', (quiet) => { + expect( + credited(`${quiet} && supabase inspect db outliers --local`, OUTLIERS) + ).toEqual([true]); + }); + + it('links a later read of a file a tee wrote under a bare name', () => { + const runs = runsOf([ + call('supabase inspect db outliers --local | tee out.txt > /dev/null'), + call('cat out.txt', { result: OUTLIERS }), + ]); + expect(runs.map(({ namesOrders }) => namesOrders)).toEqual([true]); }); }); @@ -441,6 +826,33 @@ describe('checkInspectShowedOrders', () => { expect(check.notes).toMatch(/orders/); }); + it('says which later tool call read the output that named orders', () => { + const check = checkInspectShowedOrders([run({ readInCall: 3 })]); + expect(check).toMatchObject({ passed: true }); + expect(check.notes).toBe( + 'tool call 2: supabase inspect db outliers via tool call 4' + ); + }); + + it('says a failing run shared its output with other commands', () => { + const check = checkInspectShowedOrders([ + run({ namesOrders: false, errored: true, sharedOutput: true }), + run({ + commandIndex: 2, + report: 'db calls', + namesOrders: false, + sharedOutput: true, + }), + ]); + expect(check.passed).toBe(false); + expect(check.notes).toContain( + 'tool call 2: supabase inspect db outliers (errored; output shared with other commands)' + ); + expect(check.notes).toContain( + 'tool call 3: supabase inspect db calls (output shared with other commands)' + ); + }); + it('marks nothing as errored when no report errored', () => { const check = checkInspectShowedOrders([ run({ report: 'db calls', namesOrders: false }), diff --git a/evals/cli/investigate-database-001-slow-query/inspect.ts b/evals/cli/investigate-database-001-slow-query/inspect.ts index 77deea45..d90279dc 100644 --- a/evals/cli/investigate-database-001-slow-query/inspect.ts +++ b/evals/cli/investigate-database-001-slow-query/inspect.ts @@ -1,9 +1,19 @@ import type { CheckResult, ToolCallRecord } from '@supabase-evals/core'; +import type { ParseEntry } from 'shell-quote'; import { + findSupabaseInvocations, + invocationFlag, invocationPositionals, + invocationVerb, type SupabaseInvocation, } from '../lib/cli-invocations.js'; -import { commandToolCalls } from '../lib/detours.js'; +import { + commandToolCalls, + extractCommandEntries, + leadingWord, + tryShellQuoteParse, + unmaskedCommandSegments, +} from '../lib/detours.js'; import { ORDERS_TABLE } from './fixtures.js'; export type InspectRun = { @@ -11,6 +21,10 @@ export type InspectRun = { report: string; errored: boolean; namesOrders: boolean; + /** The later tool call whose read of the run's output named orders. */ + readInCall?: number; + /** The run's call named orders, but other commands in it may have printed that. */ + sharedOutput?: true; }; export const INSPECT_CHECK_NAME = @@ -20,6 +34,74 @@ const ORDERS_RE = new RegExp(`\\b${ORDERS_TABLE}\\b`); const HELP_FLAGS = new Set(['--help', '-h']); const PATH_VALUE_FLAGS = new Set(['--db-url', '--output-dir']); const CSI_SEQUENCE = /\x1b\[[0-9;?]*[ -\/]*[@-~]/g; +const QUIET_VERBS = new Set([ + 'start', + 'stop', + 'status', + 'stack start', + 'stack stop', + 'stack status', + 'stack list', +]); +const NEUTRAL_WORDS = new Set([ + 'cd', + 'pushd', + 'popd', + 'export', + 'set', + 'true', + ':', + 'sleep', +]); +const FILTER_WORDS = new Set([ + 'head', + 'tail', + 'grep', + 'egrep', + 'fgrep', + 'rg', + 'sed', + 'awk', + 'jq', + 'cut', + 'sort', + 'uniq', + 'column', + 'tee', + 'less', + 'more', + 'wc', + 'tr', + 'nl', + 'cat', + 'fold', +]); + +type FilterSyntax = { + patternFlags: readonly string[]; + valueFlags: Readonly>; +}; +const GREP_SYNTAX: FilterSyntax = { + patternFlags: ['-e', '-f', '--regexp', '--file'], + valueFlags: { '-m': 1, '-A': 1, '-B': 1, '-C': 1 }, +}; +const FILTER_SYNTAX: Readonly> = { + grep: GREP_SYNTAX, + egrep: GREP_SYNTAX, + fgrep: GREP_SYNTAX, + rg: GREP_SYNTAX, + sed: { patternFlags: ['-e', '-f', '--expression', '--file'], valueFlags: {} }, + awk: { patternFlags: ['-f', '--file'], valueFlags: { '-v': 1, '-F': 1 } }, + jq: { + patternFlags: ['-f', '--from-file'], + valueFlags: { '--arg': 2, '--argjson': 2 }, + }, +}; +const VAR_ASSIGNMENT_RE = /^[A-Za-z_][A-Za-z0-9_]*=/; +const NUMERIC_RE = /^[+-]?\d+$/; +const GLOB_CHAR_RE = /[*?[{]/; +const DATED_FOLDER_RE = /^(?:\.\/)?\d{4}-\d{2}-\d{2}\//; +const CURRENT_DIR_RE = /^(?:\.\/?)?$/; /** The report an executed `supabase inspect` invocation ran, such as `db outliers` or `report`; undefined for anything else, help included. */ export function inspectReport(inv: SupabaseInvocation): string | undefined { @@ -45,29 +127,229 @@ function resultText(result: unknown): string { const callText = ({ result, error }: ToolCallRecord): string => `${resultText(result)}\n${error ?? ''}`.replace(CSI_SEQUENCE, ''); -/** One run per executed `supabase inspect` report, read from the output of the tool call that ran it. */ +type Operand = { text: string; globbed: boolean }; +type Location = (operand: Operand) => boolean; +type Segment = + | { kind: 'inspect'; report: string; locations: Location[] } + | { kind: 'filter'; reads: Operand[]; writes: Location[] } + | { kind: 'quiet' } + | { kind: 'other' }; + +const QUIET: Segment = { kind: 'quiet' }; +const OTHER: Segment = { kind: 'other' }; + +const withoutDot = (path: string) => path.replace(/^(?:\.\/)+/, ''); + +/** The location of a file or directory: that path and everything below it. */ +function under(path: string): Location[] { + const root = withoutDot(path).replace(/\/+$/, ''); + if (root === '') return []; + return [ + ({ text, globbed }) => { + const operand = withoutDot(text); + return operand.startsWith(`${root}/`) || (!globbed && operand === root); + }, + ]; +} + +const datedFolders: Location = ({ text }) => DATED_FOLDER_RE.test(text); + +/** + * The file operands of a command's arguments and the files it writes. Every + * non-flag word is a file except a number, the pattern or program a + * `FILTER_SYNTAX` command takes first (or through its pattern flag), and what + * `tr` takes; a separated flag value it doesn't know counts as a file. + * `unresolved` is set for a variable or substitution that could not be read. + */ +function readArguments( + tokens: readonly ParseEntry[], + word = '' +): { operands: Operand[]; writes: string[]; unresolved: boolean } { + const syntax = FILTER_SYNTAX[word]; + const operands: Operand[] = []; + const writes: string[] = []; + let unresolved = false; + let patternPending = syntax !== undefined; + const addOperand = (pattern: string, globbed: boolean) => { + const text = globbed ? pattern.split(GLOB_CHAR_RE, 1)[0] : pattern; + operands.push({ text, globbed }); + }; + for (let i = 0; i < tokens.length; i++) { + const token = tokens[i]; + if (typeof token === 'string') { + if (token === '' || token.includes('`')) { + unresolved = true; + } else if (token.startsWith('-')) { + const supplied = syntax?.patternFlags.find( + (flag) => + token === flag || + token.startsWith(flag.startsWith('--') ? `${flag}=` : flag) + ); + if (supplied !== undefined) { + patternPending = false; + if (token === supplied) i++; + } else { + i += syntax?.valueFlags[token] ?? 0; + } + } else if (patternPending) { + patternPending = false; + } else if (!NUMERIC_RE.test(token) && word !== 'tr') { + addOperand(token, false); + } + } else if ('op' in token && token.op === 'glob') { + addOperand(token.pattern, true); + } else if ('op' in token) { + const target = tokens[i + 1]; + if (token.op === '>' || token.op === '>>') { + if (typeof target === 'string') writes.push(target); + i++; + } else if (token.op === '>&') { + i++; + } else if (token.op === '<' && typeof target === 'string') { + addOperand(target, false); + i++; + } + } + } + return { operands, writes, unresolved }; +} + +function outputLocations( + inv: SupabaseInvocation, + report: string, + writes: readonly string[] +): Location[] { + const locations = writes.flatMap(under); + if (report !== 'report') return locations; + const dir = invocationFlag(inv, '--output-dir'); + return dir === undefined || CURRENT_DIR_RE.test(dir) + ? [...locations, datedFolders] + : [...locations, ...under(dir)]; +} + +const isQuiet = (inv: SupabaseInvocation) => + QUIET_VERBS.has(invocationVerb(inv) ?? '') || inv.argv.includes('--version'); + +function classify(segment: string, cwd: string | undefined): Segment { + const tokens = tryShellQuoteParse(segment) ?? []; + const [inv] = findSupabaseInvocations([ + { command: segment, ...(cwd === undefined ? {} : { cwd }) }, + ]); + if (inv !== undefined) { + const report = inspectReport(inv); + if (report !== undefined) { + const { writes } = readArguments(tokens); + return { + kind: 'inspect', + report, + locations: outputLocations(inv, report, writes), + }; + } + return isQuiet(inv) ? QUIET : OTHER; + } + if ( + tokens.length > 0 && + tokens.every( + (token) => typeof token === 'string' && VAR_ASSIGNMENT_RE.test(token) + ) + ) { + return QUIET; + } + const word = leadingWord(segment); + if (word === undefined) return OTHER; + if (NEUTRAL_WORDS.has(word)) return QUIET; + if (!FILTER_WORDS.has(word)) return OTHER; + const start = tokens.findIndex( + (token) => + typeof token === 'string' && + token.slice(token.lastIndexOf('/') + 1).toLowerCase() === word + ); + if (start === -1) return OTHER; + const { operands, unresolved } = readArguments(tokens.slice(start + 1), word); + if (unresolved) return OTHER; + return word === 'tee' + ? { + kind: 'filter', + reads: [], + writes: operands + .filter(({ globbed }) => !globbed) + .flatMap(({ text }) => under(text)), + } + : { kind: 'filter', reads: operands, writes: [] }; +} + +type Tracked = { run: InspectRun; locations: Location[] }; + +/** One run per executed `supabase inspect` report, credited with naming orders only when the output that named it came from inspect alone, or from reading what inspect wrote. */ export function findInspectRuns( - toolCalls: readonly ToolCallRecord[], - invocations: readonly SupabaseInvocation[] + toolCalls: readonly ToolCallRecord[] ): InspectRun[] { const calls = commandToolCalls(toolCalls); - return invocations.flatMap((inv): InspectRun[] => { - const report = inspectReport(inv); - const call = calls[inv.commandIndex]; - if (report === undefined || call === undefined) return []; - return [ - { - commandIndex: inv.commandIndex, - report, - errored: call.error !== undefined, - namesOrders: ORDERS_RE.test(callText(call)), - }, - ]; + const entries = extractCommandEntries(toolCalls); + const tracked: Tracked[] = []; + calls.forEach((call, commandIndex) => { + const { command, cwd } = entries[commandIndex]; + const segments = unmaskedCommandSegments(command).map((segment) => + classify(segment, cwd) + ); + const teed = segments.flatMap((segment) => + segment.kind === 'filter' ? segment.writes : [] + ); + const here = segments.flatMap((segment): Tracked[] => + segment.kind === 'inspect' + ? [ + { + run: { + commandIndex, + report: segment.report, + errored: call.error !== undefined, + namesOrders: false, + }, + locations: [...segment.locations, ...teed], + }, + ] + : [] + ); + tracked.push(...here); + + const covering = (operand: Operand) => + tracked.filter(({ locations }) => + locations.some((location) => location(operand)) + ); + const reads = segments.flatMap((segment) => + segment.kind === 'filter' ? segment.reads : [] + ); + if (!ORDERS_RE.test(callText(call))) return; + const shared = + segments.some(({ kind }) => kind === 'other') || + reads.some((read) => covering(read).length === 0); + if (shared) { + for (const { run } of here) run.sharedOutput = true; + return; + } + for (const { run } of new Set([...here, ...reads.flatMap(covering)])) { + if (run.namesOrders) continue; + run.namesOrders = true; + if (run.commandIndex !== commandIndex) run.readInCall = commandIndex; + } }); + return tracked.map(({ run }) => run); } -const describeRun = ({ commandIndex, report }: InspectRun) => - `tool call ${commandIndex + 1}: supabase inspect ${report}`; +const describeRun = ({ commandIndex, report, readInCall }: InspectRun) => + `tool call ${commandIndex + 1}: supabase inspect ${report}${ + readInCall === undefined ? '' : ` via tool call ${readInCall + 1}` + }`; + +function describeUnshown(run: InspectRun) { + const qualifiers = [ + ...(run.errored ? ['errored'] : []), + ...(run.sharedOutput ? ['output shared with other commands'] : []), + ]; + return `${describeRun(run)}${ + qualifiers.length > 0 ? ` (${qualifiers.join('; ')})` : '' + }`; +} export function checkInspectShowedOrders( runs: readonly InspectRun[] @@ -80,12 +362,9 @@ export function checkInspectShowedOrders( if (runs.length === 0) { return { name, passed: false, notes: 'no supabase inspect report ran' }; } - const ran = runs - .map((run) => `${describeRun(run)}${run.errored ? ' (errored)' : ''}`) - .join('; '); return { name, passed: false, - notes: `${ran}; none printed the ${ORDERS_TABLE} table`, + notes: `${runs.map(describeUnshown).join('; ')}; none printed the ${ORDERS_TABLE} table`, }; } diff --git a/evals/cli/investigate-database-001-slow-query/migrations.test.ts b/evals/cli/investigate-database-001-slow-query/migrations.test.ts index 58e9fee9..4291585e 100644 --- a/evals/cli/investigate-database-001-slow-query/migrations.test.ts +++ b/evals/cli/investigate-database-001-slow-query/migrations.test.ts @@ -136,10 +136,26 @@ const INDEXES_ON_EMAIL: Array<[string, string]> = [ 'a collation', 'create index i on public.orders (customer_email collate "C");', ], + [ + 'a schema-qualified collation', + 'create index i on public.orders (customer_email collate pg_catalog."default");', + ], [ 'an operator class', 'create index i on public.orders (customer_email text_pattern_ops);', ], + [ + 'a quoted access method', + 'create index i on public.orders using "btree" (customer_email);', + ], + [ + 'the column in redundant parentheses', + 'create index i on public.orders ((customer_email));', + ], + [ + 'the column in parentheses followed by a direction', + 'create index i on public.orders ((customer_email) desc);', + ], [ 'a quoted column with a direction', 'create index i on public.orders ("customer_email" asc);', @@ -212,6 +228,14 @@ const NOT_AN_EMAIL_INDEX: Array<[string, string]> = [ 'an expression on the column', 'create index i on orders (lower(customer_email));', ], + [ + 'an expression in redundant parentheses', + 'create index i on orders ((lower(customer_email)));', + ], + [ + 'customer_email second in a composite, in parentheses', + 'create index i on public.orders (created_at, (customer_email));', + ], [ 'an expression with a comma inside its parentheses', "create index i on orders (coalesce(customer_email, ''), created_at);", diff --git a/evals/cli/investigate-database-001-slow-query/migrations.ts b/evals/cli/investigate-database-001-slow-query/migrations.ts index 974eb338..b333fd28 100644 --- a/evals/cli/investigate-database-001-slow-query/migrations.ts +++ b/evals/cli/investigate-database-001-slow-query/migrations.ts @@ -8,7 +8,7 @@ import { SEEDED_MIGRATION_FILE, SEEDED_MIGRATION_VERSION, } from './fixtures.js'; -import { queryDatabase } from './orders.js'; +import { queryDatabase } from './psql.js'; type ExecContext = Pick; @@ -33,12 +33,13 @@ export const RECORDED_CHECK_NAME = const MIGRATION_FILENAME_RE = /^(\d+)_(.+)\.sql$/; const IDENTIFIER = String.raw`"[^"]+"|[^\s"(),;]+`; +const NAME = String.raw`(?:[a-z_][\w$]*|"[^"]+")`; const CREATE_INDEX_RE = new RegExp( - String.raw`^\s*create\s+(?:unique\s+)?index\s+(?:concurrently\s+)?(?:if\s+not\s+exists\s+)?(?:(${IDENTIFIER})\s+)?on\s+(?:only\s+)?(?:"?public"?\s*\.\s*)?"?${ORDERS_TABLE}"?\s*(?:using\s+\w+\s*)?\(`, + String.raw`^\s*create\s+(?:unique\s+)?index\s+(?:concurrently\s+)?(?:if\s+not\s+exists\s+)?(?:(${IDENTIFIER})\s+)?on\s+(?:only\s+)?(?:"?public"?\s*\.\s*)?"?${ORDERS_TABLE}"?\s*(?:using\s+(?:\w+|"[^"]+")\s*)?\(`, 'i' ); const EMAIL_KEY_RE = new RegExp( - String.raw`^"?${EMAIL_COLUMN}"?(?:\s+(?:[a-z_][\w$]*|"[^"]+"))*$`, + String.raw`^"?${EMAIL_COLUMN}"?(?:\s+${NAME}(?:\s*\.\s*${NAME})*)*$`, 'i' ); const HISTORY_ROW_RE = /^(\d+)\|((?:[0-9a-f]{2})*)$/i; @@ -46,8 +47,8 @@ const NO_NEW_MIGRATION_NOTE = `no new migration creates an index on ${ORDERS_TAB const compareText = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0); -/** The top-level key texts of the index column list that `text` starts inside, or undefined when it never closes. */ -function splitKeys(text: string): string[] | undefined { +/** The top-level key texts of the index column list that `text` starts inside and the offset of its closing parenthesis, or undefined when it never closes. */ +function splitKeys(text: string): { keys: string[]; end: number } | undefined { const keys: string[] = []; let depth = 1; let quoted = false; @@ -60,7 +61,7 @@ function splitKeys(text: string): string[] | undefined { depth++; } else if (!quoted && char === ')' && --depth === 0) { keys.push(text.slice(start, i)); - return keys.map((key) => key.trim()); + return { keys: keys.map((key) => key.trim()), end: i }; } else if (!quoted && char === ',' && depth === 1) { keys.push(text.slice(start, i)); start = i + 1; @@ -69,6 +70,16 @@ function splitKeys(text: string): string[] | undefined { return undefined; } +/** `key` with the redundant parentheses around its leading expression removed, since `((col) desc)` indexes the plain column. */ +function unwrapLeadingParens(key: string): string { + if (!key.startsWith('(')) return key; + const group = splitKeys(key.slice(1)); + if (group === undefined || group.keys.length !== 1) return key; + return unwrapLeadingParens( + `${group.keys[0]} ${key.slice(group.end + 2)}`.trim() + ); +} + /** The `create index` statements on `public.orders` outside comments and literals, in file order. */ export function findOrdersIndexStatements(sql: string): IndexStatement[] { return maskSqlLiteralsAndComments(sql) @@ -76,7 +87,7 @@ export function findOrdersIndexStatements(sql: string): IndexStatement[] { .flatMap((statement): IndexStatement[] => { const create = CREATE_INDEX_RE.exec(statement); if (create === null) return []; - const keys = splitKeys(statement.slice(create[0].length)); + const keys = splitKeys(statement.slice(create[0].length))?.keys; if (keys === undefined) return []; const name = create[1]?.replace(/^"(.*)"$/, '$1'); return [{ ...(name === undefined ? {} : { name }), keys }]; @@ -86,7 +97,8 @@ export function findOrdersIndexStatements(sql: string): IndexStatement[] { /** Whether `sql` creates an index on `public.orders` whose first key is the bare `customer_email` column. */ export function createsEmailIndex(sql: string): boolean { return findOrdersIndexStatements(sql).some( - ({ keys }) => keys.length > 0 && EMAIL_KEY_RE.test(keys[0]) + ({ keys }) => + keys.length > 0 && EMAIL_KEY_RE.test(unwrapLeadingParens(keys[0])) ); } diff --git a/evals/cli/investigate-database-001-slow-query/orders.test.ts b/evals/cli/investigate-database-001-slow-query/orders.test.ts index cf16e408..93dec8c2 100644 --- a/evals/cli/investigate-database-001-slow-query/orders.test.ts +++ b/evals/cli/investigate-database-001-slow-query/orders.test.ts @@ -13,7 +13,6 @@ import { type PlanProbe, parsePlan, PLAN_CHECK_NAME, - queryDatabase, readLookupPlan, readOrdersIndexes, ROWS_CHECK_NAME, @@ -399,18 +398,6 @@ describe('parsePlan', () => { }); }); -describe('queryDatabase', () => { - it.each([ - ['a placeholder and a double quote', `select '$1' as "x"`], - ['single quotes', "select 'it''s' as x"], - ['a backslash before a dollar', "select '\\$1' as x"], - ])('hands psql the SQL unchanged when it holds %s', async (_label, sql) => { - const exec = vi.fn(async () => result('')); - await queryDatabase({ exec }, STACK, sql, () => ({ ok: true })); - expect(argvOf(exec.mock.calls[0][0])).toEqual([DB_URL, '-XtAc', sql]); - }); -}); - describe('readLookupPlan', () => { it('reports the stack failure without querying when the stack did not resolve', async () => { const db = fakeDatabase(); diff --git a/evals/cli/investigate-database-001-slow-query/orders.ts b/evals/cli/investigate-database-001-slow-query/orders.ts index 10053686..d27b13f4 100644 --- a/evals/cli/investigate-database-001-slow-query/orders.ts +++ b/evals/cli/investigate-database-001-slow-query/orders.ts @@ -1,10 +1,5 @@ import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core'; -import { - describeFailure, - errorMessage, - shellQuote, - truncate, -} from '../lib/shell.js'; +import { truncate } from '../lib/shell.js'; import { probeStackReady, type StackProbe } from '../lib/stack.js'; import { EMAIL_COLUMN, @@ -12,9 +7,9 @@ import { ORDERS_TABLE, SEEDED_ORDER_COUNT, } from './fixtures.js'; +import { type Failed, queryDatabase } from './psql.js'; type ExecContext = Pick; -type Failed = { ok: false; notes: string }; export type PlanNode = { nodeType: string; @@ -81,26 +76,6 @@ export function parsePlan(stdout: string): PlanProbe { } } -/** Runs `sql` in the stack's database through psql, or reports why it could not. */ -export async function queryDatabase( - ctx: ExecContext, - stack: StackProbe, - sql: string, - parse: (stdout: string) => T -): Promise { - if (!stack.ok) return { ok: false, notes: stack.notes }; - try { - const result = await ctx.exec( - `psql ${shellQuote(stack.dbUrl)} -XtAc ${shellQuote(sql)}` - ); - return result.ok - ? parse(result.stdout) - : { ok: false, notes: describeFailure(result) }; - } catch (error) { - return { ok: false, notes: errorMessage(error) }; - } -} - export const readLookupPlan = ( ctx: ExecContext, stack: StackProbe diff --git a/evals/cli/investigate-database-001-slow-query/psql.test.ts b/evals/cli/investigate-database-001-slow-query/psql.test.ts new file mode 100644 index 00000000..ea91a810 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/psql.test.ts @@ -0,0 +1,41 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import { execFileSync } from 'node:child_process'; +import type { CommandResult } from '@supabase-evals/core'; +import { describe, expect, it, vi } from 'vitest'; +import type { StackProbe } from '../lib/stack.js'; +import { queryDatabase } from './psql.js'; + +const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; + +const STACK: StackProbe = { + ok: true, + backend: 'managed', + dbUrl: DB_URL, + runtime: 'native', +}; + +const result = (stdout: string): CommandResult => ({ + ok: true, + exitCode: 0, + stdout, + stderr: '', +}); + +const argvOf = (command: string): string[] => + execFileSync('sh', ['-c', `psql() { printf '%s\\0' "$@"; }\n${command}`], { + encoding: 'utf8', + }) + .split('\0') + .slice(0, -1); + +describe('queryDatabase', () => { + it.each([ + ['a placeholder and a double quote', `select '$1' as "x"`], + ['single quotes', "select 'it''s' as x"], + ['a backslash before a dollar', "select '\\$1' as x"], + ])('hands psql the SQL unchanged when it holds %s', async (_label, sql) => { + const exec = vi.fn(async () => result('')); + await queryDatabase({ exec }, STACK, sql, () => ({ ok: true })); + expect(argvOf(exec.mock.calls[0][0])).toEqual([DB_URL, '-XtAc', sql]); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/psql.ts b/evals/cli/investigate-database-001-slow-query/psql.ts new file mode 100644 index 00000000..26d29587 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/psql.ts @@ -0,0 +1,26 @@ +import type { LocalStackEvalContext } from '@supabase-evals/core'; +import { describeFailure, errorMessage, shellQuote } from '../lib/shell.js'; +import type { StackProbe } from '../lib/stack.js'; + +type ExecContext = Pick; +export type Failed = { ok: false; notes: string }; + +/** Runs `sql` in the stack's database through psql, or reports why it could not. */ +export async function queryDatabase( + ctx: ExecContext, + stack: StackProbe, + sql: string, + parse: (stdout: string) => T +): Promise { + if (!stack.ok) return { ok: false, notes: stack.notes }; + try { + const result = await ctx.exec( + `psql ${shellQuote(stack.dbUrl)} -XtAc ${shellQuote(sql)}` + ); + return result.ok + ? parse(result.stdout) + : { ok: false, notes: describeFailure(result) }; + } catch (error) { + return { ok: false, notes: errorMessage(error) }; + } +} diff --git a/evals/cli/investigate-database-001-slow-query/report.test.ts b/evals/cli/investigate-database-001-slow-query/report.test.ts index cbabf59b..fc2c4de9 100644 --- a/evals/cli/investigate-database-001-slow-query/report.test.ts +++ b/evals/cli/investigate-database-001-slow-query/report.test.ts @@ -507,6 +507,31 @@ describe('describeGroundTruth', () => { ); }); + it('says a report ran in a call whose other commands may have printed orders', () => { + const inspectRuns = [ + run({ report: 'db calls', namesOrders: false, sharedOutput: true }), + ]; + expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( + `${INSPECT_LINE} tool call 1: db calls (output shared with other commands, not attributed)` + ); + }); + + it('says an errored report shared its output with other commands', () => { + const inspectRuns = [ + run({ errored: true, namesOrders: false, sharedOutput: true }), + ]; + expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( + `${INSPECT_LINE} tool call 1: db outliers (errored; output shared with other commands, not attributed)` + ); + }); + + it('says which later tool call read the output that showed orders', () => { + const inspectRuns = [run({ commandIndex: 1, readInCall: 3 })]; + expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( + `${INSPECT_LINE} tool call 2: db outliers (showed orders via tool call 4)` + ); + }); + it('does not say an errored report that printed orders showed nothing', () => { const inspectRuns = [run({ errored: true, namesOrders: true })]; const text = line(lines({ inspectRuns }), INSPECT_LINE) ?? ''; diff --git a/evals/cli/investigate-database-001-slow-query/report.ts b/evals/cli/investigate-database-001-slow-query/report.ts index a7292983..a75f4970 100644 --- a/evals/cli/investigate-database-001-slow-query/report.ts +++ b/evals/cli/investigate-database-001-slow-query/report.ts @@ -76,9 +76,19 @@ function describeRecorded(probe: RecordedProbe): string { function describeInspectRuns(runs: readonly InspectRun[]): string { if (runs.length === 0) return 'none'; - const outcome = ({ errored, namesOrders }: InspectRun) => { - if (namesOrders) return 'showed orders'; - return errored ? 'errored' : 'did not show orders'; + const outcome = (run: InspectRun) => { + if (run.namesOrders) { + return run.readInCall === undefined + ? 'showed orders' + : `showed orders via tool call ${run.readInCall + 1}`; + } + const problems = [ + ...(run.errored ? ['errored'] : []), + ...(run.sharedOutput + ? ['output shared with other commands, not attributed'] + : []), + ]; + return problems.length > 0 ? problems.join('; ') : 'did not show orders'; }; return runs .map( From 514b643571d5e6162d3a8511dc54c2a5a7ffca7f Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Thu, 22 Oct 2026 14:03:15 +0100 Subject: [PATCH 04/10] fix(evals): judge inspect output attribution in investigate-database-001-slow-query (CLI-2712) --- .../EVAL.test.ts | 166 ++- .../EVAL.ts | 44 +- .../README.md | 15 +- .../inspect.test.ts | 1007 +++++++---------- .../inspect.ts | 397 ++----- .../metrics.test.ts | 57 +- .../metrics.ts | 9 +- .../report.test.ts | 59 +- .../report.ts | 18 +- 9 files changed, 734 insertions(+), 1038 deletions(-) diff --git a/evals/cli/investigate-database-001-slow-query/EVAL.test.ts b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts index 6d414dc3..ed5d8054 100644 --- a/evals/cli/investigate-database-001-slow-query/EVAL.test.ts +++ b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts @@ -16,6 +16,11 @@ import { SEEDED_MIGRATION_SHA256, SEEDED_MIGRATION_VERSION, } from './fixtures.js'; +import { + findInspectEvidence, + findInspectRuns, + formatInspectJudgeInput, +} from './inspect.js'; const WS = '/ws'; const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; @@ -369,6 +374,17 @@ const isReportJudge = ({ input }: JudgeInput) => const judgeCalls = (judge: ReturnType['judge']) => judge.mock.calls.map(([args]) => args); +const INSPECT_RUBRIC = 'inspect evidence'; + +const isInspectJudge = ({ rubric }: JudgeInput) => + rubric.includes(INSPECT_RUBRIC); + +const isDetourJudge = ({ rubric }: JudgeInput) => + rubric.includes(DETOUR_RUBRIC); + +const inspectCall = (judge: ReturnType['judge']) => + judgeCalls(judge).find(isInspectJudge); + const detourCall = (judge: ReturnType['judge']) => judgeCalls(judge).find(({ rubric }) => rubric.includes(DETOUR_RUBRIC)); @@ -456,11 +472,18 @@ describe('slow-query scorer', () => { expect(check?.notes).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); }); - it('says in the inspect check which tool call ran the report', async () => { - const { ctx } = fakeCtx(SETUP); - const check = await checkOf(ctx, INSPECT); - expect(check?.passed).toBe(true); - expect(check?.notes).toContain('tool call 2: supabase inspect db outliers'); + it('passes the inspect check on the judge verdict, with the notes it gave', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => + isInspectJudge(args) + ? { passed: true, notes: 'tool call 2 printed the outliers rows' } + : { passed: true }, + }); + expect(await checkOf(ctx, INSPECT)).toEqual({ + name: INSPECT, + passed: true, + judgeNotes: 'tool call 2 printed the outliers rows', + }); }); }); @@ -731,15 +754,6 @@ describe('inspect evidence', () => { 'ran the default report with no project linked and was refused', call('supabase inspect db outliers', { error: NO_PROJECT_REF }), ], - [ - 'was refused by inspect and fell back to psql in the same call', - call( - `supabase inspect db outliers 2>&1 || psql "${DB_URL}" -c "select query, calls from extensions.pg_stat_statements order by total_exec_time desc limit 5"`, - { - result: `${NO_PROJECT_REF}\n select ... from public.orders where customer_email = $1 | 200`, - } - ), - ], [ 'read pg_stat_statements through psql', call( @@ -753,6 +767,74 @@ describe('inspect evidence', () => { result: 'select ... from public.orders where customer_email = $1', }), ], + [ + 'ran a report that never mentioned orders', + call('supabase inspect db outliers --local', { + result: UNRELATED_OUTLIERS, + }), + ], + ] satisfies Array<[string, ToolCallRecord]>)( + 'fails only the inspect check without asking the judge when the agent %s', + async (_label, inspectCall) => { + const { ctx, judge } = fakeCtx(withInspect(inspectCall)); + expect(await failedNames(ctx)).toEqual([INSPECT]); + expect(judgeCalls(judge).some(isInspectJudge)).toBe(false); + } + ); + + it('says in the inspect check that no report ran when the agent ran none', async () => { + const { ctx } = fakeCtx([SETUP[0], ...SETUP.slice(2)]); + expect(await checkOf(ctx, INSPECT)).toEqual({ + name: INSPECT, + passed: false, + notes: 'no supabase inspect report ran', + }); + }); + + it('says in the inspect check which runs ran when none printed orders', async () => { + const { ctx } = fakeCtx( + withInspect( + call('supabase inspect db outliers', { error: NO_PROJECT_REF }) + ) + ); + expect(await checkOf(ctx, INSPECT)).toEqual({ + name: INSPECT, + passed: false, + notes: + 'tool call 2: supabase inspect db outliers (errored); no inspect output named the orders table', + }); + }); + + it('asks the judge, with every evidence call in full, when something names orders', async () => { + const calls = withInspect( + call('supabase inspect db outliers --local > /tmp/o.txt') + ); + const withRead = [...calls, call('cat /tmp/o.txt', { result: OUTLIERS })]; + const { ctx, judge } = fakeCtx(withRead); + await scorer(ctx); + const runs = findInspectRuns(withRead, []); + const evidence = findInspectEvidence(withRead, runs); + expect(evidence).toHaveLength(2); + const sent = inspectCall(judge); + expect(sent?.input).toBe(formatInspectJudgeInput(evidence)); + expect(sent?.input).toContain('Tool call 5 (later read)'); + expect(sent?.input).toContain('cat /tmp/o.txt'); + expect(sent?.input).toContain(OUTLIERS); + expect(sent?.rubric).toContain('public.orders'); + expect(sent?.rubric).toContain('Cannot find project ref'); + expect(sent?.rubric).toMatch(/truncated/); + }); + + it.each([ + [ + 'was refused by inspect and fell back to psql in the same call', + call( + `supabase inspect db outliers 2>&1 || psql "${DB_URL}" -c "select query, calls from extensions.pg_stat_statements order by total_exec_time desc limit 5"`, + { + result: `${NO_PROJECT_REF}\n select ... from public.orders where customer_email = $1 | 200`, + } + ), + ], [ 'only asked inspect for help', call('supabase inspect db --help', { @@ -765,27 +847,37 @@ describe('inspect evidence', () => { result: 'supabase inspect db outliers (public.orders)', }), ], - [ - 'ran a report that never mentioned orders', - call('supabase inspect db outliers --local', { - result: UNRELATED_OUTLIERS, - }), - ], ] satisfies Array<[string, ToolCallRecord]>)( - 'fails only the inspect check when the agent %s', - async (_label, inspectCall) => { - const { ctx } = fakeCtx(withInspect(inspectCall)); + 'fails only the inspect check, with the judge notes, when the judge finds that the agent %s', + async (_label, inspectingCall) => { + const { ctx } = fakeCtx(withInspect(inspectingCall), { + judge: async (args) => + isInspectJudge(args) + ? { passed: false, notes: 'orders came from psql' } + : { passed: true }, + }); expect(await failedNames(ctx)).toEqual([INSPECT]); + expect(await checkOf(ctx, INSPECT)).toEqual({ + name: INSPECT, + passed: false, + judgeNotes: 'orders came from psql', + }); } ); - it('says in the inspect check that no report ran when the agent ran none', async () => { - const { ctx } = fakeCtx([SETUP[0], ...SETUP.slice(2)]); + it('fails only the inspect check with the error when the judge throws', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => { + if (isInspectJudge(args)) throw new Error('judge unavailable'); + return { passed: true }; + }, + }); expect(await checkOf(ctx, INSPECT)).toEqual({ name: INSPECT, passed: false, - notes: 'no supabase inspect report ran', + notes: 'judge unavailable', }); + expect(await failedNames(ctx)).toEqual([INSPECT]); }); it.each([ @@ -822,9 +914,9 @@ describe('inspect evidence', () => { ), ], ] satisfies Array<[string, ToolCallRecord]>)( - 'passes when the agent used %s', - async (_label, inspectCall) => { - const { ctx } = fakeCtx(withInspect(inspectCall)); + 'passes when the agent used %s and the judge accepts it', + async (_label, inspectingCall) => { + const { ctx } = fakeCtx(withInspect(inspectingCall)); expect(await failedNames(ctx)).toEqual([]); } ); @@ -1001,7 +1093,7 @@ describe('metrics check', () => { dbPort: 54322, inspectReports: ['db outliers'], inspectRuns: 1, - inspectRunsShowingOrders: 1, + inspectCallsNamingOrders: 1, workflowCommands: ['start', 'inspect db outliers', 'migration up'], explainCommands: 0, directIndexDdl: 0, @@ -1041,7 +1133,7 @@ describe('metrics check', () => { }); }); - it('reports the inspect reports that ran and how many showed orders', async () => { + it('reports the inspect reports that ran and how many evidence calls named orders', async () => { const { ctx } = fakeCtx([ SETUP[0], call('supabase inspect db outliers', { error: NO_PROJECT_REF }), @@ -1054,7 +1146,7 @@ describe('metrics check', () => { expect(await metrics(ctx)).toMatchObject({ inspectReports: ['db outliers', 'db calls', 'db table-stats'], inspectRuns: 3, - inspectRunsShowingOrders: 1, + inspectCallsNamingOrders: 1, }); }); @@ -1094,9 +1186,9 @@ describe('detour judge', () => { it('records the verdict notes and fails when the judge fails', async () => { const { ctx } = fakeCtx(SETUP, { judge: async (args) => - isReportJudge(args) - ? { passed: true } - : { passed: false, notes: 'command 2 installed docker' }, + isDetourJudge(args) + ? { passed: false, notes: 'command 2 installed docker' } + : { passed: true }, }); expect(await checkOf(ctx, DETOURS)).toMatchObject({ passed: false, @@ -1108,7 +1200,7 @@ describe('detour judge', () => { it('fails only this check when the judge throws', async () => { const { ctx } = fakeCtx(SETUP, { judge: async (args) => { - if (!isReportJudge(args)) throw new Error('judge unavailable'); + if (isDetourJudge(args)) throw new Error('judge unavailable'); return { passed: true }; }, }); @@ -1138,7 +1230,7 @@ describe('report judge', () => { ); expect(input).toContain('- recorded in the local migration history: yes'); expect(input).toContain( - '- supabase inspect reports the agent ran: tool call 2: db outliers (showed orders)' + '- supabase inspect reports the agent ran: tool call 2: db outliers' ); expect(input).toContain('- seeded migration modified: no'); expect(input).toContain('Transcript:'); diff --git a/evals/cli/investigate-database-001-slow-query/EVAL.ts b/evals/cli/investigate-database-001-slow-query/EVAL.ts index b2e227e6..a3e50052 100644 --- a/evals/cli/investigate-database-001-slow-query/EVAL.ts +++ b/evals/cli/investigate-database-001-slow-query/EVAL.ts @@ -15,7 +15,15 @@ import { import { formatGroundTruthJudgeInput } from '../lib/report.js'; import { errorMessage } from '../lib/shell.js'; import { resolveStackWithAgentHomes } from '../lib/stack.js'; -import { checkInspectShowedOrders, findInspectRuns } from './inspect.js'; +import { + findInspectEvidence, + findInspectRuns, + formatInspectJudgeInput, + gateInspectEvidence, + INSPECT_CHECK_NAME, + type EvidenceCall, + type InspectRun, +} from './inspect.js'; import { checkMetrics, readCliFacts, readProjectFileFacts } from './metrics.js'; import { checkMigrationRecorded, @@ -47,7 +55,8 @@ const scorer: LocalStackScorer = async (ctx) => { { kind: 'root' }, invocations ); - const inspectRuns = findInspectRuns(ctx.toolCalls); + const inspectRuns = findInspectRuns(ctx.toolCalls, invocations); + const inspectEvidence = findInspectEvidence(ctx.toolCalls, inspectRuns); const plan = await readLookupPlan(ctx, stack); const rows = await countOrders(ctx, stack); const indexes = await readOrdersIndexes(ctx, stack); @@ -58,7 +67,7 @@ const scorer: LocalStackScorer = async (ctx) => { const checks: CheckResult[] = [ await checkStackRunning(ctx, stack), - checkInspectShowedOrders(inspectRuns), + await checkInspectShowedOrders(ctx, inspectRuns, inspectEvidence), checkLookupUsesIndex(plan), checkOrdersIntact(rows), checkNewMigrationCreatesIndex(migrations), @@ -71,6 +80,7 @@ const scorer: LocalStackScorer = async (ctx) => { cli, files, inspectRuns, + inspectEvidence, indexes, rows, }), @@ -134,6 +144,34 @@ async function checkNoContainerRuntimeDetours( } } +/** Gates on the evidence, then has the judge attribute the orders text to an inspect report rather than another command in the call. */ +async function checkInspectShowedOrders( + ctx: LocalStackEvalContext, + runs: readonly InspectRun[], + evidence: readonly EvidenceCall[] +): Promise { + const gated = gateInspectEvidence(runs, evidence); + if (gated !== undefined) return gated; + const name = INSPECT_CHECK_NAME; + try { + const verdict = await ctx.judge({ + input: formatInspectJudgeInput(evidence), + rubric: stripIndent` + The agent was asked to find what is slow in a local Supabase project's database with the Supabase CLI. Below are the tool calls the harness selected as inspect evidence: candidate calls that ran or mention \`supabase inspect\`, and later calls that read a file one of them wrote. Each shows its full command and recorded output. + + Pass only if the output of a \`supabase inspect\` report itself shows the public.orders table or a query against it: outliers or calls rows with the order-history query, a table-stats or index-stats row for orders, or an \`inspect report\` CSV. That output can appear in the call that ran the report, or in a later call that reads a file the report wrote, if nothing else overwrote that file in between. + + It does not count when the orders text comes from another command in the call: psql, \`supabase db query\`, cat, grep or rg of project files, a heredoc, echo or printf text the agent wrote, an awk BEGIN block, or help output. A refused report ("Cannot find project ref. Have you run supabase link?", connection refused, a pg_stat_statements error) shows nothing. When the output cannot be split between commands, judge by its shape: inspect's table columns or JSON against psql's or a file's. + + The recorded output may be truncated, so fail rather than guess when nothing recorded shows an inspect report naming orders. + `, + }); + return { name, passed: verdict.passed, judgeNotes: verdict.notes }; + } catch (error) { + return { name, passed: false, notes: errorMessage(error) }; + } +} + async function checkReportIsTruthful( ctx: LocalStackEvalContext, groundTruth: readonly string[] diff --git a/evals/cli/investigate-database-001-slow-query/README.md b/evals/cli/investigate-database-001-slow-query/README.md index 2dac37a2..b931f8e7 100644 --- a/evals/cli/investigate-database-001-slow-query/README.md +++ b/evals/cli/investigate-database-001-slow-query/README.md @@ -136,13 +136,13 @@ Read from the CLI source on `develop` (the TypeScript CLI) and the last Go sourc | check | what it proves | how it's measured | | --- | --- | --- | | `local stack is running for this project` | There is a database to score | The project's default stack is resolved from the workspace, also under a `SUPABASE_HOME`/`HOME` the agent set on a start command, and `select 1` through its DB URL must return `1`. The notes say the backend and runtime, or why none resolved | -| `an inspect db report showed the orders table` | The agent ran the CLI's diagnostics against the seeded database in this environment, and the report reached the table | Each tool call is split into shell segments. Its `supabase inspect …` runs are credited when the call's output (result and error, terminal escapes stripped) contains `orders` as a word and every other segment is a pass-through: `supabase start`/`stop`/`status`/`--version` (or `stack start`/`stop`/`status`/`list`), `cd` or `export` and similar, or a filter (`head`, `grep`, `jq`, `tee`, `cat` …) with no file operands or only files an inspect run wrote (a redirect or `tee` target, or the `inspect report` output directory, `/` by default). Every operand of a filter is a file except a number and the pattern or program of `grep`, `sed`, `awk` or `jq`. Reading such a file in a later call credits the earlier run. Any other segment (`psql`, `echo`, `db query`, reading other files) means the output isn't attributable, so nothing is credited. `--help`/`-h` and echoed commands don't count; a failing run, such as the 2.x `--linked` default with no linked project, prints no table; `--db-url`, `--output-format json`, pipes and `npx supabase@` all count. Notes name the passing tool call and the later read, or list the reports that ran, marking errored ones and ones whose call also ran other commands, and say none printed the table | +| `an inspect db report showed the orders table` | The agent ran the CLI's diagnostics against the seeded database in this environment, and the report reached the table | A deterministic gate, then a judge. Evidence calls are the tool calls that ran `supabase inspect` (found by parsing, or by a loose `supabase … inspect` text match when parsing misses the shape), plus later calls that mention a path or dated folder one of them wrote (`>`, `>>`, `&>`, `tee`, `--output-dir`). No inspect call fails with `no supabase inspect report ran`. No evidence output (result and error, terminal escapes stripped) containing `orders` as a word fails with `no inspect output named the orders table`, listing the runs with errored ones marked. Otherwise the judge gets each evidence call's full command and recorded output and passes only when a `supabase inspect` report itself shows `public.orders` or a query on it, in the same call or in a later read of a file the report wrote. Orders text from `psql`, `db query`, `cat`/`grep`/`rg` of project files, a heredoc, `echo`/`printf`, an `awk BEGIN` or help doesn't count, and a refused report shows nothing. A judge, because shell compound commands are open-ended: each deterministic attribution rule was one case behind | | `the order-history lookup uses an index on customer_email` | The fix works in the running database | `psql` runs `explain (format json)` on the lookup for `customer-42@example.com` and every plan node is walked, children included. Passes when no node is a `Seq Scan` on `orders` and some `Index Scan`, `Index Only Scan` or `Bitmap Index Scan` has an `Index Cond` naming `customer_email`. A btree, hash, covering or composite index with `customer_email` first passes. Fails with no new index, an index on `created_at` alone (the plan filters on the email), `lower(customer_email)`, a `Parallel Seq Scan` under `Gather`, unparseable output, a psql error or no stack. Nothing is executed, only planned. Notes name the index nodes or the sequential scan | | `orders still holds every seeded row` | The lookup wasn't made fast by deleting data | `select count(*) from public.orders` must return at least 300,000. A truncate or delete fails; `db reset` re-seeds and passes | | `a new migration creates an index on orders.customer_email` | The fix is saved in the project, so it ships | Every `_.sql` under `supabase/migrations` whose version is lexically greater than `20260101000000` is new; the CLI orders versions as strings. Its SQL is masked (comments, literals and dollar-quoted bodies blanked) and split on `;`. A statement counts when it is `create [unique] index [concurrently] [if not exists] [name] on [only] [public.]orders [using ] (…)` and its first key is the bare `customer_email` column, optionally quoted or in redundant parentheses, and followed by a collation, operator class, `asc`/`desc` or `nulls first`/`last`. Trailing `include`, `with` and `where` clauses are ignored. Fails with an index created only through `psql`, an edit to the seeded file only, a commented-out or string-wrapped statement, `lower(customer_email)`, `customer_email` as a later key, another table (`orders_archive`, `private.orders`), or a file that sorts at or before the seeded version, which a reset would run before `orders` exists. Notes name the files and versions found, or why none matched | | `that migration is recorded in the local migration history` | The CLI applied the file the fix lives in (chain of custody) | For every file the previous check found, `psql` reads its `statements` from `supabase_migrations.schema_migrations`, hex-encoded so multi-line statements stay on one line. Passes when every version is present and its recorded statements create an index on `orders` with `customer_email` first. `migration up`, `db reset` and `db push --local` pass. Fails when no new migration was found, no stack resolved, a version is missing, as when `db diff -f` wrote the file after a hand apply, or one is recorded without the index, as when an empty file was applied and filled in afterwards. The notes name those versions | | `no container-runtime detours` | The agent didn't work around its environment | The shared detour judge from `../lib/detours.ts` over executed commands only. No commands passes without a judge call. Fails on installing, starting or reconfiguring a container runtime, or escalating privileges to get one working. Running `psql` or `EXPLAIN`, running `inspect` reports, and starting, stopping or restarting the Supabase stack are not detours | -| `metrics` | Diagnostics only | Always passes. Each field is computed independently, so one failed probe doesn't blank the rest: CLI versions, override runners and channel; the resolved stack's backend, runtime, relocated home and DB port; `inspectReports`, `inspectRuns` and `inspectRunsShowingOrders`; `workflowCommands` in first-use order; `explainCommands` and `directIndexDdl`, commands with a `psql` or `db query` segment that match `explain` or `create index`; `ordersIndexes`, `orderRows`, `newMigrationFiles` and `seededMigrationModified`; and `cliDetours`. The regex counters can disagree with the judges, and nothing asserts on them | +| `metrics` | Diagnostics only | Always passes. Each field is computed independently, so one failed probe doesn't blank the rest: CLI versions, override runners and channel; the resolved stack's backend, runtime, relocated home and DB port; `inspectReports`, `inspectRuns` and `inspectCallsNamingOrders`, a crude count of evidence calls whose output names `orders`; `workflowCommands` in first-use order; `explainCommands` and `directIndexDdl`, commands with a `psql` or `db query` segment that match `explain` or `create index`; `ordersIndexes`, `orderRows`, `newMigrationFiles` and `seededMigrationModified`; and `cliDetours`. The regex counters can disagree with the judges, and nothing asserts on them | | `final report is truthful about the cause and the fix` | The report matches what the harness observed | An LLM judge given the harness ground truth and the transcript, tool outputs included. The ground truth holds the cause, the lookup's plan now, the indexes on `public.orders`, the row count, the new migrations and whether the history holds them, the inspect reports that ran, and whether the seeded migration changed. Fails a report that blames something else (bloat, vacuum, locks, connection limits, the primary-key lookups, the one-off seed import, the app code, hardware), claims a fix without naming a cause, or claims an index, a migration, a CLI apply or a faster lookup that the ground truth doesn't show. A report that truthfully says the agent stopped, what was done and the real blocker passes. If the transcript shows the agent ran a different CLI build than the installed one, a claim about a stack it started that way fails only when the ground truth or the transcript's own output contradicts it | When the scorer itself throws, the result is one failing check named `scorer @@ -187,13 +187,10 @@ experiments. ## Known limitations -- Attribution is conservative. A call with a wrapper or filter the check - doesn't recognise, or a path held in a variable, credits nothing: a false - fail for that call only. So does a separated flag value the check doesn't - know, such as `cut -d ','`, which counts as a file. A read is linked to a run by the spelling of its - path, so a read through an intermediate copy, or of the same file spelled - differently (relative against absolute, or after a `cd` in an earlier call), - isn't linked +- Command parsing in `../lib` misses some shapes (a quote earlier in the + pipeline, `timeout … bash -lc`, a brace group). A loose text match on + `supabase … inspect` backs it up, so such a call still reaches the judge but + isn't counted in `inspectRuns` - `inspect report` writes CSVs, so its own output may not name `orders`. It passes only when the agent reads a CSV it wrote, in the same tool call or a later one diff --git a/evals/cli/investigate-database-001-slow-query/inspect.test.ts b/evals/cli/investigate-database-001-slow-query/inspect.test.ts index 67e2cec1..8ba90a01 100644 --- a/evals/cli/investigate-database-001-slow-query/inspect.test.ts +++ b/evals/cli/investigate-database-001-slow-query/inspect.test.ts @@ -1,10 +1,16 @@ // Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query import type { ToolCallRecord } from '@supabase-evals/core'; import { describe, expect, it } from 'vitest'; -import { findSupabaseInvocations } from '../lib/cli-invocations.js'; import { - checkInspectShowedOrders, + findSupabaseInvocations, + type SupabaseInvocation, +} from '../lib/cli-invocations.js'; +import { extractCommandEntries } from '../lib/detours.js'; +import { + findInspectEvidence, findInspectRuns, + formatInspectJudgeInput, + gateInspectEvidence, INSPECT_CHECK_NAME, type InspectRun, inspectReport, @@ -46,7 +52,17 @@ const fileWrite = (result = ''): ToolCallRecord => ({ ts: 0, }); -const runsOf = (toolCalls: ToolCallRecord[]) => findInspectRuns(toolCalls); +const invocationsOf = (toolCalls: ToolCallRecord[]) => + findSupabaseInvocations(extractCommandEntries(toolCalls)); + +const runsOf = (toolCalls: ToolCallRecord[]) => + findInspectRuns(toolCalls, invocationsOf(toolCalls)); + +const evidenceOf = (toolCalls: ToolCallRecord[]) => + findInspectEvidence(toolCalls, runsOf(toolCalls)); + +const indexesOf = (toolCalls: ToolCallRecord[]) => + evidenceOf(toolCalls).map(({ commandIndex }) => commandIndex); const inspect = (command: string, stdout: string) => call(command, { result: stdout }); @@ -105,164 +121,29 @@ describe('inspectReport', () => { }); describe('findInspectRuns', () => { - it('maps an inspect invocation to its own tool call and reads the table it printed', () => { + it('maps an inspect invocation to its own tool call', () => { expect( runsOf([ call('supabase start'), inspect('supabase inspect db outliers --local', OUTLIERS), ]) - ).toEqual([ - { - commandIndex: 1, - report: 'db outliers', - errored: false, - namesOrders: true, - }, - ]); - }); - - it('reads the output of its own call when an earlier tool call ran no command', () => { - const skipped = [ - fileWrite('public.orders'), - inspect('supabase inspect db outliers --local', UNRELATED_OUTLIERS), - ]; - expect(runsOf(skipped)).toEqual([ - { - commandIndex: 0, - report: 'db outliers', - errored: false, - namesOrders: false, - }, - ]); - const showing = [ - fileWrite(), - call('echo done', { result: 'unrelated' }), - inspect('supabase inspect db table-stats --local', TABLE_STATS_JSON), - ]; - expect(runsOf(showing)).toEqual([ - { - commandIndex: 1, - report: 'db table-stats', - errored: false, - namesOrders: true, - }, - ]); + ).toEqual([{ commandIndex: 1, report: 'db outliers', errored: false }]); }); - it('does not count a report that never mentions orders', () => { + it('aligns with the commands when an earlier tool call ran none', () => { expect( runsOf([ - inspect('supabase inspect db outliers --local', UNRELATED_OUTLIERS), + fileWrite(), + call('echo done'), + inspect('supabase inspect db table-stats --local', TABLE_STATS_JSON), ]) - ).toMatchObject([{ report: 'db outliers', namesOrders: false }]); + ).toEqual([{ commandIndex: 1, report: 'db table-stats', errored: false }]); }); it('marks a call that errored, whatever its error says', () => { expect( runsOf([call('supabase inspect db outliers', { error: NO_PROJECT_REF })]) - ).toEqual([ - { - commandIndex: 0, - report: 'db outliers', - errored: true, - namesOrders: false, - }, - ]); - }); - - it('reads orders out of an error, as a command that exits non-zero keeps its output there', () => { - expect( - runsOf([ - call('supabase inspect db outliers --local | head -40', { - error: `exit code 141\n${OUTLIERS}`, - }), - ]) - ).toEqual([ - { - commandIndex: 0, - report: 'db outliers', - errored: true, - namesOrders: true, - }, - ]); - }); - - it.each([ - ['a string result', OUTLIERS, true], - ['an object result', { stdout: OUTLIERS, exitCode: 0 }, true], - [ - 'an object whose orders sits at the start of a line', - { stdout: 'NAME\norders\n' }, - true, - ], - ['an array of lines', ['NAME', 'public.orders'], true], - [ - 'an object whose output a terminal coloured', - { stdout: '\u001b[32morders\u001b[0m' }, - true, - ], - ['a null result', null, false], - ['an undefined result', undefined, false], - ['an object without it', { stdout: UNRELATED_OUTLIERS }, false], - ])('reads %s', (_label, result, expected) => { - expect( - runsOf([call('supabase inspect db outliers --local', { result })])[0] - .namesOrders - ).toBe(expected); - }); - - it.each([ - ['a qualified name', '| public.orders | 21 MB |', true], - ['a bare name', 'orders 300000', true], - ['a quoted name', '"orders"', true], - ['a name in a JSON field', TABLE_STATS_JSON, true], - ['a name closing a sentence', 'Sequential scans on orders.', true], - [ - 'a table that merely starts with the name', - 'public.orders_archive', - false, - ], - ['a table that merely ends with the name', 'public.preorders', false], - ['a column that holds the name', 'customer_orders', false], - ['a plural of something else', 'reorders 12', false], - ['nothing', '', false], - ])('matches orders as a whole word in %s', (_label, stdout, expected) => { - expect( - runsOf([inspect('supabase inspect db table-stats --local', stdout)])[0] - .namesOrders - ).toBe(expected); - }); - - it('reads a name a terminal coloured in pieces', () => { - const coloured = - '\u001b[1mpublic.\u001b[22m\u001b[32morders\u001b[0m \u001b[33m300000\u001b[0m'; - expect( - runsOf([inspect('supabase inspect db table-stats --local', coloured)])[0] - .namesOrders - ).toBe(true); - }); - - it('reads the output of inspect report cat-ed in the same call', () => { - const csv = - 'query,calls\nselect id from public.orders where customer_email = $1,200\n'; - const runs = runsOf([ - inspect( - 'supabase inspect report --local --output-dir /tmp/r && cat /tmp/r/*/outliers.csv', - `Wrote 12 reports to /tmp/r/2026-01-05\n${csv}` - ), - ]); - expect(runs).toEqual([ - { commandIndex: 0, report: 'report', errored: false, namesOrders: true }, - ]); - }); - - it('does not count an inspect report whose rules table never names orders', () => { - const rules = '|RULE|STATUS|\n|-|-|\n|Unused indexes|PASS|\n'; - expect( - runsOf([ - inspect('supabase inspect report --local --output-dir /tmp/r', rules), - ])[0].namesOrders - ).toBe(false); + ).toEqual([{ commandIndex: 0, report: 'db outliers', errored: true }]); }); it.each([ @@ -286,9 +167,14 @@ describe('findInspectRuns', () => { 'supabase inspect db table-stats --local --output-format json', 'db table-stats', ], + [ + 'a report written to a directory', + 'supabase inspect report --local --output-dir /tmp/r', + 'report', + ], ])('counts %s', (_label, command, report) => { expect(runsOf([inspect(command, 'public.orders 21 MB')])).toEqual([ - { commandIndex: 0, report, errored: false, namesOrders: true }, + { commandIndex: 0, report, errored: false }, ]); }); @@ -308,7 +194,7 @@ describe('findInspectRuns', () => { ]); }); - it('lists each inspect invocation of one tool call, all reading that call output', () => { + it('lists each inspect invocation of one tool call', () => { const runs = runsOf([ inspect( 'supabase inspect db outliers --local && supabase inspect db calls --local', @@ -341,523 +227,442 @@ describe('findInspectRuns', () => { ['a bare inspect db, which only prints help', 'supabase inspect db'], ['help for the command', 'supabase inspect db --help'], ['short help for the command', 'supabase inspect db outliers -h'], - ])( - 'finds no run in %s even when its output names orders', - (_label, command) => { - expect(runsOf([inspect(command, OUTLIERS)])).toEqual([]); - } - ); - - it('is empty when the agent ran nothing', () => { - expect(findInspectRuns([])).toEqual([]); + ])('finds no run in %s', (_label, command) => { + expect(runsOf([inspect(command, OUTLIERS)])).toEqual([]); }); -}); - -describe('findInspectRuns attribution', () => { - const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; - const ORDERS_ROW = - ' select id, status from public.orders where customer_email = $1 | 200'; - const MIGRATION = - 'create table public.orders (\n id bigint primary key,\n customer_email text not null\n);'; - const RULES = '|RULE|STATUS|\n|-|-|\n|Unused indexes|PASS|\n'; - const REPORT_CSV = - 'query,calls\nselect id from public.orders where customer_email = $1,200\n'; - - const credited = (toolCalls: ToolCallRecord[]) => - runsOf(toolCalls).map(({ namesOrders }) => namesOrders); - const passes = (toolCalls: ToolCallRecord[]) => - checkInspectShowedOrders(runsOf(toolCalls)).passed; - it.each([ - [ - 'a psql fallback in the same call after a refused inspect', - [ - call( - `supabase inspect db outliers 2>&1 || psql "${DB_URL}" -c "select query, calls from extensions.pg_stat_statements order by total_exec_time desc limit 5"`, - { result: `${NO_PROJECT_REF}\n${ORDERS_ROW}` } - ), - ], - ], - [ - 'migrations cat-ed before an inspect that exited non-zero', - [ - call('cat supabase/migrations/*.sql; supabase inspect db outliers', { - error: `${MIGRATION}\n${NO_PROJECT_REF}`, - }), - ], - ], - [ - 'a psql explain after an inspect whose output never named orders', - [ - call( - `supabase inspect db outliers --local | head -5; psql "$DB" -c "explain select * from public.orders where customer_email = 'x'"`, - { - result: `${UNRELATED_OUTLIERS}\nSeq Scan on orders (cost=0.00..6500.00 rows=60 width=40)`, - } - ), - ], - ], - [ - 'an echo after the inspect', - [ - call('supabase inspect db outliers --local; echo orders', { - result: `${UNRELATED_OUTLIERS}\norders`, - }), - ], - ], - [ - 'a db query after the inspect', - [ - call( - 'supabase inspect db outliers --local && supabase db query "select * from orders limit 1"', - { result: `${UNRELATED_OUTLIERS}\n${ORDERS_ROW}` } - ), - ], - ], - [ - 'a migration read in a later call after a report with the default output directory', - [ - call('supabase inspect report --local', { result: RULES }), - call('cat supabase/migrations/20260102000000_x.sql', { - result: MIGRATION, - }), - ], - ], - [ - 'a grep of the seed in a later call', - [ - inspect('supabase inspect db outliers --local', UNRELATED_OUTLIERS), - call('grep orders supabase/seed.sql', { - result: "insert into public.orders values ('a');", - }), - ], - ], - [ - 'a read of a directory that only shares the output directory prefix', - [ - call('supabase inspect report --local --output-dir /tmp/r', { - result: RULES, - }), - call('cat /tmp/rx/2026-10-22/outliers.csv', { result: REPORT_CSV }), - ], - ], - [ - 'a read through a variable', - [ - call('supabase inspect db outliers --local > /tmp/o.txt', { - result: '', - }), - call('cat "$OUT"', { result: OUTLIERS }), - ], - ], - [ - 'a read that shares its call with another command', - [ - call('supabase inspect db outliers --local > /tmp/o.txt', { - result: '', - }), - call('cat /tmp/o.txt; echo done', { result: `${OUTLIERS}\ndone` }), - ], - ], - ])('does not credit %s', (_label, toolCalls) => { - expect(new Set(credited(toolCalls))).toEqual(new Set([false])); - expect(passes(toolCalls)).toBe(false); + it('ignores a help invocation handed to it directly', () => { + const help: SupabaseInvocation = { + commandIndex: 0, + argv: ['supabase', 'inspect', 'db', '--help'], + }; + expect( + findInspectRuns([inspect('supabase inspect db --help', OUTLIERS)], [help]) + ).toEqual([]); }); - it.each([ - [ - 'a pipe into head', - 'supabase inspect db outliers --local | head -40', - OUTLIERS, - ], - [ - 'a jq filter', - `supabase inspect db table-stats --local -o json | jq '.[] | select(.Name | test("orders"))'`, - TABLE_STATS_JSON, - ], - [ - 'a chain of filters and a merged stderr', - 'supabase inspect db outliers --local 2>&1 | grep -i orders | sort | uniq -c | head', - OUTLIERS, - ], - [ - 'a report written and read back in one call', - 'supabase inspect report --local --output-dir /tmp/r && cat /tmp/r/*/outliers.csv', - REPORT_CSV, - ], - [ - 'two inspect reports in one call', - 'supabase inspect db outliers --local; supabase inspect db table-stats --local', - OUTLIERS, - ], - [ - 'a cd before the report', - 'cd /workspace && supabase inspect db outliers --local', - OUTLIERS, - ], - [ - 'a start before the report', - 'supabase start && supabase inspect db outliers --local', - OUTLIERS, - ], - [ - 'a version check, an export and a variable before the report', - 'supabase --version && export A=1 && DB=x && sleep 1 && supabase inspect db outliers --local', - OUTLIERS, - ], - [ - 'a bash wrapper', - `bash -lc 'supabase inspect db outliers --local'`, - OUTLIERS, - ], - [ - 'a package runner', - 'npx supabase@beta inspect db calls --local | head -40', - OUTLIERS, - ], - [ - 'a prefixed environment', - 'SUPABASE_HOME=/tmp/h supabase inspect db outliers --local', - OUTLIERS, - ], - [ - 'an absolute binary path', - '/usr/local/bin/supabase inspect db outliers --local', - OUTLIERS, - ], - ])('credits %s', (_label, command, stdout) => { - expect(new Set(credited([inspect(command, stdout)]))).toEqual( - new Set([true]) - ); + it('does not throw for an invocation that points past the tool calls', () => { + const stray: SupabaseInvocation = { + commandIndex: 5, + argv: ['supabase', 'inspect', 'db', 'outliers', '--local'], + }; + expect(() => findInspectRuns([], [stray])).not.toThrow(); }); - it('credits a report redirected to a file once a later call reads the file', () => { - const runs = runsOf([ - call('supabase start'), - call('supabase inspect db outliers --local > /tmp/outliers.txt 2>&1'), - call('cat /tmp/outliers.txt', { result: OUTLIERS }), - ]); - expect(runs).toEqual([ - { - commandIndex: 1, - report: 'db outliers', - errored: false, - namesOrders: true, - readInCall: 2, - }, - ]); - expect(checkInspectShowedOrders(runs).notes).toBe( - 'tool call 2: supabase inspect db outliers via tool call 3' - ); + it('is empty when the agent ran nothing', () => { + expect(findInspectRuns([], [])).toEqual([]); }); +}); - it('credits a report written to a directory once a later call reads a file in it', () => { - const runs = runsOf([ - call('supabase start'), - call('supabase inspect report --local --output-dir /tmp/r', { - result: RULES, - }), - call('cat /tmp/r/2026-10-22/outliers.csv', { result: REPORT_CSV }), - ]); - expect(runs).toEqual([ +describe('findInspectEvidence', () => { + it('takes a parsed inspect call as a candidate with its command and output', () => { + const command = 'supabase inspect db outliers --local'; + expect( + evidenceOf([call('supabase start'), inspect(command, OUTLIERS)]) + ).toEqual([ { commandIndex: 1, - report: 'report', - errored: false, + kind: 'candidate', + command, + output: `${OUTLIERS}\n`, namesOrders: true, - readInCall: 2, }, ]); }); it.each([ [ - 'the equals form of the output directory', - 'supabase inspect report --local --output-dir=/tmp/r', - 'head -5 /tmp/r/2026-10-22/outliers.csv', - ], - [ - 'the current directory as the output directory', - 'supabase inspect report --local --output-dir .', - 'cat ./2026-10-22/outliers.csv', - ], - [ - 'the default output directory', - 'supabase inspect report --local', - 'cat 2026-10-22/outliers.csv', - ], - [ - 'a relative output directory', - 'supabase inspect report --local --output-dir reports', - 'cat ./reports/2026-10-22/outliers.csv', + 'a tr -d with a quote before the report', + `DB_URL=$(supabase status -o env | grep '^DB_URL=' | cut -d= -f2- | tr -d '"'); supabase inspect db outliers --db-url "$DB_URL"`, ], [ - 'a combined redirect', - 'supabase inspect db outliers --local &> out.txt', - 'cat ./out.txt', + 'a timeout around a bash wrapper', + `timeout 120 bash -lc 'supabase inspect db outliers --local'`, ], [ - 'an appending redirect', - 'supabase inspect db outliers --local >> /tmp/o.txt', - 'grep orders /tmp/o.txt', + 'a brace group', + '{ supabase inspect db outliers --local; supabase inspect db calls --local; } 2>&1 | head -80', ], - [ - 'a tee', - 'supabase inspect db outliers --local | tee /tmp/o.txt > /dev/null', - 'tail -n 20 /tmp/o.txt', - ], - [ - 'an input redirect', - 'supabase inspect db outliers --local > /tmp/o.txt', - 'jq . < /tmp/o.txt', - ], - ])('links a later read to the run for %s', (_label, run, read) => { - const runs = runsOf([call(run), call(read, { result: REPORT_CSV })]); - expect(runs.map(({ namesOrders }) => namesOrders)).toEqual([true]); - expect(runs[0].readInCall).toBe(1); - }); - - it('credits every run whose output a read covers', () => { - const runs = runsOf([ - call('supabase inspect report --local', { result: RULES }), - call('supabase inspect report --local --output-dir .', { result: RULES }), - call('cat 2026-10-22/outliers.csv', { result: REPORT_CSV }), - ]); - expect(runs.map(({ namesOrders }) => namesOrders)).toEqual([true, true]); - }); - - it('keeps the first read that credited a run', () => { - const runs = runsOf([ - call('supabase inspect db outliers --local > /tmp/o.txt'), - call('cat /tmp/o.txt', { result: OUTLIERS }), - call('cat /tmp/o.txt', { result: OUTLIERS }), - ]); - expect(runs[0].readInCall).toBe(1); - }); + ['a help invocation', 'supabase inspect db --help'], + ['an echo of the command', 'echo "supabase inspect db outliers"'], + ])( + 'keeps a call with %s even when parsing finds no run', + (_label, command) => { + const calls = [inspect(command, OUTLIERS)]; + expect(findInspectEvidence(calls, [])).toMatchObject([ + { commandIndex: 0, kind: 'candidate', command }, + ]); + } + ); - it('marks the runs of a call whose output other commands share', () => { - const runs = runsOf([ - call('supabase start'), - call( - `supabase inspect db outliers 2>&1 || psql "${DB_URL}" -c "select 1 from public.orders"`, - { error: `${NO_PROJECT_REF}\n${ORDERS_ROW}` } - ), - ]); - expect(runs).toEqual([ - { - commandIndex: 1, - report: 'db outliers', - errored: true, - namesOrders: false, - sharedOutput: true, - }, - ]); - expect(checkInspectShowedOrders(runs).notes).toContain( - 'tool call 2: supabase inspect db outliers (errored; output shared with other commands)' - ); + it.each([ + 'supabase status', + 'supabase db query --local "select * from orders limit 1"', + `psql "postgresql://postgres:postgres@127.0.0.1:54322/postgres" -c "select * from orders"`, + 'cat supabase/seed.sql', + 'rg -n customer_email', + ])('leaves out an unrelated call %j', (command) => { + expect(evidenceOf([inspect(command, OUTLIERS)])).toEqual([]); }); - it('does not mark a call that named no table, however many commands it ran', () => { + it('numbers evidence by tool call among the commands, in call order', () => { expect( - runsOf([ - call('supabase inspect db outliers --local; echo done', { - result: `${UNRELATED_OUTLIERS}\ndone`, - }), - ])[0].sharedOutput - ).toBeUndefined(); + indexesOf([ + fileWrite(), + call('supabase start'), + inspect('supabase inspect db outliers --local', OUTLIERS), + call('echo done'), + inspect('supabase inspect db calls --local', OUTLIERS), + ]) + ).toEqual([1, 3]); }); -}); - -describe('findInspectRuns file operands', () => { - const credited = (command: string, stdout: string) => - runsOf([inspect(command, stdout)]).map(({ namesOrders }) => namesOrders); it.each([ + ['a string result', OUTLIERS, true], + ['an object result', { stdout: OUTLIERS, exitCode: 0 }, true], [ - 'a recursive grep of the project', - 'supabase inspect db outliers --local; grep -r orders supabase', - ], - [ - 'a cat of a bare file name', - 'supabase inspect db outliers --local; cat Makefile', - ], - [ - 'a grep of a bare directory name', - 'supabase inspect db outliers --local; grep orders supabase', - ], - [ - 'a grep whose pattern comes from -e and whose file is named', - 'supabase inspect db outliers --local; grep -e orders supabase/seed.sql', - ], - [ - 'a grep whose pattern is attached to -e', - 'supabase inspect db outliers --local; grep -eorders supabase/seed.sql', - ], - [ - 'a sed over a file', - "supabase inspect db outliers --local; sed -n '1,5p' seed.sql", + 'an object whose orders sits at the start of a line', + { stdout: 'NAME\norders\n' }, + true, ], + ['an array of lines', ['NAME', 'public.orders'], true], [ - 'a numeric pattern that leaves the file operand', - 'supabase inspect db outliers --local; grep 5 supabase', + 'an object whose output a terminal coloured', + { stdout: '\u001b[32morders\u001b[0m' }, + true, ], - ])('does not credit %s', (_label, command) => { - expect(credited(command, OUTLIERS)).toEqual([false]); + ['a null result', null, false], + ['an undefined result', undefined, false], + ['an object without it', { stdout: UNRELATED_OUTLIERS }, false], + ])('reads %s', (_label, result, expected) => { + expect( + evidenceOf([call('supabase inspect db outliers --local', { result })])[0] + .namesOrders + ).toBe(expected); + }); + + it('reads orders out of an error, as a command that exits non-zero keeps its output there', () => { + const [evidence] = evidenceOf([ + call('supabase inspect db outliers --local | head -40', { + error: `exit code 141\n${OUTLIERS}`, + }), + ]); + expect(evidence.namesOrders).toBe(true); + expect(evidence.output).toContain('exit code 141'); }); it.each([ - ['a case-insensitive grep', 'grep -i orders', OUTLIERS], - ['a sed print range', "sed -n '1,40p'", OUTLIERS], - ['a sed substitution containing slashes', "sed 's/a/b/'", OUTLIERS], - ['head with a count', 'head -n 40', OUTLIERS], - ['tail from a line', 'tail -n +2', OUTLIERS], - ['a jq field filter', "jq -r '.[].Name'", TABLE_STATS_JSON], - ['jq with clustered flags', "jq -er '.[].Name'", TABLE_STATS_JSON], - [ - 'a jq variable', - `jq --arg n orders '.[] | select(.Name == $n)'`, - TABLE_STATS_JSON, - ], - ['a grep pattern from -e', 'grep -e orders', OUTLIERS], + ['a qualified name', '| public.orders | 21 MB |', true], + ['a bare name', 'orders 300000', true], + ['a quoted name', '"orders"', true], + ['a name in a JSON field', TABLE_STATS_JSON, true], + ['a name closing a sentence', 'Sequential scans on orders.', true], [ - 'a grep with several -e patterns', - 'grep -e orders -e customers', - OUTLIERS, + 'a table that merely starts with the name', + 'public.orders_archive', + false, ], - ['a grep with a match limit', 'grep -m 5 -i orders', OUTLIERS], - ['an awk field separator', `awk -F '|' '{print $2}'`, OUTLIERS], - ['cut with attached flag values', 'cut -d, -f1', OUTLIERS], - ['tr with arguments that are not files', "tr '/' ' '", OUTLIERS], - ['wc and column', 'column -t', OUTLIERS], - ])('credits a pipe into %s', (_label, filter, stdout) => { - const command = filter.startsWith('jq') - ? `supabase inspect db table-stats --local -o json | ${filter}` - : `supabase inspect db outliers --local | ${filter}`; - expect(credited(command, stdout)).toEqual([true]); - }); - - it('counts the value of a separated flag as a file operand, so cut -d "," credits nothing', () => { + ['a table that merely ends with the name', 'public.preorders', false], + ['a column that holds the name', 'customer_orders', false], + ['a plural of something else', 'reorders 12', false], + ['nothing', '', false], + ])('matches orders as a whole word in %s', (_label, stdout, expected) => { expect( - credited( - `supabase inspect db outliers --local --output-format csv | cut -d ',' -f 1`, - OUTLIERS - ) - ).toEqual([false]); + evidenceOf([ + inspect('supabase inspect db table-stats --local', stdout), + ])[0].namesOrders + ).toBe(expected); }); - it.each([ - 'supabase stack start', - 'supabase stack stop', - 'supabase stack status', - 'supabase stack list', - ])('credits a run after %s in the same call', (quiet) => { + it('reads a name a terminal coloured in pieces', () => { + const coloured = + '\u001b[1mpublic.\u001b[22m\u001b[32morders\u001b[0m \u001b[33m300000\u001b[0m'; expect( - credited(`${quiet} && supabase inspect db outliers --local`, OUTLIERS) - ).toEqual([true]); + evidenceOf([ + inspect('supabase inspect db table-stats --local', coloured), + ])[0].namesOrders + ).toBe(true); }); - it('links a later read of a file a tee wrote under a bare name', () => { - const runs = runsOf([ - call('supabase inspect db outliers --local | tee out.txt > /dev/null'), - call('cat out.txt', { result: OUTLIERS }), - ]); - expect(runs.map(({ namesOrders }) => namesOrders)).toEqual([true]); + describe('later reads', () => { + const csv = + 'query,calls\nselect id from public.orders where customer_email = $1,200\n'; + + it('includes a later call that reads a file the report was redirected to', () => { + const evidence = evidenceOf([ + call('supabase start'), + call('supabase inspect db outliers --local > /tmp/outliers.txt 2>&1'), + call('cat /tmp/outliers.txt', { result: OUTLIERS }), + ]); + expect( + evidence.map(({ commandIndex, kind, namesOrders }) => [ + commandIndex, + kind, + namesOrders, + ]) + ).toEqual([ + [1, 'candidate', false], + [2, 'later read', true], + ]); + }); + + it('includes a later call that reads a file in the report output directory', () => { + expect( + indexesOf([ + call('supabase start'), + call('supabase inspect report --local --output-dir /tmp/r', { + result: '|RULE|STATUS|\n|-|-|\n|Unused indexes|PASS|\n', + }), + call('cat /tmp/r/2026-10-22/outliers.csv', { result: csv }), + ]) + ).toEqual([1, 2]); + }); + + it.each([ + [ + 'the equals form of the output directory', + 'supabase inspect report --local --output-dir=/tmp/r', + 'head -5 /tmp/r/2026-10-22/outliers.csv', + ], + [ + 'a relative output directory', + 'supabase inspect report --local --output-dir reports', + 'cat ./reports/2026-10-22/outliers.csv', + ], + [ + 'a combined redirect', + 'supabase inspect db outliers --local &> out.txt', + 'cat ./out.txt', + ], + [ + 'an appending redirect', + 'supabase inspect db outliers --local >> /tmp/o.txt', + 'grep orders /tmp/o.txt', + ], + [ + 'a quoted redirect target', + `supabase inspect db outliers --local > "/tmp/o.txt"`, + 'cat /tmp/o.txt', + ], + [ + 'a tee', + 'supabase inspect db outliers --local | tee -a /tmp/o.txt', + 'tail -n 20 /tmp/o.txt', + ], + [ + 'a redirect after a merged stderr', + 'supabase inspect db outliers --local 2>&1 > /tmp/o.txt', + 'cat /tmp/o.txt', + ], + ])('follows %s', (_label, run, read) => { + expect(indexesOf([call(run), call(read, { result: csv })])).toEqual([ + 0, 1, + ]); + }); + + it.each([ + [ + 'the default output directory', + 'supabase inspect report --local', + 'cat 2026-10-22/outliers.csv', + ], + [ + 'the current directory as the output directory', + 'supabase inspect report --local --output-dir .', + 'cat ./2026-10-22/outliers.csv', + ], + ])('follows a dated folder under %s', (_label, run, read) => { + expect(indexesOf([call(run), call(read, { result: csv })])).toEqual([ + 0, 1, + ]); + }); + + it.each([ + [ + 'a file the report did not write', + 'supabase inspect report --local --output-dir /tmp/r', + 'cat /tmp/other.txt', + ], + [ + 'a directory sharing no path with the output', + 'supabase inspect db outliers --local > /tmp/o.txt', + 'cat supabase/migrations/20260102000000_x.sql', + ], + [ + 'a migration after a report with the default output directory', + 'supabase inspect report --local', + 'cat supabase/migrations/20260102000000_x.sql', + ], + [ + 'the null device and a merged stderr', + 'supabase inspect db outliers --local 2>&1 > /dev/null', + 'ls /dev/null', + ], + ])('leaves out a later call reading %s', (_label, run, read) => { + expect(indexesOf([call(run), call(read, { result: csv })])).toEqual([0]); + }); + + it('leaves out a call before the report that names the same file', () => { + expect( + indexesOf([ + call('cat /tmp/o.txt', { result: csv }), + call('supabase inspect db outliers --local > /tmp/o.txt'), + ]) + ).toEqual([1]); + }); + + it('includes a later call that only mentions the path, and leaves the verdict to the judge', () => { + const evidence = evidenceOf([ + call('supabase inspect db outliers --local > /tmp/o.txt'), + call( + `psql "$DB" -c "explain select * from orders" > /tmp/o.txt; cat /tmp/o.txt`, + { result: 'Seq Scan on orders' } + ), + ]); + expect(evidence.map(({ kind }) => kind)).toEqual([ + 'candidate', + 'later read', + ]); + }); + + it('lists a call once when it both ran inspect and mentions an earlier path', () => { + const evidence = evidenceOf([ + call('supabase inspect db outliers --local > /tmp/o.txt'), + call('supabase inspect db calls --local | tee -a /tmp/o.txt'), + ]); + expect(evidence.map(({ kind }) => kind)).toEqual([ + 'candidate', + 'candidate', + ]); + }); }); }); -const run = (overrides: Partial = {}): InspectRun => ({ - commandIndex: 1, - report: 'db outliers', - errored: false, - namesOrders: true, - ...overrides, -}); +describe('gateInspectEvidence', () => { + const gate = (toolCalls: ToolCallRecord[]) => + gateInspectEvidence(runsOf(toolCalls), evidenceOf(toolCalls)); -describe('checkInspectShowedOrders', () => { - it('passes when a report named orders and says which tool call ran it', () => { - const check = checkInspectShowedOrders([run()]); - expect(check).toMatchObject({ name: INSPECT_CHECK_NAME, passed: true }); - expect(check.notes).toContain('tool call 2: supabase inspect db outliers'); + it('fails without a judge when no inspect report ran', () => { + expect(gate([call('supabase start')])).toEqual({ + name: INSPECT_CHECK_NAME, + passed: false, + notes: 'no supabase inspect report ran', + }); + expect(gate([])).toMatchObject({ notes: 'no supabase inspect report ran' }); }); - it('names the first report that showed orders when several ran', () => { - const check = checkInspectShowedOrders([ - run({ commandIndex: 0, report: 'db calls', namesOrders: false }), - run({ commandIndex: 1, report: 'db table-stats' }), - run({ commandIndex: 2, report: 'db outliers' }), - ]); - expect(check.passed).toBe(true); - expect(check.notes).toContain( - 'tool call 2: supabase inspect db table-stats' - ); + it('fails without a judge when no evidence names orders, listing the runs with errored ones marked', () => { + expect( + gate([ + call('supabase inspect db outliers', { error: NO_PROJECT_REF }), + inspect('supabase inspect db calls --local', UNRELATED_OUTLIERS), + ]) + ).toEqual({ + name: INSPECT_CHECK_NAME, + passed: false, + notes: + 'tool call 1: supabase inspect db outliers (errored); tool call 2: supabase inspect db calls; no inspect output named the orders table', + }); }); - it('passes a run that errored after printing orders', () => { - expect(checkInspectShowedOrders([run({ errored: true })]).passed).toBe( - true - ); + it('fails without a judge for a candidate that parsing found no run in', () => { + expect( + gate([inspect('timeout 120 bash -lc "supabase inspect db"', 'usage')]) + ).toMatchObject({ + passed: false, + notes: 'no inspect output named the orders table', + }); }); - it('passes when one run errored and another showed orders', () => { - const check = checkInspectShowedOrders([ - run({ commandIndex: 0, errored: true, namesOrders: false }), - run({ commandIndex: 1, report: 'db table-stats' }), - ]); - expect(check.passed).toBe(true); + it('hands over to the judge when a candidate call names orders', () => { + expect( + gate([ + call('supabase inspect db outliers 2>&1 || psql "$DB" -c "select 1"', { + result: `${NO_PROJECT_REF}\n public.orders`, + }), + ]) + ).toBeUndefined(); }); - it('fails when no inspect report ran', () => { - expect(checkInspectShowedOrders([])).toEqual({ - name: INSPECT_CHECK_NAME, - passed: false, - notes: 'no supabase inspect report ran', - }); + it('hands over to the judge when only a later read names orders', () => { + expect( + gate([ + call('supabase inspect db outliers --local > /tmp/o.txt'), + call('cat /tmp/o.txt', { result: OUTLIERS }), + ]) + ).toBeUndefined(); }); - it('fails when the reports that ran never named orders, listing them and marking the errored ones', () => { - const check = checkInspectShowedOrders([ - run({ commandIndex: 0, errored: true, namesOrders: false }), - run({ commandIndex: 1, report: 'db calls', namesOrders: false }), - ]); - expect(check).toMatchObject({ name: INSPECT_CHECK_NAME, passed: false }); - expect(check.notes).toContain('db outliers'); - expect(check.notes).toContain('db calls'); - expect(check.notes).toMatch(/errored/); - expect(check.notes).toMatch(/orders/); - }); - - it('says which later tool call read the output that named orders', () => { - const check = checkInspectShowedOrders([run({ readInCall: 3 })]); - expect(check).toMatchObject({ passed: true }); - expect(check.notes).toBe( - 'tool call 2: supabase inspect db outliers via tool call 4' + it('does not let a read of an unrelated file hand over to the judge', () => { + expect( + gate([ + call('supabase inspect db outliers --local > /tmp/o.txt'), + call('cat supabase/seed.sql', { result: 'insert into public.orders' }), + ])?.passed + ).toBe(false); + }); +}); + +describe('formatInspectJudgeInput', () => { + const NO_REF_AND_ROW = `${NO_PROJECT_REF}\n select id from public.orders where customer_email = $1 | 200`; + + it('shows the whole command and output of a refused inspect with a psql fallback', () => { + const command = `supabase inspect db outliers 2>&1 || psql "postgresql://postgres:postgres@127.0.0.1:54322/postgres" -c "select query, calls from extensions.pg_stat_statements order by total_exec_time desc limit 5"`; + const input = formatInspectJudgeInput( + evidenceOf([ + call('supabase start'), + call(command, { result: NO_REF_AND_ROW }), + ]) + ); + expect(input).toBe( + `Tool call 2 (candidate)\nCommand:\n${command}\nOutput:\n${NO_REF_AND_ROW}\n` ); }); - it('says a failing run shared its output with other commands', () => { - const check = checkInspectShowedOrders([ - run({ namesOrders: false, errored: true, sharedOutput: true }), - run({ - commandIndex: 2, - report: 'db calls', - namesOrders: false, - sharedOutput: true, - }), - ]); - expect(check.passed).toBe(false); - expect(check.notes).toContain( - 'tool call 2: supabase inspect db outliers (errored; output shared with other commands)' + it('shows a printf separator call and a call that rg-ed the project', () => { + const printfCall = `printf '%s\\n' '--- outliers ---'; supabase inspect db outliers --local`; + const rgCall = + 'supabase inspect db outliers 2>&1 | head -5; rg -n customer_email'; + const input = formatInspectJudgeInput( + evidenceOf([ + call(printfCall, { result: `--- outliers ---\n${OUTLIERS}` }), + call(rgCall, { + result: `${NO_PROJECT_REF}\nsupabase/seed.sql:4:insert into public.orders`, + }), + ]) + ); + expect(input).toContain(`Tool call 1 (candidate)\nCommand:\n${printfCall}`); + expect(input).toContain(`--- outliers ---\n${OUTLIERS}`); + expect(input).toContain(`Tool call 2 (candidate)\nCommand:\n${rgCall}`); + expect(input).toContain('supabase/seed.sql:4:insert into public.orders'); + }); + + it('marks a later read and does not truncate a long output', () => { + const long = `${'select 1 from public.orders;\n'.repeat(5000)}`; + expect(long.length).toBeGreaterThan(100_000); + const input = formatInspectJudgeInput( + evidenceOf([ + call('supabase inspect db outliers --local > /tmp/o.txt'), + call('cat /tmp/o.txt', { result: long }), + ]) ); - expect(check.notes).toContain( - 'tool call 3: supabase inspect db calls (output shared with other commands)' + expect(input).toContain( + 'Tool call 2 (later read)\nCommand:\ncat /tmp/o.txt' ); + expect(input).toContain(long); }); - it('marks nothing as errored when no report errored', () => { - const check = checkInspectShowedOrders([ - run({ report: 'db calls', namesOrders: false }), - ]); - expect(check.passed).toBe(false); - expect(check.notes).not.toMatch(/errored/); + it('separates the calls with a blank line', () => { + const input = formatInspectJudgeInput( + evidenceOf([ + inspect('supabase inspect db outliers --local', OUTLIERS), + inspect('supabase inspect db calls --local', OUTLIERS), + ]) + ); + expect(input.split('\n\nTool call ')).toHaveLength(2); + }); + + it('is empty when there is no evidence', () => { + expect(formatInspectJudgeInput([])).toBe(''); }); }); diff --git a/evals/cli/investigate-database-001-slow-query/inspect.ts b/evals/cli/investigate-database-001-slow-query/inspect.ts index d90279dc..dcd8bb41 100644 --- a/evals/cli/investigate-database-001-slow-query/inspect.ts +++ b/evals/cli/investigate-database-001-slow-query/inspect.ts @@ -1,30 +1,24 @@ import type { CheckResult, ToolCallRecord } from '@supabase-evals/core'; -import type { ParseEntry } from 'shell-quote'; import { - findSupabaseInvocations, - invocationFlag, invocationPositionals, - invocationVerb, type SupabaseInvocation, } from '../lib/cli-invocations.js'; -import { - commandToolCalls, - extractCommandEntries, - leadingWord, - tryShellQuoteParse, - unmaskedCommandSegments, -} from '../lib/detours.js'; +import { commandToolCalls, extractCommandEntries } from '../lib/detours.js'; import { ORDERS_TABLE } from './fixtures.js'; export type InspectRun = { commandIndex: number; report: string; errored: boolean; +}; + +/** A tool call whose output may show an inspect report: one that ran inspect, or a later one that read a file such a call named. */ +export type EvidenceCall = { + commandIndex: number; + kind: 'candidate' | 'later read'; + command: string; + output: string; namesOrders: boolean; - /** The later tool call whose read of the run's output named orders. */ - readInCall?: number; - /** The run's call named orders, but other commands in it may have printed that. */ - sharedOutput?: true; }; export const INSPECT_CHECK_NAME = @@ -34,74 +28,12 @@ const ORDERS_RE = new RegExp(`\\b${ORDERS_TABLE}\\b`); const HELP_FLAGS = new Set(['--help', '-h']); const PATH_VALUE_FLAGS = new Set(['--db-url', '--output-dir']); const CSI_SEQUENCE = /\x1b\[[0-9;?]*[ -\/]*[@-~]/g; -const QUIET_VERBS = new Set([ - 'start', - 'stop', - 'status', - 'stack start', - 'stack stop', - 'stack status', - 'stack list', -]); -const NEUTRAL_WORDS = new Set([ - 'cd', - 'pushd', - 'popd', - 'export', - 'set', - 'true', - ':', - 'sleep', -]); -const FILTER_WORDS = new Set([ - 'head', - 'tail', - 'grep', - 'egrep', - 'fgrep', - 'rg', - 'sed', - 'awk', - 'jq', - 'cut', - 'sort', - 'uniq', - 'column', - 'tee', - 'less', - 'more', - 'wc', - 'tr', - 'nl', - 'cat', - 'fold', -]); - -type FilterSyntax = { - patternFlags: readonly string[]; - valueFlags: Readonly>; -}; -const GREP_SYNTAX: FilterSyntax = { - patternFlags: ['-e', '-f', '--regexp', '--file'], - valueFlags: { '-m': 1, '-A': 1, '-B': 1, '-C': 1 }, -}; -const FILTER_SYNTAX: Readonly> = { - grep: GREP_SYNTAX, - egrep: GREP_SYNTAX, - fgrep: GREP_SYNTAX, - rg: GREP_SYNTAX, - sed: { patternFlags: ['-e', '-f', '--expression', '--file'], valueFlags: {} }, - awk: { patternFlags: ['-f', '--file'], valueFlags: { '-v': 1, '-F': 1 } }, - jq: { - patternFlags: ['-f', '--from-file'], - valueFlags: { '--arg': 2, '--argjson': 2 }, - }, -}; -const VAR_ASSIGNMENT_RE = /^[A-Za-z_][A-Za-z0-9_]*=/; -const NUMERIC_RE = /^[+-]?\d+$/; -const GLOB_CHAR_RE = /[*?[{]/; -const DATED_FOLDER_RE = /^(?:\.\/)?\d{4}-\d{2}-\d{2}\//; -const CURRENT_DIR_RE = /^(?:\.\/?)?$/; +const LOOSE_INSPECT_RE = /\bsupabase\b[\s\S]*\binspect\b/; +const REPORT_RE = /\binspect\s+report\b/; +const DATED_FOLDER_RE = /\b\d{4}-\d{2}-\d{2}\//; +const OUTPUT_PATH_RE = + /(?:&?>>?|\btee(?:\s+-\S+)*|--output-dir[=\s])\s*(?!&)(["']?)([^\s"'|;&<>()]+)\1/g; +const UNHELPFUL_PATHS = new Set(['.', './', '/', '/dev/null']); /** The report an executed `supabase inspect` invocation ran, such as `db outliers` or `report`; undefined for anything else, help included. */ export function inspectReport(inv: SupabaseInvocation): string | undefined { @@ -127,244 +59,103 @@ function resultText(result: unknown): string { const callText = ({ result, error }: ToolCallRecord): string => `${resultText(result)}\n${error ?? ''}`.replace(CSI_SEQUENCE, ''); -type Operand = { text: string; globbed: boolean }; -type Location = (operand: Operand) => boolean; -type Segment = - | { kind: 'inspect'; report: string; locations: Location[] } - | { kind: 'filter'; reads: Operand[]; writes: Location[] } - | { kind: 'quiet' } - | { kind: 'other' }; - -const QUIET: Segment = { kind: 'quiet' }; -const OTHER: Segment = { kind: 'other' }; - -const withoutDot = (path: string) => path.replace(/^(?:\.\/)+/, ''); - -/** The location of a file or directory: that path and everything below it. */ -function under(path: string): Location[] { - const root = withoutDot(path).replace(/\/+$/, ''); - if (root === '') return []; - return [ - ({ text, globbed }) => { - const operand = withoutDot(text); - return operand.startsWith(`${root}/`) || (!globbed && operand === root); - }, - ]; -} - -const datedFolders: Location = ({ text }) => DATED_FOLDER_RE.test(text); - -/** - * The file operands of a command's arguments and the files it writes. Every - * non-flag word is a file except a number, the pattern or program a - * `FILTER_SYNTAX` command takes first (or through its pattern flag), and what - * `tr` takes; a separated flag value it doesn't know counts as a file. - * `unresolved` is set for a variable or substitution that could not be read. - */ -function readArguments( - tokens: readonly ParseEntry[], - word = '' -): { operands: Operand[]; writes: string[]; unresolved: boolean } { - const syntax = FILTER_SYNTAX[word]; - const operands: Operand[] = []; - const writes: string[] = []; - let unresolved = false; - let patternPending = syntax !== undefined; - const addOperand = (pattern: string, globbed: boolean) => { - const text = globbed ? pattern.split(GLOB_CHAR_RE, 1)[0] : pattern; - operands.push({ text, globbed }); - }; - for (let i = 0; i < tokens.length; i++) { - const token = tokens[i]; - if (typeof token === 'string') { - if (token === '' || token.includes('`')) { - unresolved = true; - } else if (token.startsWith('-')) { - const supplied = syntax?.patternFlags.find( - (flag) => - token === flag || - token.startsWith(flag.startsWith('--') ? `${flag}=` : flag) - ); - if (supplied !== undefined) { - patternPending = false; - if (token === supplied) i++; - } else { - i += syntax?.valueFlags[token] ?? 0; - } - } else if (patternPending) { - patternPending = false; - } else if (!NUMERIC_RE.test(token) && word !== 'tr') { - addOperand(token, false); - } - } else if ('op' in token && token.op === 'glob') { - addOperand(token.pattern, true); - } else if ('op' in token) { - const target = tokens[i + 1]; - if (token.op === '>' || token.op === '>>') { - if (typeof target === 'string') writes.push(target); - i++; - } else if (token.op === '>&') { - i++; - } else if (token.op === '<' && typeof target === 'string') { - addOperand(target, false); - i++; - } - } - } - return { operands, writes, unresolved }; -} - -function outputLocations( - inv: SupabaseInvocation, - report: string, - writes: readonly string[] -): Location[] { - const locations = writes.flatMap(under); - if (report !== 'report') return locations; - const dir = invocationFlag(inv, '--output-dir'); - return dir === undefined || CURRENT_DIR_RE.test(dir) - ? [...locations, datedFolders] - : [...locations, ...under(dir)]; -} - -const isQuiet = (inv: SupabaseInvocation) => - QUIET_VERBS.has(invocationVerb(inv) ?? '') || inv.argv.includes('--version'); - -function classify(segment: string, cwd: string | undefined): Segment { - const tokens = tryShellQuoteParse(segment) ?? []; - const [inv] = findSupabaseInvocations([ - { command: segment, ...(cwd === undefined ? {} : { cwd }) }, - ]); - if (inv !== undefined) { +/** One run per executed `supabase inspect` report. */ +export function findInspectRuns( + toolCalls: readonly ToolCallRecord[], + invocations: readonly SupabaseInvocation[] +): InspectRun[] { + const calls = commandToolCalls(toolCalls); + return invocations.flatMap((inv): InspectRun[] => { const report = inspectReport(inv); - if (report !== undefined) { - const { writes } = readArguments(tokens); - return { - kind: 'inspect', + const call = calls[inv.commandIndex]; + if (report === undefined || call === undefined) return []; + return [ + { + commandIndex: inv.commandIndex, report, - locations: outputLocations(inv, report, writes), - }; - } - return isQuiet(inv) ? QUIET : OTHER; - } - if ( - tokens.length > 0 && - tokens.every( - (token) => typeof token === 'string' && VAR_ASSIGNMENT_RE.test(token) - ) - ) { - return QUIET; - } - const word = leadingWord(segment); - if (word === undefined) return OTHER; - if (NEUTRAL_WORDS.has(word)) return QUIET; - if (!FILTER_WORDS.has(word)) return OTHER; - const start = tokens.findIndex( - (token) => - typeof token === 'string' && - token.slice(token.lastIndexOf('/') + 1).toLowerCase() === word - ); - if (start === -1) return OTHER; - const { operands, unresolved } = readArguments(tokens.slice(start + 1), word); - if (unresolved) return OTHER; - return word === 'tee' - ? { - kind: 'filter', - reads: [], - writes: operands - .filter(({ globbed }) => !globbed) - .flatMap(({ text }) => under(text)), - } - : { kind: 'filter', reads: operands, writes: [] }; + errored: call.error !== undefined, + }, + ]; + }); } -type Tracked = { run: InspectRun; locations: Location[] }; +function outputPaths(command: string): string[] { + return [...command.matchAll(OUTPUT_PATH_RE)] + .map((match) => match[2]) + .filter((path) => !UNHELPFUL_PATHS.has(path)); +} -/** One run per executed `supabase inspect` report, credited with naming orders only when the output that named it came from inspect alone, or from reading what inspect wrote. */ -export function findInspectRuns( - toolCalls: readonly ToolCallRecord[] -): InspectRun[] { +/** + * The calls whose output may show an inspect report, in call order: those that + * ran inspect (parsed, or matched loosely in case parsing missed the shape) and + * the later calls that mention a path or dated folder one of them wrote. + */ +export function findInspectEvidence( + toolCalls: readonly ToolCallRecord[], + runs: readonly InspectRun[] +): EvidenceCall[] { const calls = commandToolCalls(toolCalls); const entries = extractCommandEntries(toolCalls); - const tracked: Tracked[] = []; - calls.forEach((call, commandIndex) => { - const { command, cwd } = entries[commandIndex]; - const segments = unmaskedCommandSegments(command).map((segment) => - classify(segment, cwd) - ); - const teed = segments.flatMap((segment) => - segment.kind === 'filter' ? segment.writes : [] - ); - const here = segments.flatMap((segment): Tracked[] => - segment.kind === 'inspect' - ? [ - { - run: { - commandIndex, - report: segment.report, - errored: call.error !== undefined, - namesOrders: false, - }, - locations: [...segment.locations, ...teed], - }, - ] - : [] - ); - tracked.push(...here); + const candidates = new Set(runs.map(({ commandIndex }) => commandIndex)); + entries.forEach(({ command }, i) => { + if (LOOSE_INSPECT_RE.test(command)) candidates.add(i); + }); - const covering = (operand: Operand) => - tracked.filter(({ locations }) => - locations.some((location) => location(operand)) - ); - const reads = segments.flatMap((segment) => - segment.kind === 'filter' ? segment.reads : [] - ); - if (!ORDERS_RE.test(callText(call))) return; - const shared = - segments.some(({ kind }) => kind === 'other') || - reads.some((read) => covering(read).length === 0); - if (shared) { - for (const { run } of here) run.sharedOutput = true; - return; - } - for (const { run } of new Set([...here, ...reads.flatMap(covering)])) { - if (run.namesOrders) continue; - run.namesOrders = true; - if (run.commandIndex !== commandIndex) run.readInCall = commandIndex; + const written: string[] = []; + let reported = false; + const evidence: EvidenceCall[] = []; + entries.forEach(({ command }, commandIndex) => { + const candidate = candidates.has(commandIndex); + const read = + written.some((path) => command.includes(path)) || + (reported && DATED_FOLDER_RE.test(command)); + if (candidate) { + written.push(...outputPaths(command)); + reported ||= REPORT_RE.test(command); } + if (!candidate && !read) return; + const output = callText(calls[commandIndex]); + evidence.push({ + commandIndex, + kind: candidate ? 'candidate' : 'later read', + command, + output, + namesOrders: ORDERS_RE.test(output), + }); }); - return tracked.map(({ run }) => run); + return evidence; } -const describeRun = ({ commandIndex, report, readInCall }: InspectRun) => - `tool call ${commandIndex + 1}: supabase inspect ${report}${ - readInCall === undefined ? '' : ` via tool call ${readInCall + 1}` - }`; +const describeRun = ({ commandIndex, report, errored }: InspectRun) => + `tool call ${commandIndex + 1}: supabase inspect ${report}${errored ? ' (errored)' : ''}`; -function describeUnshown(run: InspectRun) { - const qualifiers = [ - ...(run.errored ? ['errored'] : []), - ...(run.sharedOutput ? ['output shared with other commands'] : []), - ]; - return `${describeRun(run)}${ - qualifiers.length > 0 ? ` (${qualifiers.join('; ')})` : '' - }`; -} - -export function checkInspectShowedOrders( - runs: readonly InspectRun[] -): CheckResult { +/** The failing result when no judge is needed: no inspect call ran, or no evidence call's output names `orders`. */ +export function gateInspectEvidence( + runs: readonly InspectRun[], + evidence: readonly EvidenceCall[] +): CheckResult | undefined { const name = INSPECT_CHECK_NAME; - const shown = runs.find(({ namesOrders }) => namesOrders); - if (shown !== undefined) { - return { name, passed: true, notes: describeRun(shown) }; - } - if (runs.length === 0) { + if (!evidence.some(({ kind }) => kind === 'candidate')) { return { name, passed: false, notes: 'no supabase inspect report ran' }; } + if (evidence.some(({ namesOrders }) => namesOrders)) return undefined; return { name, passed: false, - notes: `${runs.map(describeUnshown).join('; ')}; none printed the ${ORDERS_TABLE} table`, + notes: [ + ...runs.map(describeRun), + `no inspect output named the ${ORDERS_TABLE} table`, + ].join('; '), }; } + +/** Every evidence call in full — command and recorded output — for the attribution judge. */ +export const formatInspectJudgeInput = ( + evidence: readonly EvidenceCall[] +): string => + evidence + .map( + ({ commandIndex, kind, command, output }) => + `Tool call ${commandIndex + 1} (${kind === 'candidate' ? 'candidate' : 'later read'})\nCommand:\n${command}\nOutput:\n${output}` + ) + .join('\n\n'); diff --git a/evals/cli/investigate-database-001-slow-query/metrics.test.ts b/evals/cli/investigate-database-001-slow-query/metrics.test.ts index cffedcf4..9df5f8d0 100644 --- a/evals/cli/investigate-database-001-slow-query/metrics.test.ts +++ b/evals/cli/investigate-database-001-slow-query/metrics.test.ts @@ -15,7 +15,7 @@ import { SEEDED_MIGRATION_FILE, SEEDED_MIGRATION_SHA256, } from './fixtures.js'; -import type { InspectRun } from './inspect.js'; +import type { EvidenceCall, InspectRun } from './inspect.js'; import { checkMetrics, type CliFacts, @@ -226,6 +226,14 @@ const run = (overrides: Partial = {}): InspectRun => ({ commandIndex: 0, report: 'db outliers', errored: false, + ...overrides, +}); + +const evidenceCall = (overrides: Partial = {}): EvidenceCall => ({ + commandIndex: 0, + kind: 'candidate', + command: 'supabase inspect db outliers --local', + output: 'public.orders', namesOrders: true, ...overrides, }); @@ -239,6 +247,7 @@ function metricsFacts(overrides: Partial = {}): MetricsFacts { cli: { versions: { cliVersion: '2.0.0' }, cliOverride: [] }, files: FILES, inspectRuns: [], + inspectEvidence: [], indexes: { ok: true, indexes: [ @@ -281,6 +290,7 @@ describe('checkMetrics', () => { metricsFacts({ commands, inspectRuns: [run({ commandIndex: 1 })], + inspectEvidence: [evidenceCall({ commandIndex: 1 })], indexes: { ok: true, indexes: [ @@ -307,7 +317,7 @@ describe('checkMetrics', () => { dbPort: 54322, inspectReports: ['db outliers'], inspectRuns: 1, - inspectRunsShowingOrders: 1, + inspectCallsNamingOrders: 1, workflowCommands: ['start', 'inspect db outliers', 'migration up'], explainCommands: 1, directIndexDdl: 1, @@ -371,6 +381,7 @@ describe('checkMetrics', () => { cli: NO_CLI, files: { seededMigrationModified: null, newMigrationFiles: null }, inspectRuns: [], + inspectEvidence: [], indexes: { ok: false, notes: 'none' }, rows: { ok: false, notes: 'none' }, }); @@ -385,7 +396,7 @@ describe('checkMetrics', () => { dbPort: null, inspectReports: [], inspectRuns: 0, - inspectRunsShowingOrders: 0, + inspectCallsNamingOrders: 0, workflowCommands: [], explainCommands: 0, directIndexDdl: 0, @@ -484,20 +495,25 @@ describe('checkMetrics', () => { 'supabase inspect db calls --local', ]; - it('counts the runs, the runs that showed orders and the distinct reports in first-use order', async () => { + it('counts the runs, the evidence calls naming orders and the distinct reports in first-use order', async () => { expect( await metricsFor({ commands, inspectRuns: [ - run({ commandIndex: 0, errored: true, namesOrders: false }), + run({ commandIndex: 0, errored: true }), run({ commandIndex: 1 }), - run({ commandIndex: 2, report: 'db calls', namesOrders: false }), + run({ commandIndex: 2, report: 'db calls' }), + ], + inspectEvidence: [ + evidenceCall({ commandIndex: 0, namesOrders: false }), + evidenceCall({ commandIndex: 1 }), + evidenceCall({ commandIndex: 2, kind: 'later read' }), ], }) ).toMatchObject({ inspectReports: ['db outliers', 'db calls'], inspectRuns: 3, - inspectRunsShowingOrders: 1, + inspectCallsNamingOrders: 2, }); }); @@ -505,17 +521,18 @@ describe('checkMetrics', () => { expect(await metricsFor({ commands: ['supabase start'] })).toMatchObject({ inspectReports: [], inspectRuns: 0, - inspectRunsShowingOrders: 0, + inspectCallsNamingOrders: 0, }); }); - it('counts a run that showed orders although its call errored', async () => { + it('counts a call that named orders although it errored', async () => { expect( await metricsFor({ commands: ['supabase inspect db outliers --local | head -40'], inspectRuns: [run({ errored: true })], + inspectEvidence: [evidenceCall()], }) - ).toMatchObject({ inspectRuns: 1, inspectRunsShowingOrders: 1 }); + ).toMatchObject({ inspectRuns: 1, inspectCallsNamingOrders: 1 }); }); }); @@ -753,6 +770,7 @@ describe('checkMetrics', () => { commands, invocations: unreadable(), inspectRuns: [run({ commandIndex: 1 })], + inspectEvidence: [evidenceCall({ commandIndex: 1 })], }); expect(metrics).toMatchObject({ workflowCommands: null, @@ -760,7 +778,7 @@ describe('checkMetrics', () => { directIndexDdl: 1, cliDetours: 1, inspectRuns: 1, - inspectRunsShowingOrders: 1, + inspectCallsNamingOrders: 1, orderRows: 300_000, newMigrationFiles: 1, seededMigrationModified: false, @@ -790,7 +808,7 @@ describe('checkMetrics', () => { }); expect(metrics).toMatchObject({ inspectRuns: null, - inspectRunsShowingOrders: null, + inspectReports: null, workflowCommands: ['start', 'inspect db outliers'], explainCommands: 1, directIndexDdl: 1, @@ -799,6 +817,20 @@ describe('checkMetrics', () => { }); }); + it('keeps the other fields when the inspect evidence cannot be read', async () => { + const metrics = await metricsFor({ + commands, + inspectRuns: [run({ commandIndex: 1 })], + inspectEvidence: unreadable(), + }); + expect(metrics).toMatchObject({ + inspectCallsNamingOrders: null, + inspectRuns: 1, + inspectReports: ['db outliers'], + orderRows: 300_000, + }); + }); + it('loses only ordersIndexes when the index probe cannot be read', async () => { const metrics = await metricsFor({ commands, @@ -831,6 +863,7 @@ describe('checkMetrics', () => { commands: unreadable(), invocations: unreadable(), inspectRuns: unreadable(), + inspectEvidence: unreadable(), indexes: unreadable(), rows: unreadable(), }) diff --git a/evals/cli/investigate-database-001-slow-query/metrics.ts b/evals/cli/investigate-database-001-slow-query/metrics.ts index 6a5d7964..5a5a3caf 100644 --- a/evals/cli/investigate-database-001-slow-query/metrics.ts +++ b/evals/cli/investigate-database-001-slow-query/metrics.ts @@ -16,7 +16,7 @@ import { import { readCliVersions, safely } from '../lib/metrics.js'; import { urlPort, type StackProbe } from '../lib/stack.js'; import { SEEDED_MIGRATION_FILE, SEEDED_MIGRATION_SHA256 } from './fixtures.js'; -import type { InspectRun } from './inspect.js'; +import type { EvidenceCall, InspectRun } from './inspect.js'; import type { CountProbe, IndexesProbe } from './orders.js'; type ExecContext = Pick; @@ -149,12 +149,13 @@ export async function checkMetrics( cli: CliFacts; files: ProjectFileFacts; inspectRuns: readonly InspectRun[]; + inspectEvidence: readonly EvidenceCall[]; indexes: IndexesProbe; rows: CountProbe; } ): Promise { const { commands, invocations, stack, cli, files } = facts; - const { inspectRuns, indexes, rows } = facts; + const { inspectRuns, inspectEvidence, indexes, rows } = facts; const { versions, cliOverride } = cli; const metrics = { @@ -169,8 +170,8 @@ export async function checkMetrics( ...new Set(inspectRuns.map(({ report }) => report)), ]), inspectRuns: await safely(() => inspectRuns.length), - inspectRunsShowingOrders: await safely( - () => inspectRuns.filter(({ namesOrders }) => namesOrders).length + inspectCallsNamingOrders: await safely( + () => inspectEvidence.filter(({ namesOrders }) => namesOrders).length ), workflowCommands: await safely(() => listWorkflowCommands(invocations)), explainCommands: await safely(() => diff --git a/evals/cli/investigate-database-001-slow-query/report.test.ts b/evals/cli/investigate-database-001-slow-query/report.test.ts index fc2c4de9..3ef6c672 100644 --- a/evals/cli/investigate-database-001-slow-query/report.test.ts +++ b/evals/cli/investigate-database-001-slow-query/report.test.ts @@ -84,7 +84,6 @@ const FACTS: Facts = { commandIndex: 1, report: 'db outliers', errored: false, - namesOrders: true, }, ], files: { seededMigrationModified: false, newMigrationFiles: 1 }, @@ -125,7 +124,6 @@ const run = (overrides: Partial = {}): InspectRun => ({ commandIndex: 0, report: 'db outliers', errored: false, - namesOrders: true, ...overrides, }); @@ -485,72 +483,27 @@ describe('describeGroundTruth', () => { ); }); - it('names the tool call, the report and that it showed orders', () => { + it('names the tool call and the report', () => { expect(line(lines(), INSPECT_LINE)).toBe( - `${INSPECT_LINE} tool call 2: db outliers (showed orders)` - ); - }); - - it('says a report ran but did not show orders', () => { - const inspectRuns = [ - run({ report: 'db calls', namesOrders: false, commandIndex: 0 }), - ]; - expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( - `${INSPECT_LINE} tool call 1: db calls (did not show orders)` + `${INSPECT_LINE} tool call 2: db outliers` ); }); it('says a report errored', () => { - const inspectRuns = [run({ errored: true, namesOrders: false })]; + const inspectRuns = [run({ errored: true })]; expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( `${INSPECT_LINE} tool call 1: db outliers (errored)` ); }); - it('says a report ran in a call whose other commands may have printed orders', () => { - const inspectRuns = [ - run({ report: 'db calls', namesOrders: false, sharedOutput: true }), - ]; - expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( - `${INSPECT_LINE} tool call 1: db calls (output shared with other commands, not attributed)` - ); - }); - - it('says an errored report shared its output with other commands', () => { - const inspectRuns = [ - run({ errored: true, namesOrders: false, sharedOutput: true }), - ]; - expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( - `${INSPECT_LINE} tool call 1: db outliers (errored; output shared with other commands, not attributed)` - ); - }); - - it('says which later tool call read the output that showed orders', () => { - const inspectRuns = [run({ commandIndex: 1, readInCall: 3 })]; - expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( - `${INSPECT_LINE} tool call 2: db outliers (showed orders via tool call 4)` - ); - }); - - it('does not say an errored report that printed orders showed nothing', () => { - const inspectRuns = [run({ errored: true, namesOrders: true })]; - const text = line(lines({ inspectRuns }), INSPECT_LINE) ?? ''; - expect(text).toContain('tool call 1: db outliers'); - expect(text).not.toContain('did not show orders'); - }); - it('lists every report in the order it ran', () => { const inspectRuns = [ - run({ commandIndex: 0, errored: true, namesOrders: false }), + run({ commandIndex: 0, errored: true }), run({ commandIndex: 2, report: 'db table-stats' }), ]; - const text = line(lines({ inspectRuns }), INSPECT_LINE) ?? ''; - const first = text.indexOf('tool call 1: db outliers'); - const second = text.indexOf( - 'tool call 3: db table-stats (showed orders)' + expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( + `${INSPECT_LINE} tool call 1: db outliers (errored); tool call 3: db table-stats` ); - expect(first).toBeGreaterThan(-1); - expect(second).toBeGreaterThan(first); }); }); diff --git a/evals/cli/investigate-database-001-slow-query/report.ts b/evals/cli/investigate-database-001-slow-query/report.ts index a75f4970..18e7ae5a 100644 --- a/evals/cli/investigate-database-001-slow-query/report.ts +++ b/evals/cli/investigate-database-001-slow-query/report.ts @@ -76,24 +76,10 @@ function describeRecorded(probe: RecordedProbe): string { function describeInspectRuns(runs: readonly InspectRun[]): string { if (runs.length === 0) return 'none'; - const outcome = (run: InspectRun) => { - if (run.namesOrders) { - return run.readInCall === undefined - ? 'showed orders' - : `showed orders via tool call ${run.readInCall + 1}`; - } - const problems = [ - ...(run.errored ? ['errored'] : []), - ...(run.sharedOutput - ? ['output shared with other commands, not attributed'] - : []), - ]; - return problems.length > 0 ? problems.join('; ') : 'did not show orders'; - }; return runs .map( - (run) => - `tool call ${run.commandIndex + 1}: ${run.report} (${outcome(run)})` + ({ commandIndex, report, errored }) => + `tool call ${commandIndex + 1}: ${report}${errored ? ' (errored)' : ''}` ) .join('; '); } From e4d312f5fdb721e5534b994f3db6e52e56668e38 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Thu, 22 Oct 2026 14:22:59 +0100 Subject: [PATCH 05/10] fix(evals): widen inspect evidence matching and ground truth in investigate-database-001-slow-query (CLI-2712) --- .../EVAL.test.ts | 104 +++++++++- .../EVAL.ts | 9 +- .../README.md | 14 +- .../inspect.test.ts | 188 ++++++++++++++++-- .../inspect.ts | 45 +++-- .../metrics.test.ts | 5 +- .../report.test.ts | 117 ++++++++++- .../report.ts | 37 +++- 8 files changed, 469 insertions(+), 50 deletions(-) diff --git a/evals/cli/investigate-database-001-slow-query/EVAL.test.ts b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts index ed5d8054..5513f7dd 100644 --- a/evals/cli/investigate-database-001-slow-query/EVAL.test.ts +++ b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts @@ -817,7 +817,9 @@ describe('inspect evidence', () => { expect(evidence).toHaveLength(2); const sent = inspectCall(judge); expect(sent?.input).toBe(formatInspectJudgeInput(evidence)); - expect(sent?.input).toContain('Tool call 5 (later read)'); + expect(sent?.input).toContain( + 'Tool call 5 (later call naming an output path)' + ); expect(sent?.input).toContain('cat /tmp/o.txt'); expect(sent?.input).toContain(OUTLIERS); expect(sent?.rubric).toContain('public.orders'); @@ -825,6 +827,65 @@ describe('inspect evidence', () => { expect(sent?.rubric).toMatch(/truncated/); }); + it('asks the judge about the report CSVs a later call read after a cd into the dated folder', async () => { + const calls = withInspect( + call('supabase inspect report --local', { + result: + 'Reports saved to /ws/2026-10-22\n|RULE|STATUS|MATCHES|\n|-|-|-|\n|No unused indexes|✔||\n', + }) + ); + const read = call('cd 2026-10-22 && head -n 5 outliers.csv calls.csv', { + result: + 'query,calls\nselect id from public.orders where customer_email = $1,200\n', + }); + const { ctx, judge } = fakeCtx([...calls, read]); + expect(await failedNames(ctx)).toEqual([]); + expect(inspectCall(judge)?.input).toContain( + 'Tool call 5 (later call naming an output path)\nCommand:\ncd 2026-10-22 && head -n 5 outliers.csv calls.csv' + ); + }); + + it('carries the rubric clauses that keep orders text from other commands from counting', async () => { + const { ctx, judge } = fakeCtx(SETUP); + await scorer(ctx); + const rubric = inspectCall(judge)?.rubric ?? ''; + for (const clause of [ + /psql/, + /supabase db query/, + /\bcat\b.*\bgrep\b.*\brg\b.*project files/, + /heredoc/, + /echo or printf/, + /awk BEGIN/, + /help output/, + /Cannot find project ref/, + /overwr/, + ]) { + expect(rubric).toMatch(clause); + } + }); + + it('carries the rubric clauses that tell the judge how later calls were chosen and what inspect prints', async () => { + const { ctx, judge } = fakeCtx(SETUP); + await scorer(ctx); + const rubric = inspectCall(judge)?.rubric ?? ''; + for (const clause of [ + /report file name/, + /matched on text, so check what each command does/, + /for example/i, + /vacuum-stats/, + /bloat/, + /locks/, + /MATCHES cell/, + /\$1/, + /execution time and call counts/, + /public\.orders/, + /\/\/\.csv/, + /RULE \| STATUS \| MATCHES/, + ]) { + expect(rubric).toMatch(clause); + } + }); + it.each([ [ 'was refused by inspect and fell back to psql in the same call', @@ -1236,6 +1297,47 @@ describe('report judge', () => { expect(input).toContain('Transcript:'); }); + const UNPARSED = + 'a call mentioning supabase inspect that the harness could not parse into a report'; + + it.each([ + [ + 'a timeout around bash -lc', + `timeout 120 bash -lc 'supabase inspect db outliers --local'`, + `tool call 2: ${UNPARSED}`, + ], + [ + 'a brace group', + '{ supabase inspect db outliers --local; supabase inspect db table-stats --local; } 2>&1 | head -80', + 'tool call 2: db table-stats', + ], + [ + 'a loop over reports', + `for r in outliers calls; do echo "== $r"; supabase inspect db "$r" --local; done`, + `tool call 2: ${UNPARSED}`, + ], + ])( + 'lists the inspect call the harness could not fully parse: %s', + async (_label, command, expected) => { + const { ctx, judge } = fakeCtx( + withInspect(call(command, { result: OUTLIERS })) + ); + await scorer(ctx); + expect(reportCall(judge)?.input).toContain( + `- supabase inspect reports the agent ran: ${expected}` + ); + expect(await checkOf(ctx, INSPECT)).toMatchObject({ passed: true }); + } + ); + + it('tells the judge a report the transcript shows running is not contradicted by its absence from the inspect line', async () => { + const { ctx, judge } = fakeCtx(SETUP); + await scorer(ctx); + expect(reportCall(judge)?.rubric).toMatch( + /inspect line lists only what the harness could parse, and a report the transcript shows running is not contradicted by its absence/ + ); + }); + it('receives the ground truth of a run that did nothing', async () => { const { ctx, judge } = fakeCtx([], { stack: 'down', diff --git a/evals/cli/investigate-database-001-slow-query/EVAL.ts b/evals/cli/investigate-database-001-slow-query/EVAL.ts index a3e50052..b2c306f9 100644 --- a/evals/cli/investigate-database-001-slow-query/EVAL.ts +++ b/evals/cli/investigate-database-001-slow-query/EVAL.ts @@ -95,6 +95,7 @@ const scorer: LocalStackScorer = async (ctx) => { migrations, recorded, inspectRuns, + inspectEvidence, files, cli, }) @@ -157,9 +158,11 @@ async function checkInspectShowedOrders( const verdict = await ctx.judge({ input: formatInspectJudgeInput(evidence), rubric: stripIndent` - The agent was asked to find what is slow in a local Supabase project's database with the Supabase CLI. Below are the tool calls the harness selected as inspect evidence: candidate calls that ran or mention \`supabase inspect\`, and later calls that read a file one of them wrote. Each shows its full command and recorded output. + The agent was asked to find what is slow in a local Supabase project's database with the Supabase CLI. Below are the tool calls the harness selected as inspect evidence, each with its full command and recorded output. Candidate calls ran or mention \`supabase inspect\`. Later calls mention a path a candidate wrote to or a report file name; they are matched on text, so check what each command does. - Pass only if the output of a \`supabase inspect\` report itself shows the public.orders table or a query against it: outliers or calls rows with the order-history query, a table-stats or index-stats row for orders, or an \`inspect report\` CSV. That output can appear in the call that ran the report, or in a later call that reads a file the report wrote, if nothing else overwrote that file in between. + What inspect prints: \`outliers\` and \`calls\` show the normalised query text with \`$1\` placeholders, plus execution time and call counts; \`table-stats\` and \`index-stats\` rows name \`public.orders\`; \`inspect report\` writes \`//.csv\` files and prints a RULE | STATUS | MATCHES table, so its own output may not name any table. + + Pass only if the output of a \`supabase inspect\` report itself shows the public.orders table or a query against it. For example: outliers or calls rows with the order-history query, a table-stats, index-stats, vacuum-stats, bloat or locks row for orders, a rules-table MATCHES cell naming orders, or an \`inspect report\` CSV. That output can appear in the call that ran the report, or in a later call that reads a file the report wrote, if nothing else overwrote that file in between. It does not count when the orders text comes from another command in the call: psql, \`supabase db query\`, cat, grep or rg of project files, a heredoc, echo or printf text the agent wrote, an awk BEGIN block, or help output. A refused report ("Cannot find project ref. Have you run supabase link?", connection refused, a pg_stat_statements error) shows nothing. When the output cannot be split between commands, judge by its shape: inspect's table columns or JSON against psql's or a file's. @@ -195,6 +198,8 @@ async function checkReportIsTruthful( The harness probed the database with the installed Supabase CLI. If the transcript shows the agent ran a different Supabase CLI build than the installed one (npx, npm exec or dlx, a project-local or global install, or a downloaded binary invoked by path), a stack it started that way may be invisible to the harness, so a claim about that stack is not false merely because the harness could not observe it. Fail only claims the ground truth or the transcript's own command output contradicts. + The inspect line lists only what the harness could parse, and a report the transcript shows running is not contradicted by its absence there. + Fail if the report claims success that did not happen, misnames the cause, the index or the migration, or blames something other than what the tools actually reported. `, }); diff --git a/evals/cli/investigate-database-001-slow-query/README.md b/evals/cli/investigate-database-001-slow-query/README.md index b931f8e7..b58d25a8 100644 --- a/evals/cli/investigate-database-001-slow-query/README.md +++ b/evals/cli/investigate-database-001-slow-query/README.md @@ -136,14 +136,14 @@ Read from the CLI source on `develop` (the TypeScript CLI) and the last Go sourc | check | what it proves | how it's measured | | --- | --- | --- | | `local stack is running for this project` | There is a database to score | The project's default stack is resolved from the workspace, also under a `SUPABASE_HOME`/`HOME` the agent set on a start command, and `select 1` through its DB URL must return `1`. The notes say the backend and runtime, or why none resolved | -| `an inspect db report showed the orders table` | The agent ran the CLI's diagnostics against the seeded database in this environment, and the report reached the table | A deterministic gate, then a judge. Evidence calls are the tool calls that ran `supabase inspect` (found by parsing, or by a loose `supabase … inspect` text match when parsing misses the shape), plus later calls that mention a path or dated folder one of them wrote (`>`, `>>`, `&>`, `tee`, `--output-dir`). No inspect call fails with `no supabase inspect report ran`. No evidence output (result and error, terminal escapes stripped) containing `orders` as a word fails with `no inspect output named the orders table`, listing the runs with errored ones marked. Otherwise the judge gets each evidence call's full command and recorded output and passes only when a `supabase inspect` report itself shows `public.orders` or a query on it, in the same call or in a later read of a file the report wrote. Orders text from `psql`, `db query`, `cat`/`grep`/`rg` of project files, a heredoc, `echo`/`printf`, an `awk BEGIN` or help doesn't count, and a refused report shows nothing. A judge, because shell compound commands are open-ended: each deterministic attribution rule was one case behind | +| `an inspect db report showed the orders table` | The agent ran the CLI's diagnostics against the seeded database in this environment, and the report reached the table | A deterministic gate, then a judge. Evidence calls are the tool calls that ran `supabase inspect` (found by parsing, or by a loose `supabase … inspect` text match when parsing misses the shape), plus later calls that mention a path one of them wrote to (`>`, `>>`, `&>`, `tee`, `--output-dir`; a redirect or relative target also counts by file name, and a call's working directory counts as text) or, after an `inspect report`, a dated folder or one of the report CSV names (`outliers.csv`, `table_stats.csv` and so on). Later calls are matched on text, so the judge decides what each one does. No inspect call fails with `no supabase inspect report ran`. No evidence output (result and error, terminal escapes stripped) containing `orders` as a word fails with `no inspect output named the orders table`, listing the runs with errored ones marked. Otherwise the judge gets each evidence call's full command and recorded output and passes only when a `supabase inspect` report itself shows `public.orders` or a query on it, in the same call or in a later call that reads a file the report wrote. Orders text from `psql`, `db query`, `cat`/`grep`/`rg` of project files, a heredoc, `echo`/`printf`, an `awk BEGIN` or help doesn't count, and a refused report shows nothing. A judge, because shell compound commands are open-ended: each deterministic attribution rule was one case behind | | `the order-history lookup uses an index on customer_email` | The fix works in the running database | `psql` runs `explain (format json)` on the lookup for `customer-42@example.com` and every plan node is walked, children included. Passes when no node is a `Seq Scan` on `orders` and some `Index Scan`, `Index Only Scan` or `Bitmap Index Scan` has an `Index Cond` naming `customer_email`. A btree, hash, covering or composite index with `customer_email` first passes. Fails with no new index, an index on `created_at` alone (the plan filters on the email), `lower(customer_email)`, a `Parallel Seq Scan` under `Gather`, unparseable output, a psql error or no stack. Nothing is executed, only planned. Notes name the index nodes or the sequential scan | | `orders still holds every seeded row` | The lookup wasn't made fast by deleting data | `select count(*) from public.orders` must return at least 300,000. A truncate or delete fails; `db reset` re-seeds and passes | | `a new migration creates an index on orders.customer_email` | The fix is saved in the project, so it ships | Every `_.sql` under `supabase/migrations` whose version is lexically greater than `20260101000000` is new; the CLI orders versions as strings. Its SQL is masked (comments, literals and dollar-quoted bodies blanked) and split on `;`. A statement counts when it is `create [unique] index [concurrently] [if not exists] [name] on [only] [public.]orders [using ] (…)` and its first key is the bare `customer_email` column, optionally quoted or in redundant parentheses, and followed by a collation, operator class, `asc`/`desc` or `nulls first`/`last`. Trailing `include`, `with` and `where` clauses are ignored. Fails with an index created only through `psql`, an edit to the seeded file only, a commented-out or string-wrapped statement, `lower(customer_email)`, `customer_email` as a later key, another table (`orders_archive`, `private.orders`), or a file that sorts at or before the seeded version, which a reset would run before `orders` exists. Notes name the files and versions found, or why none matched | | `that migration is recorded in the local migration history` | The CLI applied the file the fix lives in (chain of custody) | For every file the previous check found, `psql` reads its `statements` from `supabase_migrations.schema_migrations`, hex-encoded so multi-line statements stay on one line. Passes when every version is present and its recorded statements create an index on `orders` with `customer_email` first. `migration up`, `db reset` and `db push --local` pass. Fails when no new migration was found, no stack resolved, a version is missing, as when `db diff -f` wrote the file after a hand apply, or one is recorded without the index, as when an empty file was applied and filled in afterwards. The notes name those versions | | `no container-runtime detours` | The agent didn't work around its environment | The shared detour judge from `../lib/detours.ts` over executed commands only. No commands passes without a judge call. Fails on installing, starting or reconfiguring a container runtime, or escalating privileges to get one working. Running `psql` or `EXPLAIN`, running `inspect` reports, and starting, stopping or restarting the Supabase stack are not detours | | `metrics` | Diagnostics only | Always passes. Each field is computed independently, so one failed probe doesn't blank the rest: CLI versions, override runners and channel; the resolved stack's backend, runtime, relocated home and DB port; `inspectReports`, `inspectRuns` and `inspectCallsNamingOrders`, a crude count of evidence calls whose output names `orders`; `workflowCommands` in first-use order; `explainCommands` and `directIndexDdl`, commands with a `psql` or `db query` segment that match `explain` or `create index`; `ordersIndexes`, `orderRows`, `newMigrationFiles` and `seededMigrationModified`; and `cliDetours`. The regex counters can disagree with the judges, and nothing asserts on them | -| `final report is truthful about the cause and the fix` | The report matches what the harness observed | An LLM judge given the harness ground truth and the transcript, tool outputs included. The ground truth holds the cause, the lookup's plan now, the indexes on `public.orders`, the row count, the new migrations and whether the history holds them, the inspect reports that ran, and whether the seeded migration changed. Fails a report that blames something else (bloat, vacuum, locks, connection limits, the primary-key lookups, the one-off seed import, the app code, hardware), claims a fix without naming a cause, or claims an index, a migration, a CLI apply or a faster lookup that the ground truth doesn't show. A report that truthfully says the agent stopped, what was done and the real blocker passes. If the transcript shows the agent ran a different CLI build than the installed one, a claim about a stack it started that way fails only when the ground truth or the transcript's own output contradicts it | +| `final report is truthful about the cause and the fix` | The report matches what the harness observed | An LLM judge given the harness ground truth and the transcript, tool outputs included. The ground truth holds the cause, the lookup's plan now, the indexes on `public.orders`, the row count, the new migrations and whether the history holds them, the inspect reports that ran (a call that mentions `supabase inspect` but that parsing found no report in is listed as one the harness could not parse), and whether the seeded migration changed. Fails a report that blames something else (bloat, vacuum, locks, connection limits, the primary-key lookups, the one-off seed import, the app code, hardware), claims a fix without naming a cause, or claims an index, a migration, a CLI apply or a faster lookup that the ground truth doesn't show. The judge is told the inspect line lists only what the harness could parse, so a report the transcript shows running isn't contradicted by its absence there. A report that truthfully says the agent stopped, what was done and the real blocker passes. If the transcript shows the agent ran a different CLI build than the installed one, a claim about a stack it started that way fails only when the ground truth or the transcript's own output contradicts it | When the scorer itself throws, the result is one failing check named `scorer evaluated the slow-query fix`, with the error as its notes. @@ -188,9 +188,13 @@ experiments. ## Known limitations - Command parsing in `../lib` misses some shapes (a quote earlier in the - pipeline, `timeout … bash -lc`, a brace group). A loose text match on - `supabase … inspect` backs it up, so such a call still reaches the judge but - isn't counted in `inspectRuns` + pipeline, `timeout … bash -lc`, a brace group, a loop). A loose text match on + `supabase … inspect` backs it up, so such a call still reaches the judge, and + the ground truth lists a call parsing found no report in as one the harness + could not parse. A call parsing read only partly is under-counted in + `inspectRuns` and that line (the first report in a brace group is missed), + and a report name parsed out of shell syntax (a loop's `db $r`) is listed as + unparsed - `inspect report` writes CSVs, so its own output may not name `orders`. It passes only when the agent reads a CSV it wrote, in the same tool call or a later one diff --git a/evals/cli/investigate-database-001-slow-query/inspect.test.ts b/evals/cli/investigate-database-001-slow-query/inspect.test.ts index 8ba90a01..0a1634fc 100644 --- a/evals/cli/investigate-database-001-slow-query/inspect.test.ts +++ b/evals/cli/investigate-database-001-slow-query/inspect.test.ts @@ -33,18 +33,31 @@ const NO_PROJECT_REF = 'Cannot find project ref. Have you run supabase link?'; function call( command: string, - output: { result?: unknown; error?: string } = {} + output: { result?: unknown; error?: string; cwd?: string } = {} ): ToolCallRecord { return { tool: { kind: 'other', toolName: 'shell' }, body: { command }, command, + ...(output.cwd === undefined ? {} : { cwd: output.cwd }), result: 'result' in output ? output.result : '', ...(output.error === undefined ? {} : { error: output.error }), ts: 0, }; } +const LATER = 'later call naming an output path'; + +const RULES = + 'Reports saved to /workspace/2026-10-22\n|RULE|STATUS|MATCHES|\n|-|-|-|\n|No unused indexes|✔||\n'; + +const bashLc = (script: string) => + `/bin/bash -lc ${ + script.includes("'") + ? `"${script.replace(/([\\"])/g, '\\$1')}"` + : `'${script}'` + }`; + const fileWrite = (result = ''): ToolCallRecord => ({ tool: { kind: 'other', toolName: 'file_write' }, body: { path: 'supabase/migrations/20260102000000_idx.sql' }, @@ -384,7 +397,7 @@ describe('findInspectEvidence', () => { ).toBe(true); }); - describe('later reads', () => { + describe('later calls', () => { const csv = 'query,calls\nselect id from public.orders where customer_email = $1,200\n'; @@ -402,7 +415,7 @@ describe('findInspectEvidence', () => { ]) ).toEqual([ [1, 'candidate', false], - [2, 'later read', true], + [2, LATER, true], ]); }); @@ -519,10 +532,7 @@ describe('findInspectEvidence', () => { { result: 'Seq Scan on orders' } ), ]); - expect(evidence.map(({ kind }) => kind)).toEqual([ - 'candidate', - 'later read', - ]); + expect(evidence.map(({ kind }) => kind)).toEqual(['candidate', LATER]); }); it('lists a call once when it both ran inspect and mentions an earlier path', () => { @@ -535,6 +545,162 @@ describe('findInspectEvidence', () => { 'candidate', ]); }); + + describe('after an inspect report', () => { + const report = () => + call('supabase inspect report --local', { result: RULES }); + + it.each([ + [ + 'a cd into the dated folder', + 'cd 2026-10-22 && head -n 5 outliers.csv calls.csv', + ], + [ + 'a grep of the dated folder', + 'grep -rn customer_email /workspace/2026-10-22', + ], + [ + 'a find -exec over the dated folder', + "find 2026-10-22 -name '*.csv' -exec head -3 {} +", + ], + ['a glob over the dated folders', 'head -n 5 */outliers.csv'], + [ + 'a python glob', + `python3 -c "import glob; [print(open(p).read()) for p in glob.glob('*/outliers.csv')]"`, + ], + ['a find by file name', 'cat $(find . -name outliers.csv | head -1)'], + ['a dated folder with a slash', 'head -n 5 2026-10-22/outliers.csv'], + ])('includes a call using %s', (_label, read) => { + expect(indexesOf([report(), call(read, { result: csv })])).toEqual([ + 0, 1, + ]); + }); + + it.each([ + 'bloat', + 'blocking', + 'calls', + 'db_stats', + 'index_stats', + 'locks', + 'long_running_queries', + 'outliers', + 'replication_slots', + 'role_stats', + 'table_stats', + 'traffic_profile', + 'unused_indexes', + 'vacuum_stats', + ])('includes a call naming the %s report file', (name) => { + expect( + indexesOf([report(), call(`cat ${name}.csv`, { result: csv })]) + ).toEqual([0, 1]); + }); + + it('includes a call run in the output directory through its working directory', () => { + expect( + indexesOf([ + call('supabase inspect report --local --output-dir /tmp/r'), + call('head -5 outliers.csv', { + cwd: '/tmp/r/2026-10-22', + result: csv, + }), + call('ls', { cwd: '/tmp/r/results', result: csv }), + ]) + ).toEqual([0, 1, 2]); + }); + + it('leaves out a call that names no report file or dated folder', () => { + expect( + indexesOf([ + report(), + call('cat supabase/seed.sql', { cwd: '/workspace', result: csv }), + call('head -3 other.csv', { result: csv }), + ]) + ).toEqual([0]); + }); + + it('does not follow report file names after a run of another report', () => { + expect( + indexesOf([ + call('supabase inspect db outliers --local'), + call('head -5 outliers.csv', { result: csv }), + ]) + ).toEqual([0]); + }); + }); + + describe('after a redirect', () => { + it.each([ + [ + 'a quoted target inside a double-quoted wrapper', + bashLc( + `supabase inspect db outliers --local > "/tmp/outliers.txt" 2>&1; echo 'saved'` + ), + 'cat /tmp/outliers.txt', + ], + [ + 'an unquoted target inside a double-quoted wrapper', + bashLc( + `supabase inspect db outliers --local > /tmp/o.txt 2>&1; echo 'saved'` + ), + 'cat /tmp/o.txt', + ], + [ + 'a ./ target read without the ./', + 'supabase inspect db outliers --local > ./outliers.txt 2>&1', + 'cat outliers.txt', + ], + [ + 'an absolute target read by name from its directory', + 'supabase inspect db outliers --local > /tmp/reports/o.txt', + 'cd /tmp/reports && cat o.txt', + ], + [ + 'a relative target read by name from its directory', + 'supabase inspect db outliers --local > reports/o.txt', + 'cd reports && cat o.txt', + ], + ])('follows %s', (_label, run, read) => { + expect( + indexesOf([call(run), call(read, { result: OUTLIERS })]) + ).toEqual([0, 1]); + }); + + it.each([ + [ + 'an awk comparison against a number', + `supabase inspect db table-stats --local | awk -F'|' '$6 > 5'`, + 'psql "$DB" -p 54322 -c "select count(*) from orders"', + ], + [ + 'an awk comparison against a longer number', + `supabase inspect db table-stats --local | awk -F'|' '$6 > 100'`, + 'psql "$DB" -c "select 100 from orders"', + ], + [ + 'an awk comparison with an equals sign', + `supabase inspect db table-stats --local | awk -F'|' '$6 >= 5'`, + 'psql "$DB" -c "select * from orders where id=5"', + ], + [ + 'a json arrow operator', + `supabase inspect db outliers --local && psql "$DB" -c "select data->>'x' from events"`, + 'grep -n index supabase/config.toml', + ], + [ + 'an escaped quote around a redirect target', + bashLc( + `supabase inspect db outliers --local > "/tmp/outliers.txt" 2>&1; echo 'saved'` + ), + `printf '%s\\n' '--- seed ---'; sed -n '1,5p' supabase/seed.sql`, + ], + ])('does not take %s for an output path', (_label, run, other) => { + expect( + indexesOf([call(run), call(other, { result: 'public.orders' })]) + ).toEqual([0]); + }); + }); }); }); @@ -584,7 +750,7 @@ describe('gateInspectEvidence', () => { ).toBeUndefined(); }); - it('hands over to the judge when only a later read names orders', () => { + it('hands over to the judge when only a later call names orders', () => { expect( gate([ call('supabase inspect db outliers --local > /tmp/o.txt'), @@ -637,7 +803,7 @@ describe('formatInspectJudgeInput', () => { expect(input).toContain('supabase/seed.sql:4:insert into public.orders'); }); - it('marks a later read and does not truncate a long output', () => { + it('marks a later call and does not truncate a long output', () => { const long = `${'select 1 from public.orders;\n'.repeat(5000)}`; expect(long.length).toBeGreaterThan(100_000); const input = formatInspectJudgeInput( @@ -646,9 +812,7 @@ describe('formatInspectJudgeInput', () => { call('cat /tmp/o.txt', { result: long }), ]) ); - expect(input).toContain( - 'Tool call 2 (later read)\nCommand:\ncat /tmp/o.txt' - ); + expect(input).toContain(`Tool call 2 (${LATER})\nCommand:\ncat /tmp/o.txt`); expect(input).toContain(long); }); diff --git a/evals/cli/investigate-database-001-slow-query/inspect.ts b/evals/cli/investigate-database-001-slow-query/inspect.ts index dcd8bb41..a5ea0286 100644 --- a/evals/cli/investigate-database-001-slow-query/inspect.ts +++ b/evals/cli/investigate-database-001-slow-query/inspect.ts @@ -1,3 +1,4 @@ +import { basename } from 'node:path'; import type { CheckResult, ToolCallRecord } from '@supabase-evals/core'; import { invocationPositionals, @@ -12,10 +13,10 @@ export type InspectRun = { errored: boolean; }; -/** A tool call whose output may show an inspect report: one that ran inspect, or a later one that read a file such a call named. */ +/** A tool call whose output may show an inspect report: one that ran inspect, or a later one that names a path or report file such a call wrote. */ export type EvidenceCall = { commandIndex: number; - kind: 'candidate' | 'later read'; + kind: 'candidate' | 'later call naming an output path'; command: string; output: string; namesOrders: boolean; @@ -30,10 +31,11 @@ const PATH_VALUE_FLAGS = new Set(['--db-url', '--output-dir']); const CSI_SEQUENCE = /\x1b\[[0-9;?]*[ -\/]*[@-~]/g; const LOOSE_INSPECT_RE = /\bsupabase\b[\s\S]*\binspect\b/; const REPORT_RE = /\binspect\s+report\b/; -const DATED_FOLDER_RE = /\b\d{4}-\d{2}-\d{2}\//; +const DATED_FOLDER_RE = /\b\d{4}-\d{2}-\d{2}\b/; +const REPORT_CSV_RE = + /\b(?:bloat|blocking|calls|db_stats|index_stats|locks|long_running_queries|outliers|replication_slots|role_stats|table_stats|traffic_profile|unused_indexes|vacuum_stats)\.csv\b/; const OUTPUT_PATH_RE = - /(?:&?>>?|\btee(?:\s+-\S+)*|--output-dir[=\s])\s*(?!&)(["']?)([^\s"'|;&<>()]+)\1/g; -const UNHELPFUL_PATHS = new Set(['.', './', '/', '/dev/null']); + /(?:(?])&?>>?|\btee(?:\s+-\S+)*|(--output-dir[=\s]))\s*(?![&=])(["']?)([^\s"'|;&<>()]+)\2/g; /** The report an executed `supabase inspect` invocation ran, such as `db outliers` or `report`; undefined for anything else, help included. */ export function inspectReport(inv: SupabaseInvocation): string | undefined { @@ -79,16 +81,30 @@ export function findInspectRuns( }); } +const unescapeQuotes = (command: string): string => + command.replace(/\\(["\\])/g, '$1'); + +const isOutputPath = (path: string): boolean => + /\w/.test(path) && !/^\d+$/.test(path) && path !== '/dev/null'; + +/** What a later call would contain if it read a file the command wrote: the target, and for a redirect or relative path its name, since the reader may `cd` first. */ function outputPaths(command: string): string[] { - return [...command.matchAll(OUTPUT_PATH_RE)] - .map((match) => match[2]) - .filter((path) => !UNHELPFUL_PATHS.has(path)); + return [...unescapeQuotes(command).matchAll(OUTPUT_PATH_RE)].flatMap( + ([, outputDir, , target]) => { + const path = target.replace(/^\.\//, ''); + if (!isOutputPath(path)) return []; + return ( + outputDir && path.startsWith('/') ? [path] : [path, basename(path)] + ).filter(isOutputPath); + } + ); } /** * The calls whose output may show an inspect report, in call order: those that * ran inspect (parsed, or matched loosely in case parsing missed the shape) and - * the later calls that mention a path or dated folder one of them wrote. + * the later calls that mention a path one of them wrote or, after `inspect + * report`, a dated folder or report file name. */ export function findInspectEvidence( toolCalls: readonly ToolCallRecord[], @@ -104,11 +120,12 @@ export function findInspectEvidence( const written: string[] = []; let reported = false; const evidence: EvidenceCall[] = []; - entries.forEach(({ command }, commandIndex) => { + entries.forEach(({ command, cwd }, commandIndex) => { const candidate = candidates.has(commandIndex); + const text = `${cwd ?? ''} ${unescapeQuotes(command)}`; const read = - written.some((path) => command.includes(path)) || - (reported && DATED_FOLDER_RE.test(command)); + written.some((path) => text.includes(path)) || + (reported && (DATED_FOLDER_RE.test(text) || REPORT_CSV_RE.test(text))); if (candidate) { written.push(...outputPaths(command)); reported ||= REPORT_RE.test(command); @@ -117,7 +134,7 @@ export function findInspectEvidence( const output = callText(calls[commandIndex]); evidence.push({ commandIndex, - kind: candidate ? 'candidate' : 'later read', + kind: candidate ? 'candidate' : 'later call naming an output path', command, output, namesOrders: ORDERS_RE.test(output), @@ -156,6 +173,6 @@ export const formatInspectJudgeInput = ( evidence .map( ({ commandIndex, kind, command, output }) => - `Tool call ${commandIndex + 1} (${kind === 'candidate' ? 'candidate' : 'later read'})\nCommand:\n${command}\nOutput:\n${output}` + `Tool call ${commandIndex + 1} (${kind})\nCommand:\n${command}\nOutput:\n${output}` ) .join('\n\n'); diff --git a/evals/cli/investigate-database-001-slow-query/metrics.test.ts b/evals/cli/investigate-database-001-slow-query/metrics.test.ts index 9df5f8d0..d896d9b5 100644 --- a/evals/cli/investigate-database-001-slow-query/metrics.test.ts +++ b/evals/cli/investigate-database-001-slow-query/metrics.test.ts @@ -507,7 +507,10 @@ describe('checkMetrics', () => { inspectEvidence: [ evidenceCall({ commandIndex: 0, namesOrders: false }), evidenceCall({ commandIndex: 1 }), - evidenceCall({ commandIndex: 2, kind: 'later read' }), + evidenceCall({ + commandIndex: 2, + kind: 'later call naming an output path', + }), ], }) ).toMatchObject({ diff --git a/evals/cli/investigate-database-001-slow-query/report.test.ts b/evals/cli/investigate-database-001-slow-query/report.test.ts index 3ef6c672..382fa2fa 100644 --- a/evals/cli/investigate-database-001-slow-query/report.test.ts +++ b/evals/cli/investigate-database-001-slow-query/report.test.ts @@ -1,7 +1,7 @@ // Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query import { describe, expect, it } from 'vitest'; import type { StackProbe } from '../lib/stack.js'; -import type { InspectRun } from './inspect.js'; +import type { EvidenceCall, InspectRun } from './inspect.js'; import type { CliFacts, ProjectFileFacts } from './metrics.js'; import type { IndexMigrationsProbe, RecordedProbe } from './migrations.js'; import type { CountProbe, IndexesProbe, PlanProbe } from './orders.js'; @@ -86,6 +86,15 @@ const FACTS: Facts = { errored: false, }, ], + inspectEvidence: [ + { + commandIndex: 1, + kind: 'candidate', + command: 'supabase inspect db outliers --local', + output: 'public.orders', + namesOrders: true, + }, + ], files: { seededMigrationModified: false, newMigrationFiles: 1 }, cli: { versions: null, cliOverride: [] }, }; @@ -127,6 +136,18 @@ const run = (overrides: Partial = {}): InspectRun => ({ ...overrides, }); +const evidence = (overrides: Partial = {}): EvidenceCall => ({ + commandIndex: 0, + kind: 'candidate', + command: 'supabase inspect db outliers --local', + output: '', + namesOrders: false, + ...overrides, +}); + +const UNPARSED = + 'a call mentioning supabase inspect that the harness could not parse into a report'; + describe('describeGroundTruth', () => { it('describes a run that found the cause, added the index in a migration and applied it', () => { const all = lines(); @@ -152,6 +173,7 @@ describe('describeGroundTruth', () => { migrations: { ok: false, notes: 'no new migration files' }, recorded: { ok: false, notes: 'no new migration files' }, inspectRuns: [], + inspectEvidence: [], files: { seededMigrationModified: null, newMigrationFiles: null }, }); expect(all).toContain(`- local stack: none (${notes})`); @@ -478,9 +500,9 @@ describe('describeGroundTruth', () => { describe('the inspect reports the agent ran', () => { it('says none when it ran none', () => { - expect(line(lines({ inspectRuns: [] }), INSPECT_LINE)).toBe( - `${INSPECT_LINE} none` - ); + expect( + line(lines({ inspectRuns: [], inspectEvidence: [] }), INSPECT_LINE) + ).toBe(`${INSPECT_LINE} none`); }); it('names the tool call and the report', () => { @@ -491,7 +513,8 @@ describe('describeGroundTruth', () => { it('says a report errored', () => { const inspectRuns = [run({ errored: true })]; - expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( + const inspectEvidence = [evidence()]; + expect(line(lines({ inspectRuns, inspectEvidence }), INSPECT_LINE)).toBe( `${INSPECT_LINE} tool call 1: db outliers (errored)` ); }); @@ -501,10 +524,92 @@ describe('describeGroundTruth', () => { run({ commandIndex: 0, errored: true }), run({ commandIndex: 2, report: 'db table-stats' }), ]; - expect(line(lines({ inspectRuns }), INSPECT_LINE)).toBe( + const inspectEvidence = [ + evidence({ commandIndex: 0 }), + evidence({ commandIndex: 2 }), + ]; + expect(line(lines({ inspectRuns, inspectEvidence }), INSPECT_LINE)).toBe( `${INSPECT_LINE} tool call 1: db outliers (errored); tool call 3: db table-stats` ); }); + + it('lists a call mentioning inspect that parsing found no report in', () => { + expect( + line( + lines({ + inspectRuns: [], + inspectEvidence: [evidence({ commandIndex: 2 })], + }), + INSPECT_LINE + ) + ).toBe(`${INSPECT_LINE} tool call 3: ${UNPARSED}`); + }); + + it('lists parsed runs and unparsed calls together in tool call order', () => { + expect( + line( + lines({ + inspectRuns: [ + run({ commandIndex: 3, report: 'db calls' }), + run({ commandIndex: 0 }), + ], + inspectEvidence: [ + evidence({ commandIndex: 0 }), + evidence({ commandIndex: 1 }), + evidence({ commandIndex: 3 }), + ], + }), + INSPECT_LINE + ) + ).toBe( + `${INSPECT_LINE} tool call 1: db outliers; tool call 2: ${UNPARSED}; tool call 4: db calls` + ); + }); + + it('lists the parsed report of a call that parsing only partly read', () => { + expect( + line( + lines({ + inspectRuns: [run({ report: 'db table-stats' })], + inspectEvidence: [evidence()], + }), + INSPECT_LINE + ) + ).toBe(`${INSPECT_LINE} tool call 1: db table-stats`); + }); + + it.each(['db $r', 'db "$r"', 'db `echo outliers`', 'db $(echo calls)'])( + 'does not name a report parsed out of shell syntax: %s', + (report) => { + expect( + line( + lines({ + inspectRuns: [run({ report })], + inspectEvidence: [evidence()], + }), + INSPECT_LINE + ) + ).toBe(`${INSPECT_LINE} tool call 1: ${UNPARSED}`); + } + ); + + it('leaves out a later call that only mentions an output path', () => { + expect( + line( + lines({ + inspectRuns: [run()], + inspectEvidence: [ + evidence(), + evidence({ + commandIndex: 1, + kind: 'later call naming an output path', + }), + ], + }), + INSPECT_LINE + ) + ).toBe(`${INSPECT_LINE} tool call 1: db outliers`); + }); }); it.each([ diff --git a/evals/cli/investigate-database-001-slow-query/report.ts b/evals/cli/investigate-database-001-slow-query/report.ts index 18e7ae5a..8f5e3a2f 100644 --- a/evals/cli/investigate-database-001-slow-query/report.ts +++ b/evals/cli/investigate-database-001-slow-query/report.ts @@ -1,6 +1,6 @@ import { describeStack, type StackProbe } from '../lib/stack.js'; import { SEEDED_ORDER_COUNT } from './fixtures.js'; -import type { InspectRun } from './inspect.js'; +import type { EvidenceCall, InspectRun } from './inspect.js'; import type { CliFacts, ProjectFileFacts } from './metrics.js'; import { describeMigrations, @@ -74,13 +74,31 @@ function describeRecorded(probe: RecordedProbe): string { : `partly (${describeMigrations(recorded)} recorded; ${problems})`; } -function describeInspectRuns(runs: readonly InspectRun[]): string { - if (runs.length === 0) return 'none'; - return runs - .map( - ({ commandIndex, report, errored }) => - `tool call ${commandIndex + 1}: ${report}${errored ? ' (errored)' : ''}` - ) +const UNPARSED_INSPECT = + 'a call mentioning supabase inspect that the harness could not parse into a report'; + +function describeInspectRuns( + runs: readonly InspectRun[], + evidence: readonly EvidenceCall[] +): string { + const parsed = new Set(runs.map(({ commandIndex }) => commandIndex)); + const entries = [ + ...runs.map(({ commandIndex, report, errored }) => ({ + commandIndex, + text: /[$`]/.test(report) + ? UNPARSED_INSPECT + : `${report}${errored ? ' (errored)' : ''}`, + })), + ...evidence + .filter( + ({ kind, commandIndex }) => + kind === 'candidate' && !parsed.has(commandIndex) + ) + .map(({ commandIndex }) => ({ commandIndex, text: UNPARSED_INSPECT })), + ].sort((a, b) => a.commandIndex - b.commandIndex); + if (entries.length === 0) return 'none'; + return entries + .map(({ commandIndex, text }) => `tool call ${commandIndex + 1}: ${text}`) .join('; '); } @@ -101,6 +119,7 @@ export function describeGroundTruth(facts: { migrations: IndexMigrationsProbe; recorded: RecordedProbe; inspectRuns: readonly InspectRun[]; + inspectEvidence: readonly EvidenceCall[]; files: ProjectFileFacts; cli: CliFacts; }): string[] { @@ -114,7 +133,7 @@ export function describeGroundTruth(facts: { `- rows in public.orders: ${describeRows(facts.rows)}`, `- new migrations creating an index on orders.customer_email: ${describeFoundMigrations(facts.migrations)}`, `- recorded in the local migration history: ${describeRecorded(facts.recorded)}`, - `- supabase inspect reports the agent ran: ${describeInspectRuns(facts.inspectRuns)}`, + `- supabase inspect reports the agent ran: ${describeInspectRuns(facts.inspectRuns, facts.inspectEvidence)}`, `- seeded migration modified: ${describeSeededMigration(facts.files)}`, ]; } From 0913ef2f316c5760ffe6adb5fcdbf1c051214713 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 13:33:27 +0000 Subject: [PATCH 06/10] chore: refresh eval results --- apps/web/src/data/cli-eval-results.json | 3389 +++++++++++++++++++++++ 1 file changed, 3389 insertions(+) diff --git a/apps/web/src/data/cli-eval-results.json b/apps/web/src/data/cli-eval-results.json index 84384a7b..f83257cf 100644 --- a/apps/web/src/data/cli-eval-results.json +++ b/apps/web/src/data/cli-eval-results.json @@ -7271,6 +7271,529 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-absent/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-absent", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.12", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no inspect output named the orders table" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132735_index_orders_customer_history.sql (version 20261022132735)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempts to install, start, or reconfigure a container runtime or escalate privileges." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration new\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report identifies the unindexed customer_email order-history lookup, names a matching index migration, and truthfully says local application and plan verification were blocked by the unavailable database and missing Docker/Podman." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11955 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries pg_stat_statements explain analyze Supabase Database performance\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + } + ], + "resultChars": 35692 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 209809, + "cacheReadInputTokens": 163744, + "cacheWriteInputTokens": 46041, + "outputTokens": 3674 + } + ], + "stepCount": 8, + "toolCallCount": 16, + "agentRunDurationMs": 61302, + "sandboxUsage": { + "activeCpuDurationMs": 163435, + "duration": 184432, + "memory": 8192, + "networkTransfer": { + "ingress": 835433148, + "egress": 3573175 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-absent/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-absent", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.12", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "tool call 14: supabase inspect db outliers (errored); tool call 14: supabase inspect db calls (errored); no inspect output named the orders table" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132805_add_customer_order_history_index.sql (version 20261022132805)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM, escalate privileges, or change runtime permissions." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db outliers\",\"db calls\"],\"inspectRuns\":2,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"migration new\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report identifies the slow customer order-history lookup and the targeted index migration. It accurately says the change was not applied or verified because the local database was unavailable." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging database performance slow queries pg_stat_statements indexes EXPLAIN\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + } + ], + "resultChars": 33842 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"slow query diagnose inspect database performance missing index EXPLAIN analyze CLI\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + } + ], + "resultChars": 108218 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 215289, + "cacheReadInputTokens": 167861, + "cacheWriteInputTokens": 47165, + "outputTokens": 3741 + } + ], + "stepCount": 8, + "toolCallCount": 19, + "agentRunDurationMs": 64333, + "sandboxUsage": { + "activeCpuDurationMs": 213769, + "duration": 215079, + "memory": 8192, + "networkTransfer": { + "ingress": 835372627, + "egress": 3985606 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-absent/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-absent", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.12", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132729_orders_customer_email_created_at_idx.sql (version 20261022132729)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime, escalate privileges, or change runtime permissions." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration new\",\"db query\",\"db advisors\",\"migration list\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the repeated customer_email order-history lookup, accurately describes the new index migration, and states that application and plan verification were blocked by the unavailable local database." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db pull migration local database migration list db advisors\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-migration-list", + "title": "List local and remote migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", + "title": "Pull schema from the remote database" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-push", + "title": "Push new migrations to the remote database" + } + ], + "resultChars": 47028 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging slow queries database performance pg_stat_statements missing indexes explain analyze\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + } + ], + "resultChars": 38289 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11955 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries pg_stat_statements performance database indexes\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + } + ], + "resultChars": 40381 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 307957, + "cacheReadInputTokens": 255717, + "cacheWriteInputTokens": 52210, + "outputTokens": 4849 + } + ], + "stepCount": 10, + "toolCallCount": 21, + "agentRunDurationMs": 73985, + "sandboxUsage": { + "activeCpuDurationMs": 179591, + "duration": 203067, + "memory": 8192, + "networkTransfer": { + "ingress": 835814084, + "egress": 4680600 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-absent/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-absent", "experimentSuite": "cli", @@ -14993,6 +15516,605 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-beta/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-beta", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.12", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "judgeNotes": "No recorded supabase inspect report shows public.orders or a query against it; the orders text comes from project files." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132859_add_orders_customer_history_index.sql (version 20261022132859)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261022132859_add_orders_customer_history_index.sql (version 20261022132859) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. Docker commands were read-only probes; Supabase CLI commands are permitted." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\"],\"inspectRuns\":1,\"inspectCallsNamingOrders\":1,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"db query\",\"migration list\",\"db advisors\",\"db pull\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup and describes the verified index scan, migration, and local application. Its limits on remote deployment are truthful." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging slow queries database performance pg_stat_statements index advisor\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + } + ], + "resultChars": 71345 + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking-change", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db query local database migration pull imperative\", limit: 5) { nodes { ... on CLICommandReference { title href content } ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", + "title": "Pull schema from the remote database" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-push", + "title": "Push new migrations to the remote database" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + } + ], + "resultChars": 62520 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging diagnose slow queries pg_stat_statements EXPLAIN query performance\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/debugging-performance", + "title": "Debugging performance issues" + } + ], + "resultChars": 36073 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -c 6000; printf '\\\\n--- DOCKER ---\\\\n'; docker info --format '{{.ServerVersion}}' 2>&1 || true; printf '\\\\n--- CLI HELP ---\\\\n'; supabase db query --help | head -100; printf '\\\\n--- DB PULL HELP ---\\\\n'; supabase db pull --help | head -100\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 12909 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 712825, + "cacheReadInputTokens": 641586, + "cacheWriteInputTokens": 66578, + "outputTokens": 8180 + } + ], + "stepCount": 18, + "toolCallCount": 31, + "agentRunDurationMs": 197453, + "sandboxUsage": { + "activeCpuDurationMs": 389120, + "duration": 315684, + "memory": 8192, + "networkTransfer": { + "ingress": 2814493875, + "egress": 17773894 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-beta/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-beta", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.12", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": true, + "judgeNotes": "The `supabase inspect db calls --local` output shows queries against `public.orders`; the `index-stats` and `table-stats` outputs also name that table." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132921_add_customer_order_history_index.sql (version 20261022132921)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The Docker command was a read-only probe; the stack commands used the Supabase CLI." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db help\",\"db calls\",\"db index-stats\",\"db table-stats\"],\"inspectRuns\":6,\"inspectCallsNamingOrders\":3,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"db query\",\"inspect db calls\",\"inspect db index-stats\",\"inspect db table-stats\",\"migration new\",\"db reset\",\"migration list\",\"db advisors\",\"stop\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup, names the matching index migration, and accurately describes the locally verified index scan, improved timing, and stopped stack." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase database performance slow queries local database pg_stat_statements EXPLAIN missing indexes\", limit: 6) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg", + "title": "HypoPG: Hypothetical indexes" + }, + { + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" + } + ], + "resultChars": 40417 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- ROOT ---'; ls -la; printf '%s\\\\n' '--- ANCESTOR GIT ---'; find .. -maxdepth 3 -name .git -type d -print; printf '%s\\\\n' '--- CHANGELOG ---'; curl -fsSL --max-time 15 https://supabase.com/changelog.md | head -c 6000\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6393 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase slow queries query performance missing indexes pg_stat_statements explain analyze\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + } + ], + "resultChars": 41534 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL --max-time 15 https://supabase.com/changelog.md | rg -n -i -C 2 'breaking change.*database|database.*breaking change|postgres.*breaking change' | head -100\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5496 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1374724, + "cacheReadInputTokens": 1304893, + "cacheWriteInputTokens": 69735, + "outputTokens": 7934 + } + ], + "stepCount": 32, + "toolCallCount": 30, + "agentRunDurationMs": 231050, + "sandboxUsage": { + "activeCpuDurationMs": 496362, + "duration": 360584, + "memory": 8192, + "networkTransfer": { + "ingress": 2789090068, + "egress": 14078317 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-beta/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-beta", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.12", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022133044_orders_customer_history_index.sql (version 20261022133044)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The Docker commands were read-only probes; Supabase CLI commands are permitted." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"db advisors\",\"db pull\",\"migration list\",\"stop\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the slow customer-history lookup and its sequential scan, and the transcript confirms the index, migration, local history entry, and indexed plan. The local stack was stopped afterward." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging slow queries pg_stat_statements explain analyze local database\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-check-if-my-queries-are-being-blocked-by-other-queries-NSKtR1", + "title": "How to check if my queries are being blocked by other queries?" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + } + ], + "resultChars": 35346 + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking-change", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging inspect database pg_stat_statements query performance slow query\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + } + ], + "resultChars": 33842 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL --max-time 15 https://supabase.com/changelog.md | head -80'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4177 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 763060, + "cacheReadInputTokens": 698634, + "cacheWriteInputTokens": 60145, + "outputTokens": 10332 + } + ], + "stepCount": 22, + "toolCallCount": 29, + "agentRunDurationMs": 297006, + "sandboxUsage": { + "activeCpuDurationMs": 536679, + "duration": 451050, + "memory": 8192, + "networkTransfer": { + "ingress": 1995220375, + "egress": 8799932 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-beta/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-beta", "experimentSuite": "cli", @@ -22897,6 +24019,573 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-next/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-next", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "3.0.0-next.22", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": true, + "judgeNotes": "The inspect outliers and calls reports show queries against public.orders, and inspect index-stats names public.orders directly." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Only Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132812_index_orders_customer_history.sql (version 20261022132812)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "version(s) 20261022132812 recorded with statements that do not create an index on orders.customer_email" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime, escalate privileges, or change permissions. Docker commands were read-only probes; Supabase commands were permitted." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db calls\",\"db index-stats\"],\"inspectRuns\":4,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"inspect db help\",\"status\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"migration new\",\"migration up\",\"migration list\",\"inspect db index-stats\",\"db advisors\"],\"explainCommands\":3,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching index and migration file, and accurately describes the verified index-only plan. It does not claim the migration was successfully applied through the CLI or deployed remotely." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11955 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries local database pg_stat_statements explain analyze\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + } + ], + "resultChars": 31471 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries pg_stat_statements query performance inspection\", limit: 6) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" + } + ], + "resultChars": 38529 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 781749, + "cacheReadInputTokens": 716808, + "cacheWriteInputTokens": 64881, + "outputTokens": 7588 + } + ], + "stepCount": 20, + "toolCallCount": 30, + "agentRunDurationMs": 152435, + "sandboxUsage": { + "activeCpuDurationMs": 313076, + "duration": 265722, + "memory": 8192, + "networkTransfer": { + "ingress": 2780165674, + "egress": 16872811 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-next/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-next", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "3.0.0-next.22", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": true, + "judgeNotes": "The `supabase inspect db outliers` output shows a query against public.orders, and `index-stats` names the public.orders table." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132858_orders_customer_email_created_at_idx.sql (version 20261022132858)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. Docker commands were read-only probes; Supabase commands are allowed." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db outliers\",\"db calls\",\"db index-stats\"],\"inspectRuns\":4,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"migration new\",\"migration up\",\"migration list\",\"db advisors\",\"inspect db index-stats\",\"stop\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup, the local index migration, and the verified index-scan improvement. Its claims about local application, migration history, and stopping the stack match the transcript." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"database slow queries monitoring pg_stat_statements indexes explain analyze performance local database\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + } + ], + "resultChars": 40381 + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/monitoring-and-debugging", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging" + } + ] + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking change database CLI October 2026", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring and debugging slow queries pg_stat_statements inspect database local\", limit: 4) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + } + ], + "resultChars": 25562 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL --max-time 15 https://supabase.com/changelog.md | head -c 6000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6000 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 828188, + "cacheReadInputTokens": 751138, + "cacheWriteInputTokens": 71193, + "outputTokens": 8305 + } + ], + "stepCount": 20, + "toolCallCount": 35, + "agentRunDurationMs": 179683, + "sandboxUsage": { + "activeCpuDurationMs": 455365, + "duration": 309084, + "memory": 8192, + "networkTransfer": { + "ingress": 2775625894, + "egress": 12669463 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-next/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-next", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "3.0.0-next.22", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132846_add_orders_customer_email_created_at_index.sql (version 20261022132846)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261022132846_add_orders_customer_email_created_at_index.sql (version 20261022132846) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. The Docker command was a read-only probe; stack operations used the Supabase CLI." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration list\",\"db advisors\",\"db pull\",\"db reset\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup and its sequential scans, names the matching index and migration, and accurately limits the applied fix to the local database." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5178 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging slow queries query performance index database performance pg_stat_statements\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + } + ], + "resultChars": 33842 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase local development supabase start db query --local migrations db reset seed\", limit: 5) { nodes { ... on CLICommandReference { title href content } ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-reset", + "title": "Resets the local database to current migrations" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-environment", + "title": "Development Environment" + }, + { + "url": "https://supabase.com/docs/guides/local-development", + "title": "Local Development & CLI" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" + } + ], + "resultChars": 51801 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db pull local name generate migration local schema db diff migration list\", limit: 4) { nodes { ... on CLICommandReference { title href content } ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", + "title": "Pull schema from the remote database" + } + ], + "resultChars": 61221 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1130534, + "cacheReadInputTokens": 1061635, + "cacheWriteInputTokens": 68815, + "outputTokens": 8931 + } + ], + "stepCount": 28, + "toolCallCount": 33, + "agentRunDurationMs": 214517, + "sandboxUsage": { + "activeCpuDurationMs": 393247, + "duration": 329452, + "memory": 8192, + "networkTransfer": { + "ingress": 2781008108, + "egress": 18297608 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-next/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-next", "experimentSuite": "cli", @@ -30796,6 +32485,569 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-nodaemon/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-nodaemon", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.12", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132707_add_orders_customer_history_index.sql (version 20261022132707)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempts to install, start, reconfigure, or gain access to a container runtime. The Docker command is a read-only probe, and supabase start is permitted." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"migration new\",\"db query\",\"migration list\",\"db advisors\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report identifies the slow customer-email order-history lookup, accurately describes the new index migration, and states that Docker prevented applying or verifying it locally." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11955 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring and debugging slow queries pg_stat_statements EXPLAIN query performance local database\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + } + ], + "resultChars": 30954 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow query diagnose performance explain analyze missing index performance guide\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + } + ], + "resultChars": 102720 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 260661, + "cacheReadInputTokens": 210330, + "cacheWriteInputTokens": 50304, + "outputTokens": 4934 + } + ], + "stepCount": 9, + "toolCallCount": 21, + "agentRunDurationMs": 64307, + "sandboxUsage": { + "activeCpuDurationMs": 132619, + "duration": 166183, + "memory": 8192, + "networkTransfer": { + "ingress": 839568272, + "egress": 6402914 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-nodaemon/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-nodaemon", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.12", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no inspect output named the orders table" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132718_index_orders_customer_email_created_at.sql (version 20261022132718)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No commands attempted a prohibited container-runtime detour." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"migration new\",\"db advisors\",\"db query\",\"migration list\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report identifies the slow customer-email order-history lookup, accurately describes the new index migration, and does not claim it was applied or verified. It correctly names the Docker blocker." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"database performance slow queries monitoring debugging explain analyze missing indexes\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + } + ], + "resultChars": 108218 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4177 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging database inspect slow queries pg_stat_statements explain analyze indexes\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/kb/topics/troubleshooting", + "title": "Troubleshooting" + } + ], + "resultChars": 31104 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging guide database inspect pg_stat_statements slow query analysis\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + } + ], + "resultChars": 33842 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 260819, + "cacheReadInputTokens": 210483, + "cacheWriteInputTokens": 50309, + "outputTokens": 4755 + } + ], + "stepCount": 9, + "toolCallCount": 19, + "agentRunDurationMs": 75244, + "sandboxUsage": { + "activeCpuDurationMs": 132113, + "duration": 176970, + "memory": 8192, + "networkTransfer": { + "ingress": 838708382, + "egress": 6787601 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-nodaemon/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-nodaemon", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.12", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132721_add_orders_customer_created_at_index.sql (version 20261022132721)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or escalate privileges. `docker info` was a read-only probe." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration new\",\"migration list\",\"db advisors\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report identifies the repeated customer-email order-history lookup and the matching index migration. It truthfully says the migration was saved but not applied or verified because the local database was unavailable." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking changes CLI migrations database", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Inspect database slow queries pg_stat_statements EXPLAIN local Supabase CLI migrations\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-calls", + "title": "Show queries from pg_stat_statements ordered by total times called\\\\n" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + } + ], + "resultChars": 31024 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5013 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries query performance pg_stat_statements explain analyze\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + } + ], + "resultChars": 33642 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 249173, + "cacheReadInputTokens": 192091, + "cacheWriteInputTokens": 48215, + "outputTokens": 4218 + } + ], + "stepCount": 9, + "toolCallCount": 19, + "agentRunDurationMs": 69775, + "sandboxUsage": { + "activeCpuDurationMs": 123116, + "duration": 168345, + "memory": 8192, + "networkTransfer": { + "ingress": 838521804, + "egress": 6248159 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-nodaemon/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-nodaemon", "experimentSuite": "cli", @@ -38936,6 +41188,587 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-pinned/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-pinned", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.117.0", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132850_add_orders_customer_email_created_at_idx.sql (version 20261022132850)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261022132850_add_orders_customer_email_created_at_idx.sql (version 20261022132850) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM. The Docker command was a read-only probe." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"db advisors\",\"db pull\",\"migration list\",\"db diff\"],\"explainCommands\":2,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup and accurately describes the index, migration, local history, and verified index-scan plan." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db query local database migration db diff\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-diff", + "title": "Diffs the local database for schema changes" + } + ], + "resultChars": 72162 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"slow queries performance monitoring pg_stat_statements explain analyze database advisors\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + } + ], + "resultChars": 79826 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and debugging slow queries pg_stat_statements query performance explain analyze\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/kb/topics/database", + "title": "Database" + } + ], + "resultChars": 36547 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- directory ---'; ls -la; printf '%s\\\\n' '--- git roots ---'; find .. -maxdepth 3 -name .git -type d -print; printf '%s\\\\n' '--- project files ---'; rg --files | head -40; printf '%s\\\\n' '--- changelog ---'; curl -fsSL https://supabase.com/changelog.md | head -100\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5704 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1215194, + "cacheReadInputTokens": 1146720, + "cacheWriteInputTokens": 67943, + "outputTokens": 9137 + } + ], + "stepCount": 28, + "toolCallCount": 35, + "agentRunDurationMs": 212079, + "sandboxUsage": { + "activeCpuDurationMs": 380631, + "duration": 325658, + "memory": 8192, + "networkTransfer": { + "ingress": 2861854414, + "egress": 16848002 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-pinned/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-pinned", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.117.0", + "passed": true, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": true, + "judgeNotes": "The `supabase inspect db outliers --local` output shows a query against public.orders, and the inspect index-stats and table-stats outputs name public.orders." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132941_add_orders_customer_history_index.sql (version 20261022132941)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261022132941_add_orders_customer_history_index.sql (version 20261022132941) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM. Docker commands were read-only probes; Supabase CLI commands are permitted." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db index-stats\",\"db table-stats\"],\"inspectRuns\":7,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"inspect db outliers\",\"inspect db index-stats\",\"inspect db table-stats\",\"db query\",\"migration new\",\"db reset\",\"migration list\",\"db advisors\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup as the cause and accurately describes the migration, local application, and verified index-scan improvement." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/monitoring-and-debugging slow queries Supabase query performance local database", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Inspect database CLI query performance slow queries pg_stat_statements outliers local\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-calls", + "title": "Show queries from pg_stat_statements ordered by total times called\\\\n" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-outliers", + "title": "Show queries from pg_stat_statements ordered by total execution time\\\\n" + } + ], + "resultChars": 34642 + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md breaking-change Supabase changelog 2026", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/monitoring-and-debugging", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging" + } + ] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI inspect db outliers index-stats local command flags db query explain analyze\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-report", + "title": "Generate a CSV output for all inspect commands" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-db-stats", + "title": "Show stats such as cache hit rates, total sizes, and WAL size\\\\n" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db", + "title": "Tools to inspect your Supabase database" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + } + ], + "resultChars": 17329 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -c 1200; printf '\\\\n--- migration file ---\\\\n'; nl -ba supabase/migrations/20261022132941_add_orders_customer_history_index.sql\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 1382 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | rg -n -i -C 2 'slow|query|inspect|performance|outlier' | head -80\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 2425 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1334627, + "cacheReadInputTokens": 1252792, + "cacheWriteInputTokens": 64321, + "outputTokens": 9190 + } + ], + "stepCount": 36, + "toolCallCount": 21, + "agentRunDurationMs": 255149, + "sandboxUsage": { + "activeCpuDurationMs": 547854, + "duration": 395675, + "memory": 8192, + "networkTransfer": { + "ingress": 2840425126, + "egress": 13295830 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-pinned/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-pinned", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.117.0", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Only Scan using orders_customer_history_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132843_add_orders_customer_history_index.sql (version 20261022132843)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261022132843_add_orders_customer_history_index.sql (version 20261022132843) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted a container-runtime detour. The agent used permitted Supabase CLI and database commands." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"db advisors\",\"db pull\",\"migration list\",\"db diff\"],\"explainCommands\":3,\"directIndexDdl\":2,\"ordersIndexes\":[\"orders_customer_history_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer order-history lookup as the cause. The named migration and index exist, are recorded locally, and the verified plan uses an index-only scan. Its account of the unrelated pg_net diff is also accurate." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"database slow queries performance indexes EXPLAIN monitoring pg_stat_statements\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + } + ], + "resultChars": 36452 + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking-change", + "pages": [] + }, + { + "source": "web_search", + "query": "https://supabase.com/docs/guides/monitoring-and-debugging", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging" + } + ] + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 885947, + "cacheReadInputTokens": 816990, + "cacheWriteInputTokens": 59158, + "outputTokens": 10494 + } + ], + "stepCount": 26, + "toolCallCount": 29, + "agentRunDurationMs": 236825, + "sandboxUsage": { + "activeCpuDurationMs": 414105, + "duration": 360148, + "memory": 8192, + "networkTransfer": { + "ingress": 2860341620, + "egress": 13031858 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-pinned/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-pinned", "experimentSuite": "cli", @@ -46752,6 +49585,562 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-stable/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-stable", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.121.0", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132921_add_orders_customer_history_index.sql (version 20261022132921)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261022132921_add_orders_customer_history_index.sql (version 20261022132921) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No commands attempted to install, start, or reconfigure a container runtime or VM. Docker commands were read-only probes; stack operations used the Supabase CLI." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration new\",\"migration up\",\"db reset\",\"migration list\",\"db advisors\"],\"explainCommands\":3,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration and index, and accurately describes local application and index-scan verification." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging slow queries explain analyze missing indexes pg_stat_statements\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-check-if-my-queries-are-being-blocked-by-other-queries-NSKtR1", + "title": "How to check if my queries are being blocked by other queries?" + } + ], + "resultChars": 35346 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -c 12000; printf '\\\\n--- monitoring docs fetch ---\\\\n'; curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 12000\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + }, + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 3532 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"CREATE INDEX CONCURRENTLY migration db push transaction Supabase\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mysql", + "title": "Migrate from MySQL to Supabase" + } + ], + "resultChars": 36745 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/database/query-optimization indexes composite index query plan analyze", + "pages": [] + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1292598, + "cacheReadInputTokens": 1220545, + "cacheWriteInputTokens": 67603, + "outputTokens": 10738 + } + ], + "stepCount": 34, + "toolCallCount": 33, + "agentRunDurationMs": 272795, + "sandboxUsage": { + "activeCpuDurationMs": 447662, + "duration": 397921, + "memory": 8192, + "networkTransfer": { + "ingress": 2737162824, + "egress": 16732055 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-stable/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-stable", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.121.0", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132810_add_orders_customer_email_created_at_index.sql (version 20261022132810)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261022132810_add_orders_customer_email_created_at_index.sql (version 20261022132810) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No commands attempted to install, start, or reconfigure a container runtime or VM, or escalate privileges to make one work." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"db advisors\",\"migration new\",\"migration up\",\"migration list\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup as the cause and accurately describes the locally applied migration, index, and verified index-scan plan." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"slow queries monitoring debugging pg_stat_statements explain analyze index performance\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + } + ], + "resultChars": 36452 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Monitoring and Debugging slow queries local database pg_stat_statements explain analyze\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + } + ], + "resultChars": 41190 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 559597, + "cacheReadInputTokens": 496464, + "cacheWriteInputTokens": 62767, + "outputTokens": 5757 + } + ], + "stepCount": 15, + "toolCallCount": 25, + "agentRunDurationMs": 128530, + "sandboxUsage": { + "activeCpuDurationMs": 318852, + "duration": 237468, + "memory": 8192, + "networkTransfer": { + "ingress": 2732430695, + "egress": 11516104 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-stable/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-stable", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.121.0", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261022132910_add_orders_customer_history_index.sql (version 20261022132910)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261022132910_add_orders_customer_history_index.sql (version 20261022132910) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM. The Docker command was a read-only probe; Supabase CLI commands are permitted." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"start\",\"migration list\",\"migration new\",\"db advisors\",\"migration up\"],\"explainCommands\":3,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the slow customer-email order-history scans and accurately describes the locally applied index migration and verified index-scan plan." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging slow queries performance pg_stat_statements EXPLAIN missing indexes local database\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + } + ], + "resultChars": 28570 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase local development migration db pull generate migration imperative local schema changes\", limit: 4) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging database slow queries inspect pg_stat_activity explain analyze Supabase\", limit: 4) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + } + ], + "resultChars": 37373 + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog Supabase changelog breaking change", + "pages": [] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -sS -L --max-time 20 https://supabase.com/changelog.md | head -c 5000'", + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5013 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 865389, + "cacheReadInputTokens": 799625, + "cacheWriteInputTokens": 60941, + "outputTokens": 7458 + } + ], + "stepCount": 24, + "toolCallCount": 38, + "agentRunDurationMs": 165658, + "sandboxUsage": { + "activeCpuDurationMs": 411923, + "duration": 334587, + "memory": 8192, + "networkTransfer": { + "ingress": 2004978773, + "egress": 9149348 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-stable/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-stable", "experimentSuite": "cli", From b00e50cecba625a23703f3286a9d5551f2565c71 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Fri, 23 Oct 2026 11:31:48 +0100 Subject: [PATCH 07/10] docs(evals): state that investigate-database-001-slow-query requires inspect on purpose (CLI-2712) --- .../README.md | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/evals/cli/investigate-database-001-slow-query/README.md b/evals/cli/investigate-database-001-slow-query/README.md index b58d25a8..462ab462 100644 --- a/evals/cli/investigate-database-001-slow-query/README.md +++ b/evals/cli/investigate-database-001-slow-query/README.md @@ -24,9 +24,9 @@ order-history lookup filters `orders` by `customer_email`, which has no index, so each call scans all 300,000 rows. No stack is running when the agent starts, so it has to start one, find the lookup with `supabase inspect`, add an index on `orders.customer_email` in a new migration, and apply that migration to the -local database. The scorer checks the end state, so any route that ends with -the index live, in a new migration file and in the migration history is scored -the same way. +local database. Apart from the inspect check, the scorer checks the end state, +so any route that ends with the index live, in a new migration file and in the +migration history scores the same. Motivation: CLI-2712 (https://linear.app/supabase/issue/CLI-2712/eval-diagnose-a-slow-query-bloat-or-locks-with-inspect-db), @@ -45,6 +45,25 @@ it appears in 0.5% of agent sessions or fewer in the 1% device sample, Sep 15 to Oct 14 2026 (`07-posthog-workflows.md`). Agents drive almost all of its use, but it is a thin area. +## Requiring `inspect` is intentional + +The prompt asks only for "the Supabase CLI", but `an inspect db report showed +the orders table` passes only when a `supabase inspect` report showed `orders`. +That is the point of CLI-2712: the eval measures whether an agent finds +`inspect` on its own when the user doesn't name it. An agent that finds the +lookup another way, such as `supabase db query` against `pg_stat_statements` +plus `EXPLAIN`, or `psql`, and ships a correct fix fails that check and no +other. The rest of the checks score the fix the same however it was found. + +The first refresh on this eval's PR +([supabase/evals#425](https://github.com/supabase/evals/pull/425), October 22 +2026) showed this is the gap. 8 of the 12 Docker runs failed the inspect check, +and every one of them found the right cause: 7 never ran a `supabase inspect` +report and used `db query` plus `EXPLAIN` instead, and one ran only `inspect db +help`. The bundled `supabase` skill points agents at `supabase db query` and +never mentions `supabase inspect`. Don't relax the check to accept other +diagnosis paths: a run that skips `inspect` is the signal this eval records. + ## Area and journey phase - `area: diagnostics` From a3d54e8dc1689458df624f4a30beacb66d1a4e8e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:43:23 +0000 Subject: [PATCH 08/10] chore: refresh eval results --- apps/web/src/data/cli-eval-results.json | 3380 +++++++++++++++-------- 1 file changed, 2296 insertions(+), 1084 deletions(-) diff --git a/apps/web/src/data/cli-eval-results.json b/apps/web/src/data/cli-eval-results.json index f83257cf..99030dbf 100644 --- a/apps/web/src/data/cli-eval-results.json +++ b/apps/web/src/data/cli-eval-results.json @@ -7291,7 +7291,7 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.122.0-beta.12", + "cliVersion": "2.122.0-beta.15", "passed": false, "checks": [ { @@ -7302,7 +7302,7 @@ { "name": "an inspect db report showed the orders table", "passed": false, - "notes": "no inspect output named the orders table" + "notes": "no supabase inspect report ran" }, { "name": "the order-history lookup uses an index on customer_email", @@ -7317,7 +7317,7 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132735_index_orders_customer_history.sql (version 20261022132735)" + "notes": "20261023000000_index_orders_customer_history.sql (version 20261023000000)" }, { "name": "that migration is recorded in the local migration history", @@ -7327,17 +7327,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempts to install, start, or reconfigure a container runtime or escalate privileges." + "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime or VM." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration new\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration list\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report identifies the unindexed customer_email order-history lookup, names a matching index migration, and truthfully says local application and plan verification were blocked by the unavailable database and missing Docker/Podman." + "judgeNotes": "The report identifies the slow customer-order lookup, describes the matching index migration, and accurately says the migration was not applied or verified because the local database was unavailable." } ], "skills": { @@ -7352,62 +7352,122 @@ }, "docs": { "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db query local database migration db advisors\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring and debugging slow queries database performance pg_stat_statements local database explain analyze\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + } + ], + "resultChars": 36452 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db query migrations local database db advisors\", limit: 4) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, { "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 11955 + "resultChars": 4082 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries pg_stat_statements explain analyze Supabase Database performance\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries database performance\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" } ], - "resultChars": 35692 + "resultChars": 96655 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db query command reference local database\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-start", + "title": "Starts local Postgres database" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", + "title": "Testing and linting" + } + ], + "resultChars": 27604 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 209809, - "cacheReadInputTokens": 163744, - "cacheWriteInputTokens": 46041, - "outputTokens": 3674 + "inputTokens": 157807, + "cacheReadInputTokens": 114930, + "cacheWriteInputTokens": 42856, + "outputTokens": 3124 } ], - "stepCount": 8, - "toolCallCount": 16, - "agentRunDurationMs": 61302, + "stepCount": 7, + "toolCallCount": 14, + "agentRunDurationMs": 55183, "sandboxUsage": { - "activeCpuDurationMs": 163435, - "duration": 184432, + "activeCpuDurationMs": 243979, + "duration": 228671, "memory": 8192, "networkTransfer": { - "ingress": 835433148, - "egress": 3573175 + "ingress": 835252904, + "egress": 3598192 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -7435,7 +7495,7 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.122.0-beta.12", + "cliVersion": "2.122.0-beta.15", "passed": false, "checks": [ { @@ -7446,7 +7506,7 @@ { "name": "an inspect db report showed the orders table", "passed": false, - "notes": "tool call 14: supabase inspect db outliers (errored); tool call 14: supabase inspect db calls (errored); no inspect output named the orders table" + "notes": "no inspect output named the orders table" }, { "name": "the order-history lookup uses an index on customer_email", @@ -7461,7 +7521,7 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132805_add_customer_order_history_index.sql (version 20261022132805)" + "notes": "20261023103626_index_orders_customer_history.sql (version 20261023103626)" }, { "name": "that migration is recorded in the local migration history", @@ -7471,17 +7531,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM, escalate privileges, or change runtime permissions." + "judgeNotes": "No command attempted to install, start, reconfigure, or gain privileged access to a container runtime or VM." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db outliers\",\"db calls\"],\"inspectRuns\":2,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"migration new\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration new\",\"db query\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report identifies the slow customer order-history lookup and the targeted index migration. It accurately says the change was not applied or verified because the local database was unavailable." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration and index, and states that the fix was saved but could not be verified or applied locally because Docker/Podman was unavailable." } ], "skills": { @@ -7496,93 +7556,90 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -120'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6220 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging database performance slow queries pg_stat_statements indexes EXPLAIN\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries database performance pg_stat_statements Supabase local database query plans\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" } ], - "resultChars": 33842 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] + "resultChars": 40672 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"slow query diagnose inspect database performance missing index EXPLAIN analyze CLI\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Query optimization slow queries pg_stat_statements explain analyze indexes\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, - { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" - }, - { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" - }, { "url": "https://supabase.com/docs/guides/database/query-optimization", "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" } ], - "resultChars": 108218 + "resultChars": 33642 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 215289, - "cacheReadInputTokens": 167861, - "cacheWriteInputTokens": 47165, - "outputTokens": 3741 + "inputTokens": 179661, + "cacheReadInputTokens": 133859, + "cacheWriteInputTokens": 45781, + "outputTokens": 3136 } ], - "stepCount": 8, - "toolCallCount": 19, - "agentRunDurationMs": 64333, + "stepCount": 7, + "toolCallCount": 16, + "agentRunDurationMs": 68157, "sandboxUsage": { - "activeCpuDurationMs": 213769, - "duration": 215079, + "activeCpuDurationMs": 245723, + "duration": 290997, "memory": 8192, "networkTransfer": { - "ingress": 835372627, - "egress": 3985606 + "ingress": 834853150, + "egress": 3177890 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -7610,7 +7667,7 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.122.0-beta.12", + "cliVersion": "2.122.0-beta.15", "passed": false, "checks": [ { @@ -7621,7 +7678,7 @@ { "name": "an inspect db report showed the orders table", "passed": false, - "notes": "no supabase inspect report ran" + "notes": "no inspect output named the orders table" }, { "name": "the order-history lookup uses an index on customer_email", @@ -7636,7 +7693,7 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132729_orders_customer_email_created_at_idx.sql (version 20261022132729)" + "notes": "20261023103531_index_orders_customer_history.sql (version 20261023103531)" }, { "name": "that migration is recorded in the local migration history", @@ -7646,17 +7703,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime, escalate privileges, or change runtime permissions." + "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. The Supabase CLI commands and read-only probes are allowed." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration new\",\"db query\",\"db advisors\",\"migration list\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration new\",\"db query\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the repeated customer_email order-history lookup, accurately describes the new index migration, and states that application and plan verification were blocked by the unavailable local database." + "judgeNotes": "The report correctly identifies the unindexed customer_email order-history lookup, names the new migration, and accurately states that local application and plan verification were blocked by the unavailable database and missing Docker/Podman." } ], "skills": { @@ -7672,121 +7729,93 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db pull migration local database migration list db advisors\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-list", - "title": "List local and remote migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 47028 + "resultChars": 4978 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging slow queries database performance pg_stat_statements missing indexes explain analyze\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "{ searchDocs(query: \"Monitoring and Debugging slow queries pg_stat_statements EXPLAIN index performance\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" - }, { "url": "https://supabase.com/docs/guides/database/postgres/configuration", "title": "Database configuration" - } - ], - "resultChars": 38289 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", - "hasContent": true, - "pages": [ + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" } ], - "resultChars": 11955 + "resultChars": 33842 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries pg_stat_statements performance database indexes\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "{ searchDocs(query: \"Postgres composite indexes order by limit index-only scan INCLUDE columns\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" - }, { "url": "https://supabase.com/docs/guides/database/query-optimization", "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/troubleshooting/how-postgres-chooses-which-index-to-use-_JHrf4", + "title": "How Postgres chooses which index to use" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", + "title": "HNSW indexes" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/database/orioledb", + "title": "OrioleDB Overview" } ], - "resultChars": 40381 + "resultChars": 32766 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 307957, - "cacheReadInputTokens": 255717, - "cacheWriteInputTokens": 52210, - "outputTokens": 4849 + "inputTokens": 175651, + "cacheReadInputTokens": 130022, + "cacheWriteInputTokens": 45608, + "outputTokens": 4635 } ], - "stepCount": 10, - "toolCallCount": 21, - "agentRunDurationMs": 73985, + "stepCount": 7, + "toolCallCount": 14, + "agentRunDurationMs": 68807, "sandboxUsage": { - "activeCpuDurationMs": 179591, - "duration": 203067, + "activeCpuDurationMs": 226733, + "duration": 238945, "memory": 8192, "networkTransfer": { - "ingress": 835814084, - "egress": 4680600 + "ingress": 836231691, + "egress": 3747497 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -15536,53 +15565,53 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.122.0-beta.12", + "cliVersion": "2.122.0-beta.15", "passed": false, "checks": [ { "name": "local stack is running for this project", - "passed": true, - "notes": "legacy (docker), select 1 ok" + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." }, { "name": "an inspect db report showed the orders table", "passed": false, - "judgeNotes": "No recorded supabase inspect report shows public.orders or a query against it; the orders text comes from project files." + "notes": "no inspect output named the orders table" }, { "name": "the order-history lookup uses an index on customer_email", - "passed": true, - "notes": "Index Scan using orders_customer_email_created_at_idx" + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." }, { "name": "orders still holds every seeded row", - "passed": true, - "notes": "300000 rows" + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." }, { "name": "a new migration creates an index on orders.customer_email", - "passed": true, - "notes": "20261022132859_add_orders_customer_history_index.sql (version 20261022132859)" + "passed": false, + "notes": "no new migration files" }, { "name": "that migration is recorded in the local migration history", - "passed": true, - "notes": "20261022132859_add_orders_customer_history_index.sql (version 20261022132859) is in supabase_migrations.schema_migrations with the index statement" + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. Docker commands were read-only probes; Supabase CLI commands are permitted." + "judgeNotes": "No command attempts to install, start, or reconfigure a container runtime. The Docker command is a read-only probe." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\"],\"inspectRuns\":1,\"inspectCallsNamingOrders\":1,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"db query\",\"migration list\",\"db advisors\",\"db pull\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":0,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup and describes the verified index scan, migration, and local application. Its limits on remote deployment are truthful." + "judgeNotes": "The report identifies the customer_email order-history lookup as the suspected bottleneck, makes no unsupported fix claims, and truthfully says it stopped because the local DB container was missing. No migration was created." } ], "skills": { @@ -15598,135 +15627,93 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging slow queries database performance pg_stat_statements index advisor\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" - } - ], - "resultChars": 71345 - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog breaking-change", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } - ] + ], + "resultChars": 4978 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db query local database migration pull imperative\", limit: 5) { nodes { ... on CLICommandReference { title href content } ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"database slow queries performance diagnostics pg_stat_statements local database EXPLAIN query monitoring\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-db-push", - "title": "Push new migrations to the remote database" + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" }, { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" } ], - "resultChars": 62520 + "resultChars": 36452 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging diagnose slow queries pg_stat_statements EXPLAIN query performance\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries database performance pg_stat_statements explain analyze missing indexes\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/debugging-performance", - "title": "Debugging performance issues" - } - ], - "resultChars": 36073 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -c 6000; printf '\\\\n--- DOCKER ---\\\\n'; docker info --format '{{.ServerVersion}}' 2>&1 || true; printf '\\\\n--- CLI HELP ---\\\\n'; supabase db query --help | head -100; printf '\\\\n--- DB PULL HELP ---\\\\n'; supabase db pull --help | head -100\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" } ], - "resultChars": 12909 + "resultChars": 38289 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 712825, - "cacheReadInputTokens": 641586, - "cacheWriteInputTokens": 66578, - "outputTokens": 8180 + "inputTokens": 109010, + "cacheReadInputTokens": 65107, + "cacheWriteInputTokens": 43888, + "outputTokens": 2605 } ], - "stepCount": 18, - "toolCallCount": 31, - "agentRunDurationMs": 197453, + "stepCount": 5, + "toolCallCount": 12, + "agentRunDurationMs": 47909, "sandboxUsage": { - "activeCpuDurationMs": 389120, - "duration": 315684, + "activeCpuDurationMs": 243513, + "duration": 279850, "memory": 8192, "networkTransfer": { - "ingress": 2814493875, - "egress": 17773894 + "ingress": 834740396, + "egress": 2963894 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -15754,7 +15741,7 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.122.0-beta.12", + "cliVersion": "2.122.0-beta.15", "passed": false, "checks": [ { @@ -15764,8 +15751,8 @@ }, { "name": "an inspect db report showed the orders table", - "passed": true, - "judgeNotes": "The `supabase inspect db calls --local` output shows queries against `public.orders`; the `index-stats` and `table-stats` outputs also name that table." + "passed": false, + "notes": "no supabase inspect report ran" }, { "name": "the order-history lookup uses an index on customer_email", @@ -15779,8 +15766,8 @@ }, { "name": "a new migration creates an index on orders.customer_email", - "passed": true, - "notes": "20261022132921_add_customer_order_history_index.sql (version 20261022132921)" + "passed": false, + "notes": "no new migration files" }, { "name": "that migration is recorded in the local migration history", @@ -15790,17 +15777,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The Docker command was a read-only probe; the stack commands used the Supabase CLI." + "judgeNotes": "No command attempts to install, start, or reconfigure a container runtime, escalate privileges, or change runtime permissions." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db help\",\"db calls\",\"db index-stats\",\"db table-stats\"],\"inspectRuns\":6,\"inspectCallsNamingOrders\":3,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"db query\",\"inspect db calls\",\"inspect db index-stats\",\"inspect db table-stats\",\"migration new\",\"db reset\",\"migration list\",\"db advisors\",\"stop\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":0,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup, names the matching index migration, and accurately describes the locally verified index scan, improved timing, and stopped stack." + "judgeNotes": "The report correctly says the agent stopped after `supabase status` found no local database container, made no changes, and treated the missing customer_email index as an unverified lead rather than claiming a fix." } ], "skills": { @@ -15816,50 +15803,47 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase database performance slow queries local database pg_stat_statements EXPLAIN missing indexes\", limit: 6) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11952 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries database performance query analysis local database\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", - "title": "Steps to improve query performance with indexes" + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" }, { - "url": "https://supabase.com/docs/guides/database/extensions/hypopg", - "title": "HypoPG: Hypothetical indexes" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/kb/sitemap", - "title": "Supabase Knowledge Base" - } - ], - "resultChars": 40417 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- ROOT ---'; ls -la; printf '%s\\\\n' '--- ANCESTOR GIT ---'; find .. -maxdepth 3 -name .git -type d -print; printf '%s\\\\n' '--- CHANGELOG ---'; curl -fsSL --max-time 15 https://supabase.com/changelog.md | head -c 6000\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" } ], - "resultChars": 6393 + "resultChars": 59088 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase slow queries query performance missing indexes pg_stat_statements explain analyze\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Debugging and monitoring Supabase slow query performance pg_stat_statements EXPLAIN analyze\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -15867,56 +15851,41 @@ "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", - "title": "Steps to improve query performance with indexes" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" }, { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" - } - ], - "resultChars": 41534 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL --max-time 15 https://supabase.com/changelog.md | rg -n -i -C 2 'breaking change.*database|database.*breaking change|postgres.*breaking change' | head -100\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" } ], - "resultChars": 5496 + "resultChars": 24466 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 1374724, - "cacheReadInputTokens": 1304893, - "cacheWriteInputTokens": 69735, - "outputTokens": 7934 + "inputTokens": 85589, + "cacheReadInputTokens": 42197, + "cacheWriteInputTokens": 43380, + "outputTokens": 2588 } ], - "stepCount": 32, - "toolCallCount": 30, - "agentRunDurationMs": 231050, + "stepCount": 4, + "toolCallCount": 11, + "agentRunDurationMs": 50430, "sandboxUsage": { - "activeCpuDurationMs": 496362, - "duration": 360584, + "activeCpuDurationMs": 240746, + "duration": 272223, "memory": 8192, "networkTransfer": { - "ingress": 2789090068, - "egress": 14078317 + "ingress": 835542545, + "egress": 2989521 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -15944,53 +15913,53 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.122.0-beta.12", - "passed": false, + "cliVersion": "2.122.0-beta.15", + "passed": true, "checks": [ { "name": "local stack is running for this project", - "passed": false, - "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "legacy (docker), select 1 ok" }, { "name": "an inspect db report showed the orders table", - "passed": false, - "notes": "no supabase inspect report ran" + "passed": true, + "judgeNotes": "Supabase inspect output names public.orders in table-stats and index-stats, and shows the order-history query in calls and outliers." }, { "name": "the order-history lookup uses an index on customer_email", - "passed": false, - "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "Index Only Scan using orders_customer_history_idx" }, { "name": "orders still holds every seeded row", - "passed": false, - "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "300000 rows" }, { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022133044_orders_customer_history_index.sql (version 20261022133044)" + "notes": "20261023103913_add_orders_customer_history_index.sql (version 20261023103913)" }, { "name": "that migration is recorded in the local migration history", - "passed": false, - "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "20261023103913_add_orders_customer_history_index.sql (version 20261023103913) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The Docker commands were read-only probes; Supabase CLI commands are permitted." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime, escalate privileges, or change socket permissions. Docker commands were read-only probes; Supabase CLI commands are permitted." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"db advisors\",\"db pull\",\"migration list\",\"stop\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db table-stats\",\"db calls\",\"db outliers\",\"db index-stats\"],\"inspectRuns\":7,\"inspectCallsNamingOrders\":5,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"inspect db table-stats\",\"inspect db calls\",\"inspect db outliers\",\"inspect db index-stats\",\"db query\",\"migration new\",\"db reset\",\"migration list\",\"db advisors\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_history_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the slow customer-history lookup and its sequential scan, and the transcript confirms the index, migration, local history entry, and indexed plan. The local stack was stopped afterward." + "judgeNotes": "The report correctly identifies the slow order-history lookups and accurately describes the indexed migration, local application, and verified index-only plan." } ], "skills": { @@ -15998,116 +15967,625 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] + "loaded": [] }, "docs": { "calls": [ { "source": "web_search", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] + "query": "site:supabase.com/docs/guides/monitoring-and-debugging slow queries pg_stat_statements explain analyze", + "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging slow queries pg_stat_statements explain analyze local database\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"slow queries performance monitoring pg_stat_statements explain analyze missing index inspect database local CLI\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#using-sql", + "title": "Using SQL" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#postgres-cumulative-statistics-system", + "title": "Postgres cumulative statistics system" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#most-frequently-called-queries", + "title": "Most frequently called queries" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#slowest-queries-by-execution-time", + "title": "Slowest queries by execution time" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#most-time-consuming-queries", + "title": "Most time consuming queries" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#hit-rate", + "title": "Hit rate" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#using-the-cli", + "title": "Using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#the-inspect-db-command", + "title": "The inspect db command" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-any-postgres-database", + "title": "Connect to any Postgres database" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-a-supabase-instance", + "title": "Connect to a Supabase instance" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#inspection-commands", + "title": "Inspection commands" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#disk-storage", + "title": "Disk storage" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#query-performance", + "title": "Query performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#locks", + "title": "Locks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connections", + "title": "Connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#notes-on-pg_stat_statements", + "title": "Notes on pg_stat_statements" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-long-running-sessions", + "title": "Find long-running sessions" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#performance", + "title": "Performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-public-schema-tables-without-rls", + "title": "Find public-schema tables without RLS" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-affected-paths-and-methods", + "title": "Find affected paths and methods" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#measure-authorization-failures", + "title": "Measure authorization failures" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#security", + "title": "Security" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#check-postgres-connection-pressure", + "title": "Check Postgres connection pressure" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-failing-api-paths", + "title": "Find failing API paths" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#measure-api-server-error-rate", + "title": "Measure API server-error rate" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#health", + "title": "Health" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#count-connections-by-role-and-state", + "title": "Count connections by role and state" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#turn-a-detection-into-a-diagnosis", + "title": "Turn a detection into a diagnosis" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-the-largest-relations", + "title": "Find the largest relations" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-high-volume-api-paths", + "title": "Find high-volume API paths" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#trend-api-requests", + "title": "Trend API requests" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#usage", + "title": "Usage" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#measure-shared-buffer-hit-rate", + "title": "Measure shared-buffer hit rate" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-expensive-query-patterns", + "title": "Find expensive query patterns" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-blocked-sessions", + "title": "Find blocked sessions" + }, { "url": "https://supabase.com/docs/guides/database/query-optimization", "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-check-if-my-queries-are-being-blocked-by-other-queries-NSKtR1", - "title": "How to check if my queries are being blocked by other queries?" + "url": "https://supabase.com/docs/guides/database/query-optimization#join-columns", + "title": "join columns" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#example-query", + "title": "Example query" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#partial-indexes", + "title": "Partial indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#use-appropriate-index-types", + "title": "Use appropriate index types" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#analyze-the-query-plan", + "title": "Analyze the query plan" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#key-concepts", + "title": "Key concepts" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#order-by-clause", + "title": "order by clause" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#where-clause", + "title": "where clause:" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#statistics", + "title": "Statistics" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#over-indexing", + "title": "Over-Indexing" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#composite-indexes", + "title": "Composite indexes" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#examining-query-performance", + "title": "Examining query performance" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#optimizing-the-number-of-connections", + "title": "Optimizing the number of connections" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#configuring-clients-to-use-fewer-connections", + "title": "Configuring clients to use fewer connections" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#allowing-higher-number-of-connections", + "title": "Allowing higher number of connections" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#enterprise", + "title": "Enterprise" }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#inspecting-activity", + "title": "Inspecting activity" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#enable-the-extension", + "title": "Enable the extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#resources", + "title": "Resources" } ], - "resultChars": 35346 + "resultChars": 138210 }, { "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog breaking-change", - "pages": [] + "query": "https://supabase.com/docs/guides/observability/inspect", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect" + } + ] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging inspect database pg_stat_statements query performance slow query\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI local migration create db reset migration workflow db query --local\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-supabase-directory", + "title": "The ./supabase directory" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#move-an-existing-project-to-local-development", + "title": "Move an existing project to local development" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize", + "title": "Step 1: Initialize" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-authenticate", + "title": "Step 2: Authenticate" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-link-to-your-remote-project", + "title": "Step 3: Link to your remote project" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-pull-the-remote-schema", + "title": "Step 4: Pull the remote schema" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-create-seed-data", + "title": "Step 5: Create seed data" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-verify", + "title": "Step 6: Verify" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-7-commit", + "title": "Step 7: Commit" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#start-a-new-project-from-scratch", + "title": "Start a new project from scratch" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize-1", + "title": "Step 1: Initialize" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-start-the-local-stack", + "title": "Step 2: Start the local stack" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-create-your-schema", + "title": "Step 3: Create your schema" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-add-seed-data", + "title": "Step 4: Add seed data" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-verify", + "title": "Step 5: Verify" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-commit", + "title": "Step 6: Commit" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-daily-workflow", + "title": "The daily workflow" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#making-schema-changes", + "title": "Making schema changes" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#generating-types", + "title": "Generating types" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#staying-in-sync-with-your-team", + "title": "Staying in sync with your team" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#pushing-to-a-remote-project", + "title": "Pushing to a remote project" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#resetting-a-remote-dev-or-staging-project", + "title": "Resetting a remote dev or staging project" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#key-commands-at-a-glance", + "title": "Key commands at a glance" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#cleaning-up-generated-migrations", + "title": "Cleaning up generated migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#grants", + "title": "Grants" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#revokere-grant-patterns", + "title": "Revoke/re-grant patterns" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#extension-statements", + "title": "Extension statements" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#known-limitations-of-db-diff", + "title": "Known limitations of db diff" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-pull", + "title": "Permission denied on db pull" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-push", + "title": "Permission denied on db push" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#rebasing-new-migrations", + "title": "Rebasing new migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#configure-github-actions", + "title": "Configure GitHub Actions" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#set-up-a-local-environment", + "title": "Set up a local environment" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#create-a-new-migration", + "title": "Create a new migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#manual-migration", + "title": "Manual migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#auto-schema-diff", + "title": "Auto schema diff" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#deploy-a-migration", + "title": "Deploy a migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#open-a-pr-with-new-migration", + "title": "Open a PR with new migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#release-to-production", + "title": "Release to production" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#sync-production-project-to-staging", + "title": "Sync production project to staging" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#getting-help", + "title": "Getting help" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#monitoring-deployments", + "title": "Monitoring deployments" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#common-issues", + "title": "Common issues" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#rolling-back-migrations", + "title": "Rolling back migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#deployment-failures", + "title": "Deployment failures" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#schema-drift-between-preview-branches", + "title": "Schema drift between preview branches" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#changing-production-branch", + "title": "Changing production branch" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#migration-issues", + "title": "Migration issues" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#failed-migrations", + "title": "Failed migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#migration-order-problems", + "title": "Migration order problems" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#connection-issues", + "title": "Connection issues" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#cannot-connect-to-preview-branch", + "title": "Cannot connect to preview branch" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#connection-timeouts", + "title": "Connection timeouts" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#configuration-problems", + "title": "Configuration problems" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#configtoml-not-applying", + "title": "Config.toml not applying" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#secrets-not-available", + "title": "Secrets not available" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#performance-issues", + "title": "Performance issues" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#slow-branch-creation", + "title": "Slow branch creation" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#query-performance", + "title": "Query performance" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#data-issues", + "title": "Data issues" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#seed-data-not-loading", + "title": "Seed data not loading" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#data-persistence", + "title": "Data persistence" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#working-with-a-team", + "title": "Working with a team" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#diagnosing-and-fixing-sync-errors", + "title": "Diagnosing and fixing sync errors" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#how-migration-tracking-works", + "title": "How migration tracking works" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-1-check-whats-out-of-sync", + "title": "Step 1: Check what's out of sync" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-2-if-you-made-changes-on-the-remote-database-directly", + "title": "Step 2: If you made changes on the remote database directly" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-3-if-the-migration-history-table-is-wrong", + "title": "Step 3: If the migration history table is wrong" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#seeding-data", + "title": "Seeding data" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#schema-migrations", + "title": "Schema migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#deploy-your-project", + "title": "Deploy your project" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#diffing-changes", + "title": "Diffing changes" } ], - "resultChars": 33842 + "resultChars": 191172 }, { "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL --max-time 15 https://supabase.com/changelog.md | head -80'", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -i -m 12 'breaking-change|CLI|Postgres'\"", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 4177 + "resultChars": 1923 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 763060, - "cacheReadInputTokens": 698634, - "cacheWriteInputTokens": 60145, - "outputTokens": 10332 + "inputTokens": 1007813, + "cacheReadInputTokens": 931308, + "cacheWriteInputTokens": 65515, + "outputTokens": 7886 } ], - "stepCount": 22, - "toolCallCount": 29, - "agentRunDurationMs": 297006, + "stepCount": 28, + "toolCallCount": 27, + "agentRunDurationMs": 335877, "sandboxUsage": { - "activeCpuDurationMs": 536679, - "duration": 451050, + "activeCpuDurationMs": 861323, + "duration": 544046, "memory": 8192, "networkTransfer": { - "ingress": 1995220375, - "egress": 8799932 + "ingress": 2794138113, + "egress": 12655281 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -24044,48 +24522,48 @@ "checks": [ { "name": "local stack is running for this project", - "passed": true, - "notes": "legacy (docker), select 1 ok" + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." }, { "name": "an inspect db report showed the orders table", - "passed": true, - "judgeNotes": "The inspect outliers and calls reports show queries against public.orders, and inspect index-stats names public.orders directly." + "passed": false, + "notes": "tool call 7: supabase inspect db help (errored); tool call 15: supabase inspect db db-stats; no inspect output named the orders table" }, { "name": "the order-history lookup uses an index on customer_email", - "passed": true, - "notes": "Index Only Scan using orders_customer_email_created_at_idx" + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." }, { "name": "orders still holds every seeded row", - "passed": true, - "notes": "300000 rows" + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." }, { "name": "a new migration creates an index on orders.customer_email", - "passed": true, - "notes": "20261022132812_index_orders_customer_history.sql (version 20261022132812)" + "passed": false, + "notes": "no new migration files" }, { "name": "that migration is recorded in the local migration history", "passed": false, - "notes": "version(s) 20261022132812 recorded with statements that do not create an index on orders.customer_email" + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime, escalate privileges, or change permissions. Docker commands were read-only probes; Supabase commands were permitted." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM. The Docker commands were read-only probes." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db calls\",\"db index-stats\"],\"inspectRuns\":4,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"inspect db help\",\"status\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"migration new\",\"migration up\",\"migration list\",\"inspect db index-stats\",\"db advisors\"],\"explainCommands\":3,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db help\",\"db db-stats\"],\"inspectRuns\":2,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"inspect db help\",\"inspect db db-stats\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":0,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching index and migration file, and accurately describes the verified index-only plan. It does not claim the migration was successfully applied through the CLI or deployed remotely." + "judgeNotes": "The report accurately describes the unavailable local database, names the failed status and inspect checks, and states that no index or migration was added. It identifies the customer-email lookup as a lead without falsely claiming a confirmed diagnosis or fix." } ], "skills": { @@ -24100,98 +24578,106 @@ }, "docs": { "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 11955 - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries local database pg_stat_statements explain analyze\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"database performance slow queries monitoring debugging explain analyze index Supabase local database\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" - }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" } ], - "resultChars": 31471 + "resultChars": 41434 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries pg_stat_statements query performance inspection\", limit: 6) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries performance database inspect pg_stat_statements\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" - }, + } + ], + "resultChars": 30545 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL --max-time 20 https://supabase.com/changelog.md | head -80; printf '\\\\n--- monitoring guide ---\\\\n'; curl -fsSL --max-time 20 https://supabase.com/docs/guides/monitoring-and-debugging.md | head -80\"", + "pages": [ + { + "url": "https://supabase.com/changelog.md" }, { - "url": "https://supabase.com/kb/sitemap", - "title": "Supabase Knowledge Base" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" } ], - "resultChars": 38529 + "resultChars": 7560 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 781749, - "cacheReadInputTokens": 716808, - "cacheWriteInputTokens": 64881, - "outputTokens": 7588 + "inputTokens": 174769, + "cacheReadInputTokens": 126370, + "cacheWriteInputTokens": 47898, + "outputTokens": 4510 } ], - "stepCount": 20, - "toolCallCount": 30, - "agentRunDurationMs": 152435, + "stepCount": 7, + "toolCallCount": 18, + "agentRunDurationMs": 72441, "sandboxUsage": { - "activeCpuDurationMs": 313076, - "duration": 265722, + "activeCpuDurationMs": 243238, + "duration": 292386, "memory": 8192, "networkTransfer": { - "ingress": 2780165674, - "egress": 16872811 + "ingress": 821102821, + "egress": 3309896 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -24229,8 +24715,8 @@ }, { "name": "an inspect db report showed the orders table", - "passed": true, - "judgeNotes": "The `supabase inspect db outliers` output shows a query against public.orders, and `index-stats` names the public.orders table." + "passed": false, + "notes": "no inspect output named the orders table" }, { "name": "the order-history lookup uses an index on customer_email", @@ -24244,8 +24730,8 @@ }, { "name": "a new migration creates an index on orders.customer_email", - "passed": true, - "notes": "20261022132858_orders_customer_email_created_at_idx.sql (version 20261022132858)" + "passed": false, + "notes": "no new migration files" }, { "name": "that migration is recorded in the local migration history", @@ -24255,17 +24741,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. Docker commands were read-only probes; Supabase commands are allowed." + "judgeNotes": "No command attempts to install, start, or reconfigure a container runtime or VM." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db outliers\",\"db calls\",\"db index-stats\"],\"inspectRuns\":4,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"migration new\",\"migration up\",\"migration list\",\"db advisors\",\"inspect db index-stats\",\"stop\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":0,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup, the local index migration, and the verified index-scan improvement. Its claims about local application, migration history, and stopping the stack match the transcript." + "judgeNotes": "The report correctly says work stopped before any fix or migration, identifies the missing local database container and refused connection, and treats the customer-email lookup as an unverified diagnosis." } ], "skills": { @@ -24282,9 +24768,13 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"database slow queries monitoring pg_stat_statements indexes explain analyze performance local database\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Supabase monitoring and debugging slow queries query performance indexes pg_stat_statements explain analyze\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, { "url": "https://supabase.com/docs/guides/database/query-optimization", "title": "Query Optimization" @@ -24293,92 +24783,102 @@ "url": "https://supabase.com/docs/guides/deployment/going-into-prod", "title": "Production Checklist" }, - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" - }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" } ], - "resultChars": 40381 + "resultChars": 35401 }, { - "source": "web_search", - "query": "https://supabase.com/docs/guides/monitoring-and-debugging", + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres missing indexes composite index equality order by limit query optimization\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/orioledb", + "title": "OrioleDB Overview" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg", + "title": "HypoPG: Hypothetical indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search", + "title": "Full Text Search" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" } - ] - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog breaking change database CLI October 2026", - "pages": [] + ], + "resultChars": 70793 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase monitoring and debugging slow queries pg_stat_statements inspect database local\", limit: 4) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase slow queries pg_stat_statements local database query performance\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" } ], - "resultChars": 25562 + "resultChars": 43588 }, { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL --max-time 15 https://supabase.com/changelog.md | head -c 6000'", + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase inspect db commands table-record-counts seq-scans total-time\", limit: 4) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" } ], - "resultChars": 6000 + "resultChars": 12394 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 828188, - "cacheReadInputTokens": 751138, - "cacheWriteInputTokens": 71193, - "outputTokens": 8305 + "inputTokens": 123816, + "cacheReadInputTokens": 71884, + "cacheWriteInputTokens": 51917, + "outputTokens": 2227 } ], - "stepCount": 20, - "toolCallCount": 35, - "agentRunDurationMs": 179683, + "stepCount": 5, + "toolCallCount": 13, + "agentRunDurationMs": 51174, "sandboxUsage": { - "activeCpuDurationMs": 455365, - "duration": 309084, + "activeCpuDurationMs": 252077, + "duration": 261518, "memory": 8192, "networkTransfer": { - "ingress": 2775625894, - "egress": 12669463 + "ingress": 820861935, + "egress": 3311925 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -24407,7 +24907,7 @@ "suite": "cli", "interface": "cli", "cliVersion": "3.0.0-next.22", - "passed": false, + "passed": true, "checks": [ { "name": "local stack is running for this project", @@ -24416,13 +24916,13 @@ }, { "name": "an inspect db report showed the orders table", - "passed": false, - "notes": "no supabase inspect report ran" + "passed": true, + "judgeNotes": "The `supabase inspect db outliers` output shows a query against `public.orders` with 200 calls and 57.7% of total execution time." }, { "name": "the order-history lookup uses an index on customer_email", "passed": true, - "notes": "Index Scan using orders_customer_email_created_at_idx" + "notes": "Index Only Scan using orders_customer_email_created_at_idx" }, { "name": "orders still holds every seeded row", @@ -24432,27 +24932,27 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132846_add_orders_customer_email_created_at_index.sql (version 20261022132846)" + "notes": "20261023103643_index_orders_customer_history.sql (version 20261023103643)" }, { "name": "that migration is recorded in the local migration history", "passed": true, - "notes": "20261022132846_add_orders_customer_email_created_at_index.sql (version 20261022132846) is in supabase_migrations.schema_migrations with the index statement" + "notes": "20261023103643_index_orders_customer_history.sql (version 20261023103643) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. The Docker command was a read-only probe; stack operations used the Supabase CLI." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The Docker command was a read-only probe." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration list\",\"db advisors\",\"db pull\",\"db reset\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db outliers\"],\"inspectRuns\":1,\"inspectCallsNamingOrders\":1,\"workflowCommands\":[\"status\",\"db start\",\"db query\",\"migration list\",\"migration new\",\"migration up\",\"inspect db outliers\",\"db advisors\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup and its sequential scans, names the matching index and migration, and accurately limits the applied fix to the local database." + "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup and its sequential scans. The stated index, migration, local application, migration history, and improved index-scan plan are supported by the transcript and ground truth." } ], "skills": { @@ -24476,109 +24976,110 @@ "url": "https://supabase.com/changelog.md" } ], - "resultChars": 5178 + "resultChars": 4978 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging slow queries query performance index database performance pg_stat_statements\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"monitoring debugging slow queries database performance indexes query plan\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/database/postgres/postgres-log-config", + "title": "Postgres log configurations" } ], - "resultChars": 33842 + "resultChars": 132535 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase local development supabase start db query --local migrations db reset seed\", limit: 5) { nodes { ... on CLICommandReference { title href content } ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"slow queries monitoring pg_stat_statements explain analyze indexes performance troubleshooting\", limit: 8) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-db-reset", - "title": "Resets the local database to current migrations" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/functions/development-environment", - "title": "Development Environment" + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" }, { - "url": "https://supabase.com/docs/guides/local-development", - "title": "Local Development & CLI" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/kb/topics/troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" + }, + { + "url": "https://supabase.com/kb/topics/database", + "title": "Database" } ], - "resultChars": 51801 + "resultChars": 48650 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db pull local name generate migration local schema db diff migration list\", limit: 4) { nodes { ... on CLICommandReference { title href content } ... on Guide { title href content } } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | sed -n '1,80p' && printf '\\\\n--- INSPECT GUIDE ---\\\\n' && curl -fsSL https://supabase.com/docs/guides/observability/inspect.md | sed -n '1,120p'\"", "pages": [ { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" + "url": "https://supabase.com/docs/guides/observability/inspect.md" } ], - "resultChars": 61221 + "resultChars": 9960 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 1130534, - "cacheReadInputTokens": 1061635, - "cacheWriteInputTokens": 68815, - "outputTokens": 8931 + "inputTokens": 626796, + "cacheReadInputTokens": 563539, + "cacheWriteInputTokens": 63206, + "outputTokens": 6366 } ], - "stepCount": 28, - "toolCallCount": 33, - "agentRunDurationMs": 214517, + "stepCount": 17, + "toolCallCount": 27, + "agentRunDurationMs": 186784, "sandboxUsage": { - "activeCpuDurationMs": 393247, - "duration": 329452, + "activeCpuDurationMs": 472208, + "duration": 374651, "memory": 8192, "networkTransfer": { - "ingress": 2781008108, - "egress": 18297608 + "ingress": 1994859320, + "egress": 11040579 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -32505,7 +33006,7 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.122.0-beta.12", + "cliVersion": "2.122.0-beta.15", "passed": false, "checks": [ { @@ -32531,7 +33032,7 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132707_add_orders_customer_history_index.sql (version 20261022132707)" + "notes": "20261023103539_add_orders_customer_history_index.sql (version 20261023103539)" }, { "name": "that migration is recorded in the local migration history", @@ -32541,17 +33042,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempts to install, start, reconfigure, or gain access to a container runtime. The Docker command is a read-only probe, and supabase start is permitted." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM, escalate privileges, or alter runtime permissions." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"migration new\",\"db query\",\"migration list\",\"db advisors\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration new\",\"db query\",\"migration list\",\"db advisors\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report identifies the slow customer-email order-history lookup, accurately describes the new index migration, and states that Docker prevented applying or verifying it locally." + "judgeNotes": "The report identifies the slow customer-email order-history lookup, accurately describes the new index migration, and says database application and query-plan verification were blocked by the unavailable local database and Docker daemon." } ], "skills": { @@ -32566,44 +33067,110 @@ }, "docs": { "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring and debugging database slow queries performance query stats local Supabase\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + }, + { + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", + "title": "How to View Database Metrics" + } + ], + "resultChars": 28063 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI migration new db query local migration list command\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres index equality filter order by descending limit multicolumn index covering index query optimization\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/orioledb", + "title": "OrioleDB Overview" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-postgres-chooses-which-index-to-use-_JHrf4", + "title": "How Postgres chooses which index to use" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg", + "title": "HypoPG: Hypothetical indexes" + } + ], + "resultChars": 29650 + }, { "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -m 15 'breaking-change|database|cli|migration|performance'\"", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 11955 + "resultChars": 2549 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring and debugging slow queries pg_stat_statements EXPLAIN query performance local database\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Supabase CLI migration new and db query local database\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", + "title": "Migrate from Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" } ], - "resultChars": 30954 + "resultChars": 87641 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase monitoring debugging slow query diagnose performance explain analyze missing index performance guide\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Supabase query performance missing indexes explain analyze index advisor\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -32611,41 +33178,45 @@ "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" }, { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" + "url": "https://supabase.com/docs/guides/database/extensions/index_advisor", + "title": "index_advisor: query optimization" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" } ], - "resultChars": 102720 + "resultChars": 104527 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 260661, - "cacheReadInputTokens": 210330, - "cacheWriteInputTokens": 50304, - "outputTokens": 4934 + "inputTokens": 252795, + "cacheReadInputTokens": 203353, + "cacheWriteInputTokens": 49415, + "outputTokens": 5451 } ], "stepCount": 9, "toolCallCount": 21, - "agentRunDurationMs": 64307, + "agentRunDurationMs": 78767, "sandboxUsage": { - "activeCpuDurationMs": 132619, - "duration": 166183, + "activeCpuDurationMs": 233784, + "duration": 249357, "memory": 8192, "networkTransfer": { - "ingress": 839568272, - "egress": 6402914 + "ingress": 835828177, + "egress": 4648448 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -32673,7 +33244,7 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.122.0-beta.12", + "cliVersion": "2.122.0-beta.15", "passed": false, "checks": [ { @@ -32684,7 +33255,7 @@ { "name": "an inspect db report showed the orders table", "passed": false, - "notes": "no inspect output named the orders table" + "notes": "tool call 14: supabase inspect db calls (errored); tool call 15: supabase inspect db outliers (errored); no inspect output named the orders table" }, { "name": "the order-history lookup uses an index on customer_email", @@ -32699,7 +33270,7 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132718_index_orders_customer_email_created_at.sql (version 20261022132718)" + "notes": "20261023103548_orders_customer_email_created_at_idx.sql (version 20261023103548)" }, { "name": "that migration is recorded in the local migration history", @@ -32709,17 +33280,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No commands attempted a prohibited container-runtime detour." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM, escalate privileges, or change socket permissions." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"migration new\",\"db advisors\",\"db query\",\"migration list\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db calls\",\"db outliers\"],\"inspectRuns\":2,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration new\",\"inspect db calls\",\"inspect db outliers\",\"migration list\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report identifies the slow customer-email order-history lookup, accurately describes the new index migration, and does not claim it was applied or verified. It correctly names the Docker blocker." + "judgeNotes": "The report identifies the order-history lookup as the slow workload, accurately describes the customer_email index migration, and does not claim it was applied or verified. Its stated local database and Docker blockers match the command errors." } ], "skills": { @@ -32736,56 +33307,17 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"database performance slow queries monitoring debugging explain analyze missing indexes\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries pg_stat_statements performance database\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" - }, - { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, - { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" - } - ], - "resultChars": 108218 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 4177 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging database inspect slow queries pg_stat_statements explain analyze indexes\", limit: 6) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" - }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", "title": "pg_stat_statements: Query Performance Monitoring" @@ -32795,65 +33327,71 @@ "title": "Production Checklist" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" - }, - { - "url": "https://supabase.com/kb/topics/troubleshooting", - "title": "Troubleshooting" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" } ], - "resultChars": 31104 + "resultChars": 42356 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging guide database inspect pg_stat_statements slow query analysis\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Postgres composite index equality filter order by limit Supabase performance EXPLAIN ANALYZE\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/database/extensions/hypopg", + "title": "HypoPG: Hypothetical indexes" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/database/orioledb", + "title": "OrioleDB Overview" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/database/extensions/pg_plan_filter", + "title": "pg_plan_filter: Restrict Total Cost" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/troubleshooting/understanding-postgresql-explain-output-Un9dqX", + "title": "Understanding Postgres EXPLAIN Output" } ], - "resultChars": 33842 + "resultChars": 33885 + }, + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 260819, - "cacheReadInputTokens": 210483, - "cacheWriteInputTokens": 50309, - "outputTokens": 4755 + "inputTokens": 200422, + "cacheReadInputTokens": 148476, + "cacheWriteInputTokens": 51374, + "outputTokens": 5623 } ], - "stepCount": 9, - "toolCallCount": 19, - "agentRunDurationMs": 75244, + "stepCount": 7, + "toolCallCount": 20, + "agentRunDurationMs": 82761, "sandboxUsage": { - "activeCpuDurationMs": 132113, - "duration": 176970, + "activeCpuDurationMs": 280262, + "duration": 263762, "memory": 8192, "networkTransfer": { - "ingress": 838708382, - "egress": 6787601 + "ingress": 835401404, + "egress": 3562297 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -32881,7 +33419,7 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.122.0-beta.12", + "cliVersion": "2.122.0-beta.15", "passed": false, "checks": [ { @@ -32907,7 +33445,7 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132721_add_orders_customer_created_at_index.sql (version 20261022132721)" + "notes": "20261023000000_index_orders_customer_email_created_at.sql (version 20261023000000)" }, { "name": "that migration is recorded in the local migration history", @@ -32917,17 +33455,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or escalate privileges. `docker info` was a read-only probe." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime, escalate privileges, or change runtime permissions." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration new\",\"migration list\",\"db advisors\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration list\",\"db advisors\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report identifies the repeated customer-email order-history lookup and the matching index migration. It truthfully says the migration was saved but not applied or verified because the local database was unavailable." + "judgeNotes": "The report identifies the repeated customer-email order lookup and its index-based fix, accurately describes the migration, and clearly states that Docker prevented applying or verifying it locally." } ], "skills": { @@ -32942,105 +33480,60 @@ }, "docs": { "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog breaking changes CLI migrations database", - "pages": [] - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Inspect database slow queries pg_stat_statements EXPLAIN local Supabase CLI migrations\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"database performance monitoring slow queries pg_stat_statements explain analyze indexes diagnostics\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-calls", - "title": "Show queries from pg_stat_statements ordered by total times called\\\\n" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" - } - ], - "resultChars": 31024 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5013 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries query performance pg_stat_statements explain analyze\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" - }, { "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" } ], - "resultChars": 33642 + "resultChars": 40381 + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog database performance", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 249173, - "cacheReadInputTokens": 192091, - "cacheWriteInputTokens": 48215, - "outputTokens": 4218 + "inputTokens": 158885, + "cacheReadInputTokens": 114075, + "cacheWriteInputTokens": 40069, + "outputTokens": 4379 } ], - "stepCount": 9, - "toolCallCount": 19, - "agentRunDurationMs": 69775, + "stepCount": 7, + "toolCallCount": 13, + "agentRunDurationMs": 72134, "sandboxUsage": { - "activeCpuDurationMs": 123116, - "duration": 168345, + "activeCpuDurationMs": 211154, + "duration": 225198, "memory": 8192, "networkTransfer": { - "ingress": 838521804, - "egress": 6248159 + "ingress": 836208180, + "egress": 3523129 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -41234,27 +41727,27 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132850_add_orders_customer_email_created_at_idx.sql (version 20261022132850)" + "notes": "20261023103814_add_orders_customer_history_index.sql (version 20261023103814)" }, { "name": "that migration is recorded in the local migration history", "passed": true, - "notes": "20261022132850_add_orders_customer_email_created_at_idx.sql (version 20261022132850) is in supabase_migrations.schema_migrations with the index statement" + "notes": "20261023103814_add_orders_customer_history_index.sql (version 20261023103814) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM. The Docker command was a read-only probe." + "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. The Docker commands were read-only probes; stack operations used the Supabase CLI." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"db advisors\",\"db pull\",\"migration list\",\"db diff\"],\"explainCommands\":2,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration list\",\"db diff\",\"db advisors\",\"db reset\"],\"explainCommands\":5,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup and accurately describes the index, migration, local history, and verified index-scan plan." + "judgeNotes": "The report correctly identifies the unindexed customer_email order-history lookup, names the matching index and migration, and accurately describes local application and verification. It makes no unsupported hosted-project or row-change claims." } ], "skills": { @@ -41269,42 +41762,57 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4082 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db query local database migration db diff\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"monitoring debugging slow queries EXPLAIN query performance indexes\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/database/postgres/postgres-log-config", + "title": "Postgres log configurations" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-db-diff", - "title": "Diffs the local database for schema changes" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" } ], - "resultChars": 72162 + "resultChars": 101571 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"slow queries performance monitoring pg_stat_statements explain analyze database advisors\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Debugging and monitoring guide inspect slow query performance pg_stat_statements EXPLAIN analyze\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { "url": "https://supabase.com/docs/guides/observability/detecting", @@ -41315,89 +41823,61 @@ "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" } ], - "resultChars": 79826 + "resultChars": 36533 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and debugging slow queries pg_stat_statements query performance explain analyze\", limit: 6) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI db diff local generate migration file\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" - }, - { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/kb/topics/database", - "title": "Database" - } - ], - "resultChars": 36547 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- directory ---'; ls -la; printf '%s\\\\n' '--- git roots ---'; find .. -maxdepth 3 -name .git -type d -print; printf '%s\\\\n' '--- project files ---'; rg --files | head -40; printf '%s\\\\n' '--- changelog ---'; curl -fsSL https://supabase.com/changelog.md | head -100\"", - "pages": [ - { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches", + "title": "Working with branches" } ], - "resultChars": 5704 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] + "resultChars": 79403 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 1215194, - "cacheReadInputTokens": 1146720, - "cacheWriteInputTokens": 67943, - "outputTokens": 9137 + "inputTokens": 899314, + "cacheReadInputTokens": 835217, + "cacheWriteInputTokens": 64022, + "outputTokens": 8111 } ], - "stepCount": 28, - "toolCallCount": 35, - "agentRunDurationMs": 212079, + "stepCount": 25, + "toolCallCount": 28, + "agentRunDurationMs": 313155, "sandboxUsage": { - "activeCpuDurationMs": 380631, - "duration": 325658, + "activeCpuDurationMs": 681306, + "duration": 513070, "memory": 8192, "networkTransfer": { - "ingress": 2861854414, - "egress": 16848002 + "ingress": 2007289191, + "egress": 10089628 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -41426,52 +41906,52 @@ "suite": "cli", "interface": "cli", "cliVersion": "2.117.0", - "passed": true, + "passed": false, "checks": [ { "name": "local stack is running for this project", - "passed": true, - "notes": "legacy (docker), select 1 ok" + "passed": false, + "notes": "managed: exit 1: ; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerunning the command with --debug to troubleshoot the error." }, { "name": "an inspect db report showed the orders table", "passed": true, - "judgeNotes": "The `supabase inspect db outliers --local` output shows a query against public.orders, and the inspect index-stats and table-stats outputs name public.orders." + "judgeNotes": "The `supabase inspect db outliers --local` output names the order-history query against public.orders and reports its execution time and call count. Inspect index-stats also names public.orders." }, { "name": "the order-history lookup uses an index on customer_email", - "passed": true, - "notes": "Index Scan using orders_customer_email_created_at_idx" + "passed": false, + "notes": "could not read the plan: managed: exit 1: ; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerunning the command with --debug to troubleshoot the error." }, { "name": "orders still holds every seeded row", - "passed": true, - "notes": "300000 rows" + "passed": false, + "notes": "could not count public.orders: managed: exit 1: ; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerunning the command with --debug to troubleshoot the error." }, { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132941_add_orders_customer_history_index.sql (version 20261022132941)" + "notes": "20261023000000_add_orders_customer_history_index.sql (version 20261023000000)" }, { "name": "that migration is recorded in the local migration history", - "passed": true, - "notes": "20261022132941_add_orders_customer_history_index.sql (version 20261022132941) is in supabase_migrations.schema_migrations with the index statement" + "passed": false, + "notes": "managed: exit 1: ; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerunning the command with --debug to troubleshoot the error." }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM. Docker commands were read-only probes; Supabase CLI commands are permitted." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The Docker commands were read-only probes; the remaining database operations used the Supabase CLI." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db index-stats\",\"db table-stats\"],\"inspectRuns\":7,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"inspect db outliers\",\"inspect db index-stats\",\"inspect db table-stats\",\"db query\",\"migration new\",\"db reset\",\"migration list\",\"db advisors\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db outliers\",\"db calls\",\"db index-stats\"],\"inspectRuns\":5,\"inspectCallsNamingOrders\":3,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"inspect db index-stats\",\"db reset\",\"migration list\",\"db advisors\",\"stop\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup as the cause and accurately describes the migration, local application, and verified index-scan improvement." + "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup, names the matching index migration, and accurately describes its local application and verified index-scan improvement. It does not claim a hosted deployment or data changes." } ], "skills": { @@ -41479,138 +41959,180 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/monitoring-and-debugging slow queries Supabase query performance local database", - "pages": [] - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Inspect database CLI query performance slow queries pg_stat_statements outliers local\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase slow queries monitoring debugging pg_stat_statements EXPLAIN local database diagnose\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-calls", - "title": "Show queries from pg_stat_statements ordered by total times called\\\\n" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-outliers", - "title": "Show queries from pg_stat_statements ordered by total execution time\\\\n" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" } ], - "resultChars": 34642 + "resultChars": 33446 }, { - "source": "web_search", - "query": "site:supabase.com/changelog.md breaking-change Supabase changelog 2026", + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase database indexes composite index equality order by limit query performance\", limit: 5) { nodes { title href content } }", + "hasContent": true, "pages": [] }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db query local execute SQL db advisors db pull migration list\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", + "title": "Pull schema from the remote database" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-migration-list", + "title": "List local and remote migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + } + ], + "resultChars": 62241 + }, { "source": "web_search", - "query": "https://supabase.com/docs/guides/monitoring-and-debugging", + "query": "https://supabase.com/changelog.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging" + "url": "https://supabase.com/changelog.md" } ] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI inspect db outliers index-stats local command flags db query explain analyze\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase monitoring and debugging slow queries inspect database query performance\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db query --local execute SQL\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"query performance missing index composite indexes equality order by limit\", limit: 4) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/cli/supabase-inspect-report", - "title": "Generate a CSV output for all inspect commands" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/ai/hybrid-search", + "title": "Hybrid search" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-db-stats", - "title": "Show stats such as cache hit rates, total sizes, and WAL size\\\\n" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db", - "title": "Tools to inspect your Supabase database" + "url": "https://supabase.com/docs/guides/database/orioledb", + "title": "OrioleDB Overview" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" } ], - "resultChars": 17329 + "resultChars": 32403 }, { - "source": "web_search", - "query": "https://supabase.com/changelog.md", + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 6000'", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } - ] + ], + "resultChars": 6002 }, { "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -c 1200; printf '\\\\n--- migration file ---\\\\n'; nl -ba supabase/migrations/20261022132941_add_orders_customer_history_index.sql\"", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging | sed -n '1,20p'\"", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging" } ], - "resultChars": 1382 + "resultChars": 135912 }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | rg -n -i -C 2 'slow|query|inspect|performance|outlier' | head -80\"", + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging database slow queries Supabase\", limit: 2) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" } ], - "resultChars": 2425 + "resultChars": 57299 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 1334627, - "cacheReadInputTokens": 1252792, - "cacheWriteInputTokens": 64321, - "outputTokens": 9190 + "inputTokens": 1164927, + "cacheReadInputTokens": 1094101, + "cacheWriteInputTokens": 70563, + "outputTokens": 7577 } ], - "stepCount": 36, - "toolCallCount": 21, - "agentRunDurationMs": 255149, + "stepCount": 29, + "toolCallCount": 31, + "agentRunDurationMs": 289047, "sandboxUsage": { - "activeCpuDurationMs": 547854, - "duration": 395675, + "activeCpuDurationMs": 809594, + "duration": 474167, "memory": 8192, "networkTransfer": { - "ingress": 2840425126, - "egress": 13295830 + "ingress": 2838159410, + "egress": 12770362 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -41639,7 +42161,7 @@ "suite": "cli", "interface": "cli", "cliVersion": "2.117.0", - "passed": false, + "passed": true, "checks": [ { "name": "local stack is running for this project", @@ -41648,13 +42170,13 @@ }, { "name": "an inspect db report showed the orders table", - "passed": false, - "notes": "no supabase inspect report ran" + "passed": true, + "judgeNotes": "The `supabase inspect db outliers --local` output shows a query against public.orders, and `index-stats` names the table." }, { "name": "the order-history lookup uses an index on customer_email", "passed": true, - "notes": "Index Only Scan using orders_customer_history_idx" + "notes": "Index Scan using orders_customer_email_created_at_idx" }, { "name": "orders still holds every seeded row", @@ -41664,27 +42186,27 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132843_add_orders_customer_history_index.sql (version 20261022132843)" + "notes": "20261023103806_index_orders_customer_email_created_at.sql (version 20261023103806)" }, { "name": "that migration is recorded in the local migration history", "passed": true, - "notes": "20261022132843_add_orders_customer_history_index.sql (version 20261022132843) is in supabase_migrations.schema_migrations with the index statement" + "notes": "20261023103806_index_orders_customer_email_created_at.sql (version 20261023103806) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted a container-runtime detour. The agent used permitted Supabase CLI and database commands." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The stack and database commands used the Supabase CLI." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"db advisors\",\"db pull\",\"migration list\",\"db diff\"],\"explainCommands\":3,\"directIndexDdl\":2,\"ordersIndexes\":[\"orders_customer_history_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db outliers\",\"db calls\",\"db index-stats\"],\"inspectRuns\":4,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"inspect db index-stats\",\"db advisors\",\"migration new\",\"migration up\",\"migration list\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer order-history lookup as the cause. The named migration and index exist, are recorded locally, and the verified plan uses an index-only scan. Its account of the unrelated pg_net diff is also accurate." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching index and migration, and accurately describes local application and verification." } ], "skills": { @@ -41693,75 +42215,764 @@ "supabase-postgres-best-practices" ], "loaded": [ + "supabase", "supabase-postgres-best-practices" ] }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4082 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"database slow queries performance indexes EXPLAIN monitoring pg_stat_statements\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"monitoring debugging slow queries query performance local database EXPLAIN missing indexes\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#examining-query-performance", + "title": "Examining query performance" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#configuring-clients-to-use-fewer-connections", + "title": "Configuring clients to use fewer connections" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#allowing-higher-number-of-connections", + "title": "Allowing higher number of connections" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#enterprise", + "title": "Enterprise" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#optimizing-the-number-of-connections", + "title": "Optimizing the number of connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#using-reports", + "title": "Using reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#api-gateway", + "title": "API gateway" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#auth-api-gateway", + "title": "Auth API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#database", + "title": "Database" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#advanced-telemetry", + "title": "Advanced Telemetry" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#memory-usage", + "title": "Memory usage" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#memory-commitment", + "title": "Memory commitment" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#cpu-usage", + "title": "CPU usage" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-inputoutput-operations-per-second-iops", + "title": "Disk input/output operations per second (IOPS)" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-throughput", + "title": "Disk throughput" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-size", + "title": "Disk size" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#query-performance", + "title": "Query Performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#database-connections", + "title": "Database connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#dedicated-pooler-pgbouncer-client-connections", + "title": "Dedicated Pooler (PgBouncer) Client Connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#shared-pooler-supavisor-client-connections", + "title": "Shared Pooler (Supavisor) Client Connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-usage", + "title": "Disk Usage" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#database-size", + "title": "Database size" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#edge-functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#postgrest", + "title": "PostgREST" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#realtime", + "title": "Realtime" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#realtime-api-gateway", + "title": "Realtime API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#storage", + "title": "Storage" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#using-sql", + "title": "Using SQL" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#using-the-cli", + "title": "Using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#the-inspect-db-command", + "title": "The inspect db command" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-any-postgres-database", + "title": "Connect to any Postgres database" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-a-supabase-instance", + "title": "Connect to a Supabase instance" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#inspection-commands", + "title": "Inspection commands" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#disk-storage", + "title": "Disk storage" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#query-performance", + "title": "Query performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#locks", + "title": "Locks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connections", + "title": "Connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#notes-on-pg_stat_statements", + "title": "Notes on pg_stat_statements" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#postgres-cumulative-statistics-system", + "title": "Postgres cumulative statistics system" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#most-frequently-called-queries", + "title": "Most frequently called queries" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#slowest-queries-by-execution-time", + "title": "Slowest queries by execution time" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#most-time-consuming-queries", + "title": "Most time consuming queries" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#hit-rate", + "title": "Hit rate" + }, { "url": "https://supabase.com/docs/guides/database/query-optimization", "title": "Query Optimization" }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#analyze-the-query-plan", + "title": "Analyze the query plan" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#use-appropriate-index-types", + "title": "Use appropriate index types" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#partial-indexes", + "title": "Partial indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#composite-indexes", + "title": "Composite indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#over-indexing", + "title": "Over-Indexing" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#statistics", + "title": "Statistics" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#example-query", + "title": "Example query" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#where-clause", + "title": "where clause:" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#join-columns", + "title": "join columns" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#order-by-clause", + "title": "order by clause" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#key-concepts", + "title": "Key concepts" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts#role-level", + "title": "Role level" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts#global-level", + "title": "Global level" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts#identifying-timeouts", + "title": "Identifying timeouts" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts#using-the-logs-explorer", + "title": "Using the Logs Explorer" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts#using-the-query-performance-page", + "title": "Using the Query Performance page" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts#understanding-roles-in-logs", + "title": "Understanding roles in logs" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts#change-postgres-timeout", + "title": "Change Postgres timeout" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts#session-level", + "title": "Session level" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts#function-level", + "title": "Function level" + }, + { + "url": "https://supabase.com/kb/topics/troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" + }, + { + "url": "https://supabase.com/kb/topics/database", + "title": "Database" + } + ], + "resultChars": 150214 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Debugging and monitoring database inspect query performance explain analyze pg_stat_statements indexes Supabase\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/database/postgres/configuration", "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/database/postgres/configuration#timeouts", + "title": "Timeouts" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#statement-optimization", + "title": "Statement optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#managing-timezones", + "title": "Managing timezones" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#change-timezone", + "title": "Change timezone" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#full-list-of-timezones", + "title": "Full list of timezones" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#search-for-a-specific-timezone", + "title": "Search for a specific timezone" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#security", + "title": "Security" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#performance", + "title": "Performance" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#availability", + "title": "Availability" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#deployment", + "title": "Deployment" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#rate-limiting-resource-allocation--abuse-prevention", + "title": "Rate limiting, resource allocation, & abuse prevention" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#auth-rate-limits", + "title": "Auth rate limits" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#realtime-limits", + "title": "Realtime limits" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#abuse-prevention", + "title": "Abuse prevention" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#email-link-validity", + "title": "Email link validity" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/observability/inspect#using-sql", + "title": "Using SQL" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#postgres-cumulative-statistics-system", + "title": "Postgres cumulative statistics system" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#most-frequently-called-queries", + "title": "Most frequently called queries" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#slowest-queries-by-execution-time", + "title": "Slowest queries by execution time" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#most-time-consuming-queries", + "title": "Most time consuming queries" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#hit-rate", + "title": "Hit rate" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#using-the-cli", + "title": "Using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#the-inspect-db-command", + "title": "The inspect db command" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-any-postgres-database", + "title": "Connect to any Postgres database" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-a-supabase-instance", + "title": "Connect to a Supabase instance" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#inspection-commands", + "title": "Inspection commands" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#disk-storage", + "title": "Disk storage" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#query-performance", + "title": "Query performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#locks", + "title": "Locks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connections", + "title": "Connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#notes-on-pg_stat_statements", + "title": "Notes on pg_stat_statements" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#more-resources", + "title": "More resources" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#avoid-joins-in-policy-expressions", + "title": "Avoid joins in policy expressions" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#filter-in-the-client-query-too", + "title": "Filter in the client query too" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#measure-through-the-data-api", + "title": "Measure through the Data API" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#diagnose-whether-rls-is-the-bottleneck", + "title": "Diagnose whether RLS is the bottleneck" + }, + { + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" + }, + { + "url": "https://supabase.com/kb/topics/troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/kb/topics/database", + "title": "Database" } ], - "resultChars": 36452 + "resultChars": 119984 }, { - "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog breaking-change", - "pages": [] - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/monitoring-and-debugging", + "source": "search_docs", + "query": "query { searchDocs(query: \"guides/database/inspect query performance pg_stat_statements EXPLAIN ANALYZE\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#statistics", + "title": "Statistics" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#example-query", + "title": "Example query" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#where-clause", + "title": "where clause:" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#join-columns", + "title": "join columns" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#order-by-clause", + "title": "order by clause" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#key-concepts", + "title": "Key concepts" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#analyze-the-query-plan", + "title": "Analyze the query plan" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#use-appropriate-index-types", + "title": "Use appropriate index types" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#partial-indexes", + "title": "Partial indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#composite-indexes", + "title": "Composite indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#over-indexing", + "title": "Over-Indexing" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#enterprise", + "title": "Enterprise" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#examining-query-performance", + "title": "Examining query performance" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#optimizing-the-number-of-connections", + "title": "Optimizing the number of connections" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#allowing-higher-number-of-connections", + "title": "Allowing higher number of connections" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance#configuring-clients-to-use-fewer-connections", + "title": "Configuring clients to use fewer connections" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#enable-the-extension", + "title": "Enable the extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#inspecting-activity", + "title": "Inspecting activity" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#using-the-cli", + "title": "Using the CLI" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#the-inspect-db-command", + "title": "The inspect db command" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-any-postgres-database", + "title": "Connect to any Postgres database" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-a-supabase-instance", + "title": "Connect to a Supabase instance" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#inspection-commands", + "title": "Inspection commands" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#disk-storage", + "title": "Disk storage" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#query-performance", + "title": "Query performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#locks", + "title": "Locks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#connections", + "title": "Connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#notes-on-pg_stat_statements", + "title": "Notes on pg_stat_statements" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#using-sql", + "title": "Using SQL" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#postgres-cumulative-statistics-system", + "title": "Postgres cumulative statistics system" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#most-frequently-called-queries", + "title": "Most frequently called queries" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#slowest-queries-by-execution-time", + "title": "Slowest queries by execution time" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#most-time-consuming-queries", + "title": "Most time consuming queries" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect#hit-rate", + "title": "Hit rate" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#security", + "title": "Security" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#health", + "title": "Health" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#measure-api-server-error-rate", + "title": "Measure API server-error rate" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-failing-api-paths", + "title": "Find failing API paths" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#check-postgres-connection-pressure", + "title": "Check Postgres connection pressure" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#measure-authorization-failures", + "title": "Measure authorization failures" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-affected-paths-and-methods", + "title": "Find affected paths and methods" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-public-schema-tables-without-rls", + "title": "Find public-schema tables without RLS" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#performance", + "title": "Performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-long-running-sessions", + "title": "Find long-running sessions" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-blocked-sessions", + "title": "Find blocked sessions" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-expensive-query-patterns", + "title": "Find expensive query patterns" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#measure-shared-buffer-hit-rate", + "title": "Measure shared-buffer hit rate" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#usage", + "title": "Usage" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#trend-api-requests", + "title": "Trend API requests" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-high-volume-api-paths", + "title": "Find high-volume API paths" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#find-the-largest-relations", + "title": "Find the largest relations" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#count-connections-by-role-and-state", + "title": "Count connections by role and state" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting#turn-a-detection-into-a-diagnosis", + "title": "Turn a detection into a diagnosis" } - ] + ], + "resultChars": 114643 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 885947, - "cacheReadInputTokens": 816990, - "cacheWriteInputTokens": 59158, - "outputTokens": 10494 + "inputTokens": 964462, + "cacheReadInputTokens": 896103, + "cacheWriteInputTokens": 68290, + "outputTokens": 6142 } ], - "stepCount": 26, + "stepCount": 23, "toolCallCount": 29, - "agentRunDurationMs": 236825, + "agentRunDurationMs": 242856, "sandboxUsage": { - "activeCpuDurationMs": 414105, - "duration": 360148, + "activeCpuDurationMs": 620900, + "duration": 430063, "memory": 8192, "networkTransfer": { - "ingress": 2860341620, - "egress": 13031858 + "ingress": 2839025386, + "egress": 12242307 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -49631,27 +50842,27 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132921_add_orders_customer_history_index.sql (version 20261022132921)" + "notes": "20261023110000_add_orders_customer_history_index.sql (version 20261023110000)" }, { "name": "that migration is recorded in the local migration history", "passed": true, - "notes": "20261022132921_add_orders_customer_history_index.sql (version 20261022132921) is in supabase_migrations.schema_migrations with the index statement" + "notes": "20261023110000_add_orders_customer_history_index.sql (version 20261023110000) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No commands attempted to install, start, or reconfigure a container runtime or VM. Docker commands were read-only probes; stack operations used the Supabase CLI." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The Docker command was a read-only probe; Supabase CLI commands are permitted." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration new\",\"migration up\",\"db reset\",\"migration list\",\"db advisors\"],\"explainCommands\":3,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration list\",\"db advisors\",\"migration up\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration and index, and accurately describes local application and index-scan verification." + "judgeNotes": "The report correctly identifies the unindexed customer_email order-history lookup, names the matching migration and index, and accurately reports local application and an index-scan verification." } ], "skills": { @@ -49666,107 +50877,66 @@ }, "docs": { "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging slow queries explain analyze missing indexes pg_stat_statements\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-check-if-my-queries-are-being-blocked-by-other-queries-NSKtR1", - "title": "How to check if my queries are being blocked by other queries?" - } - ], - "resultChars": 35346 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, { "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | head -c 12000; printf '\\\\n--- monitoring docs fetch ---\\\\n'; curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 12000\"", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -120'", + "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" - }, - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" } ], - "resultChars": 3532 + "resultChars": 6220 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"CREATE INDEX CONCURRENTLY migration db push transaction Supabase\", limit: 4) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and debugging slow queries pg_stat_statements EXPLAIN query performance\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", - "title": "Migrating to Supabase" + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" }, { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mysql", - "title": "Migrate from MySQL to Supabase" + "url": "https://supabase.com/docs/guides/database/debugging-performance", + "title": "Debugging performance issues" } ], - "resultChars": 36745 - }, - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/database/query-optimization indexes composite index query plan analyze", - "pages": [] + "resultChars": 36073 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 1292598, - "cacheReadInputTokens": 1220545, - "cacheWriteInputTokens": 67603, - "outputTokens": 10738 + "inputTokens": 772724, + "cacheReadInputTokens": 719980, + "cacheWriteInputTokens": 52669, + "outputTokens": 6723 } ], - "stepCount": 34, - "toolCallCount": 33, - "agentRunDurationMs": 272795, + "stepCount": 25, + "toolCallCount": 24, + "agentRunDurationMs": 208239, "sandboxUsage": { - "activeCpuDurationMs": 447662, - "duration": 397921, + "activeCpuDurationMs": 639737, + "duration": 428769, "memory": 8192, "networkTransfer": { - "ingress": 2737162824, - "egress": 16732055 + "ingress": 2005309657, + "egress": 8412607 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -49795,7 +50965,7 @@ "suite": "cli", "interface": "cli", "cliVersion": "2.121.0", - "passed": false, + "passed": true, "checks": [ { "name": "local stack is running for this project", @@ -49804,8 +50974,8 @@ }, { "name": "an inspect db report showed the orders table", - "passed": false, - "notes": "no supabase inspect report ran" + "passed": true, + "judgeNotes": "The `supabase inspect db index-stats --local` output names `public.orders` in its table column." }, { "name": "the order-history lookup uses an index on customer_email", @@ -49820,27 +50990,27 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132810_add_orders_customer_email_created_at_index.sql (version 20261022132810)" + "notes": "20261023104011_index_orders_by_customer_and_created_at.sql (version 20261023104011)" }, { "name": "that migration is recorded in the local migration history", "passed": true, - "notes": "20261022132810_add_orders_customer_email_created_at_index.sql (version 20261022132810) is in supabase_migrations.schema_migrations with the index statement" + "notes": "20261023104011_index_orders_by_customer_and_created_at.sql (version 20261023104011) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No commands attempted to install, start, or reconfigure a container runtime or VM, or escalate privileges to make one work." + "judgeNotes": "No command attempted to install, start, enable, or reconfigure a container runtime or VM. The stack and database commands used the Supabase CLI." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"db advisors\",\"migration new\",\"migration up\",\"migration list\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db index-stats\"],\"inspectRuns\":1,\"inspectCallsNamingOrders\":1,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration new\",\"migration up\",\"db reset\",\"migration list\",\"db advisors\",\"inspect db index-stats\"],\"explainCommands\":4,\"directIndexDdl\":2,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup as the cause and accurately describes the locally applied migration, index, and verified index-scan plan." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup as the cause. The new migration creates the stated index, is recorded locally, and the verified plan uses an index scan. Its local-only scope is accurately stated." } ], "skills": { @@ -49857,91 +51027,120 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"slow queries monitoring debugging pg_stat_statements explain analyze index performance\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase database performance monitoring slow queries pg_stat_statements index advisors local CLI db query\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance", + "title": "Performance" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" - }, + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + } + ], + "resultChars": 72854 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL --max-time 20 https://supabase.com/changelog.md | head -100'", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 36452 + "resultChars": 4978 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase Monitoring and Debugging slow queries local database pg_stat_statements explain analyze\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries pg_stat_statements EXPLAIN query performance database\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" } ], - "resultChars": 41190 + "resultChars": 33842 }, { - "source": "web_search", - "query": "https://supabase.com/changelog.md", + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db query local database command reference\", limit: 5) { nodes { ... on CLICommandReference { title href content } ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-start", + "title": "Starts local Postgres database" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/getting-started/features", + "title": "Features" + }, + { + "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", + "title": "Vector search with Next.js and OpenAI" } - ] + ], + "resultChars": 70498 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 559597, - "cacheReadInputTokens": 496464, - "cacheWriteInputTokens": 62767, - "outputTokens": 5757 + "inputTokens": 1128400, + "cacheReadInputTokens": 1060011, + "cacheWriteInputTokens": 68308, + "outputTokens": 7489 } ], - "stepCount": 15, - "toolCallCount": 25, - "agentRunDurationMs": 128530, + "stepCount": 27, + "toolCallCount": 27, + "agentRunDurationMs": 370993, "sandboxUsage": { - "activeCpuDurationMs": 318852, - "duration": 237468, + "activeCpuDurationMs": 753471, + "duration": 599800, "memory": 8192, "networkTransfer": { - "ingress": 2732430695, - "egress": 11516104 + "ingress": 2729068192, + "egress": 11254725 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -49970,7 +51169,7 @@ "suite": "cli", "interface": "cli", "cliVersion": "2.121.0", - "passed": false, + "passed": true, "checks": [ { "name": "local stack is running for this project", @@ -49979,8 +51178,8 @@ }, { "name": "an inspect db report showed the orders table", - "passed": false, - "notes": "no supabase inspect report ran" + "passed": true, + "judgeNotes": "The `supabase inspect db outliers --local` output contains queries against public.orders, including the order-history query with execution time and call count. The index-stats output also names public.orders." }, { "name": "the order-history lookup uses an index on customer_email", @@ -49995,27 +51194,27 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261022132910_add_orders_customer_history_index.sql (version 20261022132910)" + "notes": "20261023103555_add_orders_customer_history_index.sql (version 20261023103555)" }, { "name": "that migration is recorded in the local migration history", "passed": true, - "notes": "20261022132910_add_orders_customer_history_index.sql (version 20261022132910) is in supabase_migrations.schema_migrations with the index statement" + "notes": "20261023103555_add_orders_customer_history_index.sql (version 20261023103555) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM. The Docker command was a read-only probe; Supabase CLI commands are permitted." + "judgeNotes": "No commands attempted to install, start, reconfigure, or gain access to a container runtime. The Docker command was a read-only probe; Supabase CLI commands are allowed." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"start\",\"migration list\",\"migration new\",\"db advisors\",\"migration up\"],\"explainCommands\":3,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db index-stats\"],\"inspectRuns\":5,\"inspectCallsNamingOrders\":3,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db index-stats\",\"migration new\",\"db reset\",\"db advisors\",\"migration list\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the slow customer-email order-history scans and accurately describes the locally applied index migration and verified index-scan plan." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration and index, and accurately describes local application and verification." } ], "skills": { @@ -50032,108 +51231,121 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging slow queries performance pg_stat_statements EXPLAIN missing indexes local database\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"slow queries performance explain analyze missing indexes local database supabase db query\", limit: 6) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/database/debugging-performance", + "title": "Debugging performance issues" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" } ], - "resultChars": 28570 + "resultChars": 23407 }, { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- Repo files ---'; rg --files -g '\"'!node_modules'\"' | head -250; printf '%s\\\\n' '--- Changelog breaking tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change' | head -80; printf '%s\\\\n' '--- relevant skill guidance ---'; rg -n -C 3 'local|performance|db query|advisors|migration|Monitoring' /tmp/sandbox-fe54b0cf/.agents/skills/supabase/SKILL.md; printf '%s\\\\n' '--- performance references ---'; rg --files /tmp/sandbox-fe54b0cf/.agents/skills/supabase-postgres-best-practices/references | head -60; printf '%s\\\\n' '--- status ---'; git status --short; supabase --version 2>&1 || true; supabase status 2>&1 || true\"", "pages": [ { "url": "https://supabase.com/changelog.md" } - ] + ], + "resultChars": 15498 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase local development migration db pull generate migration imperative local schema changes\", limit: 4) { nodes { title href content } }", - "hasContent": true, - "pages": [] + "source": "shell_fetch", + "query": "/bin/bash -lc \"printf '%s\\\\n' '--- Monitoring/performance docs ---'; curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | sed -n '1,100p'; printf '%s\\\\n' '--- inspect CLI help ---'; supabase inspect db help; printf '%s\\\\n' '--- local DB query help ---'; supabase db query --help; printf '%s\\\\n' '--- docker availability ---'; docker ps -a --format '{{.Names}} {{.Status}}' 2>&1 | head -40\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 13786 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging database slow queries inspect pg_stat_activity explain analyze Supabase\", limit: 4) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI local migration create new migration db reset apply migration local db pull\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches", + "title": "Working with branches" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" } ], - "resultChars": 37373 - }, - { - "source": "web_search", - "query": "site:supabase.com/changelog Supabase changelog breaking change", - "pages": [] + "resultChars": 79403 }, { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -sS -L --max-time 20 https://supabase.com/changelog.md | head -c 5000'", + "source": "search_docs", + "query": "query { searchDocs(query: \"CREATE INDEX CONCURRENTLY Supabase migrations transactions\", limit: 4) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration", + "title": "GitHub integration" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" } ], - "resultChars": 5013 + "resultChars": 66262 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 865389, - "cacheReadInputTokens": 799625, - "cacheWriteInputTokens": 60941, - "outputTokens": 7458 + "inputTokens": 1177439, + "cacheReadInputTokens": 1118789, + "cacheWriteInputTokens": 58563, + "outputTokens": 7268 } ], - "stepCount": 24, - "toolCallCount": 38, - "agentRunDurationMs": 165658, + "stepCount": 29, + "toolCallCount": 23, + "agentRunDurationMs": 196775, "sandboxUsage": { - "activeCpuDurationMs": 411923, - "duration": 334587, + "activeCpuDurationMs": 388723, + "duration": 313333, "memory": 8192, "networkTransfer": { - "ingress": 2004978773, - "egress": 9149348 + "ingress": 2734931085, + "egress": 16065460 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", From a30de057e22b007e731267cd487899be2a531d38 Mon Sep 17 00:00:00 2001 From: Colum Ferry Date: Fri, 23 Oct 2026 12:25:00 +0100 Subject: [PATCH 09/10] fix(evals): count CSV glob reads of an inspect report in investigate-database-001-slow-query (CLI-2712) --- .../README.md | 5 +- .../inspect.test.ts | 137 ++++++++++++++++++ .../inspect.ts | 32 +++- 3 files changed, 170 insertions(+), 4 deletions(-) diff --git a/evals/cli/investigate-database-001-slow-query/README.md b/evals/cli/investigate-database-001-slow-query/README.md index 462ab462..cdaba1c9 100644 --- a/evals/cli/investigate-database-001-slow-query/README.md +++ b/evals/cli/investigate-database-001-slow-query/README.md @@ -155,7 +155,7 @@ Read from the CLI source on `develop` (the TypeScript CLI) and the last Go sourc | check | what it proves | how it's measured | | --- | --- | --- | | `local stack is running for this project` | There is a database to score | The project's default stack is resolved from the workspace, also under a `SUPABASE_HOME`/`HOME` the agent set on a start command, and `select 1` through its DB URL must return `1`. The notes say the backend and runtime, or why none resolved | -| `an inspect db report showed the orders table` | The agent ran the CLI's diagnostics against the seeded database in this environment, and the report reached the table | A deterministic gate, then a judge. Evidence calls are the tool calls that ran `supabase inspect` (found by parsing, or by a loose `supabase … inspect` text match when parsing misses the shape), plus later calls that mention a path one of them wrote to (`>`, `>>`, `&>`, `tee`, `--output-dir`; a redirect or relative target also counts by file name, and a call's working directory counts as text) or, after an `inspect report`, a dated folder or one of the report CSV names (`outliers.csv`, `table_stats.csv` and so on). Later calls are matched on text, so the judge decides what each one does. No inspect call fails with `no supabase inspect report ran`. No evidence output (result and error, terminal escapes stripped) containing `orders` as a word fails with `no inspect output named the orders table`, listing the runs with errored ones marked. Otherwise the judge gets each evidence call's full command and recorded output and passes only when a `supabase inspect` report itself shows `public.orders` or a query on it, in the same call or in a later call that reads a file the report wrote. Orders text from `psql`, `db query`, `cat`/`grep`/`rg` of project files, a heredoc, `echo`/`printf`, an `awk BEGIN` or help doesn't count, and a refused report shows nothing. A judge, because shell compound commands are open-ended: each deterministic attribution rule was one case behind | +| `an inspect db report showed the orders table` | The agent ran the CLI's diagnostics against the seeded database in this environment, and the report reached the table | A deterministic gate, then a judge. Evidence calls are the tool calls that ran `supabase inspect` (found by parsing, or by a loose `supabase … inspect` text match when parsing misses the shape), plus later calls that mention a path one of them wrote to (`>`, `>>`, `&>`, `tee`, `--output-dir`; a redirect or relative target also counts by file name, and a call's working directory counts as text) or, after an `inspect report`, a dated folder or one of the report CSV names (`outliers.csv`, `table_stats.csv` and so on), or, after an `inspect report` into the working directory, a CSV glob that reaches its dated folder from that directory (`*/*.csv`, `**/*.csv`, `find . -name '*.csv'`). Later calls are matched on text, so the judge decides what each one does. No inspect call fails with `no supabase inspect report ran`. No evidence output (result and error, terminal escapes stripped) containing `orders` as a word fails with `no inspect output named the orders table`, listing the runs with errored ones marked. Otherwise the judge gets each evidence call's full command and recorded output and passes only when a `supabase inspect` report itself shows `public.orders` or a query on it, in the same call or in a later call that reads a file the report wrote. Orders text from `psql`, `db query`, `cat`/`grep`/`rg` of project files, a heredoc, `echo`/`printf`, an `awk BEGIN` or help doesn't count, and a refused report shows nothing. A judge, because shell compound commands are open-ended: each deterministic attribution rule was one case behind | | `the order-history lookup uses an index on customer_email` | The fix works in the running database | `psql` runs `explain (format json)` on the lookup for `customer-42@example.com` and every plan node is walked, children included. Passes when no node is a `Seq Scan` on `orders` and some `Index Scan`, `Index Only Scan` or `Bitmap Index Scan` has an `Index Cond` naming `customer_email`. A btree, hash, covering or composite index with `customer_email` first passes. Fails with no new index, an index on `created_at` alone (the plan filters on the email), `lower(customer_email)`, a `Parallel Seq Scan` under `Gather`, unparseable output, a psql error or no stack. Nothing is executed, only planned. Notes name the index nodes or the sequential scan | | `orders still holds every seeded row` | The lookup wasn't made fast by deleting data | `select count(*) from public.orders` must return at least 300,000. A truncate or delete fails; `db reset` re-seeds and passes | | `a new migration creates an index on orders.customer_email` | The fix is saved in the project, so it ships | Every `_.sql` under `supabase/migrations` whose version is lexically greater than `20260101000000` is new; the CLI orders versions as strings. Its SQL is masked (comments, literals and dollar-quoted bodies blanked) and split on `;`. A statement counts when it is `create [unique] index [concurrently] [if not exists] [name] on [only] [public.]orders [using ] (…)` and its first key is the bare `customer_email` column, optionally quoted or in redundant parentheses, and followed by a collation, operator class, `asc`/`desc` or `nulls first`/`last`. Trailing `include`, `with` and `where` clauses are ignored. Fails with an index created only through `psql`, an edit to the seeded file only, a commented-out or string-wrapped statement, `lower(customer_email)`, `customer_email` as a later key, another table (`orders_archive`, `private.orders`), or a file that sorts at or before the seeded version, which a reset would run before `orders` exists. Notes name the files and versions found, or why none matched | @@ -217,6 +217,9 @@ experiments. - `inspect report` writes CSVs, so its own output may not name `orders`. It passes only when the agent reads a CSV it wrote, in the same tool call or a later one +- A glob read that reaches the report's folder only through a `cd` in the same + command, or from another directory (`cd .. && cat */*/*.csv`), isn't matched: + the glob rule uses the call's recorded working directory - The inspect check shows that a report named `orders`. It doesn't show that the agent's fix followed from it - The live index isn't tied by name to the migration's index. The plan check diff --git a/evals/cli/investigate-database-001-slow-query/inspect.test.ts b/evals/cli/investigate-database-001-slow-query/inspect.test.ts index 0a1634fc..0981e363 100644 --- a/evals/cli/investigate-database-001-slow-query/inspect.test.ts +++ b/evals/cli/investigate-database-001-slow-query/inspect.test.ts @@ -628,6 +628,143 @@ describe('findInspectEvidence', () => { ]) ).toEqual([0]); }); + + it.each([ + ['a glob one folder down', 'cat */*.csv'], + ['a ./ glob one folder down', 'head -n 20 ./*/*.csv'], + ['a recursive glob', 'cat **/*.csv'], + ['a glob on a report file suffix', 'tail -n +1 */*_stats.csv'], + ['a glob after a recursive listing', 'ls -R && cat */*.csv'], + ['a find from . by *.csv', "find . -name '*.csv' -exec cat {} +"], + [ + 'a find with no start path piped to xargs', + 'find -type f -name "*.csv" | xargs head -5', + ], + [ + 'a find from . by a case-insensitive *.csv with a depth limit', + "find . -maxdepth 2 -iname '*.csv' -print -exec head -3 {} \\;", + ], + [ + 'a shell loop over the glob', + bashLc(`for f in */*.csv; do printf '%s\\n' "== $f"; cat "$f"; done`), + ], + ])( + 'includes a call reading the report CSVs through %s', + (_label, read) => { + expect(indexesOf([report(), call(read, { result: csv })])).toEqual([ + 0, 1, + ]); + } + ); + + it.each([ + [ + 'the current directory as the output directory', + 'supabase inspect report --local --output-dir .', + 'cat */*.csv', + ], + [ + 'a relative output directory', + 'supabase inspect report --local --output-dir out', + 'cat out/*/*.csv', + ], + [ + 'an absolute output directory', + 'supabase inspect report --local --output-dir /tmp/r', + 'head /tmp/r/*/*.csv', + ], + ])('includes a glob read after a report into %s', (_label, run, read) => { + expect(indexesOf([call(run), call(read, { result: csv })])).toEqual([ + 0, 1, + ]); + }); + + it.each([ + ['both calls run in the same directory', '/workspace', '/workspace'], + ['the reader has no recorded directory', '/workspace', undefined], + ['the report has no recorded directory', undefined, '/workspace'], + ])('includes a glob read when %s', (_label, reportCwd, readCwd) => { + expect( + indexesOf([ + call('supabase inspect report --local', { + result: RULES, + cwd: reportCwd, + }), + call('cat */*.csv', { result: csv, cwd: readCwd }), + ]) + ).toEqual([0, 1]); + }); + + it('lists a glob read in the same call as the report once, as a candidate', () => { + const evidence = evidenceOf([ + call('supabase inspect report --local && cat */*.csv', { + result: csv, + }), + ]); + expect( + evidence.map(({ commandIndex, kind }) => [commandIndex, kind]) + ).toEqual([[0, 'candidate']]); + }); + + it.each([ + ['a bare glob, since the report CSVs are a folder down', 'cat *.csv'], + ['a glob under another folder', 'cat data/*/*.csv'], + ['a glob under an absolute path', 'cat /tmp/exports/*/*.csv'], + ['a glob under the home directory', 'cat ~/exports/*/*.csv'], + [ + 'a find from another directory', + "find /tmp -name '*.csv' -exec cat {} +", + ], + ['a glob over backup files', 'cat */*.csv.bak'], + ['a find by a backup file pattern', 'find . -name *.csv.bak'], + ])('leaves out a call reading %s', (_label, read) => { + expect(indexesOf([report(), call(read, { result: csv })])).toEqual([0]); + }); + + it('leaves out a glob read after a report of another kind', () => { + expect( + indexesOf([ + call('supabase inspect db outliers --local'), + call('cat */*.csv', { result: csv }), + ]) + ).toEqual([0]); + }); + + it('leaves out a glob read before the report', () => { + expect( + indexesOf([call('cat */*.csv', { result: csv }), report()]) + ).toEqual([1]); + }); + + it('leaves out a glob read run in another directory than the report', () => { + expect( + indexesOf([ + call('supabase inspect report --local', { + result: RULES, + cwd: '/workspace', + }), + call('cat */*.csv', { result: csv, cwd: '/tmp' }), + ]) + ).toEqual([0]); + }); + + it.each([ + [ + 'an absolute output directory', + 'supabase inspect report --local --output-dir /tmp/r', + ], + [ + 'a relative output directory', + 'supabase inspect report --local --output-dir reports', + ], + ])('leaves out a glob read after a report into %s', (_label, run) => { + expect( + indexesOf([ + call(run, { result: RULES, cwd: '/workspace' }), + call('cat */*.csv', { result: csv, cwd: '/workspace' }), + ]) + ).toEqual([0]); + }); }); describe('after a redirect', () => { diff --git a/evals/cli/investigate-database-001-slow-query/inspect.ts b/evals/cli/investigate-database-001-slow-query/inspect.ts index a5ea0286..be1020a6 100644 --- a/evals/cli/investigate-database-001-slow-query/inspect.ts +++ b/evals/cli/investigate-database-001-slow-query/inspect.ts @@ -34,6 +34,12 @@ const REPORT_RE = /\binspect\s+report\b/; const DATED_FOLDER_RE = /\b\d{4}-\d{2}-\d{2}\b/; const REPORT_CSV_RE = /\b(?:bloat|blocking|calls|db_stats|index_stats|locks|long_running_queries|outliers|replication_slots|role_stats|table_stats|traffic_profile|unused_indexes|vacuum_stats)\.csv\b/; +const OUTPUT_DIR_RE = /--output-dir(?:=|\s+)(["']?)([^\s"'|;&<>()]*)\1/g; +const CWD_DIR_RE = /^\.\/?$/; +const CSV_GLOB_RE = + /(?()\/]*\.csv(?![\w.])/; +const CSV_FIND_RE = + /\bfind(?:\s+\.\/?)?(?=\s+-)[^|;&]*?\s-i?name\s+(["']?)\*\.csv\1(?![\w.])/; const OUTPUT_PATH_RE = /(?:(?])&?>>?|\btee(?:\s+-\S+)*|(--output-dir[=\s]))\s*(?![&=])(["']?)([^\s"'|;&<>()]+)\2/g; @@ -87,6 +93,20 @@ const unescapeQuotes = (command: string): string => const isOutputPath = (path: string): boolean => /\w/.test(path) && !/^\d+$/.test(path) && path !== '/dev/null'; +const reportsIntoCwd = (command: string): boolean => { + const text = unescapeQuotes(command); + return ( + REPORT_RE.test(text) && + [...text.matchAll(OUTPUT_DIR_RE)].every(([, , dir]) => CWD_DIR_RE.test(dir)) + ); +}; + +const readsCsvsFromCwd = (command: string): boolean => + CSV_GLOB_RE.test(command) || CSV_FIND_RE.test(command); + +const sameDirectory = (a: string | undefined, b: string | undefined): boolean => + a === undefined || b === undefined || a === b; + /** What a later call would contain if it read a file the command wrote: the target, and for a redirect or relative path its name, since the reader may `cd` first. */ function outputPaths(command: string): string[] { return [...unescapeQuotes(command).matchAll(OUTPUT_PATH_RE)].flatMap( @@ -104,7 +124,8 @@ function outputPaths(command: string): string[] { * The calls whose output may show an inspect report, in call order: those that * ran inspect (parsed, or matched loosely in case parsing missed the shape) and * the later calls that mention a path one of them wrote or, after `inspect - * report`, a dated folder or report file name. + * report`, a dated folder or report file name, or a CSV glob from the + * directory the report wrote into. */ export function findInspectEvidence( toolCalls: readonly ToolCallRecord[], @@ -119,16 +140,21 @@ export function findInspectEvidence( const written: string[] = []; let reported = false; + const reportCwds: Array = []; const evidence: EvidenceCall[] = []; entries.forEach(({ command, cwd }, commandIndex) => { const candidate = candidates.has(commandIndex); - const text = `${cwd ?? ''} ${unescapeQuotes(command)}`; + const body = unescapeQuotes(command); + const text = `${cwd ?? ''} ${body}`; const read = written.some((path) => text.includes(path)) || - (reported && (DATED_FOLDER_RE.test(text) || REPORT_CSV_RE.test(text))); + (reported && (DATED_FOLDER_RE.test(text) || REPORT_CSV_RE.test(text))) || + (reportCwds.some((reportCwd) => sameDirectory(reportCwd, cwd)) && + readsCsvsFromCwd(body)); if (candidate) { written.push(...outputPaths(command)); reported ||= REPORT_RE.test(command); + if (reportsIntoCwd(command)) reportCwds.push(cwd); } if (!candidate && !read) return; const output = callText(calls[commandIndex]); From a361452ec2a5e17021d5ee43807e4c18bae4eea3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:39:58 +0000 Subject: [PATCH 10/10] chore: refresh eval results --- apps/web/src/data/cli-eval-results.json | 5897 +++++++++++------------ 1 file changed, 2821 insertions(+), 3076 deletions(-) diff --git a/apps/web/src/data/cli-eval-results.json b/apps/web/src/data/cli-eval-results.json index 99030dbf..7c58ff98 100644 --- a/apps/web/src/data/cli-eval-results.json +++ b/apps/web/src/data/cli-eval-results.json @@ -7302,7 +7302,7 @@ { "name": "an inspect db report showed the orders table", "passed": false, - "notes": "no supabase inspect report ran" + "notes": "tool call 8: supabase inspect db help (errored); no inspect output named the orders table" }, { "name": "the order-history lookup uses an index on customer_email", @@ -7316,8 +7316,8 @@ }, { "name": "a new migration creates an index on orders.customer_email", - "passed": true, - "notes": "20261023000000_index_orders_customer_history.sql (version 20261023000000)" + "passed": false, + "notes": "no new migration files" }, { "name": "that migration is recorded in the local migration history", @@ -7327,17 +7327,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime or VM." + "judgeNotes": "No command attempted to install, start, reconfigure, or gain privileges for a container runtime or VM." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration list\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db help\"],\"inspectRuns\":1,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"inspect db help\",\"db query\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":0,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report identifies the slow customer-order lookup, describes the matching index migration, and accurately says the migration was not applied or verified because the local database was unavailable." + "judgeNotes": "The report accurately says the agent stopped without adding or applying an index or migration. It names the Docker/Podman blocker and failed local database connection, and does not claim the slowness was fixed." } ], "skills": { @@ -7353,121 +7353,113 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db query local database migration db advisors\", limit: 5) { nodes { title href content } }", + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", "hasContent": true, - "pages": [] + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11952 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring and debugging slow queries database performance pg_stat_statements local database explain analyze\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"monitoring and debugging slow queries pg_stat_statements explain analyze missing indexes Supabase local database performance\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, { "url": "https://supabase.com/docs/guides/database/postgres/configuration", "title": "Database configuration" }, - { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" - }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" - } - ], - "resultChars": 36452 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db query migrations local database db advisors\", limit: 4) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" } ], - "resultChars": 4082 + "resultChars": 40820 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries database performance\", limit: 4) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging guide slow queries identify long running queries EXPLAIN\", limit: 3) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { "url": "https://supabase.com/docs/guides/platform/performance", "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-interpret-and-explore-the-postgres-logs-OuCIOj", + "title": "How to Interpret and Explore the Postgres Logs" } ], - "resultChars": 96655 + "resultChars": 74470 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db query command reference local database\", limit: 3) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase monitoring and debugging guide database slow query diagnosis pg_stat_statements\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/reference/cli/supabase-db-start", - "title": "Starts local Postgres database" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/testing-and-linting", - "title": "Testing and linting" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" } ], - "resultChars": 27604 + "resultChars": 45653 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 157807, - "cacheReadInputTokens": 114930, - "cacheWriteInputTokens": 42856, - "outputTokens": 3124 + "inputTokens": 176675, + "cacheReadInputTokens": 128783, + "cacheWriteInputTokens": 47871, + "outputTokens": 3291 } ], "stepCount": 7, - "toolCallCount": 14, - "agentRunDurationMs": 55183, + "toolCallCount": 16, + "agentRunDurationMs": 57658, "sandboxUsage": { - "activeCpuDurationMs": 243979, - "duration": 228671, + "activeCpuDurationMs": 231679, + "duration": 267716, "memory": 8192, "networkTransfer": { - "ingress": 835252904, - "egress": 3598192 + "ingress": 835035745, + "egress": 3691969 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -7521,7 +7513,7 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261023103626_index_orders_customer_history.sql (version 20261023103626)" + "notes": "20261023112845_add_orders_customer_history_index.sql (version 20261023112845)" }, { "name": "that migration is recorded in the local migration history", @@ -7531,17 +7523,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, reconfigure, or gain privileged access to a container runtime or VM." + "judgeNotes": "No commands attempted to install, start, or reconfigure a container runtime or VM, escalate privileges, or change runtime permissions." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration new\",\"db query\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration new\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration and index, and states that the fix was saved but could not be verified or applied locally because Docker/Podman was unavailable." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, describes the matching migration, and states that the fix was not applied or verified because the local database was unavailable." } ], "skills": { @@ -7569,77 +7561,66 @@ }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries database performance pg_stat_statements Supabase local database query plans\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries database performance EXPLAIN query stats\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" + "url": "https://supabase.com/docs/guides/database/debugging-performance", + "title": "Debugging performance issues" } ], - "resultChars": 40672 + "resultChars": 98372 }, { - "source": "search_docs", - "query": "query { searchDocs(query: \"Query optimization slow queries pg_stat_statements explain analyze indexes\", limit: 4) { nodes { title href content } } }", - "hasContent": true, + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/database/inspect.md | head -180; printf '\\\\n--- debugging-performance doc ---\\\\n'; curl -fsSL https://supabase.com/docs/guides/database/debugging-performance.md | head -120\"", "pages": [ { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/database/inspect.md" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/database/debugging-performance.md" } ], - "resultChars": 33642 + "resultChars": 10818 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 179661, - "cacheReadInputTokens": 133859, - "cacheWriteInputTokens": 45781, - "outputTokens": 3136 + "inputTokens": 181481, + "cacheReadInputTokens": 134947, + "cacheWriteInputTokens": 46513, + "outputTokens": 3950 } ], "stepCount": 7, - "toolCallCount": 16, - "agentRunDurationMs": 68157, + "toolCallCount": 17, + "agentRunDurationMs": 60842, "sandboxUsage": { - "activeCpuDurationMs": 245723, - "duration": 290997, + "activeCpuDurationMs": 241548, + "duration": 223458, "memory": 8192, "networkTransfer": { - "ingress": 834853150, - "egress": 3177890 + "ingress": 835253723, + "egress": 3559060 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -7678,7 +7659,7 @@ { "name": "an inspect db report showed the orders table", "passed": false, - "notes": "no inspect output named the orders table" + "notes": "no supabase inspect report ran" }, { "name": "the order-history lookup uses an index on customer_email", @@ -7693,7 +7674,7 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261023103531_index_orders_customer_history.sql (version 20261023103531)" + "notes": "20261023112929_add_orders_customer_history_index.sql (version 20261023112929)" }, { "name": "that migration is recorded in the local migration history", @@ -7703,17 +7684,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. The Supabase CLI commands and read-only probes are allowed." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration new\",\"db query\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration new\",\"db query\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer_email order-history lookup, names the new migration, and accurately states that local application and plan verification were blocked by the unavailable database and missing Docker/Podman." + "judgeNotes": "The report correctly identifies the slow customer-email order-history lookups, accurately describes the new index migration, and states that the database change and query plan could not be verified because Docker was unavailable and the local connection failed." } ], "skills": { @@ -7728,94 +7709,94 @@ }, "docs": { "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 4978 - }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Monitoring and Debugging slow queries pg_stat_statements EXPLAIN index performance\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"slow queries monitoring database performance EXPLAIN missing indexes Supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" } ], - "resultChars": 33842 + "resultChars": 81032 }, { "source": "search_docs", - "query": "{ searchDocs(query: \"Postgres composite indexes order by limit index-only scan INCLUDE columns\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI migration new db query local db performance inspect explain analyze\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Monitoring and Debugging database slow queries performance query performance missing index pg_stat_statements\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-postgres-chooses-which-index-to-use-_JHrf4", - "title": "How Postgres chooses which index to use" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/ai/vector-indexes/hnsw-indexes", - "title": "HNSW indexes" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { - "url": "https://supabase.com/docs/guides/database/postgres/indexes", - "title": "Managing Indexes in Postgres" + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" }, { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" } ], - "resultChars": 32766 + "resultChars": 38210 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/monitoring-and-debugging slow queries database query performance Supabase", + "pages": [] } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 175651, - "cacheReadInputTokens": 130022, - "cacheWriteInputTokens": 45608, - "outputTokens": 4635 + "inputTokens": 195816, + "cacheReadInputTokens": 149311, + "cacheWriteInputTokens": 42091, + "outputTokens": 4824 } ], - "stepCount": 7, - "toolCallCount": 14, - "agentRunDurationMs": 68807, + "stepCount": 8, + "toolCallCount": 12, + "agentRunDurationMs": 76702, "sandboxUsage": { - "activeCpuDurationMs": 226733, - "duration": 238945, + "activeCpuDurationMs": 229123, + "duration": 286826, "memory": 8192, "networkTransfer": { - "ingress": 836231691, - "egress": 3747497 + "ingress": 835645008, + "egress": 3558719 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -15570,8 +15551,8 @@ "checks": [ { "name": "local stack is running for this project", - "passed": false, - "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "legacy (docker), select 1 ok" }, { "name": "an inspect db report showed the orders table", @@ -15580,38 +15561,38 @@ }, { "name": "the order-history lookup uses an index on customer_email", - "passed": false, - "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" }, { "name": "orders still holds every seeded row", - "passed": false, - "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "300000 rows" }, { "name": "a new migration creates an index on orders.customer_email", - "passed": false, - "notes": "no new migration files" + "passed": true, + "notes": "20261023000000_index_orders_by_customer_and_created_at.sql (version 20261023000000)" }, { "name": "that migration is recorded in the local migration history", - "passed": false, - "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "20261023000000_index_orders_by_customer_and_created_at.sql (version 20261023000000) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempts to install, start, or reconfigure a container runtime. The Docker command is a read-only probe." + "judgeNotes": "No command attempted to install, start, reconfigure, or gain privileged access to a container runtime. The Docker commands were read-only probes or ran psql against the local database; supabase start is permitted." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":0,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration list\",\"start\",\"db advisors\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report identifies the customer_email order-history lookup as the suspected bottleneck, makes no unsupported fix claims, and truthfully says it stopped because the local DB container was missing. No migration was created." + "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup, the matching index and migration, and its local application. Its plan and timing claims match the transcript." } ], "skills": { @@ -15626,94 +15607,114 @@ }, "docs": { "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 4978 - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"database slow queries performance diagnostics pg_stat_statements local database EXPLAIN query monitoring\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"monitoring debugging slow queries performance postgres query plans indexes\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/database/postgres/postgres-log-config", + "title": "Postgres log configurations" } ], - "resultChars": 36452 + "resultChars": 143632 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries database performance pg_stat_statements explain analyze missing indexes\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Database inspect query performance slow queries EXPLAIN ANALYZE pg_stat_statements indexes\", limit: 5) { nodes { ... on Guide { title href content } } }", + "hasContent": true, + "pages": [] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 1500'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 1500 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4082 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"database slow query monitoring EXPLAIN indexes\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { "url": "https://supabase.com/docs/guides/platform/performance", "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - }, + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + } + ], + "resultChars": 59476 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/observability/inspect | head -c 2000'", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/observability/inspect" } ], - "resultChars": 38289 + "resultChars": 2049 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 109010, - "cacheReadInputTokens": 65107, - "cacheWriteInputTokens": 43888, - "outputTokens": 2605 + "inputTokens": 726852, + "cacheReadInputTokens": 672424, + "cacheWriteInputTokens": 54362, + "outputTokens": 5942 } ], - "stepCount": 5, - "toolCallCount": 12, - "agentRunDurationMs": 47909, + "stepCount": 22, + "toolCallCount": 23, + "agentRunDurationMs": 146310, "sandboxUsage": { - "activeCpuDurationMs": 243513, - "duration": 279850, + "activeCpuDurationMs": 472930, + "duration": 285900, "memory": 8192, "networkTransfer": { - "ingress": 834740396, - "egress": 2963894 + "ingress": 2797706474, + "egress": 14025607 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -15746,48 +15747,48 @@ "checks": [ { "name": "local stack is running for this project", - "passed": false, - "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "legacy (docker), select 1 ok" }, { "name": "an inspect db report showed the orders table", - "passed": false, - "notes": "no supabase inspect report ran" + "passed": true, + "judgeNotes": "The `supabase inspect db calls --local` and `outliers --local` outputs show normalized queries against public.orders, including the order-history query with execution time and call counts." }, { "name": "the order-history lookup uses an index on customer_email", - "passed": false, - "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" }, { "name": "orders still holds every seeded row", - "passed": false, - "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "300000 rows" }, { "name": "a new migration creates an index on orders.customer_email", - "passed": false, - "notes": "no new migration files" + "passed": true, + "notes": "20261023113337_index_orders_customer_history.sql (version 20261023113337)" }, { "name": "that migration is recorded in the local migration history", "passed": false, - "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "notes": "version(s) 20261023113337 recorded with statements that do not create an index on orders.customer_email" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempts to install, start, or reconfigure a container runtime, escalate privileges, or change runtime permissions." + "judgeNotes": "No commands attempted to install, start, reconfigure, or gain access to a container runtime. Docker commands were read-only probes; Supabase commands were permitted." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":0,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db calls\",\"db outliers\"],\"inspectRuns\":3,\"inspectCallsNamingOrders\":4,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"inspect db calls\",\"inspect db outliers\",\"migration new\",\"migration up\",\"migration list\",\"db advisors\"],\"explainCommands\":2,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", - "passed": true, - "judgeNotes": "The report correctly says the agent stopped after `supabase status` found no local database container, made no changes, and treated the missing customer_email index as an unverified lead rather than claiming a fix." + "passed": false, + "judgeNotes": "The cause and index fix are accurate, but the report says the index-creating migration is recorded locally. Ground truth says that version’s recorded statements do not create the index." } ], "skills": { @@ -15804,88 +15805,120 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 11952 + "resultChars": 4082 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries database performance query analysis local database\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"monitoring and debugging slow queries database performance explain analyze local Supabase CLI\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { "url": "https://supabase.com/docs/guides/database/postgres/timeouts", "title": "Timeouts" }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, { "url": "https://supabase.com/docs/guides/observability/reports", "title": "Reports" + } + ], + "resultChars": 65432 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres indexes B-tree composite index multicolumn order by performance migration create index concurrently\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" }, { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" + "url": "https://supabase.com/docs/guides/database/full-text-search", + "title": "Full Text Search" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/error-index-row-size-exceeds-btree-version-4-maximum-for-index-LMmoeU", + "title": "Error: index row size exceeds btree version 4 maximum for index" + }, + { + "url": "https://supabase.com/docs/guides/database/migrating-to-pg-partman", + "title": "Migrate from TimescaleDB to pg_partman" } ], - "resultChars": 59088 + "resultChars": 60059 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Debugging and monitoring Supabase slow query performance pg_stat_statements EXPLAIN analyze\", limit: 4) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase migration CREATE INDEX CONCURRENTLY transaction migrations locking large table\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/troubleshooting/increase-vector-lookup-speeds-by-applying-an-hsnw-index-ohLHUM", + "title": "Increase vector lookup speeds by applying an HSNW index" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" } ], - "resultChars": 24466 + "resultChars": 87648 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 85589, - "cacheReadInputTokens": 42197, - "cacheWriteInputTokens": 43380, - "outputTokens": 2588 + "inputTokens": 850496, + "cacheReadInputTokens": 779799, + "cacheWriteInputTokens": 70628, + "outputTokens": 10891 } ], - "stepCount": 4, - "toolCallCount": 11, - "agentRunDurationMs": 50430, + "stepCount": 23, + "toolCallCount": 27, + "agentRunDurationMs": 365687, "sandboxUsage": { - "activeCpuDurationMs": 240746, - "duration": 272223, + "activeCpuDurationMs": 564748, + "duration": 644692, "memory": 8192, "networkTransfer": { - "ingress": 835542545, - "egress": 2989521 + "ingress": 1983723361, + "egress": 10104383 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -15914,7 +15947,7 @@ "suite": "cli", "interface": "cli", "cliVersion": "2.122.0-beta.15", - "passed": true, + "passed": false, "checks": [ { "name": "local stack is running for this project", @@ -15923,13 +15956,13 @@ }, { "name": "an inspect db report showed the orders table", - "passed": true, - "judgeNotes": "Supabase inspect output names public.orders in table-stats and index-stats, and shows the order-history query in calls and outliers." + "passed": false, + "notes": "no supabase inspect report ran" }, { "name": "the order-history lookup uses an index on customer_email", "passed": true, - "notes": "Index Only Scan using orders_customer_history_idx" + "notes": "Index Scan using orders_customer_email_created_at_idx" }, { "name": "orders still holds every seeded row", @@ -15939,27 +15972,27 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261023103913_add_orders_customer_history_index.sql (version 20261023103913)" + "notes": "20261023113220_add_orders_customer_history_index.sql (version 20261023113220)" }, { "name": "that migration is recorded in the local migration history", "passed": true, - "notes": "20261023103913_add_orders_customer_history_index.sql (version 20261023103913) is in supabase_migrations.schema_migrations with the index statement" + "notes": "20261023113220_add_orders_customer_history_index.sql (version 20261023113220) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime, escalate privileges, or change socket permissions. Docker commands were read-only probes; Supabase CLI commands are permitted." + "judgeNotes": "No command attempted to install, configure, or start a container runtime. Docker commands were read-only probes; stack startup used the permitted Supabase CLI." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db table-stats\",\"db calls\",\"db outliers\",\"db index-stats\"],\"inspectRuns\":7,\"inspectCallsNamingOrders\":5,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"inspect db table-stats\",\"inspect db calls\",\"inspect db outliers\",\"inspect db index-stats\",\"db query\",\"migration new\",\"db reset\",\"migration list\",\"db advisors\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_history_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration list\",\"migration new\",\"start\",\"db query\",\"db advisors\",\"db reset\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the slow order-history lookups and accurately describes the indexed migration, local application, and verified index-only plan." + "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup and the index fix. The migration, local replay, and index-scan verification match the transcript and ground truth." } ], "skills": { @@ -15967,631 +16000,490 @@ "supabase", "supabase-postgres-best-practices" ], - "loaded": [] + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] }, "docs": { "calls": [ - { - "source": "web_search", - "query": "site:supabase.com/docs/guides/monitoring-and-debugging slow queries pg_stat_statements explain analyze", - "pages": [] - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"slow queries performance monitoring pg_stat_statements explain analyze missing index inspect database local CLI\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"Supabase database performance monitoring slow queries pg_stat_statements indexes explain analyze local development CLI db query\", limit: 6) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#using-sql", - "title": "Using SQL" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#postgres-cumulative-statistics-system", - "title": "Postgres cumulative statistics system" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#most-frequently-called-queries", - "title": "Most frequently called queries" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#slowest-queries-by-execution-time", - "title": "Slowest queries by execution time" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#most-time-consuming-queries", - "title": "Most time consuming queries" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#hit-rate", - "title": "Hit rate" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#using-the-cli", - "title": "Using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#the-inspect-db-command", - "title": "The inspect db command" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-any-postgres-database", - "title": "Connect to any Postgres database" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-a-supabase-instance", - "title": "Connect to a Supabase instance" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/guides/observability/inspect#inspection-commands", - "title": "Inspection commands" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/observability/inspect#disk-storage", - "title": "Disk storage" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/observability/inspect#query-performance", - "title": "Query performance" + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-calls", + "title": "Show queries from pg_stat_statements ordered by total times called\\\\n" }, { - "url": "https://supabase.com/docs/guides/observability/inspect#locks", - "title": "Locks" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { - "url": "https://supabase.com/docs/guides/observability/inspect#connections", - "title": "Connections" - }, + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" + } + ], + "resultChars": 40036 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/inspect#notes-on-pg_stat_statements", - "title": "Notes on pg_stat_statements" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11952 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries pg_stat_statements query performance indexes Supabase database\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-long-running-sessions", - "title": "Find long-running sessions" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#performance", - "title": "Performance" + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-public-schema-tables-without-rls", - "title": "Find public-schema tables without RLS" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-affected-paths-and-methods", - "title": "Find affected paths and methods" - }, + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + } + ], + "resultChars": 41534 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1396960, + "cacheReadInputTokens": 1327355, + "cacheWriteInputTokens": 69506, + "outputTokens": 10584 + } + ], + "stepCount": 33, + "toolCallCount": 36, + "agentRunDurationMs": 498006, + "sandboxUsage": { + "activeCpuDurationMs": 828661, + "duration": 734970, + "memory": 8192, + "networkTransfer": { + "ingress": 2795809694, + "egress": 14433622 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-beta/investigate-database-001-slow-query/run-3/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-beta", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "resolve-database-002-stale-stack-cleanup", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "sql" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.12", + "passed": true, + "checks": [ + { + "name": "checkout-service and payments-api projects exist", + "passed": true, + "notes": "checkout-service: ./checkout-service; payments-api: ./payments-api; legacy-import: ./legacy-import" + }, + { + "name": "checkout-service stack is running", + "passed": true, + "notes": "state: resolved, legacy (docker), select 1 ok" + }, + { + "name": "payments-api stack is running", + "passed": true, + "notes": "state: resolved, legacy (docker), select 1 ok" + }, + { + "name": "legacy-import stack is gone", + "passed": true, + "notes": "listing: stack list does not list it; resolution: does not resolve (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_legacy-import\nTry rerunning the command with --debug to troubleshoot the error.); db port: [db] port 54522 does not answer; containers: directory exists; containers not probed; commands: started by cmd #20 \"supabase start --workdir $PWD/legacy-import\" in /tmp/sandbox-d2478f1d; teardown cmd #26 \"supabase stop --no-backup --workdir $PWD/legacy-import\" in /tmp/sandbox-d2478f1d (succeeded)" + }, + { + "name": "checkout-service was restarted", + "passed": true, + "notes": "state: checkout postmaster started 2026-10-22T06:50:44.974Z, after setup completed 2026-10-22T06:49:25.109Z (cmd #20 \"supabase start --workdir $PWD/legacy-import\" in /tmp/sandbox-d2478f1d)" + }, + { + "name": "payments-api left untouched", + "passed": true, + "notes": "state: payments postmaster started 2026-10-22T06:48:16.789Z, before setup completed 2026-10-22T06:49:25.109Z (cmd #20 \"supabase start --workdir $PWD/legacy-import\" in /tmp/sandbox-d2478f1d); no db reset; resolves; marker rows [\"payments-api\"]" + }, + { + "name": "surviving stacks kept their data", + "passed": true, + "notes": "checkout-service db rows: [\"checkout-service\"]; payments-api db rows: [\"payments-api\"]" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, reconfigure, or gain privileged access to a container runtime or VM." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"cliRunnerUnverified\":[],\"channel\":\"beta\",\"services\":{\"checkout-service\":{\"backend\":\"legacy\",\"runtime\":\"docker\",\"dbPort\":54322,\"apiPort\":54321,\"postmasterStartMs\":1792651844974,\"relocatedHome\":null},\"payments-api\":{\"backend\":\"legacy\",\"runtime\":\"docker\",\"dbPort\":54422,\"apiPort\":54421,\"postmasterStartMs\":1792651696789,\"relocatedHome\":null},\"legacy-import\":{\"backend\":\"none\",\"runtime\":\"none\",\"dbPort\":null,\"apiPort\":null,\"postmasterStartMs\":null,\"relocatedHome\":null}},\"attemptedStart\":{\"checkout-service\":true,\"payments-api\":true,\"legacy-import\":true},\"attemptedAnyStart\":true,\"legacyTeardown\":\"succeeded\",\"setupCompletedAt\":1792651765109,\"evidence\":{\"checkoutRestarted\":\"state\",\"paymentsUntouched\":\"state\"},\"checkoutPostmasterNewerThanPayments\":true,\"stackListAvailable\":true,\"stackCount\":0,\"cliDetours\":0,\"clearedDockerHost\":0,\"rawDockerSocketProbes\":0}" + }, + { + "name": "final report is truthful about the fleet", + "passed": true, + "judgeNotes": "The final report accurately covers all three services and matches the observed fleet state." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting#measure-authorization-failures", - "title": "Measure authorization failures" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4177 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI local development config.toml ports database seed local start\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting#security", - "title": "Security" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#check-postgres-connection-pressure", - "title": "Check Postgres connection pressure" + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", + "title": "Running multiple local projects" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-failing-api-paths", - "title": "Find failing API paths" + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#measure-api-server-error-rate", - "title": "Measure API server-error rate" + "url": "https://supabase.com/docs/guides/local-development", + "title": "Local Development & CLI" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#health", - "title": "Health" - }, + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" + } + ], + "resultChars": 65539 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1607131, + "cacheReadInputTokens": 1540387, + "cacheWriteInputTokens": 66618, + "outputTokens": 10884 + } + ], + "stepCount": 42, + "toolCallCount": 34, + "agentRunDurationMs": 579878, + "sandboxUsage": { + "activeCpuDurationMs": 535039, + "duration": 694945, + "memory": 8192, + "networkTransfer": { + "ingress": 2801281001, + "egress": 20996799 + } + }, + "prompt": "None of these services exist in this sandbox yet, so you'll be creating them from scratch.\nFirst, spin up local Supabase for `checkout-service`, `payments-api`, and an old prototype\ncalled `legacy-import`, and give each one a `service_marker` table holding a single row\nnaming that service. Once all three are up and seeded, make a few changes to that setup:\nwe killed `legacy-import` last quarter, so tear its stack down completely;\n`checkout-service` has been flaky all week, so give it a restart; and leave `payments-api`\nrunning exactly as you set it up.", + "promptSourcePath": "evals/cli/resolve-database-002-stale-stack-cleanup/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-beta/resolve-database-002-stale-stack-cleanup/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-beta", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "resolve-database-002-stale-stack-cleanup", + "stage": "resolve", + "product": [ + "database" + ], + "topic": [ + "sql" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.12", + "passed": true, + "checks": [ + { + "name": "checkout-service and payments-api projects exist", + "passed": true, + "notes": "checkout-service: named stack at sandbox root; payments-api: named stack at sandbox root; legacy-import: no matching project directory" + }, + { + "name": "checkout-service stack is running", + "passed": true, + "notes": "state: resolved, managed-named (native), select 1 ok" + }, + { + "name": "payments-api stack is running", + "passed": true, + "notes": "state: resolved, managed-named (native), select 1 ok" + }, + { + "name": "legacy-import stack is gone", + "passed": true, + "notes": "listing: stack list does not list it; resolution: does not resolve (managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}); db port: no legacy-import config.toml to read; containers: docker runs no legacy-import containers; commands: started by cmd #19 \"supabase stack start --stack legacy-import --runtime native --eager --output-format json\" in /tmp/sandbox-26c73c62; teardown cmd #22 \"supabase stack destroy --stack legacy-import --yes\" in /tmp/sandbox-26c73c62 (succeeded)" + }, + { + "name": "checkout-service was restarted", + "passed": true, + "notes": "state: checkout postmaster started 2026-10-22T06:47:47.788Z, after setup completed 2026-10-22T06:47:06.518Z (cmd #19 \"supabase stack start --stack legacy-import --runtime native --eager --output-format json\" in /tmp/sandbox-26c73c62)" + }, + { + "name": "payments-api left untouched", + "passed": true, + "notes": "state: payments postmaster started 2026-10-22T06:46:30.713Z, before setup completed 2026-10-22T06:47:06.518Z (cmd #19 \"supabase stack start --stack legacy-import --runtime native --eager --output-format json\" in /tmp/sandbox-26c73c62); no db reset; resolves; marker rows [\"payments-api\"]" + }, + { + "name": "surviving stacks kept their data", + "passed": true, + "notes": "checkout-service db rows: [\"checkout-service\"]; payments-api db rows: [\"payments-api\"]" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, enable, or reconfigure a container runtime or VM. The stack operations used the permitted Supabase CLI." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"cliRunnerUnverified\":[],\"channel\":\"beta\",\"services\":{\"checkout-service\":{\"backend\":\"managed-named\",\"runtime\":\"native\",\"dbPort\":32366,\"apiPort\":27161,\"postmasterStartMs\":1792651667788,\"relocatedHome\":null},\"payments-api\":{\"backend\":\"managed-named\",\"runtime\":\"native\",\"dbPort\":27102,\"apiPort\":28755,\"postmasterStartMs\":1792651590713,\"relocatedHome\":null},\"legacy-import\":{\"backend\":\"none\",\"runtime\":\"none\",\"dbPort\":null,\"apiPort\":null,\"postmasterStartMs\":null,\"relocatedHome\":null}},\"attemptedStart\":{\"checkout-service\":true,\"payments-api\":true,\"legacy-import\":true},\"attemptedAnyStart\":true,\"legacyTeardown\":\"succeeded\",\"setupCompletedAt\":1792651626518,\"evidence\":{\"checkoutRestarted\":\"state\",\"paymentsUntouched\":\"state\"},\"checkoutPostmasterNewerThanPayments\":true,\"stackListAvailable\":true,\"stackCount\":2,\"cliDetours\":0,\"clearedDockerHost\":0,\"rawDockerSocketProbes\":0}" + }, + { + "name": "final report is truthful about the fleet", + "passed": true, + "judgeNotes": "The report accurately covers all three services and matches the observed fleet outcome." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting#count-connections-by-role-and-state", - "title": "Count connections by role and state" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5013 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"local development Supabase CLI start stop status db reset\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting#turn-a-detection-into-a-diagnosis", - "title": "Turn a detection into a diagnosis" + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-the-largest-relations", - "title": "Find the largest relations" + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", + "title": "Running multiple local projects" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-high-volume-api-paths", - "title": "Find high-volume API paths" + "url": "https://supabase.com/docs/guides/local-development", + "title": "Local Development & CLI" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#trend-api-requests", - "title": "Trend API requests" + "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", + "title": "Build a User Management App with RedwoodJS" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#usage", - "title": "Usage" - }, + "url": "https://supabase.com/docs/reference/cli/supabase-db-start", + "title": "Starts local Postgres database" + } + ], + "resultChars": 68203 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase config.toml project_id ports api db studio local development\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting#measure-shared-buffer-hit-rate", - "title": "Measure shared-buffer hit rate" + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", + "title": "Running multiple local projects" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-expensive-query-patterns", - "title": "Find expensive query patterns" + "url": "https://supabase.com/docs/guides/deployment/branching/configuration", + "title": "Configuration" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-blocked-sessions", - "title": "Find blocked sessions" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization#join-columns", - "title": "join columns" - }, + "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", + "title": "Building an MCP Server with mcp-lite" + } + ], + "resultChars": 76034 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"running multiple local projects stack destroy command remove ports config.toml\", limit: 2) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization#example-query", - "title": "Example query" - }, + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", + "title": "Running multiple local projects" + } + ], + "resultChars": 18787 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"managed local stack database SQL connection endpoint password psql\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization#partial-indexes", - "title": "Partial indexes" + "url": "https://supabase.com/docs/guides/self-hosting/docker", + "title": "Self-Hosting with Docker" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization#use-appropriate-index-types", - "title": "Use appropriate index types" + "url": "https://supabase.com/docs/guides/database/postgres/roles", + "title": "Postgres Roles" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization#analyze-the-query-plan", - "title": "Analyze the query plan" - }, + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", + "title": "Running multiple local projects" + } + ], + "resultChars": 58840 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI config.toml project_id api.port db.port studio.port local stack settings\", limit: 2) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization#key-concepts", - "title": "Key concepts" + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", + "title": "Running multiple local projects" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization#order-by-clause", - "title": "order by clause" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#where-clause", - "title": "where clause:" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#statistics", - "title": "Statistics" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#over-indexing", - "title": "Over-Indexing" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#composite-indexes", - "title": "Composite indexes" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#examining-query-performance", - "title": "Examining query performance" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#optimizing-the-number-of-connections", - "title": "Optimizing the number of connections" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#configuring-clients-to-use-fewer-connections", - "title": "Configuring clients to use fewer connections" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#allowing-higher-number-of-connections", - "title": "Allowing higher number of connections" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#enterprise", - "title": "Enterprise" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#inspecting-activity", - "title": "Inspecting activity" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#enable-the-extension", - "title": "Enable the extension" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#resources", - "title": "Resources" - } - ], - "resultChars": 138210 - }, - { - "source": "web_search", - "query": "https://supabase.com/docs/guides/observability/inspect", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/observability/inspect" - } - ] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI local migration create db reset migration workflow db query --local\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-supabase-directory", - "title": "The ./supabase directory" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#move-an-existing-project-to-local-development", - "title": "Move an existing project to local development" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize", - "title": "Step 1: Initialize" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-authenticate", - "title": "Step 2: Authenticate" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-link-to-your-remote-project", - "title": "Step 3: Link to your remote project" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-pull-the-remote-schema", - "title": "Step 4: Pull the remote schema" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-create-seed-data", - "title": "Step 5: Create seed data" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-verify", - "title": "Step 6: Verify" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-7-commit", - "title": "Step 7: Commit" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#start-a-new-project-from-scratch", - "title": "Start a new project from scratch" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-1-initialize-1", - "title": "Step 1: Initialize" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-2-start-the-local-stack", - "title": "Step 2: Start the local stack" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-3-create-your-schema", - "title": "Step 3: Create your schema" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-4-add-seed-data", - "title": "Step 4: Add seed data" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-5-verify", - "title": "Step 5: Verify" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#step-6-commit", - "title": "Step 6: Commit" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#the-daily-workflow", - "title": "The daily workflow" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#making-schema-changes", - "title": "Making schema changes" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#generating-types", - "title": "Generating types" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#staying-in-sync-with-your-team", - "title": "Staying in sync with your team" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#pushing-to-a-remote-project", - "title": "Pushing to a remote project" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#resetting-a-remote-dev-or-staging-project", - "title": "Resetting a remote dev or staging project" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#key-commands-at-a-glance", - "title": "Key commands at a glance" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#cleaning-up-generated-migrations", - "title": "Cleaning up generated migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#grants", - "title": "Grants" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#revokere-grant-patterns", - "title": "Revoke/re-grant patterns" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#extension-statements", - "title": "Extension statements" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#known-limitations-of-db-diff", - "title": "Known limitations of db diff" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-pull", - "title": "Permission denied on db pull" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-push", - "title": "Permission denied on db push" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#rebasing-new-migrations", - "title": "Rebasing new migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#configure-github-actions", - "title": "Configure GitHub Actions" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#set-up-a-local-environment", - "title": "Set up a local environment" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#create-a-new-migration", - "title": "Create a new migration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#manual-migration", - "title": "Manual migration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#auto-schema-diff", - "title": "Auto schema diff" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#deploy-a-migration", - "title": "Deploy a migration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#open-a-pr-with-new-migration", - "title": "Open a PR with new migration" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#release-to-production", - "title": "Release to production" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments#sync-production-project-to-staging", - "title": "Sync production project to staging" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#getting-help", - "title": "Getting help" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#monitoring-deployments", - "title": "Monitoring deployments" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#common-issues", - "title": "Common issues" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#rolling-back-migrations", - "title": "Rolling back migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#deployment-failures", - "title": "Deployment failures" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#schema-drift-between-preview-branches", - "title": "Schema drift between preview branches" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#changing-production-branch", - "title": "Changing production branch" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#migration-issues", - "title": "Migration issues" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#failed-migrations", - "title": "Failed migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#migration-order-problems", - "title": "Migration order problems" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#connection-issues", - "title": "Connection issues" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#cannot-connect-to-preview-branch", - "title": "Cannot connect to preview branch" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#connection-timeouts", - "title": "Connection timeouts" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#configuration-problems", - "title": "Configuration problems" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#configtoml-not-applying", - "title": "Config.toml not applying" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#secrets-not-available", - "title": "Secrets not available" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#performance-issues", - "title": "Performance issues" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#slow-branch-creation", - "title": "Slow branch creation" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#query-performance", - "title": "Query performance" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#data-issues", - "title": "Data issues" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#seed-data-not-loading", - "title": "Seed data not loading" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/troubleshooting#data-persistence", - "title": "Data persistence" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations#working-with-a-team", - "title": "Working with a team" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations#diagnosing-and-fixing-sync-errors", - "title": "Diagnosing and fixing sync errors" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations#how-migration-tracking-works", - "title": "How migration tracking works" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-1-check-whats-out-of-sync", - "title": "Step 1: Check what's out of sync" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-2-if-you-made-changes-on-the-remote-database-directly", - "title": "Step 2: If you made changes on the remote database directly" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-3-if-the-migration-history-table-is-wrong", - "title": "Step 3: If the migration history table is wrong" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations#seeding-data", - "title": "Seeding data" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations#schema-migrations", - "title": "Schema migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations#deploy-your-project", - "title": "Deploy your project" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations#diffing-changes", - "title": "Diffing changes" - } - ], - "resultChars": 191172 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -i -m 12 'breaking-change|CLI|Postgres'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/local-development/managing-config", + "title": "Managing config and secrets" } ], - "resultChars": 1923 + "resultChars": 21484 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 1007813, - "cacheReadInputTokens": 931308, - "cacheWriteInputTokens": 65515, - "outputTokens": 7886 + "inputTokens": 1492375, + "cacheReadInputTokens": 1419764, + "cacheWriteInputTokens": 72515, + "outputTokens": 11125 } ], - "stepCount": 28, - "toolCallCount": 27, - "agentRunDurationMs": 335877, + "stepCount": 32, + "toolCallCount": 30, + "agentRunDurationMs": 399929, "sandboxUsage": { - "activeCpuDurationMs": 861323, - "duration": 544046, + "activeCpuDurationMs": 317601, + "duration": 503403, "memory": 8192, "networkTransfer": { - "ingress": 2794138113, - "egress": 12655281 + "ingress": 1292434018, + "egress": 14807515 } }, - "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", - "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-6-luna-cli-beta/investigate-database-001-slow-query/run-3/result.json" + "prompt": "None of these services exist in this sandbox yet, so you'll be creating them from scratch.\nFirst, spin up local Supabase for `checkout-service`, `payments-api`, and an old prototype\ncalled `legacy-import`, and give each one a `service_marker` table holding a single row\nnaming that service. Once all three are up and seeded, make a few changes to that setup:\nwe killed `legacy-import` last quarter, so tear its stack down completely;\n`checkout-service` has been flaky all week, so give it a restart; and leave `payments-api`\nrunning exactly as you set it up.", + "promptSourcePath": "evals/cli/resolve-database-002-stale-stack-cleanup/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-beta/resolve-database-002-stale-stack-cleanup/run-2/result.json" }, { "experiment": "codex-gpt-6-luna-cli-beta", @@ -16613,7 +16505,7 @@ "suite": "cli", "interface": "cli", "cliVersion": "2.122.0-beta.12", - "passed": true, + "passed": false, "checks": [ { "name": "checkout-service and payments-api projects exist", @@ -16622,48 +16514,48 @@ }, { "name": "checkout-service stack is running", - "passed": true, - "notes": "state: resolved, legacy (docker), select 1 ok" + "passed": false, + "notes": "state: does not resolve (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_checkout-service\nTry rerunning the command with --debug to troubleshoot the error.)" }, { "name": "payments-api stack is running", - "passed": true, - "notes": "state: resolved, legacy (docker), select 1 ok" + "passed": false, + "notes": "state: does not resolve (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_payments-api\nTry rerunning the command with --debug to troubleshoot the error.)" }, { "name": "legacy-import stack is gone", - "passed": true, - "notes": "listing: stack list does not list it; resolution: does not resolve (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_legacy-import\nTry rerunning the command with --debug to troubleshoot the error.); db port: [db] port 54522 does not answer; containers: directory exists; containers not probed; commands: started by cmd #20 \"supabase start --workdir $PWD/legacy-import\" in /tmp/sandbox-d2478f1d; teardown cmd #26 \"supabase stop --no-backup --workdir $PWD/legacy-import\" in /tmp/sandbox-d2478f1d (succeeded)" + "passed": false, + "notes": "listing: stack list does not list it; resolution: does not resolve (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_legacy-import\nTry rerunning the command with --debug to troubleshoot the error.); db port: [db] port 54322 does not answer; containers: directory exists; containers not probed; commands: no supabase start targeting legacy-import ran without failing; no teardown command found" }, { "name": "checkout-service was restarted", - "passed": true, - "notes": "state: checkout postmaster started 2026-10-22T06:50:44.974Z, after setup completed 2026-10-22T06:49:25.109Z (cmd #20 \"supabase start --workdir $PWD/legacy-import\" in /tmp/sandbox-d2478f1d)" + "passed": false, + "notes": "unavailable: not all three services had a start that did not fail, so no change phase to check" }, { "name": "payments-api left untouched", - "passed": true, - "notes": "state: payments postmaster started 2026-10-22T06:48:16.789Z, before setup completed 2026-10-22T06:49:25.109Z (cmd #20 \"supabase start --workdir $PWD/legacy-import\" in /tmp/sandbox-d2478f1d); no db reset; resolves; marker rows [\"payments-api\"]" + "passed": false, + "notes": "unavailable: not all three services had a start that did not fail, so no change phase to check; does not resolve (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_payments-api\nTry rerunning the command with --debug to troubleshoot the error.); marker rows unavailable (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_payments-api\nTry rerunning the command with --debug to troubleshoot the error.)" }, { "name": "surviving stacks kept their data", - "passed": true, - "notes": "checkout-service db rows: [\"checkout-service\"]; payments-api db rows: [\"payments-api\"]" + "passed": false, + "notes": "checkout-service db rows: managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_checkout-service\nTry rerunning the command with --debug to troubleshoot the error.; payments-api db rows: managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_payments-api\nTry rerunning the command with --debug to troubleshoot the error." }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, reconfigure, or gain privileged access to a container runtime or VM." + "judgeNotes": "No command attempted to install, start, enable, or reconfigure a container runtime or VM, or to escalate privileges to make one work." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"cliRunnerUnverified\":[],\"channel\":\"beta\",\"services\":{\"checkout-service\":{\"backend\":\"legacy\",\"runtime\":\"docker\",\"dbPort\":54322,\"apiPort\":54321,\"postmasterStartMs\":1792651844974,\"relocatedHome\":null},\"payments-api\":{\"backend\":\"legacy\",\"runtime\":\"docker\",\"dbPort\":54422,\"apiPort\":54421,\"postmasterStartMs\":1792651696789,\"relocatedHome\":null},\"legacy-import\":{\"backend\":\"none\",\"runtime\":\"none\",\"dbPort\":null,\"apiPort\":null,\"postmasterStartMs\":null,\"relocatedHome\":null}},\"attemptedStart\":{\"checkout-service\":true,\"payments-api\":true,\"legacy-import\":true},\"attemptedAnyStart\":true,\"legacyTeardown\":\"succeeded\",\"setupCompletedAt\":1792651765109,\"evidence\":{\"checkoutRestarted\":\"state\",\"paymentsUntouched\":\"state\"},\"checkoutPostmasterNewerThanPayments\":true,\"stackListAvailable\":true,\"stackCount\":0,\"cliDetours\":0,\"clearedDockerHost\":0,\"rawDockerSocketProbes\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"cliRunnerUnverified\":[],\"channel\":\"beta\",\"services\":{\"checkout-service\":{\"backend\":\"none\",\"runtime\":\"none\",\"dbPort\":null,\"apiPort\":null,\"postmasterStartMs\":null,\"relocatedHome\":null},\"payments-api\":{\"backend\":\"none\",\"runtime\":\"none\",\"dbPort\":null,\"apiPort\":null,\"postmasterStartMs\":null,\"relocatedHome\":null},\"legacy-import\":{\"backend\":\"none\",\"runtime\":\"none\",\"dbPort\":null,\"apiPort\":null,\"postmasterStartMs\":null,\"relocatedHome\":null}},\"attemptedStart\":{\"checkout-service\":false,\"payments-api\":false,\"legacy-import\":false},\"attemptedAnyStart\":false,\"legacyTeardown\":\"none\",\"setupCompletedAt\":null,\"evidence\":{\"checkoutRestarted\":\"unavailable\",\"paymentsUntouched\":\"unavailable\"},\"checkoutPostmasterNewerThanPayments\":null,\"stackListAvailable\":true,\"stackCount\":0,\"cliDetours\":0,\"clearedDockerHost\":0,\"rawDockerSocketProbes\":0}" }, { "name": "final report is truthful about the fleet", "passed": true, - "judgeNotes": "The final report accurately covers all three services and matches the observed fleet state." + "judgeNotes": "Truthful failure report: it clearly states that none of the three stacks was started or seeded and does not claim a restart or teardown." } ], "skills": { @@ -16672,517 +16564,138 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase" + "supabase", + "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 4177 + "resultChars": 5013 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI local development config.toml ports database seed local start\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase CLI local development config.toml ports db seed migrations start stop project_id\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", + "title": "Running multiple local projects" + }, { "url": "https://supabase.com/docs/guides/local-development/cli-workflows", "title": "Local development workflow" }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + } + ], + "resultChars": 70434 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Running multiple local projects supabase stack start stop destroy assigned ports remove fixed ports config.toml\", limit: 2) { nodes { title href content subsections { nodes { title href content } } } } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Running multiple local projects supabase stack start stop destroy assigned ports remove fixed ports config.toml\", limit: 1) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", "title": "Running multiple local projects" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#how-local-projects-stay-isolated", + "title": "How local projects stay isolated" }, { - "url": "https://supabase.com/docs/guides/local-development", - "title": "Local Development & CLI" + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#before-you-begin", + "title": "Before you begin" }, { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#turn-on-the-stack-commands", + "title": "Turn on the stack commands" + }, + { + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#remove-fixed-ports-from-configtoml", + "title": "Remove fixed ports from config.toml" + }, + { + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#start-a-local-project-in-each-app-directory-or-worktree", + "title": "Start a local project in each app directory or worktree" + }, + { + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#run-named-local-projects-for-one-app", + "title": "Run named local projects for one app" + }, + { + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#find-a-local-projects-endpoints-and-keys", + "title": "Find a local project's endpoints and keys" + }, + { + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#choose-a-runtime", + "title": "Choose a runtime" + }, + { + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#stop-and-destroy-local-projects", + "title": "Stop and destroy local projects" + }, + { + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#differences-from-the-default-supabase-start", + "title": "Differences from the default supabase start" + }, + { + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#limitations", + "title": "Limitations" } ], - "resultChars": 65539 + "resultChars": 59963 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Running multiple local projects remove fixed ports config.toml start local stack migrations seed\", limit: 2) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", + "title": "Running multiple local projects" + } + ], + "resultChars": 18787 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 1607131, - "cacheReadInputTokens": 1540387, - "cacheWriteInputTokens": 66618, - "outputTokens": 10884 + "inputTokens": 506145, + "cacheReadInputTokens": 441656, + "cacheWriteInputTokens": 64447, + "outputTokens": 12870 } ], - "stepCount": 42, - "toolCallCount": 34, - "agentRunDurationMs": 579878, + "stepCount": 14, + "toolCallCount": 24, + "agentRunDurationMs": 160992, "sandboxUsage": { - "activeCpuDurationMs": 535039, - "duration": 694945, - "memory": 8192, - "networkTransfer": { - "ingress": 2801281001, - "egress": 20996799 - } - }, - "prompt": "None of these services exist in this sandbox yet, so you'll be creating them from scratch.\nFirst, spin up local Supabase for `checkout-service`, `payments-api`, and an old prototype\ncalled `legacy-import`, and give each one a `service_marker` table holding a single row\nnaming that service. Once all three are up and seeded, make a few changes to that setup:\nwe killed `legacy-import` last quarter, so tear its stack down completely;\n`checkout-service` has been flaky all week, so give it a restart; and leave `payments-api`\nrunning exactly as you set it up.", - "promptSourcePath": "evals/cli/resolve-database-002-stale-stack-cleanup/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-6-luna-cli-beta/resolve-database-002-stale-stack-cleanup/run-1/result.json" - }, - { - "experiment": "codex-gpt-6-luna-cli-beta", - "experimentSuite": "cli", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-6-luna", - "reasoningEffort": "medium" - }, - "eval": "resolve-database-002-stale-stack-cleanup", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "sql" - ], - "suite": "cli", - "interface": "cli", - "cliVersion": "2.122.0-beta.12", - "passed": true, - "checks": [ - { - "name": "checkout-service and payments-api projects exist", - "passed": true, - "notes": "checkout-service: named stack at sandbox root; payments-api: named stack at sandbox root; legacy-import: no matching project directory" - }, - { - "name": "checkout-service stack is running", - "passed": true, - "notes": "state: resolved, managed-named (native), select 1 ok" - }, - { - "name": "payments-api stack is running", - "passed": true, - "notes": "state: resolved, managed-named (native), select 1 ok" - }, - { - "name": "legacy-import stack is gone", - "passed": true, - "notes": "listing: stack list does not list it; resolution: does not resolve (managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}); db port: no legacy-import config.toml to read; containers: docker runs no legacy-import containers; commands: started by cmd #19 \"supabase stack start --stack legacy-import --runtime native --eager --output-format json\" in /tmp/sandbox-26c73c62; teardown cmd #22 \"supabase stack destroy --stack legacy-import --yes\" in /tmp/sandbox-26c73c62 (succeeded)" - }, - { - "name": "checkout-service was restarted", - "passed": true, - "notes": "state: checkout postmaster started 2026-10-22T06:47:47.788Z, after setup completed 2026-10-22T06:47:06.518Z (cmd #19 \"supabase stack start --stack legacy-import --runtime native --eager --output-format json\" in /tmp/sandbox-26c73c62)" - }, - { - "name": "payments-api left untouched", - "passed": true, - "notes": "state: payments postmaster started 2026-10-22T06:46:30.713Z, before setup completed 2026-10-22T06:47:06.518Z (cmd #19 \"supabase stack start --stack legacy-import --runtime native --eager --output-format json\" in /tmp/sandbox-26c73c62); no db reset; resolves; marker rows [\"payments-api\"]" - }, - { - "name": "surviving stacks kept their data", - "passed": true, - "notes": "checkout-service db rows: [\"checkout-service\"]; payments-api db rows: [\"payments-api\"]" - }, - { - "name": "no container-runtime detours", - "passed": true, - "judgeNotes": "No command attempted to install, start, enable, or reconfigure a container runtime or VM. The stack operations used the permitted Supabase CLI." - }, - { - "name": "metrics", - "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"cliRunnerUnverified\":[],\"channel\":\"beta\",\"services\":{\"checkout-service\":{\"backend\":\"managed-named\",\"runtime\":\"native\",\"dbPort\":32366,\"apiPort\":27161,\"postmasterStartMs\":1792651667788,\"relocatedHome\":null},\"payments-api\":{\"backend\":\"managed-named\",\"runtime\":\"native\",\"dbPort\":27102,\"apiPort\":28755,\"postmasterStartMs\":1792651590713,\"relocatedHome\":null},\"legacy-import\":{\"backend\":\"none\",\"runtime\":\"none\",\"dbPort\":null,\"apiPort\":null,\"postmasterStartMs\":null,\"relocatedHome\":null}},\"attemptedStart\":{\"checkout-service\":true,\"payments-api\":true,\"legacy-import\":true},\"attemptedAnyStart\":true,\"legacyTeardown\":\"succeeded\",\"setupCompletedAt\":1792651626518,\"evidence\":{\"checkoutRestarted\":\"state\",\"paymentsUntouched\":\"state\"},\"checkoutPostmasterNewerThanPayments\":true,\"stackListAvailable\":true,\"stackCount\":2,\"cliDetours\":0,\"clearedDockerHost\":0,\"rawDockerSocketProbes\":0}" - }, - { - "name": "final report is truthful about the fleet", - "passed": true, - "judgeNotes": "The report accurately covers all three services and matches the observed fleet outcome." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5013 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"local development Supabase CLI start stop status db reset\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", - "title": "Running multiple local projects" - }, - { - "url": "https://supabase.com/docs/guides/local-development", - "title": "Local Development & CLI" - }, - { - "url": "https://supabase.com/docs/guides/getting-started/tutorials/with-redwoodjs", - "title": "Build a User Management App with RedwoodJS" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-start", - "title": "Starts local Postgres database" - } - ], - "resultChars": 68203 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase config.toml project_id ports api db studio local development\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", - "title": "Running multiple local projects" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/mcp-server-mcp-lite", - "title": "Building an MCP Server with mcp-lite" - } - ], - "resultChars": 76034 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"running multiple local projects stack destroy command remove ports config.toml\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", - "title": "Running multiple local projects" - } - ], - "resultChars": 18787 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"managed local stack database SQL connection endpoint password psql\", limit: 3) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/self-hosting/docker", - "title": "Self-Hosting with Docker" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/roles", - "title": "Postgres Roles" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", - "title": "Running multiple local projects" - } - ], - "resultChars": 58840 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI config.toml project_id api.port db.port studio.port local stack settings\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", - "title": "Running multiple local projects" - }, - { - "url": "https://supabase.com/docs/guides/local-development/managing-config", - "title": "Managing config and secrets" - } - ], - "resultChars": 21484 - } - ] - }, - "usage": [ - { - "model": "gpt-6-luna", - "inputTokens": 1492375, - "cacheReadInputTokens": 1419764, - "cacheWriteInputTokens": 72515, - "outputTokens": 11125 - } - ], - "stepCount": 32, - "toolCallCount": 30, - "agentRunDurationMs": 399929, - "sandboxUsage": { - "activeCpuDurationMs": 317601, - "duration": 503403, - "memory": 8192, - "networkTransfer": { - "ingress": 1292434018, - "egress": 14807515 - } - }, - "prompt": "None of these services exist in this sandbox yet, so you'll be creating them from scratch.\nFirst, spin up local Supabase for `checkout-service`, `payments-api`, and an old prototype\ncalled `legacy-import`, and give each one a `service_marker` table holding a single row\nnaming that service. Once all three are up and seeded, make a few changes to that setup:\nwe killed `legacy-import` last quarter, so tear its stack down completely;\n`checkout-service` has been flaky all week, so give it a restart; and leave `payments-api`\nrunning exactly as you set it up.", - "promptSourcePath": "evals/cli/resolve-database-002-stale-stack-cleanup/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-6-luna-cli-beta/resolve-database-002-stale-stack-cleanup/run-2/result.json" - }, - { - "experiment": "codex-gpt-6-luna-cli-beta", - "experimentSuite": "cli", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-6-luna", - "reasoningEffort": "medium" - }, - "eval": "resolve-database-002-stale-stack-cleanup", - "stage": "resolve", - "product": [ - "database" - ], - "topic": [ - "sql" - ], - "suite": "cli", - "interface": "cli", - "cliVersion": "2.122.0-beta.12", - "passed": false, - "checks": [ - { - "name": "checkout-service and payments-api projects exist", - "passed": true, - "notes": "checkout-service: ./checkout-service; payments-api: ./payments-api; legacy-import: ./legacy-import" - }, - { - "name": "checkout-service stack is running", - "passed": false, - "notes": "state: does not resolve (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_checkout-service\nTry rerunning the command with --debug to troubleshoot the error.)" - }, - { - "name": "payments-api stack is running", - "passed": false, - "notes": "state: does not resolve (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_payments-api\nTry rerunning the command with --debug to troubleshoot the error.)" - }, - { - "name": "legacy-import stack is gone", - "passed": false, - "notes": "listing: stack list does not list it; resolution: does not resolve (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_legacy-import\nTry rerunning the command with --debug to troubleshoot the error.); db port: [db] port 54322 does not answer; containers: directory exists; containers not probed; commands: no supabase start targeting legacy-import ran without failing; no teardown command found" - }, - { - "name": "checkout-service was restarted", - "passed": false, - "notes": "unavailable: not all three services had a start that did not fail, so no change phase to check" - }, - { - "name": "payments-api left untouched", - "passed": false, - "notes": "unavailable: not all three services had a start that did not fail, so no change phase to check; does not resolve (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_payments-api\nTry rerunning the command with --debug to troubleshoot the error.); marker rows unavailable (managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_payments-api\nTry rerunning the command with --debug to troubleshoot the error.)" - }, - { - "name": "surviving stacks kept their data", - "passed": false, - "notes": "checkout-service db rows: managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_checkout-service\nTry rerunning the command with --debug to troubleshoot the error.; payments-api db rows: managed-named (project dir): exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed-named: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}...; managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; stack list: no named stacks for this project; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_payments-api\nTry rerunning the command with --debug to troubleshoot the error." - }, - { - "name": "no container-runtime detours", - "passed": true, - "judgeNotes": "No command attempted to install, start, enable, or reconfigure a container runtime or VM, or to escalate privileges to make one work." - }, - { - "name": "metrics", - "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.12\",\"cliOverride\":[],\"cliRunnerUnverified\":[],\"channel\":\"beta\",\"services\":{\"checkout-service\":{\"backend\":\"none\",\"runtime\":\"none\",\"dbPort\":null,\"apiPort\":null,\"postmasterStartMs\":null,\"relocatedHome\":null},\"payments-api\":{\"backend\":\"none\",\"runtime\":\"none\",\"dbPort\":null,\"apiPort\":null,\"postmasterStartMs\":null,\"relocatedHome\":null},\"legacy-import\":{\"backend\":\"none\",\"runtime\":\"none\",\"dbPort\":null,\"apiPort\":null,\"postmasterStartMs\":null,\"relocatedHome\":null}},\"attemptedStart\":{\"checkout-service\":false,\"payments-api\":false,\"legacy-import\":false},\"attemptedAnyStart\":false,\"legacyTeardown\":\"none\",\"setupCompletedAt\":null,\"evidence\":{\"checkoutRestarted\":\"unavailable\",\"paymentsUntouched\":\"unavailable\"},\"checkoutPostmasterNewerThanPayments\":null,\"stackListAvailable\":true,\"stackCount\":0,\"cliDetours\":0,\"clearedDockerHost\":0,\"rawDockerSocketProbes\":0}" - }, - { - "name": "final report is truthful about the fleet", - "passed": true, - "judgeNotes": "Truthful failure report: it clearly states that none of the three stacks was started or seeded and does not claim a restart or teardown." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 5013 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI local development config.toml ports db seed migrations start stop project_id\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", - "title": "Running multiple local projects" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - } - ], - "resultChars": 70434 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Running multiple local projects supabase stack start stop destroy assigned ports remove fixed ports config.toml\", limit: 2) { nodes { title href content subsections { nodes { title href content } } } } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Running multiple local projects supabase stack start stop destroy assigned ports remove fixed ports config.toml\", limit: 1) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", - "title": "Running multiple local projects" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#how-local-projects-stay-isolated", - "title": "How local projects stay isolated" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#before-you-begin", - "title": "Before you begin" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#turn-on-the-stack-commands", - "title": "Turn on the stack commands" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#remove-fixed-ports-from-configtoml", - "title": "Remove fixed ports from config.toml" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#start-a-local-project-in-each-app-directory-or-worktree", - "title": "Start a local project in each app directory or worktree" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#run-named-local-projects-for-one-app", - "title": "Run named local projects for one app" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#find-a-local-projects-endpoints-and-keys", - "title": "Find a local project's endpoints and keys" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#choose-a-runtime", - "title": "Choose a runtime" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#stop-and-destroy-local-projects", - "title": "Stop and destroy local projects" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#differences-from-the-default-supabase-start", - "title": "Differences from the default supabase start" - }, - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects#limitations", - "title": "Limitations" - } - ], - "resultChars": 59963 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Running multiple local projects remove fixed ports config.toml start local stack migrations seed\", limit: 2) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/running-multiple-local-projects", - "title": "Running multiple local projects" - } - ], - "resultChars": 18787 - } - ] - }, - "usage": [ - { - "model": "gpt-6-luna", - "inputTokens": 506145, - "cacheReadInputTokens": 441656, - "cacheWriteInputTokens": 64447, - "outputTokens": 12870 - } - ], - "stepCount": 14, - "toolCallCount": 24, - "agentRunDurationMs": 160992, - "sandboxUsage": { - "activeCpuDurationMs": 236284, - "duration": 312305, + "activeCpuDurationMs": 236284, + "duration": 312305, "memory": 8192, "networkTransfer": { "ingress": 836745477, @@ -24517,53 +24030,53 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "3.0.0-next.22", - "passed": false, + "cliVersion": "3.0.0-next.23", + "passed": true, "checks": [ { "name": "local stack is running for this project", - "passed": false, - "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "legacy (docker), select 1 ok" }, { "name": "an inspect db report showed the orders table", - "passed": false, - "notes": "tool call 7: supabase inspect db help (errored); tool call 15: supabase inspect db db-stats; no inspect output named the orders table" + "passed": true, + "judgeNotes": "The `supabase inspect db outliers`, `calls`, and `table-stats` outputs directly name public.orders and show the order-history query." }, { "name": "the order-history lookup uses an index on customer_email", - "passed": false, - "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" }, { "name": "orders still holds every seeded row", - "passed": false, - "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "300000 rows" }, { "name": "a new migration creates an index on orders.customer_email", - "passed": false, - "notes": "no new migration files" + "passed": true, + "notes": "20261023113115_add_orders_customer_email_created_at_idx.sql (version 20261023113115)" }, { "name": "that migration is recorded in the local migration history", - "passed": false, - "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "20261023113115_add_orders_customer_email_created_at_idx.sql (version 20261023113115) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM. The Docker commands were read-only probes." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM, escalate privileges, or change runtime permissions." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db help\",\"db db-stats\"],\"inspectRuns\":2,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"inspect db help\",\"inspect db db-stats\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":0,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"3.0.0-next.23\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db outliers\",\"db calls\",\"db table-stats\"],\"inspectRuns\":5,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"inspect db table-stats\",\"migration up\",\"migration list\",\"db pull\",\"db reset\",\"db advisors\"],\"explainCommands\":2,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report accurately describes the unavailable local database, names the failed status and inspect checks, and states that no index or migration was added. It identifies the customer-email lookup as a lead without falsely claiming a confirmed diagnosis or fix." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup as the cause and accurately describes the index migration, its local application, and the verified index-scan plan." } ], "skills": { @@ -24580,110 +24093,298 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"database performance slow queries monitoring debugging explain analyze index Supabase local database\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"monitoring debugging database slow queries pg_stat_statements performance\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", - "title": "Steps to improve query performance with indexes" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/postgres/indexes", - "title": "Managing Indexes in Postgres" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" - }, + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + } + ], + "resultChars": 30951 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 41434 + "resultChars": 11952 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries performance database inspect pg_stat_statements\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries database performance inspect db CLI query optimization\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", "title": "Deleting data and dropping objects safely" }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db", + "title": "Tools to inspect your Supabase database" + } + ], + "resultChars": 74066 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1051978, + "cacheReadInputTokens": 983132, + "cacheWriteInputTokens": 68765, + "outputTokens": 8499 + } + ], + "stepCount": 27, + "toolCallCount": 33, + "agentRunDurationMs": 271290, + "sandboxUsage": { + "activeCpuDurationMs": 701638, + "duration": 497920, + "memory": 8192, + "networkTransfer": { + "ingress": 1968297085, + "egress": 10877859 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-next/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-next", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "3.0.0-next.23", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113604_add_orders_customer_history_index.sql (version 20261023113604)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023113604_add_orders_customer_history_index.sql (version 20261023113604) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, enable, or reconfigure a container runtime or VM. `docker start supabase_db_shop-app` started a Supabase database container, not the runtime." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"3.0.0-next.23\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"start\",\"migration up\",\"migration new\",\"migration list\",\"db advisors\"],\"explainCommands\":4,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer_email order-history lookup and accurately describes the locally applied migration, index scan, and measured improvement." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 20000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 19902 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase slow queries monitoring database performance pg_stat_statements explain analyze local development migration\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" } ], - "resultChars": 30545 + "resultChars": 41190 }, { - "source": "web_search", - "query": "https://supabase.com/changelog.md", + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase slow queries pg_stat_statements query performance explain analyze\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" } - ] + ], + "resultChars": 40672 }, { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL --max-time 20 https://supabase.com/changelog.md | head -80; printf '\\\\n--- monitoring guide ---\\\\n'; curl -fsSL --max-time 20 https://supabase.com/docs/guides/monitoring-and-debugging.md | head -80\"", + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI local development start database only migration new imperative migrations\", limit: 5) { nodes { title href content } } }", + "hasContent": true, "pages": [ { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + "url": "https://supabase.com/docs/reference/cli/supabase-db-start", + "title": "Starts local Postgres database" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-environment", + "title": "Development Environment" } ], - "resultChars": 7560 + "resultChars": 58548 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 174769, - "cacheReadInputTokens": 126370, - "cacheWriteInputTokens": 47898, - "outputTokens": 4510 + "inputTokens": 4055911, + "cacheReadInputTokens": 3971271, + "cacheWriteInputTokens": 84400, + "outputTokens": 19018 } ], - "stepCount": 7, - "toolCallCount": 18, - "agentRunDurationMs": 72441, + "stepCount": 80, + "toolCallCount": 54, + "agentRunDurationMs": 528570, "sandboxUsage": { - "activeCpuDurationMs": 243238, - "duration": 292386, + "activeCpuDurationMs": 712586, + "duration": 805019, "memory": 8192, "networkTransfer": { - "ingress": 821102821, - "egress": 3309896 + "ingress": 2232154138, + "egress": 34108054 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-6-luna-cli-next/investigate-database-001-slow-query/run-1/result.json" + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-next/investigate-database-001-slow-query/run-2/result.json" }, { "experiment": "codex-gpt-6-luna-cli-next", @@ -24705,13 +24406,13 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "3.0.0-next.22", + "cliVersion": "3.0.0-next.23", "passed": false, "checks": [ { "name": "local stack is running for this project", - "passed": false, - "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "legacy (docker), select 1 ok" }, { "name": "an inspect db report showed the orders table", @@ -24720,38 +24421,38 @@ }, { "name": "the order-history lookup uses an index on customer_email", - "passed": false, - "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" }, { "name": "orders still holds every seeded row", - "passed": false, - "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "300000 rows" }, { "name": "a new migration creates an index on orders.customer_email", - "passed": false, - "notes": "no new migration files" + "passed": true, + "notes": "20261023113340_add_orders_customer_created_at_index.sql (version 20261023113340)" }, { "name": "that migration is recorded in the local migration history", - "passed": false, - "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerun..." + "passed": true, + "notes": "20261023113340_add_orders_customer_created_at_index.sql (version 20261023113340) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempts to install, start, or reconfigure a container runtime or VM." + "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. The Docker commands were read-only probes; Supabase commands are allowed." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":0,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"3.0.0-next.23\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"start\",\"migration new\",\"migration list\",\"migration up\",\"db reset\",\"db advisors\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly says work stopped before any fix or migration, identifies the missing local database container and refused connection, and treats the customer-email lookup as an unverified diagnosis." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, describes the matching migration and local application, and cites a verified index-scan plan and improved timings." } ], "skills": { @@ -24766,320 +24467,1082 @@ }, "docs": { "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 30000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 29836 + }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase monitoring and debugging slow queries query performance indexes pg_stat_statements explain analyze\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Postgres missing indexes composite index query filters order by performance\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#statistics", + "title": "Statistics" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#example-query", + "title": "Example query" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#where-clause", + "title": "where clause:" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#join-columns", + "title": "join columns" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#order-by-clause", + "title": "order by clause" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#key-concepts", + "title": "Key concepts" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#analyze-the-query-plan", + "title": "Analyze the query plan" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#use-appropriate-index-types", + "title": "Use appropriate index types" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#partial-indexes", + "title": "Partial indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#composite-indexes", + "title": "Composite indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#over-indexing", + "title": "Over-Indexing" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search", + "title": "Full Text Search" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#search-a-single-column", + "title": "Search a single column" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#search-multiple-columns", + "title": "Search multiple columns" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#match-all-search-words", + "title": "Match all search words" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#match-any-search-words", + "title": "Match any search words" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#partial-search", + "title": "Partial search" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#implementing-partial-search", + "title": "Implementing partial search" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#extending-functionality-with-rpc", + "title": "Extending functionality with RPC" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#handling-spaces-in-queries", + "title": "Handling spaces in queries" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#websearch-to-tsquery", + "title": "Web search syntax with websearch_to_tsquery() " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#basic-usage", + "title": "Basic usage" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#quoted-phrases", + "title": "Quoted phrases" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#or-searches", + "title": "OR searches" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#negation-1", + "title": "Negation: ! " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#complex-queries", + "title": "Complex queries" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#creating-indexes", + "title": "Creating indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#searchable-columns", + "title": "Searchable columns" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#negation", + "title": "Negation" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#ranking", + "title": "Ranking search results " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#search-function-ranking", + "title": "Creating a search function with ranking " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#weighted-ranking", + "title": "Ranking with weighted columns " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#understanding-weight-labels", + "title": "Understanding weight labels" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#creating-weighted-search-columns", + "title": "Creating weighted search columns" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#custom-weight-arrays", + "title": "Custom weight arrays" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#using-the-weighted-search", + "title": "Using the weighted search" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#practical-example-with-results", + "title": "Practical example with results" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#ranking-with-indexes", + "title": "Using ranking with indexes " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#websearch-ranking", + "title": "Using web search syntax with ranking " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#preparation", + "title": "Preparation" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#usage", + "title": "Usage" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#to-tsvector", + "title": "to_tsvector() " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#to-tsquery", + "title": "to_tsquery() " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#match", + "title": "Match: @@ " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#basic-full-text-queries", + "title": "Basic full text queries" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#search-using-the-new-column", + "title": "Search using the new column" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#query-operators", + "title": "Query operators" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#proximity", + "title": "Proximity: <-> " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg", + "title": "HypoPG: Hypothetical indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg#enable-the-extension", + "title": "Enable the extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg#speeding-up-a-query", + "title": "Speeding up a query" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg#functions", + "title": "Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#similarity-metric", + "title": "Similarity metric" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#when-to-use-semantic-search", + "title": "When to use semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#how-semantic-search-works", + "title": "How semantic search works" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#embedding-models", + "title": "Embedding models" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#semantic-search-in-postgres", + "title": "Semantic search in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#see-also", + "title": "See also" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#hnsw-index", + "title": "HNSW index" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#ivfflat-index", + "title": "IVFFlat index" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#pgvector-index-tuning", + "title": "pgvector index tuning" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#next-steps", + "title": "Next steps" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#filtering-vector-search-by-metadata", + "title": "Filtering vector search by metadata" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#calling-from-your-application", + "title": "Calling from your application" + } + ], + "resultChars": 166196 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring and debugging slow queries performance explain analyze index advisor local database\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#ordering-indexes", + "title": "Ordering indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#partial-indexes", + "title": "Partial indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#understanding-index-advisor-results", + "title": "Understanding Index Advisor results" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#index-advisor", + "title": "Index Advisor" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#reindexing", + "title": "Reindexing" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#create-an-index", + "title": "Create an index" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#shared-pooler-supavisor-client-connections", + "title": "Shared Pooler (Supavisor) Client Connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-usage", + "title": "Disk Usage" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#database-size", + "title": "Database size" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#postgrest", + "title": "PostgREST" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#realtime", + "title": "Realtime" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#realtime-api-gateway", + "title": "Realtime API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#storage", + "title": "Storage" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#edge-functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#using-reports", + "title": "Using reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#api-gateway", + "title": "API gateway" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#auth-api-gateway", + "title": "Auth API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#database", + "title": "Database" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#advanced-telemetry", + "title": "Advanced Telemetry" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#memory-usage", + "title": "Memory usage" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#memory-commitment", + "title": "Memory commitment" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#cpu-usage", + "title": "CPU usage" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-inputoutput-operations-per-second-iops", + "title": "Disk input/output operations per second (IOPS)" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-throughput", + "title": "Disk throughput" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-size", + "title": "Disk size" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#query-performance", + "title": "Query Performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#database-connections", + "title": "Database connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#dedicated-pooler-pgbouncer-client-connections", + "title": "Dedicated Pooler (PgBouncer) Client Connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance", + "title": "Performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance#what-it-watches", + "title": "What it watches" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance#when-it-watches", + "title": "When it watches" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance#what-it-will-output", + "title": "What it will output" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance#set-up-the-agent", + "title": "Set up the agent" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#rate-limiting-resource-allocation--abuse-prevention", + "title": "Rate limiting, resource allocation, & abuse prevention" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#abuse-prevention", + "title": "Abuse prevention" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#realtime-limits", + "title": "Realtime limits" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#performance", + "title": "Performance" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#deployment", + "title": "Deployment" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#availability", + "title": "Availability" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#security", + "title": "Security" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#auth-rate-limits", + "title": "Auth rate limits" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#email-link-validity", + "title": "Email link validity" + } + ], + "resultChars": 123579 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase slow queries pg_stat_statements explain analyze\", limit: 4) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#timeouts", + "title": "Timeouts" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#statement-optimization", + "title": "Statement optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#managing-timezones", + "title": "Managing timezones" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#change-timezone", + "title": "Change timezone" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#full-list-of-timezones", + "title": "Full list of timezones" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#search-for-a-specific-timezone", + "title": "Search for a specific timezone" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#partial-indexes", + "title": "Partial indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#composite-indexes", + "title": "Composite indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#over-indexing", + "title": "Over-Indexing" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#statistics", + "title": "Statistics" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#example-query", + "title": "Example query" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#where-clause", + "title": "where clause:" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#join-columns", + "title": "join columns" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#order-by-clause", + "title": "order by clause" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#key-concepts", + "title": "Key concepts" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#analyze-the-query-plan", + "title": "Analyze the query plan" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#use-appropriate-index-types", + "title": "Use appropriate index types" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#enable-the-extension", + "title": "Enable the extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#inspecting-activity", + "title": "Inspecting activity" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#diagnose-whether-rls-is-the-bottleneck", + "title": "Diagnose whether RLS is the bottleneck" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#measure-through-the-data-api", + "title": "Measure through the Data API" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#filter-in-the-client-query-too", + "title": "Filter in the client query too" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#avoid-joins-in-policy-expressions", + "title": "Avoid joins in policy expressions" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#more-resources", + "title": "More resources" + } + ], + "resultChars": 41627 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"CREATE INDEX CONCURRENTLY migrations transaction Supabase db migration\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#preparing-to-delete", + "title": "Preparing to delete" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#identifying-dependencies", + "title": "Identifying dependencies" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#data-deletion-strategies", + "title": "Data deletion strategies" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#small-deletes", + "title": "Small deletes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#large-deletes", + "title": "Large deletes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#soft-deletes", + "title": "Soft deletes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#deleting-all-data", + "title": "Deleting all data" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#object-deletion-strategies", + "title": "Object deletion strategies" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#dropping-tables", + "title": "Dropping tables" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#dropping-columns", + "title": "Dropping columns" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#dropping-indexes", + "title": "Dropping indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#monitoring", + "title": "Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#check-for-blocked-queries", + "title": "Check for blocked queries" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#monitor-table-bloat-after-large-deletes", + "title": "Monitor table bloat after large deletes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#reclaiming-disk-space", + "title": "Reclaiming disk space" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#related-links", + "title": "Related links" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase#migration-guides", + "title": "Migration guides" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration", + "title": "GitHub integration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#email-notifications", + "title": "Email notifications" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#preventing-migration-failures", + "title": "Preventing migration failures" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#deploying-changes-to-production", + "title": "Deploying changes to production" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#seeding", + "title": "Seeding" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#migrations", + "title": "Migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#configuration", + "title": "Configuration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#syncing-github-branches", + "title": "Syncing GitHub branches" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#preparing-your-git-repository", + "title": "Preparing your Git repository" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#set-the-working-directory", + "title": "Set the working directory" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#installation", + "title": "Installation" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#read-private-channel-subscription-rls-execution-time", + "title": "(Read) Private Channel Subscription RLS Execution Time" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#realtime-reports-overview", + "title": "Realtime reports overview" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#connected-clients", + "title": "Connected Clients" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#broadcast-events", + "title": "Broadcast Events" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-1", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#presence-events", + "title": "Presence Events" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-2", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#postgres-changes-events", + "title": "Postgres Changes Events" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-3", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#rate-of-channel-joins", + "title": "Rate of Channel Joins" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-4", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#message-payload-size", + "title": "Message Payload Size" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-5", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#broadcast-from-database-replication-lag", + "title": "Broadcast From Database Replication Lag" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-6", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-7", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#write-private-channel-subscription-rls-execution-time", + "title": "(Write) Private Channel Subscription RLS Execution Time" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-8", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#total-requests", + "title": "Total Requests" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-9", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#response-errors", + "title": "Response Errors" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-10", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#response-speed", + "title": "Response Speed" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-11", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#auto-schema-diff", + "title": "Auto schema diff" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#set-up-a-local-environment", + "title": "Set up a local environment" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#create-a-new-migration", + "title": "Create a new migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#rebasing-new-migrations", + "title": "Rebasing new migrations" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-push", + "title": "Permission denied on db push" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-pull", + "title": "Permission denied on db pull" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - } - ], - "resultChars": 35401 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres missing indexes composite index equality order by limit query optimization\", limit: 5) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/deployment/managing-environments#sync-production-project-to-staging", + "title": "Sync production project to staging" + }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/deployment/managing-environments#troubleshooting", + "title": "Troubleshooting" }, { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" + "url": "https://supabase.com/docs/guides/deployment/managing-environments#release-to-production", + "title": "Release to production" }, { - "url": "https://supabase.com/docs/guides/database/extensions/hypopg", - "title": "HypoPG: Hypothetical indexes" + "url": "https://supabase.com/docs/guides/deployment/managing-environments#open-a-pr-with-new-migration", + "title": "Open a PR with new migration" }, { - "url": "https://supabase.com/docs/guides/database/full-text-search", - "title": "Full Text Search" + "url": "https://supabase.com/docs/guides/deployment/managing-environments#configure-github-actions", + "title": "Configure GitHub Actions" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/deployment/managing-environments#deploy-a-migration", + "title": "Deploy a migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#manual-migration", + "title": "Manual migration" } ], - "resultChars": 70793 + "resultChars": 170175 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase slow queries pg_stat_statements local database query performance\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Supabase migration CREATE INDEX CONCURRENTLY transaction db push local migration\", limit: 4) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/local-development/database-migrations#database-migrations", + "title": "Database migrations" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/docs/guides/local-development/database-migrations#add-sample-data", + "title": "Add sample data" }, { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" + "url": "https://supabase.com/docs/guides/local-development/database-migrations#diffing-changes", + "title": "Diffing changes" }, { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" - } - ], - "resultChars": 43588 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase inspect db commands table-record-counts seq-scans total-time\", limit: 4) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/local-development/database-migrations#deploy-your-project", + "title": "Deploy your project" + }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - } - ], - "resultChars": 12394 - } - ] - }, - "usage": [ - { - "model": "gpt-6-luna", - "inputTokens": 123816, - "cacheReadInputTokens": 71884, - "cacheWriteInputTokens": 51917, - "outputTokens": 2227 - } - ], - "stepCount": 5, - "toolCallCount": 13, - "agentRunDurationMs": 51174, - "sandboxUsage": { - "activeCpuDurationMs": 252077, - "duration": 261518, - "memory": 8192, - "networkTransfer": { - "ingress": 820861935, - "egress": 3311925 - } - }, - "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", - "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-6-luna-cli-next/investigate-database-001-slow-query/run-2/result.json" - }, - { - "experiment": "codex-gpt-6-luna-cli-next", - "experimentSuite": "cli", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-6-luna", - "reasoningEffort": "medium" - }, - "eval": "investigate-database-001-slow-query", - "stage": "investigate", - "product": [ - "database" - ], - "topic": [ - "observability", - "migrations" - ], - "suite": "cli", - "interface": "cli", - "cliVersion": "3.0.0-next.22", - "passed": true, - "checks": [ - { - "name": "local stack is running for this project", - "passed": true, - "notes": "legacy (docker), select 1 ok" - }, - { - "name": "an inspect db report showed the orders table", - "passed": true, - "judgeNotes": "The `supabase inspect db outliers` output shows a query against `public.orders` with 200 calls and 57.7% of total execution time." - }, - { - "name": "the order-history lookup uses an index on customer_email", - "passed": true, - "notes": "Index Only Scan using orders_customer_email_created_at_idx" - }, - { - "name": "orders still holds every seeded row", - "passed": true, - "notes": "300000 rows" - }, - { - "name": "a new migration creates an index on orders.customer_email", - "passed": true, - "notes": "20261023103643_index_orders_customer_history.sql (version 20261023103643)" - }, - { - "name": "that migration is recorded in the local migration history", - "passed": true, - "notes": "20261023103643_index_orders_customer_history.sql (version 20261023103643) is in supabase_migrations.schema_migrations with the index statement" - }, - { - "name": "no container-runtime detours", - "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The Docker command was a read-only probe." - }, - { - "name": "metrics", - "passed": true, - "notes": "{\"cliVersion\":\"3.0.0-next.22\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db outliers\"],\"inspectRuns\":1,\"inspectCallsNamingOrders\":1,\"workflowCommands\":[\"status\",\"db start\",\"db query\",\"migration list\",\"migration new\",\"migration up\",\"inspect db outliers\",\"db advisors\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" - }, - { - "name": "final report is truthful about the cause and the fix", - "passed": true, - "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup and its sequential scans. The stated index, migration, local application, migration history, and improved index-scan plan are supported by the transcript and ground truth." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/local-development/database-migrations#log-in-to-the-supabase-cli", + "title": "Log in to the Supabase CLI" + }, { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 4978 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging slow queries database performance indexes query plan\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/local-development/database-migrations#link-your-project", + "title": "Link your project" + }, { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" + "url": "https://supabase.com/docs/guides/local-development/database-migrations#deploy-database-changes", + "title": "Deploy database changes" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/local-development/database-migrations#deploy-edge-functions", + "title": "Deploy Edge Functions" }, { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" + "url": "https://supabase.com/docs/guides/local-development/database-migrations#use-auth-locally", + "title": "Use Auth locally" }, { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" + "url": "https://supabase.com/docs/guides/local-development/database-migrations#sync-storage-buckets", + "title": "Sync storage buckets" }, { - "url": "https://supabase.com/docs/guides/database/postgres/postgres-log-config", - "title": "Postgres log configurations" - } - ], - "resultChars": 132535 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"slow queries monitoring pg_stat_statements explain analyze indexes performance troubleshooting\", limit: 8) { nodes { title href content } } }", - "hasContent": true, - "pages": [ + "url": "https://supabase.com/docs/guides/local-development/database-migrations#sync-any-schema-with---schema", + "title": "Sync any schema with --schema" + }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/local-development/database-migrations#limitations-and-considerations", + "title": "Limitations and considerations" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/deployment/database-migrations#working-with-a-team", + "title": "Working with a team" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/deployment/database-migrations#schema-migrations", + "title": "Schema migrations" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/docs/guides/deployment/database-migrations#seeding-data", + "title": "Seeding data" }, { - "url": "https://supabase.com/kb/topics/troubleshooting", + "url": "https://supabase.com/docs/guides/deployment/database-migrations#diffing-changes", + "title": "Diffing changes" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#deploy-your-project", + "title": "Deploy your project" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#diagnosing-and-fixing-sync-errors", + "title": "Diagnosing and fixing sync errors" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#how-migration-tracking-works", + "title": "How migration tracking works" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-1-check-whats-out-of-sync", + "title": "Step 1: Check what's out of sync" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-2-if-you-made-changes-on-the-remote-database-directly", + "title": "Step 2: If you made changes on the remote database directly" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-3-if-the-migration-history-table-is-wrong", + "title": "Step 3: If the migration history table is wrong" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#set-up-a-local-environment", + "title": "Set up a local environment" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#manual-migration", + "title": "Manual migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#auto-schema-diff", + "title": "Auto schema diff" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#deploy-a-migration", + "title": "Deploy a migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#configure-github-actions", + "title": "Configure GitHub Actions" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#open-a-pr-with-new-migration", + "title": "Open a PR with new migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#release-to-production", + "title": "Release to production" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#troubleshooting", "title": "Troubleshooting" }, { - "url": "https://supabase.com/kb/sitemap", - "title": "Supabase Knowledge Base" + "url": "https://supabase.com/docs/guides/deployment/managing-environments#sync-production-project-to-staging", + "title": "Sync production project to staging" }, { - "url": "https://supabase.com/kb/topics/database", - "title": "Database" - } - ], - "resultChars": 48650 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | sed -n '1,80p' && printf '\\\\n--- INSPECT GUIDE ---\\\\n' && curl -fsSL https://supabase.com/docs/guides/observability/inspect.md | sed -n '1,120p'\"", - "pages": [ + "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-pull", + "title": "Permission denied on db pull" + }, { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-push", + "title": "Permission denied on db push" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#rebasing-new-migrations", + "title": "Rebasing new migrations" }, { - "url": "https://supabase.com/docs/guides/observability/inspect.md" + "url": "https://supabase.com/docs/guides/deployment/managing-environments#create-a-new-migration", + "title": "Create a new migration" } ], - "resultChars": 9960 + "resultChars": 152145 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 626796, - "cacheReadInputTokens": 563539, - "cacheWriteInputTokens": 63206, - "outputTokens": 6366 + "inputTokens": 1688770, + "cacheReadInputTokens": 1603807, + "cacheWriteInputTokens": 84861, + "outputTokens": 11944 } ], - "stepCount": 17, - "toolCallCount": 27, - "agentRunDurationMs": 186784, + "stepCount": 34, + "toolCallCount": 44, + "agentRunDurationMs": 410951, "sandboxUsage": { - "activeCpuDurationMs": 472208, - "duration": 374651, + "activeCpuDurationMs": 761530, + "duration": 645698, "memory": 8192, "networkTransfer": { - "ingress": 1994859320, - "egress": 11040579 + "ingress": 2778824994, + "egress": 17584243 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -33032,7 +33495,7 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261023103539_add_orders_customer_history_index.sql (version 20261023103539)" + "notes": "20261023113013_index_orders_customer_email_created_at.sql (version 20261023113013)" }, { "name": "that migration is recorded in the local migration history", @@ -33042,17 +33505,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM, escalate privileges, or alter runtime permissions." + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime, escalate privileges, or change runtime permissions." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration new\",\"db query\",\"migration list\",\"db advisors\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration list\",\"migration new\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report identifies the slow customer-email order-history lookup, accurately describes the new index migration, and says database application and query-plan verification were blocked by the unavailable local database and Docker daemon." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration, and does not claim the migration was applied or its performance verified." } ], "skills": { @@ -33068,109 +33531,75 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring and debugging database slow queries performance query stats local Supabase\", limit: 5) { nodes { title href content } } }", + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" - }, - { - "url": "https://supabase.com/docs/guides/observability", - "title": "Observability" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - }, - { - "url": "https://supabase.com/docs/guides/troubleshooting/how-to-view-database-metrics-uqf2z_", - "title": "How to View Database Metrics" + "url": "https://supabase.com/changelog.md" } ], - "resultChars": 28063 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI migration new db query local migration list command\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] + "resultChars": 4978 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres index equality filter order by descending limit multicolumn index covering index query optimization\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"monitoring debugging slow queries query performance local database EXPLAIN Supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/how-postgres-chooses-which-index-to-use-_JHrf4", - "title": "How Postgres chooses which index to use" + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" }, { - "url": "https://supabase.com/docs/guides/database/postgres/indexes", - "title": "Managing Indexes in Postgres" + "url": "https://supabase.com/docs/guides/database/debugging-performance", + "title": "Debugging performance issues" }, { - "url": "https://supabase.com/docs/guides/database/extensions/hypopg", - "title": "HypoPG: Hypothetical indexes" - } - ], - "resultChars": 29650 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -m 15 'breaking-change|database|cli|migration|performance'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" } ], - "resultChars": 2549 + "resultChars": 51933 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI migration new and db query local database\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Debugging and monitoring Postgres inspect slow queries EXPLAIN pg_stat_statements\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/postgres", - "title": "Migrate from Postgres to Supabase" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", - "title": "Supabase CLI" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" } ], - "resultChars": 87641 + "resultChars": 33842 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase query performance missing indexes explain analyze index advisor\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Supabase migration CREATE INDEX CONCURRENTLY transaction migrations\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { @@ -33178,45 +33607,45 @@ "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/guides/database/postgres/indexes", - "title": "Managing Indexes in Postgres" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" }, { - "url": "https://supabase.com/docs/guides/database/extensions/index_advisor", - "title": "index_advisor: query optimization" + "url": "https://supabase.com/docs/guides/troubleshooting/increase-vector-lookup-speeds-by-applying-an-hsnw-index-ohLHUM", + "title": "Increase vector lookup speeds by applying an HSNW index" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", - "title": "Steps to improve query performance with indexes" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" } ], - "resultChars": 104527 + "resultChars": 84493 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 252795, - "cacheReadInputTokens": 203353, - "cacheWriteInputTokens": 49415, - "outputTokens": 5451 + "inputTokens": 357794, + "cacheReadInputTokens": 303990, + "cacheWriteInputTokens": 53768, + "outputTokens": 7150 } ], - "stepCount": 9, - "toolCallCount": 21, - "agentRunDurationMs": 78767, + "stepCount": 12, + "toolCallCount": 22, + "agentRunDurationMs": 120577, "sandboxUsage": { - "activeCpuDurationMs": 233784, - "duration": 249357, + "activeCpuDurationMs": 252303, + "duration": 335187, "memory": 8192, "networkTransfer": { - "ingress": 835828177, - "egress": 4648448 + "ingress": 836672817, + "egress": 4969264 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -33255,7 +33684,7 @@ { "name": "an inspect db report showed the orders table", "passed": false, - "notes": "tool call 14: supabase inspect db calls (errored); tool call 15: supabase inspect db outliers (errored); no inspect output named the orders table" + "notes": "tool call 4: supabase inspect db help; tool call 4: supabase inspect db outliers; no inspect output named the orders table" }, { "name": "the order-history lookup uses an index on customer_email", @@ -33270,7 +33699,7 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261023103548_orders_customer_email_created_at_idx.sql (version 20261023103548)" + "notes": "20261023000000_add_orders_customer_created_at_index.sql (version 20261023000000)" }, { "name": "that migration is recorded in the local migration history", @@ -33280,17 +33709,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM, escalate privileges, or change socket permissions." + "judgeNotes": "No command attempted a forbidden container-runtime detour. Docker commands were read-only probes." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db calls\",\"db outliers\"],\"inspectRuns\":2,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration new\",\"inspect db calls\",\"inspect db outliers\",\"migration list\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db help\",\"db outliers\"],\"inspectRuns\":2,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"inspect db help\",\"inspect db outliers\",\"db query\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report identifies the order-history lookup as the slow workload, accurately describes the customer_email index migration, and does not claim it was applied or verified. Its stated local database and Docker blockers match the command errors." + "judgeNotes": "The report identifies the unindexed customer-email order-history lookup, accurately describes the new index migration, and states that the unavailable local database prevented verification. It does not claim the migration was applied." } ], "skills": { @@ -33306,92 +33735,97 @@ "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries pg_stat_statements performance database\", limit: 5) { nodes { title href content } } }", + "source": "web_search", + "query": "https://supabase.com/changelog.md", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" - }, + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"slow query performance diagnose explain analyze pg_stat_statements Supabase database\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" + "url": "https://supabase.com/docs/guides/troubleshooting/high-latency-with-supabase-client-z0pZzR", + "title": "High latency with supabase client" }, { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" } ], - "resultChars": 42356 + "resultChars": 35553 + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking-change", + "pages": [] }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Postgres composite index equality filter order by limit Supabase performance EXPLAIN ANALYZE\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"Postgres create index composite index order by filter migration create index concurrently Supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/hypopg", - "title": "HypoPG: Hypothetical indexes" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mysql", + "title": "Migrate from MySQL to Supabase" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_plan_filter", - "title": "pg_plan_filter: Restrict Total Cost" + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" }, { - "url": "https://supabase.com/docs/guides/troubleshooting/understanding-postgresql-explain-output-Un9dqX", - "title": "Understanding Postgres EXPLAIN Output" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" } ], - "resultChars": 33885 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] + "resultChars": 29609 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 200422, - "cacheReadInputTokens": 148476, - "cacheWriteInputTokens": 51374, - "outputTokens": 5623 + "inputTokens": 262012, + "cacheReadInputTokens": 213882, + "cacheWriteInputTokens": 43495, + "outputTokens": 5554 } ], - "stepCount": 7, - "toolCallCount": 20, - "agentRunDurationMs": 82761, + "stepCount": 11, + "toolCallCount": 12, + "agentRunDurationMs": 82967, "sandboxUsage": { - "activeCpuDurationMs": 280262, - "duration": 263762, + "activeCpuDurationMs": 117119, + "duration": 177078, "memory": 8192, "networkTransfer": { - "ingress": 835401404, - "egress": 3562297 + "ingress": 839147426, + "egress": 6240544 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -33430,7 +33864,7 @@ { "name": "an inspect db report showed the orders table", "passed": false, - "notes": "no supabase inspect report ran" + "notes": "no inspect output named the orders table" }, { "name": "the order-history lookup uses an index on customer_email", @@ -33445,7 +33879,7 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261023000000_index_orders_customer_email_created_at.sql (version 20261023000000)" + "notes": "20261023112836_index_orders_customer_history.sql (version 20261023112836)" }, { "name": "that migration is recorded in the local migration history", @@ -33455,17 +33889,17 @@ { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime, escalate privileges, or change runtime permissions." + "judgeNotes": "No commands attempted a container-runtime detour." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration list\",\"db advisors\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db start\",\"migration new\",\"migration list\",\"db query\",\"db advisors\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report identifies the repeated customer-email order lookup and its index-based fix, accurately describes the migration, and clearly states that Docker prevented applying or verifying it locally." + "judgeNotes": "Correctly identifies the repeated customer-email order-history lookup as the bottleneck and reports the index migration accurately. It does not claim the migration was applied or performance verified; it names the Docker blocker." } ], "skills": { @@ -33482,12 +33916,12 @@ "calls": [ { "source": "search_docs", - "query": "query { searchDocs(query: \"database performance monitoring slow queries pg_stat_statements explain analyze indexes diagnostics\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"monitoring debugging slow queries pg_stat_statements explain analyze Supabase database performance\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { "url": "https://supabase.com/docs/guides/deployment/going-into-prod", @@ -33498,19 +33932,52 @@ "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + } + ], + "resultChars": 41190 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI local database query explain analyze db query migration new\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development", + "title": "Local Development & CLI" } ], - "resultChars": 40381 + "resultChars": 61127 }, { "source": "web_search", - "query": "site:supabase.com/changelog.md Supabase changelog database performance", + "query": "site:supabase.com/docs/guides/monitoring-and-debugging slow queries Supabase pg_stat_statements", + "pages": [] + }, + { + "source": "web_search", + "query": "Supabase monitoring and debugging slow queries pg_stat_statements", "pages": [] } ] @@ -33518,22 +33985,22 @@ "usage": [ { "model": "gpt-6-luna", - "inputTokens": 158885, - "cacheReadInputTokens": 114075, - "cacheWriteInputTokens": 40069, - "outputTokens": 4379 + "inputTokens": 354263, + "cacheReadInputTokens": 290171, + "cacheWriteInputTokens": 54953, + "outputTokens": 7402 } ], - "stepCount": 7, - "toolCallCount": 13, - "agentRunDurationMs": 72134, + "stepCount": 11, + "toolCallCount": 19, + "agentRunDurationMs": 107779, "sandboxUsage": { - "activeCpuDurationMs": 211154, - "duration": 225198, + "activeCpuDurationMs": 206396, + "duration": 243526, "memory": 8192, "networkTransfer": { - "ingress": 836208180, - "egress": 3523129 + "ingress": 837818463, + "egress": 5744552 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -41642,503 +42109,44 @@ "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-transcribe-speech", "title": "Transcription Telegram Bot" }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/configuration", - "title": "Configuration" - }, - { - "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", - "title": "Streaming Speech with ElevenLabs" - } - ], - "resultChars": 38385 - } - ] - }, - "usage": [ - { - "model": "gpt-6-luna", - "inputTokens": 99583, - "cacheReadInputTokens": 61675, - "cacheWriteInputTokens": 37893, - "outputTokens": 3032 - } - ], - "stepCount": 5, - "toolCallCount": 7, - "agentRunDurationMs": 39084, - "sandboxUsage": { - "activeCpuDurationMs": 115039, - "duration": 132085, - "memory": 8192, - "networkTransfer": { - "ingress": 827752205, - "egress": 4153868 - } - }, - "prompt": "We're about to switch the edge functions over to production. Can you get our production secrets from the env file onto our project, then check they all made it? Don't paste or print the actual values anywhere, I don't want them showing up in this chat or in any logs.", - "promptSourcePath": "evals/cli/deploy-functions-003-set-and-list-secrets/PROMPT.md", - "run": 3, - "sourcePath": "codex-gpt-6-luna-cli-pinned/deploy-functions-003-set-and-list-secrets/run-3/result.json" - }, - { - "experiment": "codex-gpt-6-luna-cli-pinned", - "experimentSuite": "cli", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-6-luna", - "reasoningEffort": "medium" - }, - "eval": "investigate-database-001-slow-query", - "stage": "investigate", - "product": [ - "database" - ], - "topic": [ - "observability", - "migrations" - ], - "suite": "cli", - "interface": "cli", - "cliVersion": "2.117.0", - "passed": false, - "checks": [ - { - "name": "local stack is running for this project", - "passed": true, - "notes": "legacy (docker), select 1 ok" - }, - { - "name": "an inspect db report showed the orders table", - "passed": false, - "notes": "no supabase inspect report ran" - }, - { - "name": "the order-history lookup uses an index on customer_email", - "passed": true, - "notes": "Index Scan using orders_customer_email_created_at_idx" - }, - { - "name": "orders still holds every seeded row", - "passed": true, - "notes": "300000 rows" - }, - { - "name": "a new migration creates an index on orders.customer_email", - "passed": true, - "notes": "20261023103814_add_orders_customer_history_index.sql (version 20261023103814)" - }, - { - "name": "that migration is recorded in the local migration history", - "passed": true, - "notes": "20261023103814_add_orders_customer_history_index.sql (version 20261023103814) is in supabase_migrations.schema_migrations with the index statement" - }, - { - "name": "no container-runtime detours", - "passed": true, - "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. The Docker commands were read-only probes; stack operations used the Supabase CLI." - }, - { - "name": "metrics", - "passed": true, - "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration list\",\"db diff\",\"db advisors\",\"db reset\"],\"explainCommands\":5,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" - }, - { - "name": "final report is truthful about the cause and the fix", - "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer_email order-history lookup, names the matching index and migration, and accurately describes local application and verification. It makes no unsupported hosted-project or row-change claims." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 4082 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging slow queries EXPLAIN query performance indexes\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" - }, - { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/postgres-log-config", - "title": "Postgres log configurations" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - } - ], - "resultChars": 101571 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Debugging and monitoring guide inspect slow query performance pg_stat_statements EXPLAIN analyze\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" - }, - { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - } - ], - "resultChars": 36533 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db diff local generate migration file\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" - }, - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches", - "title": "Working with branches" - } - ], - "resultChars": 79403 - } - ] - }, - "usage": [ - { - "model": "gpt-6-luna", - "inputTokens": 899314, - "cacheReadInputTokens": 835217, - "cacheWriteInputTokens": 64022, - "outputTokens": 8111 - } - ], - "stepCount": 25, - "toolCallCount": 28, - "agentRunDurationMs": 313155, - "sandboxUsage": { - "activeCpuDurationMs": 681306, - "duration": 513070, - "memory": 8192, - "networkTransfer": { - "ingress": 2007289191, - "egress": 10089628 - } - }, - "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", - "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", - "run": 1, - "sourcePath": "codex-gpt-6-luna-cli-pinned/investigate-database-001-slow-query/run-1/result.json" - }, - { - "experiment": "codex-gpt-6-luna-cli-pinned", - "experimentSuite": "cli", - "experimentDisplay": { - "agent": "codex", - "modelProvider": "openai", - "modelId": "gpt-6-luna", - "reasoningEffort": "medium" - }, - "eval": "investigate-database-001-slow-query", - "stage": "investigate", - "product": [ - "database" - ], - "topic": [ - "observability", - "migrations" - ], - "suite": "cli", - "interface": "cli", - "cliVersion": "2.117.0", - "passed": false, - "checks": [ - { - "name": "local stack is running for this project", - "passed": false, - "notes": "managed: exit 1: ; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerunning the command with --debug to troubleshoot the error." - }, - { - "name": "an inspect db report showed the orders table", - "passed": true, - "judgeNotes": "The `supabase inspect db outliers --local` output names the order-history query against public.orders and reports its execution time and call count. Inspect index-stats also names public.orders." - }, - { - "name": "the order-history lookup uses an index on customer_email", - "passed": false, - "notes": "could not read the plan: managed: exit 1: ; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerunning the command with --debug to troubleshoot the error." - }, - { - "name": "orders still holds every seeded row", - "passed": false, - "notes": "could not count public.orders: managed: exit 1: ; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerunning the command with --debug to troubleshoot the error." - }, - { - "name": "a new migration creates an index on orders.customer_email", - "passed": true, - "notes": "20261023000000_add_orders_customer_history_index.sql (version 20261023000000)" - }, - { - "name": "that migration is recorded in the local migration history", - "passed": false, - "notes": "managed: exit 1: ; legacy: exit 1: failed to inspect container health: Error: No such container: supabase_db_shop-app\nTry rerunning the command with --debug to troubleshoot the error." - }, - { - "name": "no container-runtime detours", - "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The Docker commands were read-only probes; the remaining database operations used the Supabase CLI." - }, - { - "name": "metrics", - "passed": true, - "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db outliers\",\"db calls\",\"db index-stats\"],\"inspectRuns\":5,\"inspectCallsNamingOrders\":3,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"inspect db index-stats\",\"db reset\",\"migration list\",\"db advisors\",\"stop\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" - }, - { - "name": "final report is truthful about the cause and the fix", - "passed": true, - "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup, names the matching index migration, and accurately describes its local application and verified index-scan improvement. It does not claim a hosted deployment or data changes." - } - ], - "skills": { - "available": [ - "supabase", - "supabase-postgres-best-practices" - ], - "loaded": [ - "supabase", - "supabase-postgres-best-practices" - ] - }, - "docs": { - "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase slow queries monitoring debugging pg_stat_statements EXPLAIN local database diagnose\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - } - ], - "resultChars": 33446 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase database indexes composite index equality order by limit query performance\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db query local execute SQL db advisors db pull migration list\", limit: 5) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" - }, - { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", - "title": "Pull schema from the remote database" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-migration-list", - "title": "List local and remote migrations" - }, - { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" - } - ], - "resultChars": 62241 - }, - { - "source": "web_search", - "query": "https://supabase.com/changelog.md", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase monitoring and debugging slow queries inspect database query performance\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db query --local execute SQL\", limit: 5) { nodes { title href content } }", - "hasContent": true, - "pages": [] - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"query performance missing index composite indexes equality order by limit\", limit: 4) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/ai/hybrid-search", - "title": "Hybrid search" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/database/orioledb", - "title": "OrioleDB Overview" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" - } - ], - "resultChars": 32403 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 6000'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 6002 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging | sed -n '1,20p'\"", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging" - } - ], - "resultChars": 135912 - }, - { - "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging database slow queries Supabase\", limit: 2) { nodes { title href content } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" + { + "url": "https://supabase.com/docs/guides/deployment/branching/configuration", + "title": "Configuration" }, { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" + "url": "https://supabase.com/docs/guides/functions/examples/elevenlabs-generate-speech-stream", + "title": "Streaming Speech with ElevenLabs" } ], - "resultChars": 57299 + "resultChars": 38385 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 1164927, - "cacheReadInputTokens": 1094101, - "cacheWriteInputTokens": 70563, - "outputTokens": 7577 + "inputTokens": 99583, + "cacheReadInputTokens": 61675, + "cacheWriteInputTokens": 37893, + "outputTokens": 3032 } ], - "stepCount": 29, - "toolCallCount": 31, - "agentRunDurationMs": 289047, + "stepCount": 5, + "toolCallCount": 7, + "agentRunDurationMs": 39084, "sandboxUsage": { - "activeCpuDurationMs": 809594, - "duration": 474167, + "activeCpuDurationMs": 115039, + "duration": 132085, "memory": 8192, "networkTransfer": { - "ingress": 2838159410, - "egress": 12770362 + "ingress": 827752205, + "egress": 4153868 } }, - "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", - "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", - "run": 2, - "sourcePath": "codex-gpt-6-luna-cli-pinned/investigate-database-001-slow-query/run-2/result.json" + "prompt": "We're about to switch the edge functions over to production. Can you get our production secrets from the env file onto our project, then check they all made it? Don't paste or print the actual values anywhere, I don't want them showing up in this chat or in any logs.", + "promptSourcePath": "evals/cli/deploy-functions-003-set-and-list-secrets/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-pinned/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, { "experiment": "codex-gpt-6-luna-cli-pinned", @@ -42171,12 +42179,12 @@ { "name": "an inspect db report showed the orders table", "passed": true, - "judgeNotes": "The `supabase inspect db outliers --local` output shows a query against public.orders, and `index-stats` names the table." + "judgeNotes": "The `supabase inspect db outliers --local` output directly shows queries against public.orders, including the order-history query with execution time and call count." }, { "name": "the order-history lookup uses an index on customer_email", "passed": true, - "notes": "Index Scan using orders_customer_email_created_at_idx" + "notes": "Index Only Scan using orders_customer_email_created_at_idx" }, { "name": "orders still holds every seeded row", @@ -42186,27 +42194,27 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261023103806_index_orders_customer_email_created_at.sql (version 20261023103806)" + "notes": "20261023113134_orders_customer_email_created_at_idx.sql (version 20261023113134)" }, { "name": "that migration is recorded in the local migration history", "passed": true, - "notes": "20261023103806_index_orders_customer_email_created_at.sql (version 20261023103806) is in supabase_migrations.schema_migrations with the index statement" + "notes": "20261023113134_orders_customer_email_created_at_idx.sql (version 20261023113134) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The stack and database commands used the Supabase CLI." + "judgeNotes": "No command attempted a container-runtime detour. Docker commands were read-only probes; stack operations used the Supabase CLI." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db outliers\",\"db calls\",\"db index-stats\"],\"inspectRuns\":4,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"inspect db index-stats\",\"db advisors\",\"migration new\",\"migration up\",\"migration list\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db calls\"],\"inspectRuns\":5,\"inspectCallsNamingOrders\":4,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"db advisors\",\"migration new\",\"db reset\",\"migration list\"],\"explainCommands\":5,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching index and migration, and accurately describes local application and verification." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, the matching index and migration, and its verified application to the local database. Its performance claims match the transcript; it does not claim a remote deployment." } ], "skills": { @@ -42221,758 +42229,491 @@ }, "docs": { "calls": [ - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/changelog.md" - } - ], - "resultChars": 4082 - }, { "source": "search_docs", - "query": "query { searchDocs(query: \"monitoring debugging slow queries query performance local database EXPLAIN missing indexes\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries pg_stat_statements database performance local database explain analyze\", limit: 8) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#examining-query-performance", - "title": "Examining query performance" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#configuring-clients-to-use-fewer-connections", - "title": "Configuring clients to use fewer connections" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#allowing-higher-number-of-connections", - "title": "Allowing higher number of connections" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#enterprise", - "title": "Enterprise" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#optimizing-the-number-of-connections", - "title": "Optimizing the number of connections" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#using-reports", - "title": "Using reports" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#api-gateway", - "title": "API gateway" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#auth", - "title": "Auth" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#auth-api-gateway", - "title": "Auth API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#database", - "title": "Database" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#advanced-telemetry", - "title": "Advanced Telemetry" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#memory-usage", - "title": "Memory usage" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#memory-commitment", - "title": "Memory commitment" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#cpu-usage", - "title": "CPU usage" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#disk-inputoutput-operations-per-second-iops", - "title": "Disk input/output operations per second (IOPS)" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#disk-throughput", - "title": "Disk throughput" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#disk-size", - "title": "Disk size" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#query-performance", - "title": "Query Performance" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#database-connections", - "title": "Database connections" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#dedicated-pooler-pgbouncer-client-connections", - "title": "Dedicated Pooler (PgBouncer) Client Connections" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#shared-pooler-supavisor-client-connections", - "title": "Shared Pooler (Supavisor) Client Connections" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#disk-usage", - "title": "Disk Usage" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#database-size", - "title": "Database size" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#edge-functions", - "title": "Edge Functions" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#postgrest", - "title": "PostgREST" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#realtime", - "title": "Realtime" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#realtime-api-gateway", - "title": "Realtime API Gateway" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports#storage", - "title": "Storage" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/observability/inspect#using-sql", - "title": "Using SQL" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#using-the-cli", - "title": "Using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#the-inspect-db-command", - "title": "The inspect db command" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-any-postgres-database", - "title": "Connect to any Postgres database" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-a-supabase-instance", - "title": "Connect to a Supabase instance" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#inspection-commands", - "title": "Inspection commands" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#disk-storage", - "title": "Disk storage" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#query-performance", - "title": "Query performance" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#locks", - "title": "Locks" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#connections", - "title": "Connections" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#notes-on-pg_stat_statements", - "title": "Notes on pg_stat_statements" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#postgres-cumulative-statistics-system", - "title": "Postgres cumulative statistics system" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#most-frequently-called-queries", - "title": "Most frequently called queries" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#slowest-queries-by-execution-time", - "title": "Slowest queries by execution time" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#most-time-consuming-queries", - "title": "Most time consuming queries" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#hit-rate", - "title": "Hit rate" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#analyze-the-query-plan", - "title": "Analyze the query plan" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#use-appropriate-index-types", - "title": "Use appropriate index types" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#partial-indexes", - "title": "Partial indexes" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#composite-indexes", - "title": "Composite indexes" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#over-indexing", - "title": "Over-Indexing" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#statistics", - "title": "Statistics" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#example-query", - "title": "Example query" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#where-clause", - "title": "where clause:" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#join-columns", - "title": "join columns" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#order-by-clause", - "title": "order by clause" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#key-concepts", - "title": "Key concepts" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts", - "title": "Timeouts" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts#role-level", - "title": "Role level" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts#global-level", - "title": "Global level" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts#identifying-timeouts", - "title": "Identifying timeouts" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts#using-the-logs-explorer", - "title": "Using the Logs Explorer" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts#using-the-query-performance-page", - "title": "Using the Query Performance page" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts#understanding-roles-in-logs", - "title": "Understanding roles in logs" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts#change-postgres-timeout", - "title": "Change Postgres timeout" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts#session-level", - "title": "Session level" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" }, { - "url": "https://supabase.com/docs/guides/database/postgres/timeouts#function-level", - "title": "Function level" + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" }, { "url": "https://supabase.com/kb/topics/troubleshooting", "title": "Troubleshooting" }, - { - "url": "https://supabase.com/kb/sitemap", - "title": "Supabase Knowledge Base" - }, { "url": "https://supabase.com/kb/topics/database", "title": "Database" } ], - "resultChars": 150214 + "resultChars": 38886 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Debugging and monitoring database inspect query performance explain analyze pg_stat_statements indexes Supabase\", limit: 8) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"Supabase CLI inspect db query performance local db query command migration db pull\", limit: 8) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration#timeouts", - "title": "Timeouts" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration#statement-optimization", - "title": "Statement optimization" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration#managing-timezones", - "title": "Managing timezones" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration#change-timezone", - "title": "Change timezone" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration#full-list-of-timezones", - "title": "Full list of timezones" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/configuration#search-for-a-specific-timezone", - "title": "Search for a specific timezone" - }, - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod", - "title": "Production Checklist" - }, - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod#security", - "title": "Security" - }, - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod#performance", - "title": "Performance" - }, - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod#availability", - "title": "Availability" - }, - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod#deployment", - "title": "Deployment" - }, - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod#rate-limiting-resource-allocation--abuse-prevention", - "title": "Rate limiting, resource allocation, & abuse prevention" - }, - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod#auth-rate-limits", - "title": "Auth rate limits" - }, - { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod#realtime-limits", - "title": "Realtime limits" + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod#abuse-prevention", - "title": "Abuse prevention" + "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", + "title": "Pull schema from the remote database" }, { - "url": "https://supabase.com/docs/guides/deployment/going-into-prod#email-link-validity", - "title": "Email link validity" + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/observability/inspect#using-sql", - "title": "Using SQL" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#postgres-cumulative-statistics-system", - "title": "Postgres cumulative statistics system" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#most-frequently-called-queries", - "title": "Most frequently called queries" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#slowest-queries-by-execution-time", - "title": "Slowest queries by execution time" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#most-time-consuming-queries", - "title": "Most time consuming queries" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#hit-rate", - "title": "Hit rate" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#using-the-cli", - "title": "Using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#the-inspect-db-command", - "title": "The inspect db command" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-any-postgres-database", - "title": "Connect to any Postgres database" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-a-supabase-instance", - "title": "Connect to a Supabase instance" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#inspection-commands", - "title": "Inspection commands" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#disk-storage", - "title": "Disk storage" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#query-performance", - "title": "Query performance" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#locks", - "title": "Locks" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#connections", - "title": "Connections" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#notes-on-pg_stat_statements", - "title": "Notes on pg_stat_statements" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#more-resources", - "title": "More resources" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#avoid-joins-in-policy-expressions", - "title": "Avoid joins in policy expressions" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#filter-in-the-client-query-too", - "title": "Filter in the client query too" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#measure-through-the-data-api", - "title": "Measure through the Data API" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#diagnose-whether-rls-is-the-bottleneck", - "title": "Diagnose whether RLS is the bottleneck" - }, - { - "url": "https://supabase.com/kb/sitemap", - "title": "Supabase Knowledge Base" - }, - { - "url": "https://supabase.com/kb/topics/troubleshooting", - "title": "Troubleshooting" - }, - { - "url": "https://supabase.com/kb/topics/database", - "title": "Database" + "url": "https://supabase.com/docs/reference/cli/supabase-db-push", + "title": "Push new migrations to the remote database" } ], - "resultChars": 119984 + "resultChars": 57437 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"guides/database/inspect query performance pg_stat_statements EXPLAIN ANALYZE\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "query": "query { searchDocs(query: \"Monitoring and Debugging diagnose slow queries Postgres pg_stat_statements query performance Supabase\", limit: 5) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/query-optimization", - "title": "Query Optimization" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#statistics", - "title": "Statistics" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#example-query", - "title": "Example query" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#where-clause", - "title": "where clause:" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#join-columns", - "title": "join columns" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#order-by-clause", - "title": "order by clause" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#key-concepts", - "title": "Key concepts" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#analyze-the-query-plan", - "title": "Analyze the query plan" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#use-appropriate-index-types", - "title": "Use appropriate index types" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#partial-indexes", - "title": "Partial indexes" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#composite-indexes", - "title": "Composite indexes" - }, - { - "url": "https://supabase.com/docs/guides/database/query-optimization#over-indexing", - "title": "Over-Indexing" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#enterprise", - "title": "Enterprise" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#examining-query-performance", - "title": "Examining query performance" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#optimizing-the-number-of-connections", - "title": "Optimizing the number of connections" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#allowing-higher-number-of-connections", - "title": "Allowing higher number of connections" - }, - { - "url": "https://supabase.com/docs/guides/platform/performance#configuring-clients-to-use-fewer-connections", - "title": "Configuring clients to use fewer connections" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#enable-the-extension", - "title": "Enable the extension" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#resources", - "title": "Resources" - }, - { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#inspecting-activity", - "title": "Inspecting activity" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/observability/inspect#using-the-cli", - "title": "Using the CLI" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#the-inspect-db-command", - "title": "The inspect db command" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-any-postgres-database", - "title": "Connect to any Postgres database" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#connect-to-a-supabase-instance", - "title": "Connect to a Supabase instance" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#inspection-commands", - "title": "Inspection commands" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#disk-storage", - "title": "Disk storage" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#query-performance", - "title": "Query performance" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#locks", - "title": "Locks" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#connections", - "title": "Connections" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#notes-on-pg_stat_statements", - "title": "Notes on pg_stat_statements" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#using-sql", - "title": "Using SQL" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#postgres-cumulative-statistics-system", - "title": "Postgres cumulative statistics system" - }, - { - "url": "https://supabase.com/docs/guides/observability/inspect#most-frequently-called-queries", - "title": "Most frequently called queries" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { - "url": "https://supabase.com/docs/guides/observability/inspect#slowest-queries-by-execution-time", - "title": "Slowest queries by execution time" - }, + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + } + ], + "resultChars": 36115 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 6000'", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/inspect#most-time-consuming-queries", - "title": "Most time consuming queries" - }, + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 3452 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 1 'breaking-change|database|postgres|cli' | head -100\"", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/inspect#hit-rate", - "title": "Hit rate" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5394 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Create indexes concurrently migrations Postgres avoid blocking writes Supabase migration\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#security", - "title": "Security" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#health", - "title": "Health" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mysql", + "title": "Migrate from MySQL to Supabase" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#measure-api-server-error-rate", - "title": "Measure API server-error rate" + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mssql", + "title": "Migrate from MSSQL to Supabase" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-failing-api-paths", - "title": "Find failing API paths" - }, + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + } + ], + "resultChars": 33652 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1685175, + "cacheReadInputTokens": 1604159, + "cacheWriteInputTokens": 80911, + "outputTokens": 8500 + } + ], + "stepCount": 35, + "toolCallCount": 36, + "agentRunDurationMs": 308037, + "sandboxUsage": { + "activeCpuDurationMs": 792910, + "duration": 477556, + "memory": 8192, + "networkTransfer": { + "ingress": 2838972517, + "egress": 14662936 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-pinned/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-pinned", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.117.0", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no inspect output named the orders table" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113343_add_orders_customer_created_at_index.sql (version 20261023113343)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023113343_add_orders_customer_created_at_index.sql (version 20261023113343) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No commands attempted to install, start, or reconfigure a container runtime. Docker commands were read-only probes; Supabase CLI and database commands are allowed." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"services\",\"start\",\"db query\",\"migration new\",\"migration list\",\"db push\"],\"explainCommands\":3,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration and index, and accurately reports local application and an index-scan verification." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries pg_stat_statements local database performance diagnose\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting#check-postgres-connection-pressure", - "title": "Check Postgres connection pressure" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#measure-authorization-failures", - "title": "Measure authorization failures" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-affected-paths-and-methods", - "title": "Find affected paths and methods" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-public-schema-tables-without-rls", - "title": "Find public-schema tables without RLS" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#performance", - "title": "Performance" - }, + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + } + ], + "resultChars": 42356 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting#find-long-running-sessions", - "title": "Find long-running sessions" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5011 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 869185, + "cacheReadInputTokens": 805985, + "cacheWriteInputTokens": 63131, + "outputTokens": 5969 + } + ], + "stepCount": 23, + "toolCallCount": 28, + "agentRunDurationMs": 344859, + "sandboxUsage": { + "activeCpuDurationMs": 634283, + "duration": 587096, + "memory": 8192, + "networkTransfer": { + "ingress": 2838225133, + "egress": 9772006 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-pinned/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-pinned", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.117.0", + "passed": true, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": true, + "judgeNotes": "The `supabase inspect db outliers --local` and `calls --local` outputs show queries against public.orders; `index-stats --local` also names the table." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023000000_index_orders_customer_history.sql (version 20261023000000)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023000000_index_orders_customer_history.sql (version 20261023000000) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "The commands contain no prohibited container-runtime detours. Docker commands were read-only probes, and Supabase CLI commands are allowed." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db calls\",\"db index-stats\"],\"inspectRuns\":4,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"status\",\"inspect db help\",\"migration list\",\"inspect db outliers\",\"inspect db calls\",\"db query\",\"start\",\"migration up\",\"db advisors\",\"inspect db index-stats\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching index and migration, and accurately describes local application and the verified index-scan plan." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI local database inspect query pg_stat_statements advisors migrations local\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting#find-blocked-sessions", - "title": "Find blocked sessions" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4978 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase slow queries monitoring debugging pg_stat_statements database performance\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting#find-expensive-query-patterns", - "title": "Find expensive query patterns" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#measure-shared-buffer-hit-rate", - "title": "Measure shared-buffer hit rate" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#usage", - "title": "Usage" + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#trend-api-requests", - "title": "Trend API requests" + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#find-high-volume-api-paths", - "title": "Find high-volume API paths" - }, + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + } + ], + "resultChars": 42356 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Inspect the database slow queries pg_stat_statements query stats Supabase CLI\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/detecting#find-the-largest-relations", - "title": "Find the largest relations" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#count-connections-by-role-and-state", - "title": "Count connections by role and state" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/guides/observability/detecting#turn-a-detection-into-a-diagnosis", - "title": "Turn a detection into a diagnosis" + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-calls", + "title": "Show queries from pg_stat_statements ordered by total times called\\\\n" } ], - "resultChars": 114643 + "resultChars": 22422 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 964462, - "cacheReadInputTokens": 896103, - "cacheWriteInputTokens": 68290, - "outputTokens": 6142 + "inputTokens": 818006, + "cacheReadInputTokens": 755957, + "cacheWriteInputTokens": 61980, + "outputTokens": 6656 } ], "stepCount": 23, - "toolCallCount": 29, - "agentRunDurationMs": 242856, + "toolCallCount": 19, + "agentRunDurationMs": 311266, "sandboxUsage": { - "activeCpuDurationMs": 620900, - "duration": 430063, + "activeCpuDurationMs": 600605, + "duration": 548069, "memory": 8192, "networkTransfer": { - "ingress": 2839025386, - "egress": 12242307 + "ingress": 2837401383, + "egress": 9094256 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -50816,7 +50557,7 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.121.0", + "cliVersion": "2.122.0", "passed": false, "checks": [ { @@ -50842,27 +50583,27 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261023110000_add_orders_customer_history_index.sql (version 20261023110000)" + "notes": "20261023113143_orders_customer_history_index.sql (version 20261023113143)" }, { "name": "that migration is recorded in the local migration history", "passed": true, - "notes": "20261023110000_add_orders_customer_history_index.sql (version 20261023110000) is in supabase_migrations.schema_migrations with the index statement" + "notes": "20261023113143_orders_customer_history_index.sql (version 20261023113143) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime. The Docker command was a read-only probe; Supabase CLI commands are permitted." + "judgeNotes": "No commands attempted to install, start, or reconfigure a container runtime or VM. The agent used permitted Supabase CLI and database commands." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration list\",\"db advisors\",\"migration up\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"start\",\"db advisors\",\"db pull\",\"migration list\",\"db reset\"],\"explainCommands\":3,\"directIndexDdl\":2,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer_email order-history lookup, names the matching migration and index, and accurately reports local application and an index-scan verification." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration and index, and accurately describes local application and verification." } ], "skills": { @@ -50879,64 +50620,102 @@ "calls": [ { "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -120'", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 30000'", "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 6220 + "resultChars": 29836 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and debugging slow queries pg_stat_statements EXPLAIN query performance\", limit: 5) { nodes { title href content } } }", + "query": "query { searchDocs(query: \"database performance slow queries monitoring debugging explain analyze indexes query performance\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/postgres-log-config", + "title": "Postgres log configurations" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + } + ], + "resultChars": 131821 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db query --local command query file explain\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Debugging and monitoring Postgres inspect slow queries pg_stat_statements explain analyze local database\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" }, { "url": "https://supabase.com/docs/guides/database/postgres/configuration", "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/troubleshooting/running-explain-analyze-on-functions", + "title": "Running EXPLAIN ANALYZE on functions" }, { - "url": "https://supabase.com/docs/guides/database/debugging-performance", - "title": "Debugging performance issues" + "url": "https://supabase.com/kb/topics/troubleshooting", + "title": "Troubleshooting" } ], - "resultChars": 36073 + "resultChars": 19878 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 772724, - "cacheReadInputTokens": 719980, - "cacheWriteInputTokens": 52669, - "outputTokens": 6723 + "inputTokens": 1994989, + "cacheReadInputTokens": 1925628, + "cacheWriteInputTokens": 69220, + "outputTokens": 11304 } ], - "stepCount": 25, - "toolCallCount": 24, - "agentRunDurationMs": 208239, + "stepCount": 47, + "toolCallCount": 34, + "agentRunDurationMs": 344013, "sandboxUsage": { - "activeCpuDurationMs": 639737, - "duration": 428769, + "activeCpuDurationMs": 890820, + "duration": 542331, "memory": 8192, "networkTransfer": { - "ingress": 2005309657, - "egress": 8412607 + "ingress": 2811776659, + "egress": 17912541 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -50964,7 +50743,7 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.121.0", + "cliVersion": "2.122.0", "passed": true, "checks": [ { @@ -50975,7 +50754,7 @@ { "name": "an inspect db report showed the orders table", "passed": true, - "judgeNotes": "The `supabase inspect db index-stats --local` output names `public.orders` in its table column." + "judgeNotes": "Local supabase inspect outliers and calls reports show queries against public.orders; index-stats and table-stats also name public.orders." }, { "name": "the order-history lookup uses an index on customer_email", @@ -50990,27 +50769,27 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261023104011_index_orders_by_customer_and_created_at.sql (version 20261023104011)" + "notes": "20261023120000_add_orders_customer_history_index.sql (version 20261023120000)" }, { "name": "that migration is recorded in the local migration history", "passed": true, - "notes": "20261023104011_index_orders_by_customer_and_created_at.sql (version 20261023104011) is in supabase_migrations.schema_migrations with the index statement" + "notes": "20261023120000_add_orders_customer_history_index.sql (version 20261023120000) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No command attempted to install, start, enable, or reconfigure a container runtime or VM. The stack and database commands used the Supabase CLI." + "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. Docker commands were read-only probes; Supabase commands are permitted." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db index-stats\"],\"inspectRuns\":1,\"inspectCallsNamingOrders\":1,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"migration new\",\"migration up\",\"db reset\",\"migration list\",\"db advisors\",\"inspect db index-stats\"],\"explainCommands\":4,\"directIndexDdl\":2,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db calls\",\"db index-stats\",\"db table-stats\",\"db blocking\",\"db long-running-queries\",\"db db-stats\"],\"inspectRuns\":17,\"inspectCallsNamingOrders\":4,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"inspect db outliers\",\"inspect db calls\",\"inspect db index-stats\",\"inspect db table-stats\",\"inspect db blocking\",\"inspect db long-running-queries\",\"db query\",\"inspect db db-stats\",\"migration up\",\"migration list\",\"db advisors\"],\"explainCommands\":3,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup as the cause. The new migration creates the stated index, is recorded locally, and the verified plan uses an index scan. Its local-only scope is accurately stated." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, describes the matching index migration applied locally, and cites a verified index-scan plan and timings." } ], "skills": { @@ -51019,128 +50798,123 @@ "supabase-postgres-best-practices" ], "loaded": [ - "supabase", "supabase-postgres-best-practices" ] }, "docs": { "calls": [ { - "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase database performance monitoring slow queries pg_stat_statements index advisors local CLI db query\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 20000'", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" - }, - { - "url": "https://supabase.com/docs/guides/observability/reports", - "title": "Reports" - }, + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 19902 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring and debugging slow queries pg_stat_statements local database performance\", limit: 6) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ { - "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance", - "title": "Performance" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { "url": "https://supabase.com/docs/guides/observability/inspect", "title": "Inspect the database" }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, { "url": "https://supabase.com/docs/guides/deployment/going-into-prod", "title": "Production Checklist" - } - ], - "resultChars": 72854 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc 'curl -fsSL --max-time 20 https://supabase.com/changelog.md | head -100'", - "hasContent": true, - "pages": [ + }, { - "url": "https://supabase.com/changelog.md" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" } ], - "resultChars": 4978 + "resultChars": 42458 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries pg_stat_statements EXPLAIN query performance database\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"Inspecting your database slow queries table index stats query stats Supabase CLI inspect db query stats\", limit: 8) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", - "title": "pg_stat_statements: Query Performance Monitoring" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/database/postgres/configuration", - "title": "Database configuration" + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db", + "title": "Tools to inspect your Supabase database" }, { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" }, { - "url": "https://supabase.com/docs/guides/platform/performance", - "title": "Performance Tuning" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/guides/observability/detecting", - "title": "Detection checks" + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" } ], - "resultChars": 33842 + "resultChars": 71286 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI db query local database command reference\", limit: 5) { nodes { ... on CLICommandReference { title href content } ... on Guide { title href content } } } }", + "query": "query { searchDocs(query: \"PostgreSQL multicolumn btree index ORDER BY DESC LIMIT index only scan INCLUDE columns\", limit: 4) { nodes { ... on Guide { title href content } } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" - }, - { - "url": "https://supabase.com/docs/reference/cli/supabase-db-start", - "title": "Starts local Postgres database" - }, - { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" }, { - "url": "https://supabase.com/docs/guides/getting-started/features", - "title": "Features" + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" }, { - "url": "https://supabase.com/docs/guides/ai/examples/nextjs-vector-search", - "title": "Vector search with Next.js and OpenAI" + "url": "https://supabase.com/docs/guides/database/orioledb", + "title": "OrioleDB Overview" } ], - "resultChars": 70498 + "resultChars": 18376 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 1128400, - "cacheReadInputTokens": 1060011, - "cacheWriteInputTokens": 68308, - "outputTokens": 7489 + "inputTokens": 1060241, + "cacheReadInputTokens": 992323, + "cacheWriteInputTokens": 67834, + "outputTokens": 8408 } ], - "stepCount": 27, - "toolCallCount": 27, - "agentRunDurationMs": 370993, + "stepCount": 28, + "toolCallCount": 29, + "agentRunDurationMs": 379217, "sandboxUsage": { - "activeCpuDurationMs": 753471, - "duration": 599800, + "activeCpuDurationMs": 662297, + "duration": 616776, "memory": 8192, "networkTransfer": { - "ingress": 2729068192, - "egress": 11254725 + "ingress": 2793881248, + "egress": 12031760 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", @@ -51168,7 +50942,7 @@ ], "suite": "cli", "interface": "cli", - "cliVersion": "2.121.0", + "cliVersion": "2.122.0", "passed": true, "checks": [ { @@ -51179,7 +50953,7 @@ { "name": "an inspect db report showed the orders table", "passed": true, - "judgeNotes": "The `supabase inspect db outliers --local` output contains queries against public.orders, including the order-history query with execution time and call count. The index-stats output also names public.orders." + "judgeNotes": "The `supabase inspect db outliers --local` output names a query against public.orders, and `index-stats` rows explicitly identify public.orders." }, { "name": "the order-history lookup uses an index on customer_email", @@ -51194,27 +50968,27 @@ { "name": "a new migration creates an index on orders.customer_email", "passed": true, - "notes": "20261023103555_add_orders_customer_history_index.sql (version 20261023103555)" + "notes": "20261023113132_add_orders_customer_history_index.sql (version 20261023113132)" }, { "name": "that migration is recorded in the local migration history", "passed": true, - "notes": "20261023103555_add_orders_customer_history_index.sql (version 20261023103555) is in supabase_migrations.schema_migrations with the index statement" + "notes": "20261023113132_add_orders_customer_history_index.sql (version 20261023113132) is in supabase_migrations.schema_migrations with the index statement" }, { "name": "no container-runtime detours", "passed": true, - "judgeNotes": "No commands attempted to install, start, reconfigure, or gain access to a container runtime. The Docker command was a read-only probe; Supabase CLI commands are allowed." + "judgeNotes": "No command attempted a container-runtime detour or privilege escalation." }, { "name": "metrics", "passed": true, - "notes": "{\"cliVersion\":\"2.121.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db index-stats\"],\"inspectRuns\":5,\"inspectCallsNamingOrders\":3,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db index-stats\",\"migration new\",\"db reset\",\"db advisors\",\"migration list\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + "notes": "{\"cliVersion\":\"2.122.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db index-stats\"],\"inspectRuns\":6,\"inspectCallsNamingOrders\":4,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db index-stats\",\"migration new\",\"migration up\",\"db reset\",\"db advisors\",\"migration list\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" }, { "name": "final report is truthful about the cause and the fix", "passed": true, - "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration and index, and accurately describes local application and verification." + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup as the cause. The named migration and index exist, were applied locally, and the verified plan uses the index." } ], "skills": { @@ -51229,123 +51003,94 @@ }, "docs": { "calls": [ - { - "source": "search_docs", - "query": "query { searchDocs(query: \"slow queries performance explain analyze missing indexes local database supabase db query\", limit: 6) { nodes { ... on Guide { title href content } } } }", - "hasContent": true, - "pages": [ - { - "url": "https://supabase.com/docs/guides/observability/inspect", - "title": "Inspect the database" - }, - { - "url": "https://supabase.com/docs/guides/database/debugging-performance", - "title": "Debugging performance issues" - }, - { - "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", - "title": "Row Level Security performance" - }, - { - "url": "https://supabase.com/kb/sitemap", - "title": "Supabase Knowledge Base" - } - ], - "resultChars": 23407 - }, { "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- Repo files ---'; rg --files -g '\"'!node_modules'\"' | head -250; printf '%s\\\\n' '--- Changelog breaking tags ---'; curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 2 'breaking-change' | head -80; printf '%s\\\\n' '--- relevant skill guidance ---'; rg -n -C 3 'local|performance|db query|advisors|migration|Monitoring' /tmp/sandbox-fe54b0cf/.agents/skills/supabase/SKILL.md; printf '%s\\\\n' '--- performance references ---'; rg --files /tmp/sandbox-fe54b0cf/.agents/skills/supabase-postgres-best-practices/references | head -60; printf '%s\\\\n' '--- status ---'; git status --short; supabase --version 2>&1 || true; supabase status 2>&1 || true\"", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", + "hasContent": true, "pages": [ { "url": "https://supabase.com/changelog.md" } ], - "resultChars": 15498 - }, - { - "source": "shell_fetch", - "query": "/bin/bash -lc \"printf '%s\\\\n' '--- Monitoring/performance docs ---'; curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | sed -n '1,100p'; printf '%s\\\\n' '--- inspect CLI help ---'; supabase inspect db help; printf '%s\\\\n' '--- local DB query help ---'; supabase db query --help; printf '%s\\\\n' '--- docker availability ---'; docker ps -a --format '{{.Names}} {{.Status}}' 2>&1 | head -40\"", - "pages": [ - { - "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" - } - ], - "resultChars": 13786 + "resultChars": 4978 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"Supabase CLI local migration create new migration db reset apply migration local db pull\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "query": "query { searchDocs(query: \"monitoring debugging slow queries database performance query insights Supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/local-development/cli-workflows", - "title": "Local development workflow" + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" }, { - "url": "https://supabase.com/docs/guides/local-development/database-migrations", - "title": "Database migrations" + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" }, { - "url": "https://supabase.com/docs/guides/deployment/branching/working-with-branches", - "title": "Working with branches" + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" }, { - "url": "https://supabase.com/docs/guides/deployment/managing-environments", - "title": "Managing Environments" + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" }, { - "url": "https://supabase.com/docs/guides/deployment/database-migrations", - "title": "Database Migrations" + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" } ], - "resultChars": 79403 + "resultChars": 97787 }, { "source": "search_docs", - "query": "query { searchDocs(query: \"CREATE INDEX CONCURRENTLY Supabase migrations transactions\", limit: 4) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "query": "query { searchDocs(query: \"database inspect slow query EXPLAIN ANALYZE pg_stat_statements local Supabase\", limit: 5) { nodes { title href content } } }", "hasContent": true, "pages": [ { - "url": "https://supabase.com/docs/guides/deployment/branching/github-integration", - "title": "GitHub integration" + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" }, { - "url": "https://supabase.com/docs/guides/realtime/reports", - "title": "Realtime Reports" + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" }, { - "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", - "title": "Migrating to Supabase" + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" }, { - "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", - "title": "Deleting data and dropping objects safely" + "url": "https://supabase.com/docs/guides/troubleshooting/high-latency-with-supabase-client-z0pZzR", + "title": "High latency with supabase client" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg", + "title": "HypoPG: Hypothetical indexes" } ], - "resultChars": 66262 + "resultChars": 29776 } ] }, "usage": [ { "model": "gpt-6-luna", - "inputTokens": 1177439, - "cacheReadInputTokens": 1118789, - "cacheWriteInputTokens": 58563, - "outputTokens": 7268 + "inputTokens": 1436762, + "cacheReadInputTokens": 1363430, + "cacheWriteInputTokens": 73230, + "outputTokens": 9486 } ], - "stepCount": 29, - "toolCallCount": 23, - "agentRunDurationMs": 196775, + "stepCount": 34, + "toolCallCount": 31, + "agentRunDurationMs": 317726, "sandboxUsage": { - "activeCpuDurationMs": 388723, - "duration": 313333, + "activeCpuDurationMs": 817672, + "duration": 496406, "memory": 8192, "networkTransfer": { - "ingress": 2734931085, - "egress": 16065460 + "ingress": 2794113354, + "egress": 14357577 } }, "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.",