diff --git a/apps/web/src/data/cli-eval-results.json b/apps/web/src/data/cli-eval-results.json index 84384a7b..7c58ff98 100644 --- a/apps/web/src/data/cli-eval-results.json +++ b/apps/web/src/data/cli-eval-results.json @@ -7271,6 +7271,539 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-absent/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-absent", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.15", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "tool call 8: supabase inspect db help (errored); no inspect output named the orders table" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": false, + "notes": "no new migration files" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, reconfigure, or gain privileges for a container runtime or VM." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db help\"],\"inspectRuns\":1,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"inspect db help\",\"db query\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":0,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report accurately says the agent stopped without adding or applying an index or migration. It names the Docker/Podman blocker and failed local database connection, and does not claim the slowness was fixed." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11952 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring and debugging slow queries pg_stat_statements explain analyze missing indexes Supabase local database performance\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + } + ], + "resultChars": 40820 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging guide slow queries identify long running queries EXPLAIN\", limit: 3) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/how-to-interpret-and-explore-the-postgres-logs-OuCIOj", + "title": "How to Interpret and Explore the Postgres Logs" + } + ], + "resultChars": 74470 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring and debugging guide database slow query diagnosis pg_stat_statements\", limit: 5) { nodes { ... on Guide { title href content } ... on TroubleshootingGuide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + } + ], + "resultChars": 45653 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 176675, + "cacheReadInputTokens": 128783, + "cacheWriteInputTokens": 47871, + "outputTokens": 3291 + } + ], + "stepCount": 7, + "toolCallCount": 16, + "agentRunDurationMs": 57658, + "sandboxUsage": { + "activeCpuDurationMs": 231679, + "duration": 267716, + "memory": 8192, + "networkTransfer": { + "ingress": 835035745, + "egress": 3691969 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-absent/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-absent", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.15", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no inspect output named the orders table" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023112845_add_orders_customer_history_index.sql (version 20261023112845)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No commands attempted to install, start, or reconfigure a container runtime or VM, escalate privileges, or change runtime permissions." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration new\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, describes the matching migration, and states that the fix was not applied or verified because the local database was unavailable." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -120'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 6220 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries database performance EXPLAIN query stats\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/debugging-performance", + "title": "Debugging performance issues" + } + ], + "resultChars": 98372 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/docs/guides/database/inspect.md | head -180; printf '\\\\n--- debugging-performance doc ---\\\\n'; curl -fsSL https://supabase.com/docs/guides/database/debugging-performance.md | head -120\"", + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/inspect.md" + }, + { + "url": "https://supabase.com/docs/guides/database/debugging-performance.md" + } + ], + "resultChars": 10818 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 181481, + "cacheReadInputTokens": 134947, + "cacheWriteInputTokens": 46513, + "outputTokens": 3950 + } + ], + "stepCount": 7, + "toolCallCount": 17, + "agentRunDurationMs": 60842, + "sandboxUsage": { + "activeCpuDurationMs": 241548, + "duration": 223458, + "memory": 8192, + "networkTransfer": { + "ingress": 835253723, + "egress": 3559060 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-absent/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-absent", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.15", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023112929_add_orders_customer_history_index.sql (version 20261023112929)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: docker: command not found (podman also not found) — inst..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration new\",\"db query\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the slow customer-email order-history lookups, accurately describes the new index migration, and states that the database change and query plan could not be verified because Docker was unavailable and the local connection failed." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"slow queries monitoring database performance EXPLAIN missing indexes Supabase\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + } + ], + "resultChars": 81032 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI migration new db query local db performance inspect explain analyze\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase Monitoring and Debugging database slow queries performance query performance missing index pg_stat_statements\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + } + ], + "resultChars": 38210 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/monitoring-and-debugging slow queries database query performance Supabase", + "pages": [] + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 195816, + "cacheReadInputTokens": 149311, + "cacheWriteInputTokens": 42091, + "outputTokens": 4824 + } + ], + "stepCount": 8, + "toolCallCount": 12, + "agentRunDurationMs": 76702, + "sandboxUsage": { + "activeCpuDurationMs": 229123, + "duration": 286826, + "memory": 8192, + "networkTransfer": { + "ingress": 835645008, + "egress": 3558719 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-absent/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-absent", "experimentSuite": "cli", @@ -14993,6 +15526,586 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-beta/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-beta", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.15", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no inspect output named the orders table" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023000000_index_orders_by_customer_and_created_at.sql (version 20261023000000)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023000000_index_orders_by_customer_and_created_at.sql (version 20261023000000) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, reconfigure, or gain privileged access to a container runtime. The Docker commands were read-only probes or ran psql against the local database; supabase start is permitted." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration list\",\"start\",\"db advisors\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup, the matching index and migration, and its local application. Its plan and timing claims match the transcript." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging slow queries performance postgres query plans indexes\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/postgres-log-config", + "title": "Postgres log configurations" + } + ], + "resultChars": 143632 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Database inspect query performance slow queries EXPLAIN ANALYZE pg_stat_statements indexes\", limit: 5) { nodes { ... on Guide { title href content } } }", + "hasContent": true, + "pages": [] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 1500'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 1500 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4082 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"database slow query monitoring EXPLAIN indexes\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + } + ], + "resultChars": 59476 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/observability/inspect | head -c 2000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect" + } + ], + "resultChars": 2049 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 726852, + "cacheReadInputTokens": 672424, + "cacheWriteInputTokens": 54362, + "outputTokens": 5942 + } + ], + "stepCount": 22, + "toolCallCount": 23, + "agentRunDurationMs": 146310, + "sandboxUsage": { + "activeCpuDurationMs": 472930, + "duration": 285900, + "memory": 8192, + "networkTransfer": { + "ingress": 2797706474, + "egress": 14025607 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-beta/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-beta", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.15", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": true, + "judgeNotes": "The `supabase inspect db calls --local` and `outliers --local` outputs show normalized queries against public.orders, including the order-history query with execution time and call counts." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113337_index_orders_customer_history.sql (version 20261023113337)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "version(s) 20261023113337 recorded with statements that do not create an index on orders.customer_email" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No commands attempted to install, start, reconfigure, or gain access to a container runtime. Docker commands were read-only probes; Supabase commands were permitted." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db calls\",\"db outliers\"],\"inspectRuns\":3,\"inspectCallsNamingOrders\":4,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"inspect db calls\",\"inspect db outliers\",\"migration new\",\"migration up\",\"migration list\",\"db advisors\"],\"explainCommands\":2,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": false, + "judgeNotes": "The cause and index fix are accurate, but the report says the index-creating migration is recorded locally. Ground truth says that version’s recorded statements do not create the index." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -80'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4082 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring and debugging slow queries database performance explain analyze local Supabase CLI\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + } + ], + "resultChars": 65432 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres indexes B-tree composite index multicolumn order by performance migration create index concurrently\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search", + "title": "Full Text Search" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/error-index-row-size-exceeds-btree-version-4-maximum-for-index-LMmoeU", + "title": "Error: index row size exceeds btree version 4 maximum for index" + }, + { + "url": "https://supabase.com/docs/guides/database/migrating-to-pg-partman", + "title": "Migrate from TimescaleDB to pg_partman" + } + ], + "resultChars": 60059 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase migration CREATE INDEX CONCURRENTLY transaction migrations locking large table\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/increase-vector-lookup-speeds-by-applying-an-hsnw-index-ohLHUM", + "title": "Increase vector lookup speeds by applying an HSNW index" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + } + ], + "resultChars": 87648 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 850496, + "cacheReadInputTokens": 779799, + "cacheWriteInputTokens": 70628, + "outputTokens": 10891 + } + ], + "stepCount": 23, + "toolCallCount": 27, + "agentRunDurationMs": 365687, + "sandboxUsage": { + "activeCpuDurationMs": 564748, + "duration": 644692, + "memory": 8192, + "networkTransfer": { + "ingress": 1983723361, + "egress": 10104383 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-beta/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-beta", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.15", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113220_add_orders_customer_history_index.sql (version 20261023113220)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023113220_add_orders_customer_history_index.sql (version 20261023113220) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, configure, or start a container runtime. Docker commands were read-only probes; stack startup used the permitted Supabase CLI." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"migration list\",\"migration new\",\"start\",\"db query\",\"db advisors\",\"db reset\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the slow customer-email order-history lookup and the index fix. The migration, local replay, and index-scan verification match the transcript and ground truth." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase database performance monitoring slow queries pg_stat_statements indexes explain analyze local development CLI db query\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-calls", + "title": "Show queries from pg_stat_statements ordered by total times called\\\\n" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" + } + ], + "resultChars": 40036 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11952 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging slow queries pg_stat_statements query performance indexes Supabase database\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/steps-to-improve-query-performance-with-indexes-q8PoC9", + "title": "Steps to improve query performance with indexes" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + } + ], + "resultChars": 41534 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1396960, + "cacheReadInputTokens": 1327355, + "cacheWriteInputTokens": 69506, + "outputTokens": 10584 + } + ], + "stepCount": 33, + "toolCallCount": 36, + "agentRunDurationMs": 498006, + "sandboxUsage": { + "activeCpuDurationMs": 828661, + "duration": 734970, + "memory": 8192, + "networkTransfer": { + "ingress": 2795809694, + "egress": 14433622 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-beta/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-beta", "experimentSuite": "cli", @@ -22897,6 +24010,1546 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-next/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-next", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "3.0.0-next.23", + "passed": true, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": true, + "judgeNotes": "The `supabase inspect db outliers`, `calls`, and `table-stats` outputs directly name public.orders and show the order-history query." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113115_add_orders_customer_email_created_at_idx.sql (version 20261023113115)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023113115_add_orders_customer_email_created_at_idx.sql (version 20261023113115) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime or VM, escalate privileges, or change runtime permissions." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"3.0.0-next.23\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db outliers\",\"db calls\",\"db table-stats\"],\"inspectRuns\":5,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"status\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"inspect db table-stats\",\"migration up\",\"migration list\",\"db pull\",\"db reset\",\"db advisors\"],\"explainCommands\":2,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup as the cause and accurately describes the index migration, its local application, and the verified index-scan plan." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging database slow queries pg_stat_statements performance\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + } + ], + "resultChars": 30951 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 12000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 11952 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries database performance inspect db CLI query optimization\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db", + "title": "Tools to inspect your Supabase database" + } + ], + "resultChars": 74066 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1051978, + "cacheReadInputTokens": 983132, + "cacheWriteInputTokens": 68765, + "outputTokens": 8499 + } + ], + "stepCount": 27, + "toolCallCount": 33, + "agentRunDurationMs": 271290, + "sandboxUsage": { + "activeCpuDurationMs": 701638, + "duration": 497920, + "memory": 8192, + "networkTransfer": { + "ingress": 1968297085, + "egress": 10877859 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-next/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-next", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "3.0.0-next.23", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113604_add_orders_customer_history_index.sql (version 20261023113604)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023113604_add_orders_customer_history_index.sql (version 20261023113604) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, enable, or reconfigure a container runtime or VM. `docker start supabase_db_shop-app` started a Supabase database container, not the runtime." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"3.0.0-next.23\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"start\",\"migration up\",\"migration new\",\"migration list\",\"db advisors\"],\"explainCommands\":4,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer_email order-history lookup and accurately describes the locally applied migration, index scan, and measured improvement." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 20000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 19902 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase slow queries monitoring database performance pg_stat_statements explain analyze local development migration\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + } + ], + "resultChars": 41190 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase slow queries pg_stat_statements query performance explain analyze\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + } + ], + "resultChars": 40672 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI local development start database only migration new imperative migrations\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/declarative-database-schemas", + "title": "Declarative database schemas" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-start", + "title": "Starts local Postgres database" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli/getting-started", + "title": "Supabase CLI" + }, + { + "url": "https://supabase.com/docs/guides/functions/development-environment", + "title": "Development Environment" + } + ], + "resultChars": 58548 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 4055911, + "cacheReadInputTokens": 3971271, + "cacheWriteInputTokens": 84400, + "outputTokens": 19018 + } + ], + "stepCount": 80, + "toolCallCount": 54, + "agentRunDurationMs": 528570, + "sandboxUsage": { + "activeCpuDurationMs": 712586, + "duration": 805019, + "memory": 8192, + "networkTransfer": { + "ingress": 2232154138, + "egress": 34108054 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-next/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-next", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "3.0.0-next.23", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no inspect output named the orders table" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113340_add_orders_customer_created_at_index.sql (version 20261023113340)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023113340_add_orders_customer_created_at_index.sql (version 20261023113340) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. The Docker commands were read-only probes; Supabase commands are allowed." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"3.0.0-next.23\",\"cliOverride\":[],\"channel\":\"next\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"start\",\"migration new\",\"migration list\",\"migration up\",\"db reset\",\"db advisors\"],\"explainCommands\":4,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, describes the matching migration and local application, and cites a verified index-scan plan and improved timings." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 30000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 29836 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres missing indexes composite index query filters order by performance\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#statistics", + "title": "Statistics" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#example-query", + "title": "Example query" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#where-clause", + "title": "where clause:" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#join-columns", + "title": "join columns" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#order-by-clause", + "title": "order by clause" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#key-concepts", + "title": "Key concepts" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#analyze-the-query-plan", + "title": "Analyze the query plan" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#use-appropriate-index-types", + "title": "Use appropriate index types" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#partial-indexes", + "title": "Partial indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#composite-indexes", + "title": "Composite indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#over-indexing", + "title": "Over-Indexing" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search", + "title": "Full Text Search" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#search-a-single-column", + "title": "Search a single column" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#search-multiple-columns", + "title": "Search multiple columns" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#match-all-search-words", + "title": "Match all search words" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#match-any-search-words", + "title": "Match any search words" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#partial-search", + "title": "Partial search" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#implementing-partial-search", + "title": "Implementing partial search" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#extending-functionality-with-rpc", + "title": "Extending functionality with RPC" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#handling-spaces-in-queries", + "title": "Handling spaces in queries" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#websearch-to-tsquery", + "title": "Web search syntax with websearch_to_tsquery() " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#basic-usage", + "title": "Basic usage" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#quoted-phrases", + "title": "Quoted phrases" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#or-searches", + "title": "OR searches" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#negation-1", + "title": "Negation: ! " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#complex-queries", + "title": "Complex queries" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#creating-indexes", + "title": "Creating indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#searchable-columns", + "title": "Searchable columns" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#negation", + "title": "Negation" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#ranking", + "title": "Ranking search results " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#search-function-ranking", + "title": "Creating a search function with ranking " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#weighted-ranking", + "title": "Ranking with weighted columns " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#understanding-weight-labels", + "title": "Understanding weight labels" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#creating-weighted-search-columns", + "title": "Creating weighted search columns" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#custom-weight-arrays", + "title": "Custom weight arrays" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#using-the-weighted-search", + "title": "Using the weighted search" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#practical-example-with-results", + "title": "Practical example with results" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#ranking-with-indexes", + "title": "Using ranking with indexes " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#websearch-ranking", + "title": "Using web search syntax with ranking " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#preparation", + "title": "Preparation" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#usage", + "title": "Usage" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#to-tsvector", + "title": "to_tsvector() " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#to-tsquery", + "title": "to_tsquery() " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#match", + "title": "Match: @@ " + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#basic-full-text-queries", + "title": "Basic full text queries" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#search-using-the-new-column", + "title": "Search using the new column" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#query-operators", + "title": "Query operators" + }, + { + "url": "https://supabase.com/docs/guides/database/full-text-search#proximity", + "title": "Proximity: <-> " + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg", + "title": "HypoPG: Hypothetical indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg#enable-the-extension", + "title": "Enable the extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg#speeding-up-a-query", + "title": "Speeding up a query" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg#functions", + "title": "Functions" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search", + "title": "Semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#similarity-metric", + "title": "Similarity metric" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#when-to-use-semantic-search", + "title": "When to use semantic search" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#how-semantic-search-works", + "title": "How semantic search works" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#embedding-models", + "title": "Embedding models" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#semantic-search-in-postgres", + "title": "Semantic search in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#see-also", + "title": "See also" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#hnsw-index", + "title": "HNSW index" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#ivfflat-index", + "title": "IVFFlat index" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#pgvector-index-tuning", + "title": "pgvector index tuning" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#next-steps", + "title": "Next steps" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#filtering-vector-search-by-metadata", + "title": "Filtering vector search by metadata" + }, + { + "url": "https://supabase.com/docs/guides/ai/semantic-search#calling-from-your-application", + "title": "Calling from your application" + } + ], + "resultChars": 166196 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring and debugging slow queries performance explain analyze index advisor local database\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#ordering-indexes", + "title": "Ordering indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#partial-indexes", + "title": "Partial indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#understanding-index-advisor-results", + "title": "Understanding Index Advisor results" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#index-advisor", + "title": "Index Advisor" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#reindexing", + "title": "Reindexing" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes#create-an-index", + "title": "Create an index" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#shared-pooler-supavisor-client-connections", + "title": "Shared Pooler (Supavisor) Client Connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-usage", + "title": "Disk Usage" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#database-size", + "title": "Database size" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#postgrest", + "title": "PostgREST" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#realtime", + "title": "Realtime" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#realtime-api-gateway", + "title": "Realtime API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#storage", + "title": "Storage" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#edge-functions", + "title": "Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#using-reports", + "title": "Using reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#api-gateway", + "title": "API gateway" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#auth", + "title": "Auth" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#auth-api-gateway", + "title": "Auth API Gateway" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#database", + "title": "Database" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#advanced-telemetry", + "title": "Advanced Telemetry" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#memory-usage", + "title": "Memory usage" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#memory-commitment", + "title": "Memory commitment" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#cpu-usage", + "title": "CPU usage" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-inputoutput-operations-per-second-iops", + "title": "Disk input/output operations per second (IOPS)" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-throughput", + "title": "Disk throughput" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#disk-size", + "title": "Disk size" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#query-performance", + "title": "Query Performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#database-connections", + "title": "Database connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports#dedicated-pooler-pgbouncer-client-connections", + "title": "Dedicated Pooler (PgBouncer) Client Connections" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance", + "title": "Performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance#what-it-watches", + "title": "What it watches" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance#when-it-watches", + "title": "When it watches" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance#what-it-will-output", + "title": "What it will output" + }, + { + "url": "https://supabase.com/docs/guides/observability/automate-with-agents/performance#set-up-the-agent", + "title": "Set up the agent" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#rate-limiting-resource-allocation--abuse-prevention", + "title": "Rate limiting, resource allocation, & abuse prevention" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#abuse-prevention", + "title": "Abuse prevention" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#realtime-limits", + "title": "Realtime limits" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#performance", + "title": "Performance" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#deployment", + "title": "Deployment" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#availability", + "title": "Availability" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#security", + "title": "Security" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#auth-rate-limits", + "title": "Auth rate limits" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod#email-link-validity", + "title": "Email link validity" + } + ], + "resultChars": 123579 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging Supabase slow queries pg_stat_statements explain analyze\", limit: 4) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#timeouts", + "title": "Timeouts" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#statement-optimization", + "title": "Statement optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#managing-timezones", + "title": "Managing timezones" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#change-timezone", + "title": "Change timezone" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#full-list-of-timezones", + "title": "Full list of timezones" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration#search-for-a-specific-timezone", + "title": "Search for a specific timezone" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#partial-indexes", + "title": "Partial indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#composite-indexes", + "title": "Composite indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#over-indexing", + "title": "Over-Indexing" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#statistics", + "title": "Statistics" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#example-query", + "title": "Example query" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#where-clause", + "title": "where clause:" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#join-columns", + "title": "join columns" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#order-by-clause", + "title": "order by clause" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#key-concepts", + "title": "Key concepts" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#analyze-the-query-plan", + "title": "Analyze the query plan" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization#use-appropriate-index-types", + "title": "Use appropriate index types" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#enable-the-extension", + "title": "Enable the extension" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#inspecting-activity", + "title": "Inspecting activity" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements#resources", + "title": "Resources" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#diagnose-whether-rls-is-the-bottleneck", + "title": "Diagnose whether RLS is the bottleneck" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#measure-through-the-data-api", + "title": "Measure through the Data API" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#filter-in-the-client-query-too", + "title": "Filter in the client query too" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#avoid-joins-in-policy-expressions", + "title": "Avoid joins in policy expressions" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance#more-resources", + "title": "More resources" + } + ], + "resultChars": 41627 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"CREATE INDEX CONCURRENTLY migrations transaction Supabase db migration\", limit: 5) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#preparing-to-delete", + "title": "Preparing to delete" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#identifying-dependencies", + "title": "Identifying dependencies" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#data-deletion-strategies", + "title": "Data deletion strategies" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#small-deletes", + "title": "Small deletes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#large-deletes", + "title": "Large deletes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#soft-deletes", + "title": "Soft deletes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#deleting-all-data", + "title": "Deleting all data" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#object-deletion-strategies", + "title": "Object deletion strategies" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#dropping-tables", + "title": "Dropping tables" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#dropping-columns", + "title": "Dropping columns" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#dropping-indexes", + "title": "Dropping indexes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#monitoring", + "title": "Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#check-for-blocked-queries", + "title": "Check for blocked queries" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#monitor-table-bloat-after-large-deletes", + "title": "Monitor table bloat after large deletes" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#reclaiming-disk-space", + "title": "Reclaiming disk space" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion#related-links", + "title": "Related links" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase#migration-guides", + "title": "Migration guides" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration", + "title": "GitHub integration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#email-notifications", + "title": "Email notifications" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#preventing-migration-failures", + "title": "Preventing migration failures" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#deploying-changes-to-production", + "title": "Deploying changes to production" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#seeding", + "title": "Seeding" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#migrations", + "title": "Migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#configuration", + "title": "Configuration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#syncing-github-branches", + "title": "Syncing GitHub branches" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#preparing-your-git-repository", + "title": "Preparing your Git repository" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#set-the-working-directory", + "title": "Set the working directory" + }, + { + "url": "https://supabase.com/docs/guides/deployment/branching/github-integration#installation", + "title": "Installation" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#read-private-channel-subscription-rls-execution-time", + "title": "(Read) Private Channel Subscription RLS Execution Time" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#realtime-reports-overview", + "title": "Realtime reports overview" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#connected-clients", + "title": "Connected Clients" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#broadcast-events", + "title": "Broadcast Events" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-1", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#presence-events", + "title": "Presence Events" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-2", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#postgres-changes-events", + "title": "Postgres Changes Events" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-3", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#rate-of-channel-joins", + "title": "Rate of Channel Joins" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-4", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#message-payload-size", + "title": "Message Payload Size" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-5", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#broadcast-from-database-replication-lag", + "title": "Broadcast From Database Replication Lag" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-6", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-7", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#write-private-channel-subscription-rls-execution-time", + "title": "(Write) Private Channel Subscription RLS Execution Time" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-8", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#total-requests", + "title": "Total Requests" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-9", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#response-errors", + "title": "Response Errors" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-10", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#response-speed", + "title": "Response Speed" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports#actions-you-can-take-11", + "title": "Actions you can take" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#auto-schema-diff", + "title": "Auto schema diff" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#set-up-a-local-environment", + "title": "Set up a local environment" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#create-a-new-migration", + "title": "Create a new migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#rebasing-new-migrations", + "title": "Rebasing new migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-push", + "title": "Permission denied on db push" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-pull", + "title": "Permission denied on db pull" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#sync-production-project-to-staging", + "title": "Sync production project to staging" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#release-to-production", + "title": "Release to production" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#open-a-pr-with-new-migration", + "title": "Open a PR with new migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#configure-github-actions", + "title": "Configure GitHub Actions" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#deploy-a-migration", + "title": "Deploy a migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#manual-migration", + "title": "Manual migration" + } + ], + "resultChars": 170175 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase migration CREATE INDEX CONCURRENTLY transaction db push local migration\", limit: 4) { nodes { ... on Guide { title href content subsections { nodes { title href content } } } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#add-sample-data", + "title": "Add sample data" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#diffing-changes", + "title": "Diffing changes" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#deploy-your-project", + "title": "Deploy your project" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#log-in-to-the-supabase-cli", + "title": "Log in to the Supabase CLI" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#link-your-project", + "title": "Link your project" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#deploy-database-changes", + "title": "Deploy database changes" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#deploy-edge-functions", + "title": "Deploy Edge Functions" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#use-auth-locally", + "title": "Use Auth locally" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#sync-storage-buckets", + "title": "Sync storage buckets" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#sync-any-schema-with---schema", + "title": "Sync any schema with --schema" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations#limitations-and-considerations", + "title": "Limitations and considerations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#working-with-a-team", + "title": "Working with a team" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#schema-migrations", + "title": "Schema migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#seeding-data", + "title": "Seeding data" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#diffing-changes", + "title": "Diffing changes" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#deploy-your-project", + "title": "Deploy your project" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#diagnosing-and-fixing-sync-errors", + "title": "Diagnosing and fixing sync errors" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#how-migration-tracking-works", + "title": "How migration tracking works" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-1-check-whats-out-of-sync", + "title": "Step 1: Check what's out of sync" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-2-if-you-made-changes-on-the-remote-database-directly", + "title": "Step 2: If you made changes on the remote database directly" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations#step-3-if-the-migration-history-table-is-wrong", + "title": "Step 3: If the migration history table is wrong" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments", + "title": "Managing Environments" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#set-up-a-local-environment", + "title": "Set up a local environment" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#manual-migration", + "title": "Manual migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#auto-schema-diff", + "title": "Auto schema diff" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#deploy-a-migration", + "title": "Deploy a migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#configure-github-actions", + "title": "Configure GitHub Actions" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#open-a-pr-with-new-migration", + "title": "Open a PR with new migration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#release-to-production", + "title": "Release to production" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#sync-production-project-to-staging", + "title": "Sync production project to staging" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-pull", + "title": "Permission denied on db pull" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#permission-denied-on-db-push", + "title": "Permission denied on db push" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#rebasing-new-migrations", + "title": "Rebasing new migrations" + }, + { + "url": "https://supabase.com/docs/guides/deployment/managing-environments#create-a-new-migration", + "title": "Create a new migration" + } + ], + "resultChars": 152145 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1688770, + "cacheReadInputTokens": 1603807, + "cacheWriteInputTokens": 84861, + "outputTokens": 11944 + } + ], + "stepCount": 34, + "toolCallCount": 44, + "agentRunDurationMs": 410951, + "sandboxUsage": { + "activeCpuDurationMs": 761530, + "duration": 645698, + "memory": 8192, + "networkTransfer": { + "ingress": 2778824994, + "egress": 17584243 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-next/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-next", "experimentSuite": "cli", @@ -30796,6 +33449,565 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-nodaemon/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-nodaemon", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.15", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113013_index_orders_customer_email_created_at.sql (version 20261023113013)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, or reconfigure a container runtime, escalate privileges, or change runtime permissions." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"migration list\",\"migration new\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration, and does not claim the migration was applied or its performance verified." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4978 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging slow queries query performance local database EXPLAIN Supabase\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + }, + { + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" + }, + { + "url": "https://supabase.com/docs/guides/database/debugging-performance", + "title": "Debugging performance issues" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + } + ], + "resultChars": 51933 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Debugging and monitoring Postgres inspect slow queries EXPLAIN pg_stat_statements\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + } + ], + "resultChars": 33842 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase migration CREATE INDEX CONCURRENTLY transaction migrations\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/increase-vector-lookup-speeds-by-applying-an-hsnw-index-ohLHUM", + "title": "Increase vector lookup speeds by applying an HSNW index" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/platform/sso/multiple-providers", + "title": "Multiple SSO Providers" + } + ], + "resultChars": 84493 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 357794, + "cacheReadInputTokens": 303990, + "cacheWriteInputTokens": 53768, + "outputTokens": 7150 + } + ], + "stepCount": 12, + "toolCallCount": 22, + "agentRunDurationMs": 120577, + "sandboxUsage": { + "activeCpuDurationMs": 252303, + "duration": 335187, + "memory": 8192, + "networkTransfer": { + "ingress": 836672817, + "egress": 4969264 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-nodaemon/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-nodaemon", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.15", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "tool call 4: supabase inspect db help; tool call 4: supabase inspect db outliers; no inspect output named the orders table" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023000000_add_orders_customer_created_at_index.sql (version 20261023000000)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted a forbidden container-runtime detour. Docker commands were read-only probes." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[\"db help\",\"db outliers\"],\"inspectRuns\":2,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"inspect db help\",\"inspect db outliers\",\"db query\"],\"explainCommands\":1,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report identifies the unindexed customer-email order-history lookup, accurately describes the new index migration, and states that the unavailable local database prevented verification. It does not claim the migration was applied." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "web_search", + "query": "https://supabase.com/changelog.md", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"slow query performance diagnose explain analyze pg_stat_statements Supabase database\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/high-latency-with-supabase-client-z0pZzR", + "title": "High latency with supabase client" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + }, + { + "url": "https://supabase.com/docs/guides/observability/detecting", + "title": "Detection checks" + } + ], + "resultChars": 35553 + }, + { + "source": "web_search", + "query": "site:supabase.com/changelog.md Supabase changelog breaking-change", + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Postgres create index composite index order by filter migration create index concurrently Supabase\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mysql", + "title": "Migrate from MySQL to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + } + ], + "resultChars": 29609 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 262012, + "cacheReadInputTokens": 213882, + "cacheWriteInputTokens": 43495, + "outputTokens": 5554 + } + ], + "stepCount": 11, + "toolCallCount": 12, + "agentRunDurationMs": 82967, + "sandboxUsage": { + "activeCpuDurationMs": 117119, + "duration": 177078, + "memory": 8192, + "networkTransfer": { + "ingress": 839147426, + "egress": 6240544 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-nodaemon/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-nodaemon", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0-beta.15", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no inspect output named the orders table" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": false, + "notes": "could not read the plan: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "orders still holds every seeded row", + "passed": false, + "notes": "could not count public.orders: managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023112836_index_orders_customer_history.sql (version 20261023112836)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": false, + "notes": "managed: exit 1: {\"_tag\":\"Error\",\"error\":{\"code\":\"ExperimentalStackStatusError\",\"message\":\"No managed stack exists for the selected project.\",\"suggestion\":\"Run supabase stack start first.\"}}; legacy: exit 1: failed to inspect container health: Cannot connect to the Docker daemon at tcp://127.0.0.1:1..." + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No commands attempted a container-runtime detour." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0-beta.15\",\"cliOverride\":[],\"channel\":\"beta\",\"backend\":\"none\",\"runtime\":\"none\",\"relocatedHome\":null,\"dbPort\":null,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db start\",\"migration new\",\"migration list\",\"db query\",\"db advisors\"],\"explainCommands\":0,\"directIndexDdl\":0,\"ordersIndexes\":null,\"orderRows\":null,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "Correctly identifies the repeated customer-email order-history lookup as the bottleneck and reports the index migration accurately. It does not claim the migration was applied or performance verified; it names the Docker blocker." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging slow queries pg_stat_statements explain analyze Supabase database performance\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + } + ], + "resultChars": 41190 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI local database query explain analyze db query migration new\", limit: 5) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase", + "title": "Migrating to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/deployment/database-migrations", + "title": "Database Migrations" + }, + { + "url": "https://supabase.com/docs/guides/local-development", + "title": "Local Development & CLI" + } + ], + "resultChars": 61127 + }, + { + "source": "web_search", + "query": "site:supabase.com/docs/guides/monitoring-and-debugging slow queries Supabase pg_stat_statements", + "pages": [] + }, + { + "source": "web_search", + "query": "Supabase monitoring and debugging slow queries pg_stat_statements", + "pages": [] + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 354263, + "cacheReadInputTokens": 290171, + "cacheWriteInputTokens": 54953, + "outputTokens": 7402 + } + ], + "stepCount": 11, + "toolCallCount": 19, + "agentRunDurationMs": 107779, + "sandboxUsage": { + "activeCpuDurationMs": 206396, + "duration": 243526, + "memory": 8192, + "networkTransfer": { + "ingress": 837818463, + "egress": 5744552 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-nodaemon/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-nodaemon", "experimentSuite": "cli", @@ -38936,6 +42148,579 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-pinned/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-pinned", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.117.0", + "passed": true, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": true, + "judgeNotes": "The `supabase inspect db outliers --local` output directly shows queries against public.orders, including the order-history query with execution time and call count." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Only Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113134_orders_customer_email_created_at_idx.sql (version 20261023113134)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023113134_orders_customer_email_created_at_idx.sql (version 20261023113134) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted a container-runtime detour. Docker commands were read-only probes; stack operations used the Supabase CLI." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db calls\"],\"inspectRuns\":5,\"inspectCallsNamingOrders\":4,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db calls\",\"db advisors\",\"migration new\",\"db reset\",\"migration list\"],\"explainCommands\":5,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, the matching index and migration, and its verified application to the local database. Its performance claims match the transcript; it does not claim a remote deployment." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries pg_stat_statements database performance local database explain analyze\", limit: 8) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + }, + { + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" + }, + { + "url": "https://supabase.com/kb/topics/troubleshooting", + "title": "Troubleshooting" + }, + { + "url": "https://supabase.com/kb/topics/database", + "title": "Database" + } + ], + "resultChars": 38886 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI inspect db query performance local db query command migration db pull\", limit: 8) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/local-development/cli-workflows", + "title": "Local development workflow" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-pull", + "title": "Pull schema from the remote database" + }, + { + "url": "https://supabase.com/docs/guides/local-development/database-migrations", + "title": "Database migrations" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-db-push", + "title": "Push new migrations to the remote database" + } + ], + "resultChars": 57437 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Monitoring and Debugging diagnose slow queries Postgres pg_stat_statements query performance Supabase\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + } + ], + "resultChars": 36115 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/docs/guides/monitoring-and-debugging.md | head -c 6000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/monitoring-and-debugging.md" + } + ], + "resultChars": 3452 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc \"curl -fsSL https://supabase.com/changelog.md | rg -n -i -C 1 'breaking-change|database|postgres|cli' | head -100\"", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5394 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Create indexes concurrently migrations Postgres avoid blocking writes Supabase migration\", limit: 5) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/vercel-postgres", + "title": "Migrate from Vercel Postgres to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mysql", + "title": "Migrate from MySQL to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/migrating-to-supabase/mssql", + "title": "Migrate from MSSQL to Supabase" + }, + { + "url": "https://supabase.com/docs/guides/platform/upgrading", + "title": "Upgrading" + } + ], + "resultChars": 33652 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1685175, + "cacheReadInputTokens": 1604159, + "cacheWriteInputTokens": 80911, + "outputTokens": 8500 + } + ], + "stepCount": 35, + "toolCallCount": 36, + "agentRunDurationMs": 308037, + "sandboxUsage": { + "activeCpuDurationMs": 792910, + "duration": 477556, + "memory": 8192, + "networkTransfer": { + "ingress": 2838972517, + "egress": 14662936 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-pinned/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-pinned", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.117.0", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no inspect output named the orders table" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113343_add_orders_customer_created_at_index.sql (version 20261023113343)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023113343_add_orders_customer_created_at_index.sql (version 20261023113343) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No commands attempted to install, start, or reconfigure a container runtime. Docker commands were read-only probes; Supabase CLI and database commands are allowed." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"services\",\"start\",\"db query\",\"migration new\",\"migration list\",\"db push\"],\"explainCommands\":3,\"directIndexDdl\":1,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration and index, and accurately reports local application and an index-scan verification." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring debugging slow queries pg_stat_statements local database performance diagnose\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + } + ], + "resultChars": 42356 + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 5000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 5011 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 869185, + "cacheReadInputTokens": 805985, + "cacheWriteInputTokens": 63131, + "outputTokens": 5969 + } + ], + "stepCount": 23, + "toolCallCount": 28, + "agentRunDurationMs": 344859, + "sandboxUsage": { + "activeCpuDurationMs": 634283, + "duration": 587096, + "memory": 8192, + "networkTransfer": { + "ingress": 2838225133, + "egress": 9772006 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-pinned/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-pinned", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.117.0", + "passed": true, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": true, + "judgeNotes": "The `supabase inspect db outliers --local` and `calls --local` outputs show queries against public.orders; `index-stats --local` also names the table." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023000000_index_orders_customer_history.sql (version 20261023000000)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023000000_index_orders_customer_history.sql (version 20261023000000) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "The commands contain no prohibited container-runtime detours. Docker commands were read-only probes, and Supabase CLI commands are allowed." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.117.0\",\"cliOverride\":[],\"channel\":\"pinned\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db calls\",\"db index-stats\"],\"inspectRuns\":4,\"inspectCallsNamingOrders\":2,\"workflowCommands\":[\"status\",\"inspect db help\",\"migration list\",\"inspect db outliers\",\"inspect db calls\",\"db query\",\"start\",\"migration up\",\"db advisors\",\"inspect db index-stats\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching index and migration, and accurately describes local application and the verified index-scan plan." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI local database inspect query pg_stat_statements advisors migrations local\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4978 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase slow queries monitoring debugging pg_stat_statements database performance\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + } + ], + "resultChars": 42356 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Inspect the database slow queries pg_stat_statements query stats Supabase CLI\", limit: 3) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db-calls", + "title": "Show queries from pg_stat_statements ordered by total times called\\\\n" + } + ], + "resultChars": 22422 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 818006, + "cacheReadInputTokens": 755957, + "cacheWriteInputTokens": 61980, + "outputTokens": 6656 + } + ], + "stepCount": 23, + "toolCallCount": 19, + "agentRunDurationMs": 311266, + "sandboxUsage": { + "activeCpuDurationMs": 600605, + "duration": 548069, + "memory": 8192, + "networkTransfer": { + "ingress": 2837401383, + "egress": 9094256 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-pinned/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-pinned", "experimentSuite": "cli", @@ -46752,6 +50537,567 @@ "run": 3, "sourcePath": "codex-gpt-6-luna-cli-stable/deploy-functions-003-set-and-list-secrets/run-3/result.json" }, + { + "experiment": "codex-gpt-6-luna-cli-stable", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0", + "passed": false, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": false, + "notes": "no supabase inspect report ran" + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113143_orders_customer_history_index.sql (version 20261023113143)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023113143_orders_customer_history_index.sql (version 20261023113143) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No commands attempted to install, start, or reconfigure a container runtime or VM. The agent used permitted Supabase CLI and database commands." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[],\"inspectRuns\":0,\"inspectCallsNamingOrders\":0,\"workflowCommands\":[\"status\",\"db query\",\"start\",\"db advisors\",\"db pull\",\"migration list\",\"db reset\"],\"explainCommands\":3,\"directIndexDdl\":2,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, names the matching migration and index, and accurately describes local application and verification." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 30000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 29836 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"database performance slow queries monitoring debugging explain analyze indexes query performance\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/postgres-log-config", + "title": "Postgres log configurations" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + } + ], + "resultChars": 131821 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase CLI db query --local command query file explain\", limit: 5) { nodes { title href content } }", + "hasContent": true, + "pages": [] + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Debugging and monitoring Postgres inspect slow queries pg_stat_statements explain analyze local database\", limit: 6) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/running-explain-analyze-on-functions", + "title": "Running EXPLAIN ANALYZE on functions" + }, + { + "url": "https://supabase.com/kb/topics/troubleshooting", + "title": "Troubleshooting" + } + ], + "resultChars": 19878 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1994989, + "cacheReadInputTokens": 1925628, + "cacheWriteInputTokens": 69220, + "outputTokens": 11304 + } + ], + "stepCount": 47, + "toolCallCount": 34, + "agentRunDurationMs": 344013, + "sandboxUsage": { + "activeCpuDurationMs": 890820, + "duration": 542331, + "memory": 8192, + "networkTransfer": { + "ingress": 2811776659, + "egress": 17912541 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 1, + "sourcePath": "codex-gpt-6-luna-cli-stable/investigate-database-001-slow-query/run-1/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-stable", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0", + "passed": true, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": true, + "judgeNotes": "Local supabase inspect outliers and calls reports show queries against public.orders; index-stats and table-stats also name public.orders." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023120000_add_orders_customer_history_index.sql (version 20261023120000)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023120000_add_orders_customer_history_index.sql (version 20261023120000) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted to install, start, reconfigure, or gain access to a container runtime. Docker commands were read-only probes; Supabase commands are permitted." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db calls\",\"db index-stats\",\"db table-stats\",\"db blocking\",\"db long-running-queries\",\"db db-stats\"],\"inspectRuns\":17,\"inspectCallsNamingOrders\":4,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"inspect db outliers\",\"inspect db calls\",\"inspect db index-stats\",\"inspect db table-stats\",\"inspect db blocking\",\"inspect db long-running-queries\",\"db query\",\"inspect db db-stats\",\"migration up\",\"migration list\",\"db advisors\"],\"explainCommands\":3,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup, describes the matching index migration applied locally, and cites a verified index-scan plan and timings." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -c 20000'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 19902 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Supabase monitoring and debugging slow queries pg_stat_statements local database performance\", limit: 6) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/pg_stat_statements", + "title": "pg_stat_statements: Query Performance Monitoring" + }, + { + "url": "https://supabase.com/docs/guides/deployment/going-into-prod", + "title": "Production Checklist" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" + } + ], + "resultChars": 42458 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"Inspecting your database slow queries table index stats query stats Supabase CLI inspect db query stats\", limit: 8) { nodes { ... on Guide { title href content } ... on CLICommandReference { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/reference/cli/supabase-inspect-db", + "title": "Tools to inspect your Supabase database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/data-deletion", + "title": "Deleting data and dropping objects safely" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/kb/sitemap", + "title": "Supabase Knowledge Base" + } + ], + "resultChars": 71286 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"PostgreSQL multicolumn btree index ORDER BY DESC LIMIT index only scan INCLUDE columns\", limit: 4) { nodes { ... on Guide { title href content } } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/database/postgres/indexes", + "title": "Managing Indexes in Postgres" + }, + { + "url": "https://supabase.com/docs/guides/database/query-optimization", + "title": "Query Optimization" + }, + { + "url": "https://supabase.com/docs/guides/database/orioledb", + "title": "OrioleDB Overview" + } + ], + "resultChars": 18376 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1060241, + "cacheReadInputTokens": 992323, + "cacheWriteInputTokens": 67834, + "outputTokens": 8408 + } + ], + "stepCount": 28, + "toolCallCount": 29, + "agentRunDurationMs": 379217, + "sandboxUsage": { + "activeCpuDurationMs": 662297, + "duration": 616776, + "memory": 8192, + "networkTransfer": { + "ingress": 2793881248, + "egress": 12031760 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 2, + "sourcePath": "codex-gpt-6-luna-cli-stable/investigate-database-001-slow-query/run-2/result.json" + }, + { + "experiment": "codex-gpt-6-luna-cli-stable", + "experimentSuite": "cli", + "experimentDisplay": { + "agent": "codex", + "modelProvider": "openai", + "modelId": "gpt-6-luna", + "reasoningEffort": "medium" + }, + "eval": "investigate-database-001-slow-query", + "stage": "investigate", + "product": [ + "database" + ], + "topic": [ + "observability", + "migrations" + ], + "suite": "cli", + "interface": "cli", + "cliVersion": "2.122.0", + "passed": true, + "checks": [ + { + "name": "local stack is running for this project", + "passed": true, + "notes": "legacy (docker), select 1 ok" + }, + { + "name": "an inspect db report showed the orders table", + "passed": true, + "judgeNotes": "The `supabase inspect db outliers --local` output names a query against public.orders, and `index-stats` rows explicitly identify public.orders." + }, + { + "name": "the order-history lookup uses an index on customer_email", + "passed": true, + "notes": "Index Scan using orders_customer_email_created_at_idx" + }, + { + "name": "orders still holds every seeded row", + "passed": true, + "notes": "300000 rows" + }, + { + "name": "a new migration creates an index on orders.customer_email", + "passed": true, + "notes": "20261023113132_add_orders_customer_history_index.sql (version 20261023113132)" + }, + { + "name": "that migration is recorded in the local migration history", + "passed": true, + "notes": "20261023113132_add_orders_customer_history_index.sql (version 20261023113132) is in supabase_migrations.schema_migrations with the index statement" + }, + { + "name": "no container-runtime detours", + "passed": true, + "judgeNotes": "No command attempted a container-runtime detour or privilege escalation." + }, + { + "name": "metrics", + "passed": true, + "notes": "{\"cliVersion\":\"2.122.0\",\"cliOverride\":[],\"channel\":\"stable\",\"backend\":\"legacy\",\"runtime\":\"docker\",\"relocatedHome\":null,\"dbPort\":54322,\"inspectReports\":[\"db help\",\"db outliers\",\"db index-stats\"],\"inspectRuns\":6,\"inspectCallsNamingOrders\":4,\"workflowCommands\":[\"status\",\"inspect db help\",\"start\",\"db query\",\"inspect db outliers\",\"inspect db index-stats\",\"migration new\",\"migration up\",\"db reset\",\"db advisors\",\"migration list\"],\"explainCommands\":2,\"directIndexDdl\":0,\"ordersIndexes\":[\"orders_customer_email_created_at_idx\",\"orders_pkey\"],\"orderRows\":300000,\"newMigrationFiles\":1,\"seededMigrationModified\":false,\"cliDetours\":0}" + }, + { + "name": "final report is truthful about the cause and the fix", + "passed": true, + "judgeNotes": "The report correctly identifies the unindexed customer-email order-history lookup as the cause. The named migration and index exist, were applied locally, and the verified plan uses the index." + } + ], + "skills": { + "available": [ + "supabase", + "supabase-postgres-best-practices" + ], + "loaded": [ + "supabase", + "supabase-postgres-best-practices" + ] + }, + "docs": { + "calls": [ + { + "source": "shell_fetch", + "query": "/bin/bash -lc 'curl -fsSL https://supabase.com/changelog.md | head -100'", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/changelog.md" + } + ], + "resultChars": 4978 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"monitoring debugging slow queries database performance query insights Supabase\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/reports", + "title": "Reports" + }, + { + "url": "https://supabase.com/docs/guides/observability", + "title": "Observability" + }, + { + "url": "https://supabase.com/docs/guides/realtime/reports", + "title": "Realtime Reports" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/timeouts", + "title": "Timeouts" + }, + { + "url": "https://supabase.com/docs/guides/platform/performance", + "title": "Performance Tuning" + } + ], + "resultChars": 97787 + }, + { + "source": "search_docs", + "query": "query { searchDocs(query: \"database inspect slow query EXPLAIN ANALYZE pg_stat_statements local Supabase\", limit: 5) { nodes { title href content } } }", + "hasContent": true, + "pages": [ + { + "url": "https://supabase.com/docs/guides/observability/inspect", + "title": "Inspect the database" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/configuration", + "title": "Database configuration" + }, + { + "url": "https://supabase.com/docs/guides/database/postgres/row-level-security-performance", + "title": "Row Level Security performance" + }, + { + "url": "https://supabase.com/docs/guides/troubleshooting/high-latency-with-supabase-client-z0pZzR", + "title": "High latency with supabase client" + }, + { + "url": "https://supabase.com/docs/guides/database/extensions/hypopg", + "title": "HypoPG: Hypothetical indexes" + } + ], + "resultChars": 29776 + } + ] + }, + "usage": [ + { + "model": "gpt-6-luna", + "inputTokens": 1436762, + "cacheReadInputTokens": 1363430, + "cacheWriteInputTokens": 73230, + "outputTokens": 9486 + } + ], + "stepCount": 34, + "toolCallCount": 31, + "agentRunDurationMs": 317726, + "sandboxUsage": { + "activeCpuDurationMs": 817672, + "duration": 496406, + "memory": 8192, + "networkTransfer": { + "ingress": 2794113354, + "egress": 14357577 + } + }, + "prompt": "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine.", + "promptSourcePath": "evals/cli/investigate-database-001-slow-query/PROMPT.md", + "run": 3, + "sourcePath": "codex-gpt-6-luna-cli-stable/investigate-database-001-slow-query/run-3/result.json" + }, { "experiment": "codex-gpt-6-luna-cli-stable", "experimentSuite": "cli", diff --git a/evals/cli/investigate-database-001-slow-query/EVAL.test.ts b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts new file mode 100644 index 00000000..5513f7dd --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/EVAL.test.ts @@ -0,0 +1,1489 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import { createHash } from 'node:crypto'; +import type { + CommandResult, + JudgeInput, + LocalStackEnvironmentMarker, + LocalStackEvalContext, + ToolCallRecord, + TranscriptPart, +} from '@supabase-evals/core'; +import { describe, expect, it, vi } from 'vitest'; +import scorer from './EVAL.js'; +import { + LOOKUP_QUERY, + SEEDED_MIGRATION_FILE, + SEEDED_MIGRATION_SHA256, + SEEDED_MIGRATION_VERSION, +} from './fixtures.js'; +import { + findInspectEvidence, + findInspectRuns, + formatInspectJudgeInput, +} from './inspect.js'; + +const WS = '/ws'; +const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; + +const SEEDED_SQL = `create table public.orders ( + id bigint generated always as identity primary key, + customer_email text not null, + status text not null default 'pending', + total_cents integer not null, + created_at timestamptz not null default now() +); +`; +const CREATE_INDEX = + 'create index if not exists orders_customer_email_idx on public.orders (customer_email);\n'; +const NEW_FILE = '20260102000000_add_orders_customer_email_idx.sql'; +const NEW_VERSION = '20260102000000'; +const SECOND_FILE = '20260103000000_more_indexes.sql'; +const SECOND_VERSION = '20260103000000'; +const SECOND_INDEX = + 'create index if not exists orders_customer_email_again_idx on public.orders (customer_email);\n'; + +const MARKER: LocalStackEnvironmentMarker = { + runtime: 'local-stack', + channel: 'beta', + cliVersion: '2.0.0', + docker: 'available', + sessionStartedMs: 1_000, +}; + +const NO_PROJECT_REF = 'Cannot find project ref. Have you run supabase link?'; + +const OUTLIERS = `|QUERY|EXECUTION TIME|PROPORTION OF EXEC TIME|NUMBER CALLS|SYNC IO TIME| +|-|-|-|-|-| +|\`select id, status, total_cents, created_at from public.orders where customer_email = $1 order by created_at desc limit $2\`|00:00:02.810|71.4%|200|00:00:00| +`; +const TABLE_STATS_JSON = `[{"name":"public.orders","table_size":"21 MB","index_size":"6600 kB","estimated_row_count":300000,"seq_scans":200}]`; +const UNRELATED_OUTLIERS = `|QUERY|EXECUTION TIME|PROPORTION OF EXEC TIME|NUMBER CALLS|SYNC IO TIME| +|-|-|-|-|-| +|\`select name from pg_timezone_names\`|00:00:00.120|88.0%|3|00:00:00| +`; + +function result(stdout: string, ok = true, stderr = ''): CommandResult { + return { ok, exitCode: ok ? 0 : 1, stdout, stderr }; +} + +const unquote = (arg: string) => arg.slice(1, -1).replaceAll("'\\''", "'"); + +function call( + command: string, + output: { result?: string; error?: string } = {} +): ToolCallRecord { + return { + tool: { kind: 'other', toolName: 'shell' }, + body: { command }, + command, + result: output.result ?? '', + ...(output.error === undefined ? {} : { error: output.error }), + ts: 0, + }; +} + +const COND = "(customer_email = 'customer-42@example.com'::text)"; + +type Fields = Record; + +const node = (type: string, fields: Fields = {}, plans: Fields[] = []) => ({ + 'Node Type': type, + 'Parallel Aware': false, + 'Plan Rows': 20, + ...fields, + ...(plans.length === 0 ? {} : { Plans: plans }), +}); + +const scan = (type: string, fields: Fields = {}, plans: Fields[] = []) => + node(type, { 'Relation Name': 'orders', Schema: 'public', ...fields }, plans); + +const explain = (root: Fields) => + `${JSON.stringify([{ Plan: root }], null, 2)}\n`; + +const SORT = { 'Sort Key': ['created_at DESC'] }; + +const PLANS = { + seqScan: explain( + node('Limit', {}, [ + node('Sort', SORT, [scan('Seq Scan', { Filter: COND })]), + ]) + ), + parallelSeqScan: explain( + node('Limit', {}, [ + node('Gather Merge', {}, [ + node('Sort', SORT, [ + scan('Seq Scan', { 'Parallel Aware': true, Filter: COND }), + ]), + ]), + ]) + ), + bitmap: explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Bitmap Heap Scan', { 'Recheck Cond': COND }, [ + node('Bitmap Index Scan', { + 'Index Name': 'orders_customer_email_idx', + 'Index Cond': COND, + }), + ]), + ]), + ]) + ), + composite: explain( + node('Limit', {}, [ + scan('Index Scan', { + 'Index Name': 'orders_customer_email_created_at_idx', + 'Index Cond': COND, + }), + ]) + ), + hash: explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Index Scan', { + 'Index Name': 'orders_customer_email_hash_idx', + 'Index Cond': COND, + }), + ]), + ]) + ), + createdAtOnly: explain( + node('Limit', {}, [ + scan('Index Scan', { + 'Scan Direction': 'Backward', + 'Index Name': 'orders_created_at_idx', + Filter: COND, + }), + ]) + ), +}; + +const hex = (text: string) => Buffer.from(text, 'utf8').toString('hex'); + +type CatalogIndex = { name: string; definition: string }; + +const PKEY: CatalogIndex = { + name: 'orders_pkey', + definition: + 'CREATE UNIQUE INDEX orders_pkey ON public.orders USING btree (id)', +}; +const EMAIL_INDEX: CatalogIndex = { + name: 'orders_customer_email_idx', + definition: + 'CREATE INDEX orders_customer_email_idx ON public.orders USING btree (customer_email)', +}; + +type World = { + stack?: 'up' | 'down' | 'refusing'; + home?: string; + files?: Record; + plan?: string | CommandResult; + catalog?: CatalogIndex[]; + rows?: string; + history?: string[]; + recordedStatements?: Record; + judge?: (args: JudgeInput) => Promise<{ passed: boolean; notes?: string }>; + transcript?: TranscriptPart[]; + marker?: LocalStackEnvironmentMarker | 'throws'; + execThrows?: boolean; +}; + +const HAPPY: Required< + Pick +> = { + stack: 'up', + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL, [NEW_FILE]: CREATE_INDEX }, + plan: PLANS.bitmap, + catalog: [PKEY, EMAIL_INDEX], + rows: '300000', + history: [SEEDED_MIGRATION_VERSION, NEW_VERSION], +}; + +function fakeCtx(toolCalls: ToolCallRecord[], world: World = {}) { + const { stack, files, plan, catalog, rows, history } = { + ...HAPPY, + ...world, + }; + const judge = vi.fn( + world.judge ?? (async () => ({ passed: true, notes: 'ok' })) + ); + + const homeMatches = (command: string) => + world.home === undefined + ? !/(?:SUPABASE_HOME|HOME)=/.test(command) + : command.includes(`SUPABASE_HOME='${world.home}'`); + + const exec = async (command: string): Promise => { + if (world.execThrows) throw new Error('sandbox gone'); + if (command.endsWith('supabase --version')) return result('2.0.0\n'); + if (/supabase stack status/.test(command)) { + if (stack === 'down' || !homeMatches(command)) { + return result('', false, 'no stack running'); + } + return command.includes('--env') + ? result(JSON.stringify({ DB_URL })) + : result(JSON.stringify({ runtime: 'native' })); + } + if (/supabase (stack list|status)/.test(command)) { + return result('', false, 'no stack running'); + } + + const psql = command.match(/^psql '([^']+)' -X?tAc ([\s\S]+)$/); + if (psql) { + const sql = unquote(psql[2]); + if (psql[1] !== DB_URL || stack !== 'up') { + return result('', false, 'psql: error: connection refused'); + } + if (sql.includes('explain (format json)')) { + return typeof plan === 'string' ? result(plan) : plan; + } + if (sql.includes('supabase_migrations.schema_migrations')) { + const filtered = /\bversion\s+in\s*\(/i.test(sql); + const asked = [...sql.matchAll(/'(\d+)'/g)].map((m) => m[1]); + const found = history + .filter((v) => !filtered || asked.includes(v)) + .map((version) => { + const name = Object.keys(files).find((file) => + file.startsWith(`${version}_`) + ); + const statements = + world.recordedStatements?.[version] ?? + (name === undefined ? '' : files[name]); + return `${version}|${hex(statements)}`; + }); + return result(found.length > 0 ? `${found.join('\n')}\n` : ''); + } + if (sql.includes('pg_indexes')) { + const asHex = /'hex'/.test(sql); + const rowsOut = catalog.map( + ({ name, definition }) => + `${name}|${asHex ? hex(definition) : definition}` + ); + return result(rowsOut.length > 0 ? `${rowsOut.join('\n')}\n` : ''); + } + if (/count\(\*\)/.test(sql)) return result(`${rows}\n`); + if (sql.includes('select 1')) return result('1\n'); + } + + if (command.includes('sha256sum')) { + const seeded = files[SEEDED_MIGRATION_FILE]; + if (seeded === undefined) { + return result('', false, 'sha256sum: No such file or directory'); + } + const digest = + seeded === SEEDED_SQL + ? SEEDED_MIGRATION_SHA256 + : createHash('sha256').update(seeded).digest('hex'); + return result( + `${digest} supabase/migrations/${SEEDED_MIGRATION_FILE}\n` + ); + } + if (command.includes('supabase/migrations')) { + const cat = command.match(/^cat 'supabase\/migrations\/(.+)'$/); + if (cat) return result(files[cat[1]] ?? ''); + const names = Object.keys(files).sort(); + return command.includes('find ') + ? result(names.map((name) => `supabase/migrations/${name}`).join('\n')) + : result(names.join('\n')); + } + return result('', false, `unexpected command: ${command}`); + }; + + const ctx = { + workspace: WS, + exec, + folderExists: async (path: string) => path === 'supabase/migrations', + toolCalls, + transcript: world.transcript ?? [], + environmentMarker: async () => { + if (world.marker === 'throws') throw new Error('marker unreadable'); + return world.marker ?? MARKER; + }, + judge, + } as unknown as LocalStackEvalContext; + return { ctx, judge }; +} + +const CHECK_NAMES = [ + 'local stack is running for this project', + 'an inspect db report showed the orders table', + 'the order-history lookup uses an index on customer_email', + 'orders still holds every seeded row', + 'a new migration creates an index on orders.customer_email', + 'that migration is recorded in the local migration history', + 'no container-runtime detours', + 'metrics', + 'final report is truthful about the cause and the fix', +]; + +const [ + STACK, + INSPECT, + PLAN, + ROWS, + NEW_MIGRATION, + RECORDED, + DETOURS, + METRICS, + REPORT, +] = CHECK_NAMES; + +const INSPECT_CALL = call('supabase inspect db outliers --local', { + result: OUTLIERS, +}); +const WRITE_MIGRATION = call( + `cat > supabase/migrations/${NEW_FILE} <<'EOF'\n${CREATE_INDEX}EOF` +); +const CREATE_INDEX_BY_HAND = call( + `psql "$DATABASE_URL" -c "create index if not exists orders_customer_email_idx on public.orders (customer_email)"` +); + +const SETUP = [ + call('supabase start'), + INSPECT_CALL, + WRITE_MIGRATION, + call('supabase migration up'), +]; + +const inWorkspace = (record: ToolCallRecord): ToolCallRecord => ({ + ...record, + cwd: WS, +}); + +const withInspect = (inspectCall: ToolCallRecord) => [ + SETUP[0], + inspectCall, + ...SETUP.slice(2), +]; + +const failedNames = async (ctx: LocalStackEvalContext) => + ((await scorer(ctx)).checks ?? []) + .filter(({ passed }) => !passed) + .map(({ name }) => name); + +const checkOf = async (ctx: LocalStackEvalContext, name: string) => + (await scorer(ctx)).checks?.find((check) => check.name === name); + +const DETOUR_RUBRIC = 'container runtime'; +const GROUND_TRUTH_HEADER = + 'Ground truth observed by the harness after the run:'; + +const isReportJudge = ({ input }: JudgeInput) => + input.startsWith(GROUND_TRUTH_HEADER); + +const judgeCalls = (judge: ReturnType['judge']) => + judge.mock.calls.map(([args]) => args); + +const INSPECT_RUBRIC = 'inspect evidence'; + +const isInspectJudge = ({ rubric }: JudgeInput) => + rubric.includes(INSPECT_RUBRIC); + +const isDetourJudge = ({ rubric }: JudgeInput) => + rubric.includes(DETOUR_RUBRIC); + +const inspectCall = (judge: ReturnType['judge']) => + judgeCalls(judge).find(isInspectJudge); + +const detourCall = (judge: ReturnType['judge']) => + judgeCalls(judge).find(({ rubric }) => rubric.includes(DETOUR_RUBRIC)); + +const reportCall = (judge: ReturnType['judge']) => + judgeCalls(judge).find(isReportJudge); + +const mentionsApplied = (line: string) => /applied/i.test(line); +const groundTruthLines = (input: string) => + input.split('\nTranscript:')[0].split('\n'); + +describe('slow-query scorer', () => { + it('passes a run that found the cause with inspect and shipped an applied index migration, in the declared check order', async () => { + const { ctx } = fakeCtx(SETUP); + const scored = await scorer(ctx); + expect(scored.checks?.map(({ name }) => name)).toEqual(CHECK_NAMES); + expect(scored.checks?.filter(({ passed }) => !passed)).toEqual([]); + expect(scored.passed).toBe(true); + }); + + it('fails when any check fails', async () => { + const { ctx } = fakeCtx(SETUP, { rows: '0' }); + expect((await scorer(ctx)).passed).toBe(false); + }); + + it('returns a single failing check when the scorer itself throws', async () => { + const { ctx } = fakeCtx(SETUP, { marker: 'throws' }); + expect(await scorer(ctx)).toEqual({ + passed: false, + checks: [ + { + name: 'scorer evaluated the slow-query fix', + passed: false, + notes: 'marker unreadable', + }, + ], + }); + }); + + it('still reports all nine checks when every command in the sandbox throws', async () => { + const { ctx } = fakeCtx(SETUP, { execThrows: true }); + const scored = await scorer(ctx); + expect(scored.checks?.map(({ name }) => name)).toEqual(CHECK_NAMES); + expect(await failedNames(ctx)).toEqual([ + STACK, + PLAN, + ROWS, + NEW_MIGRATION, + RECORDED, + ]); + }); + + it('fails the checks that need a database with the reason when the stack is down, without throwing', async () => { + const { ctx } = fakeCtx(SETUP, { stack: 'down' }); + expect(await failedNames(ctx)).toEqual([STACK, PLAN, ROWS, RECORDED]); + for (const name of [STACK, PLAN, ROWS, RECORDED]) { + expect((await checkOf(ctx, name))?.notes).toContain('no stack running'); + } + }); + + it('says in the plan check which index the lookup uses', async () => { + const { ctx } = fakeCtx(SETUP); + const check = await checkOf(ctx, PLAN); + expect(check?.passed).toBe(true); + expect(check?.notes).toContain('orders_customer_email_idx'); + }); + + it('says in the plan check that the lookup still scans orders sequentially', async () => { + const { ctx } = fakeCtx(SETUP, { plan: PLANS.seqScan }); + const check = await checkOf(ctx, PLAN); + expect(check?.passed).toBe(false); + expect(check?.notes).toMatch(/seq scan/i); + }); + + it('says in the rows check how many rows are left', async () => { + const { ctx } = fakeCtx(SETUP, { rows: '1000' }); + const check = await checkOf(ctx, ROWS); + expect(check?.passed).toBe(false); + expect(check?.notes).toContain('1000'); + }); + + it('says in the migration check which file and version create the index', async () => { + const { ctx } = fakeCtx(SETUP); + const check = await checkOf(ctx, NEW_MIGRATION); + expect(check?.passed).toBe(true); + expect(check?.notes).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + }); + + it('passes the inspect check on the judge verdict, with the notes it gave', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => + isInspectJudge(args) + ? { passed: true, notes: 'tool call 2 printed the outliers rows' } + : { passed: true }, + }); + expect(await checkOf(ctx, INSPECT)).toEqual({ + name: INSPECT, + passed: true, + judgeNotes: 'tool call 2 printed the outliers rows', + }); + }); +}); + +describe('known-bad runs', () => { + it.each([ + [ + 'never started the stack', + [], + { stack: 'down', files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL } }, + [STACK, INSPECT, PLAN, ROWS, NEW_MIGRATION, RECORDED], + ], + [ + 'wrote the migration but never started the stack', + [WRITE_MIGRATION], + { stack: 'down' }, + [STACK, INSPECT, PLAN, ROWS, RECORDED], + ], + [ + 'started the stack but its database refuses connections', + SETUP, + { stack: 'refusing' }, + [STACK, PLAN, ROWS, RECORDED], + ], + [ + 'wrote the migration but never applied it', + SETUP.slice(0, 3), + { + plan: PLANS.seqScan, + catalog: [PKEY], + history: [SEEDED_MIGRATION_VERSION], + }, + [PLAN, RECORDED], + ], + [ + 'inspected the database and then did nothing about it', + SETUP.slice(0, 2), + { + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL }, + plan: PLANS.seqScan, + catalog: [PKEY], + history: [SEEDED_MIGRATION_VERSION], + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'created the index with psql and never wrote a file', + [SETUP[0], INSPECT_CALL, CREATE_INDEX_BY_HAND], + { + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL }, + history: [SEEDED_MIGRATION_VERSION], + }, + [NEW_MIGRATION, RECORDED], + ], + [ + 'wrote the migration file but created the index with psql only', + [SETUP[0], INSPECT_CALL, WRITE_MIGRATION, CREATE_INDEX_BY_HAND], + { history: [SEEDED_MIGRATION_VERSION] }, + [RECORDED], + ], + [ + 'edited the seeded migration instead of adding one', + [SETUP[0], INSPECT_CALL, call('supabase db reset')], + { + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL + CREATE_INDEX }, + history: [SEEDED_MIGRATION_VERSION], + }, + [NEW_MIGRATION, RECORDED], + ], + [ + 'indexed created_at only', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index orders_created_at_idx on public.orders (created_at);\n', + }, + plan: PLANS.createdAtOnly, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'indexed lower(customer_email)', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index orders_email_lower_idx on public.orders (lower(customer_email));\n', + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'indexed (created_at, customer_email) with customer_email second', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index orders_created_email_idx on public.orders (created_at, customer_email);\n', + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'left the lookup on a parallel sequential scan', + SETUP, + { plan: PLANS.parallelSeqScan }, + [PLAN], + ], + [ + 'renamed the column so the probe query errors', + SETUP, + { + plan: result( + '', + false, + 'ERROR: column "customer_email" does not exist' + ), + }, + [PLAN], + ], + ['truncated the table', SETUP, { rows: '0' }, [ROWS]], + ['deleted most of the rows', SETUP, { rows: '1000' }, [ROWS]], + [ + 'indexed the look-alike table orders_archive', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index on public.orders_archive (customer_email);\n', + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'indexed the look-alike table private.orders', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: 'create index on private.orders (customer_email);\n', + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'wrote the migration with a date older than the seeded one', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + '20250101000000_idx.sql': CREATE_INDEX, + }, + history: [SEEDED_MIGRATION_VERSION, '20250101000000'], + }, + [NEW_MIGRATION, RECORDED], + ], + [ + 'commented the statement out', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: `-- ${CREATE_INDEX}`, + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'put the statement inside a function body', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create function public.make_idx() returns void language sql as $$ create index on public.orders (customer_email) $$;\n', + }, + plan: PLANS.seqScan, + }, + [PLAN, NEW_MIGRATION, RECORDED], + ], + [ + 'wrote a unique index the data cannot satisfy and could not apply it', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create unique index orders_customer_email_key on public.orders (customer_email);\n', + }, + plan: PLANS.seqScan, + history: [SEEDED_MIGRATION_VERSION], + }, + [PLAN, RECORDED], + ], + [ + 'applied an empty migration file, then filled it and created the index by hand', + [ + SETUP[0], + INSPECT_CALL, + call('supabase migration new add_orders_customer_email_idx'), + call('supabase migration up'), + CREATE_INDEX_BY_HAND, + ], + { recordedStatements: { [NEW_VERSION]: '' } }, + [RECORDED], + ], + [ + 'wrote two migrations that create the index and applied only the first', + SETUP, + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: CREATE_INDEX, + [SECOND_FILE]: SECOND_INDEX, + }, + history: [SEEDED_MIGRATION_VERSION, NEW_VERSION], + }, + [RECORDED], + ], + ] satisfies Array<[string, ToolCallRecord[], World, string[]]>)( + 'fails only the checks the run got wrong when it %s', + async (_label, toolCalls, world, expected) => { + const { ctx } = fakeCtx(toolCalls, world); + expect(await failedNames(ctx)).toEqual(expected); + } + ); + + it('fails only the history check when the index is created by hand and saved with db diff -f, which never records the version', async () => { + const { ctx } = fakeCtx( + [ + SETUP[0], + INSPECT_CALL, + CREATE_INDEX_BY_HAND, + call('supabase db diff -f add_orders_customer_email_idx --local', { + result: `Created new migration at supabase/migrations/${NEW_FILE}`, + }), + ], + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'CREATE INDEX orders_customer_email_idx ON public.orders USING btree (customer_email);\n', + }, + history: [SEEDED_MIGRATION_VERSION], + } + ); + expect(await failedNames(ctx)).toEqual([RECORDED]); + const check = await checkOf(ctx, RECORDED); + expect(check?.notes).toContain(NEW_VERSION); + expect(check?.notes).toContain( + 'not in supabase_migrations.schema_migrations' + ); + }); +}); + +describe('inspect evidence', () => { + it.each([ + [ + 'ran the default report with no project linked and was refused', + call('supabase inspect db outliers', { error: NO_PROJECT_REF }), + ], + [ + 'read pg_stat_statements through psql', + call( + `psql "$DATABASE_URL" -c "select query, calls from pg_stat_statements order by total_exec_time desc"`, + { result: 'select ... from public.orders where customer_email = $1' } + ), + ], + [ + 'read pg_stat_statements through db query', + call(`supabase db query --local "select query from pg_stat_statements"`, { + result: 'select ... from public.orders where customer_email = $1', + }), + ], + [ + 'ran a report that never mentioned orders', + call('supabase inspect db outliers --local', { + result: UNRELATED_OUTLIERS, + }), + ], + ] satisfies Array<[string, ToolCallRecord]>)( + 'fails only the inspect check without asking the judge when the agent %s', + async (_label, inspectCall) => { + const { ctx, judge } = fakeCtx(withInspect(inspectCall)); + expect(await failedNames(ctx)).toEqual([INSPECT]); + expect(judgeCalls(judge).some(isInspectJudge)).toBe(false); + } + ); + + it('says in the inspect check that no report ran when the agent ran none', async () => { + const { ctx } = fakeCtx([SETUP[0], ...SETUP.slice(2)]); + expect(await checkOf(ctx, INSPECT)).toEqual({ + name: INSPECT, + passed: false, + notes: 'no supabase inspect report ran', + }); + }); + + it('says in the inspect check which runs ran when none printed orders', async () => { + const { ctx } = fakeCtx( + withInspect( + call('supabase inspect db outliers', { error: NO_PROJECT_REF }) + ) + ); + expect(await checkOf(ctx, INSPECT)).toEqual({ + name: INSPECT, + passed: false, + notes: + 'tool call 2: supabase inspect db outliers (errored); no inspect output named the orders table', + }); + }); + + it('asks the judge, with every evidence call in full, when something names orders', async () => { + const calls = withInspect( + call('supabase inspect db outliers --local > /tmp/o.txt') + ); + const withRead = [...calls, call('cat /tmp/o.txt', { result: OUTLIERS })]; + const { ctx, judge } = fakeCtx(withRead); + await scorer(ctx); + const runs = findInspectRuns(withRead, []); + const evidence = findInspectEvidence(withRead, runs); + expect(evidence).toHaveLength(2); + const sent = inspectCall(judge); + expect(sent?.input).toBe(formatInspectJudgeInput(evidence)); + expect(sent?.input).toContain( + 'Tool call 5 (later call naming an output path)' + ); + expect(sent?.input).toContain('cat /tmp/o.txt'); + expect(sent?.input).toContain(OUTLIERS); + expect(sent?.rubric).toContain('public.orders'); + expect(sent?.rubric).toContain('Cannot find project ref'); + expect(sent?.rubric).toMatch(/truncated/); + }); + + it('asks the judge about the report CSVs a later call read after a cd into the dated folder', async () => { + const calls = withInspect( + call('supabase inspect report --local', { + result: + 'Reports saved to /ws/2026-10-22\n|RULE|STATUS|MATCHES|\n|-|-|-|\n|No unused indexes|✔||\n', + }) + ); + const read = call('cd 2026-10-22 && head -n 5 outliers.csv calls.csv', { + result: + 'query,calls\nselect id from public.orders where customer_email = $1,200\n', + }); + const { ctx, judge } = fakeCtx([...calls, read]); + expect(await failedNames(ctx)).toEqual([]); + expect(inspectCall(judge)?.input).toContain( + 'Tool call 5 (later call naming an output path)\nCommand:\ncd 2026-10-22 && head -n 5 outliers.csv calls.csv' + ); + }); + + it('carries the rubric clauses that keep orders text from other commands from counting', async () => { + const { ctx, judge } = fakeCtx(SETUP); + await scorer(ctx); + const rubric = inspectCall(judge)?.rubric ?? ''; + for (const clause of [ + /psql/, + /supabase db query/, + /\bcat\b.*\bgrep\b.*\brg\b.*project files/, + /heredoc/, + /echo or printf/, + /awk BEGIN/, + /help output/, + /Cannot find project ref/, + /overwr/, + ]) { + expect(rubric).toMatch(clause); + } + }); + + it('carries the rubric clauses that tell the judge how later calls were chosen and what inspect prints', async () => { + const { ctx, judge } = fakeCtx(SETUP); + await scorer(ctx); + const rubric = inspectCall(judge)?.rubric ?? ''; + for (const clause of [ + /report file name/, + /matched on text, so check what each command does/, + /for example/i, + /vacuum-stats/, + /bloat/, + /locks/, + /MATCHES cell/, + /\$1/, + /execution time and call counts/, + /public\.orders/, + /\/\/\.csv/, + /RULE \| STATUS \| MATCHES/, + ]) { + expect(rubric).toMatch(clause); + } + }); + + it.each([ + [ + 'was refused by inspect and fell back to psql in the same call', + call( + `supabase inspect db outliers 2>&1 || psql "${DB_URL}" -c "select query, calls from extensions.pg_stat_statements order by total_exec_time desc limit 5"`, + { + result: `${NO_PROJECT_REF}\n select ... from public.orders where customer_email = $1 | 200`, + } + ), + ], + [ + 'only asked inspect for help', + call('supabase inspect db --help', { + result: 'Usage: supabase inspect db [command]\n outliers orders ...', + }), + ], + [ + 'only echoed the inspect command', + call('echo "supabase inspect db outliers"', { + result: 'supabase inspect db outliers (public.orders)', + }), + ], + ] satisfies Array<[string, ToolCallRecord]>)( + 'fails only the inspect check, with the judge notes, when the judge finds that the agent %s', + async (_label, inspectingCall) => { + const { ctx } = fakeCtx(withInspect(inspectingCall), { + judge: async (args) => + isInspectJudge(args) + ? { passed: false, notes: 'orders came from psql' } + : { passed: true }, + }); + expect(await failedNames(ctx)).toEqual([INSPECT]); + expect(await checkOf(ctx, INSPECT)).toEqual({ + name: INSPECT, + passed: false, + judgeNotes: 'orders came from psql', + }); + } + ); + + it('fails only the inspect check with the error when the judge throws', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => { + if (isInspectJudge(args)) throw new Error('judge unavailable'); + return { passed: true }; + }, + }); + expect(await checkOf(ctx, INSPECT)).toEqual({ + name: INSPECT, + passed: false, + notes: 'judge unavailable', + }); + expect(await failedNames(ctx)).toEqual([INSPECT]); + }); + + it.each([ + [ + 'table-stats as JSON', + call('supabase inspect db table-stats --local -o json', { + result: TABLE_STATS_JSON, + }), + ], + [ + 'calls through a package runner and a pipe', + call('npx supabase@beta inspect db calls --local | head -40', { + result: OUTLIERS, + }), + ], + [ + 'outliers over a connection string in JSON', + call( + `supabase inspect db outliers --db-url ${DB_URL} --output-format json`, + { result: OUTLIERS } + ), + ], + [ + 'the deprecated seq-scans alias', + call('supabase inspect db seq-scans --local', { + result: TABLE_STATS_JSON, + }), + ], + [ + 'a report written to disk and read back in the same call', + call( + 'supabase inspect report --local --output-dir /tmp/r && cat /tmp/r/*/outliers.csv', + { result: 'query,calls\nselect ... from public.orders ...,200\n' } + ), + ], + ] satisfies Array<[string, ToolCallRecord]>)( + 'passes when the agent used %s and the judge accepts it', + async (_label, inspectingCall) => { + const { ctx } = fakeCtx(withInspect(inspectingCall)); + expect(await failedNames(ctx)).toEqual([]); + } + ); + + it('passes when the first report was refused and a later --local report showed orders', async () => { + const { ctx } = fakeCtx([ + SETUP[0], + call('supabase inspect db outliers', { error: NO_PROJECT_REF }), + INSPECT_CALL, + ...SETUP.slice(2), + ]); + expect(await failedNames(ctx)).toEqual([]); + }); +}); + +describe('valid alternatives', () => { + it.each([ + [ + 'writes db diff output with quoted identifiers', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'CREATE INDEX "orders_customer_email_idx" ON "public"."orders" USING btree ("customer_email");\n', + }, + }, + ], + [ + 'creates the index concurrently', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index concurrently if not exists orders_customer_email_idx on public.orders (customer_email);\n', + }, + }, + ], + [ + 'indexes (customer_email, created_at desc)', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index on orders(customer_email, created_at desc);\n', + }, + plan: PLANS.composite, + }, + ], + [ + 'uses a hash index', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index orders_customer_email_hash_idx on public.orders using hash (customer_email);\n', + }, + plan: PLANS.hash, + }, + ], + [ + 'adds a covering index', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: + 'create index orders_customer_email_idx on public.orders (customer_email) include (status, total_cents);\n', + }, + }, + ], + [ + 'writes two migrations that create the index and applies both', + { + files: { + [SEEDED_MIGRATION_FILE]: SEEDED_SQL, + [NEW_FILE]: CREATE_INDEX, + [SECOND_FILE]: SECOND_INDEX, + }, + history: [SEEDED_MIGRATION_VERSION, NEW_VERSION, SECOND_VERSION], + }, + ], + ['leaves 300001 rows after a test insert', { rows: '300001' }], + ] satisfies Array<[string, World]>)( + 'passes when the agent %s', + async (_label, world) => { + const { ctx } = fakeCtx(SETUP, world); + expect(await failedNames(ctx)).toEqual([]); + } + ); + + it('passes when the agent explains the lookup through psql while investigating', async () => { + const { ctx } = fakeCtx([ + SETUP[0], + INSPECT_CALL, + call(`psql "$DATABASE_URL" -c "explain analyze ${LOOKUP_QUERY}"`), + ...SETUP.slice(2), + ]); + expect(await failedNames(ctx)).toEqual([]); + }); + + it.each([ + ['db reset', 'supabase db reset'], + ['db push --local', 'supabase db push --local'], + ])( + 'passes when the agent applies the file with %s', + async (_label, apply) => { + const { ctx } = fakeCtx([...SETUP.slice(0, 3), call(apply)]); + expect(await failedNames(ctx)).toEqual([]); + } + ); + + it('passes a manual apply recorded with migration repair', async () => { + const { ctx } = fakeCtx([ + SETUP[0], + INSPECT_CALL, + WRITE_MIGRATION, + CREATE_INDEX_BY_HAND, + call(`supabase migration repair --status applied ${NEW_VERSION} --local`), + ]); + expect(await failedNames(ctx)).toEqual([]); + }); + + it('passes when the recorded statements match the file apart from formatting', async () => { + const { ctx } = fakeCtx(SETUP, { + recordedStatements: { + [NEW_VERSION]: + '-- index the lookup\nCREATE INDEX IF NOT EXISTS orders_customer_email_idx\n ON public.orders (customer_email)', + }, + }); + expect(await failedNames(ctx)).toEqual([]); + }); + + it('finds a stack the agent started under a SUPABASE_HOME it set itself', async () => { + const home = '/home/agent/.supabase-local'; + const { ctx } = fakeCtx( + [ + inWorkspace(call(`SUPABASE_HOME=${home} supabase start`)), + ...SETUP.slice(1), + ], + { home } + ); + expect(await failedNames(ctx)).toEqual([]); + }); + + it('finds a stack under the SUPABASE_HOME the sandbox was started with', async () => { + const home = '/sandbox/.supabase-home'; + const { ctx } = fakeCtx([inWorkspace(SETUP[0]), ...SETUP.slice(1)], { + home, + marker: { ...MARKER, env: { SUPABASE_HOME: home } }, + }); + expect(await failedNames(ctx)).toEqual([]); + }); + + it('does not find a relocated stack when nothing says where it lives', async () => { + const { ctx } = fakeCtx(SETUP, { home: '/home/agent/.supabase-local' }); + expect(await failedNames(ctx)).toEqual([STACK, PLAN, ROWS, RECORDED]); + }); +}); + +describe('metrics check', () => { + const metrics = async (ctx: LocalStackEvalContext) => { + const check = await checkOf(ctx, METRICS); + return JSON.parse(check?.notes ?? '{}'); + }; + + it('reports the facts of a finished run', async () => { + const { ctx } = fakeCtx(SETUP); + expect(await metrics(ctx)).toEqual({ + cliVersion: '2.0.0', + cliOverride: [], + channel: 'beta', + backend: 'managed', + runtime: 'native', + relocatedHome: null, + dbPort: 54322, + inspectReports: ['db outliers'], + inspectRuns: 1, + inspectCallsNamingOrders: 1, + workflowCommands: ['start', 'inspect db outliers', 'migration up'], + explainCommands: 0, + directIndexDdl: 0, + ordersIndexes: ['orders_pkey', 'orders_customer_email_idx'], + orderRows: 300_000, + newMigrationFiles: 1, + seededMigrationModified: false, + cliDetours: 0, + }); + }); + + it('counts the plans the agent explained and the index it created straight against the database', async () => { + const { ctx } = fakeCtx([ + SETUP[0], + INSPECT_CALL, + call(`psql "$DATABASE_URL" -c "explain ${LOOKUP_QUERY}"`), + CREATE_INDEX_BY_HAND, + ]); + expect(await metrics(ctx)).toMatchObject({ + explainCommands: 1, + directIndexDdl: 1, + }); + }); + + it('reports the edited seeded migration and the facts it could not read when no stack is up', async () => { + const { ctx } = fakeCtx(SETUP, { + stack: 'down', + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL + CREATE_INDEX }, + }); + expect(await metrics(ctx)).toMatchObject({ + seededMigrationModified: true, + newMigrationFiles: 0, + backend: 'none', + dbPort: null, + ordersIndexes: null, + orderRows: null, + }); + }); + + it('reports the inspect reports that ran and how many evidence calls named orders', async () => { + const { ctx } = fakeCtx([ + SETUP[0], + call('supabase inspect db outliers', { error: NO_PROJECT_REF }), + call('supabase inspect db calls --local', { result: UNRELATED_OUTLIERS }), + call('supabase inspect db table-stats --local', { + result: TABLE_STATS_JSON, + }), + ...SETUP.slice(2), + ]); + expect(await metrics(ctx)).toMatchObject({ + inspectReports: ['db outliers', 'db calls', 'db table-stats'], + inspectRuns: 3, + inspectCallsNamingOrders: 1, + }); + }); + + it('always passes', async () => { + const { ctx } = fakeCtx([], { stack: 'down', execThrows: true }); + expect((await checkOf(ctx, METRICS))?.passed).toBe(true); + }); +}); + +describe('detour judge', () => { + it('receives every command numbered and in full, with the scenario', async () => { + const longCommand = `cat > supabase/migrations/${NEW_FILE} <<'EOF'\n${CREATE_INDEX.repeat(60)}EOF`; + expect(longCommand.length).toBeGreaterThan(2000); + const calls = [SETUP[0], INSPECT_CALL, call(longCommand), SETUP[3]]; + const { ctx, judge } = fakeCtx(calls); + await scorer(ctx); + const sent = detourCall(judge); + expect(sent?.input).toBe( + calls.map((record, i) => `${i + 1}. ${record.command}`).join('\n') + ); + expect(sent?.rubric).toContain('find what is slow'); + expect(sent?.rubric).toContain( + 'Running psql, EXPLAIN or other client tools against the local database' + ); + expect(sent?.rubric).toContain('supabase inspect'); + }); + + it('is not called when the agent ran no commands', async () => { + const { ctx, judge } = fakeCtx([]); + expect(await checkOf(ctx, DETOURS)).toMatchObject({ + name: DETOURS, + passed: true, + }); + expect(detourCall(judge)).toBeUndefined(); + }); + + it('records the verdict notes and fails when the judge fails', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => + isDetourJudge(args) + ? { passed: false, notes: 'command 2 installed docker' } + : { passed: true }, + }); + expect(await checkOf(ctx, DETOURS)).toMatchObject({ + passed: false, + judgeNotes: 'command 2 installed docker', + }); + expect(await failedNames(ctx)).toEqual([DETOURS]); + }); + + it('fails only this check when the judge throws', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => { + if (isDetourJudge(args)) throw new Error('judge unavailable'); + return { passed: true }; + }, + }); + expect(await checkOf(ctx, DETOURS)).toEqual({ + name: DETOURS, + passed: false, + notes: 'judge unavailable', + }); + expect(await failedNames(ctx)).toEqual([DETOURS]); + }); +}); + +describe('report judge', () => { + it('receives the harness ground truth for a finished run', async () => { + const { ctx, judge } = fakeCtx(SETUP); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('- project directory: /ws'); + expect(input).toContain('- local stack: resolved: managed/native'); + expect(input).toContain('- what the seed made slow:'); + expect(input).toContain('- rows in public.orders: 300000 (seeded 300000)'); + expect(input).toMatch( + /- order-history lookup plan now: .*orders_customer_email_idx/ + ); + expect(input).toContain( + `- new migrations creating an index on orders.customer_email: ${NEW_FILE} (version ${NEW_VERSION})` + ); + expect(input).toContain('- recorded in the local migration history: yes'); + expect(input).toContain( + '- supabase inspect reports the agent ran: tool call 2: db outliers' + ); + expect(input).toContain('- seeded migration modified: no'); + expect(input).toContain('Transcript:'); + }); + + const UNPARSED = + 'a call mentioning supabase inspect that the harness could not parse into a report'; + + it.each([ + [ + 'a timeout around bash -lc', + `timeout 120 bash -lc 'supabase inspect db outliers --local'`, + `tool call 2: ${UNPARSED}`, + ], + [ + 'a brace group', + '{ supabase inspect db outliers --local; supabase inspect db table-stats --local; } 2>&1 | head -80', + 'tool call 2: db table-stats', + ], + [ + 'a loop over reports', + `for r in outliers calls; do echo "== $r"; supabase inspect db "$r" --local; done`, + `tool call 2: ${UNPARSED}`, + ], + ])( + 'lists the inspect call the harness could not fully parse: %s', + async (_label, command, expected) => { + const { ctx, judge } = fakeCtx( + withInspect(call(command, { result: OUTLIERS })) + ); + await scorer(ctx); + expect(reportCall(judge)?.input).toContain( + `- supabase inspect reports the agent ran: ${expected}` + ); + expect(await checkOf(ctx, INSPECT)).toMatchObject({ passed: true }); + } + ); + + it('tells the judge a report the transcript shows running is not contradicted by its absence from the inspect line', async () => { + const { ctx, judge } = fakeCtx(SETUP); + await scorer(ctx); + expect(reportCall(judge)?.rubric).toMatch( + /inspect line lists only what the harness could parse, and a report the transcript shows running is not contradicted by its absence/ + ); + }); + + it('receives the ground truth of a run that did nothing', async () => { + const { ctx, judge } = fakeCtx([], { + stack: 'down', + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL }, + }); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('- local stack: none ('); + expect(input).toContain('- order-history lookup plan now: unavailable ('); + expect(input).toContain('- rows in public.orders: unavailable ('); + expect(input).toContain( + '- new migrations creating an index on orders.customer_email: none (' + ); + expect(input).toContain( + '- recorded in the local migration history: unknown (' + ); + expect(input).toContain('- supabase inspect reports the agent ran: none'); + }); + + it('shows the sequential scan when the index never reached the database', async () => { + const { ctx, judge } = fakeCtx(SETUP.slice(0, 3), { + plan: PLANS.seqScan, + catalog: [PKEY], + history: [SEEDED_MIGRATION_VERSION], + }); + await scorer(ctx); + const truth = groundTruthLines(reportCall(judge)?.input ?? ''); + const planLine = truth.find((entry) => + entry.startsWith('- order-history lookup plan now:') + ); + const indexesLine = truth.find((entry) => + entry.startsWith('- indexes on public.orders:') + ); + expect(planLine).toMatch(/seq scan on orders/i); + expect(planLine).not.toContain('orders_customer_email_idx'); + expect(indexesLine).toBe( + `- indexes on public.orders: ${PKEY.name}: ${PKEY.definition}` + ); + }); + + it('receives the tool outputs in the transcript', async () => { + const { ctx, judge } = fakeCtx(SETUP, { + transcript: [ + { + type: 'tool_call', + name: 'shell', + input: { command: 'supabase inspect db outliers --local' }, + output: 'toomanyrequests: Rate exceeded', + }, + ], + }); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('supabase inspect db outliers --local'); + expect(input).toContain('toomanyrequests: Rate exceeded'); + }); + + it('says the history is unknown, never applied, when the stack is down and a migration was written', async () => { + const { ctx, judge } = fakeCtx([WRITE_MIGRATION], { stack: 'down' }); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain( + '- recorded in the local migration history: unknown (' + ); + expect(groundTruthLines(input).filter(mentionsApplied)).toEqual([]); + }); + + it('says the migration is not in the history, never applied, when it was written but not run', async () => { + const { ctx, judge } = fakeCtx(SETUP.slice(0, 3), { + plan: PLANS.seqScan, + history: [SEEDED_MIGRATION_VERSION], + }); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('- recorded in the local migration history: no ('); + expect(input).toContain(NEW_VERSION); + expect(groundTruthLines(input).filter(mentionsApplied)).toEqual([]); + }); + + it('says the history has the version with statements that lack the index', async () => { + const { ctx, judge } = fakeCtx(SETUP, { + recordedStatements: { [NEW_VERSION]: '' }, + }); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('- recorded in the local migration history: no ('); + expect(input).toMatch(/statements/); + }); + + it('states the installed CLI and the override runners the agent used', async () => { + const { ctx, judge } = fakeCtx([ + call('npx --yes supabase@2.1.0 start'), + ...SETUP.slice(1), + ]); + await scorer(ctx); + const input = reportCall(judge)?.input ?? ''; + expect(input).toContain('- installed CLI: 2.0.0;'); + expect(input).toContain( + '- agent ran npx --yes supabase@2.1.0; scorer uses the installed CLI' + ); + }); + + it('carries the rubric clauses that keep the verdict on the cause and the fix', async () => { + const { ctx, judge } = fakeCtx(SETUP); + await scorer(ctx); + const rubric = reportCall(judge)?.rubric ?? ''; + for (const clause of [ + /customer_email/, + /bloat/i, + /vacuum/i, + /lock/i, + /connection limits/i, + /primary[- ]key/i, + /migration/i, + /different Supabase CLI build than the installed one/, + ]) { + expect(rubric).toMatch(clause); + } + }); + + it('records the verdict notes and fails when the judge fails', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => + isReportJudge(args) + ? { passed: false, notes: 'report blames vacuum' } + : { passed: true }, + }); + expect(await checkOf(ctx, REPORT)).toMatchObject({ + passed: false, + judgeNotes: 'report blames vacuum', + }); + expect(await failedNames(ctx)).toEqual([REPORT]); + }); + + it('fails only this check when the judge throws', async () => { + const { ctx } = fakeCtx(SETUP, { + judge: async (args) => { + if (isReportJudge(args)) throw new Error('judge unavailable'); + return { passed: true }; + }, + }); + expect(await checkOf(ctx, REPORT)).toEqual({ + name: REPORT, + passed: false, + notes: 'judge unavailable', + }); + expect(await failedNames(ctx)).toEqual([REPORT]); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/EVAL.ts b/evals/cli/investigate-database-001-slow-query/EVAL.ts new file mode 100644 index 00000000..b2c306f9 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/EVAL.ts @@ -0,0 +1,210 @@ +import { + type CheckResult, + type LocalStackEvalContext, + type LocalStackScorer, +} from '@supabase-evals/core'; +import { stripIndent } from 'common-tags'; +import { findSupabaseInvocations } from '../lib/cli-invocations.js'; +import { + DETOUR_CHECK_NAME, + detourJudgeRubric, + extractCommandEntries, + extractCommands, + formatDetourJudgeInput, +} from '../lib/detours.js'; +import { formatGroundTruthJudgeInput } from '../lib/report.js'; +import { errorMessage } from '../lib/shell.js'; +import { resolveStackWithAgentHomes } from '../lib/stack.js'; +import { + findInspectEvidence, + findInspectRuns, + formatInspectJudgeInput, + gateInspectEvidence, + INSPECT_CHECK_NAME, + type EvidenceCall, + type InspectRun, +} from './inspect.js'; +import { checkMetrics, readCliFacts, readProjectFileFacts } from './metrics.js'; +import { + checkMigrationRecorded, + checkNewMigrationCreatesIndex, + findEmailIndexMigrations, + readRecordedIndexMigrations, +} from './migrations.js'; +import { + checkLookupUsesIndex, + checkOrdersIntact, + checkStackRunning, + countOrders, + readLookupPlan, + readOrdersIndexes, +} from './orders.js'; +import { describeGroundTruth } from './report.js'; + +/** Scores "find what is slow with the Supabase CLI and ship the fix" without branching on which CLI or Docker arm was staged. */ +const scorer: LocalStackScorer = async (ctx) => { + try { + const marker = await ctx.environmentMarker(); + const commands = extractCommands(ctx.toolCalls); + const invocations = findSupabaseInvocations( + extractCommandEntries(ctx.toolCalls), + { sandboxEnv: marker?.env } + ); + const stack = await resolveStackWithAgentHomes( + ctx, + { kind: 'root' }, + invocations + ); + const inspectRuns = findInspectRuns(ctx.toolCalls, invocations); + const inspectEvidence = findInspectEvidence(ctx.toolCalls, inspectRuns); + const plan = await readLookupPlan(ctx, stack); + const rows = await countOrders(ctx, stack); + const indexes = await readOrdersIndexes(ctx, stack); + const migrations = await findEmailIndexMigrations(ctx); + const recorded = await readRecordedIndexMigrations(ctx, stack, migrations); + const cli = await readCliFacts(ctx, marker, invocations); + const files = await readProjectFileFacts(ctx); + + const checks: CheckResult[] = [ + await checkStackRunning(ctx, stack), + await checkInspectShowedOrders(ctx, inspectRuns, inspectEvidence), + checkLookupUsesIndex(plan), + checkOrdersIntact(rows), + checkNewMigrationCreatesIndex(migrations), + checkMigrationRecorded(recorded), + await checkNoContainerRuntimeDetours(ctx, commands), + await checkMetrics(marker, { + commands, + invocations, + stack, + cli, + files, + inspectRuns, + inspectEvidence, + indexes, + rows, + }), + await checkReportIsTruthful( + ctx, + describeGroundTruth({ + workspace: ctx.workspace, + stack, + plan, + indexes, + rows, + migrations, + recorded, + inspectRuns, + inspectEvidence, + files, + cli, + }) + ), + ]; + + return { passed: checks.every((check) => check.passed), checks }; + } catch (error) { + return { + passed: false, + checks: [ + { + name: 'scorer evaluated the slow-query fix', + passed: false, + notes: errorMessage(error), + }, + ], + }; + } +}; + +export default scorer; + +/** Judges executed commands only, so naming the blocker in a report can't count as a detour. */ +async function checkNoContainerRuntimeDetours( + ctx: LocalStackEvalContext, + commands: readonly string[] +): Promise { + const name = DETOUR_CHECK_NAME; + if (commands.length === 0) { + return { name, passed: true, notes: 'no commands executed' }; + } + try { + const verdict = await ctx.judge({ + input: formatDetourJudgeInput(commands), + rubric: detourJudgeRubric( + "The agent was asked to find what is slow in a local Supabase project's database with the Supabase CLI and fix it.", + { + additionalGuidance: + 'Running psql, EXPLAIN or other client tools against the local database, running `supabase inspect` reports, and starting, stopping or restarting the Supabase stack are not container-runtime detours.', + } + ), + }); + return { name, passed: verdict.passed, judgeNotes: verdict.notes }; + } catch (error) { + return { name, passed: false, notes: errorMessage(error) }; + } +} + +/** Gates on the evidence, then has the judge attribute the orders text to an inspect report rather than another command in the call. */ +async function checkInspectShowedOrders( + ctx: LocalStackEvalContext, + runs: readonly InspectRun[], + evidence: readonly EvidenceCall[] +): Promise { + const gated = gateInspectEvidence(runs, evidence); + if (gated !== undefined) return gated; + const name = INSPECT_CHECK_NAME; + try { + const verdict = await ctx.judge({ + input: formatInspectJudgeInput(evidence), + rubric: stripIndent` + The agent was asked to find what is slow in a local Supabase project's database with the Supabase CLI. Below are the tool calls the harness selected as inspect evidence, each with its full command and recorded output. Candidate calls ran or mention \`supabase inspect\`. Later calls mention a path a candidate wrote to or a report file name; they are matched on text, so check what each command does. + + What inspect prints: \`outliers\` and \`calls\` show the normalised query text with \`$1\` placeholders, plus execution time and call counts; \`table-stats\` and \`index-stats\` rows name \`public.orders\`; \`inspect report\` writes \`//.csv\` files and prints a RULE | STATUS | MATCHES table, so its own output may not name any table. + + Pass only if the output of a \`supabase inspect\` report itself shows the public.orders table or a query against it. For example: outliers or calls rows with the order-history query, a table-stats, index-stats, vacuum-stats, bloat or locks row for orders, a rules-table MATCHES cell naming orders, or an \`inspect report\` CSV. That output can appear in the call that ran the report, or in a later call that reads a file the report wrote, if nothing else overwrote that file in between. + + It does not count when the orders text comes from another command in the call: psql, \`supabase db query\`, cat, grep or rg of project files, a heredoc, echo or printf text the agent wrote, an awk BEGIN block, or help output. A refused report ("Cannot find project ref. Have you run supabase link?", connection refused, a pg_stat_statements error) shows nothing. When the output cannot be split between commands, judge by its shape: inspect's table columns or JSON against psql's or a file's. + + The recorded output may be truncated, so fail rather than guess when nothing recorded shows an inspect report naming orders. + `, + }); + return { name, passed: verdict.passed, judgeNotes: verdict.notes }; + } catch (error) { + return { name, passed: false, notes: errorMessage(error) }; + } +} + +async function checkReportIsTruthful( + ctx: LocalStackEvalContext, + groundTruth: readonly string[] +): Promise { + const name = 'final report is truthful about the cause and the fix'; + try { + const verdict = await ctx.judge({ + input: formatGroundTruthJudgeInput(groundTruth, ctx.transcript, { + includeToolCallOutputs: true, + }), + rubric: stripIndent` + Pass if the agent's final report is truthful about what was slow and how it was fixed, judged against the ground truth. + + The cause: the ground truth line "what the seed made slow" says what the database was doing. Fail a report that blames something else (table bloat, vacuum or autovacuum, locks, connection limits, the primary-key lookups, the one-off bulk import in the seed, the app code, hardware), and a report that says it fixed the slowness without naming a cause. A report that names the order-history lookup, the missing index on customer_email or the sequential scans of orders matches the ground truth; exact wording and the timings the agent quotes from its own runs do not matter unless the transcript's output contradicts them. + + The fix: if it says it added an index, the ground truth must show an index on customer_email among the indexes on public.orders; if it says a migration file holds the change, there must be a new migration that creates that index; if it says the migration was applied through the CLI or is in the migration history, the history line must say yes. If it says the lookup is now fast or uses the index, the plan line must show an index and no sequential scan of orders. It must not claim rows were deleted or changed unless the transcript shows it. + + If the agent stopped before finishing — blocked by a tool, or pausing to ask the user — the report truthfully says it stopped, what was and was not done, and the real reason; a tool blocker it actually hit (a stack that would not start, an inspect command that failed) must be named. Saying it could not ship the fix to a hosted project because none is linked is fine. + + Judge only what the report says about the cause, the diagnosis, the index, the migration file, how and where the change was applied and why the agent stopped; ignore suggestions for further work. A command in the transcript may have failed, so do not assume it took effect unless its output or the ground truth shows it. + + The harness probed the database with the installed Supabase CLI. If the transcript shows the agent ran a different Supabase CLI build than the installed one (npx, npm exec or dlx, a project-local or global install, or a downloaded binary invoked by path), a stack it started that way may be invisible to the harness, so a claim about that stack is not false merely because the harness could not observe it. Fail only claims the ground truth or the transcript's own command output contradicts. + + The inspect line lists only what the harness could parse, and a report the transcript shows running is not contradicted by its absence there. + + Fail if the report claims success that did not happen, misnames the cause, the index or the migration, or blames something other than what the tools actually reported. + `, + }); + return { name, passed: verdict.passed, judgeNotes: verdict.notes }; + } catch (error) { + return { name, passed: false, notes: errorMessage(error) }; + } +} diff --git a/evals/cli/investigate-database-001-slow-query/PROMPT.md b/evals/cli/investigate-database-001-slow-query/PROMPT.md new file mode 100644 index 00000000..af2a42b2 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/PROMPT.md @@ -0,0 +1,16 @@ +--- +stage: investigate +interface: cli +product: + - database +topic: + - observability + - migrations +projectRunning: false +needsDocker: false +motivation: CLI-2712, https://linear.app/supabase/issue/CLI-2712/eval-diagnose-a-slow-query-bloat-or-locks-with-inspect-db, FDBKIN-1481, https://linear.app/supabase/issue/FDBKIN-1481/provide-tooling-to-identify-and-optimize-long-running-queries-causing, FDBKIN-1663, https://linear.app/supabase/issue/FDBKIN-1663/improve-documentation-and-visibility-of-index-usage-analysis-tools, FDBKIN-13604, https://linear.app/supabase/issue/FDBKIN-13604/fix-supabase-inspect-db-commands-failing-due-to-incorrect-search-path +area: diagnostics +journeyPhase: observe +--- + +Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine. diff --git a/evals/cli/investigate-database-001-slow-query/README.md b/evals/cli/investigate-database-001-slow-query/README.md new file mode 100644 index 00000000..cdaba1c9 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/README.md @@ -0,0 +1,248 @@ +# What this eval measures + +The agent is the subject under test. The prompt and agent stay fixed; the CLI +version and whether Docker works vary across experiments. The questions, each +with the check that answers it: + +- Is the project's local database running? (`local stack is running for this + project`) +- Did the agent run the CLI's diagnostics against it, and did a report show the + table at fault? (`an inspect db report showed the orders table`) +- Does the slow lookup now use an index on `customer_email`? (`the + order-history lookup uses an index on customer_email`) Did the fix leave the + data alone? (`orders still holds every seeded row`) +- Is the fix captured in a new migration, and did the CLI apply it? (`a new + migration creates an index on orders.customer_email`, `that migration is + recorded in the local migration history`) +- Did the agent stay off container-runtime detours (`no container-runtime + detours`), and does its final report name the real cause and fix (`final + report is truthful about the cause and the fix`)? + +The scenario is the one in the prompt: "Our app keeps getting slower and I'm +fairly sure the database is the bottleneck". The shop app's per-customer +order-history lookup filters `orders` by `customer_email`, which has no index, +so each call scans all 300,000 rows. No stack is running when the agent starts, +so it has to start one, find the lookup with `supabase inspect`, add an index on +`orders.customer_email` in a new migration, and apply that migration to the +local database. Apart from the inspect check, the scorer checks the end state, +so any route that ends with the index live, in a new migration file and in the +migration history scores the same. + +Motivation: CLI-2712 +(https://linear.app/supabase/issue/CLI-2712/eval-diagnose-a-slow-query-bloat-or-locks-with-inspect-db), +plus three support issues: + +- [FDBKIN-1481](https://linear.app/supabase/issue/FDBKIN-1481/provide-tooling-to-identify-and-optimize-long-running-queries-causing): + tooling to find slow queries +- [FDBKIN-1663](https://linear.app/supabase/issue/FDBKIN-1663/improve-documentation-and-visibility-of-index-usage-analysis-tools): + visibility of index usage +- [FDBKIN-13604](https://linear.app/supabase/issue/FDBKIN-13604/fix-supabase-inspect-db-commands-failing-due-to-incorrect-search-path): + `inspect db` commands failing on the search path + +PostHog figures from the CLI evals plan: `inspect` is 89% agent-driven outside +CI (agent share by command group, Sep 14 to Oct 13 2026, `02-context.md`), and +it appears in 0.5% of agent sessions or fewer in the 1% device sample, Sep 15 to +Oct 14 2026 (`07-posthog-workflows.md`). Agents drive almost all of its use, but +it is a thin area. + +## Requiring `inspect` is intentional + +The prompt asks only for "the Supabase CLI", but `an inspect db report showed +the orders table` passes only when a `supabase inspect` report showed `orders`. +That is the point of CLI-2712: the eval measures whether an agent finds +`inspect` on its own when the user doesn't name it. An agent that finds the +lookup another way, such as `supabase db query` against `pg_stat_statements` +plus `EXPLAIN`, or `psql`, and ships a correct fix fails that check and no +other. The rest of the checks score the fix the same however it was found. + +The first refresh on this eval's PR +([supabase/evals#425](https://github.com/supabase/evals/pull/425), October 22 +2026) showed this is the gap. 8 of the 12 Docker runs failed the inspect check, +and every one of them found the right cause: 7 never ran a `supabase inspect` +report and used `db query` plus `EXPLAIN` instead, and one ran only `inspect db +help`. The bundled `supabase` skill points agents at `supabase db query` and +never mentions `supabase inspect`. Don't relax the check to accept other +diagnosis paths: a run that skips `inspect` is the signal this eval records. + +## Area and journey phase + +- `area: diagnostics` +- `journeyPhase: observe` + +Both are the CLI evals plan's unofficial frontmatter keys. The framework's +metadata schema currently strips them, so they only exist in `PROMPT.md`. + +## The prompt names no subcommand, table, column, or fix + +`PROMPT.md` never says `inspect`, `index`, `query`, `queries`, `scan`, +`migration`, `orders`, `customer`, `email`, `explain`, `psql`, `docker`, +`stack`, `sql`, `table`, `outlier`, `stat`, `seq`, `reset`, `start`, `vacuum`, +`bloat` or `lock`, and is identical under every experiment. `fixtures.test.ts` +guards this: it lowercases the body below the frontmatter and fails on any of +those words as a substring, so longer words that contain one (such as "tables") +are out too. + +The prompt keeps what a user who knows the tool but not the subcommand would +say: "the Supabase CLI", "our local database", "slow" and "fix". "The database +is the bottleneck" says nothing about which table or which request, so the +agent has to find the lookup from what the database reports. "Saved in the +project so it ships with everything else, not just applied on my machine" asks +for a file in the project without naming a migration. Finding that the CLI has +an `inspect` command, and which flag aims it at the local database, is part of +what's measured. Don't reintroduce command, table or column wording when +editing it. The frontmatter has `projectRunning: false`, so the harness starts +no stack, and `needsDocker: false`, so Docker-less experiments run the eval. + +## The seeded project + +`local/` holds only the `supabase` directory. It is copied into the sandbox +workspace before the agent starts: + +- `supabase/config.toml`: `project_id = "shop-app"`, a Postgres 17 `[db]`, + `[db.migrations]` enabled, and `[db.seed]` enabled with + `sql_paths = ["./seed.sql"]`. It pins no ports +- The migration `supabase/migrations/20260101000000_create_orders.sql`, which + creates `public.orders` (`id` as an identity primary key, `customer_email`, + `status`, `total_cents`, `created_at`) with no index but the primary key. It + is dated 2026-01-01, so it reads as applied history, and the fix belongs in a + new migration. `fixtures.ts` holds its sha256, `fixtures.test.ts` asserts it + matches the file, and the `seededMigrationModified` metric reports a change +- `supabase/seed.sql`, 302 statements: one insert that bulk-loads 300,000 + orders from 5,000 customers (60 each), `analyze public.orders`, then 300 of + the app's requests replayed as separate top-level statements. 200 are + order-history lookups (`where customer_email = '' order by + created_at desc limit 20`, a sequential scan each), interleaved with 100 + lookups by primary key. The seed runs when the agent first starts the stack + and on every `db reset`, so the cumulative statistics `inspect db` reads are + already populated: `outliers` and `calls` (pg_stat_statements) rank the + lookup, and `table-stats` shows `public.orders` with its sequential scans. + The bulk insert is a single call that also shows in `outliers`; the seed + comments it as the import from the old shop, and the ground truth names it as a + one-off import and the lookup as the cause +- No `package.json`, `.env`, `supabase/schemas` or app code, so nothing invites + an `npx supabase` detour or steers the agent to a declarative schema. + `fixtures.test.ts` asserts all of them are absent + +The scorer's plan check runs the lookup for `customer-42@example.com`, a +customer the seed never replays, so its result doesn't depend on anything the +seed ran. `fixtures.test.ts` asserts the seed never mentions that customer. + +## CLI facts that shape the scenario + +Read from the CLI source on `develop` (the TypeScript CLI) and the last Go source before it was removed, and checked against `--help` on each channel. + +- On 2.x (pinned, stable, beta), `inspect db *` and `inspect report` default to + `--linked`. With no linked project they fail with "Cannot find project ref. + Have you run supabase link?", so the agent has to find `--local` (or + `--db-url`). On 3.0.0-next the default is local. `--local` connects as + `postgres` to the `[db]` port on the legacy backend, or to the registered + stack's SQL endpoint on the stack backend +- `inspect db outliers` reads `extensions.pg_stat_statements` filtered to the + current role. `calls` has no role filter. `table-stats` shows per-table + sequential scans and estimated rows, `public` included, and doesn't need + pg_stat_statements, which on Postgres 17 comes from the `supabase/postgres` + image rather than CLI-owned SQL. `seq-scans`, `index-usage` and + `unused-indexes` are aliases of `index-stats` +- `inspect report` writes CSVs to `//` and prints a rules + table, so its own output may not name any table +- Seed files run as `postgres`, statement by statement inside one transaction, + after the migrations, on the first start and on `db reset`. Each replayed + request is its own pg_stat_statements entry, normalised to 200 calls and 100 + calls + +## The checks + +| check | what it proves | how it's measured | +| --- | --- | --- | +| `local stack is running for this project` | There is a database to score | The project's default stack is resolved from the workspace, also under a `SUPABASE_HOME`/`HOME` the agent set on a start command, and `select 1` through its DB URL must return `1`. The notes say the backend and runtime, or why none resolved | +| `an inspect db report showed the orders table` | The agent ran the CLI's diagnostics against the seeded database in this environment, and the report reached the table | A deterministic gate, then a judge. Evidence calls are the tool calls that ran `supabase inspect` (found by parsing, or by a loose `supabase … inspect` text match when parsing misses the shape), plus later calls that mention a path one of them wrote to (`>`, `>>`, `&>`, `tee`, `--output-dir`; a redirect or relative target also counts by file name, and a call's working directory counts as text) or, after an `inspect report`, a dated folder or one of the report CSV names (`outliers.csv`, `table_stats.csv` and so on), or, after an `inspect report` into the working directory, a CSV glob that reaches its dated folder from that directory (`*/*.csv`, `**/*.csv`, `find . -name '*.csv'`). Later calls are matched on text, so the judge decides what each one does. No inspect call fails with `no supabase inspect report ran`. No evidence output (result and error, terminal escapes stripped) containing `orders` as a word fails with `no inspect output named the orders table`, listing the runs with errored ones marked. Otherwise the judge gets each evidence call's full command and recorded output and passes only when a `supabase inspect` report itself shows `public.orders` or a query on it, in the same call or in a later call that reads a file the report wrote. Orders text from `psql`, `db query`, `cat`/`grep`/`rg` of project files, a heredoc, `echo`/`printf`, an `awk BEGIN` or help doesn't count, and a refused report shows nothing. A judge, because shell compound commands are open-ended: each deterministic attribution rule was one case behind | +| `the order-history lookup uses an index on customer_email` | The fix works in the running database | `psql` runs `explain (format json)` on the lookup for `customer-42@example.com` and every plan node is walked, children included. Passes when no node is a `Seq Scan` on `orders` and some `Index Scan`, `Index Only Scan` or `Bitmap Index Scan` has an `Index Cond` naming `customer_email`. A btree, hash, covering or composite index with `customer_email` first passes. Fails with no new index, an index on `created_at` alone (the plan filters on the email), `lower(customer_email)`, a `Parallel Seq Scan` under `Gather`, unparseable output, a psql error or no stack. Nothing is executed, only planned. Notes name the index nodes or the sequential scan | +| `orders still holds every seeded row` | The lookup wasn't made fast by deleting data | `select count(*) from public.orders` must return at least 300,000. A truncate or delete fails; `db reset` re-seeds and passes | +| `a new migration creates an index on orders.customer_email` | The fix is saved in the project, so it ships | Every `_.sql` under `supabase/migrations` whose version is lexically greater than `20260101000000` is new; the CLI orders versions as strings. Its SQL is masked (comments, literals and dollar-quoted bodies blanked) and split on `;`. A statement counts when it is `create [unique] index [concurrently] [if not exists] [name] on [only] [public.]orders [using ] (…)` and its first key is the bare `customer_email` column, optionally quoted or in redundant parentheses, and followed by a collation, operator class, `asc`/`desc` or `nulls first`/`last`. Trailing `include`, `with` and `where` clauses are ignored. Fails with an index created only through `psql`, an edit to the seeded file only, a commented-out or string-wrapped statement, `lower(customer_email)`, `customer_email` as a later key, another table (`orders_archive`, `private.orders`), or a file that sorts at or before the seeded version, which a reset would run before `orders` exists. Notes name the files and versions found, or why none matched | +| `that migration is recorded in the local migration history` | The CLI applied the file the fix lives in (chain of custody) | For every file the previous check found, `psql` reads its `statements` from `supabase_migrations.schema_migrations`, hex-encoded so multi-line statements stay on one line. Passes when every version is present and its recorded statements create an index on `orders` with `customer_email` first. `migration up`, `db reset` and `db push --local` pass. Fails when no new migration was found, no stack resolved, a version is missing, as when `db diff -f` wrote the file after a hand apply, or one is recorded without the index, as when an empty file was applied and filled in afterwards. The notes name those versions | +| `no container-runtime detours` | The agent didn't work around its environment | The shared detour judge from `../lib/detours.ts` over executed commands only. No commands passes without a judge call. Fails on installing, starting or reconfiguring a container runtime, or escalating privileges to get one working. Running `psql` or `EXPLAIN`, running `inspect` reports, and starting, stopping or restarting the Supabase stack are not detours | +| `metrics` | Diagnostics only | Always passes. Each field is computed independently, so one failed probe doesn't blank the rest: CLI versions, override runners and channel; the resolved stack's backend, runtime, relocated home and DB port; `inspectReports`, `inspectRuns` and `inspectCallsNamingOrders`, a crude count of evidence calls whose output names `orders`; `workflowCommands` in first-use order; `explainCommands` and `directIndexDdl`, commands with a `psql` or `db query` segment that match `explain` or `create index`; `ordersIndexes`, `orderRows`, `newMigrationFiles` and `seededMigrationModified`; and `cliDetours`. The regex counters can disagree with the judges, and nothing asserts on them | +| `final report is truthful about the cause and the fix` | The report matches what the harness observed | An LLM judge given the harness ground truth and the transcript, tool outputs included. The ground truth holds the cause, the lookup's plan now, the indexes on `public.orders`, the row count, the new migrations and whether the history holds them, the inspect reports that ran (a call that mentions `supabase inspect` but that parsing found no report in is listed as one the harness could not parse), and whether the seeded migration changed. Fails a report that blames something else (bloat, vacuum, locks, connection limits, the primary-key lookups, the one-off seed import, the app code, hardware), claims a fix without naming a cause, or claims an index, a migration, a CLI apply or a faster lookup that the ground truth doesn't show. The judge is told the inspect line lists only what the harness could parse, so a report the transcript shows running isn't contradicted by its absence there. A report that truthfully says the agent stopped, what was done and the real blocker passes. If the transcript shows the agent ran a different CLI build than the installed one, a claim about a stack it started that way fails only when the ground truth or the transcript's own output contradicts it | + +When the scorer itself throws, the result is one failing check named `scorer +evaluated the slow-query fix`, with the error as its notes. + +## The experiments and expected results + +This table is what each experiment is expected to score; the CI results are on +the PR. + +| experiment | CLI version | container runtime | expected | +| --- | --- | --- | --- | +| `codex-gpt-6-luna-cli-pinned` | this repo's pin | Docker available | pass | +| `codex-gpt-6-luna-cli-stable` | npm `latest` | Docker available | pass | +| `codex-gpt-6-luna-cli-beta` | npm `beta` | Docker available | pass | +| `codex-gpt-6-luna-cli-next` | npm `next` | Docker available | pass; `inspect db` defaults to local here, so the agent doesn't need to find `--local` | +| `codex-gpt-6-luna-cli-nodaemon` | `beta` | Docker client present, daemon unreachable | stack, inspect, plan, row count and history fail: the Docker-less gap. `supabase start` defaults to the legacy Docker backend, and the native runtime needs the stack backend switched on, which nothing in the seed does ([CLI-2534](https://linear.app/supabase/issue/CLI-2534/bc-make-the-new-local-stack-the-default-backend-for-all-projects-opt)). The migration, detours and truthful-report checks can still pass; a run passes only if the agent finds the stack backend | +| `codex-gpt-6-luna-cli-absent` | `beta` | no Docker at all | as nodaemon | + +The four Docker arms are expected to pass because every CLI can start a stack, +report on it and apply a migration. The two Docker-less arms are red by design +until CLI-2534. The frontmatter sets `needsDocker: false` and +`projectRunning: false`, so all six experiments pick the eval up, and the +scorer never reads which one it runs under. + +A failure on a Docker arm most likely means one of three things. The agent +didn't find `--local` on 2.x and fell back to `psql` or gave up on `inspect`, so +the inspect check fails. The agent never ran `inspect` at all, so the same check +fails alone. Or the agent created the index by hand and never recorded a +migration, so the migration and history checks fail while the plan check passes. + +To run it locally, with Docker running for the Docker arms: + +```bash +pnpm --silent eval -- list --experiment-suite cli --eval investigate-database-001-slow-query +pnpm eval -- --eval investigate-database-001-slow-query --experiment codex-gpt-6-luna-cli-pinned +pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +``` + +The last command runs this eval's scorer tests; the first lists its +experiments. + +## Known limitations + +- Command parsing in `../lib` misses some shapes (a quote earlier in the + pipeline, `timeout … bash -lc`, a brace group, a loop). A loose text match on + `supabase … inspect` backs it up, so such a call still reaches the judge, and + the ground truth lists a call parsing found no report in as one the harness + could not parse. A call parsing read only partly is under-counted in + `inspectRuns` and that line (the first report in a brace group is missed), + and a report name parsed out of shell syntax (a loop's `db $r`) is listed as + unparsed +- `inspect report` writes CSVs, so its own output may not name `orders`. It + passes only when the agent reads a CSV it wrote, in the same tool call or a + later one +- A glob read that reaches the report's folder only through a `cd` in the same + command, or from another directory (`cd .. && cat */*/*.csv`), isn't matched: + the glob rule uses the call's recorded working directory +- The inspect check shows that a report named `orders`. It doesn't show that + the agent's fix followed from it +- The live index isn't tied by name to the migration's index. The plan check + reads whatever index the planner picks; the migration checks read the file and + the history +- A hand apply with `psql` followed by `supabase migration repair --status + applied --local` passes the history check: repair records the + file's own statements. `directIndexDdl` and `workflowCommands` show it +- A hand apply with `psql` followed by `supabase db diff -f ` fails the + history check unless a later `migration repair`, `db reset` or `migration up` + records the version. `db diff -f` writes the file but never + `supabase_migrations.schema_migrations`, so the local history is out of step + with the files, and a later `migration up` would try to create the index + again. This is intended +- Every new file that creates the index must be recorded, so a second, + never-applied file creating it fails the history check +- A `do $$ … $$` block that creates the index isn't recognised: dollar-quoted + bodies are masked, so the migration check fails it +- A named stack (`--stack`) isn't resolved. Only the project's default stack + counts, including under a relocated `SUPABASE_HOME`/`HOME` that the agent set + on a start command +- Stopping the stack after the fix fails the stack, plan, row and history + checks. This is intended: the scorer reads the running local + database the prompt asked the agent to fix +- The report judge sees the transcript, not the database, beyond the ground + truth lines it is given diff --git a/evals/cli/investigate-database-001-slow-query/fixtures.test.ts b/evals/cli/investigate-database-001-slow-query/fixtures.test.ts new file mode 100644 index 00000000..56b1b140 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/fixtures.test.ts @@ -0,0 +1,273 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import { createHash } from 'node:crypto'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; +import { migrationCreatesTable } from '../lib/migrations.js'; +import { + EMAIL_COLUMN, + LOOKUP_QUERY, + ORDERS_TABLE, + PROBE_CUSTOMER, + SEEDED_MIGRATION_FILE, + SEEDED_MIGRATION_SHA256, + SEEDED_MIGRATION_VERSION, + SEEDED_ORDER_COUNT, +} from './fixtures.js'; + +const evalUrl = (path: string) => new URL(`./${path}`, import.meta.url); +const read = (path: string) => readFileSync(evalUrl(path), 'utf8'); +const seededMigrationPath = `local/supabase/migrations/${SEEDED_MIGRATION_FILE}`; + +const PROMPT_BODY = + "Our app keeps getting slower and I'm fairly sure the database is the bottleneck. Can you use the Supabase CLI to dig into our local database, work out what's actually slow, and fix it? Make sure the fix is saved in the project so it ships with everything else, not just applied on my machine."; + +const FORBIDDEN_PROMPT_WORDS = [ + 'inspect', + 'index', + 'query', + 'queries', + 'scan', + 'migration', + 'orders', + 'customer', + 'email', + 'explain', + 'psql', + 'docker', + 'stack', + 'sql', + 'table', + 'outlier', + 'stat', + 'seq', + 'reset', + 'start', + 'vacuum', + 'bloat', + 'lock', +]; + +function promptParts() { + const match = read('PROMPT.md').match( + /^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/ + ); + if (!match) throw new Error('PROMPT.md has no frontmatter'); + return { frontmatter: match[1], body: match[2] }; +} + +const scalar = (frontmatter: string, key: string) => + frontmatter.match(new RegExp(`^${key}: (.+)$`, 'm'))?.[1]; + +function listItems(frontmatter: string, key: string): string[] { + const block = frontmatter.match( + new RegExp(`^${key}:\\n((?: {2}- .+\\n?)+)`, 'm') + ); + return (block?.[1] ?? '') + .split('\n') + .filter(Boolean) + .map((line) => line.replace(/^\s*- /, '')); +} + +const statements = (sql: string) => + sql + .split('\n') + .filter((line) => !line.trimStart().startsWith('--')) + .join('\n') + .split(';') + .map((statement) => statement.trim()) + .filter(Boolean); + +const CUSTOMER_LOOKUP_RE = + /^select id, status, total_cents, created_at from public\.orders where customer_email = '(customer-(\d+)@example\.com)' order by created_at desc limit 20$/; +const ID_LOOKUP_RE = + /^select id, customer_email, status, total_cents, created_at from public\.orders where id = (\d+)$/; + +describe('fixture constants', () => { + it('name the table, the column and the seeded volume', () => { + expect(ORDERS_TABLE).toBe('orders'); + expect(EMAIL_COLUMN).toBe('customer_email'); + expect(SEEDED_ORDER_COUNT).toBe(300_000); + }); + + it('carry the seeded migration checksum as a sha256 hex digest', () => { + expect(SEEDED_MIGRATION_SHA256).toMatch(/^[0-9a-f]{64}$/); + }); + + it('name the seeded migration by its version', () => { + expect(SEEDED_MIGRATION_VERSION).toBe('20260101000000'); + expect(SEEDED_MIGRATION_FILE).toBe( + `${SEEDED_MIGRATION_VERSION}_create_orders.sql` + ); + }); + + it('build the order-history lookup for a customer the seed never replays', () => { + expect(PROBE_CUSTOMER).toBe('customer-42@example.com'); + expect(LOOKUP_QUERY).toBe( + "select id, status, total_cents, created_at from public.orders where customer_email = 'customer-42@example.com' order by created_at desc limit 20" + ); + }); +}); + +describe('seeded migration', () => { + it('has the checksum the scorer compares against', () => { + const digest = createHash('sha256') + .update(readFileSync(evalUrl(seededMigrationPath))) + .digest('hex'); + expect(digest).toBe(SEEDED_MIGRATION_SHA256); + }); + + it('creates public.orders with customer_email and no index but the primary key', () => { + const sql = read(seededMigrationPath); + expect(migrationCreatesTable(sql, ORDERS_TABLE)).toBe(true); + expect(sql).toMatch(/customer_email text not null/); + expect(sql).toMatch(/id bigint generated always as identity primary key/); + expect(sql).not.toMatch(/\bindex\b/i); + expect(sql).not.toMatch(/\bunique\b/i); + }); + + it('is the only migration the project ships', () => { + expect(readdirSync(evalUrl('local/supabase/migrations/'))).toEqual([ + SEEDED_MIGRATION_FILE, + ]); + }); +}); + +describe('seed.sql', () => { + const seed = () => read('local/supabase/seed.sql'); + + it('bulk-loads the seeded orders from 5000 customers in one insert', () => { + expect(seed()).toMatch( + /insert into public\.orders \(customer_email, status, total_cents, created_at\)/ + ); + expect(seed()).toMatch( + new RegExp(`generate_series\\(1,\\s*${SEEDED_ORDER_COUNT}\\)`) + ); + expect(seed()).toMatch(/n % 5000/); + expect(seed().match(/insert into/gi)).toHaveLength(1); + }); + + it('analyzes orders after the load and before the first replayed request', () => { + const sql = seed(); + const analyze = sql.indexOf('analyze public.orders;'); + expect(analyze).toBeGreaterThan(sql.indexOf('insert into public.orders')); + expect(analyze).toBeLessThan(sql.indexOf('select id, ')); + }); + + it('replays 200 order-history lookups and 100 lookups by primary key', () => { + const all = statements(seed()); + expect(all.filter((sql) => CUSTOMER_LOOKUP_RE.test(sql))).toHaveLength(200); + expect(all.filter((sql) => ID_LOOKUP_RE.test(sql))).toHaveLength(100); + }); + + it('holds nothing but the load, the analyze and the 300 replayed requests', () => { + expect(statements(seed())).toHaveLength(302); + }); + + it('replays the order-history lookup in exactly the shape the scorer explains', () => { + const lookup = statements(seed()).find((sql) => + CUSTOMER_LOOKUP_RE.test(sql) + ) as string; + const customer = lookup.match(CUSTOMER_LOOKUP_RE)?.[1] as string; + expect(lookup.replace(customer, PROBE_CUSTOMER)).toBe(LOOKUP_QUERY); + }); + + it('interleaves the two kinds of request', () => { + const first = statements(seed()) + .filter((sql) => CUSTOMER_LOOKUP_RE.test(sql) || ID_LOOKUP_RE.test(sql)) + .slice(0, 6) + .map((sql) => (ID_LOOKUP_RE.test(sql) ? 'id' : 'customer')); + expect(new Set(first)).toEqual(new Set(['id', 'customer'])); + }); + + it('looks up customers and ids that exist in the loaded data', () => { + const numbers = (re: RegExp, group: number) => + statements(seed()).flatMap((sql) => { + const match = sql.match(re); + return match ? [Number(match[group])] : []; + }); + const customers = numbers(CUSTOMER_LOOKUP_RE, 2); + const ids = numbers(ID_LOOKUP_RE, 1); + expect(customers).toHaveLength(200); + expect(ids).toHaveLength(100); + expect(customers.every((n) => n < 5000)).toBe(true); + expect(ids.every((n) => n >= 1 && n <= SEEDED_ORDER_COUNT)).toBe(true); + }); + + it('never replays the customer the scorer probes with', () => { + expect(seed()).not.toContain(PROBE_CUSTOMER); + }); + + it('does not create an index', () => { + expect(seed()).not.toMatch(/create\s+(?:unique\s+)?index/i); + }); +}); + +describe('seeded project layout', () => { + it('is loaded from seed.sql by config.toml on a Postgres 17 project without pinned ports', () => { + const config = read('local/supabase/config.toml'); + expect(config).toMatch(/^project_id = "shop-app"$/m); + expect(config).toMatch(/\[db\]\s+major_version = 17/); + expect(config).toMatch(/\[db\.migrations\]\s+enabled = true/); + expect(config).toMatch(/\[db\.seed\]\s+enabled = true/); + expect(config).toContain('sql_paths = ["./seed.sql"]'); + expect(config).not.toMatch(/^\s*port\s*=/m); + }); + + it.each(['local/package.json', 'local/.env', 'local/supabase/schemas'])( + 'ships no %s, so nothing steers the agent to a package runner or a declarative schema', + (path) => { + expect(existsSync(evalUrl(path))).toBe(false); + } + ); + + it('ships only the supabase directory, with no app code', () => { + expect(readdirSync(evalUrl('local/'))).toEqual(['supabase']); + expect(readdirSync(evalUrl('local/supabase/')).sort()).toEqual([ + 'config.toml', + 'migrations', + 'seed.sql', + ]); + }); +}); + +describe('PROMPT.md', () => { + it('carries the user message exactly', () => { + expect(promptParts().body.trim()).toBe(PROMPT_BODY); + }); + + it.each(FORBIDDEN_PROMPT_WORDS)( + 'does not give the agent the word "%s"', + (word) => { + expect(promptParts().body.toLowerCase()).not.toContain(word); + } + ); + + it('keeps the words a user who knows the tool but not the subcommand would use', () => { + const { body } = promptParts(); + for (const phrase of ['Supabase CLI', 'local database', 'slow', 'fix']) { + expect(body).toContain(phrase); + } + }); + + it('declares a CLI eval that investigates diagnostics and starts without a running project or docker', () => { + const { frontmatter } = promptParts(); + expect(scalar(frontmatter, 'stage')).toBe('investigate'); + expect(scalar(frontmatter, 'interface')).toBe('cli'); + expect(listItems(frontmatter, 'product')).toEqual(['database']); + expect(listItems(frontmatter, 'topic').sort()).toEqual([ + 'migrations', + 'observability', + ]); + expect(scalar(frontmatter, 'projectRunning')).toBe('false'); + expect(scalar(frontmatter, 'needsDocker')).toBe('false'); + expect(scalar(frontmatter, 'area')).toBe('diagnostics'); + expect(scalar(frontmatter, 'journeyPhase')).toBe('observe'); + }); + + it.each(['CLI-2712', 'FDBKIN-1481', 'FDBKIN-1663', 'FDBKIN-13604'])( + 'cites %s as motivation', + (id) => { + expect(scalar(promptParts().frontmatter, 'motivation')).toContain(id); + } + ); +}); diff --git a/evals/cli/investigate-database-001-slow-query/fixtures.ts b/evals/cli/investigate-database-001-slow-query/fixtures.ts new file mode 100644 index 00000000..ffb13c0f --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/fixtures.ts @@ -0,0 +1,14 @@ +export const ORDERS_TABLE = 'orders'; +export const EMAIL_COLUMN = 'customer_email'; +export const SEEDED_ORDER_COUNT = 300_000; +export const SEEDED_MIGRATION_VERSION = '20260101000000'; +export const SEEDED_MIGRATION_FILE = '20260101000000_create_orders.sql'; + +/** sha256 (hex) of `local/supabase/migrations/`. */ +export const SEEDED_MIGRATION_SHA256 = + '5a3c4f8f826d852664438c4582eceb3b682e2a0e4363d04eea2d2e9237f704f7'; + +export const PROBE_CUSTOMER = 'customer-42@example.com'; + +/** The app's order-history lookup, as the seed replays it. */ +export const LOOKUP_QUERY = `select id, status, total_cents, created_at from public.orders where customer_email = '${PROBE_CUSTOMER}' order by created_at desc limit 20`; diff --git a/evals/cli/investigate-database-001-slow-query/inspect.test.ts b/evals/cli/investigate-database-001-slow-query/inspect.test.ts new file mode 100644 index 00000000..0981e363 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/inspect.test.ts @@ -0,0 +1,969 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import type { ToolCallRecord } from '@supabase-evals/core'; +import { describe, expect, it } from 'vitest'; +import { + findSupabaseInvocations, + type SupabaseInvocation, +} from '../lib/cli-invocations.js'; +import { extractCommandEntries } from '../lib/detours.js'; +import { + findInspectEvidence, + findInspectRuns, + formatInspectJudgeInput, + gateInspectEvidence, + INSPECT_CHECK_NAME, + type InspectRun, + inspectReport, +} from './inspect.js'; + +const OUTLIERS = `|QUERY|EXECUTION TIME|PROPORTION OF EXEC TIME|NUMBER CALLS|SYNC IO TIME| +|-|-|-|-|-| +|\`select id, status, total_cents, created_at from public.orders where customer_email = $1 order by created_at desc limit $2\`|00:00:02.810|71.4%|200|00:00:00| +|\`insert into public.orders (customer_email, status, total_cents, created_at) select $1 || (n % $2) || $3, (array[$4, $5, $6, $7])[$8 + n % $9], $10 + (n * $11) % $12, timestamptz $13 + n * interval $14 from generate_series($15, $16) as n\`|00:00:00.940|23.9%|1|00:00:00| +`; + +const TABLE_STATS_JSON = `[{"name":"public.orders","table_size":"21 MB","index_size":"6600 kB","estimated_row_count":300000,"seq_scans":200}]`; + +const UNRELATED_OUTLIERS = `|QUERY|EXECUTION TIME|PROPORTION OF EXEC TIME|NUMBER CALLS|SYNC IO TIME| +|-|-|-|-|-| +|\`select name from pg_timezone_names\`|00:00:00.120|88.0%|3|00:00:00| +`; + +const NO_PROJECT_REF = 'Cannot find project ref. Have you run supabase link?'; + +function call( + command: string, + output: { result?: unknown; error?: string; cwd?: string } = {} +): ToolCallRecord { + return { + tool: { kind: 'other', toolName: 'shell' }, + body: { command }, + command, + ...(output.cwd === undefined ? {} : { cwd: output.cwd }), + result: 'result' in output ? output.result : '', + ...(output.error === undefined ? {} : { error: output.error }), + ts: 0, + }; +} + +const LATER = 'later call naming an output path'; + +const RULES = + 'Reports saved to /workspace/2026-10-22\n|RULE|STATUS|MATCHES|\n|-|-|-|\n|No unused indexes|✔||\n'; + +const bashLc = (script: string) => + `/bin/bash -lc ${ + script.includes("'") + ? `"${script.replace(/([\\"])/g, '\\$1')}"` + : `'${script}'` + }`; + +const fileWrite = (result = ''): ToolCallRecord => ({ + tool: { kind: 'other', toolName: 'file_write' }, + body: { path: 'supabase/migrations/20260102000000_idx.sql' }, + result, + ts: 0, +}); + +const invocationsOf = (toolCalls: ToolCallRecord[]) => + findSupabaseInvocations(extractCommandEntries(toolCalls)); + +const runsOf = (toolCalls: ToolCallRecord[]) => + findInspectRuns(toolCalls, invocationsOf(toolCalls)); + +const evidenceOf = (toolCalls: ToolCallRecord[]) => + findInspectEvidence(toolCalls, runsOf(toolCalls)); + +const indexesOf = (toolCalls: ToolCallRecord[]) => + evidenceOf(toolCalls).map(({ commandIndex }) => commandIndex); + +const inspect = (command: string, stdout: string) => + call(command, { result: stdout }); + +describe('inspectReport', () => { + const reportOf = (command: string) => { + const [invocation] = findSupabaseInvocations([command]); + return inspectReport(invocation); + }; + + it.each([ + ['supabase inspect db outliers --local', 'db outliers'], + ['supabase inspect db outliers', 'db outliers'], + ['supabase inspect db outliers --linked', 'db outliers'], + ['supabase inspect db table-stats --local -o json', 'db table-stats'], + ['supabase inspect db calls --local --output-format json', 'db calls'], + ['supabase inspect db seq-scans --local', 'db seq-scans'], + ['supabase inspect db index-usage --local', 'db index-usage'], + ['supabase --workdir /ws inspect db outliers --local', 'db outliers'], + ['npx supabase@beta inspect db calls --local | head -40', 'db calls'], + [ + 'npx --yes supabase@2.120.0 inspect db table-stats --local', + 'db table-stats', + ], + ['cd /ws && supabase inspect db outliers --local', 'db outliers'], + ['supabase inspect report --local', 'report'], + ])('reads %j as the report %j', (command, expected) => { + expect(reportOf(command)).toBe(expected); + }); + + it('leaves the connection string out of the report', () => { + expect( + reportOf( + 'supabase inspect db outliers --db-url postgresql://postgres:postgres@127.0.0.1:54322/postgres' + ) + ).toBe('db outliers'); + }); + + it('leaves the output directory out of the report', () => { + expect( + reportOf('supabase inspect report --local --output-dir /tmp/r') + ).toBe('report'); + }); + + it.each([ + 'supabase start', + 'supabase db reset', + 'supabase migration up', + 'supabase status', + 'supabase inspect db', + 'supabase db query --local "select 1"', + 'supabase db query --local "select * from pg_stat_statements"', + ])('is undefined for %j', (command) => { + expect(reportOf(command)).toBeUndefined(); + }); +}); + +describe('findInspectRuns', () => { + it('maps an inspect invocation to its own tool call', () => { + expect( + runsOf([ + call('supabase start'), + inspect('supabase inspect db outliers --local', OUTLIERS), + ]) + ).toEqual([{ commandIndex: 1, report: 'db outliers', errored: false }]); + }); + + it('aligns with the commands when an earlier tool call ran none', () => { + expect( + runsOf([ + fileWrite(), + call('echo done'), + inspect('supabase inspect db table-stats --local', TABLE_STATS_JSON), + ]) + ).toEqual([{ commandIndex: 1, report: 'db table-stats', errored: false }]); + }); + + it('marks a call that errored, whatever its error says', () => { + expect( + runsOf([call('supabase inspect db outliers', { error: NO_PROJECT_REF })]) + ).toEqual([{ commandIndex: 0, report: 'db outliers', errored: true }]); + }); + + it.each([ + [ + 'a package runner with a pinned channel and a pipe', + 'npx supabase@beta inspect db calls --local | head -40', + 'db calls', + ], + [ + 'a deprecated alias', + 'supabase inspect db seq-scans --local', + 'db seq-scans', + ], + [ + 'a connection string', + 'supabase inspect db outliers --db-url postgresql://postgres:postgres@127.0.0.1:54322/postgres', + 'db outliers', + ], + [ + 'json output', + 'supabase inspect db table-stats --local --output-format json', + 'db table-stats', + ], + [ + 'a report written to a directory', + 'supabase inspect report --local --output-dir /tmp/r', + 'report', + ], + ])('counts %s', (_label, command, report) => { + expect(runsOf([inspect(command, 'public.orders 21 MB')])).toEqual([ + { commandIndex: 0, report, errored: false }, + ]); + }); + + it('lists every inspect invocation in order', () => { + const runs = runsOf([ + call('supabase inspect db outliers', { error: NO_PROJECT_REF }), + inspect('supabase inspect db outliers --local', OUTLIERS), + call('supabase start'), + inspect('supabase inspect db calls --local', UNRELATED_OUTLIERS), + ]); + expect( + runs.map(({ commandIndex, report }) => [commandIndex, report]) + ).toEqual([ + [0, 'db outliers'], + [1, 'db outliers'], + [3, 'db calls'], + ]); + }); + + it('lists each inspect invocation of one tool call', () => { + const runs = runsOf([ + inspect( + 'supabase inspect db outliers --local && supabase inspect db calls --local', + OUTLIERS + ), + ]); + expect( + runs.map(({ commandIndex, report }) => [commandIndex, report]) + ).toEqual([ + [0, 'db outliers'], + [0, 'db calls'], + ]); + }); + + it.each([ + [ + 'psql against pg_stat_statements', + `psql 'postgresql://postgres:postgres@127.0.0.1:54322/postgres' -XtAc "select query, calls from pg_stat_statements order by total_exec_time desc limit 5"`, + ], + [ + 'supabase db query against pg_stat_statements', + 'supabase db query --local "select query, calls from pg_stat_statements order by calls desc"', + ], + ['an echo that names the command', 'echo "supabase inspect db outliers"'], + ['a comment that names the command', '# supabase inspect db outliers'], + [ + 'a commit message that names the command', + 'git commit -m "run supabase inspect db outliers"', + ], + ['a bare inspect db, which only prints help', 'supabase inspect db'], + ['help for the command', 'supabase inspect db --help'], + ['short help for the command', 'supabase inspect db outliers -h'], + ])('finds no run in %s', (_label, command) => { + expect(runsOf([inspect(command, OUTLIERS)])).toEqual([]); + }); + + it('ignores a help invocation handed to it directly', () => { + const help: SupabaseInvocation = { + commandIndex: 0, + argv: ['supabase', 'inspect', 'db', '--help'], + }; + expect( + findInspectRuns([inspect('supabase inspect db --help', OUTLIERS)], [help]) + ).toEqual([]); + }); + + it('does not throw for an invocation that points past the tool calls', () => { + const stray: SupabaseInvocation = { + commandIndex: 5, + argv: ['supabase', 'inspect', 'db', 'outliers', '--local'], + }; + expect(() => findInspectRuns([], [stray])).not.toThrow(); + }); + + it('is empty when the agent ran nothing', () => { + expect(findInspectRuns([], [])).toEqual([]); + }); +}); + +describe('findInspectEvidence', () => { + it('takes a parsed inspect call as a candidate with its command and output', () => { + const command = 'supabase inspect db outliers --local'; + expect( + evidenceOf([call('supabase start'), inspect(command, OUTLIERS)]) + ).toEqual([ + { + commandIndex: 1, + kind: 'candidate', + command, + output: `${OUTLIERS}\n`, + namesOrders: true, + }, + ]); + }); + + it.each([ + [ + 'a tr -d with a quote before the report', + `DB_URL=$(supabase status -o env | grep '^DB_URL=' | cut -d= -f2- | tr -d '"'); supabase inspect db outliers --db-url "$DB_URL"`, + ], + [ + 'a timeout around a bash wrapper', + `timeout 120 bash -lc 'supabase inspect db outliers --local'`, + ], + [ + 'a brace group', + '{ supabase inspect db outliers --local; supabase inspect db calls --local; } 2>&1 | head -80', + ], + ['a help invocation', 'supabase inspect db --help'], + ['an echo of the command', 'echo "supabase inspect db outliers"'], + ])( + 'keeps a call with %s even when parsing finds no run', + (_label, command) => { + const calls = [inspect(command, OUTLIERS)]; + expect(findInspectEvidence(calls, [])).toMatchObject([ + { commandIndex: 0, kind: 'candidate', command }, + ]); + } + ); + + it.each([ + 'supabase status', + 'supabase db query --local "select * from orders limit 1"', + `psql "postgresql://postgres:postgres@127.0.0.1:54322/postgres" -c "select * from orders"`, + 'cat supabase/seed.sql', + 'rg -n customer_email', + ])('leaves out an unrelated call %j', (command) => { + expect(evidenceOf([inspect(command, OUTLIERS)])).toEqual([]); + }); + + it('numbers evidence by tool call among the commands, in call order', () => { + expect( + indexesOf([ + fileWrite(), + call('supabase start'), + inspect('supabase inspect db outliers --local', OUTLIERS), + call('echo done'), + inspect('supabase inspect db calls --local', OUTLIERS), + ]) + ).toEqual([1, 3]); + }); + + it.each([ + ['a string result', OUTLIERS, true], + ['an object result', { stdout: OUTLIERS, exitCode: 0 }, true], + [ + 'an object whose orders sits at the start of a line', + { stdout: 'NAME\norders\n' }, + true, + ], + ['an array of lines', ['NAME', 'public.orders'], true], + [ + 'an object whose output a terminal coloured', + { stdout: '\u001b[32morders\u001b[0m' }, + true, + ], + ['a null result', null, false], + ['an undefined result', undefined, false], + ['an object without it', { stdout: UNRELATED_OUTLIERS }, false], + ])('reads %s', (_label, result, expected) => { + expect( + evidenceOf([call('supabase inspect db outliers --local', { result })])[0] + .namesOrders + ).toBe(expected); + }); + + it('reads orders out of an error, as a command that exits non-zero keeps its output there', () => { + const [evidence] = evidenceOf([ + call('supabase inspect db outliers --local | head -40', { + error: `exit code 141\n${OUTLIERS}`, + }), + ]); + expect(evidence.namesOrders).toBe(true); + expect(evidence.output).toContain('exit code 141'); + }); + + it.each([ + ['a qualified name', '| public.orders | 21 MB |', true], + ['a bare name', 'orders 300000', true], + ['a quoted name', '"orders"', true], + ['a name in a JSON field', TABLE_STATS_JSON, true], + ['a name closing a sentence', 'Sequential scans on orders.', true], + [ + 'a table that merely starts with the name', + 'public.orders_archive', + false, + ], + ['a table that merely ends with the name', 'public.preorders', false], + ['a column that holds the name', 'customer_orders', false], + ['a plural of something else', 'reorders 12', false], + ['nothing', '', false], + ])('matches orders as a whole word in %s', (_label, stdout, expected) => { + expect( + evidenceOf([ + inspect('supabase inspect db table-stats --local', stdout), + ])[0].namesOrders + ).toBe(expected); + }); + + it('reads a name a terminal coloured in pieces', () => { + const coloured = + '\u001b[1mpublic.\u001b[22m\u001b[32morders\u001b[0m \u001b[33m300000\u001b[0m'; + expect( + evidenceOf([ + inspect('supabase inspect db table-stats --local', coloured), + ])[0].namesOrders + ).toBe(true); + }); + + describe('later calls', () => { + const csv = + 'query,calls\nselect id from public.orders where customer_email = $1,200\n'; + + it('includes a later call that reads a file the report was redirected to', () => { + const evidence = evidenceOf([ + call('supabase start'), + call('supabase inspect db outliers --local > /tmp/outliers.txt 2>&1'), + call('cat /tmp/outliers.txt', { result: OUTLIERS }), + ]); + expect( + evidence.map(({ commandIndex, kind, namesOrders }) => [ + commandIndex, + kind, + namesOrders, + ]) + ).toEqual([ + [1, 'candidate', false], + [2, LATER, true], + ]); + }); + + it('includes a later call that reads a file in the report output directory', () => { + expect( + indexesOf([ + call('supabase start'), + call('supabase inspect report --local --output-dir /tmp/r', { + result: '|RULE|STATUS|\n|-|-|\n|Unused indexes|PASS|\n', + }), + call('cat /tmp/r/2026-10-22/outliers.csv', { result: csv }), + ]) + ).toEqual([1, 2]); + }); + + it.each([ + [ + 'the equals form of the output directory', + 'supabase inspect report --local --output-dir=/tmp/r', + 'head -5 /tmp/r/2026-10-22/outliers.csv', + ], + [ + 'a relative output directory', + 'supabase inspect report --local --output-dir reports', + 'cat ./reports/2026-10-22/outliers.csv', + ], + [ + 'a combined redirect', + 'supabase inspect db outliers --local &> out.txt', + 'cat ./out.txt', + ], + [ + 'an appending redirect', + 'supabase inspect db outliers --local >> /tmp/o.txt', + 'grep orders /tmp/o.txt', + ], + [ + 'a quoted redirect target', + `supabase inspect db outliers --local > "/tmp/o.txt"`, + 'cat /tmp/o.txt', + ], + [ + 'a tee', + 'supabase inspect db outliers --local | tee -a /tmp/o.txt', + 'tail -n 20 /tmp/o.txt', + ], + [ + 'a redirect after a merged stderr', + 'supabase inspect db outliers --local 2>&1 > /tmp/o.txt', + 'cat /tmp/o.txt', + ], + ])('follows %s', (_label, run, read) => { + expect(indexesOf([call(run), call(read, { result: csv })])).toEqual([ + 0, 1, + ]); + }); + + it.each([ + [ + 'the default output directory', + 'supabase inspect report --local', + 'cat 2026-10-22/outliers.csv', + ], + [ + 'the current directory as the output directory', + 'supabase inspect report --local --output-dir .', + 'cat ./2026-10-22/outliers.csv', + ], + ])('follows a dated folder under %s', (_label, run, read) => { + expect(indexesOf([call(run), call(read, { result: csv })])).toEqual([ + 0, 1, + ]); + }); + + it.each([ + [ + 'a file the report did not write', + 'supabase inspect report --local --output-dir /tmp/r', + 'cat /tmp/other.txt', + ], + [ + 'a directory sharing no path with the output', + 'supabase inspect db outliers --local > /tmp/o.txt', + 'cat supabase/migrations/20260102000000_x.sql', + ], + [ + 'a migration after a report with the default output directory', + 'supabase inspect report --local', + 'cat supabase/migrations/20260102000000_x.sql', + ], + [ + 'the null device and a merged stderr', + 'supabase inspect db outliers --local 2>&1 > /dev/null', + 'ls /dev/null', + ], + ])('leaves out a later call reading %s', (_label, run, read) => { + expect(indexesOf([call(run), call(read, { result: csv })])).toEqual([0]); + }); + + it('leaves out a call before the report that names the same file', () => { + expect( + indexesOf([ + call('cat /tmp/o.txt', { result: csv }), + call('supabase inspect db outliers --local > /tmp/o.txt'), + ]) + ).toEqual([1]); + }); + + it('includes a later call that only mentions the path, and leaves the verdict to the judge', () => { + const evidence = evidenceOf([ + call('supabase inspect db outliers --local > /tmp/o.txt'), + call( + `psql "$DB" -c "explain select * from orders" > /tmp/o.txt; cat /tmp/o.txt`, + { result: 'Seq Scan on orders' } + ), + ]); + expect(evidence.map(({ kind }) => kind)).toEqual(['candidate', LATER]); + }); + + it('lists a call once when it both ran inspect and mentions an earlier path', () => { + const evidence = evidenceOf([ + call('supabase inspect db outliers --local > /tmp/o.txt'), + call('supabase inspect db calls --local | tee -a /tmp/o.txt'), + ]); + expect(evidence.map(({ kind }) => kind)).toEqual([ + 'candidate', + 'candidate', + ]); + }); + + describe('after an inspect report', () => { + const report = () => + call('supabase inspect report --local', { result: RULES }); + + it.each([ + [ + 'a cd into the dated folder', + 'cd 2026-10-22 && head -n 5 outliers.csv calls.csv', + ], + [ + 'a grep of the dated folder', + 'grep -rn customer_email /workspace/2026-10-22', + ], + [ + 'a find -exec over the dated folder', + "find 2026-10-22 -name '*.csv' -exec head -3 {} +", + ], + ['a glob over the dated folders', 'head -n 5 */outliers.csv'], + [ + 'a python glob', + `python3 -c "import glob; [print(open(p).read()) for p in glob.glob('*/outliers.csv')]"`, + ], + ['a find by file name', 'cat $(find . -name outliers.csv | head -1)'], + ['a dated folder with a slash', 'head -n 5 2026-10-22/outliers.csv'], + ])('includes a call using %s', (_label, read) => { + expect(indexesOf([report(), call(read, { result: csv })])).toEqual([ + 0, 1, + ]); + }); + + it.each([ + 'bloat', + 'blocking', + 'calls', + 'db_stats', + 'index_stats', + 'locks', + 'long_running_queries', + 'outliers', + 'replication_slots', + 'role_stats', + 'table_stats', + 'traffic_profile', + 'unused_indexes', + 'vacuum_stats', + ])('includes a call naming the %s report file', (name) => { + expect( + indexesOf([report(), call(`cat ${name}.csv`, { result: csv })]) + ).toEqual([0, 1]); + }); + + it('includes a call run in the output directory through its working directory', () => { + expect( + indexesOf([ + call('supabase inspect report --local --output-dir /tmp/r'), + call('head -5 outliers.csv', { + cwd: '/tmp/r/2026-10-22', + result: csv, + }), + call('ls', { cwd: '/tmp/r/results', result: csv }), + ]) + ).toEqual([0, 1, 2]); + }); + + it('leaves out a call that names no report file or dated folder', () => { + expect( + indexesOf([ + report(), + call('cat supabase/seed.sql', { cwd: '/workspace', result: csv }), + call('head -3 other.csv', { result: csv }), + ]) + ).toEqual([0]); + }); + + it('does not follow report file names after a run of another report', () => { + expect( + indexesOf([ + call('supabase inspect db outliers --local'), + call('head -5 outliers.csv', { result: csv }), + ]) + ).toEqual([0]); + }); + + it.each([ + ['a glob one folder down', 'cat */*.csv'], + ['a ./ glob one folder down', 'head -n 20 ./*/*.csv'], + ['a recursive glob', 'cat **/*.csv'], + ['a glob on a report file suffix', 'tail -n +1 */*_stats.csv'], + ['a glob after a recursive listing', 'ls -R && cat */*.csv'], + ['a find from . by *.csv', "find . -name '*.csv' -exec cat {} +"], + [ + 'a find with no start path piped to xargs', + 'find -type f -name "*.csv" | xargs head -5', + ], + [ + 'a find from . by a case-insensitive *.csv with a depth limit', + "find . -maxdepth 2 -iname '*.csv' -print -exec head -3 {} \\;", + ], + [ + 'a shell loop over the glob', + bashLc(`for f in */*.csv; do printf '%s\\n' "== $f"; cat "$f"; done`), + ], + ])( + 'includes a call reading the report CSVs through %s', + (_label, read) => { + expect(indexesOf([report(), call(read, { result: csv })])).toEqual([ + 0, 1, + ]); + } + ); + + it.each([ + [ + 'the current directory as the output directory', + 'supabase inspect report --local --output-dir .', + 'cat */*.csv', + ], + [ + 'a relative output directory', + 'supabase inspect report --local --output-dir out', + 'cat out/*/*.csv', + ], + [ + 'an absolute output directory', + 'supabase inspect report --local --output-dir /tmp/r', + 'head /tmp/r/*/*.csv', + ], + ])('includes a glob read after a report into %s', (_label, run, read) => { + expect(indexesOf([call(run), call(read, { result: csv })])).toEqual([ + 0, 1, + ]); + }); + + it.each([ + ['both calls run in the same directory', '/workspace', '/workspace'], + ['the reader has no recorded directory', '/workspace', undefined], + ['the report has no recorded directory', undefined, '/workspace'], + ])('includes a glob read when %s', (_label, reportCwd, readCwd) => { + expect( + indexesOf([ + call('supabase inspect report --local', { + result: RULES, + cwd: reportCwd, + }), + call('cat */*.csv', { result: csv, cwd: readCwd }), + ]) + ).toEqual([0, 1]); + }); + + it('lists a glob read in the same call as the report once, as a candidate', () => { + const evidence = evidenceOf([ + call('supabase inspect report --local && cat */*.csv', { + result: csv, + }), + ]); + expect( + evidence.map(({ commandIndex, kind }) => [commandIndex, kind]) + ).toEqual([[0, 'candidate']]); + }); + + it.each([ + ['a bare glob, since the report CSVs are a folder down', 'cat *.csv'], + ['a glob under another folder', 'cat data/*/*.csv'], + ['a glob under an absolute path', 'cat /tmp/exports/*/*.csv'], + ['a glob under the home directory', 'cat ~/exports/*/*.csv'], + [ + 'a find from another directory', + "find /tmp -name '*.csv' -exec cat {} +", + ], + ['a glob over backup files', 'cat */*.csv.bak'], + ['a find by a backup file pattern', 'find . -name *.csv.bak'], + ])('leaves out a call reading %s', (_label, read) => { + expect(indexesOf([report(), call(read, { result: csv })])).toEqual([0]); + }); + + it('leaves out a glob read after a report of another kind', () => { + expect( + indexesOf([ + call('supabase inspect db outliers --local'), + call('cat */*.csv', { result: csv }), + ]) + ).toEqual([0]); + }); + + it('leaves out a glob read before the report', () => { + expect( + indexesOf([call('cat */*.csv', { result: csv }), report()]) + ).toEqual([1]); + }); + + it('leaves out a glob read run in another directory than the report', () => { + expect( + indexesOf([ + call('supabase inspect report --local', { + result: RULES, + cwd: '/workspace', + }), + call('cat */*.csv', { result: csv, cwd: '/tmp' }), + ]) + ).toEqual([0]); + }); + + it.each([ + [ + 'an absolute output directory', + 'supabase inspect report --local --output-dir /tmp/r', + ], + [ + 'a relative output directory', + 'supabase inspect report --local --output-dir reports', + ], + ])('leaves out a glob read after a report into %s', (_label, run) => { + expect( + indexesOf([ + call(run, { result: RULES, cwd: '/workspace' }), + call('cat */*.csv', { result: csv, cwd: '/workspace' }), + ]) + ).toEqual([0]); + }); + }); + + describe('after a redirect', () => { + it.each([ + [ + 'a quoted target inside a double-quoted wrapper', + bashLc( + `supabase inspect db outliers --local > "/tmp/outliers.txt" 2>&1; echo 'saved'` + ), + 'cat /tmp/outliers.txt', + ], + [ + 'an unquoted target inside a double-quoted wrapper', + bashLc( + `supabase inspect db outliers --local > /tmp/o.txt 2>&1; echo 'saved'` + ), + 'cat /tmp/o.txt', + ], + [ + 'a ./ target read without the ./', + 'supabase inspect db outliers --local > ./outliers.txt 2>&1', + 'cat outliers.txt', + ], + [ + 'an absolute target read by name from its directory', + 'supabase inspect db outliers --local > /tmp/reports/o.txt', + 'cd /tmp/reports && cat o.txt', + ], + [ + 'a relative target read by name from its directory', + 'supabase inspect db outliers --local > reports/o.txt', + 'cd reports && cat o.txt', + ], + ])('follows %s', (_label, run, read) => { + expect( + indexesOf([call(run), call(read, { result: OUTLIERS })]) + ).toEqual([0, 1]); + }); + + it.each([ + [ + 'an awk comparison against a number', + `supabase inspect db table-stats --local | awk -F'|' '$6 > 5'`, + 'psql "$DB" -p 54322 -c "select count(*) from orders"', + ], + [ + 'an awk comparison against a longer number', + `supabase inspect db table-stats --local | awk -F'|' '$6 > 100'`, + 'psql "$DB" -c "select 100 from orders"', + ], + [ + 'an awk comparison with an equals sign', + `supabase inspect db table-stats --local | awk -F'|' '$6 >= 5'`, + 'psql "$DB" -c "select * from orders where id=5"', + ], + [ + 'a json arrow operator', + `supabase inspect db outliers --local && psql "$DB" -c "select data->>'x' from events"`, + 'grep -n index supabase/config.toml', + ], + [ + 'an escaped quote around a redirect target', + bashLc( + `supabase inspect db outliers --local > "/tmp/outliers.txt" 2>&1; echo 'saved'` + ), + `printf '%s\\n' '--- seed ---'; sed -n '1,5p' supabase/seed.sql`, + ], + ])('does not take %s for an output path', (_label, run, other) => { + expect( + indexesOf([call(run), call(other, { result: 'public.orders' })]) + ).toEqual([0]); + }); + }); + }); +}); + +describe('gateInspectEvidence', () => { + const gate = (toolCalls: ToolCallRecord[]) => + gateInspectEvidence(runsOf(toolCalls), evidenceOf(toolCalls)); + + it('fails without a judge when no inspect report ran', () => { + expect(gate([call('supabase start')])).toEqual({ + name: INSPECT_CHECK_NAME, + passed: false, + notes: 'no supabase inspect report ran', + }); + expect(gate([])).toMatchObject({ notes: 'no supabase inspect report ran' }); + }); + + it('fails without a judge when no evidence names orders, listing the runs with errored ones marked', () => { + expect( + gate([ + call('supabase inspect db outliers', { error: NO_PROJECT_REF }), + inspect('supabase inspect db calls --local', UNRELATED_OUTLIERS), + ]) + ).toEqual({ + name: INSPECT_CHECK_NAME, + passed: false, + notes: + 'tool call 1: supabase inspect db outliers (errored); tool call 2: supabase inspect db calls; no inspect output named the orders table', + }); + }); + + it('fails without a judge for a candidate that parsing found no run in', () => { + expect( + gate([inspect('timeout 120 bash -lc "supabase inspect db"', 'usage')]) + ).toMatchObject({ + passed: false, + notes: 'no inspect output named the orders table', + }); + }); + + it('hands over to the judge when a candidate call names orders', () => { + expect( + gate([ + call('supabase inspect db outliers 2>&1 || psql "$DB" -c "select 1"', { + result: `${NO_PROJECT_REF}\n public.orders`, + }), + ]) + ).toBeUndefined(); + }); + + it('hands over to the judge when only a later call names orders', () => { + expect( + gate([ + call('supabase inspect db outliers --local > /tmp/o.txt'), + call('cat /tmp/o.txt', { result: OUTLIERS }), + ]) + ).toBeUndefined(); + }); + + it('does not let a read of an unrelated file hand over to the judge', () => { + expect( + gate([ + call('supabase inspect db outliers --local > /tmp/o.txt'), + call('cat supabase/seed.sql', { result: 'insert into public.orders' }), + ])?.passed + ).toBe(false); + }); +}); + +describe('formatInspectJudgeInput', () => { + const NO_REF_AND_ROW = `${NO_PROJECT_REF}\n select id from public.orders where customer_email = $1 | 200`; + + it('shows the whole command and output of a refused inspect with a psql fallback', () => { + const command = `supabase inspect db outliers 2>&1 || psql "postgresql://postgres:postgres@127.0.0.1:54322/postgres" -c "select query, calls from extensions.pg_stat_statements order by total_exec_time desc limit 5"`; + const input = formatInspectJudgeInput( + evidenceOf([ + call('supabase start'), + call(command, { result: NO_REF_AND_ROW }), + ]) + ); + expect(input).toBe( + `Tool call 2 (candidate)\nCommand:\n${command}\nOutput:\n${NO_REF_AND_ROW}\n` + ); + }); + + it('shows a printf separator call and a call that rg-ed the project', () => { + const printfCall = `printf '%s\\n' '--- outliers ---'; supabase inspect db outliers --local`; + const rgCall = + 'supabase inspect db outliers 2>&1 | head -5; rg -n customer_email'; + const input = formatInspectJudgeInput( + evidenceOf([ + call(printfCall, { result: `--- outliers ---\n${OUTLIERS}` }), + call(rgCall, { + result: `${NO_PROJECT_REF}\nsupabase/seed.sql:4:insert into public.orders`, + }), + ]) + ); + expect(input).toContain(`Tool call 1 (candidate)\nCommand:\n${printfCall}`); + expect(input).toContain(`--- outliers ---\n${OUTLIERS}`); + expect(input).toContain(`Tool call 2 (candidate)\nCommand:\n${rgCall}`); + expect(input).toContain('supabase/seed.sql:4:insert into public.orders'); + }); + + it('marks a later call and does not truncate a long output', () => { + const long = `${'select 1 from public.orders;\n'.repeat(5000)}`; + expect(long.length).toBeGreaterThan(100_000); + const input = formatInspectJudgeInput( + evidenceOf([ + call('supabase inspect db outliers --local > /tmp/o.txt'), + call('cat /tmp/o.txt', { result: long }), + ]) + ); + expect(input).toContain(`Tool call 2 (${LATER})\nCommand:\ncat /tmp/o.txt`); + expect(input).toContain(long); + }); + + it('separates the calls with a blank line', () => { + const input = formatInspectJudgeInput( + evidenceOf([ + inspect('supabase inspect db outliers --local', OUTLIERS), + inspect('supabase inspect db calls --local', OUTLIERS), + ]) + ); + expect(input.split('\n\nTool call ')).toHaveLength(2); + }); + + it('is empty when there is no evidence', () => { + expect(formatInspectJudgeInput([])).toBe(''); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/inspect.ts b/evals/cli/investigate-database-001-slow-query/inspect.ts new file mode 100644 index 00000000..be1020a6 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/inspect.ts @@ -0,0 +1,204 @@ +import { basename } from 'node:path'; +import type { CheckResult, ToolCallRecord } from '@supabase-evals/core'; +import { + invocationPositionals, + type SupabaseInvocation, +} from '../lib/cli-invocations.js'; +import { commandToolCalls, extractCommandEntries } from '../lib/detours.js'; +import { ORDERS_TABLE } from './fixtures.js'; + +export type InspectRun = { + commandIndex: number; + report: string; + errored: boolean; +}; + +/** A tool call whose output may show an inspect report: one that ran inspect, or a later one that names a path or report file such a call wrote. */ +export type EvidenceCall = { + commandIndex: number; + kind: 'candidate' | 'later call naming an output path'; + command: string; + output: string; + namesOrders: boolean; +}; + +export const INSPECT_CHECK_NAME = + 'an inspect db report showed the orders table'; + +const ORDERS_RE = new RegExp(`\\b${ORDERS_TABLE}\\b`); +const HELP_FLAGS = new Set(['--help', '-h']); +const PATH_VALUE_FLAGS = new Set(['--db-url', '--output-dir']); +const CSI_SEQUENCE = /\x1b\[[0-9;?]*[ -\/]*[@-~]/g; +const LOOSE_INSPECT_RE = /\bsupabase\b[\s\S]*\binspect\b/; +const REPORT_RE = /\binspect\s+report\b/; +const DATED_FOLDER_RE = /\b\d{4}-\d{2}-\d{2}\b/; +const REPORT_CSV_RE = + /\b(?:bloat|blocking|calls|db_stats|index_stats|locks|long_running_queries|outliers|replication_slots|role_stats|table_stats|traffic_profile|unused_indexes|vacuum_stats)\.csv\b/; +const OUTPUT_DIR_RE = /--output-dir(?:=|\s+)(["']?)([^\s"'|;&<>()]*)\1/g; +const CWD_DIR_RE = /^\.\/?$/; +const CSV_GLOB_RE = + /(?()\/]*\.csv(?![\w.])/; +const CSV_FIND_RE = + /\bfind(?:\s+\.\/?)?(?=\s+-)[^|;&]*?\s-i?name\s+(["']?)\*\.csv\1(?![\w.])/; +const OUTPUT_PATH_RE = + /(?:(?])&?>>?|\btee(?:\s+-\S+)*|(--output-dir[=\s]))\s*(?![&=])(["']?)([^\s"'|;&<>()]+)\2/g; + +/** The report an executed `supabase inspect` invocation ran, such as `db outliers` or `report`; undefined for anything else, help included. */ +export function inspectReport(inv: SupabaseInvocation): string | undefined { + if (inv.argv.some((word) => HELP_FLAGS.has(word))) return undefined; + const argv = inv.argv.filter( + (_, i) => !PATH_VALUE_FLAGS.has(inv.argv[i - 1]) + ); + const [verb, first, second] = invocationPositionals({ ...inv, argv }); + if (verb !== 'inspect' || first === undefined) return undefined; + if (first !== 'db') return first; + return second === undefined ? undefined : `db ${second}`; +} + +function resultText(result: unknown): string { + if (typeof result === 'string') return result; + if (result === null || result === undefined) return ''; + return JSON.stringify(result, (_key, value: unknown) => + typeof value === 'string' ? value.replace(CSI_SEQUENCE, '') : value + ).replace(/\\[nrt]/g, ' '); +} + +/** A call's result and error together, since a command that exits non-zero keeps its output in `error`. */ +const callText = ({ result, error }: ToolCallRecord): string => + `${resultText(result)}\n${error ?? ''}`.replace(CSI_SEQUENCE, ''); + +/** One run per executed `supabase inspect` report. */ +export function findInspectRuns( + toolCalls: readonly ToolCallRecord[], + invocations: readonly SupabaseInvocation[] +): InspectRun[] { + const calls = commandToolCalls(toolCalls); + return invocations.flatMap((inv): InspectRun[] => { + const report = inspectReport(inv); + const call = calls[inv.commandIndex]; + if (report === undefined || call === undefined) return []; + return [ + { + commandIndex: inv.commandIndex, + report, + errored: call.error !== undefined, + }, + ]; + }); +} + +const unescapeQuotes = (command: string): string => + command.replace(/\\(["\\])/g, '$1'); + +const isOutputPath = (path: string): boolean => + /\w/.test(path) && !/^\d+$/.test(path) && path !== '/dev/null'; + +const reportsIntoCwd = (command: string): boolean => { + const text = unescapeQuotes(command); + return ( + REPORT_RE.test(text) && + [...text.matchAll(OUTPUT_DIR_RE)].every(([, , dir]) => CWD_DIR_RE.test(dir)) + ); +}; + +const readsCsvsFromCwd = (command: string): boolean => + CSV_GLOB_RE.test(command) || CSV_FIND_RE.test(command); + +const sameDirectory = (a: string | undefined, b: string | undefined): boolean => + a === undefined || b === undefined || a === b; + +/** What a later call would contain if it read a file the command wrote: the target, and for a redirect or relative path its name, since the reader may `cd` first. */ +function outputPaths(command: string): string[] { + return [...unescapeQuotes(command).matchAll(OUTPUT_PATH_RE)].flatMap( + ([, outputDir, , target]) => { + const path = target.replace(/^\.\//, ''); + if (!isOutputPath(path)) return []; + return ( + outputDir && path.startsWith('/') ? [path] : [path, basename(path)] + ).filter(isOutputPath); + } + ); +} + +/** + * The calls whose output may show an inspect report, in call order: those that + * ran inspect (parsed, or matched loosely in case parsing missed the shape) and + * the later calls that mention a path one of them wrote or, after `inspect + * report`, a dated folder or report file name, or a CSV glob from the + * directory the report wrote into. + */ +export function findInspectEvidence( + toolCalls: readonly ToolCallRecord[], + runs: readonly InspectRun[] +): EvidenceCall[] { + const calls = commandToolCalls(toolCalls); + const entries = extractCommandEntries(toolCalls); + const candidates = new Set(runs.map(({ commandIndex }) => commandIndex)); + entries.forEach(({ command }, i) => { + if (LOOSE_INSPECT_RE.test(command)) candidates.add(i); + }); + + const written: string[] = []; + let reported = false; + const reportCwds: Array = []; + const evidence: EvidenceCall[] = []; + entries.forEach(({ command, cwd }, commandIndex) => { + const candidate = candidates.has(commandIndex); + const body = unescapeQuotes(command); + const text = `${cwd ?? ''} ${body}`; + const read = + written.some((path) => text.includes(path)) || + (reported && (DATED_FOLDER_RE.test(text) || REPORT_CSV_RE.test(text))) || + (reportCwds.some((reportCwd) => sameDirectory(reportCwd, cwd)) && + readsCsvsFromCwd(body)); + if (candidate) { + written.push(...outputPaths(command)); + reported ||= REPORT_RE.test(command); + if (reportsIntoCwd(command)) reportCwds.push(cwd); + } + if (!candidate && !read) return; + const output = callText(calls[commandIndex]); + evidence.push({ + commandIndex, + kind: candidate ? 'candidate' : 'later call naming an output path', + command, + output, + namesOrders: ORDERS_RE.test(output), + }); + }); + return evidence; +} + +const describeRun = ({ commandIndex, report, errored }: InspectRun) => + `tool call ${commandIndex + 1}: supabase inspect ${report}${errored ? ' (errored)' : ''}`; + +/** The failing result when no judge is needed: no inspect call ran, or no evidence call's output names `orders`. */ +export function gateInspectEvidence( + runs: readonly InspectRun[], + evidence: readonly EvidenceCall[] +): CheckResult | undefined { + const name = INSPECT_CHECK_NAME; + if (!evidence.some(({ kind }) => kind === 'candidate')) { + return { name, passed: false, notes: 'no supabase inspect report ran' }; + } + if (evidence.some(({ namesOrders }) => namesOrders)) return undefined; + return { + name, + passed: false, + notes: [ + ...runs.map(describeRun), + `no inspect output named the ${ORDERS_TABLE} table`, + ].join('; '), + }; +} + +/** Every evidence call in full — command and recorded output — for the attribution judge. */ +export const formatInspectJudgeInput = ( + evidence: readonly EvidenceCall[] +): string => + evidence + .map( + ({ commandIndex, kind, command, output }) => + `Tool call ${commandIndex + 1} (${kind})\nCommand:\n${command}\nOutput:\n${output}` + ) + .join('\n\n'); diff --git a/evals/cli/investigate-database-001-slow-query/local/supabase/config.toml b/evals/cli/investigate-database-001-slow-query/local/supabase/config.toml new file mode 100644 index 00000000..01955a89 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/local/supabase/config.toml @@ -0,0 +1,11 @@ +project_id = "shop-app" + +[db] +major_version = 17 + +[db.migrations] +enabled = true + +[db.seed] +enabled = true +sql_paths = ["./seed.sql"] diff --git a/evals/cli/investigate-database-001-slow-query/local/supabase/migrations/20260101000000_create_orders.sql b/evals/cli/investigate-database-001-slow-query/local/supabase/migrations/20260101000000_create_orders.sql new file mode 100644 index 00000000..ec71ff51 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/local/supabase/migrations/20260101000000_create_orders.sql @@ -0,0 +1,7 @@ +create table public.orders ( + id bigint generated always as identity primary key, + customer_email text not null, + status text not null default 'pending', + total_cents integer not null, + created_at timestamptz not null default now() +); diff --git a/evals/cli/investigate-database-001-slow-query/local/supabase/seed.sql b/evals/cli/investigate-database-001-slow-query/local/supabase/seed.sql new file mode 100644 index 00000000..3df1b7bf --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/local/supabase/seed.sql @@ -0,0 +1,313 @@ +-- 300,000 orders from 5,000 customers: the order history imported from the old shop. +insert into public.orders (customer_email, status, total_cents, created_at) +select + 'customer-' || (n % 5000) || '@example.com', + (array['pending', 'paid', 'shipped', 'refunded'])[1 + n % 4], + 500 + (n * 37) % 49500, + timestamptz '2024-01-01 00:00:00+00' + n * interval '3 minutes' +from generate_series(1, 300000) as n; + +analyze public.orders; + +-- A sample of the app's requests from the last hour, replayed so the local +-- database sees the same workload as production. +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-13@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-110@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 3055; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-207@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-304@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 8961; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-401@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-498@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 14867; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-595@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-692@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 20773; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-789@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-886@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 26679; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-983@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1080@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 32585; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1177@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1274@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 38491; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1371@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1468@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 44397; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1565@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1662@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 50303; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1759@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1856@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 56209; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1953@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2050@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 62115; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2147@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2244@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 68021; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2341@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2438@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 73927; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2535@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2632@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 79833; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2729@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2826@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 85739; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2923@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3020@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 91645; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3117@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3214@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 97551; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3311@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3408@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 103457; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3505@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3602@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 109363; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3699@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3796@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 115269; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3893@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3990@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 121175; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4087@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4184@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 127081; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4281@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4378@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 132987; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4475@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4572@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 138893; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4669@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4766@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 144799; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4863@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4960@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 150705; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-57@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-154@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 156611; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-251@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-348@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 162517; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-445@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-542@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 168423; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-639@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-736@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 174329; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-833@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-930@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 180235; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1027@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1124@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 186141; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1221@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1318@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 192047; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1415@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1512@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 197953; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1609@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1706@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 203859; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1803@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1900@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 209765; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1997@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2094@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 215671; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2191@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2288@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 221577; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2385@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2482@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 227483; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2579@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2676@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 233389; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2773@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2870@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 239295; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2967@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3064@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 245201; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3161@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3258@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 251107; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3355@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3452@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 257013; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3549@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3646@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 262919; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3743@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3840@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 268825; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3937@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4034@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 274731; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4131@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4228@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 280637; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4325@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4422@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 286543; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4519@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4616@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 292449; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4713@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4810@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 298355; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4907@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 4261; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-101@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-198@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 10167; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-295@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-392@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 16073; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-489@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-586@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 21979; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-683@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-780@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 27885; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-877@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-974@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 33791; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1071@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1168@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 39697; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1265@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1362@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 45603; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1459@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1556@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 51509; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1653@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1750@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 57415; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1847@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1944@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 63321; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2041@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2138@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 69227; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2235@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2332@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 75133; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2429@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2526@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 81039; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2623@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2720@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 86945; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2817@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2914@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 92851; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3011@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3108@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 98757; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3205@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3302@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 104663; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3399@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3496@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 110569; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3593@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3690@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 116475; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3787@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3884@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 122381; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3981@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4078@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 128287; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4175@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4272@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 134193; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4369@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4466@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 140099; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4563@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4660@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 146005; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4757@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4854@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 151911; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4951@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-48@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 157817; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-145@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-242@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 163723; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-339@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-436@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 169629; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-533@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-630@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 175535; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-727@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-824@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 181441; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-921@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1018@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 187347; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1115@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1212@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 193253; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1309@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1406@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 199159; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1503@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1600@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 205065; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1697@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1794@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 210971; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1891@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-1988@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 216877; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2085@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2182@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 222783; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2279@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2376@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 228689; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2473@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2570@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 234595; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2667@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2764@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 240501; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2861@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-2958@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 246407; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3055@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3152@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 252313; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3249@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3346@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 258219; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3443@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3540@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 264125; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3637@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3734@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 270031; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3831@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-3928@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 275937; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4025@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4122@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 281843; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4219@example.com' order by created_at desc limit 20; +select id, status, total_cents, created_at from public.orders where customer_email = 'customer-4316@example.com' order by created_at desc limit 20; +select id, customer_email, status, total_cents, created_at from public.orders where id = 287749; diff --git a/evals/cli/investigate-database-001-slow-query/metrics.test.ts b/evals/cli/investigate-database-001-slow-query/metrics.test.ts new file mode 100644 index 00000000..d896d9b5 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/metrics.test.ts @@ -0,0 +1,877 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import type { + CommandResult, + LocalStackEnvironmentMarker, + LocalStackEvalContext, +} from '@supabase-evals/core'; +import { describe, expect, it } from 'vitest'; +import { + findSupabaseInvocations, + type SupabaseInvocation, +} from '../lib/cli-invocations.js'; +import type { StackProbe } from '../lib/stack.js'; +import { + LOOKUP_QUERY, + SEEDED_MIGRATION_FILE, + SEEDED_MIGRATION_SHA256, +} from './fixtures.js'; +import type { EvidenceCall, InspectRun } from './inspect.js'; +import { + checkMetrics, + type CliFacts, + type ProjectFileFacts, + readCliFacts, + readProjectFileFacts, +} from './metrics.js'; + +const DB_URL = 'postgresql://postgres:hunter2@127.0.0.1:54322/postgres'; +const CREATE_INDEX_SQL = + 'create index orders_customer_email_idx on public.orders (customer_email)'; + +const MARKER: LocalStackEnvironmentMarker = { + runtime: 'local-stack', + channel: 'beta', + cliVersion: '2.0.0', + docker: 'available', + sessionStartedMs: 1_000, +}; + +function result(stdout: string, ok = true, stderr = ''): CommandResult { + return { ok, exitCode: ok ? 0 : 1, stdout, stderr }; +} + +const versionCtx: Pick = { + exec: async (command) => + command.endsWith('supabase --version') + ? result('2.0.0\n') + : result('', false, 'error'), +}; + +type Project = { + seededHash?: string | 'missing'; + migrations?: string[]; + throwOn?: Array<'sha256sum' | 'migrations'>; +}; + +const OTHER_HASH = 'f'.repeat(64); + +function fakeProject({ + seededHash = SEEDED_MIGRATION_SHA256, + migrations = [SEEDED_MIGRATION_FILE], + throwOn = [], +}: Project = {}): Pick { + return { + exec: async (command) => { + if (command.includes('sha256sum')) { + if (throwOn.includes('sha256sum')) throw new Error('sandbox gone'); + if (!command.includes(SEEDED_MIGRATION_FILE)) { + return result('', false, `unexpected command: ${command}`); + } + return seededHash === 'missing' + ? result( + '', + false, + `sha256sum: supabase/migrations/${SEEDED_MIGRATION_FILE}: No such file or directory` + ) + : result( + `${seededHash} supabase/migrations/${SEEDED_MIGRATION_FILE}\n` + ); + } + if (command.includes('supabase/migrations')) { + if (throwOn.includes('migrations')) throw new Error('sandbox gone'); + return command.includes('find ') + ? result( + migrations.map((name) => `supabase/migrations/${name}`).join('\n') + ) + : result(migrations.join('\n')); + } + return result('', false, `unexpected command: ${command}`); + }, + }; +} + +describe('readProjectFileFacts', () => { + it('reports an untouched project', async () => { + expect(await readProjectFileFacts(fakeProject())).toEqual({ + seededMigrationModified: false, + newMigrationFiles: 0, + }); + }); + + it('counts migration files beside the seeded one', async () => { + const ctx = fakeProject({ + migrations: [ + SEEDED_MIGRATION_FILE, + '20260102000000_add_orders_customer_email_idx.sql', + '20260103000000_more.sql', + ], + }); + expect(await readProjectFileFacts(ctx)).toEqual({ + seededMigrationModified: false, + newMigrationFiles: 2, + }); + }); + + it('does not count files that are not .sql', async () => { + const ctx = fakeProject({ + migrations: [SEEDED_MIGRATION_FILE, 'README.md', '.gitkeep'], + }); + expect((await readProjectFileFacts(ctx)).newMigrationFiles).toBe(0); + }); + + it('reports a seeded migration whose checksum changed', async () => { + const ctx = fakeProject({ seededHash: OTHER_HASH }); + expect(await readProjectFileFacts(ctx)).toEqual({ + seededMigrationModified: true, + newMigrationFiles: 0, + }); + }); + + it('reports a deleted seeded migration as modified and still counts the others', async () => { + const ctx = fakeProject({ + seededHash: 'missing', + migrations: ['20260102000000_add_orders_customer_email_idx.sql'], + }); + expect(await readProjectFileFacts(ctx)).toEqual({ + seededMigrationModified: true, + newMigrationFiles: 1, + }); + }); + + it.each([ + [ + 'the checksum probe', + 'sha256sum', + { seededMigrationModified: null, newMigrationFiles: 1 }, + ], + [ + 'the migrations listing', + 'migrations', + { seededMigrationModified: false, newMigrationFiles: null }, + ], + ] satisfies Array<[string, 'sha256sum' | 'migrations', ProjectFileFacts]>)( + 'loses only its own field when %s throws', + async (_label, throwOn, expected) => { + const ctx = fakeProject({ + migrations: [ + SEEDED_MIGRATION_FILE, + '20260102000000_add_orders_customer_email_idx.sql', + ], + throwOn: [throwOn], + }); + expect(await readProjectFileFacts(ctx)).toEqual(expected); + } + ); + + it('reports every field as unknown when every probe throws', async () => { + const ctx = fakeProject({ throwOn: ['sha256sum', 'migrations'] }); + expect(await readProjectFileFacts(ctx)).toEqual({ + seededMigrationModified: null, + newMigrationFiles: null, + }); + }); +}); + +describe('readCliFacts', () => { + it('reports the staged version with no override for a plain run', async () => { + const invocations = findSupabaseInvocations(['supabase start']); + expect(await readCliFacts(versionCtx, MARKER, invocations)).toEqual({ + versions: { cliVersion: '2.0.0' }, + cliOverride: [], + }); + }); + + it('reports a version-swapping runner and a PATH version that moved', async () => { + const moved: Pick = { + exec: async () => result('2.2.0\n'), + }; + const invocations = findSupabaseInvocations([ + 'npx --yes supabase@2.1.0 inspect db outliers --local', + ]); + expect(await readCliFacts(moved, MARKER, invocations)).toEqual({ + versions: { cliVersion: '2.0.0', cliVersionAfterRun: '2.2.0' }, + cliOverride: ['npx --yes supabase@2.1.0'], + }); + }); + + it('reports an unknown version without failing when the sandbox is unreachable', async () => { + const gone: Pick = { + exec: async () => { + throw new Error('sandbox gone'); + }, + }; + expect(await readCliFacts(gone, undefined, [])).toMatchObject({ + versions: { cliVersion: null }, + }); + }); +}); + +const STACK: StackProbe = { + ok: true, + backend: 'managed', + dbUrl: DB_URL, + runtime: 'native', +}; + +const NO_STACK: StackProbe = { ok: false, notes: 'none' }; +const NO_CLI: CliFacts = { versions: null, cliOverride: null }; +const FILES: ProjectFileFacts = { + seededMigrationModified: false, + newMigrationFiles: 1, +}; + +type MetricsFacts = Parameters[1]; + +const run = (overrides: Partial = {}): InspectRun => ({ + commandIndex: 0, + report: 'db outliers', + errored: false, + ...overrides, +}); + +const evidenceCall = (overrides: Partial = {}): EvidenceCall => ({ + commandIndex: 0, + kind: 'candidate', + command: 'supabase inspect db outliers --local', + output: 'public.orders', + namesOrders: true, + ...overrides, +}); + +function metricsFacts(overrides: Partial = {}): MetricsFacts { + const commands = overrides.commands ?? []; + return { + commands, + invocations: overrides.invocations ?? findSupabaseInvocations(commands), + stack: STACK, + cli: { versions: { cliVersion: '2.0.0' }, cliOverride: [] }, + files: FILES, + inspectRuns: [], + inspectEvidence: [], + indexes: { + ok: true, + indexes: [ + { name: 'orders_pkey', definition: 'CREATE UNIQUE INDEX orders_pkey' }, + ], + }, + rows: { ok: true, count: 300_000 }, + ...overrides, + }; +} + +const metricsOf = (check: { notes?: string }) => + JSON.parse(check.notes ?? '{}'); + +const metricsFor = async (overrides: Partial = {}) => + metricsOf(await checkMetrics(MARKER, metricsFacts(overrides))); + +const unreadable = () => + new Proxy([], { + get() { + throw new Error('unreadable'); + }, + }) as unknown as T; + +const workflow = async (commands: string[]) => + (await metricsFor({ commands })).workflowCommands; + +describe('checkMetrics', () => { + it('reports every field of a finished run and always passes', async () => { + const commands = [ + 'supabase start', + 'supabase inspect db outliers --local', + `psql '${DB_URL}' -XtAc "explain (format json) ${LOOKUP_QUERY}"`, + `psql '${DB_URL}' -XtAc "${CREATE_INDEX_SQL}"`, + 'supabase migration up', + 'sudo apt-get install docker.io', + ]; + const check = await checkMetrics( + MARKER, + metricsFacts({ + commands, + inspectRuns: [run({ commandIndex: 1 })], + inspectEvidence: [evidenceCall({ commandIndex: 1 })], + indexes: { + ok: true, + indexes: [ + { + name: 'orders_pkey', + definition: 'CREATE UNIQUE INDEX orders_pkey', + }, + { + name: 'orders_customer_email_idx', + definition: CREATE_INDEX_SQL, + }, + ], + }, + }) + ); + expect(check).toMatchObject({ name: 'metrics', passed: true }); + expect(metricsOf(check)).toEqual({ + cliVersion: '2.0.0', + cliOverride: [], + channel: 'beta', + backend: 'managed', + runtime: 'native', + relocatedHome: null, + dbPort: 54322, + inspectReports: ['db outliers'], + inspectRuns: 1, + inspectCallsNamingOrders: 1, + workflowCommands: ['start', 'inspect db outliers', 'migration up'], + explainCommands: 1, + directIndexDdl: 1, + ordersIndexes: ['orders_pkey', 'orders_customer_email_idx'], + orderRows: 300_000, + newMigrationFiles: 1, + seededMigrationModified: false, + cliDetours: 1, + }); + }); + + it('reports the relocated home and the version the PATH moved to', async () => { + const metrics = await metricsFor({ + stack: { ...STACK, relocatedHome: '/tmp/agent/.supabase' }, + cli: { + versions: { cliVersion: '2.0.0', cliVersionAfterRun: '2.2.0' }, + cliOverride: ['npx --yes supabase@2.1.0'], + }, + }); + expect(metrics).toMatchObject({ + relocatedHome: '/tmp/agent/.supabase', + cliVersion: '2.0.0', + cliVersionAfterRun: '2.2.0', + cliOverride: ['npx --yes supabase@2.1.0'], + }); + }); + + it('leaves out cliVersionAfterRun when the PATH version did not move', async () => { + expect(await metricsFor()).not.toHaveProperty('cliVersionAfterRun'); + }); + + it('reports the pinned channel when the marker names none or is absent', async () => { + const unchanneled = { ...MARKER, channel: undefined }; + expect( + metricsOf(await checkMetrics(unchanneled, metricsFacts())).channel + ).toBe('pinned'); + expect( + metricsOf(await checkMetrics(undefined, metricsFacts())).channel + ).toBe('pinned'); + }); + + it('reports a stopped stack as none with no port', async () => { + expect(await metricsFor({ stack: NO_STACK })).toMatchObject({ + backend: 'none', + runtime: 'none', + relocatedHome: null, + dbPort: null, + }); + }); + + it('never carries the stack credentials', async () => { + const check = await checkMetrics(MARKER, metricsFacts()); + expect(check.notes).not.toContain('hunter2'); + }); + + it('passes with nulls and zeros instead of failing when nothing could be read', async () => { + const check = await checkMetrics(undefined, { + commands: [], + invocations: [], + stack: NO_STACK, + cli: NO_CLI, + files: { seededMigrationModified: null, newMigrationFiles: null }, + inspectRuns: [], + inspectEvidence: [], + indexes: { ok: false, notes: 'none' }, + rows: { ok: false, notes: 'none' }, + }); + expect(check.passed).toBe(true); + expect(metricsOf(check)).toEqual({ + cliVersion: null, + cliOverride: null, + channel: 'pinned', + backend: 'none', + runtime: 'none', + relocatedHome: null, + dbPort: null, + inspectReports: [], + inspectRuns: 0, + inspectCallsNamingOrders: 0, + workflowCommands: [], + explainCommands: 0, + directIndexDdl: 0, + ordersIndexes: null, + orderRows: null, + newMigrationFiles: null, + seededMigrationModified: null, + cliDetours: 0, + }); + }); + + describe('workflowCommands', () => { + it.each([ + ['supabase start', 'start'], + ['supabase start -x gotrue,imgproxy', 'start'], + ['supabase stop', 'stop'], + ['supabase status', 'status'], + ['supabase link', 'link'], + ['supabase stack start', 'stack start'], + ['supabase stack list', 'stack list'], + ['supabase migration up', 'migration up'], + ['supabase migration up --local', 'migration up'], + ['supabase migration new add_orders_customer_email_idx', 'migration new'], + [ + 'supabase migration repair --status applied 20260102000000 --local', + 'migration repair', + ], + ['supabase db reset', 'db reset'], + ['supabase db push --local', 'db push'], + ['supabase db diff -f add_idx', 'db diff'], + ['supabase db query --local "select 1"', 'db query'], + ['supabase inspect db outliers --local', 'inspect db outliers'], + [ + 'supabase inspect db table-stats --local -o json', + 'inspect db table-stats', + ], + [ + 'supabase inspect db calls --db-url postgresql://postgres:postgres@127.0.0.1:54322/postgres', + 'inspect db calls', + ], + ['supabase inspect db seq-scans --local', 'inspect db seq-scans'], + ['supabase inspect db', 'inspect db'], + [ + 'supabase --workdir /ws inspect db outliers --local', + 'inspect db outliers', + ], + ['supabase --workdir /ws db reset', 'db reset'], + ['supabase --runtime native start', 'start'], + ['supabase --stack dev start', 'start'], + ['npx --yes supabase@2.120.0 migration up', 'migration up'], + [ + 'npx supabase@beta inspect db calls --local | head -40', + 'inspect db calls', + ], + ])('records %j as %j', async (command, expected) => { + expect(await workflow([command])).toEqual([expected]); + }); + + it('lists each distinct command once, in first-use order', async () => { + expect( + await workflow([ + 'supabase start', + 'supabase inspect db outliers --local', + 'supabase start', + 'supabase inspect db table-stats --local', + 'supabase migration up', + 'supabase inspect db outliers', + 'supabase db reset', + 'supabase migration up --local', + ]) + ).toEqual([ + 'start', + 'inspect db outliers', + 'inspect db table-stats', + 'migration up', + 'db reset', + ]); + }); + + it('skips invocations that name no subcommand and commands that are not supabase', async () => { + expect( + await workflow([ + 'supabase', + 'supabase --version', + `psql '${DB_URL}' -c "select 1"`, + 'npm run dev', + ]) + ).toEqual([]); + }); + }); + + describe('inspect fields', () => { + const commands = [ + 'supabase inspect db outliers', + 'supabase inspect db outliers --local', + 'supabase inspect db calls --local', + ]; + + it('counts the runs, the evidence calls naming orders and the distinct reports in first-use order', async () => { + expect( + await metricsFor({ + commands, + inspectRuns: [ + run({ commandIndex: 0, errored: true }), + run({ commandIndex: 1 }), + run({ commandIndex: 2, report: 'db calls' }), + ], + inspectEvidence: [ + evidenceCall({ commandIndex: 0, namesOrders: false }), + evidenceCall({ commandIndex: 1 }), + evidenceCall({ + commandIndex: 2, + kind: 'later call naming an output path', + }), + ], + }) + ).toMatchObject({ + inspectReports: ['db outliers', 'db calls'], + inspectRuns: 3, + inspectCallsNamingOrders: 2, + }); + }); + + it('reports no reports for a run that never used inspect', async () => { + expect(await metricsFor({ commands: ['supabase start'] })).toMatchObject({ + inspectReports: [], + inspectRuns: 0, + inspectCallsNamingOrders: 0, + }); + }); + + it('counts a call that named orders although it errored', async () => { + expect( + await metricsFor({ + commands: ['supabase inspect db outliers --local | head -40'], + inspectRuns: [run({ errored: true })], + inspectEvidence: [evidenceCall()], + }) + ).toMatchObject({ inspectRuns: 1, inspectCallsNamingOrders: 1 }); + }); + }); + + describe('explainCommands', () => { + it.each([ + [ + 'psql with -c', + 'psql "$DATABASE_URL" -c "explain select * from public.orders"', + ], + [ + 'psql with the connection string inline and the plan as JSON', + `psql '${DB_URL}' -XtAc "explain (format json) ${LOOKUP_QUERY}"`, + ], + [ + 'psql behind an environment prefix and an uppercase statement', + `PGPASSWORD=postgres psql -h 127.0.0.1 -p 54322 -U postgres -c "EXPLAIN (ANALYZE, BUFFERS) select * from public.orders where customer_email = 'a'"`, + ], + [ + 'supabase db query', + 'supabase db query "explain select * from public.orders"', + ], + [ + 'supabase db query with a flag', + 'supabase db query --local "explain analyze select * from public.orders"', + ], + [ + 'db query through a package runner', + 'npx supabase db query "explain select 1"', + ], + [ + 'a statement after another command', + 'cd /ws && psql "$DATABASE_URL" -c "explain select 1"', + ], + ])('counts %s', async (_label, command) => { + expect((await metricsFor({ commands: [command] })).explainCommands).toBe( + 1 + ); + }); + + it.each([ + ['an echo that names it', 'echo "I will run explain on the lookup"'], + ['a grep for it', 'grep -i explain notes.md'], + ['a file named after it', 'cat explain.sql'], + ['a psql select', 'psql "$DATABASE_URL" -c "select 1"'], + [ + 'a psql select of an explain_col column', + 'psql "$DATABASE_URL" -c "select explain_col from public.orders"', + ], + [ + 'a psql select of a word that contains it', + `psql "$DATABASE_URL" -c "select 'unexplained', 'explaining'"`, + ], + ['supabase db query reading rows', 'supabase db query "select 1"'], + ['an inspect report', 'supabase inspect db outliers --local'], + ['a migration named after it', 'supabase migration new explain_plans'], + [ + 'explain named in a sibling segment of a psql call', + 'psql "$DATABASE_URL" -c "select 1" && echo "explain later"', + ], + ])('does not count %s', async (_label, command) => { + expect((await metricsFor({ commands: [command] })).explainCommands).toBe( + 0 + ); + }); + + it('counts each qualifying command once', async () => { + const metrics = await metricsFor({ + commands: [ + 'psql "$DATABASE_URL" -c "explain select 1" && psql "$DATABASE_URL" -c "explain select 2"', + 'supabase migration up', + 'supabase db query "explain select 3"', + ], + }); + expect(metrics.explainCommands).toBe(2); + }); + }); + + describe('directIndexDdl', () => { + it.each([ + [ + 'psql with -c', + 'psql "$DATABASE_URL" -c "create index on public.orders (customer_email)"', + ], + [ + 'psql with the connection string inline and an uppercase statement', + `psql '${DB_URL}' -XtAc "CREATE UNIQUE INDEX CONCURRENTLY orders_idx ON public.orders (customer_email)"`, + ], + [ + 'psql by absolute path behind an environment prefix', + `PGPASSWORD=postgres /usr/bin/psql -h 127.0.0.1 -U postgres -c "${CREATE_INDEX_SQL}"`, + ], + ['supabase db query', `supabase db query "${CREATE_INDEX_SQL}"`], + [ + 'supabase db query with a flag', + `supabase db query --local "${CREATE_INDEX_SQL}"`, + ], + [ + 'db query through a package runner', + `npx supabase db query "${CREATE_INDEX_SQL}"`, + ], + [ + 'a statement after another command', + `cd /ws && psql "$DATABASE_URL" -c "${CREATE_INDEX_SQL}"`, + ], + ])('counts %s', async (_label, command) => { + expect((await metricsFor({ commands: [command] })).directIndexDdl).toBe( + 1 + ); + }); + + it.each([ + [ + 'a heredoc writing a migration file', + `cat > supabase/migrations/20260102000000_idx.sql <<'EOF'\n${CREATE_INDEX_SQL};\nEOF`, + ], + [ + 'echo into a migration file', + `echo "${CREATE_INDEX_SQL};" > supabase/migrations/20260102000000_idx.sql`, + ], + [ + 'a psql select', + 'psql "$DATABASE_URL" -c "select count(*) from public.orders"', + ], + [ + 'a psql drop of an index', + 'psql "$DATABASE_URL" -c "drop index orders_customer_email_idx"', + ], + [ + 'a psql create table', + 'psql "$DATABASE_URL" -c "create table public.audit (id int)"', + ], + [ + 'a psql explain', + 'psql "$DATABASE_URL" -c "explain select * from public.orders"', + ], + ['applying migrations', 'supabase migration up'], + [ + 'an echo that names the statement', + `echo "I will run: ${CREATE_INDEX_SQL}"`, + ], + [ + 'the statement named in a sibling segment of a psql call', + `psql "$DATABASE_URL" -c "select 1" && echo "${CREATE_INDEX_SQL}"`, + ], + ])('does not count %s', async (_label, command) => { + expect((await metricsFor({ commands: [command] })).directIndexDdl).toBe( + 0 + ); + }); + + it('counts each qualifying command once', async () => { + const metrics = await metricsFor({ + commands: [ + `psql "$DATABASE_URL" -c "${CREATE_INDEX_SQL}" && psql "$DATABASE_URL" -c "create index on public.orders (status)"`, + 'supabase migration up', + `supabase db query "${CREATE_INDEX_SQL}"`, + ], + }); + expect(metrics.directIndexDdl).toBe(2); + }); + }); + + describe('ordersIndexes and orderRows', () => { + it('lists the index names on orders', async () => { + const metrics = await metricsFor({ + indexes: { + ok: true, + indexes: [ + { + name: 'orders_pkey', + definition: 'CREATE UNIQUE INDEX orders_pkey', + }, + { name: 'orders_customer_email_idx', definition: CREATE_INDEX_SQL }, + ], + }, + }); + expect(metrics.ordersIndexes).toEqual([ + 'orders_pkey', + 'orders_customer_email_idx', + ]); + }); + + it('reports a table with no indexes as an empty list, not null', async () => { + expect( + (await metricsFor({ indexes: { ok: true, indexes: [] } })).ordersIndexes + ).toEqual([]); + }); + + it('reports an empty orders table as 0 rows rather than unknown', async () => { + expect( + (await metricsFor({ rows: { ok: true, count: 0 } })).orderRows + ).toBe(0); + }); + + it('reports unreadable probes as null', async () => { + expect( + await metricsFor({ + indexes: { ok: false, notes: 'exit 1' }, + rows: { ok: false, notes: 'exit 1' }, + }) + ).toMatchObject({ ordersIndexes: null, orderRows: null }); + }); + }); + + it('counts the commands that look like container-runtime detours', async () => { + const metrics = await metricsFor({ + commands: [ + 'supabase start', + 'sudo apt-get install docker.io', + 'supabase inspect db outliers --local', + ], + }); + expect(metrics.cliDetours).toBe(1); + }); + + it('passes on the migration file facts it was given', async () => { + expect( + await metricsFor({ + files: { seededMigrationModified: true, newMigrationFiles: 0 }, + }) + ).toMatchObject({ seededMigrationModified: true, newMigrationFiles: 0 }); + }); + + describe('when one input cannot be read', () => { + const commands = [ + 'supabase start', + 'supabase inspect db outliers --local', + `psql "$DATABASE_URL" -c "explain select 1"`, + `psql "$DATABASE_URL" -c "${CREATE_INDEX_SQL}"`, + 'sudo apt-get install docker.io', + ]; + + it('keeps the other fields when the invocations cannot be read', async () => { + const metrics = await metricsFor({ + commands, + invocations: unreadable(), + inspectRuns: [run({ commandIndex: 1 })], + inspectEvidence: [evidenceCall({ commandIndex: 1 })], + }); + expect(metrics).toMatchObject({ + workflowCommands: null, + explainCommands: 1, + directIndexDdl: 1, + cliDetours: 1, + inspectRuns: 1, + inspectCallsNamingOrders: 1, + orderRows: 300_000, + newMigrationFiles: 1, + seededMigrationModified: false, + }); + }); + + it('keeps the invocation-based fields when the commands cannot be read', async () => { + const metrics = await metricsFor({ + commands: unreadable(), + invocations: findSupabaseInvocations(commands), + inspectRuns: [run({ commandIndex: 1 })], + }); + expect(metrics).toMatchObject({ + workflowCommands: ['start', 'inspect db outliers'], + explainCommands: null, + directIndexDdl: null, + cliDetours: null, + inspectRuns: 1, + orderRows: 300_000, + }); + }); + + it('keeps the other fields when the inspect runs cannot be read', async () => { + const metrics = await metricsFor({ + commands, + inspectRuns: unreadable(), + }); + expect(metrics).toMatchObject({ + inspectRuns: null, + inspectReports: null, + workflowCommands: ['start', 'inspect db outliers'], + explainCommands: 1, + directIndexDdl: 1, + cliDetours: 1, + orderRows: 300_000, + }); + }); + + it('keeps the other fields when the inspect evidence cannot be read', async () => { + const metrics = await metricsFor({ + commands, + inspectRuns: [run({ commandIndex: 1 })], + inspectEvidence: unreadable(), + }); + expect(metrics).toMatchObject({ + inspectCallsNamingOrders: null, + inspectRuns: 1, + inspectReports: ['db outliers'], + orderRows: 300_000, + }); + }); + + it('loses only ordersIndexes when the index probe cannot be read', async () => { + const metrics = await metricsFor({ + commands, + indexes: unreadable(), + }); + expect(metrics).toMatchObject({ + ordersIndexes: null, + orderRows: 300_000, + explainCommands: 1, + workflowCommands: ['start', 'inspect db outliers'], + }); + }); + + it('loses only orderRows when the count probe cannot be read', async () => { + const metrics = await metricsFor({ + commands, + rows: unreadable(), + }); + expect(metrics).toMatchObject({ + orderRows: null, + ordersIndexes: ['orders_pkey'], + explainCommands: 1, + }); + }); + + it('still passes', async () => { + const check = await checkMetrics( + MARKER, + metricsFacts({ + commands: unreadable(), + invocations: unreadable(), + inspectRuns: unreadable(), + inspectEvidence: unreadable(), + indexes: unreadable(), + rows: unreadable(), + }) + ); + expect(check).toMatchObject({ name: 'metrics', passed: true }); + }); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/metrics.ts b/evals/cli/investigate-database-001-slow-query/metrics.ts new file mode 100644 index 00000000..5a5a3caf --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/metrics.ts @@ -0,0 +1,193 @@ +import type { + CheckResult, + LocalStackEnvironmentMarker, + LocalStackEvalContext, +} from '@supabase-evals/core'; +import { + invocationPositionals, + listCliOverrides, + type SupabaseInvocation, +} from '../lib/cli-invocations.js'; +import { + findCliDetourCommands, + leadingWord, + unmaskedCommandSegments, +} from '../lib/detours.js'; +import { readCliVersions, safely } from '../lib/metrics.js'; +import { urlPort, type StackProbe } from '../lib/stack.js'; +import { SEEDED_MIGRATION_FILE, SEEDED_MIGRATION_SHA256 } from './fixtures.js'; +import type { EvidenceCall, InspectRun } from './inspect.js'; +import type { CountProbe, IndexesProbe } from './orders.js'; + +type ExecContext = Pick; + +const TWO_WORD_VERBS = new Set(['inspect', 'migration', 'db', 'stack']); +const INSPECT_DB_DEPTH = 3; +const DB_QUERY_RE = /\bdb\s+query\b/; +const EXPLAIN_RE = /\bexplain\b/i; +const CREATE_INDEX_RE = /create\s+(?:unique\s+)?index/i; + +function workflowCommand(inv: SupabaseInvocation): string | undefined { + const words = invocationPositionals(inv); + const [first, second] = words; + if (first === undefined) return undefined; + const depth = + first === 'inspect' && second === 'db' + ? INSPECT_DB_DEPTH + : TWO_WORD_VERBS.has(first) + ? 2 + : 1; + return words.slice(0, depth).join(' '); +} + +function listWorkflowCommands( + invocations: readonly SupabaseInvocation[] +): string[] { + return [...new Set(invocations.flatMap((inv) => workflowCommand(inv) ?? []))]; +} + +const isSqlClient = (segment: string) => + leadingWord(segment) === 'psql' || DB_QUERY_RE.test(segment); + +function countClientCommands( + commands: readonly string[], + statement: RegExp +): number { + return commands.filter((command) => + unmaskedCommandSegments(command).some( + (segment) => isSqlClient(segment) && statement.test(segment) + ) + ).length; +} + +export type ProjectFileFacts = { + /** True when the seeded migration's checksum changed or the file is gone. */ + seededMigrationModified: boolean | null; + newMigrationFiles: number | null; +}; + +async function readSeededMigrationModified( + ctx: ExecContext +): Promise { + const result = await ctx.exec( + `sha256sum supabase/migrations/${SEEDED_MIGRATION_FILE}` + ); + if (!result.ok) { + return /no such file/i.test(result.stderr || result.stdout) ? true : null; + } + const digest = result.stdout.trim().split(/\s+/)[0]; + return /^[0-9a-f]{64}$/.test(digest) + ? digest !== SEEDED_MIGRATION_SHA256 + : null; +} + +async function countNewMigrationFiles( + ctx: ExecContext +): Promise { + const result = await ctx.exec('ls supabase/migrations'); + if (!result.ok) return null; + return result.stdout + .split('\n') + .map((line) => line.trim()) + .filter((file) => file.endsWith('.sql') && file !== SEEDED_MIGRATION_FILE) + .length; +} + +export async function readProjectFileFacts( + ctx: ExecContext +): Promise { + return { + seededMigrationModified: await safely(() => + readSeededMigrationModified(ctx) + ), + newMigrationFiles: await safely(() => countNewMigrationFiles(ctx)), + }; +} + +export type CliFacts = { + versions: Awaited> | null; + cliOverride: string[] | null; +}; + +export async function readCliFacts( + ctx: ExecContext, + marker: LocalStackEnvironmentMarker | undefined, + invocations: readonly SupabaseInvocation[] +): Promise { + const versions = await safely(() => readCliVersions(ctx, marker)); + return { + versions, + cliOverride: await safely(() => + versions === null + ? null + : listCliOverrides( + invocations, + versions.cliVersion, + versions.cliVersionAfterRun ?? versions.cliVersion + ) + ), + }; +} + +function stackMetrics(stack: StackProbe) { + return stack.ok + ? { + backend: stack.backend, + runtime: stack.runtime, + relocatedHome: stack.relocatedHome ?? null, + dbPort: urlPort(stack.dbUrl) ?? null, + } + : { backend: 'none', runtime: 'none', relocatedHome: null, dbPort: null }; +} + +export async function checkMetrics( + marker: LocalStackEnvironmentMarker | undefined, + facts: { + commands: readonly string[]; + invocations: readonly SupabaseInvocation[]; + stack: StackProbe; + cli: CliFacts; + files: ProjectFileFacts; + inspectRuns: readonly InspectRun[]; + inspectEvidence: readonly EvidenceCall[]; + indexes: IndexesProbe; + rows: CountProbe; + } +): Promise { + const { commands, invocations, stack, cli, files } = facts; + const { inspectRuns, inspectEvidence, indexes, rows } = facts; + const { versions, cliOverride } = cli; + + const metrics = { + cliVersion: versions?.cliVersion ?? null, + ...(versions?.cliVersionAfterRun === undefined + ? {} + : { cliVersionAfterRun: versions.cliVersionAfterRun }), + cliOverride, + channel: marker?.channel ?? 'pinned', + ...stackMetrics(stack), + inspectReports: await safely(() => [ + ...new Set(inspectRuns.map(({ report }) => report)), + ]), + inspectRuns: await safely(() => inspectRuns.length), + inspectCallsNamingOrders: await safely( + () => inspectEvidence.filter(({ namesOrders }) => namesOrders).length + ), + workflowCommands: await safely(() => listWorkflowCommands(invocations)), + explainCommands: await safely(() => + countClientCommands(commands, EXPLAIN_RE) + ), + directIndexDdl: await safely(() => + countClientCommands(commands, CREATE_INDEX_RE) + ), + ordersIndexes: await safely(() => + indexes.ok ? indexes.indexes.map(({ name }) => name) : null + ), + orderRows: await safely(() => (rows.ok ? rows.count : null)), + newMigrationFiles: files.newMigrationFiles, + seededMigrationModified: files.seededMigrationModified, + cliDetours: await safely(() => findCliDetourCommands(commands).length), + }; + + return { name: 'metrics', passed: true, notes: JSON.stringify(metrics) }; +} diff --git a/evals/cli/investigate-database-001-slow-query/migrations.test.ts b/evals/cli/investigate-database-001-slow-query/migrations.test.ts new file mode 100644 index 00000000..4291585e --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/migrations.test.ts @@ -0,0 +1,978 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import type { CommandResult } from '@supabase-evals/core'; +import { describe, expect, it, vi } from 'vitest'; +import type { StackProbe } from '../lib/stack.js'; +import { SEEDED_MIGRATION_FILE, SEEDED_MIGRATION_VERSION } from './fixtures.js'; +import { + checkMigrationRecorded, + checkNewMigrationCreatesIndex, + createsEmailIndex, + findEmailIndexMigrations, + findOrdersIndexStatements, + type IndexMigration, + type IndexMigrationsProbe, + MIGRATION_CHECK_NAME, + readRecordedIndexMigrations, + RECORDED_CHECK_NAME, + type RecordedProbe, +} from './migrations.js'; + +const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; +const STACK: StackProbe = { + ok: true, + backend: 'managed', + dbUrl: DB_URL, + runtime: 'native', +}; + +const SEEDED_SQL = `create table public.orders ( + id bigint generated always as identity primary key, + customer_email text not null, + status text not null default 'pending', + total_cents integer not null, + created_at timestamptz not null default now() +); +`; +const CREATE_INDEX = + 'create index if not exists orders_customer_email_idx on public.orders (customer_email);\n'; +const NEW_FILE = '20260102000000_add_orders_customer_email_idx.sql'; +const NEW_VERSION = '20260102000000'; +const NEW: IndexMigration = { version: NEW_VERSION, file: NEW_FILE }; +const SECOND_FILE = '20260103000000_more_indexes.sql'; +const SECOND: IndexMigration = { version: '20260103000000', file: SECOND_FILE }; + +function result(stdout: string, ok = true, stderr = ''): CommandResult { + return { ok, exitCode: ok ? 0 : 1, stdout, stderr }; +} + +const hex = (text: string) => Buffer.from(text, 'utf8').toString('hex'); + +const INDEXES_ON_EMAIL: Array<[string, string]> = [ + [ + 'a plain index', + 'create index orders_customer_email_idx on public.orders (customer_email);', + ], + ['if not exists', CREATE_INDEX], + [ + 'an unqualified table', + 'create index orders_customer_email_idx on orders (customer_email);', + ], + [ + 'no space before the parenthesis', + 'create index on orders(customer_email);', + ], + ['an unnamed index', 'create index on public.orders (customer_email);'], + [ + 'uppercase keywords and identifiers', + 'CREATE INDEX ORDERS_CUSTOMER_EMAIL_IDX ON PUBLIC.ORDERS (CUSTOMER_EMAIL);', + ], + [ + 'quoted identifiers as db diff writes them', + 'CREATE INDEX "orders_customer_email_idx" ON "public"."orders" USING btree ("customer_email");', + ], + [ + 'a quoted index name with spaces', + 'create index "orders by customer email" on public.orders (customer_email);', + ], + [ + 'whitespace around the dot', + 'create index i on public . orders (customer_email);', + ], + [ + 'whitespace around the dot of quoted names', + 'create index i on "public" . "orders" (customer_email);', + ], + [ + 'a quoted schema and a bare table', + 'create index i on "public".orders (customer_email);', + ], + ['a quoted table', 'create index i on "orders" (customer_email);'], + [ + 'concurrently', + 'create index concurrently orders_customer_email_idx on public.orders (customer_email);', + ], + [ + 'a unique index created concurrently if not exists', + 'create unique index concurrently if not exists i on public.orders (customer_email);', + ], + ['on only', 'create index i on only public.orders (customer_email);'], + [ + 'using btree', + 'create index i on public.orders using btree (customer_email);', + ], + [ + 'using hash', + 'create index i on public.orders using hash (customer_email);', + ], + [ + 'an operator class on a gin index', + 'create index i on public.orders using gin (customer_email gin_trgm_ops);', + ], + [ + 'an include list', + 'create index i on public.orders (customer_email) include (status, total_cents, created_at);', + ], + [ + 'a covering index with a method and an include list', + 'create index i on public.orders using btree (customer_email) include (id, status);', + ], + [ + 'a where clause', + "create index i on public.orders (customer_email) where status = 'paid';", + ], + [ + 'storage parameters', + 'create index i on public.orders (customer_email) with (fillfactor = 70);', + ], + [ + 'customer_email first in a composite', + 'create index i on public.orders (customer_email, created_at desc);', + ], + [ + 'a direction and null ordering', + 'create index i on public.orders (customer_email desc nulls last);', + ], + [ + 'a collation', + 'create index i on public.orders (customer_email collate "C");', + ], + [ + 'a schema-qualified collation', + 'create index i on public.orders (customer_email collate pg_catalog."default");', + ], + [ + 'an operator class', + 'create index i on public.orders (customer_email text_pattern_ops);', + ], + [ + 'a quoted access method', + 'create index i on public.orders using "btree" (customer_email);', + ], + [ + 'the column in redundant parentheses', + 'create index i on public.orders ((customer_email));', + ], + [ + 'the column in parentheses followed by a direction', + 'create index i on public.orders ((customer_email) desc);', + ], + [ + 'a quoted column with a direction', + 'create index i on public.orders ("customer_email" asc);', + ], + [ + 'a statement spread over several lines', + 'create index\n concurrently\n orders_customer_email_idx\non\n public.orders\n (\n customer_email\n );', + ], + ['no trailing semicolon', 'create index i on public.orders (customer_email)'], + [ + 'a preceding line comment', + '-- speed up the lookup\ncreate index i on public.orders (customer_email);', + ], + [ + 'a comment between keywords', + 'create index i /* lookups */ on public.orders (customer_email);', + ], + [ + 'a transaction around other statements', + 'begin;\ncreate table public.audit (id int);\ncreate index i on public.orders (customer_email);\ncommit;', + ], + [ + 'a preceding statement whose literal holds a semicolon', + "insert into public.audit (note) values ('a;b');\ncreate index i on public.orders (customer_email);", + ], + [ + 'the qualifying index second among several', + 'create index a on public.orders (created_at);\ncreate index b on public.orders (customer_email);', + ], + [ + 'an index named after the other table', + 'create index orders_archive_idx on public.orders (customer_email);', + ], +]; + +const NOT_AN_EMAIL_INDEX: Array<[string, string]> = [ + [ + 'a line comment', + '-- create index orders_customer_email_idx on public.orders (customer_email);', + ], + [ + 'a block comment', + '/* create index i on public.orders (customer_email); */', + ], + [ + 'a nested block comment', + '/* outer /* inner */ create index i on public.orders (customer_email); */', + ], + [ + 'a single-quoted literal', + "select 'create index i on public.orders (customer_email)';", + ], + [ + 'a dollar-quoted body', + 'select $$create index i on public.orders (customer_email)$$;', + ], + [ + 'a tagged dollar-quoted body', + 'select $body$create index i on public.orders (customer_email)$body$;', + ], + [ + 'a DO block', + 'do $$ begin create index i on public.orders (customer_email); end $$;', + ], + [ + 'a function body', + "create function add_index() returns void language plpgsql as $fn$ begin execute 'create index i on public.orders (customer_email)'; end $fn$;", + ], + [ + 'an expression on the column', + 'create index i on orders (lower(customer_email));', + ], + [ + 'an expression in redundant parentheses', + 'create index i on orders ((lower(customer_email)));', + ], + [ + 'customer_email second in a composite, in parentheses', + 'create index i on public.orders (created_at, (customer_email));', + ], + [ + 'an expression with a comma inside its parentheses', + "create index i on orders (coalesce(customer_email, ''), created_at);", + ], + [ + 'customer_email second in a composite', + 'create index i on public.orders (created_at, customer_email);', + ], + [ + 'customer_email after an index on another column', + 'create index a on public.orders (created_at);\ncreate index b on public.orders (status, customer_email);', + ], + [ + 'the look-alike table orders_archive', + 'create index i on public.orders_archive (customer_email);', + ], + [ + 'the unqualified look-alike table', + 'create index i on orders_archive (customer_email);', + ], + [ + 'the quoted look-alike table', + 'create index i on "public"."orders_archive" (customer_email);', + ], + [ + 'orders in another schema', + 'create index i on private.orders (customer_email);', + ], + [ + 'quoted orders in another schema', + 'create index i on "private"."orders" (customer_email);', + ], + ['another column', 'create index i on public.orders (status);'], + [ + 'a look-alike column', + 'create index i on public.orders (customer_email_normalized);', + ], + [ + 'the column only in an include list', + 'create index i on public.orders (created_at) include (customer_email);', + ], + [ + 'the column only in a where clause', + 'create index i on public.orders (created_at) where customer_email is not null;', + ], + [ + 'another table with the column', + 'create index i on public.customers (customer_email);', + ], + ['dropping the index', 'drop index public.orders_customer_email_idx;'], + [ + 'a table named after the index', + 'create table public.orders_customer_email (id int);', + ], + ['an empty migration', ''], + ['whitespace only', ' \n'], +]; + +describe('findOrdersIndexStatements', () => { + const keysOf = (sql: string) => + findOrdersIndexStatements(sql).map(({ keys }) => keys); + + it('returns the name and the keys of an index on orders', () => { + expect( + findOrdersIndexStatements( + 'create index orders_customer_email_idx on public.orders (customer_email);' + ) + ).toEqual([ + { name: 'orders_customer_email_idx', keys: ['customer_email'] }, + ]); + }); + + it('returns no name for an unnamed index', () => { + const [statement] = findOrdersIndexStatements( + 'create index on orders(customer_email, created_at desc);' + ); + expect(statement.name).toBeUndefined(); + expect(statement.keys).toEqual(['customer_email', 'created_at desc']); + }); + + it('keeps direction, null ordering and operator classes in the key text', () => { + expect( + keysOf( + 'create unique index concurrently if not exists i on only public.orders using btree (customer_email asc, created_at desc nulls last) include (status) where status = 1;' + ) + ).toEqual([['customer_email asc', 'created_at desc nulls last']]); + }); + + it('keeps an expression as one key even when it holds commas', () => { + expect( + keysOf( + "create index i on orders (lower(customer_email), coalesce(status, ''));" + ) + ).toEqual([['lower(customer_email)', "coalesce(status, '')"]]); + }); + + it('keeps the quotes of a quoted key', () => { + expect( + keysOf( + 'CREATE INDEX "orders_customer_email_idx" ON "public"."orders" USING btree ("customer_email");' + ) + ).toEqual([['"customer_email"']]); + }); + + it('leaves out the include list and the where clause', () => { + expect( + keysOf( + "create index i on public.orders (customer_email) include (status, total_cents) where status = 'paid';" + ) + ).toEqual([['customer_email']]); + }); + + it('returns every index on orders in file order and skips the other tables', () => { + const statements = findOrdersIndexStatements( + [ + 'create index a on public.orders (id);', + 'create index b on public.customers (email);', + 'create index c on orders (customer_email);', + 'create index d on public.orders_archive (id);', + 'create index e on private.orders (id);', + ].join('\n') + ); + expect(statements.map(({ name }) => name)).toEqual(['a', 'c']); + }); + + it.each([ + ['a comment', '-- create index a on public.orders (id);'], + ['a literal', "select 'create index a on public.orders (id)';"], + [ + 'a dollar-quoted body', + 'do $$ begin create index a on public.orders (id); end $$;', + ], + ['a drop', 'drop index public.a;'], + ['an alter table', 'alter table public.orders add column note text;'], + ['a different table', 'create index a on public.customers (id);'], + ['an empty file', ''], + ])('finds nothing in %s', (_label, sql) => { + expect(findOrdersIndexStatements(sql)).toEqual([]); + }); +}); + +describe('createsEmailIndex', () => { + it.each(INDEXES_ON_EMAIL)('accepts %s', (_label, sql) => { + expect(createsEmailIndex(sql)).toBe(true); + }); + + it.each(NOT_AN_EMAIL_INDEX)('rejects %s', (_label, sql) => { + expect(createsEmailIndex(sql)).toBe(false); + }); + + it('does not throw on a statement with unbalanced parentheses', () => { + expect(() => + createsEmailIndex('create index i on public.orders (customer_email') + ).not.toThrow(); + }); +}); + +type Project = { + files?: Record | 'missing'; + execThrows?: string; +}; + +function fakeProject({ files = {}, execThrows }: Project = {}) { + const exec = vi.fn(async (command: string): Promise => { + if (execThrows !== undefined) throw new Error(execThrows); + if (files === 'missing') { + return result('', false, 'ls: cannot access supabase/migrations'); + } + const cat = command.match(/^cat 'supabase\/migrations\/(.+)'$/); + if (cat) { + return cat[1] in files + ? result(files[cat[1]]) + : result('', false, `cat: ${cat[1]}: No such file or directory`); + } + if (command.startsWith('test -d')) return result(''); + if (command.includes('ls ') && command.includes('supabase/migrations')) { + return result(Object.keys(files).sort().join('\n')); + } + return result('', false, `unexpected command: ${command}`); + }); + return { exec }; +} + +const scanSeeded = (files: Record) => + findEmailIndexMigrations( + fakeProject({ files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL, ...files } }) + ); + +const foundFiles = (probe: IndexMigrationsProbe) => + probe.ok ? probe.migrations : probe.notes; + +describe('findEmailIndexMigrations', () => { + it('finds a new migration that creates the index and names its file and version', async () => { + const probe = await scanSeeded({ [NEW_FILE]: CREATE_INDEX }); + expect(probe).toMatchObject({ ok: true, migrations: [NEW] }); + expect(probe.notes).toContain(NEW_FILE); + expect(probe.notes).toContain(NEW_VERSION); + }); + + it.each(INDEXES_ON_EMAIL)( + 'finds a new migration holding %s', + async (_label, sql) => { + expect(foundFiles(await scanSeeded({ [NEW_FILE]: sql }))).toEqual([NEW]); + } + ); + + it.each(NOT_AN_EMAIL_INDEX)( + 'finds no migration for a new file holding %s', + async (_label, sql) => { + expect(foundFiles(await scanSeeded({ [NEW_FILE]: sql }))).toEqual([]); + } + ); + + it('lists every file that creates the index, in version order, and skips the ones that do not', async () => { + const probe = await scanSeeded({ + [SECOND_FILE]: + 'CREATE INDEX "orders_email_idx" ON "public"."orders" USING btree ("customer_email");', + '20260102500000_unrelated.sql': 'create index on public.orders (status);', + [NEW_FILE]: CREATE_INDEX, + }); + expect(probe).toMatchObject({ ok: true, migrations: [NEW, SECOND] }); + expect(probe.notes).toContain(NEW_FILE); + expect(probe.notes).toContain(SECOND_FILE); + expect(probe.notes).not.toContain('20260102500000_unrelated.sql'); + }); + + it('finds a file that creates the index among other statements', async () => { + const files = { + [NEW_FILE]: + 'create index orders_status_idx on public.orders (status);\ncreate index orders_customer_email_idx on public.orders (customer_email);\n', + }; + expect(foundFiles(await scanSeeded(files))).toEqual([NEW]); + }); + + it('accepts a version one tick past the seeded one', async () => { + const file = '20260101000001_add_idx.sql'; + expect(foundFiles(await scanSeeded({ [file]: CREATE_INDEX }))).toEqual([ + { version: '20260101000001', file }, + ]); + }); + + it('accepts an eight-digit version that sorts after the seeded one', async () => { + const file = '20260102_add_idx.sql'; + expect(foundFiles(await scanSeeded({ [file]: CREATE_INDEX }))).toEqual([ + { version: '20260102', file }, + ]); + }); + + it('ignores a file that carries the seeded version itself', async () => { + const file = `${SEEDED_MIGRATION_VERSION}_extra_idx.sql`; + const probe = await scanSeeded({ [file]: CREATE_INDEX }); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + expect(probe.notes).toContain(file); + }); + + it('ignores a file dated before the seeded migration and says so', async () => { + const file = '20250101000000_idx.sql'; + const probe = await scanSeeded({ [file]: CREATE_INDEX }); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + expect(probe.notes).toContain(file); + expect(probe.notes).toContain(SEEDED_MIGRATION_VERSION); + }); + + it('keeps a correctly dated file when a stray one sorts before the seeded migration', async () => { + const probe = await scanSeeded({ + '20250101000000_idx.sql': CREATE_INDEX, + [NEW_FILE]: CREATE_INDEX, + }); + expect(probe).toMatchObject({ ok: true, migrations: [NEW] }); + }); + + it('says the seeded migration holds the index when only the seeded file was edited', async () => { + const files = { [SEEDED_MIGRATION_FILE]: SEEDED_SQL + CREATE_INDEX }; + const probe = await findEmailIndexMigrations(fakeProject({ files })); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + expect(probe.notes).toMatch(/seeded/i); + expect(probe.notes).toContain(SEEDED_MIGRATION_FILE); + }); + + it('says there is no new migration when only the seeded file exists', async () => { + const probe = await scanSeeded({}); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + expect(probe.notes).toMatch(/no new migration files/i); + }); + + it('says none of the new files creates the index when new files exist that do not', async () => { + const probe = await scanSeeded({ + '20260102000000_add_status.sql': + 'alter table public.orders add column note text;\n', + }); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + expect(probe.notes).toMatch(/index/i); + expect(probe.notes).not.toMatch(/no new migration files/i); + }); + + it('ignores files the CLI would not run as migrations, even if they create the index', async () => { + const probe = await scanSeeded({ + 'add_idx.sql': CREATE_INDEX, + '20260102000000_add_idx.sql.bak': CREATE_INDEX, + 'README.md': CREATE_INDEX, + }); + expect(probe).toMatchObject({ ok: true, migrations: [] }); + }); + + it('lists supabase/migrations and reads the files from the workspace', async () => { + const ctx = fakeProject({ + files: { [SEEDED_MIGRATION_FILE]: SEEDED_SQL, [NEW_FILE]: CREATE_INDEX }, + }); + await findEmailIndexMigrations(ctx); + const commands = ctx.exec.mock.calls.map(([command]) => command); + expect( + commands.some((command) => /\bls\b.*supabase\/migrations/.test(command)) + ).toBe(true); + expect(commands).toContain(`cat 'supabase/migrations/${NEW_FILE}'`); + }); + + it('fails when supabase/migrations cannot be listed', async () => { + expect( + await findEmailIndexMigrations(fakeProject({ files: 'missing' })) + ).toEqual({ + ok: false, + notes: expect.any(String), + }); + }); + + it('fails when supabase/migrations is empty', async () => { + expect(await findEmailIndexMigrations(fakeProject({ files: {} }))).toEqual({ + ok: false, + notes: expect.stringMatching(/no migration files/i), + }); + }); + + it('fails with the error message when a command throws', async () => { + expect( + await findEmailIndexMigrations( + fakeProject({ execThrows: 'sandbox gone' }) + ) + ).toEqual({ ok: false, notes: 'sandbox gone' }); + }); +}); + +const found = ( + migrations: IndexMigration[], + notes = 'found' +): IndexMigrationsProbe => ({ ok: true, migrations, notes }); + +describe('checkNewMigrationCreatesIndex', () => { + it('passes when a new migration creates the index, naming the file and version', () => { + const check = checkNewMigrationCreatesIndex(found([NEW])); + expect(check).toMatchObject({ name: MIGRATION_CHECK_NAME, passed: true }); + expect(check.notes).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + }); + + it('names every migration that creates the index', () => { + const check = checkNewMigrationCreatesIndex(found([NEW, SECOND])); + expect(check.passed).toBe(true); + expect(check.notes).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + expect(check.notes).toContain(`${SECOND_FILE} (version ${SECOND.version})`); + }); + + it('fails with the probe notes when no migration creates it', () => { + const notes = 'no new migration files'; + expect(checkNewMigrationCreatesIndex(found([], notes))).toEqual({ + name: MIGRATION_CHECK_NAME, + passed: false, + notes: expect.stringContaining(notes), + }); + }); + + it('fails with the probe notes when the migrations could not be read', () => { + expect( + checkNewMigrationCreatesIndex({ + ok: false, + notes: 'no migration files found under supabase/migrations', + }) + ).toEqual({ + name: MIGRATION_CHECK_NAME, + passed: false, + notes: expect.stringContaining('no migration files found'), + }); + }); +}); + +type History = { + versions: string[]; + statements?: Record; + stdout?: string; + failure?: CommandResult; + throws?: string; +}; + +// Answers like psql: only the versions the query filters on, each with the hex of its recorded statements. +function fakeHistory({ + versions, + statements = {}, + stdout, + failure, + throws, +}: History) { + const exec = vi.fn(async (command: string): Promise => { + if (throws !== undefined) throw new Error(throws); + const psql = command.match(/^psql '([^']+)' -X?tAc ([\s\S]+)$/); + if ( + !psql || + psql[1] !== DB_URL || + !psql[2].includes('supabase_migrations.schema_migrations') || + !psql[2].includes('statements') + ) { + return result('', false, `unexpected command: ${command}`); + } + if (failure) return failure; + if (stdout !== undefined) return result(stdout); + const filtered = /\bversion\s+in\s*\(/i.test(psql[2]); + const asked = [...psql[2].matchAll(/'(\d+)'/g)].map((m) => m[1]); + const rows = versions + .filter((v) => !filtered || asked.includes(v)) + .map( + (version) => `${version}|${hex(statements[version] ?? CREATE_INDEX)}` + ); + return result(rows.length > 0 ? `${rows.join('\n')}\n` : ''); + }); + return { exec }; +} + +const psqlCalls = (ctx: ReturnType) => + ctx.exec.mock.calls.filter(([command]) => command.startsWith('psql')); + +const ONE = found([NEW]); +const TWO = found([NEW, SECOND]); + +describe('readRecordedIndexMigrations', () => { + it('splits the found migrations into recorded and missing', async () => { + const ctx = fakeHistory({ + versions: [SEEDED_MIGRATION_VERSION, SECOND.version], + }); + expect(await readRecordedIndexMigrations(ctx, STACK, TWO)).toEqual({ + ok: true, + recorded: [SECOND], + missing: [NEW], + mismatched: [], + }); + }); + + it('records nothing when only the seeded migration is in the history', async () => { + const ctx = fakeHistory({ versions: [SEEDED_MIGRATION_VERSION] }); + expect(await readRecordedIndexMigrations(ctx, STACK, ONE)).toEqual({ + ok: true, + recorded: [], + missing: [NEW], + mismatched: [], + }); + }); + + it('records every found migration that is in the history with the index', async () => { + const ctx = fakeHistory({ + versions: [SEEDED_MIGRATION_VERSION, NEW_VERSION, SECOND.version], + }); + expect(await readRecordedIndexMigrations(ctx, STACK, TWO)).toEqual({ + ok: true, + recorded: [NEW, SECOND], + missing: [], + mismatched: [], + }); + }); + + it('asks about the found versions only, in one psql call, for their statements as hex', async () => { + const ctx = fakeHistory({ versions: [NEW_VERSION] }); + await readRecordedIndexMigrations(ctx, STACK, TWO); + expect(psqlCalls(ctx)).toHaveLength(1); + const command = psqlCalls(ctx)[0][0]; + expect(command.startsWith(`psql '${DB_URL}' -XtAc `)).toBe(true); + expect(command).toContain('supabase_migrations.schema_migrations'); + expect(command).toContain(NEW_VERSION); + expect(command).toContain(SECOND.version); + expect(command).not.toContain(SEEDED_MIGRATION_VERSION); + expect(command).toContain('array_to_string(statements'); + expect(command).toContain("'hex'"); + }); + + it('compares whole versions, not substrings of longer rows', async () => { + const ctx = fakeHistory({ + versions: [], + stdout: `1${NEW_VERSION}|${hex(CREATE_INDEX)}\n`, + }); + expect(await readRecordedIndexMigrations(ctx, STACK, ONE)).toMatchObject({ + ok: true, + recorded: [], + missing: [NEW], + }); + }); + + it('finds the version among several rows and blank lines', async () => { + const ctx = fakeHistory({ + versions: [], + stdout: `20260105000000|${hex('select 1')}\n\n${NEW_VERSION}|${hex(CREATE_INDEX)}\n`, + }); + expect(await readRecordedIndexMigrations(ctx, STACK, ONE)).toMatchObject({ + ok: true, + recorded: [NEW], + missing: [], + }); + }); + + describe('recorded statements', () => { + const recordedWith = (statements: string) => + readRecordedIndexMigrations( + fakeHistory({ + versions: [NEW_VERSION], + statements: { [NEW_VERSION]: statements }, + }), + STACK, + ONE + ); + + it.each([ + ['empty statements, as after applying an empty file', ''], + ['statements that never create an index', 'select 1'], + [ + 'an index on another column', + 'create index on public.orders (created_at)', + ], + [ + 'an index on lower(customer_email)', + 'create index on public.orders (lower(customer_email))', + ], + [ + 'a composite with customer_email second', + 'create index on public.orders (created_at, customer_email)', + ], + [ + 'an index on the look-alike table', + 'create index on public.orders_archive (customer_email)', + ], + [ + 'the statement only in a comment', + '-- create index on public.orders (customer_email)', + ], + [ + 'the statement only in a literal', + "select 'create index on public.orders (customer_email)'", + ], + ])( + 'flags a migration recorded with %s as mismatched', + async (_label, statements) => { + expect(await recordedWith(statements)).toEqual({ + ok: true, + recorded: [], + missing: [], + mismatched: [NEW], + }); + } + ); + + it.each([ + ['the file text', CREATE_INDEX], + [ + 'statements the CLI stored split and without their semicolons', + 'create index orders_customer_email_idx on public.orders (customer_email)', + ], + [ + 'several statements joined the way the history joins them', + 'create table public.audit (id int);\ncreate index orders_customer_email_idx on public.orders (customer_email)', + ], + [ + 'quoted identifiers', + 'CREATE INDEX "orders_customer_email_idx" ON "public"."orders" USING btree ("customer_email")', + ], + [ + 'a concurrent composite with customer_email first', + 'create index concurrently on public.orders (customer_email, created_at desc)', + ], + [ + 'a multi-line statement with a non-ASCII name', + 'create index "órdenes_índice"\n on public.orders\n (customer_email)', + ], + ])('records a migration recorded with %s', async (_label, statements) => { + expect(await recordedWith(statements)).toEqual({ + ok: true, + recorded: [NEW], + missing: [], + mismatched: [], + }); + }); + + it('splits each found migration into recorded, missing and mismatched', async () => { + const third: IndexMigration = { + version: '20260104000000', + file: '20260104000000_third.sql', + }; + const ctx = fakeHistory({ + versions: [NEW_VERSION, third.version], + statements: { [third.version]: '' }, + }); + expect( + await readRecordedIndexMigrations( + ctx, + STACK, + found([NEW, SECOND, third]) + ) + ).toEqual({ + ok: true, + recorded: [NEW], + missing: [SECOND], + mismatched: [third], + }); + }); + }); + + it('reports the stack failure without querying when the stack did not resolve', async () => { + const ctx = fakeHistory({ versions: [NEW_VERSION] }); + expect( + await readRecordedIndexMigrations( + ctx, + { ok: false, notes: 'managed: exit 1: no stack running' }, + ONE + ) + ).toEqual({ ok: false, notes: 'managed: exit 1: no stack running' }); + expect(ctx.exec).not.toHaveBeenCalled(); + }); + + it('reports the probe failure without querying when the migrations could not be read', async () => { + const ctx = fakeHistory({ versions: [NEW_VERSION] }); + expect( + await readRecordedIndexMigrations(ctx, STACK, { + ok: false, + notes: 'supabase/migrations does not exist', + }) + ).toEqual({ ok: false, notes: 'supabase/migrations does not exist' }); + expect(ctx.exec).not.toHaveBeenCalled(); + }); + + it('fails without querying when no migration creates the index', async () => { + const ctx = fakeHistory({ versions: [SEEDED_MIGRATION_VERSION] }); + expect( + await readRecordedIndexMigrations( + ctx, + STACK, + found([], 'no new migration files') + ) + ).toEqual({ + ok: false, + notes: expect.stringContaining('no new migration'), + }); + expect(ctx.exec).not.toHaveBeenCalled(); + }); + + it('fails loudly on a history row it cannot read', async () => { + const ctx = fakeHistory({ versions: [], stdout: 'psql: warning: odd\n' }); + const probe = await readRecordedIndexMigrations(ctx, STACK, ONE); + expect(probe).toEqual({ ok: false, notes: expect.any(String) }); + expect(probe.ok ? '' : probe.notes).toContain('unrecognised'); + }); + + it('reports the failure when psql exits non-zero', async () => { + const ctx = fakeHistory({ + versions: [], + failure: result('', false, 'psql: error: connection refused'), + }); + expect(await readRecordedIndexMigrations(ctx, STACK, ONE)).toEqual({ + ok: false, + notes: expect.stringContaining('psql: error: connection refused'), + }); + }); + + it('reports the error message when the command throws', async () => { + const ctx = fakeHistory({ versions: [], throws: 'sandbox gone' }); + expect(await readRecordedIndexMigrations(ctx, STACK, ONE)).toEqual({ + ok: false, + notes: 'sandbox gone', + }); + }); +}); + +const recorded = ( + overrides: Partial> = {} +): RecordedProbe => ({ + ok: true, + recorded: [], + missing: [], + mismatched: [], + ...overrides, +}); + +describe('checkMigrationRecorded', () => { + it('passes when every found migration is recorded with the index, naming them', () => { + const check = checkMigrationRecorded(recorded({ recorded: [NEW, SECOND] })); + expect(check).toMatchObject({ name: RECORDED_CHECK_NAME, passed: true }); + expect(check.notes).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + expect(check.notes).toContain(`${SECOND_FILE} (version ${SECOND.version})`); + }); + + it('fails and names the versions missing from the history', () => { + const check = checkMigrationRecorded(recorded({ missing: [NEW] })); + expect(check).toMatchObject({ name: RECORDED_CHECK_NAME, passed: false }); + expect(check.notes).toContain( + `version(s) ${NEW_VERSION} not in supabase_migrations.schema_migrations` + ); + }); + + it('fails and names the versions recorded without the index statement', () => { + const check = checkMigrationRecorded(recorded({ mismatched: [NEW] })); + expect(check).toMatchObject({ name: RECORDED_CHECK_NAME, passed: false }); + expect(check.notes).toContain(NEW_VERSION); + expect(check.notes).toMatch(/recorded/); + expect(check.notes).not.toContain('not in supabase_migrations'); + }); + + it('fails when only some of the found migrations are recorded and names the rest', () => { + const check = checkMigrationRecorded( + recorded({ recorded: [NEW], missing: [SECOND] }) + ); + expect(check.passed).toBe(false); + expect(check.notes).toContain(SECOND.version); + }); + + it('fails when one is missing and another is recorded without the index', () => { + const check = checkMigrationRecorded( + recorded({ missing: [NEW], mismatched: [SECOND] }) + ); + expect(check.passed).toBe(false); + expect(check.notes).toContain(NEW_VERSION); + expect(check.notes).toContain(SECOND.version); + }); + + it('fails when nothing was recorded, even with nothing missing', () => { + expect(checkMigrationRecorded(recorded()).passed).toBe(false); + }); + + it.each([ + ['the stack did not resolve', 'managed: exit 1: no stack running'], + ['no migration creates the index', 'no new migration files'], + ['psql failed', 'exit 1: psql: error: connection refused'], + ])('fails with the probe notes when %s', (_label, notes) => { + expect(checkMigrationRecorded({ ok: false, notes })).toEqual({ + name: RECORDED_CHECK_NAME, + passed: false, + notes: expect.stringContaining(notes), + }); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/migrations.ts b/evals/cli/investigate-database-001-slow-query/migrations.ts new file mode 100644 index 00000000..b333fd28 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/migrations.ts @@ -0,0 +1,287 @@ +import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core'; +import { maskSqlLiteralsAndComments } from '../lib/migrations.js'; +import { describeFailure, errorMessage, shellQuote } from '../lib/shell.js'; +import type { StackProbe } from '../lib/stack.js'; +import { + EMAIL_COLUMN, + ORDERS_TABLE, + SEEDED_MIGRATION_FILE, + SEEDED_MIGRATION_VERSION, +} from './fixtures.js'; +import { queryDatabase } from './psql.js'; + +type ExecContext = Pick; + +export type IndexStatement = { name?: string; keys: string[] }; +export type IndexMigration = { file: string; version: string }; +export type IndexMigrationsProbe = + | { ok: true; migrations: IndexMigration[]; notes: string } + | { ok: false; notes: string }; +export type RecordedProbe = + | { + ok: true; + recorded: IndexMigration[]; + missing: IndexMigration[]; + mismatched: IndexMigration[]; + } + | { ok: false; notes: string }; + +export const MIGRATION_CHECK_NAME = + 'a new migration creates an index on orders.customer_email'; +export const RECORDED_CHECK_NAME = + 'that migration is recorded in the local migration history'; + +const MIGRATION_FILENAME_RE = /^(\d+)_(.+)\.sql$/; +const IDENTIFIER = String.raw`"[^"]+"|[^\s"(),;]+`; +const NAME = String.raw`(?:[a-z_][\w$]*|"[^"]+")`; +const CREATE_INDEX_RE = new RegExp( + String.raw`^\s*create\s+(?:unique\s+)?index\s+(?:concurrently\s+)?(?:if\s+not\s+exists\s+)?(?:(${IDENTIFIER})\s+)?on\s+(?:only\s+)?(?:"?public"?\s*\.\s*)?"?${ORDERS_TABLE}"?\s*(?:using\s+(?:\w+|"[^"]+")\s*)?\(`, + 'i' +); +const EMAIL_KEY_RE = new RegExp( + String.raw`^"?${EMAIL_COLUMN}"?(?:\s+${NAME}(?:\s*\.\s*${NAME})*)*$`, + 'i' +); +const HISTORY_ROW_RE = /^(\d+)\|((?:[0-9a-f]{2})*)$/i; +const NO_NEW_MIGRATION_NOTE = `no new migration creates an index on ${ORDERS_TABLE}.${EMAIL_COLUMN}`; + +const compareText = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0); + +/** The top-level key texts of the index column list that `text` starts inside and the offset of its closing parenthesis, or undefined when it never closes. */ +function splitKeys(text: string): { keys: string[]; end: number } | undefined { + const keys: string[] = []; + let depth = 1; + let quoted = false; + let start = 0; + for (let i = 0; i < text.length; i++) { + const char = text[i]; + if (char === '"') { + quoted = !quoted; + } else if (!quoted && char === '(') { + depth++; + } else if (!quoted && char === ')' && --depth === 0) { + keys.push(text.slice(start, i)); + return { keys: keys.map((key) => key.trim()), end: i }; + } else if (!quoted && char === ',' && depth === 1) { + keys.push(text.slice(start, i)); + start = i + 1; + } + } + return undefined; +} + +/** `key` with the redundant parentheses around its leading expression removed, since `((col) desc)` indexes the plain column. */ +function unwrapLeadingParens(key: string): string { + if (!key.startsWith('(')) return key; + const group = splitKeys(key.slice(1)); + if (group === undefined || group.keys.length !== 1) return key; + return unwrapLeadingParens( + `${group.keys[0]} ${key.slice(group.end + 2)}`.trim() + ); +} + +/** The `create index` statements on `public.orders` outside comments and literals, in file order. */ +export function findOrdersIndexStatements(sql: string): IndexStatement[] { + return maskSqlLiteralsAndComments(sql) + .split(';') + .flatMap((statement): IndexStatement[] => { + const create = CREATE_INDEX_RE.exec(statement); + if (create === null) return []; + const keys = splitKeys(statement.slice(create[0].length))?.keys; + if (keys === undefined) return []; + const name = create[1]?.replace(/^"(.*)"$/, '$1'); + return [{ ...(name === undefined ? {} : { name }), keys }]; + }); +} + +/** Whether `sql` creates an index on `public.orders` whose first key is the bare `customer_email` column. */ +export function createsEmailIndex(sql: string): boolean { + return findOrdersIndexStatements(sql).some( + ({ keys }) => + keys.length > 0 && EMAIL_KEY_RE.test(unwrapLeadingParens(keys[0])) + ); +} + +export const describeMigrations = (migrations: readonly IndexMigration[]) => + migrations + .map(({ file, version }) => `${file} (version ${version})`) + .join(', '); + +function describeNone( + newFiles: readonly string[], + seededCreates: boolean, + earlier: readonly string[] +): string { + return [ + newFiles.length === 0 + ? 'no new migration files' + : `none of ${newFiles.join(', ')} creates an index on ${ORDERS_TABLE}.${EMAIL_COLUMN}`, + ...(seededCreates + ? [`only the seeded migration ${SEEDED_MIGRATION_FILE} creates it`] + : []), + ...earlier.map( + (file) => + `${file} creates it but is dated at or before the seeded version ${SEEDED_MIGRATION_VERSION}, so a reset runs it before ${ORDERS_TABLE} exists` + ), + ].join('; '); +} + +/** Finds the migration files newer than the seeded one that create an index on `orders.customer_email`. */ +export async function findEmailIndexMigrations( + ctx: ExecContext +): Promise { + try { + const listing = await ctx.exec('ls supabase/migrations 2>/dev/null | sort'); + if (!listing.ok || listing.stdout.trim() === '') { + return { + ok: false, + notes: 'no migration files found under supabase/migrations', + }; + } + + const newFiles: IndexMigration[] = []; + const found: IndexMigration[] = []; + const earlier: string[] = []; + let seededCreates = false; + for (const line of listing.stdout.split('\n')) { + const file = line.trim(); + const version = MIGRATION_FILENAME_RE.exec(file)?.[1]; + if (version === undefined) continue; + const result = await ctx.exec( + `cat ${shellQuote(`supabase/migrations/${file}`)}` + ); + if (!result.ok) { + return { + ok: false, + notes: `could not read ${file}: ${describeFailure(result)}`, + }; + } + const creates = createsEmailIndex(result.stdout); + if (file === SEEDED_MIGRATION_FILE) { + seededCreates = creates; + } else if (version > SEEDED_MIGRATION_VERSION) { + newFiles.push({ file, version }); + if (creates) found.push({ file, version }); + } else if (creates) { + earlier.push(file); + } + } + + const byVersion = (a: IndexMigration, b: IndexMigration) => + compareText(a.version, b.version) || compareText(a.file, b.file); + found.sort(byVersion); + return { + ok: true, + migrations: found, + notes: + found.length > 0 + ? describeMigrations(found) + : describeNone( + newFiles.map(({ file }) => file), + seededCreates, + earlier + ), + }; + } catch (error) { + return { ok: false, notes: errorMessage(error) }; + } +} + +export function checkNewMigrationCreatesIndex( + probe: IndexMigrationsProbe +): CheckResult { + const name = MIGRATION_CHECK_NAME; + if (!probe.ok || probe.migrations.length === 0) { + return { name, passed: false, notes: probe.notes }; + } + return { name, passed: true, notes: describeMigrations(probe.migrations) }; +} + +type HistoryProbe = + | { ok: true; history: Map } + | { ok: false; notes: string }; + +function parseHistory(stdout: string): HistoryProbe { + const history = new Map(); + for (const line of stdout.split('\n')) { + const row = line.trim(); + if (row === '') continue; + const match = HISTORY_ROW_RE.exec(row); + if (match === null) { + return { + ok: false, + notes: `unrecognised migration history row: ${row.slice(0, 120)}`, + }; + } + history.set(match[1], Buffer.from(match[2], 'hex').toString('utf8')); + } + return { ok: true, history }; +} + +/** Reads which of the found migrations the local migration history holds with a statement that creates the index. */ +export async function readRecordedIndexMigrations( + ctx: ExecContext, + stack: StackProbe, + probe: IndexMigrationsProbe +): Promise { + if (!stack.ok) return { ok: false, notes: stack.notes }; + if (!probe.ok) return { ok: false, notes: probe.notes }; + const { migrations } = probe; + if (migrations.length === 0) { + return { ok: false, notes: NO_NEW_MIGRATION_NOTE }; + } + const versions = migrations.map(({ version }) => `'${version}'`).join(', '); + const read = await queryDatabase( + ctx, + stack, + `select version || '|' || encode(convert_to(coalesce(array_to_string(statements, ';' || chr(10)), ''), 'UTF8'), 'hex') from supabase_migrations.schema_migrations where version in (${versions})`, + parseHistory + ); + if (!read.ok) return read; + + const recorded: IndexMigration[] = []; + const missing: IndexMigration[] = []; + const mismatched: IndexMigration[] = []; + for (const migration of migrations) { + const statements = read.history.get(migration.version); + if (statements === undefined) { + missing.push(migration); + } else { + (createsEmailIndex(statements) ? recorded : mismatched).push(migration); + } + } + return { ok: true, recorded, missing, mismatched }; +} + +export const describeMissingVersions = (missing: readonly IndexMigration[]) => + `version(s) ${missing.map(({ version }) => version).join(', ')} not in supabase_migrations.schema_migrations`; + +export const describeMismatchedVersions = ( + mismatched: readonly IndexMigration[] +) => + `version(s) ${mismatched.map(({ version }) => version).join(', ')} recorded with statements that do not create an index on ${ORDERS_TABLE}.${EMAIL_COLUMN}`; + +export function checkMigrationRecorded(probe: RecordedProbe): CheckResult { + const name = RECORDED_CHECK_NAME; + if (!probe.ok) return { name, passed: false, notes: probe.notes }; + const problems = [ + ...(probe.missing.length > 0 + ? [describeMissingVersions(probe.missing)] + : []), + ...(probe.mismatched.length > 0 + ? [describeMismatchedVersions(probe.mismatched)] + : []), + ]; + if (problems.length > 0 || probe.recorded.length === 0) { + return { + name, + passed: false, + notes: problems.join('; ') || 'no migration is recorded', + }; + } + return { + name, + passed: true, + notes: `${describeMigrations(probe.recorded)} ${probe.recorded.length === 1 ? 'is' : 'are'} in supabase_migrations.schema_migrations with the index statement`, + }; +} diff --git a/evals/cli/investigate-database-001-slow-query/orders.test.ts b/evals/cli/investigate-database-001-slow-query/orders.test.ts new file mode 100644 index 00000000..93dec8c2 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/orders.test.ts @@ -0,0 +1,836 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import { execFileSync } from 'node:child_process'; +import type { CommandResult } from '@supabase-evals/core'; +import { describe, expect, it, vi } from 'vitest'; +import type { StackProbe } from '../lib/stack.js'; +import { LOOKUP_QUERY, SEEDED_ORDER_COUNT } from './fixtures.js'; +import { + checkLookupUsesIndex, + checkOrdersIntact, + checkStackRunning, + countOrders, + type CountProbe, + type PlanProbe, + parsePlan, + PLAN_CHECK_NAME, + readLookupPlan, + readOrdersIndexes, + ROWS_CHECK_NAME, + STACK_CHECK_NAME, +} from './orders.js'; + +const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; + +const STACK: StackProbe = { + ok: true, + backend: 'managed', + dbUrl: DB_URL, + runtime: 'native', +}; + +const NO_STACK_NOTES = 'managed: exit 1: no stack running'; +const NO_STACK: StackProbe = { ok: false, notes: NO_STACK_NOTES }; + +function result(stdout: string, ok = true, stderr = ''): CommandResult { + return { ok, exitCode: ok ? 0 : 1, stdout, stderr }; +} + +const unquote = (arg: string) => arg.slice(1, -1).replaceAll("'\\''", "'"); + +const argvOf = (command: string): string[] => + execFileSync('sh', ['-c', `psql() { printf '%s\\0' "$@"; }\n${command}`], { + encoding: 'utf8', + }) + .split('\0') + .slice(0, -1); + +const COND = "(customer_email = 'customer-42@example.com'::text)"; + +type Fields = Record; + +const node = (type: string, fields: Fields = {}, plans: Fields[] = []) => ({ + 'Node Type': type, + 'Parallel Aware': false, + 'Async Capable': false, + 'Startup Cost': 0.42, + 'Total Cost': 12.5, + 'Plan Rows': 20, + 'Plan Width': 36, + ...fields, + ...(plans.length === 0 + ? {} + : { + Plans: plans.map((child) => ({ + 'Parent Relationship': 'Outer', + ...child, + })), + }), +}); + +const scan = (type: string, fields: Fields = {}, plans: Fields[] = []) => + node( + type, + { 'Relation Name': 'orders', Schema: 'public', Alias: 'orders', ...fields }, + plans + ); + +const explain = (root: Fields) => + `${JSON.stringify([{ Plan: root }], null, 2)}\n`; + +const SORT = { 'Sort Key': ['created_at DESC'] }; + +const SEQ_SCAN = explain( + node('Limit', {}, [node('Sort', SORT, [scan('Seq Scan', { Filter: COND })])]) +); + +const PARALLEL_SEQ_SCAN = explain( + node('Limit', {}, [ + node('Gather Merge', { 'Workers Planned': 2 }, [ + node('Sort', SORT, [ + scan('Seq Scan', { 'Parallel Aware': true, Filter: COND }), + ]), + ]), + ]) +); + +const BITMAP = explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Bitmap Heap Scan', { 'Recheck Cond': COND }, [ + node('Bitmap Index Scan', { + 'Index Name': 'orders_customer_email_idx', + 'Index Cond': COND, + }), + ]), + ]), + ]) +); + +const COMPOSITE = explain( + node('Limit', {}, [ + scan('Index Scan', { + 'Scan Direction': 'Forward', + 'Index Name': 'orders_customer_email_created_at_idx', + 'Index Cond': COND, + }), + ]) +); + +const COMPOSITE_BACKWARD = explain( + node('Limit', {}, [ + scan('Index Scan', { + 'Scan Direction': 'Backward', + 'Index Name': 'orders_customer_email_created_at_idx', + 'Index Cond': COND, + }), + ]) +); + +const INDEX_ONLY = explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Index Only Scan', { + 'Scan Direction': 'Forward', + 'Index Name': 'orders_customer_email_covering_idx', + 'Index Cond': COND, + }), + ]), + ]) +); + +const HASH = explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Index Scan', { + 'Scan Direction': 'Forward', + 'Index Name': 'orders_customer_email_hash_idx', + 'Index Cond': COND, + }), + ]), + ]) +); + +const CREATED_AT_ONLY = explain( + node('Limit', {}, [ + scan('Index Scan', { + 'Scan Direction': 'Backward', + 'Index Name': 'orders_created_at_idx', + Filter: COND, + }), + ]) +); + +const PKEY_WITH_FILTER = explain( + node('Limit', {}, [ + node('Sort', SORT, [ + scan('Index Scan', { + 'Index Name': 'orders_pkey', + 'Index Cond': '(id > 0)', + Filter: COND, + }), + ]), + ]) +); + +const JOINED_TO_SCANNED_CUSTOMERS = explain( + node('Nested Loop', {}, [ + scan('Seq Scan', { 'Relation Name': 'customers', Alias: 'customers' }), + scan('Index Scan', { + 'Index Name': 'orders_customer_email_idx', + 'Index Cond': COND, + }), + ]) +); + +const index = ( + nodeType: string, + indexCond: string | undefined, + filter?: string +) => ({ + nodeType, + relation: 'orders', + index: 'orders_customer_email_idx', + indexCond, + filter, +}); + +const hex = (text: string) => Buffer.from(text, 'utf8').toString('hex'); + +type CatalogIndex = { + schema: string; + table: string; + name: string; + definition: string; +}; + +const PKEY: CatalogIndex = { + schema: 'public', + table: 'orders', + name: 'orders_pkey', + definition: + 'CREATE UNIQUE INDEX orders_pkey ON public.orders USING btree (id)', +}; +const EMAIL_INDEX: CatalogIndex = { + schema: 'public', + table: 'orders', + name: 'orders_customer_email_idx', + definition: + 'CREATE INDEX orders_customer_email_idx ON public.orders USING btree (customer_email)', +}; +const CATALOG: CatalogIndex[] = [ + PKEY, + EMAIL_INDEX, + { + schema: 'public', + table: 'orders_archive', + name: 'orders_archive_pkey', + definition: + 'CREATE UNIQUE INDEX orders_archive_pkey ON public.orders_archive USING btree (id)', + }, + { + schema: 'private', + table: 'orders', + name: 'orders_secret_idx', + definition: + 'CREATE INDEX orders_secret_idx ON private.orders USING btree (status)', + }, +]; + +// Answers like pg_indexes: only the rows the query's schemaname/tablename filters keep, hex-encoded when the query asks for hex. +function indexRows(sql: string, catalog: readonly CatalogIndex[]): string { + const schema = sql.match(/schemaname\s*=\s*'(\w+)'/)?.[1]; + const table = sql.match(/tablename\s*=\s*'(\w+)'/)?.[1]; + const asHex = /'hex'/.test(sql); + const rows = catalog + .filter( + (entry) => + (schema === undefined || entry.schema === schema) && + (table === undefined || entry.table === table) + ) + .map( + ({ name, definition }) => + `${name}|${asHex ? hex(definition) : definition}` + ); + return rows.length > 0 ? `${rows.join('\n')}\n` : ''; +} + +type Database = { + plan?: CommandResult; + rows?: CommandResult; + indexes?: CommandResult; + catalog?: CatalogIndex[]; + ready?: CommandResult; + throws?: string; +}; + +function fakeDatabase(database: Database = {}) { + const exec = vi.fn(async (command: string): Promise => { + if (database.throws !== undefined) throw new Error(database.throws); + const psql = command.match(/^psql '([^']+)' -X?tAc ([\s\S]+)$/); + if (!psql || psql[1] !== DB_URL) { + return result('', false, `unexpected command: ${command}`); + } + const sql = unquote(psql[2]); + if (sql.includes('explain (format json)')) { + return database.plan ?? result(SEQ_SCAN); + } + if (sql.includes('pg_indexes')) { + return ( + database.indexes ?? result(indexRows(sql, database.catalog ?? CATALOG)) + ); + } + if (/count\(\*\)/.test(sql)) return database.rows ?? result('300000\n'); + if (sql.includes('select 1')) return database.ready ?? result('1\n'); + return result('', false, `unexpected query: ${sql}`); + }); + return { exec }; +} + +describe('parsePlan', () => { + it('lists every node of a nested plan with its relation, index and conditions', () => { + const probe = parsePlan(BITMAP); + expect(probe.ok).toBe(true); + const nodes = probe.ok ? probe.nodes : []; + expect(nodes).toHaveLength(4); + expect(nodes).toEqual( + expect.arrayContaining([ + { nodeType: 'Limit' }, + { nodeType: 'Sort' }, + { nodeType: 'Bitmap Heap Scan', relation: 'orders' }, + { + nodeType: 'Bitmap Index Scan', + index: 'orders_customer_email_idx', + indexCond: COND, + }, + ]) + ); + }); + + it('reads a sequential scan with its filter', () => { + const probe = parsePlan(SEQ_SCAN); + expect(probe.ok && probe.nodes).toEqual( + expect.arrayContaining([ + { nodeType: 'Seq Scan', relation: 'orders', filter: COND }, + ]) + ); + }); + + it('reads a parallel sequential scan under a gather as a Seq Scan', () => { + const probe = parsePlan(PARALLEL_SEQ_SCAN); + expect(probe.ok && probe.nodes.map(({ nodeType }) => nodeType)).toEqual( + expect.arrayContaining(['Limit', 'Gather Merge', 'Sort', 'Seq Scan']) + ); + expect(probe.ok && probe.nodes).toHaveLength(4); + }); + + it('reads the index of an index scan and the condition it searches by', () => { + const probe = parsePlan(COMPOSITE); + expect(probe.ok && probe.nodes).toEqual( + expect.arrayContaining([ + { + nodeType: 'Index Scan', + relation: 'orders', + index: 'orders_customer_email_created_at_idx', + indexCond: COND, + }, + ]) + ); + }); + + it('keeps the filter and leaves the index condition empty when the index only orders rows', () => { + const probe = parsePlan(CREATED_AT_ONLY); + const found = probe.ok + ? probe.nodes.find(({ nodeType }) => nodeType === 'Index Scan') + : undefined; + expect(found).toMatchObject({ + index: 'orders_created_at_idx', + filter: COND, + }); + expect(found?.indexCond).toBeUndefined(); + }); + + it('walks the sibling branches of a join', () => { + const probe = parsePlan(JOINED_TO_SCANNED_CUSTOMERS); + expect(probe.ok && probe.nodes).toHaveLength(3); + expect(probe.ok && probe.nodes).toEqual( + expect.arrayContaining([ + { nodeType: 'Seq Scan', relation: 'customers' }, + { + nodeType: 'Index Scan', + relation: 'orders', + index: 'orders_customer_email_idx', + indexCond: COND, + }, + ]) + ); + }); + + it('reads the single-line JSON a client prints without pretty-printing', () => { + const compact = JSON.stringify([{ Plan: scan('Seq Scan') }]); + const probe = parsePlan(`${compact}\n`); + expect(probe.ok && probe.nodes).toEqual([ + { nodeType: 'Seq Scan', relation: 'orders' }, + ]); + }); + + it.each([ + ['empty output', ''], + ['whitespace only', ' \n'], + ['plain text', 'not json at all'], + ['a psql error', 'ERROR: column "customer_email" does not exist\nLINE 1:'], + ['a text-format plan', 'Limit (cost=0.00..1.00 rows=20 width=36)\n'], + ['an empty array', '[]'], + ['an array without a plan', '[{}]'], + ['null', 'null'], + ['a number', '42'], + ['an array holding null', '[null]'], + ['truncated JSON', '[{"Plan": {"Node Type": "Limit"'], + ])('reports %s as unrecognised instead of throwing', (_label, stdout) => { + expect(parsePlan(stdout)).toEqual({ + ok: false, + notes: expect.stringContaining('unrecognised plan output'), + }); + }); + + it('shows the output it could not read', () => { + const probe = parsePlan('not json at all'); + expect(probe.ok ? '' : probe.notes).toContain('not json at all'); + }); +}); + +describe('readLookupPlan', () => { + it('reports the stack failure without querying when the stack did not resolve', async () => { + const db = fakeDatabase(); + expect(await readLookupPlan(db, NO_STACK)).toEqual({ + ok: false, + notes: NO_STACK_NOTES, + }); + expect(db.exec).not.toHaveBeenCalled(); + }); + + it("explains the app's order-history lookup in the stack's database with one psql call", async () => { + const db = fakeDatabase(); + await readLookupPlan(db, STACK); + expect(db.exec).toHaveBeenCalledTimes(1); + expect(argvOf(db.exec.mock.calls[0][0])).toEqual([ + DB_URL, + '-XtAc', + `explain (format json) ${LOOKUP_QUERY}`, + ]); + }); + + it('returns the nodes of the plan psql printed', async () => { + const db = fakeDatabase({ plan: result(BITMAP) }); + const probe = await readLookupPlan(db, STACK); + expect(probe.ok && probe.nodes.map(({ nodeType }) => nodeType)).toEqual( + expect.arrayContaining(['Bitmap Index Scan']) + ); + }); + + it('reports unreadable output as a failed probe', async () => { + const db = fakeDatabase({ plan: result('psql: warning: something odd\n') }); + expect(await readLookupPlan(db, STACK)).toEqual({ + ok: false, + notes: expect.stringContaining('unrecognised plan output'), + }); + }); + + it('reports the failure when psql exits non-zero, even if it printed a plan', async () => { + const db = fakeDatabase({ + plan: result(BITMAP, false, 'psql: error: connection refused'), + }); + expect(await readLookupPlan(db, STACK)).toEqual({ + ok: false, + notes: expect.stringContaining('psql: error: connection refused'), + }); + }); + + it('reports the database error when the lookup no longer parses against the schema', async () => { + const db = fakeDatabase({ + plan: result( + '', + false, + 'ERROR: column "customer_email" does not exist\nLINE 1: ...' + ), + }); + expect(await readLookupPlan(db, STACK)).toEqual({ + ok: false, + notes: expect.stringContaining('column "customer_email" does not exist'), + }); + }); + + it('reports the error message when the command throws', async () => { + const db = fakeDatabase({ throws: 'sandbox gone' }); + expect(await readLookupPlan(db, STACK)).toEqual({ + ok: false, + notes: 'sandbox gone', + }); + }); +}); + +describe('checkLookupUsesIndex', () => { + it.each([ + [ + 'a bitmap index scan on a plain index', + BITMAP, + 'orders_customer_email_idx', + ], + [ + 'an index scan on a composite index', + COMPOSITE, + 'orders_customer_email_created_at_idx', + ], + [ + 'a backward index scan on a composite index that matches the sort', + COMPOSITE_BACKWARD, + 'orders_customer_email_created_at_idx', + ], + [ + 'an index-only scan on a covering index', + INDEX_ONLY, + 'orders_customer_email_covering_idx', + ], + ['an index scan on a hash index', HASH, 'orders_customer_email_hash_idx'], + [ + 'an index scan beside a sequential scan of another table', + JOINED_TO_SCANNED_CUSTOMERS, + 'orders_customer_email_idx', + ], + ])('passes %s and names the index', (_label, stdout, indexName) => { + const check = checkLookupUsesIndex(parsePlan(stdout)); + expect(check).toMatchObject({ name: PLAN_CHECK_NAME, passed: true }); + expect(check.notes).toContain(indexName); + }); + + it.each([ + ['no index', SEQ_SCAN], + ['a parallel sequential scan under a gather', PARALLEL_SEQ_SCAN], + ])('fails %s and names the sequential scan', (_label, stdout) => { + const check = checkLookupUsesIndex(parsePlan(stdout)); + expect(check).toMatchObject({ name: PLAN_CHECK_NAME, passed: false }); + expect(check.notes).toMatch(/seq scan/i); + expect(check.notes).toContain('orders'); + }); + + it.each([ + [ + 'an index on created_at that only filters customer_email', + CREATED_AT_ONLY, + ], + ['an index scan whose condition is on another column', PKEY_WITH_FILTER], + ])('fails %s', (_label, stdout) => { + const check = checkLookupUsesIndex(parsePlan(stdout)); + expect(check).toMatchObject({ name: PLAN_CHECK_NAME, passed: false }); + expect(check.notes).toContain('customer_email'); + }); + + it('fails an index scan beside a sequential scan of orders itself', () => { + const plan: PlanProbe = { + ok: true, + nodes: [ + { nodeType: 'Append' }, + index('Index Scan', COND), + { nodeType: 'Seq Scan', relation: 'orders', filter: COND }, + ], + }; + const check = checkLookupUsesIndex(plan); + expect(check.passed).toBe(false); + expect(check.notes).toMatch(/seq scan/i); + }); + + it('does not take a sequential scan of orders_archive for a scan of orders', () => { + const plan: PlanProbe = { + ok: true, + nodes: [ + index('Index Scan', COND), + { nodeType: 'Seq Scan', relation: 'orders_archive' }, + ], + }; + expect(checkLookupUsesIndex(plan).passed).toBe(true); + }); + + it.each([ + [ + 'a cast column', + "((customer_email)::text = 'customer-42@example.com'::text)", + ], + ['an any-list search', "(customer_email = ANY ('{a,b}'::text[]))"], + ['a bare column', 'customer_email'], + ])('passes an index condition on %s', (_label, indexCond) => { + expect( + checkLookupUsesIndex({ + ok: true, + nodes: [index('Index Scan', indexCond)], + }).passed + ).toBe(true); + }); + + it.each([ + ['a look-alike column', "(customer_emails = 'x'::text)"], + ['a column that extends the name', "(customer_email_hash = 'x'::text)"], + ['a column that prefixes the name', "(old_customer_email = 'x'::text)"], + ['another column', '(id = 1)'], + ['no condition', undefined], + ])('fails an index scan whose condition names %s', (_label, indexCond) => { + expect( + checkLookupUsesIndex({ + ok: true, + nodes: [index('Index Scan', indexCond)], + }).passed + ).toBe(false); + }); + + it.each(['Index Scan', 'Index Only Scan', 'Bitmap Index Scan'])( + 'accepts a %s node', + (nodeType) => { + expect( + checkLookupUsesIndex({ ok: true, nodes: [index(nodeType, COND)] }) + .passed + ).toBe(true); + } + ); + + it('fails a plan with no nodes at all', () => { + expect(checkLookupUsesIndex({ ok: true, nodes: [] }).passed).toBe(false); + }); + + it.each([ + ['the stack did not resolve', NO_STACK_NOTES], + ['the plan could not be read', 'unrecognised plan output: nope'], + ['psql failed', 'exit 1: psql: error: connection refused'], + ])('fails with the probe notes when %s', (_label, notes) => { + expect(checkLookupUsesIndex({ ok: false, notes })).toEqual({ + name: PLAN_CHECK_NAME, + passed: false, + notes: expect.stringContaining(notes), + }); + }); +}); + +describe('countOrders', () => { + it('reports the stack failure without querying when the stack did not resolve', async () => { + const db = fakeDatabase(); + expect(await countOrders(db, NO_STACK)).toEqual({ + ok: false, + notes: NO_STACK_NOTES, + }); + expect(db.exec).not.toHaveBeenCalled(); + }); + + it('counts the rows of public.orders with one psql call', async () => { + const db = fakeDatabase({ rows: result('300000\n') }); + expect(await countOrders(db, STACK)).toEqual({ ok: true, count: 300_000 }); + expect(db.exec).toHaveBeenCalledTimes(1); + const command = db.exec.mock.calls[0][0]; + expect(command.startsWith(`psql '${DB_URL}' -XtAc `)).toBe(true); + expect(command).toContain('select count(*) from public.orders'); + }); + + it.each([ + ['300001\n', 300_001], + [' 1000 \n', 1000], + ['0\n', 0], + ])( + 'reads %j as %d rows, so an empty table is a count and not a failure', + async (stdout, count) => { + const db = fakeDatabase({ rows: result(stdout) }); + expect(await countOrders(db, STACK)).toEqual({ ok: true, count }); + } + ); + + it.each([ + 'abc', + '', + '3.5', + '-1', + '300000 rows', + '3,000', + 'psql: warning: odd', + ])( + 'fails loudly on the unrecognised output %j instead of reading it as a count', + async (stdout) => { + const db = fakeDatabase({ rows: result(stdout) }); + const probe = await countOrders(db, STACK); + expect(probe).toEqual({ ok: false, notes: expect.any(String) }); + expect(probe.ok ? '' : probe.notes).toContain('unrecognised'); + } + ); + + it('shows the text it could not read', async () => { + const db = fakeDatabase({ rows: result('abc\n') }); + const probe = await countOrders(db, STACK); + expect(probe.ok ? '' : probe.notes).toContain('abc'); + }); + + it('reports the failure when psql exits non-zero, even if it printed a number', async () => { + const db = fakeDatabase({ + rows: result('5\n', false, 'psql: error: connection refused'), + }); + expect(await countOrders(db, STACK)).toEqual({ + ok: false, + notes: expect.stringContaining('psql: error: connection refused'), + }); + }); + + it('reports the error message when the command throws', async () => { + const db = fakeDatabase({ throws: 'sandbox gone' }); + expect(await countOrders(db, STACK)).toEqual({ + ok: false, + notes: 'sandbox gone', + }); + }); +}); + +describe('checkOrdersIntact', () => { + it.each([SEEDED_ORDER_COUNT, SEEDED_ORDER_COUNT + 1, SEEDED_ORDER_COUNT * 2])( + 'passes %d rows', + (count) => { + const check = checkOrdersIntact({ ok: true, count }); + expect(check).toMatchObject({ name: ROWS_CHECK_NAME, passed: true }); + expect(check.notes).toContain(String(count)); + } + ); + + it.each([ + [SEEDED_ORDER_COUNT - 1, 'one row short'], + [1000, 'a delete'], + [0, 'a truncate'], + ])('fails %d rows after %s and says how many are left', (count) => { + const check = checkOrdersIntact({ ok: true, count }); + expect(check).toMatchObject({ name: ROWS_CHECK_NAME, passed: false }); + expect(check.notes).toContain(String(count)); + expect(check.notes).toContain(String(SEEDED_ORDER_COUNT)); + }); + + it.each([ + ['the stack did not resolve', NO_STACK_NOTES], + ['the count was not a number', 'unrecognised count output: abc'], + ['psql failed', 'exit 1: psql: error: connection refused'], + ])('fails with the probe notes when %s', (_label, notes) => { + const probe: CountProbe = { ok: false, notes }; + expect(checkOrdersIntact(probe)).toEqual({ + name: ROWS_CHECK_NAME, + passed: false, + notes: expect.stringContaining(notes), + }); + }); +}); + +describe('readOrdersIndexes', () => { + it('reports the stack failure without querying when the stack did not resolve', async () => { + const db = fakeDatabase(); + expect(await readOrdersIndexes(db, NO_STACK)).toEqual({ + ok: false, + notes: NO_STACK_NOTES, + }); + expect(db.exec).not.toHaveBeenCalled(); + }); + + it('lists the indexes of public.orders only, with their definitions', async () => { + const db = fakeDatabase(); + expect(await readOrdersIndexes(db, STACK)).toEqual({ + ok: true, + indexes: [ + { name: PKEY.name, definition: PKEY.definition }, + { name: EMAIL_INDEX.name, definition: EMAIL_INDEX.definition }, + ], + }); + expect(db.exec).toHaveBeenCalledTimes(1); + const command = db.exec.mock.calls[0][0]; + expect(command.startsWith(`psql '${DB_URL}' -XtAc `)).toBe(true); + expect(command).toContain('pg_indexes'); + }); + + it('keeps a definition that contains the column separator intact', async () => { + const definition = + "CREATE INDEX orders_expr_idx ON public.orders USING btree (((customer_email || '|'::text)))"; + const db = fakeDatabase({ + catalog: [{ ...EMAIL_INDEX, name: 'orders_expr_idx', definition }], + }); + expect(await readOrdersIndexes(db, STACK)).toEqual({ + ok: true, + indexes: [{ name: 'orders_expr_idx', definition }], + }); + }); + + it('reports a table with no indexes as an empty list, not as unavailable', async () => { + const db = fakeDatabase({ catalog: [] }); + expect(await readOrdersIndexes(db, STACK)).toEqual({ + ok: true, + indexes: [], + }); + }); + + it.each(['psql: warning: something unexpected', 'orders_pkey'])( + 'fails loudly on the unrecognised output %j', + async (stdout) => { + const db = fakeDatabase({ indexes: result(`${stdout}\n`) }); + const probe = await readOrdersIndexes(db, STACK); + expect(probe).toEqual({ ok: false, notes: expect.any(String) }); + expect(probe.ok ? '' : probe.notes).toContain('unrecognised'); + } + ); + + it('reports the failure when psql exits non-zero', async () => { + const db = fakeDatabase({ + indexes: result('', false, 'psql: error: connection refused'), + }); + expect(await readOrdersIndexes(db, STACK)).toEqual({ + ok: false, + notes: expect.stringContaining('psql: error: connection refused'), + }); + }); + + it('reports the error message when the command throws', async () => { + const db = fakeDatabase({ throws: 'sandbox gone' }); + expect(await readOrdersIndexes(db, STACK)).toEqual({ + ok: false, + notes: 'sandbox gone', + }); + }); +}); + +describe('checkStackRunning', () => { + it('passes when the resolved stack answers select 1', async () => { + const db = fakeDatabase(); + expect(await checkStackRunning(db, STACK)).toMatchObject({ + name: STACK_CHECK_NAME, + passed: true, + }); + }); + + it('fails with the stack notes, without querying, when no stack resolved', async () => { + const db = fakeDatabase(); + expect(await checkStackRunning(db, NO_STACK)).toEqual({ + name: STACK_CHECK_NAME, + passed: false, + notes: NO_STACK_NOTES, + }); + expect(db.exec).not.toHaveBeenCalled(); + }); + + it('fails when the stack resolved but its database refuses connections', async () => { + const db = fakeDatabase({ + ready: result('', false, 'psql: error: connection refused'), + }); + expect(await checkStackRunning(db, STACK)).toEqual({ + name: STACK_CHECK_NAME, + passed: false, + notes: 'exit 1: psql: error: connection refused', + }); + }); + + it('fails when select 1 answers anything but 1', async () => { + const db = fakeDatabase({ ready: result('0\n') }); + expect((await checkStackRunning(db, STACK)).passed).toBe(false); + }); + + it('fails instead of throwing when the command throws', async () => { + const db = fakeDatabase({ throws: 'sandbox gone' }); + expect(await checkStackRunning(db, STACK)).toEqual({ + name: STACK_CHECK_NAME, + passed: false, + notes: 'sandbox gone', + }); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/orders.ts b/evals/cli/investigate-database-001-slow-query/orders.ts new file mode 100644 index 00000000..d27b13f4 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/orders.ts @@ -0,0 +1,199 @@ +import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core'; +import { truncate } from '../lib/shell.js'; +import { probeStackReady, type StackProbe } from '../lib/stack.js'; +import { + EMAIL_COLUMN, + LOOKUP_QUERY, + ORDERS_TABLE, + SEEDED_ORDER_COUNT, +} from './fixtures.js'; +import { type Failed, queryDatabase } from './psql.js'; + +type ExecContext = Pick; + +export type PlanNode = { + nodeType: string; + relation?: string; + index?: string; + indexCond?: string; + filter?: string; +}; +export type PlanProbe = { ok: true; nodes: PlanNode[] } | Failed; +export type CountProbe = { ok: true; count: number } | Failed; +export type IndexesProbe = + | { ok: true; indexes: { name: string; definition: string }[] } + | Failed; + +export const STACK_CHECK_NAME = 'local stack is running for this project'; +export const PLAN_CHECK_NAME = + 'the order-history lookup uses an index on customer_email'; +export const ROWS_CHECK_NAME = 'orders still holds every seeded row'; + +const INDEX_NODE_TYPES = new Set([ + 'Index Scan', + 'Index Only Scan', + 'Bitmap Index Scan', +]); +const NODE_FIELDS = [ + ['relation', 'Relation Name'], + ['index', 'Index Name'], + ['indexCond', 'Index Cond'], + ['filter', 'Filter'], +] as const; +const EMAIL_COLUMN_RE = new RegExp(`(? => + typeof value === 'object' && value !== null && !Array.isArray(value); + +function readNodes(plan: unknown): PlanNode[] { + if (!isRecord(plan) || typeof plan['Node Type'] !== 'string') { + throw new Error('plan node without a Node Type'); + } + const node: PlanNode = { nodeType: plan['Node Type'] }; + for (const [field, key] of NODE_FIELDS) { + const value = plan[key]; + if (typeof value === 'string') node[field] = value; + } + const children = Array.isArray(plan.Plans) ? plan.Plans : []; + return [node, ...children.flatMap(readNodes)]; +} + +/** Flattens `explain (format json)` output into its plan nodes, parent before children. */ +export function parsePlan(stdout: string): PlanProbe { + try { + const parsed: unknown = JSON.parse(stdout); + const root = + Array.isArray(parsed) && isRecord(parsed[0]) ? parsed[0].Plan : undefined; + return { ok: true, nodes: readNodes(root) }; + } catch { + return { + ok: false, + notes: `unrecognised plan output: ${truncate(stdout.trim(), MAX_ECHOED_CHARS)}`, + }; + } +} + +export const readLookupPlan = ( + ctx: ExecContext, + stack: StackProbe +): Promise => + queryDatabase(ctx, stack, `explain (format json) ${LOOKUP_QUERY}`, parsePlan); + +const scansOrdersSequentially = ({ nodeType, relation }: PlanNode) => + nodeType === 'Seq Scan' && relation === ORDERS_TABLE; + +const searchesByEmail = ({ nodeType, indexCond }: PlanNode) => + INDEX_NODE_TYPES.has(nodeType) && + indexCond !== undefined && + EMAIL_COLUMN_RE.test(indexCond); + +/** What the lookup does to `orders`: the sequential scan, else the index nodes searching by `customer_email`. */ +export function describeLookupPlan(nodes: readonly PlanNode[]): string { + if (nodes.some(scansOrdersSequentially)) return `Seq Scan on ${ORDERS_TABLE}`; + const searching = nodes.filter(searchesByEmail); + return searching.length > 0 + ? searching + .map(({ nodeType, index }) => + index === undefined ? nodeType : `${nodeType} using ${index}` + ) + .join(', ') + : `no index searches ${ORDERS_TABLE} by ${EMAIL_COLUMN}`; +} + +export function checkLookupUsesIndex(plan: PlanProbe): CheckResult { + const name = PLAN_CHECK_NAME; + if (!plan.ok) { + return { + name, + passed: false, + notes: `could not read the plan: ${plan.notes}`, + }; + } + return { + name, + passed: + !plan.nodes.some(scansOrdersSequentially) && + plan.nodes.some(searchesByEmail), + notes: describeLookupPlan(plan.nodes), + }; +} + +export const countOrders = ( + ctx: ExecContext, + stack: StackProbe +): Promise => + queryDatabase( + ctx, + stack, + `select count(*) from public.${ORDERS_TABLE}`, + (stdout): CountProbe => { + const output = stdout.trim(); + return COUNT_RE.test(output) + ? { ok: true, count: Number(output) } + : { + ok: false, + notes: `unrecognised count output: ${truncate(output, MAX_ECHOED_CHARS)}`, + }; + } + ); + +export function checkOrdersIntact(count: CountProbe): CheckResult { + const name = ROWS_CHECK_NAME; + if (!count.ok) { + return { + name, + passed: false, + notes: `could not count public.${ORDERS_TABLE}: ${count.notes}`, + }; + } + const passed = count.count >= SEEDED_ORDER_COUNT; + return { + name, + passed, + notes: passed + ? `${count.count} rows` + : `${count.count} rows left of ${SEEDED_ORDER_COUNT} seeded`, + }; +} + +function parseIndexes(stdout: string): IndexesProbe { + const indexes: { name: string; definition: string }[] = []; + for (const line of stdout.split('\n')) { + const row = line.trim(); + if (row === '') continue; + const match = INDEX_ROW_RE.exec(row); + if (match === null) { + return { + ok: false, + notes: `unrecognised index row: ${truncate(row, MAX_ECHOED_CHARS)}`, + }; + } + indexes.push({ + name: match[1], + definition: Buffer.from(match[2], 'hex').toString('utf8'), + }); + } + return { ok: true, indexes }; +} + +export const readOrdersIndexes = ( + ctx: ExecContext, + stack: StackProbe +): Promise => + queryDatabase( + ctx, + stack, + `select indexname || '|' || encode(convert_to(indexdef, 'UTF8'), 'hex') from pg_indexes where schemaname = 'public' and tablename = '${ORDERS_TABLE}' order by indexname`, + parseIndexes + ); + +export async function checkStackRunning( + ctx: ExecContext, + stack: StackProbe +): Promise { + const { ready, notes } = await probeStackReady(ctx, stack); + return { name: STACK_CHECK_NAME, passed: ready, notes }; +} diff --git a/evals/cli/investigate-database-001-slow-query/psql.test.ts b/evals/cli/investigate-database-001-slow-query/psql.test.ts new file mode 100644 index 00000000..ea91a810 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/psql.test.ts @@ -0,0 +1,41 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import { execFileSync } from 'node:child_process'; +import type { CommandResult } from '@supabase-evals/core'; +import { describe, expect, it, vi } from 'vitest'; +import type { StackProbe } from '../lib/stack.js'; +import { queryDatabase } from './psql.js'; + +const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres'; + +const STACK: StackProbe = { + ok: true, + backend: 'managed', + dbUrl: DB_URL, + runtime: 'native', +}; + +const result = (stdout: string): CommandResult => ({ + ok: true, + exitCode: 0, + stdout, + stderr: '', +}); + +const argvOf = (command: string): string[] => + execFileSync('sh', ['-c', `psql() { printf '%s\\0' "$@"; }\n${command}`], { + encoding: 'utf8', + }) + .split('\0') + .slice(0, -1); + +describe('queryDatabase', () => { + it.each([ + ['a placeholder and a double quote', `select '$1' as "x"`], + ['single quotes', "select 'it''s' as x"], + ['a backslash before a dollar', "select '\\$1' as x"], + ])('hands psql the SQL unchanged when it holds %s', async (_label, sql) => { + const exec = vi.fn(async () => result('')); + await queryDatabase({ exec }, STACK, sql, () => ({ ok: true })); + expect(argvOf(exec.mock.calls[0][0])).toEqual([DB_URL, '-XtAc', sql]); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/psql.ts b/evals/cli/investigate-database-001-slow-query/psql.ts new file mode 100644 index 00000000..26d29587 --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/psql.ts @@ -0,0 +1,26 @@ +import type { LocalStackEvalContext } from '@supabase-evals/core'; +import { describeFailure, errorMessage, shellQuote } from '../lib/shell.js'; +import type { StackProbe } from '../lib/stack.js'; + +type ExecContext = Pick; +export type Failed = { ok: false; notes: string }; + +/** Runs `sql` in the stack's database through psql, or reports why it could not. */ +export async function queryDatabase( + ctx: ExecContext, + stack: StackProbe, + sql: string, + parse: (stdout: string) => T +): Promise { + if (!stack.ok) return { ok: false, notes: stack.notes }; + try { + const result = await ctx.exec( + `psql ${shellQuote(stack.dbUrl)} -XtAc ${shellQuote(sql)}` + ); + return result.ok + ? parse(result.stdout) + : { ok: false, notes: describeFailure(result) }; + } catch (error) { + return { ok: false, notes: errorMessage(error) }; + } +} diff --git a/evals/cli/investigate-database-001-slow-query/report.test.ts b/evals/cli/investigate-database-001-slow-query/report.test.ts new file mode 100644 index 00000000..382fa2fa --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/report.test.ts @@ -0,0 +1,676 @@ +// Run: pnpm --filter @supabase-evals/framework exec vitest run --config vitest.cli-lib.config.ts evals/cli/investigate-database-001-slow-query +import { describe, expect, it } from 'vitest'; +import type { StackProbe } from '../lib/stack.js'; +import type { EvidenceCall, InspectRun } from './inspect.js'; +import type { CliFacts, ProjectFileFacts } from './metrics.js'; +import type { IndexMigrationsProbe, RecordedProbe } from './migrations.js'; +import type { CountProbe, IndexesProbe, PlanProbe } from './orders.js'; +import { describeGroundTruth } from './report.js'; + +const PROBE_NOTE = + 'every stack probe below ran with the installed CLI, so stacks the agent created through a different Supabase CLI build may not appear here'; + +const STACK: StackProbe = { + ok: true, + backend: 'managed', + dbUrl: 'postgresql://postgres:hunter2@127.0.0.1:54322/postgres', + runtime: 'native', +}; + +const NEW_FILE = '20260102000000_add_orders_customer_email_idx.sql'; +const NEW_VERSION = '20260102000000'; +const NEW_MIGRATION = { version: NEW_VERSION, file: NEW_FILE }; +const SECOND_FILE = '20260103000000_more_indexes.sql'; +const SECOND_VERSION = '20260103000000'; +const SECOND_MIGRATION = { version: SECOND_VERSION, file: SECOND_FILE }; + +const COND = "(customer_email = 'customer-42@example.com'::text)"; + +const PKEY_INDEX = { + name: 'orders_pkey', + definition: + 'CREATE UNIQUE INDEX orders_pkey ON public.orders USING btree (id)', +}; +const EMAIL_INDEX = { + name: 'orders_customer_email_idx', + definition: + 'CREATE INDEX orders_customer_email_idx ON public.orders USING btree (customer_email)', +}; + +const BITMAP_PLAN: PlanProbe = { + ok: true, + nodes: [ + { nodeType: 'Limit' }, + { nodeType: 'Sort' }, + { nodeType: 'Bitmap Heap Scan', relation: 'orders', filter: COND }, + { + nodeType: 'Bitmap Index Scan', + index: 'orders_customer_email_idx', + indexCond: COND, + }, + ], +}; + +const SEQ_SCAN_PLAN: PlanProbe = { + ok: true, + nodes: [ + { nodeType: 'Limit' }, + { nodeType: 'Sort' }, + { nodeType: 'Seq Scan', relation: 'orders', filter: COND }, + ], +}; + +type Facts = Parameters[0]; + +const FACTS: Facts = { + workspace: '/ws', + stack: STACK, + plan: BITMAP_PLAN, + indexes: { ok: true, indexes: [PKEY_INDEX, EMAIL_INDEX] }, + rows: { ok: true, count: 300_000 }, + migrations: { + ok: true, + migrations: [NEW_MIGRATION], + notes: `${NEW_FILE} creates an index on customer_email`, + }, + recorded: { + ok: true, + recorded: [NEW_MIGRATION], + missing: [], + mismatched: [], + }, + inspectRuns: [ + { + commandIndex: 1, + report: 'db outliers', + errored: false, + }, + ], + inspectEvidence: [ + { + commandIndex: 1, + kind: 'candidate', + command: 'supabase inspect db outliers --local', + output: 'public.orders', + namesOrders: true, + }, + ], + files: { seededMigrationModified: false, newMigrationFiles: 1 }, + cli: { versions: null, cliOverride: [] }, +}; + +const lines = (overrides: Partial = {}) => + describeGroundTruth({ ...FACTS, ...overrides }); + +const line = (all: string[], prefix: string) => + all.find((candidate) => candidate.startsWith(prefix)); + +const SLOW_LINE = '- what the seed made slow:'; +const PLAN_LINE = '- order-history lookup plan now:'; +const INDEXES_LINE = '- indexes on public.orders:'; +const ROWS_LINE = '- rows in public.orders:'; +const MIGRATION_LINE = + '- new migrations creating an index on orders.customer_email:'; +const RECORDED_LINE = '- recorded in the local migration history:'; +const INSPECT_LINE = '- supabase inspect reports the agent ran:'; +const SEEDED_LINE = '- seeded migration modified:'; + +const LINE_PREFIXES = [ + '- project directory:', + '- installed CLI:', + '- local stack:', + SLOW_LINE, + PLAN_LINE, + INDEXES_LINE, + ROWS_LINE, + MIGRATION_LINE, + RECORDED_LINE, + INSPECT_LINE, + SEEDED_LINE, +]; + +const run = (overrides: Partial = {}): InspectRun => ({ + commandIndex: 0, + report: 'db outliers', + errored: false, + ...overrides, +}); + +const evidence = (overrides: Partial = {}): EvidenceCall => ({ + commandIndex: 0, + kind: 'candidate', + command: 'supabase inspect db outliers --local', + output: '', + namesOrders: false, + ...overrides, +}); + +const UNPARSED = + 'a call mentioning supabase inspect that the harness could not parse into a report'; + +describe('describeGroundTruth', () => { + it('describes a run that found the cause, added the index in a migration and applied it', () => { + const all = lines(); + expect( + all.map((entry) => + LINE_PREFIXES.find((prefix) => entry.startsWith(prefix)) + ) + ).toEqual(LINE_PREFIXES); + expect(all[0]).toBe('- project directory: /ws'); + expect(all[1]).toBe(`- installed CLI: unknown; ${PROBE_NOTE}`); + expect(all[2]).toBe('- local stack: resolved: managed/native'); + expect(line(all, ROWS_LINE)).toBe(`${ROWS_LINE} 300000 (seeded 300000)`); + expect(line(all, SEEDED_LINE)).toBe(`${SEEDED_LINE} no`); + }); + + it('describes a run that did nothing, without throwing and with the cause intact', () => { + const notes = 'managed: exit 1: no stack running'; + const all = lines({ + stack: { ok: false, notes }, + plan: { ok: false, notes }, + indexes: { ok: false, notes }, + rows: { ok: false, notes }, + migrations: { ok: false, notes: 'no new migration files' }, + recorded: { ok: false, notes: 'no new migration files' }, + inspectRuns: [], + inspectEvidence: [], + files: { seededMigrationModified: null, newMigrationFiles: null }, + }); + expect(all).toContain(`- local stack: none (${notes})`); + expect(line(all, PLAN_LINE)).toBe(`${PLAN_LINE} unavailable (${notes})`); + expect(line(all, INDEXES_LINE)).toBe( + `${INDEXES_LINE} unavailable (${notes})` + ); + expect(line(all, ROWS_LINE)).toBe(`${ROWS_LINE} unavailable (${notes})`); + expect(line(all, MIGRATION_LINE)).toBe( + `${MIGRATION_LINE} none (no new migration files)` + ); + expect(line(all, RECORDED_LINE)).toBe( + `${RECORDED_LINE} unknown (no new migration files)` + ); + expect(line(all, INSPECT_LINE)).toBe(`${INSPECT_LINE} none`); + expect(line(all, SEEDED_LINE)).toBe(`${SEEDED_LINE} unknown`); + expect(line(all, SLOW_LINE)).toBe(line(lines(), SLOW_LINE)); + }); + + describe('the cause', () => { + it('names the lookup, the column, the call count, the volume and the request that was already fast', () => { + const cause = line(lines(), SLOW_LINE) ?? ''; + expect(cause).toContain('public.orders'); + expect(cause).toContain('customer_email'); + expect(cause).toContain('200'); + expect(cause).toContain('300000'); + expect(cause).toContain('primary key'); + }); + + it('names the seed bulk insert as a one-off import outside the workload', () => { + const cause = line(lines(), SLOW_LINE) ?? ''; + expect(cause).toMatch(/one-off/); + expect(cause).toMatch(/import/); + expect(cause).toContain('pg_stat_statements'); + }); + }); + + describe('the plan the lookup gets now', () => { + it.each([ + ['a bitmap scan', BITMAP_PLAN, 'orders_customer_email_idx'], + [ + 'an index scan on a composite index', + { + ok: true, + nodes: [ + { nodeType: 'Limit' }, + { + nodeType: 'Index Scan', + relation: 'orders', + index: 'orders_email_created_idx', + indexCond: COND, + }, + ], + }, + 'orders_email_created_idx', + ], + ] satisfies Array<[string, PlanProbe, string]>)( + 'names the index %s uses and no sequential scan', + (_label, plan, indexName) => { + const text = line(lines({ plan }), PLAN_LINE) ?? ''; + expect(text).toContain(indexName); + expect(text).not.toMatch(/seq scan/i); + } + ); + + it.each([ + ['a sequential scan', SEQ_SCAN_PLAN], + [ + 'a parallel sequential scan under a gather', + { + ok: true, + nodes: [ + { nodeType: 'Limit' }, + { nodeType: 'Gather Merge' }, + { nodeType: 'Sort' }, + { nodeType: 'Seq Scan', relation: 'orders', filter: COND }, + ], + }, + ], + ] satisfies Array<[string, PlanProbe]>)( + 'says orders is scanned sequentially for %s and names no index', + (_label, plan) => { + const text = line(lines({ plan }), PLAN_LINE) ?? ''; + expect(text).toMatch(/seq scan on orders/i); + expect(text).not.toContain('orders_customer_email_idx'); + } + ); + + it('does not call a sequential scan of another table a scan of orders', () => { + const plan: PlanProbe = { + ok: true, + nodes: [ + { nodeType: 'Nested Loop' }, + { nodeType: 'Seq Scan', relation: 'customers' }, + { + nodeType: 'Index Scan', + relation: 'orders', + index: 'orders_customer_email_idx', + indexCond: COND, + }, + ], + }; + expect(line(lines({ plan }), PLAN_LINE)).not.toMatch( + /seq scan on orders/i + ); + }); + + it('shows why the plan could not be read', () => { + const plan: PlanProbe = { + ok: false, + notes: 'unrecognised plan output: psql: warning', + }; + expect(line(lines({ plan }), PLAN_LINE)).toBe( + `${PLAN_LINE} unavailable (unrecognised plan output: psql: warning)` + ); + }); + }); + + describe('the indexes on orders', () => { + it('lists each index with its definition', () => { + expect(line(lines(), INDEXES_LINE)).toBe( + `${INDEXES_LINE} ${PKEY_INDEX.name}: ${PKEY_INDEX.definition}; ${EMAIL_INDEX.name}: ${EMAIL_INDEX.definition}` + ); + }); + + it('lists only the primary key for a table nobody indexed', () => { + const indexes: IndexesProbe = { ok: true, indexes: [PKEY_INDEX] }; + const text = line(lines({ indexes }), INDEXES_LINE) ?? ''; + expect(text).toContain(PKEY_INDEX.definition); + expect(text).not.toContain('customer_email'); + }); + + it('says a table with no indexes has none rather than that the probe failed', () => { + const indexes: IndexesProbe = { ok: true, indexes: [] }; + const text = line(lines({ indexes }), INDEXES_LINE) ?? ''; + expect(text).not.toContain('unavailable'); + }); + + it('shows why the indexes could not be read', () => { + const indexes: IndexesProbe = { + ok: false, + notes: 'exit 2: connection refused', + }; + expect(line(lines({ indexes }), INDEXES_LINE)).toBe( + `${INDEXES_LINE} unavailable (exit 2: connection refused)` + ); + }); + }); + + describe('the rows in orders', () => { + it.each([ + [{ ok: true, count: 300_000 }, '300000 (seeded 300000)'], + [{ ok: true, count: 300_001 }, '300001 (seeded 300000)'], + [{ ok: true, count: 1000 }, '1000 (seeded 300000)'], + [{ ok: true, count: 0 }, '0 (seeded 300000)'], + [ + { ok: false, notes: 'exit 2: connection refused' }, + 'unavailable (exit 2: connection refused)', + ], + ] satisfies Array<[CountProbe, string]>)( + 'describes the count probe %j as %s', + (rows, expected) => { + expect(line(lines({ rows }), ROWS_LINE)).toBe( + `${ROWS_LINE} ${expected}` + ); + } + ); + }); + + describe('the migrations that create the index', () => { + it('names the file and version of a migration that creates it', () => { + expect(line(lines(), MIGRATION_LINE)).toMatch( + new RegExp( + `^${MIGRATION_LINE} ${NEW_FILE} \\(version ${NEW_VERSION}\\)` + ) + ); + }); + + it('names every migration that creates it, in order', () => { + const migrations: IndexMigrationsProbe = { + ok: true, + migrations: [NEW_MIGRATION, SECOND_MIGRATION], + notes: 'two files', + }; + const text = line(lines({ migrations }), MIGRATION_LINE) ?? ''; + const first = `${NEW_FILE} (version ${NEW_VERSION})`; + const second = `${SECOND_FILE} (version ${SECOND_VERSION})`; + expect(text).toContain(first); + expect(text).toContain(second); + expect(text.indexOf(first)).toBeLessThan(text.indexOf(second)); + }); + + it('says none, with the reason, when the files were read and none creates it', () => { + const migrations: IndexMigrationsProbe = { + ok: true, + migrations: [], + notes: 'no new migration files', + }; + expect(line(lines({ migrations }), MIGRATION_LINE)).toBe( + `${MIGRATION_LINE} none (no new migration files)` + ); + }); + + it('says none, with the reason, when the files could not be read', () => { + const migrations: IndexMigrationsProbe = { + ok: false, + notes: 'no migration files found under supabase/migrations', + }; + expect(line(lines({ migrations }), MIGRATION_LINE)).toBe( + `${MIGRATION_LINE} none (no migration files found under supabase/migrations)` + ); + }); + }); + + describe('the migration history', () => { + it('says yes and names the migrations when every one is recorded', () => { + const text = line(lines(), RECORDED_LINE) ?? ''; + expect(text.startsWith(`${RECORDED_LINE} yes (`)).toBe(true); + expect(text).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + }); + + it('says no, naming the version, when none is recorded', () => { + const recorded: RecordedProbe = { + ok: true, + recorded: [], + missing: [NEW_MIGRATION], + mismatched: [], + }; + const text = line(lines({ recorded }), RECORDED_LINE) ?? ''; + expect(text.startsWith(`${RECORDED_LINE} no (`)).toBe(true); + expect(text).toContain(NEW_VERSION); + expect(text).not.toMatch(/statements/); + }); + + it('says partly, naming what is recorded and what is missing', () => { + const recorded: RecordedProbe = { + ok: true, + recorded: [NEW_MIGRATION], + missing: [SECOND_MIGRATION], + mismatched: [], + }; + const text = line(lines({ recorded }), RECORDED_LINE) ?? ''; + expect(text.startsWith(`${RECORDED_LINE} partly (`)).toBe(true); + expect(text).toContain(`${NEW_FILE} (version ${NEW_VERSION})`); + expect(text).toContain(SECOND_VERSION); + }); + + it('says no, naming the version and its statements, when what was recorded lacks the index', () => { + const recorded: RecordedProbe = { + ok: true, + recorded: [], + missing: [], + mismatched: [NEW_MIGRATION], + }; + const text = line(lines({ recorded }), RECORDED_LINE) ?? ''; + expect(text.startsWith(`${RECORDED_LINE} no (`)).toBe(true); + expect(text).toContain(NEW_VERSION); + expect(text).toMatch(/statements/); + }); + + it('says partly, naming the mismatched version, when some are recorded and one lacks the index', () => { + const recorded: RecordedProbe = { + ok: true, + recorded: [NEW_MIGRATION], + missing: [], + mismatched: [SECOND_MIGRATION], + }; + const text = line(lines({ recorded }), RECORDED_LINE) ?? ''; + expect(text.startsWith(`${RECORDED_LINE} partly (`)).toBe(true); + expect(text).toContain(SECOND_VERSION); + expect(text).toMatch(/statements/); + }); + + it('names both the missing and the mismatched versions', () => { + const recorded: RecordedProbe = { + ok: true, + recorded: [], + missing: [NEW_MIGRATION], + mismatched: [SECOND_MIGRATION], + }; + const text = line(lines({ recorded }), RECORDED_LINE) ?? ''; + expect(text).toContain(NEW_VERSION); + expect(text).toContain(SECOND_VERSION); + }); + + it('says the history is unknown when the probe could not read it', () => { + const recorded: RecordedProbe = { + ok: false, + notes: 'exit 2: connection refused', + }; + expect(line(lines({ recorded }), RECORDED_LINE)).toBe( + `${RECORDED_LINE} unknown (exit 2: connection refused)` + ); + }); + + it('never says anything was applied when the stack is down', () => { + const notes = 'managed: exit 1: no stack running'; + const all = lines({ + stack: { ok: false, notes }, + plan: { ok: false, notes }, + indexes: { ok: false, notes }, + rows: { ok: false, notes }, + recorded: { ok: false, notes }, + }); + expect(line(all, RECORDED_LINE)).toBe( + `${RECORDED_LINE} unknown (${notes})` + ); + expect(all.filter((entry) => /applied/i.test(entry))).toEqual([]); + }); + + it('never says anything was applied when the migration is not recorded', () => { + const all = lines({ + plan: SEQ_SCAN_PLAN, + recorded: { + ok: true, + recorded: [], + missing: [NEW_MIGRATION], + mismatched: [], + }, + }); + expect(all.filter((entry) => /applied/i.test(entry))).toEqual([]); + }); + }); + + describe('the inspect reports the agent ran', () => { + it('says none when it ran none', () => { + expect( + line(lines({ inspectRuns: [], inspectEvidence: [] }), INSPECT_LINE) + ).toBe(`${INSPECT_LINE} none`); + }); + + it('names the tool call and the report', () => { + expect(line(lines(), INSPECT_LINE)).toBe( + `${INSPECT_LINE} tool call 2: db outliers` + ); + }); + + it('says a report errored', () => { + const inspectRuns = [run({ errored: true })]; + const inspectEvidence = [evidence()]; + expect(line(lines({ inspectRuns, inspectEvidence }), INSPECT_LINE)).toBe( + `${INSPECT_LINE} tool call 1: db outliers (errored)` + ); + }); + + it('lists every report in the order it ran', () => { + const inspectRuns = [ + run({ commandIndex: 0, errored: true }), + run({ commandIndex: 2, report: 'db table-stats' }), + ]; + const inspectEvidence = [ + evidence({ commandIndex: 0 }), + evidence({ commandIndex: 2 }), + ]; + expect(line(lines({ inspectRuns, inspectEvidence }), INSPECT_LINE)).toBe( + `${INSPECT_LINE} tool call 1: db outliers (errored); tool call 3: db table-stats` + ); + }); + + it('lists a call mentioning inspect that parsing found no report in', () => { + expect( + line( + lines({ + inspectRuns: [], + inspectEvidence: [evidence({ commandIndex: 2 })], + }), + INSPECT_LINE + ) + ).toBe(`${INSPECT_LINE} tool call 3: ${UNPARSED}`); + }); + + it('lists parsed runs and unparsed calls together in tool call order', () => { + expect( + line( + lines({ + inspectRuns: [ + run({ commandIndex: 3, report: 'db calls' }), + run({ commandIndex: 0 }), + ], + inspectEvidence: [ + evidence({ commandIndex: 0 }), + evidence({ commandIndex: 1 }), + evidence({ commandIndex: 3 }), + ], + }), + INSPECT_LINE + ) + ).toBe( + `${INSPECT_LINE} tool call 1: db outliers; tool call 2: ${UNPARSED}; tool call 4: db calls` + ); + }); + + it('lists the parsed report of a call that parsing only partly read', () => { + expect( + line( + lines({ + inspectRuns: [run({ report: 'db table-stats' })], + inspectEvidence: [evidence()], + }), + INSPECT_LINE + ) + ).toBe(`${INSPECT_LINE} tool call 1: db table-stats`); + }); + + it.each(['db $r', 'db "$r"', 'db `echo outliers`', 'db $(echo calls)'])( + 'does not name a report parsed out of shell syntax: %s', + (report) => { + expect( + line( + lines({ + inspectRuns: [run({ report })], + inspectEvidence: [evidence()], + }), + INSPECT_LINE + ) + ).toBe(`${INSPECT_LINE} tool call 1: ${UNPARSED}`); + } + ); + + it('leaves out a later call that only mentions an output path', () => { + expect( + line( + lines({ + inspectRuns: [run()], + inspectEvidence: [ + evidence(), + evidence({ + commandIndex: 1, + kind: 'later call naming an output path', + }), + ], + }), + INSPECT_LINE + ) + ).toBe(`${INSPECT_LINE} tool call 1: db outliers`); + }); + }); + + it.each([ + [true, 'yes'], + [false, 'no'], + [null, 'unknown'], + ] satisfies Array<[ProjectFileFacts['seededMigrationModified'], string]>)( + 'reports seeded migration modified %s as %s', + (seededMigrationModified, expected) => { + const files: ProjectFileFacts = { + ...FACTS.files, + seededMigrationModified, + }; + expect(line(lines({ files }), SEEDED_LINE)).toBe( + `${SEEDED_LINE} ${expected}` + ); + } + ); + + it('says the stack was found under a relocated CLI home', () => { + const stack: StackProbe = { + ...STACK, + relocatedHome: '/tmp/agent/.supabase', + }; + expect(line(lines({ stack }), '- local stack')).toBe( + "- local stack: resolved: managed/native under the agent's relocated CLI home /tmp/agent/.supabase" + ); + }); + + it('states the installed CLI, the PATH version after the run and the runners the agent used', () => { + const cli: CliFacts = { + versions: { cliVersion: '2.117.0', cliVersionAfterRun: '2.120.0' }, + cliOverride: ['npx --yes supabase@2.120.0'], + }; + const all = lines({ cli }); + expect(all[1]).toContain( + 'installed CLI: 2.117.0; PATH supabase reported 2.120.0 after the run' + ); + expect(all[1]).toContain(PROBE_NOTE); + expect(all[2]).toBe( + '- agent ran npx --yes supabase@2.120.0; scorer uses the installed CLI' + ); + }); + + it('adds no runner line when the agent used the installed CLI', () => { + const all = lines({ + cli: { versions: { cliVersion: '2.0.0' }, cliOverride: [] }, + }); + expect(all.filter((entry) => entry.startsWith('- agent ran'))).toEqual([]); + }); + + it('adds no runner line when the runners could not be read', () => { + const all = lines({ cli: { versions: null, cliOverride: null } }); + expect(all.filter((entry) => entry.startsWith('- agent ran'))).toEqual([]); + }); + + it('starts every line with a bullet', () => { + expect(lines().every((entry) => entry.startsWith('- '))).toBe(true); + }); + + it('never carries the stack credentials', () => { + expect(lines().join('\n')).not.toContain('hunter2'); + }); +}); diff --git a/evals/cli/investigate-database-001-slow-query/report.ts b/evals/cli/investigate-database-001-slow-query/report.ts new file mode 100644 index 00000000..8f5e3a2f --- /dev/null +++ b/evals/cli/investigate-database-001-slow-query/report.ts @@ -0,0 +1,139 @@ +import { describeStack, type StackProbe } from '../lib/stack.js'; +import { SEEDED_ORDER_COUNT } from './fixtures.js'; +import type { EvidenceCall, InspectRun } from './inspect.js'; +import type { CliFacts, ProjectFileFacts } from './metrics.js'; +import { + describeMigrations, + describeMismatchedVersions, + describeMissingVersions, + type IndexMigrationsProbe, + type RecordedProbe, +} from './migrations.js'; +import { + describeLookupPlan, + type CountProbe, + type IndexesProbe, + type PlanProbe, +} from './orders.js'; + +const SLOW_CAUSE = `the order-history lookup on public.orders filtering by customer_email (one of the app's two replayed requests, 200 calls) had no index on customer_email, so every call scanned all ${SEEDED_ORDER_COUNT} rows; the other request, lookups by primary key, was already fast; the seed's single bulk insert of ${SEEDED_ORDER_COUNT} rows is the one-off import of the order history and also appears in pg_stat_statements, but it is not part of the app's workload`; + +function describeProbeCli({ versions, cliOverride }: CliFacts): string[] { + const installed = versions?.cliVersion ?? 'unknown'; + const afterRun = + versions?.cliVersionAfterRun === undefined + ? '' + : `; PATH supabase reported ${versions.cliVersionAfterRun} after the run`; + return [ + `- installed CLI: ${installed}${afterRun}; every stack probe below ran with the installed CLI, so stacks the agent created through a different Supabase CLI build may not appear here`, + ...(cliOverride !== null && cliOverride.length > 0 + ? [`- agent ran ${cliOverride.join(', ')}; scorer uses the installed CLI`] + : []), + ]; +} + +const unavailable = (notes: string) => `unavailable (${notes})`; + +function describePlan(plan: PlanProbe): string { + return plan.ok ? describeLookupPlan(plan.nodes) : unavailable(plan.notes); +} + +function describeIndexes(probe: IndexesProbe): string { + if (!probe.ok) return unavailable(probe.notes); + return probe.indexes.length > 0 + ? probe.indexes + .map(({ name, definition }) => `${name}: ${definition}`) + .join('; ') + : 'none'; +} + +function describeRows(rows: CountProbe): string { + return rows.ok + ? `${rows.count} (seeded ${SEEDED_ORDER_COUNT})` + : unavailable(rows.notes); +} + +function describeFoundMigrations(probe: IndexMigrationsProbe): string { + return probe.ok && probe.migrations.length > 0 + ? describeMigrations(probe.migrations) + : `none (${probe.notes})`; +} + +function describeRecorded(probe: RecordedProbe): string { + if (!probe.ok) return `unknown (${probe.notes})`; + const { recorded, missing, mismatched } = probe; + if (missing.length === 0 && mismatched.length === 0) { + return `yes (${describeMigrations(recorded)})`; + } + const problems = [ + ...(missing.length > 0 ? [describeMissingVersions(missing)] : []), + ...(mismatched.length > 0 ? [describeMismatchedVersions(mismatched)] : []), + ].join('; '); + return recorded.length === 0 + ? `no (${problems})` + : `partly (${describeMigrations(recorded)} recorded; ${problems})`; +} + +const UNPARSED_INSPECT = + 'a call mentioning supabase inspect that the harness could not parse into a report'; + +function describeInspectRuns( + runs: readonly InspectRun[], + evidence: readonly EvidenceCall[] +): string { + const parsed = new Set(runs.map(({ commandIndex }) => commandIndex)); + const entries = [ + ...runs.map(({ commandIndex, report, errored }) => ({ + commandIndex, + text: /[$`]/.test(report) + ? UNPARSED_INSPECT + : `${report}${errored ? ' (errored)' : ''}`, + })), + ...evidence + .filter( + ({ kind, commandIndex }) => + kind === 'candidate' && !parsed.has(commandIndex) + ) + .map(({ commandIndex }) => ({ commandIndex, text: UNPARSED_INSPECT })), + ].sort((a, b) => a.commandIndex - b.commandIndex); + if (entries.length === 0) return 'none'; + return entries + .map(({ commandIndex, text }) => `tool call ${commandIndex + 1}: ${text}`) + .join('; '); +} + +function describeSeededMigration({ + seededMigrationModified, +}: ProjectFileFacts): string { + if (seededMigrationModified === null) return 'unknown'; + return seededMigrationModified ? 'yes' : 'no'; +} + +/** Harness-observed facts about the slow lookup and its fix, for the truthful-report judge. */ +export function describeGroundTruth(facts: { + workspace: string; + stack: StackProbe; + plan: PlanProbe; + indexes: IndexesProbe; + rows: CountProbe; + migrations: IndexMigrationsProbe; + recorded: RecordedProbe; + inspectRuns: readonly InspectRun[]; + inspectEvidence: readonly EvidenceCall[]; + files: ProjectFileFacts; + cli: CliFacts; +}): string[] { + return [ + `- project directory: ${facts.workspace}`, + ...describeProbeCli(facts.cli), + `- local stack: ${describeStack(facts.stack)}`, + `- what the seed made slow: ${SLOW_CAUSE}`, + `- order-history lookup plan now: ${describePlan(facts.plan)}`, + `- indexes on public.orders: ${describeIndexes(facts.indexes)}`, + `- rows in public.orders: ${describeRows(facts.rows)}`, + `- new migrations creating an index on orders.customer_email: ${describeFoundMigrations(facts.migrations)}`, + `- recorded in the local migration history: ${describeRecorded(facts.recorded)}`, + `- supabase inspect reports the agent ran: ${describeInspectRuns(facts.inspectRuns, facts.inspectEvidence)}`, + `- seeded migration modified: ${describeSeededMigration(facts.files)}`, + ]; +}