refactor(evals): share migration-history, psql and stack-lifecycle helpers in cli/lib

evals/cli/build-database-005-apply-schema-change/EVAL.test.ts

@@ -107,7 +107,7 @@ function fakeCtx(toolCalls: ToolCallRecord[], world: World = {}) {
       return result('', false, 'no stack running');
     }
 
-    const psql = command.match(/^psql '([^']+)' -tAc ([\s\S]+)$/);
+    const psql = command.match(/^psql '([^']+)' -X?tAc ([\s\S]+)$/);
     if (psql) {
       const sql = psql[2];
       if (psql[1] !== DB_URL || stack !== 'up') {

evals/cli/build-database-005-apply-schema-change/migrations.test.ts

@@ -1248,7 +1248,7 @@ function fakeHistory({
         ? result(files[cat[1]])
         : result('', false, `cat: ${cat[1]}: No such file or directory`);
     }
-    const psql = command.match(/^psql '([^']+)' -tAc ([\s\S]+)$/);
+    const psql = command.match(/^psql '([^']+)' -X?tAc ([\s\S]+)$/);
     if (
       !psql ||
       psql[1] !== DB_URL ||

evals/cli/build-database-005-apply-schema-change/migrations.ts

@@ -1,5 +1,11 @@
 import type { CheckResult, LocalStackEvalContext } from '@supabase-evals/core';
-import { maskSqlLiteralsAndComments } from '../lib/migrations.js';
+import {
+  checkMigrationsRecorded,
+  maskSqlLiteralsAndComments,
+  readMigrationHistory,
+  sortMigrationsByHistory,
+  type SortedMigrations,
+} from '../lib/migrations.js';
 import { describeFailure, errorMessage, shellQuote } from '../lib/shell.js';
 import type { StackProbe } from '../lib/stack.js';
 import {
@@ -18,14 +24,7 @@ export type ColumnMigrationsProbe =
       notes: string;
     }
   | { ok: false; notes: string };
-export type RecordedMigrationsProbe =
-  | {
-      ok: true;
-      recorded: ColumnMigration[];
-      missing: ColumnMigration[];
-      mismatched: ColumnMigration[];
-    }
-  | { ok: false; notes: string };
+export type RecordedMigrationsProbe = SortedMigrations<ColumnMigration>;
 
 export const NEW_MIGRATION_CHECK_NAME =
   'a new migration adds due_date to tasks as a nullable date or timestamp';
@@ -37,7 +36,6 @@ const ACCEPTED_TYPE_RE =
   /^(?:"?pg_catalog"?\s*\.\s*)?"?(?:date|timestamptz|timestamp)"?(?:\s*\(\s*\d+\s*\))?(?:\s+(?:with|without)\s+time\s+zone)?(?![\w$"])/i;
 const ARRAY_SUFFIX_RE = /^\s*(?:\[|array(?![\w$"]))/i;
 const NOT_NULL_RE = /\bnot\s+null\b/i;
-const HISTORY_ROW_RE = /^(\d+)\|((?:[0-9a-f]{2})*)$/i;
 const GENERATED_RE = /\bgenerated\b/i;
 const NO_NEW_MIGRATION_NOTE = `no new migration adds ${DUE_DATE_COLUMN} to ${TASKS_TABLE}`;
 const SEEDED_ONLY_NOTE = `only the seeded migration ${SEEDED_MIGRATION_FILE} adds ${DUE_DATE_COLUMN}; no new migration does`;
@@ -443,9 +441,6 @@ export function checkNewMigrationAddsDueDate(
   };
 }
 
-export const describeMissingVersions = (missing: readonly ColumnMigration[]) =>
-  `version(s) ${missing.map(({ version }) => version).join(', ')} not in supabase_migrations.schema_migrations`;
-
 export const describeMismatchedVersions = (
   mismatched: readonly ColumnMigration[]
 ) =>
@@ -472,69 +467,31 @@ export async function readRecordedMigrations(
   if (migrations.length === 0) {
     return { ok: false, notes: `no new migration touches ${DUE_DATE_COLUMN}` };
   }
-  try {
-    const versions = migrations.map(({ version }) => `'${version}'`);
-    const result = await ctx.exec(
-      `psql ${shellQuote(stack.dbUrl)} -tAc "select version || '|' || encode(convert_to(coalesce(array_to_string(statements, ';' || chr(10)), ''), 'UTF8'), 'hex') from supabase_migrations.schema_migrations where version in (${versions.join(', ')})"`
-    );
-    if (!result.ok) return { ok: false, notes: describeFailure(result) };
-    const history = new Map<string, string>();
-    for (const line of result.stdout.split('\n')) {
-      const trimmed = line.trim();
-      if (trimmed === '') continue;
-      const row = HISTORY_ROW_RE.exec(trimmed);
-      if (row === null) {
-        return {
-          ok: false,
-          notes: `unrecognised migration history row: ${trimmed.slice(0, 120)}`,
-        };
-      }
-      history.set(row[1], Buffer.from(row[2], 'hex').toString('utf8'));
-    }
-
-    const recorded: ColumnMigration[] = [];
-    const missing: ColumnMigration[] = [];
-    const mismatched: ColumnMigration[] = [];
-    for (const migration of migrations) {
-      const statements = history.get(migration.version);
-      if (statements === undefined) {
-        missing.push(migration);
-        continue;
-      }
-      const file = await ctx.exec(
-        `cat ${shellQuote(`supabase/migrations/${migration.file}`)}`
+  const read = await readMigrationHistory(
+    ctx,
+    stack,
+    migrations.map(({ version }) => version)
+  );
+  if (!read.ok) return read;
+  return sortMigrationsByHistory(
+    migrations,
+    read.history,
+    async (statements, { file }) => {
+      const current = await ctx.exec(
+        `cat ${shellQuote(`supabase/migrations/${file}`)}`
       );
-      if (!file.ok) return { ok: false, notes: describeFailure(file) };
-      const matches =
-        actionSignature(statements) === actionSignature(file.stdout);
-      (matches ? recorded : mismatched).push(migration);
+      if (!current.ok) throw new Error(describeFailure(current));
+      return actionSignature(statements) === actionSignature(current.stdout);
     }
-    return { ok: true, recorded, missing, mismatched };
-  } catch (error) {
-    return { ok: false, notes: errorMessage(error) };
-  }
+  );
 }
 
 /** Passes when every migration that touches the column is recorded in the CLI's migration history with the due_date statements the file holds now. */
 export function checkMigrationRecorded(
   probe: RecordedMigrationsProbe
 ): CheckResult {
-  const name = MIGRATION_RECORDED_CHECK_NAME;
-  if (!probe.ok) return { name, passed: false, notes: probe.notes };
-  const problems = [
-    ...(probe.missing.length > 0
-      ? [describeMissingVersions(probe.missing)]
-      : []),
-    ...(probe.mismatched.length > 0
-      ? [describeMismatchedVersions(probe.mismatched)]
-      : []),
-  ];
-  if (problems.length > 0 || probe.recorded.length === 0) {
-    return { name, passed: false, notes: problems.join('; ') };
-  }
-  return {
-    name,
-    passed: true,
-    notes: `${describeMigrations(probe.recorded)} ${probe.recorded.length === 1 ? 'is' : 'are'} in supabase_migrations.schema_migrations`,
-  };
+  return checkMigrationsRecorded(MIGRATION_RECORDED_CHECK_NAME, probe, {
+    describeRecorded: describeMigrations,
+    describeMismatched: describeMismatchedVersions,
+  });
 }

evals/cli/build-database-005-apply-schema-change/report.ts

@@ -1,9 +1,9 @@
 import { serializeTranscript, type TranscriptPart } from '@supabase-evals/core';
+import { describeMissingVersions } from '../lib/migrations.js';
 import { describeStack, type StackProbe } from '../lib/stack.js';
 import type { CliFacts, ProjectFileFacts } from './metrics.js';
 import {
   describeMigrations,
-  describeMissingVersions,
   type ColumnMigrationsProbe,
   type RecordedMigrationsProbe,
 } from './migrations.js';

evals/cli/build-database-006-regenerate-types/EVAL.ts

@@ -6,7 +6,11 @@ import {
   type LocalStackScorer,
 } from '@supabase-evals/core';
 import { stripIndent } from 'common-tags';
-import { findSupabaseInvocations } from '../lib/cli-invocations.js';
+import {
+  findSupabaseInvocations,
+  readStackLifecycle,
+  type StackLifecycle,
+} from '../lib/cli-invocations.js';
 import {
   DETOUR_CHECK_NAME,
   detourJudgeRubric,
@@ -30,10 +34,8 @@ import {
   countGenTypesRuns,
   listSilentFailures,
   readCliFacts,
-  readStackLifecycle,
   type CliFacts,
   type GenTypesRuns,
-  type StackLifecycle,
 } from './metrics.js';
 import { describeGroundTruth } from './report.js';
 import {

evals/cli/build-database-006-regenerate-types/metrics.test.ts

@@ -6,7 +6,10 @@ import type {
   ToolCallRecord,
 } from '@supabase-evals/core';
 import { describe, expect, it } from 'vitest';
-import { findSupabaseInvocations } from '../lib/cli-invocations.js';
+import {
+  findSupabaseInvocations,
+  type StackLifecycle,
+} from '../lib/cli-invocations.js';
 import { extractCommandEntries } from '../lib/detours.js';
 import type { StackProbe } from '../lib/stack.js';
 import {
@@ -30,9 +33,7 @@ import {
   countGenTypesRuns,
   listSilentFailures,
   readCliFacts,
-  readStackLifecycle,
   type GenTypesRuns,
-  type StackLifecycle,
 } from './metrics.js';
 
 const DB_URL = 'postgresql://postgres:postgres@127.0.0.1:54322/postgres';
@@ -543,72 +544,6 @@ describe('checkTypesRegenerated', () => {
   });
 });
 
-describe('readStackLifecycle', () => {
-  const lifecycle = (commands: string[]) =>
-    readStackLifecycle(findSupabaseInvocations(commands));
-
-  it.each([
-    ['no supabase command ran', [] as string[]],
-    [
-      'only non-lifecycle commands ran',
-      ['supabase gen types typescript --local', 'supabase migration up'],
-    ],
-  ])('reports no lifecycle command when %s', (_name, commands) => {
-    expect(lifecycle(commands)).toEqual(NO_LIFECYCLE);
-  });
-
-  it.each([
-    ['supabase start', 'supabase start'],
-    ['supabase stack start', 'supabase stack start'],
-    ['supabase stack restart', 'supabase stack restart'],
-    ['supabase db start', 'supabase db start'],
-  ])('treats %s as a start', (command, last) => {
-    expect(lifecycle([command])).toEqual({
-      lastCommand: last,
-      stoppedByAgent: false,
-    });
-  });
-
-  it.each([
-    ['supabase stop', 'supabase stop'],
-    ['supabase stop --no-backup', 'supabase stop --no-backup'],
-    [
-      'supabase stop --project-id scratch',
-      'supabase stop --project-id scratch',
-    ],
-    ['supabase stack stop', 'supabase stack stop'],
-    ['supabase stack destroy', 'supabase stack destroy'],
-  ])('treats %s as a stop', (command, last) => {
-    expect(lifecycle(['supabase start', command])).toEqual({
-      lastCommand: last,
-      stoppedByAgent: true,
-    });
-  });
-
-  it('uses the last lifecycle command when the agent stopped and then started again', () => {
-    expect(
-      lifecycle(['supabase start', 'supabase stop', 'supabase start'])
-    ).toEqual({ lastCommand: 'supabase start', stoppedByAgent: false });
-  });
-
-  it('ignores commands that follow the last lifecycle command', () => {
-    expect(
-      lifecycle([
-        'supabase start',
-        'supabase stop',
-        'supabase gen types typescript --local',
-      ])
-    ).toEqual({ lastCommand: 'supabase stop', stoppedByAgent: true });
-  });
-
-  it('does not count a stop that was only echoed', () => {
-    expect(lifecycle(['supabase start', 'echo "supabase stop"'])).toEqual({
-      lastCommand: 'supabase start',
-      stoppedByAgent: false,
-    });
-  });
-});
-
 describe('checkMetrics', () => {
   it('reports every documented field and always passes', async () => {
     const commands = [

evals/cli/build-database-006-regenerate-types/metrics.ts

@@ -7,6 +7,7 @@ import type {
 import {
   invocationVerb,
   listCliOverrides,
+  type StackLifecycle,
   type SupabaseInvocation,
 } from '../lib/cli-invocations.js';
 import {
@@ -188,46 +189,6 @@ export function checkTypesRegenerated({
   };
 }
 
-export type StackLifecycle = {
-  lastCommand: string | null;
-  stoppedByAgent: boolean;
-};
-
-const STACK_START_COMMANDS = new Set([
-  'start',
-  'stack start',
-  'stack restart',
-  'db start',
-]);
-const STACK_STOP_COMMANDS = new Set(['stop', 'stack stop', 'stack destroy']);
-
-function lifecycleCommand(invocation: SupabaseInvocation): string | undefined {
-  const verb = invocationVerb(invocation);
-  if (verb === undefined) return undefined;
-  const command = verb === 'db' ? `db ${subcommand(invocation)}` : verb;
-  return STACK_START_COMMANDS.has(command) || STACK_STOP_COMMANDS.has(command)
-    ? command
-    : undefined;
-}
-
-/** The last stack start or stop invocation the agent ran, flags included, and whether it left the stack stopped. */
-export function readStackLifecycle(
-  invocations: readonly SupabaseInvocation[]
-): StackLifecycle {
-  const last = invocations
-    .flatMap((invocation) => {
-      const command = lifecycleCommand(invocation);
-      return command === undefined ? [] : [{ command, argv: invocation.argv }];
-    })
-    .at(-1);
-  return last === undefined
-    ? { lastCommand: null, stoppedByAgent: false }
-    : {
-        lastCommand: last.argv.join(' '),
-        stoppedByAgent: STACK_STOP_COMMANDS.has(last.command),
-      };
-}
-
 export type CliFacts = {
   versions: Awaited<ReturnType<typeof readCliVersions>> | null;
   cliOverride: string[] | null;

evals/cli/build-database-006-regenerate-types/report.test.ts

@@ -2,7 +2,10 @@
 import type { ToolCallRecord } from '@supabase-evals/core';
 import { describe, expect, it } from 'vitest';
 import type { StackProbe } from '../lib/stack.js';
-import { findSupabaseInvocations } from '../lib/cli-invocations.js';
+import {
+  findSupabaseInvocations,
+  type StackLifecycle,
+} from '../lib/cli-invocations.js';
 import { extractCommandEntries } from '../lib/detours.js';
 import {
   analysisFixture,
@@ -15,7 +18,6 @@ import {
   listSilentFailures,
   type CliFacts,
   type GenTypesRuns,
-  type StackLifecycle,
 } from './metrics.js';
 import { describeGroundTruth } from './report.js';
 import type { SchemaProbe } from './schema.js';

evals/cli/build-database-006-regenerate-types/report.ts

@@ -1,5 +1,6 @@
+import type { StackLifecycle } from '../lib/cli-invocations.js';
 import { describeStack, type StackProbe } from '../lib/stack.js';
-import type { CliFacts, GenTypesRuns, StackLifecycle } from './metrics.js';
+import type { CliFacts, GenTypesRuns } from './metrics.js';
 import {
   type MigrationEvidence,
   NEW_MIGRATION_VERSION,

evals/cli/build-database-006-regenerate-types/schema.ts

@@ -6,6 +6,7 @@ import type {
 import {
   invocationFlag,
   invocationVerb,
+  type StackLifecycle,
   type SupabaseInvocation,
 } from '../lib/cli-invocations.js';
 import { commandToolCalls } from '../lib/detours.js';
@@ -16,7 +17,6 @@ import {
   isGenTypes,
   OUTPUT_FAILURE,
   subcommand,
-  type StackLifecycle,
 } from './metrics.js';
 
 export const NEW_MIGRATION_VERSION = '20260315000000';