fix(projects): honor operator cancellation and status exits

Author: mzxchandra

apps/sim/scripts/backfill-projects.ts

@@ -131,7 +131,7 @@ async function main() {
 From apps/sim: bun --no-env-file scripts/backfill-projects.ts <command> [options]\n
 plan    Read-only discovery; atomically writes a new --manifest PATH (never overwrites).\nrepair  Repair only archived environments listed in --manifest; requires --report PATH.\napply   Assign reviewed families from --manifest; requires --report PATH.\nverify  Validate the database and repair completion; requires --manifest and --report.\nstatus  Read --report; does not connect or resume.\n
 Writes require MIGRATION_DATABASE_URL pointing directly to the primary and --ack-release-drained.\nDeploy #8830 and verify old servers/workers drained first. Keep the reviewed manifest and report\nin durable private job storage. Review conflicts and repairs before repair/apply. Repair requires\nthe app runtime environment (DATABASE_URL must target the same database) for provider cleanup.\n
-Defaults: --batch-size 50 (1–50), --max-batches 100 (1–10000), --seconds 60 (1–3600), --pause-ms 100 (1–60000).\nA budget limits scheduling. Transactions use 3s statement limits and a 5s total limit on PG17+,\notherwise a 5s idle-transaction limit on PG16. Reuse the same report\nto resume; lost reports can be recreated safely from the same manifest. Changed families need a\nnew plan/report. Complete verify plus deployment evidence is required before #8590 enforcement.\nExit: 0 complete, 2 paused/incomplete, 1 failure. Never run db:migrate to launch this tool.\n`)
+Defaults: --batch-size 50 (1–50), --max-batches 100 (1–10000), --seconds 60 (1–3600), --pause-ms 100 (1–60000).\nA budget limits scheduling. Transactions use 3s statement limits and a 5s total limit on PG17+,\notherwise a 5s idle-transaction limit on PG16. Reuse the same report\nto resume; lost reports can be recreated safely from the same manifest. Changed families need a\nnew plan/report. Complete verify plus deployment evidence is required before #8590 enforcement.\nExit: 0 complete, 2 running/paused/incomplete, 1 failure. Never run db:migrate to launch this tool.\n`)
     return
   }
   const [command] = positionals
@@ -146,7 +146,9 @@ Defaults: --batch-size 50 (1–50), --max-batches 100 (1–10000), --seconds 60
     throw new Error('Manifest and report must be different files')
   if (command === 'status') {
     if (!reportPath) throw new Error('status requires --report')
-    logger.info('Project backfill status', progressSchema.parse(await readJson(reportPath)))
+    const progress = progressSchema.parse(await readJson(reportPath))
+    logger.info('Project backfill status', progress)
+    process.exitCode = progress.status === 'complete' ? 0 : progress.status === 'failed' ? 1 : 2
     return
   }
   const rawUrl = process.env.MIGRATION_DATABASE_URL
@@ -181,8 +183,10 @@ Defaults: --batch-size 50 (1–50), --max-batches 100 (1–10000), --seconds 60
   const requestStop = () => {
     stop = true
   }
-  process.on('SIGINT', requestStop)
-  process.on('SIGTERM', requestStop)
+  if (live) {
+    process.on('SIGINT', requestStop)
+    process.on('SIGTERM', requestStop)
+  }
   let lockPid: number | undefined
   let repairRuntimeLoaded = false
   async function assertLock() {

packages/db/scripts/project-contract.integration.ts

@@ -204,6 +204,20 @@ describe('Project expand/backfill/contract against PostgreSQL', () => {
           projectsCreated: 50,
           status: 'paused',
         })
+        for (const [status, exitCode] of [
+          ['running', 2],
+          ['paused', 2],
+          ['incomplete', 2],
+          ['failed', 1],
+          ['complete', 0],
+        ] as const) {
+          const checkpoint = JSON.stringify({ ...report, status })
+          await writeFile(reportPath, checkpoint)
+          const statusRun = run('status')
+          if (exitCode === 0) await statusRun
+          else await expect(statusRun).rejects.toMatchObject({ code: exitCode })
+          expect(await readFile(reportPath, 'utf8')).toBe(checkpoint)
+        }
         const first =
           await sql`SELECT id,project_id FROM workspace WHERE project_id IS NOT NULL ORDER BY id`
         await rm(reportPath)
@@ -284,6 +298,84 @@ describe('Project expand/backfill/contract against PostgreSQL', () => {
     })
   }, 60000)
 
+  it.each([
+    ['plan', 'SIGINT'],
+    ['verify', 'SIGTERM'],
+  ] as const)(
+    'terminates a stalled %s scan on %s',
+    async (command, signal) => {
+      await database(async (sql, url) => {
+        const directory = await mkdtemp(join(tmpdir(), 'project-read-cancel-test-'))
+        const artifacts = [
+          '--manifest',
+          join(directory, 'manifest.json'),
+          '--report',
+          join(directory, 'report.json'),
+        ]
+        const options = {
+          cwd: new URL('../../../apps/sim/', import.meta.url),
+          env: { ...process.env, MIGRATION_DATABASE_URL: url },
+          timeout: 15000,
+        }
+        await sql`INSERT INTO workspace (id,name,owner_id) VALUES ('root','Root','owner')`
+        if (command === 'verify') {
+          for (const step of ['plan', 'apply'])
+            await promisify(execFile)(
+              'bun',
+              [
+                '--no-env-file',
+                'scripts/backfill-projects.ts',
+                step,
+                ...artifacts,
+                '--ack-release-drained',
+              ],
+              options
+            )
+        }
+        const files = await readdir(directory)
+        const contents = await Promise.all(
+          files.map((file) => readFile(join(directory, file), 'utf8'))
+        )
+        await sql.unsafe(`
+        ALTER TABLE workspace RENAME TO workspace_fixture_source;
+        CREATE FUNCTION pause_fixture_scan() RETURNS boolean LANGUAGE plpgsql AS $$
+          BEGIN PERFORM pg_sleep(5); RETURN true; END $$;
+        CREATE VIEW workspace AS SELECT * FROM workspace_fixture_source WHERE pause_fixture_scan();
+      `)
+        const child = execFile(
+          'bun',
+          ['--no-env-file', 'scripts/backfill-projects.ts', command, ...artifacts],
+          options
+        )
+        const exited = once(child, 'close')
+        try {
+          await expect
+            .poll(
+              async () => {
+                const [state] = await sql`SELECT count(*)::int AS waiting FROM pg_stat_activity
+                WHERE datname = current_database() AND application_name = 'sim-project-backfill'
+                  AND wait_event = 'PgSleep'`
+                return state.waiting
+              },
+              { timeout: 5000, interval: 20 }
+            )
+            .toBe(1)
+          child.kill(signal)
+          expect((await exited)[1]).toBe(signal)
+          expect(await readdir(directory)).toEqual(files)
+          expect(
+            await Promise.all(files.map((file) => readFile(join(directory, file), 'utf8')))
+          ).toEqual(contents)
+        } finally {
+          if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL')
+          await exited
+          await rm(directory, { recursive: true, force: true })
+        }
+      })
+    },
+    20000
+  )
+
   it('refuses a generated plan exceeding the artifact size limit before publishing it', async () => {
     await database(async (sql, url) => {
       const directory = await mkdtemp(join(tmpdir(), 'project-plan-size-test-'))