fix(db): harden interrupted restore recovery across schema versions and descriptor cleanup (#1219)

main/db.ts

@@ -990,7 +990,7 @@ interface SchemaRecoveryReference {
   definitions: Map<string, string>;
 }
 
-let cachedSchemaRecoveryReference: SchemaRecoveryReference | null = null;
+const cachedSchemaRecoveryReferences = new Map<number, SchemaRecoveryReference>();
 let buildingIdealSchema = false;
 
 let recoverySchemaBuildCount = 0;
@@ -999,25 +999,35 @@ export function getRecoverySchemaBuildCount(): number {
   return recoverySchemaBuildCount;
 }
 
-function getMigrationRegistryIdentity(): string {
+export function clearRecoverySchemaReference(): void {
+  cachedSchemaRecoveryReferences.clear();
+}
+
+function getMigrationRegistryIdentity(targetVersion?: number): string {
+  const supportedVersion = MIGRATIONS[MIGRATIONS.length - 1]?.version || 0;
+  const version = targetVersion !== undefined ? targetVersion : supportedVersion;
   const last = MIGRATIONS[MIGRATIONS.length - 1];
-  return `${MIGRATIONS.length}:${last?.version ?? 0}:${last?.name ?? ''}`;
+  return `${version}:${MIGRATIONS.length}:${last?.version ?? 0}:${last?.name ?? ''}`;
 }
 
-function getRecoverySchemaReference(): SchemaRecoveryReference {
-  const currentIdentity = getMigrationRegistryIdentity();
-  if (cachedSchemaRecoveryReference && cachedSchemaRecoveryReference.identity === currentIdentity) {
-    return cachedSchemaRecoveryReference;
+function getRecoverySchemaReference(targetVersion?: number): SchemaRecoveryReference {
+  const supportedVersion = MIGRATIONS[MIGRATIONS.length - 1]?.version || 0;
+  const version = targetVersion !== undefined ? targetVersion : supportedVersion;
+  const currentIdentity = getMigrationRegistryIdentity(version);
+  const cached = cachedSchemaRecoveryReferences.get(version);
+  if (cached && cached.identity === currentIdentity) {
+    return cached;
   }
   recoverySchemaBuildCount += 1;
-  const idealDb = buildIdealSchemaDb();
+  const idealDb = buildIdealSchemaDb(version);
   try {
-    cachedSchemaRecoveryReference = {
+    const ref: SchemaRecoveryReference = {
       identity: currentIdentity,
       tableColumns: new Map(getTables(idealDb).map((table) => [table, getColumns(idealDb, table)])),
       definitions: normalizedSchemaDefinitions(idealDb),
     };
-    return cachedSchemaRecoveryReference;
+    cachedSchemaRecoveryReferences.set(version, ref);
+    return ref;
   } finally {
     idealDb.close();
   }
@@ -1061,14 +1071,20 @@ export function isHealthyDatabaseFile(
         .prepare("SELECT value FROM _flo_meta WHERE key = 'schema_version'")
         .get() as { value: string } | undefined;
     } catch {}
+
+    const supportedVersion = MIGRATIONS[MIGRATIONS.length - 1]?.version || 0;
+    if (schemaVersion < 1 || schemaVersion > supportedVersion) {
+      candidate.close();
+      return false;
+    }
+
     const tables = new Set(getTables(candidate));
-    const schemaRef = getRecoverySchemaReference();
+    const schemaRef = getRecoverySchemaReference(schemaVersion);
     const expectedSchema = schemaRef.tableColumns;
     const columnsValid = [...expectedSchema.entries()].every(([table, columns]) => {
       const available = new Set(getColumns(candidate, table));
       return columns.every((column) => available.has(column));
     });
-    const supportedVersion = MIGRATIONS[MIGRATIONS.length - 1]?.version || 0;
     const actualDefinitions = normalizedSchemaDefinitions(candidate);
     const definitionsValid =
       schemaRef.definitions.size === actualDefinitions.size &&
@@ -3854,6 +3870,15 @@ function materializeRestoreSource(
     sourcePath,
     fs.constants.O_RDONLY | ((fs.constants as any).O_NOFOLLOW || 0),
   );
+  let snapshotDir: string | null = null;
+  let cleaned = false;
+  const cleanup = () => {
+    if (cleaned || !snapshotDir) return;
+    cleaned = true;
+    try {
+      fs.rmSync(snapshotDir, { recursive: true, force: true });
+    } catch {}
+  };
   try {
     const openedStat = fs.fstatSync(sourceFd);
     if (
@@ -3879,27 +3904,20 @@ function materializeRestoreSource(
     if (pathEntryExists(`${sourcePath}-wal`) || pathEntryExists(`${sourcePath}-shm`)) {
       throw new Error('Restore source acquired SQLite sidecars while it was being read');
     }
-    const snapshotDir = fs.mkdtempSync(path.join(os.tmpdir(), 'flo-restore-source-'));
+    snapshotDir = fs.mkdtempSync(path.join(os.tmpdir(), 'flo-restore-source-'));
+    const snapshotPath = path.join(snapshotDir, 'source.db');
+    fs.writeFileSync(snapshotPath, sourceBytes, { flag: 'wx', mode: 0o600 });
+    return { snapshotPath, cleanup };
+  } catch (error) {
+    cleanup();
+    throw error;
+  } finally {
     try {
-      const snapshotPath = path.join(snapshotDir, 'source.db');
-      fs.writeFileSync(snapshotPath, sourceBytes, { flag: 'wx', mode: 0o600 });
-      let cleaned = false;
-      const cleanup = () => {
-        if (cleaned) return;
-        cleaned = true;
-        try {
-          fs.rmSync(snapshotDir, { recursive: true, force: true });
-        } catch {}
-      };
-      return { snapshotPath, cleanup };
-    } catch (writeError) {
-      try {
-        fs.rmSync(snapshotDir, { recursive: true, force: true });
-      } catch {}
-      throw writeError;
+      fs.closeSync(sourceFd);
+    } catch (closeError) {
+      cleanup();
+      throw closeError;
     }
-  } finally {
-    fs.closeSync(sourceFd);
   }
 }
 
@@ -4288,15 +4306,15 @@ export function rebuildSearchIndex(targetDb?: Database.Database): void {
 }
 
 /** Builds in-memory database using migration pipeline as ideal schema reference. */
-export function buildIdealSchemaDb(): Database.Database {
+export function buildIdealSchemaDb(targetVersion?: number): Database.Database {
   const idealDb = new Database(':memory:');
   registerOrderSearchFunctions(idealDb);
   idealDb.pragma('foreign_keys = OFF'); // Off during migrations
   const previousDb = db;
   db = idealDb;
   try {
     buildingIdealSchema = true;
-    runMigrations();
+    runMigrations(targetVersion);
   } finally {
     buildingIdealSchema = false;
     db = previousDb;
@@ -7528,9 +7546,10 @@ export class SchemaVersionMismatchError extends Error {
   }
 }
 
-function runMigrations(): void {
+function runMigrations(targetVersion?: number): void {
   const current = getCurrentSchemaVersion();
-  const target = MIGRATIONS.length > 0 ? MIGRATIONS[MIGRATIONS.length - 1].version : 0;
+  const maxSupported = MIGRATIONS.length > 0 ? MIGRATIONS[MIGRATIONS.length - 1].version : 0;
+  const target = targetVersion !== undefined ? targetVersion : maxSupported;
   const verboseMigrationLogs = process.env.FLOCAFE_TEST_VERBOSE === '1';
 
   if (current > target) {
@@ -7539,27 +7558,30 @@ function runMigrations(): void {
   }
 
   if (current === target) {
-    console.log(`[DB] Schema up to date (v${current})`);
+    if (!buildingIdealSchema) console.log(`[DB] Schema up to date (v${current})`);
     return;
   }
 
-  console.log(`[DB] Schema: v${current} → v${target}`);
+  if (!buildingIdealSchema) console.log(`[DB] Schema: v${current} → v${target}`);
 
   // Create automatic pre-migration backup before running pending batch.
-  console.log(`[DB] Triggering auto-backup before migrating v${current} → v${target}...`);
-  syncBackupBeforeMigration(current, target);
+  if (!buildingIdealSchema) {
+    console.log(`[DB] Triggering auto-backup before migrating v${current} → v${target}...`);
+    syncBackupBeforeMigration(current, target);
+  }
 
   for (const migration of MIGRATIONS) {
     if (migration.version <= current) continue;
+    if (migration.version > target) break;
 
-    if (verboseMigrationLogs) {
+    if (verboseMigrationLogs && !buildingIdealSchema) {
       console.log(`[DB] Applying migration v${migration.version}: ${migration.name}`);
     }
     db.transaction(() => {
       migration.up();
       db.pragma(`user_version = ${migration.version}`);
     })();
-    if (verboseMigrationLogs) {
+    if (verboseMigrationLogs && !buildingIdealSchema) {
       console.log(`[DB] Migration v${migration.version} complete`);
     }
   }

tests/backup-restore-production.test.ts

@@ -34,6 +34,7 @@ import {
   initDatabase,
   restoreBackup,
   validateInventoryLedgerDatabase,
+  buildIdealSchemaDb,
 } from '../main/db';
 import { adjustProductStock } from '../main/services/inventory';
 
@@ -899,10 +900,93 @@ async function run() {
       } finally {
         (nativeFs as any).writeFileSync = originalWriteFileSync;
       }
+
+      // 5. Failed materialization descriptor close (e.g. closeSync error): temp dir is cleaned up immediately, no leak
+      const originalCloseSync = nativeFs.closeSync;
+      const failedCloseStart = trackedRestoreDirs.length;
+      let injectCloseSyncFailure = false;
+      (nativeFs as any).closeSync = (fd: any) => {
+        if (injectCloseSyncFailure) {
+          originalCloseSync(fd);
+          throw new Error('injected snapshot closeSync failure');
+        }
+        return originalCloseSync(fd);
+      };
+      try {
+        injectCloseSyncFailure = true;
+        const failedClose = restoreBackup(sameSchemaBackup, true);
+        assert.equal(
+          failedClose.success,
+          false,
+          'restore fails gracefully on materialization closeSync failure',
+        );
+        assert.match(
+          failedClose.error ?? '',
+          /injected snapshot closeSync failure/,
+          'error matches injected failure',
+        );
+        const failedDirs = trackedRestoreDirs.slice(failedCloseStart);
+        assert.equal(
+          failedDirs.length,
+          1,
+          'materialize created a temp directory before closeSync failure',
+        );
+        assert.equal(
+          fs.existsSync(failedDirs[0]),
+          false,
+          'failed materialization closeSync cleaned up its temp directory',
+        );
+      } finally {
+        injectCloseSyncFailure = false;
+        (nativeFs as any).closeSync = originalCloseSync;
+      }
     } finally {
       (nativeFs as any).mkdtempSync = originalMkdtempSync;
     }
 
+    // ── 6. Interrupted restore recovery across schema versions ───────────
+    console.log('Testing interrupted restore recovery across historical schema versions...');
+    closeDatabase();
+
+    const historicalRecoveryId = 'hist-v114-recovery';
+    const backupDir = path.join(testDir, 'backups');
+    fs.mkdirSync(backupDir, { recursive: true });
+
+    const v114DbPath = path.join(backupDir, `flo-restore-recovery-${historicalRecoveryId}.db`);
+    const v114IdealDb = buildIdealSchemaDb(114);
+    try {
+      await v114IdealDb.backup(v114DbPath);
+    } finally {
+      v114IdealDb.close();
+    }
+
+    const journalPath = path.join(backupDir, `flo-restore-recovery-${historicalRecoveryId}.json`);
+    const liveDbPath = getDbPath();
+    const journalData = {
+      phase: 'prepared',
+      recoveryPath: v114DbPath,
+      dbPath: liveDbPath,
+    };
+    fs.writeFileSync(journalPath, JSON.stringify(journalData));
+    fs.writeFileSync(liveDbPath, 'corrupted data from interrupted write');
+
+    initDatabase();
+    assert.equal(
+      getCurrentSchemaVersion(),
+      115,
+      'interrupted v114 replacement recovered and migrated to v115',
+    );
+    assert.equal(
+      fs.existsSync(journalPath),
+      false,
+      'recovery journal cleaned up after successful recovery',
+    );
+    assert.equal(
+      fs.existsSync(v114DbPath),
+      false,
+      'recovery snapshot cleaned up after successful recovery',
+    );
+
     console.log('✅ Production database restore tests passed');
   } finally {
     closeDatabase();