Add Netlify DB migration tooling

docs/storage-json.md

@@ -7,8 +7,7 @@ This app has two writable storage backends:
 - Netlify deployments use Netlify Blobs in the `kid-a-data` store by default,
   seeded from committed JSON files in `server/data` or `src/data`.
 - Netlify deployments can use Netlify DB/Postgres by setting
-  `KID_A_STORE_BACKEND=db`. Magic-link tokens follow the same backend unless
-  `KID_A_TOKEN_BACKEND` is set explicitly.
+  `KID_A_STORE_BACKEND=db`. Magic-link tokens always follow the same backend.
 
 Netlify Blobs are optimized for reads and infrequent writes. Avoid
 read-modify-write on shared JSON blobs for high-frequency event data because
@@ -56,3 +55,17 @@ app-owned IDs for kids, prizes, prize awards, and magic-link token hashes; the
 database does not auto-generate user-facing IDs or maintain a separate kid ID
 counter table. The DB adapter derives prize `given` counts from `prize_awards`
 and builds passport responses from `passport_activities` rows.
+
+Apply the schema and populate Netlify DB from the current source with:
+
+```sh
+NETLIFY_DATABASE_URL=... KID_A_MIGRATION_SOURCE=file npm run db:migrate
+```
+
+Set `KID_A_MIGRATION_SOURCE=blob` to migrate from Netlify Blobs instead of local
+JSON files. The same command verifies DB parity after writing. To verify later
+without writing, run:
+
+```sh
+NETLIFY_DATABASE_URL=... KID_A_MIGRATION_SOURCE=file npm run db:verify
+```

netlify/functions/api.ts

@@ -24,19 +24,14 @@ function definedHeaders(headers: NetlifyEvent['headers']) {
 
 function configureStores() {
   const storeBackend = process.env.KID_A_STORE_BACKEND ?? 'blob';
-  const tokenBackend = process.env.KID_A_TOKEN_BACKEND ?? storeBackend;
 
   if (storeBackend !== 'blob' && storeBackend !== 'db') {
     throw new Error('KID_A_STORE_BACKEND must be blob or db');
   }
 
-  if (tokenBackend !== 'blob' && tokenBackend !== 'db') {
-    throw new Error('KID_A_TOKEN_BACKEND must be blob or db');
-  }
-
   setStoreAdapter(storeBackend === 'db' ? createDbStore() : createBlobStore());
   setMagicTokenStore(
-    tokenBackend === 'db' ? createDbMagicTokenStore() : createBlobMagicTokenStore(),
+    storeBackend === 'db' ? createDbMagicTokenStore() : createBlobMagicTokenStore(),
   );
 }
 

package.json

@@ -10,6 +10,9 @@
     "build:netlify": "VITE_DATA_LAYER=remote VITE_BASE_PATH=/ VITE_API_BASE_URL=/api npm run build && npm run build:server",
     "build:node": "VITE_DATA_LAYER=remote VITE_BASE_PATH=/ VITE_API_BASE_URL=/api npm run build && npm run build:server",
     "build:server": "tsc -p tsconfig.server.json",
+    "db:apply-schema": "npm run build:server && node dist-server/server/db-migrate.js apply-schema",
+    "db:migrate": "npm run build:server && node dist-server/server/db-migrate.js migrate",
+    "db:verify": "npm run build:server && node dist-server/server/db-migrate.js verify",
     "lint": "eslint .",
     "preview": "vite preview",
     "start:node": "node dist-server/server/index.js"

server/access-tokens.ts

@@ -66,7 +66,7 @@ function toSession(token: MagicLinkTokenRecord): MagicLinkSession {
   };
 }
 
-function createFileMagicTokenStore(filePath = defaultMagicTokensFile) {
+export function createFileMagicTokenStore(filePath = defaultMagicTokensFile) {
   return {
     async readTokens() {
       try {

server/db-migrate.ts

@@ -0,0 +1,275 @@
+import { createHash } from 'node:crypto';
+import { readdir, readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { neon } from '@netlify/neon';
+import {
+  createBlobMagicTokenStore,
+  createFileMagicTokenStore,
+  type MagicLinkTokenRecord,
+  type MagicLinkTokenStore,
+} from './access-tokens.js';
+import { createBlobStore } from './blob-store.js';
+import { createDbMagicTokenStore, createDbStore } from './db-store.js';
+import { createFileStore, syncPrizeGivenCache, type StoreAdapter } from './store.js';
+import type { StoreData } from './types.js';
+
+type MigrationSource = 'blob' | 'file';
+type Command = 'apply-schema' | 'migrate' | 'verify';
+type SqlClient = ReturnType<typeof neon>;
+
+type ComparableData = {
+  conference: StoreData['conference'];
+  kids: StoreData['kids'];
+  passportActivitiesByKid: StoreData['passportActivitiesByKid'];
+  prizeAwards: StoreData['prizeAwards'];
+  prizes: StoreData['prizes'];
+};
+
+const migrationsDir = path.resolve('db/migrations');
+const validCommands = new Set<Command>(['apply-schema', 'migrate', 'verify']);
+const validSources = new Set<MigrationSource>(['blob', 'file']);
+
+function getCommand(): Command {
+  const command = process.argv[2] ?? 'migrate';
+
+  if (!validCommands.has(command as Command)) {
+    throw new Error(
+      `Unsupported command "${command}". Use apply-schema, migrate, or verify.`,
+    );
+  }
+
+  return command as Command;
+}
+
+function getSource(): MigrationSource {
+  const source = process.env.KID_A_MIGRATION_SOURCE ?? 'file';
+
+  if (!validSources.has(source as MigrationSource)) {
+    throw new Error('KID_A_MIGRATION_SOURCE must be file or blob');
+  }
+
+  return source as MigrationSource;
+}
+
+function createSourceStore(source: MigrationSource): StoreAdapter {
+  return source === 'blob' ? createBlobStore() : createFileStore();
+}
+
+function createSourceMagicTokenStore(source: MigrationSource): MagicLinkTokenStore {
+  return source === 'blob' ? createBlobMagicTokenStore() : createFileMagicTokenStore();
+}
+
+function splitSqlStatements(sqlText: string) {
+  return sqlText
+    .split(/;\s*(?:\r?\n|$)/)
+    .map((statement) => statement.trim())
+    .filter(Boolean);
+}
+
+function stableStringify(value: unknown): string {
+  if (Array.isArray(value)) {
+    return `[${value.map(stableStringify).join(',')}]`;
+  }
+
+  if (value && typeof value === 'object') {
+    return `{${Object.entries(value)
+      .sort(([left], [right]) => left.localeCompare(right))
+      .map(([key, nestedValue]) => `${JSON.stringify(key)}:${stableStringify(nestedValue)}`)
+      .join(',')}}`;
+  }
+
+  return JSON.stringify(value);
+}
+
+function hashValue(value: unknown) {
+  return createHash('sha256').update(stableStringify(value)).digest('hex');
+}
+
+function normalizeStoreData(data: StoreData): ComparableData {
+  const prizeAwards = data.prizeAwards
+    .map((award) => ({ ...award }))
+    .sort((left, right) => left.id.localeCompare(right.id));
+
+  return {
+    conference: { ...data.conference },
+    kids: data.kids
+      .map((kid) => ({ ...kid }))
+      .sort((left, right) => left.id.localeCompare(right.id)),
+    passportActivitiesByKid: Object.fromEntries(
+      Object.entries(data.passportActivitiesByKid)
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([kidId, passport]) => [
+          kidId,
+          passport
+            .map((activity) => ({ ...activity }))
+            .sort((left, right) => left.id - right.id),
+        ]),
+    ),
+    prizeAwards,
+    prizes: syncPrizeGivenCache(data.prizes, prizeAwards)
+      .map((prize) => ({ ...prize }))
+      .sort((left, right) => left.id.localeCompare(right.id)),
+  };
+}
+
+function normalizeTokens(tokens: MagicLinkTokenRecord[]) {
+  return tokens
+    .map((token) => ({ ...token }))
+    .sort((left, right) => left.tokenHash.localeCompare(right.tokenHash));
+}
+
+function summarizeStoreData(data: ComparableData) {
+  return {
+    kids: data.kids.length,
+    passportKids: Object.keys(data.passportActivitiesByKid).length,
+    passportRows: Object.values(data.passportActivitiesByKid).reduce(
+      (total, passport) => total + passport.length,
+      0,
+    ),
+    prizeAwards: data.prizeAwards.length,
+    prizes: data.prizes.length,
+  };
+}
+
+async function applySchema(sql: SqlClient) {
+  await sql`
+    CREATE TABLE IF NOT EXISTS schema_migrations (
+      id text PRIMARY KEY,
+      applied_at timestamptz NOT NULL DEFAULT now()
+    )
+  `;
+
+  const migrationFiles = (await readdir(migrationsDir))
+    .filter((fileName) => fileName.endsWith('.sql'))
+    .sort();
+
+  for (const migrationFile of migrationFiles) {
+    const existingRows = (await sql`
+      SELECT id
+      FROM schema_migrations
+      WHERE id = ${migrationFile}
+    `) as Array<{ id: string }>;
+
+    if (existingRows.length > 0) {
+      continue;
+    }
+
+    const sqlText = await readFile(path.join(migrationsDir, migrationFile), 'utf8');
+
+    for (const statement of splitSqlStatements(sqlText)) {
+      await sql.query(statement);
+    }
+
+    await sql`
+      INSERT INTO schema_migrations (id)
+      VALUES (${migrationFile})
+      ON CONFLICT DO NOTHING
+    `;
+    console.log(`Applied ${migrationFile}`);
+  }
+}
+
+async function migrateData(source: MigrationSource) {
+  const sourceStore = createSourceStore(source);
+  const sourceTokenStore = createSourceMagicTokenStore(source);
+  const dbStore = createDbStore();
+  const dbTokenStore = createDbMagicTokenStore();
+  const [snapshot, tokens] = await Promise.all([
+    sourceStore.readSnapshot(),
+    sourceTokenStore.readTokens(),
+  ]);
+
+  await dbStore.updateSnapshot(
+    (targetSnapshot) => {
+      targetSnapshot.conference = snapshot.conference;
+      targetSnapshot.kids = snapshot.kids;
+      targetSnapshot.passportActivitiesByKid = snapshot.passportActivitiesByKid;
+      targetSnapshot.prizeAwards = snapshot.prizeAwards;
+      targetSnapshot.prizes = syncPrizeGivenCache(snapshot.prizes, snapshot.prizeAwards);
+    },
+    ['conference', 'kids', 'passportActivitiesByKid', 'prizeAwards', 'prizes'],
+  );
+  await dbTokenStore.writeTokens(tokens);
+
+  console.log(
+    `Migrated ${source} data to Netlify DB: ${JSON.stringify(
+      summarizeStoreData(normalizeStoreData(snapshot)),
+    )}, tokens=${tokens.length}`,
+  );
+}
+
+async function verifyData(source: MigrationSource) {
+  const sourceStore = createSourceStore(source);
+  const sourceTokenStore = createSourceMagicTokenStore(source);
+  const dbStore = createDbStore();
+  const dbTokenStore = createDbMagicTokenStore();
+  const [sourceSnapshot, dbSnapshot, sourceTokens, dbTokens] = await Promise.all([
+    sourceStore.readSnapshot(),
+    dbStore.readSnapshot(),
+    sourceTokenStore.readTokens(),
+    dbTokenStore.readTokens(),
+  ]);
+  const sourceData = normalizeStoreData(sourceSnapshot);
+  const dbData = normalizeStoreData(dbSnapshot);
+  const sourceTokenData = normalizeTokens(sourceTokens);
+  const dbTokenData = normalizeTokens(dbTokens);
+  const checks = [
+    ['conference', sourceData.conference, dbData.conference],
+    ['kids', sourceData.kids, dbData.kids],
+    [
+      'passportActivitiesByKid',
+      sourceData.passportActivitiesByKid,
+      dbData.passportActivitiesByKid,
+    ],
+    ['prizes', sourceData.prizes, dbData.prizes],
+    ['prizeAwards', sourceData.prizeAwards, dbData.prizeAwards],
+    ['magicLinkTokens', sourceTokenData, dbTokenData],
+  ] as const;
+  const failures = checks
+    .map(([label, sourceValue, dbValue]) => ({
+      dbHash: hashValue(dbValue),
+      label,
+      sourceHash: hashValue(sourceValue),
+    }))
+    .filter((check) => check.sourceHash !== check.dbHash);
+
+  console.log(`Source summary: ${JSON.stringify(summarizeStoreData(sourceData))}`);
+  console.log(`DB summary: ${JSON.stringify(summarizeStoreData(dbData))}`);
+  console.log(`Source tokens: ${sourceTokenData.length}; DB tokens: ${dbTokenData.length}`);
+
+  if (failures.length > 0) {
+    throw new Error(
+      `Netlify DB verification failed: ${failures
+        .map(
+          (failure) =>
+            `${failure.label} source=${failure.sourceHash} db=${failure.dbHash}`,
+        )
+        .join('; ')}`,
+    );
+  }
+
+  console.log('Netlify DB verification passed');
+}
+
+async function main() {
+  const command = getCommand();
+  const source = getSource();
+  const sql = neon();
+
+  if (command === 'apply-schema' || command === 'migrate') {
+    await applySchema(sql);
+  }
+
+  if (command === 'migrate') {
+    await migrateData(source);
+  }
+
+  if (command === 'migrate' || command === 'verify') {
+    await verifyData(source);
+  }
+}
+
+main().catch((error: unknown) => {
+  console.error(error);
+  process.exitCode = 1;
+});