Enable dual writes for Blob and DB stores

docs/storage-json.md

@@ -7,8 +7,11 @@ This app has two writable storage backends:
 - Netlify deployments use Netlify Blobs in the `kid-a-data` store by default,
   seeded from committed JSON files in `server/data` or `src/data`.
 - Netlify deployments can use Netlify DB/Postgres by setting
-  `KID_A_STORE_BACKEND=db`. Magic-link tokens follow the same backend unless
-  `KID_A_TOKEN_BACKEND` is set explicitly.
+  `KID_A_STORE_BACKEND=db`.
+- Netlify deployments can read from one backend and dual-write to Blobs and DB
+  by setting `KID_A_STORE_BACKEND=dual`. `KID_A_STORE_READ=blob` keeps Blob
+  reads while shadow-writing DB; `KID_A_STORE_READ=db` reads from DB instead.
+  Magic-link tokens always follow the same backend mode as app data.
 
 Netlify Blobs are optimized for reads and infrequent writes. Avoid
 read-modify-write on shared JSON blobs for high-frequency event data because
@@ -44,9 +47,9 @@ app for DB and dual-write backends.
 
 ## Follow-up migration targets
 
-1. Move writable event state to Netlify DB/Postgres behind the store interface.
-2. Add migration and verification tooling from file/blob JSON to DB rows.
-3. Enable config-driven dual writes before switching reads to DB.
+1. Reset Blob and DB writable state from committed seed JSON.
+2. Enable config-driven dual writes with Blob reads.
+3. Switch reads to DB after live verification.
 4. Keep blobs as a rollback target until DB reads are verified.
 
 ## Netlify DB schema
@@ -56,3 +59,19 @@ app-owned IDs for kids, prizes, prize awards, and magic-link token hashes; the
 database does not auto-generate user-facing IDs or maintain a separate kid ID
 counter table. The DB adapter derives prize `given` counts from `prize_awards`
 and builds passport responses from `passport_activities` rows.
+
+The app is not in production yet, so existing Blob and DB testing data can be
+discarded. To apply the DB schema, reset both stores from committed seed JSON,
+and clear magic-link tokens in both places, run:
+
+```sh
+NETLIFY_DATABASE_URL=... npm run data:reset-stores
+```
+
+After reset, start the rollout with Blob reads and dual writes:
+
+```sh
+KID_A_STORE_BACKEND=dual
+KID_A_STORE_READ=blob
+KID_A_DUAL_WRITE_STRICT=true
+```

netlify/functions/api.ts

@@ -3,6 +3,7 @@ import { createBlobMagicTokenStore, setMagicTokenStore } from '../../server/acce
 import { handleApiRequest } from '../../server/api.js';
 import { createBlobStore } from '../../server/blob-store.js';
 import { createDbMagicTokenStore, createDbStore } from '../../server/db-store.js';
+import { createDualMagicTokenStore, createDualStore } from '../../server/dual-store.js';
 import { setStoreAdapter } from '../../server/store.js';
 
 type NetlifyEvent = {
@@ -24,19 +25,43 @@ function definedHeaders(headers: NetlifyEvent['headers']) {
 
 function configureStores() {
   const storeBackend = process.env.KID_A_STORE_BACKEND ?? 'blob';
-  const tokenBackend = process.env.KID_A_TOKEN_BACKEND ?? storeBackend;
+  const storeRead = process.env.KID_A_STORE_READ ?? 'blob';
+  const strictDualWrites = process.env.KID_A_DUAL_WRITE_STRICT !== 'false';
 
-  if (storeBackend !== 'blob' && storeBackend !== 'db') {
-    throw new Error('KID_A_STORE_BACKEND must be blob or db');
+  if (storeBackend !== 'blob' && storeBackend !== 'db' && storeBackend !== 'dual') {
+    throw new Error('KID_A_STORE_BACKEND must be blob, db, or dual');
   }
 
-  if (tokenBackend !== 'blob' && tokenBackend !== 'db') {
-    throw new Error('KID_A_TOKEN_BACKEND must be blob or db');
+  if (storeRead !== 'blob' && storeRead !== 'db') {
+    throw new Error('KID_A_STORE_READ must be blob or db');
+  }
+
+  if (storeBackend === 'dual') {
+    const blobStore = createBlobStore();
+    const dbStore = createDbStore();
+    const blobTokenStore = createBlobMagicTokenStore();
+    const dbTokenStore = createDbMagicTokenStore();
+    const primary = storeRead === 'db' ? dbStore : blobStore;
+    const secondary = storeRead === 'db' ? blobStore : dbStore;
+    const primaryTokenStore = storeRead === 'db' ? dbTokenStore : blobTokenStore;
+    const secondaryTokenStore = storeRead === 'db' ? blobTokenStore : dbTokenStore;
+
+    setStoreAdapter(
+      createDualStore({ primary, secondary, strict: strictDualWrites }),
+    );
+    setMagicTokenStore(
+      createDualMagicTokenStore({
+        primary: primaryTokenStore,
+        secondary: secondaryTokenStore,
+        strict: strictDualWrites,
+      }),
+    );
+    return;
   }
 
   setStoreAdapter(storeBackend === 'db' ? createDbStore() : createBlobStore());
   setMagicTokenStore(
-    tokenBackend === 'db' ? createDbMagicTokenStore() : createBlobMagicTokenStore(),
+    storeBackend === 'db' ? createDbMagicTokenStore() : createBlobMagicTokenStore(),
   );
 }
 

package.json

@@ -10,6 +10,7 @@
     "build:netlify": "VITE_DATA_LAYER=remote VITE_BASE_PATH=/ VITE_API_BASE_URL=/api npm run build && npm run build:server",
     "build:node": "VITE_DATA_LAYER=remote VITE_BASE_PATH=/ VITE_API_BASE_URL=/api npm run build && npm run build:server",
     "build:server": "tsc -p tsconfig.server.json",
+    "data:reset-stores": "npm run build:server && node dist-server/server/reset-stores.js",
     "lint": "eslint .",
     "preview": "vite preview",
     "start:node": "node dist-server/server/index.js"

server/dual-store.ts

@@ -0,0 +1,225 @@
+import type { MagicLinkTokenRecord, MagicLinkTokenStore } from './access-tokens.js';
+import type {
+  AwardPrizeCommand,
+  CompletePassportActivityCommand,
+  PrizeMutationResult,
+  RegisterKidCommand,
+  SavePrizeCommand,
+  StoreAdapter,
+  StoreFile,
+  WritableStoreData,
+} from './store.js';
+import type { Kid, PassportActivity, PrizeAward, StoreData } from './types.js';
+
+type DualStoreOptions = {
+  primary: StoreAdapter;
+  secondary: StoreAdapter;
+  strict: boolean;
+};
+
+function checkMatchingJson(
+  strict: boolean,
+  label: string,
+  primary: unknown,
+  secondary: unknown,
+) {
+  if (JSON.stringify(primary) !== JSON.stringify(secondary)) {
+    const message = `Dual store ${label} mismatch between primary and secondary`;
+
+    if (strict) {
+      throw new Error(message);
+    }
+
+    console.error(message);
+  }
+}
+
+async function writeSecondary<T>(
+  strict: boolean,
+  label: string,
+  write: () => Promise<T>,
+) {
+  try {
+    return await write();
+  } catch (error) {
+    if (strict) {
+      throw error;
+    }
+
+    console.error(`Dual store secondary ${label} failed`, error);
+    return undefined;
+  }
+}
+
+export function createDualStore({
+  primary,
+  secondary,
+  strict,
+}: DualStoreOptions): StoreAdapter {
+  async function mirrorSnapshotToSecondary(
+    primarySnapshot: StoreData,
+    changedFiles: readonly StoreFile[],
+  ) {
+    await writeSecondary(strict, 'snapshot write', () =>
+      secondary.updateSnapshot((secondarySnapshot) => {
+        for (const changedFile of changedFiles) {
+          switch (changedFile) {
+            case 'conference':
+              secondarySnapshot.conference = primarySnapshot.conference;
+              break;
+            case 'kids':
+              secondarySnapshot.kids = primarySnapshot.kids;
+              break;
+            case 'passportActivitiesByKid':
+              secondarySnapshot.passportActivitiesByKid =
+                primarySnapshot.passportActivitiesByKid;
+              break;
+            case 'prizeAwards':
+              secondarySnapshot.prizeAwards = primarySnapshot.prizeAwards;
+              break;
+            case 'prizes':
+              secondarySnapshot.prizes = primarySnapshot.prizes;
+              break;
+          }
+        }
+      }, changedFiles),
+    );
+  }
+
+  return {
+    async awardPrize(command: AwardPrizeCommand): Promise<PrizeAward[]> {
+      const primaryResult = await primary.awardPrize(command);
+      const secondaryResult = await writeSecondary(strict, 'awardPrize', () =>
+        secondary.awardPrize(command),
+      );
+
+      if (secondaryResult) {
+        checkMatchingJson(strict, 'awardPrize result', primaryResult, secondaryResult);
+      }
+
+      return primaryResult;
+    },
+    async completePassportActivity(
+      command: CompletePassportActivityCommand,
+    ): Promise<PassportActivity[]> {
+      const primaryResult = await primary.completePassportActivity(command);
+      const secondaryResult = await writeSecondary(
+        strict,
+        'completePassportActivity',
+        () => secondary.completePassportActivity(command),
+      );
+
+      if (secondaryResult) {
+        checkMatchingJson(
+          strict,
+          'completePassportActivity result',
+          primaryResult,
+          secondaryResult,
+        );
+      }
+
+      return primaryResult;
+    },
+    readSnapshot() {
+      return primary.readSnapshot();
+    },
+    async registerKid(command: RegisterKidCommand): Promise<Kid> {
+      const primaryResult = await primary.registerKid(command);
+      const secondaryResult = await writeSecondary(strict, 'registerKid', () =>
+        secondary.registerKid(command),
+      );
+
+      if (secondaryResult) {
+        checkMatchingJson(strict, 'registerKid result', primaryResult, secondaryResult);
+      }
+
+      return primaryResult;
+    },
+    async restoreWritableData(data: WritableStoreData): Promise<StoreData> {
+      const primaryResult = await primary.restoreWritableData(data);
+      const secondaryResult = await writeSecondary(strict, 'restoreWritableData', () =>
+        secondary.restoreWritableData(data),
+      );
+
+      if (secondaryResult) {
+        checkMatchingJson(
+          strict,
+          'restoreWritableData result',
+          primaryResult,
+          secondaryResult,
+        );
+      }
+
+      return primaryResult;
+    },
+    async savePrize(command: SavePrizeCommand): Promise<PrizeMutationResult> {
+      const primaryResult = await primary.savePrize(command);
+      const secondaryResult = await writeSecondary(strict, 'savePrize', () =>
+        secondary.savePrize(command),
+      );
+
+      if (secondaryResult) {
+        checkMatchingJson(strict, 'savePrize result', primaryResult, secondaryResult);
+      }
+
+      return primaryResult;
+    },
+    async updatePassportForKid<T>(
+      kidId: string,
+      mutator: (snapshot: StoreData) => T | Promise<T>,
+    ) {
+      const primaryResult = await primary.updatePassportForKid(kidId, mutator);
+      const primarySnapshot = await primary.readSnapshot();
+      await writeSecondary(strict, 'updatePassportForKid', () =>
+        secondary.updatePassportForKid(kidId, (secondarySnapshot) => {
+          secondarySnapshot.passportActivitiesByKid[kidId] =
+            primarySnapshot.passportActivitiesByKid[kidId] ?? [];
+        }),
+      );
+      return primaryResult;
+    },
+    async updatePrizeAwardsForKid<T>(
+      kidId: string,
+      mutator: (snapshot: StoreData) => T | Promise<T>,
+    ) {
+      const primaryResult = await primary.updatePrizeAwardsForKid(kidId, mutator);
+      const primarySnapshot = await primary.readSnapshot();
+      await writeSecondary(strict, 'updatePrizeAwardsForKid', () =>
+        secondary.updatePrizeAwardsForKid(kidId, (secondarySnapshot) => {
+          secondarySnapshot.prizeAwards = primarySnapshot.prizeAwards;
+        }),
+      );
+      return primaryResult;
+    },
+    async updateSnapshot<T>(
+      mutator: (snapshot: StoreData) => T | Promise<T>,
+      changedFiles: readonly StoreFile[],
+    ) {
+      const primaryResult = await primary.updateSnapshot(mutator, changedFiles);
+      await mirrorSnapshotToSecondary(await primary.readSnapshot(), changedFiles);
+      return primaryResult;
+    },
+  };
+}
+
+export function createDualMagicTokenStore({
+  primary,
+  secondary,
+  strict,
+}: {
+  primary: MagicLinkTokenStore;
+  secondary: MagicLinkTokenStore;
+  strict: boolean;
+}): MagicLinkTokenStore {
+  return {
+    readTokens() {
+      return primary.readTokens();
+    },
+    async writeTokens(tokens: MagicLinkTokenRecord[]) {
+      await primary.writeTokens(tokens);
+      await writeSecondary(strict, 'magic token write', () =>
+        secondary.writeTokens(tokens),
+      );
+    },
+  };
+}