From 8cc00487dc6b70e5df49fb48ae00b4d12f63c351 Mon Sep 17 00:00:00 2001 From: pablonete Date: Wed, 24 Jun 2026 23:16:53 +0200 Subject: [PATCH] Enable dual writes for Blob and DB stores Add dual store composition for Blob and Netlify DB backends, a reset command that seeds both stores from committed JSON, and Netlify config for Blob reads with dual writes. Magic-link tokens now follow the same backend mode as application data. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/storage-json.md | 29 ++++- netlify/functions/api.ts | 37 +++++-- package.json | 1 + server/dual-store.ts | 225 +++++++++++++++++++++++++++++++++++++++ server/reset-stores.ts | 105 ++++++++++++++++++ 5 files changed, 386 insertions(+), 11 deletions(-) create mode 100644 server/dual-store.ts create mode 100644 server/reset-stores.ts diff --git a/docs/storage-json.md b/docs/storage-json.md index 9cb0e07..f2bf426 100644 --- a/docs/storage-json.md +++ b/docs/storage-json.md @@ -7,8 +7,11 @@ This app has two writable storage backends: - Netlify deployments use Netlify Blobs in the `kid-a-data` store by default, seeded from committed JSON files in `server/data` or `src/data`. - Netlify deployments can use Netlify DB/Postgres by setting - `KID_A_STORE_BACKEND=db`. Magic-link tokens follow the same backend unless - `KID_A_TOKEN_BACKEND` is set explicitly. + `KID_A_STORE_BACKEND=db`. +- Netlify deployments can read from one backend and dual-write to Blobs and DB + by setting `KID_A_STORE_BACKEND=dual`. `KID_A_STORE_READ=blob` keeps Blob + reads while shadow-writing DB; `KID_A_STORE_READ=db` reads from DB instead. + Magic-link tokens always follow the same backend mode as app data. Netlify Blobs are optimized for reads and infrequent writes. Avoid read-modify-write on shared JSON blobs for high-frequency event data because @@ -44,9 +47,9 @@ app for DB and dual-write backends. ## Follow-up migration targets -1. Move writable event state to Netlify DB/Postgres behind the store interface. -2. Add migration and verification tooling from file/blob JSON to DB rows. -3. Enable config-driven dual writes before switching reads to DB. +1. Reset Blob and DB writable state from committed seed JSON. +2. Enable config-driven dual writes with Blob reads. +3. Switch reads to DB after live verification. 4. Keep blobs as a rollback target until DB reads are verified. ## Netlify DB schema @@ -56,3 +59,19 @@ app-owned IDs for kids, prizes, prize awards, and magic-link token hashes; the database does not auto-generate user-facing IDs or maintain a separate kid ID counter table. The DB adapter derives prize `given` counts from `prize_awards` and builds passport responses from `passport_activities` rows. + +The app is not in production yet, so existing Blob and DB testing data can be +discarded. To apply the DB schema, reset both stores from committed seed JSON, +and clear magic-link tokens in both places, run: + +```sh +NETLIFY_DATABASE_URL=... npm run data:reset-stores +``` + +After reset, start the rollout with Blob reads and dual writes: + +```sh +KID_A_STORE_BACKEND=dual +KID_A_STORE_READ=blob +KID_A_DUAL_WRITE_STRICT=true +``` diff --git a/netlify/functions/api.ts b/netlify/functions/api.ts index d350873..6c322ab 100644 --- a/netlify/functions/api.ts +++ b/netlify/functions/api.ts @@ -3,6 +3,7 @@ import { createBlobMagicTokenStore, setMagicTokenStore } from '../../server/acce import { handleApiRequest } from '../../server/api.js'; import { createBlobStore } from '../../server/blob-store.js'; import { createDbMagicTokenStore, createDbStore } from '../../server/db-store.js'; +import { createDualMagicTokenStore, createDualStore } from '../../server/dual-store.js'; import { setStoreAdapter } from '../../server/store.js'; type NetlifyEvent = { @@ -24,19 +25,43 @@ function definedHeaders(headers: NetlifyEvent['headers']) { function configureStores() { const storeBackend = process.env.KID_A_STORE_BACKEND ?? 'blob'; - const tokenBackend = process.env.KID_A_TOKEN_BACKEND ?? storeBackend; + const storeRead = process.env.KID_A_STORE_READ ?? 'blob'; + const strictDualWrites = process.env.KID_A_DUAL_WRITE_STRICT !== 'false'; - if (storeBackend !== 'blob' && storeBackend !== 'db') { - throw new Error('KID_A_STORE_BACKEND must be blob or db'); + if (storeBackend !== 'blob' && storeBackend !== 'db' && storeBackend !== 'dual') { + throw new Error('KID_A_STORE_BACKEND must be blob, db, or dual'); } - if (tokenBackend !== 'blob' && tokenBackend !== 'db') { - throw new Error('KID_A_TOKEN_BACKEND must be blob or db'); + if (storeRead !== 'blob' && storeRead !== 'db') { + throw new Error('KID_A_STORE_READ must be blob or db'); + } + + if (storeBackend === 'dual') { + const blobStore = createBlobStore(); + const dbStore = createDbStore(); + const blobTokenStore = createBlobMagicTokenStore(); + const dbTokenStore = createDbMagicTokenStore(); + const primary = storeRead === 'db' ? dbStore : blobStore; + const secondary = storeRead === 'db' ? blobStore : dbStore; + const primaryTokenStore = storeRead === 'db' ? dbTokenStore : blobTokenStore; + const secondaryTokenStore = storeRead === 'db' ? blobTokenStore : dbTokenStore; + + setStoreAdapter( + createDualStore({ primary, secondary, strict: strictDualWrites }), + ); + setMagicTokenStore( + createDualMagicTokenStore({ + primary: primaryTokenStore, + secondary: secondaryTokenStore, + strict: strictDualWrites, + }), + ); + return; } setStoreAdapter(storeBackend === 'db' ? createDbStore() : createBlobStore()); setMagicTokenStore( - tokenBackend === 'db' ? createDbMagicTokenStore() : createBlobMagicTokenStore(), + storeBackend === 'db' ? createDbMagicTokenStore() : createBlobMagicTokenStore(), ); } diff --git a/package.json b/package.json index 61f7b0b..84fdaff 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "build:netlify": "VITE_DATA_LAYER=remote VITE_BASE_PATH=/ VITE_API_BASE_URL=/api npm run build && npm run build:server", "build:node": "VITE_DATA_LAYER=remote VITE_BASE_PATH=/ VITE_API_BASE_URL=/api npm run build && npm run build:server", "build:server": "tsc -p tsconfig.server.json", + "data:reset-stores": "npm run build:server && node dist-server/server/reset-stores.js", "lint": "eslint .", "preview": "vite preview", "start:node": "node dist-server/server/index.js" diff --git a/server/dual-store.ts b/server/dual-store.ts new file mode 100644 index 0000000..c629709 --- /dev/null +++ b/server/dual-store.ts @@ -0,0 +1,225 @@ +import type { MagicLinkTokenRecord, MagicLinkTokenStore } from './access-tokens.js'; +import type { + AwardPrizeCommand, + CompletePassportActivityCommand, + PrizeMutationResult, + RegisterKidCommand, + SavePrizeCommand, + StoreAdapter, + StoreFile, + WritableStoreData, +} from './store.js'; +import type { Kid, PassportActivity, PrizeAward, StoreData } from './types.js'; + +type DualStoreOptions = { + primary: StoreAdapter; + secondary: StoreAdapter; + strict: boolean; +}; + +function checkMatchingJson( + strict: boolean, + label: string, + primary: unknown, + secondary: unknown, +) { + if (JSON.stringify(primary) !== JSON.stringify(secondary)) { + const message = `Dual store ${label} mismatch between primary and secondary`; + + if (strict) { + throw new Error(message); + } + + console.error(message); + } +} + +async function writeSecondary( + strict: boolean, + label: string, + write: () => Promise, +) { + try { + return await write(); + } catch (error) { + if (strict) { + throw error; + } + + console.error(`Dual store secondary ${label} failed`, error); + return undefined; + } +} + +export function createDualStore({ + primary, + secondary, + strict, +}: DualStoreOptions): StoreAdapter { + async function mirrorSnapshotToSecondary( + primarySnapshot: StoreData, + changedFiles: readonly StoreFile[], + ) { + await writeSecondary(strict, 'snapshot write', () => + secondary.updateSnapshot((secondarySnapshot) => { + for (const changedFile of changedFiles) { + switch (changedFile) { + case 'conference': + secondarySnapshot.conference = primarySnapshot.conference; + break; + case 'kids': + secondarySnapshot.kids = primarySnapshot.kids; + break; + case 'passportActivitiesByKid': + secondarySnapshot.passportActivitiesByKid = + primarySnapshot.passportActivitiesByKid; + break; + case 'prizeAwards': + secondarySnapshot.prizeAwards = primarySnapshot.prizeAwards; + break; + case 'prizes': + secondarySnapshot.prizes = primarySnapshot.prizes; + break; + } + } + }, changedFiles), + ); + } + + return { + async awardPrize(command: AwardPrizeCommand): Promise { + const primaryResult = await primary.awardPrize(command); + const secondaryResult = await writeSecondary(strict, 'awardPrize', () => + secondary.awardPrize(command), + ); + + if (secondaryResult) { + checkMatchingJson(strict, 'awardPrize result', primaryResult, secondaryResult); + } + + return primaryResult; + }, + async completePassportActivity( + command: CompletePassportActivityCommand, + ): Promise { + const primaryResult = await primary.completePassportActivity(command); + const secondaryResult = await writeSecondary( + strict, + 'completePassportActivity', + () => secondary.completePassportActivity(command), + ); + + if (secondaryResult) { + checkMatchingJson( + strict, + 'completePassportActivity result', + primaryResult, + secondaryResult, + ); + } + + return primaryResult; + }, + readSnapshot() { + return primary.readSnapshot(); + }, + async registerKid(command: RegisterKidCommand): Promise { + const primaryResult = await primary.registerKid(command); + const secondaryResult = await writeSecondary(strict, 'registerKid', () => + secondary.registerKid(command), + ); + + if (secondaryResult) { + checkMatchingJson(strict, 'registerKid result', primaryResult, secondaryResult); + } + + return primaryResult; + }, + async restoreWritableData(data: WritableStoreData): Promise { + const primaryResult = await primary.restoreWritableData(data); + const secondaryResult = await writeSecondary(strict, 'restoreWritableData', () => + secondary.restoreWritableData(data), + ); + + if (secondaryResult) { + checkMatchingJson( + strict, + 'restoreWritableData result', + primaryResult, + secondaryResult, + ); + } + + return primaryResult; + }, + async savePrize(command: SavePrizeCommand): Promise { + const primaryResult = await primary.savePrize(command); + const secondaryResult = await writeSecondary(strict, 'savePrize', () => + secondary.savePrize(command), + ); + + if (secondaryResult) { + checkMatchingJson(strict, 'savePrize result', primaryResult, secondaryResult); + } + + return primaryResult; + }, + async updatePassportForKid( + kidId: string, + mutator: (snapshot: StoreData) => T | Promise, + ) { + const primaryResult = await primary.updatePassportForKid(kidId, mutator); + const primarySnapshot = await primary.readSnapshot(); + await writeSecondary(strict, 'updatePassportForKid', () => + secondary.updatePassportForKid(kidId, (secondarySnapshot) => { + secondarySnapshot.passportActivitiesByKid[kidId] = + primarySnapshot.passportActivitiesByKid[kidId] ?? []; + }), + ); + return primaryResult; + }, + async updatePrizeAwardsForKid( + kidId: string, + mutator: (snapshot: StoreData) => T | Promise, + ) { + const primaryResult = await primary.updatePrizeAwardsForKid(kidId, mutator); + const primarySnapshot = await primary.readSnapshot(); + await writeSecondary(strict, 'updatePrizeAwardsForKid', () => + secondary.updatePrizeAwardsForKid(kidId, (secondarySnapshot) => { + secondarySnapshot.prizeAwards = primarySnapshot.prizeAwards; + }), + ); + return primaryResult; + }, + async updateSnapshot( + mutator: (snapshot: StoreData) => T | Promise, + changedFiles: readonly StoreFile[], + ) { + const primaryResult = await primary.updateSnapshot(mutator, changedFiles); + await mirrorSnapshotToSecondary(await primary.readSnapshot(), changedFiles); + return primaryResult; + }, + }; +} + +export function createDualMagicTokenStore({ + primary, + secondary, + strict, +}: { + primary: MagicLinkTokenStore; + secondary: MagicLinkTokenStore; + strict: boolean; +}): MagicLinkTokenStore { + return { + readTokens() { + return primary.readTokens(); + }, + async writeTokens(tokens: MagicLinkTokenRecord[]) { + await primary.writeTokens(tokens); + await writeSecondary(strict, 'magic token write', () => + secondary.writeTokens(tokens), + ); + }, + }; +} diff --git a/server/reset-stores.ts b/server/reset-stores.ts new file mode 100644 index 0000000..819a444 --- /dev/null +++ b/server/reset-stores.ts @@ -0,0 +1,105 @@ +import { readdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { neon } from '@netlify/neon'; +import { createBlobMagicTokenStore } from './access-tokens.js'; +import { createBlobStore } from './blob-store.js'; +import { createDbMagicTokenStore, createDbStore } from './db-store.js'; +import { + getStoreFileName, + syncPrizeGivenCache, + type StoreAdapter, +} from './store.js'; +import type { + ConferenceData, + Kid, + PassportActivitiesByKid, + Prize, + PrizeAward, + StoreData, +} from './types.js'; + +type SqlClient = ReturnType; + +const migrationsDir = path.resolve('db/migrations'); +const seedDataDir = path.resolve(process.env.KID_A_SEED_DATA_DIR ?? 'src/data'); + +async function readSeedJson(fileName: string): Promise { + return JSON.parse(await readFile(path.join(seedDataDir, fileName), 'utf8')) as T; +} + +async function readSeedSnapshot(): Promise { + const [conference, kids, passportActivitiesByKid, prizeAwards, prizes] = + await Promise.all([ + readSeedJson(getStoreFileName('conference')), + readSeedJson(getStoreFileName('kids')), + readSeedJson( + getStoreFileName('passportActivitiesByKid'), + ), + readSeedJson(getStoreFileName('prizeAwards')), + readSeedJson(getStoreFileName('prizes')), + ]); + + return { + conference, + kids, + passportActivitiesByKid, + prizeAwards, + prizes: syncPrizeGivenCache(prizes, prizeAwards), + }; +} + +function splitSqlStatements(sqlText: string) { + return sqlText + .split(/;\s*(?:\r?\n|$)/) + .map((statement) => statement.trim()) + .filter(Boolean); +} + +async function applySchema(sql: SqlClient) { + const migrationFiles = (await readdir(migrationsDir)) + .filter((fileName) => fileName.endsWith('.sql')) + .sort(); + + for (const migrationFile of migrationFiles) { + const sqlText = await readFile(path.join(migrationsDir, migrationFile), 'utf8'); + + for (const statement of splitSqlStatements(sqlText)) { + await sql.query(statement); + } + } +} + +async function resetStore(name: string, store: StoreAdapter) { + const seedSnapshot = await readSeedSnapshot(); + + await store.updateSnapshot( + (snapshot) => { + snapshot.conference = seedSnapshot.conference; + snapshot.kids = seedSnapshot.kids; + snapshot.passportActivitiesByKid = seedSnapshot.passportActivitiesByKid; + snapshot.prizeAwards = seedSnapshot.prizeAwards; + snapshot.prizes = seedSnapshot.prizes; + }, + ['conference', 'kids', 'passportActivitiesByKid', 'prizeAwards', 'prizes'], + ); + + console.log(`Reset ${name} store from ${seedDataDir}`); +} + +async function main() { + const sql = neon(); + + await applySchema(sql); + await resetStore('blob', createBlobStore()); + await resetStore('db', createDbStore(sql)); + await Promise.all([ + createBlobMagicTokenStore().writeTokens([]), + createDbMagicTokenStore(sql).writeTokens([]), + ]); + console.log('Cleared blob and DB magic-link tokens'); +} + +main().catch((error: unknown) => { + console.error(error); + process.exitCode = 1; +});