From 346a88f2715b2fb749c1bb7744bae6e8c59fb270 Mon Sep 17 00:00:00 2001 From: Aswin C Date: Fri, 23 Oct 2026 01:30:10 +0530 Subject: [PATCH 1/2] feat(sync): per-show check cadence in shadow --- scripts/README.md | 1 + scripts/package.json | 2 +- scripts/sync/03-sync-episodes.js | 69 ++++++- scripts/sync/lib/cadence.js | 246 +++++++++++++++++++++++ scripts/sync/lib/cadence.test.js | 143 +++++++++++++ scripts/sync/lib/chart-countries.js | 1 + scripts/sync/lib/chart-countries.test.js | 2 + scripts/sync/lib/config.js | 4 + scripts/sync/lib/stale-tips.js | 3 +- scripts/sync/lib/stale-tips.test.js | 1 + scripts/sync/lib/state.js | 4 +- 11 files changed, 469 insertions(+), 7 deletions(-) create mode 100644 scripts/sync/lib/cadence.js create mode 100644 scripts/sync/lib/cadence.test.js diff --git a/scripts/README.md b/scripts/README.md index 994de0ef2..c59850b98 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -33,6 +33,7 @@ A Git push does not deploy this pipeline. Changes to `scripts/sync/` or `scripts | [`scripts/sync/lib/catalog-rows.js`](sync/lib/catalog-rows.js) | Scrubbed values for stage 2's API import and stage 3's latest-episode update: an odd optional field from Podcast Index becomes empty instead of failing the write. The latest-episode update stores the episode's served ID in `latest_ep_id` (what Home, trending and bootstrap send), plus `latest_ep_key` and `latest_ep_eid`, from `tipIdentity` in `lib/episode-identity.js` (by catalog key, else Podcast Index's ID as an alias in the same show, else that ID). Change detection and the tip queue keep Podcast Index's own ID | | [`scripts/sync/lib/stale-tips.js`](sync/lib/stale-tips.js) | Stage 3's tips for stale shows (`R5_MODE`: `off` by default, `shadow`, `on`). After the Podcast Index pass, checked chart shows that the published feed-address registry (`REGISTRY_DB`) flags `episode_stale` ask the local feed reader (`FEEDS_URL`, `POST /feeds/latest`, 40 shows per call) for their newest episode. When it is playable, a different episode and newer than the stored tip, it becomes the tip under its served ID, catalog key and recipe ID. Only the tip columns change: `qdrant_vectorized` and the tip queue stay with the Podcast Index pass. `shadow` reads only the feed's dates and never writes. A registry or feed-reader failure skips it, and the Podcast Index tips stay | | [`scripts/sync/lib/show-window.js`](sync/lib/show-window.js) | Saved episode windows (`WINDOWS_MODE`: `off` by default, `shadow`, `on`). Stage 3 records each full list it fetches as episode-book rows, with descriptions, and a `show_window` row: the book row IDs in list order. It fetches at the cap stage 4 enforces, chart or earned. Shows whose R5 tip came from the feed also save the feed reader's merged list (`GET /feeds/episodes`), so their feed-only episodes get vectors. With `on`, stage 4 reads a saved window instead of fetching the list from Podcast Index, when it is usable, fresh (3 days) and long enough. `shadow` records windows and has stage 4 compare them with its own fetch. A window with an episode that has no catalog key, or that another show serves, is unusable, and stage 4 takes the fetch path. Tip-queue rows carry `gen`, and stage 4 clears only rows enqueued before it started | +| [`scripts/sync/lib/cadence.js`](sync/lib/cadence.js) | Per-show check schedule for stage 3, in shadow (`CADENCE_MODE`: `shadow` by default, `off`). Checks still follow the news/core/relaxed tiers; the run report adds what the schedule would have checked. Each show's interval follows its pace, the average gap between its newest 11 episodes: news shows hourly, others a quarter of the gap within 3 to 48 hours, and dormant shows weekly when both the newest episode and a fresh Apple answer are over 180 days old. A show with no known pace keeps its tier. A show Apple lists a newer episode for (`APPLE_LATEST_DB`, the applefeeds snapshot) is due until it has been checked 6 hours past that date. Pace and the newest date come from each fetched list and from the episode book about daily; show records gain `g`, `p` and `i` (iTunes ID). The report line counts shows the schedule would skip or check earlier, new tips it would have found later and by how much, and checks a day against today's tiers. A book or snapshot failure only skips that part | | [`scripts/sync/lib/attribution.js`](sync/lib/attribution.js) | Vector attribution log: one JSON line per episode point stored or deleted (prune, cap trim), show vector stored, and show imported or deleted. Written to `SYNC_ATTRIBUTION_DIR` (default `scripts/data/attribution/`, gitignored) as daily files, gzipped after the day ends and kept 60 days. Best effort: a write failure never fails a stage. Stage 4 adds `stored`, `pruned` and `reasons …` to its run line | | [`scripts/sync/attribution-report.js`](sync/attribution-report.js) | Read-only report over that log: embeddings by reason (new episode, first coverage, returning show, re-embed after cap trim or prune, Podcast Index ID change, backfill), deletions, and how long chart entrants stay. `--days N`, `--json` | | [`scripts/sync/lib/episode-book.js`](sync/lib/episode-book.js) + [`episode-book-cli.js`](sync/episode-book-cli.js) | The episode book in sqld: one permanent row per (show, catalog key) with its recipe ID and served ID, plus Podcast Index aliases, `served_show` and `show_window`. Every book write also marks the show in `served_show` when one of its episodes is listed by Podcast Index under an ID other than its served ID (renumbered, duplicated or recipe-served), so the Worker's v1 overlay translates that ID back. That check reaches a show's aliases through `episode_pi_alias_rid` (an index on `rid`); without it each check scanned every alias inside the write. `ensure` creates the tables, indexes and columns; `stats` counts. Writes go through `turso.atomic()` (one Hrana batch: every step runs only after the previous one succeeded, then COMMIT, else ROLLBACK). Rules: [`docs/EPISODE_IDS.md`](../docs/EPISODE_IDS.md) | diff --git a/scripts/package.json b/scripts/package.json index 9b02adbb3..62d4b13d4 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -16,7 +16,7 @@ "test:sync-scalars": "node --test sync/lib/scalars.test.js", "test:sync-turso-page": "node --test sync/lib/turso-page.test.js", "test:sync-turso": "node --test sync/lib/turso.test.js", - "test:sync": "node --test sync/lib/csv.test.js sync/lib/pi-unavailable.test.js sync/lib/scalars.test.js sync/lib/turso-page.test.js sync/lib/episode-caps.test.js sync/lib/staleness.test.js sync/lib/podcast-index.test.js sync/lib/chart-countries.test.js sync/lib/log.test.js sync/lib/text.test.js sync/lib/vectorize-lanes.test.js sync/lib/tip-queue.test.js sync/lib/import-missing.test.js sync/lib/episode-recipe-id.test.js sync/lib/attribution.test.js sync/lib/vector-window.test.js sync/lib/trim-episode-caps.test.js sync/lib/remediate-qdrant.test.js sync/lib/turso.test.js sync/lib/catalog-rows.test.js sync/lib/episode-book.test.js sync/lib/book-bootstrap.test.js sync/lib/episode-identity.test.js sync/lib/stale-tips.test.js sync/lib/show-window.test.js", + "test:sync": "node --test sync/lib/csv.test.js sync/lib/pi-unavailable.test.js sync/lib/scalars.test.js sync/lib/turso-page.test.js sync/lib/episode-caps.test.js sync/lib/staleness.test.js sync/lib/podcast-index.test.js sync/lib/chart-countries.test.js sync/lib/log.test.js sync/lib/text.test.js sync/lib/vectorize-lanes.test.js sync/lib/tip-queue.test.js sync/lib/import-missing.test.js sync/lib/episode-recipe-id.test.js sync/lib/attribution.test.js sync/lib/vector-window.test.js sync/lib/trim-episode-caps.test.js sync/lib/remediate-qdrant.test.js sync/lib/turso.test.js sync/lib/catalog-rows.test.js sync/lib/episode-book.test.js sync/lib/book-bootstrap.test.js sync/lib/episode-identity.test.js sync/lib/stale-tips.test.js sync/lib/show-window.test.js sync/lib/cadence.test.js", "test:sync-text": "node --test sync/lib/text.test.js", "test:sync-recipe": "node --test sync/lib/episode-recipe-id.test.js", "test:sync-attribution": "node --test sync/lib/attribution.test.js", diff --git a/scripts/sync/03-sync-episodes.js b/scripts/sync/03-sync-episodes.js index 3d20f7465..97ba5f3b3 100644 --- a/scripts/sync/03-sync-episodes.js +++ b/scripts/sync/03-sync-episodes.js @@ -13,6 +13,8 @@ * full items[] written to PI handoff for stage 4 (no second PI call same run). * - R5 (R5_MODE, lib/stale-tips.js): afterwards, checked shows the registry flags stale take the * live feed's newest episode as their tip when it is newer than Podcast Index's. + * - Check cadence (CADENCE_MODE, lib/cadence.js): in shadow, reports what a per-show schedule would + * have checked; checks still follow the tiers above. */ const log = require('./lib/log'); @@ -35,6 +37,7 @@ const { const staleness = require('./lib/staleness'); const staleTips = require('./lib/stale-tips'); const showWindow = require('./lib/show-window'); +const cadence = require('./lib/cadence'); const CANDIDATE_CACHE_MAX_AGE_MS = 20 * 60 * 60 * 1000; const CONCURRENCY = 5; @@ -63,9 +66,10 @@ async function refreshCandidates(st) { const ids = []; let newsCount = 0; let relaxedCount = 0; - for (const { id: podId, latestEpId, categories, medium, countryCsv } of chartShows) { + for (const { id: podId, latestEpId, categories, medium, itunesId, countryCsv } of chartShows) { ids.push(podId); const rec = st.shows[podId] || {}; + if (itunesId) rec.i = itunesId; if (rec.e === undefined && latestEpId) rec.e = String(latestEpId); if ((categories || '').includes('News')) { rec.n = 1; @@ -143,6 +147,7 @@ async function main() { 'Concurrency': String(CONCURRENCY), 'Stale tips (R5)': cfg.R5_MODE, 'Saved windows': cfg.WINDOWS_MODE, + 'Check cadence': cfg.CADENCE_MODE, }); const cacheAge = Date.now() - (st.candidatesRefreshedAt || 0); @@ -163,12 +168,47 @@ async function main() { needRefresh = true; } } + if (!needRefresh && st.candidateIds && cfg.CADENCE_MODE !== 'off') { + // One-shot: the cadence reads Apple's dates by iTunes ID, which only a refresh records. + const missingI = st.candidateIds.filter((id) => !st.shows[id]?.i).length; + if (missingI > Math.max(50, st.candidateIds.length * 0.05)) { + log.info(`[CANDIDATES] Refreshing to record iTunes IDs (${log.fmt(missingI)} shows without one)`); + needRefresh = true; + } + } if (needRefresh) { await refreshCandidates(st); } else { log.info(`[CANDIDATES] Using cached list (${log.fmt(st.candidateIds.length)} shows, age ${Math.round(cacheAge / 3600000)}h) - 0 Turso reads`); } + // Cadence shadow: pace from the book about daily, then the plan from the state as it was before + // this run's checks. Never fails the stage. + let shadow = null; + let appleError = null; + if (cfg.CADENCE_MODE === 'shadow') { + if (needRefresh || Date.now() - (st.cadenceSeededAt || 0) > CANDIDATE_CACHE_MAX_AGE_MS) { + try { + const seeded = await cadence.seedFromBook(book, st.shows, st.candidateIds); + st.cadenceSeededAt = Date.now(); + log.info(`[CADENCE] pace seeded from the book for ${log.fmt(seeded)} shows`); + } catch (e) { + log.warn(`[CADENCE] pace seed skipped: ${e.message}`); + } + } + const apple = cadence.openApple(cfg.APPLE_LATEST_DB); + appleError = apple.error; + if (appleError) log.warn(`[CADENCE] Apple snapshot unread: ${appleError}`); + try { + shadow = cadence.shadowPlan(st.candidateIds, st.shows, { appleOf: apple.lookup }); + } catch (e) { + log.warn(`[CADENCE] shadow plan skipped: ${e.message}`); + } finally { + apple.close(); + } + } + const newTipIds = []; + const plan = staleness.planDue(st.candidateIds, st.shows); const { allDue, neverChecked, staleNews, staleCore, staleRelaxed, poolNews, poolCore, poolRelaxed } = plan; @@ -252,7 +292,8 @@ async function main() { const latestTip = tip[0]; if (rec.e && String(latestTip.id) === String(rec.e)) { unchanged++; - state.recordCheck(st, podId); + const checked = state.recordCheck(st, podId); + if (shadow) cadence.noteNewest(checked, latestTip.datePublished); return; } @@ -292,7 +333,12 @@ async function main() { updated++; tipReflags++; tipUpdatedThisBatch = true; - state.recordCheck(st, podId, { latestEpId: latest.id }).m = medium; + const checked = state.recordCheck(st, podId, { latestEpId: latest.id }); + checked.m = medium; + if (shadow) { + cadence.notePace(checked, (full || tip).map((ep) => ep.datePublished)); + newTipIds.push(podId); + } } catch (e) { if (pi.isSuspended(e)) { piPaused = piPaused || e; @@ -329,6 +375,20 @@ async function main() { const r5Line = staleTips.summary(r5); if (cfg.R5_MODE !== 'off') log.info(`[R5] ${r5Line}`); + let cadenceLine = ''; + if (shadow) { + for (const [id, date] of Object.entries(r5.tipDates || {})) { + cadence.noteNewest(st.shows[id] || (st.shows[id] = {}), date); + newTipIds.push(id); + } + state.save(st); + const paceShows = st.candidateIds.filter((id) => st.shows[id]?.g != null).length; + cadenceLine = cadence.summary(cfg.CADENCE_MODE, shadow, cadence.compare(shadow, allDue, newTipIds), { + appleError, paceShows, candidates: st.candidateIds.length, + }); + log.info(`[CADENCE] ${cadenceLine}`); + } + // R5 shows whose tip came from the feed also save the feed's merged list as their window, so // stage 4 embeds the feed-only episodes (WINDOWS_MODE on flags them; shadow only records). if (cfg.WINDOWS_MODE !== 'off' && r5.updatedIds?.length) { @@ -388,7 +448,8 @@ async function main() { ` · ${pi.summary()}` + (piPaused ? ` · ${due.length - checkedBeforePause} checks left for the next run` : '') + (cfg.R5_MODE !== 'off' ? ` · ${r5Line}` : '') + - (windowLine ? ` · ${windowLine}` : ''), + (windowLine ? ` · ${windowLine}` : '') + + (cadenceLine ? ` · ${cadenceLine}` : ''), }]); // A pause is "try later", not a failure; only real errors over half the checks fail the stage. diff --git a/scripts/sync/lib/cadence.js b/scripts/sync/lib/cadence.js new file mode 100644 index 000000000..014d34f65 --- /dev/null +++ b/scripts/sync/lib/cadence.js @@ -0,0 +1,246 @@ +'use strict'; + +/** + * Per-show check cadence (own-episode-IDs P1b, plan section 10b items 1–2), in shadow: stage 3 + * still checks on today's tiers (lib/staleness.js) and reports what this schedule would have done. + * P3b switches it on together with hourly runs. + * + * A show's interval follows how often it publishes: + * news → 1 h + * dormant (newest > 180 d, and Apple agrees) → 7 d + * known pace → a quarter of its typical gap, 3 h to 48 h + * unknown pace → today's tier threshold + * Apple's latest-episode date is the safety net: a show Apple lists a newer episode for than the + * newest one we know is due at once, until we have checked it for a few hours past that date. + * Never-checked shows are always due. Intervals carry the same per-show jitter as today's tiers. + * + * Show record fields (lib/state.js): g = typical gap between episodes in hours (newest 11 dates), + * p = newest episode date we know (epoch seconds), i = iTunes ID (for Apple's snapshot). + */ + +const cfg = require('./config'); +const staleness = require('./staleness'); + +const HOUR_MS = 3_600_000; +const NEWS_MS = HOUR_MS; +const MIN_MS = 3 * HOUR_MS; +const MAX_MS = 48 * HOUR_MS; +const DORMANT_MS = 7 * 24 * HOUR_MS; +const DORMANT_AFTER_S = 180 * 86_400; +/** Apple's answer counts as evidence for dormancy only this long after Apple last confirmed it. */ +const APPLE_FRESH_S = 30 * 86_400; +/** After Apple's newer date, checks are forced this long (Podcast Index indexes hours late). */ +const APPLE_FORCE_WINDOW_S = 6 * 3_600; +/** Episode dates used for a show's pace. */ +const PACE_DATES = 11; + +/** + * Typical gap in hours between a show's newest 11 dated episodes: the span over the gaps. The book + * seed (paceFromSpan) uses the same rule, so a list and the book agree. Null under 3 dates. + */ +function paceFromDates(dates) { + const sorted = (dates || []).map(Number).filter((d) => d > 0).sort((a, b) => b - a).slice(0, PACE_DATES); + return paceFromSpan(sorted.length, sorted[sorted.length - 1], sorted[0]); +} + +/** Average gap in hours from a count and date span, or null under 3 episodes. */ +function paceFromSpan(count, oldest, newest) { + const n = Number(count); + const span = Number(newest) - Number(oldest); + if (!(n >= 3) || !(span >= 0)) return null; + return Math.round((span / (n - 1) / 3600) * 10) / 10; +} + +/** Records a fetched list's pace and newest date on the show record (dates in epoch seconds). */ +function notePace(rec, dates) { + const g = paceFromDates(dates); + if (g != null) rec.g = g; + noteNewest(rec, Math.max(0, ...(dates || []).map((d) => Number(d) || 0))); + return rec; +} + +/** Keeps the newest episode date we know. */ +function noteNewest(rec, dateS) { + const d = Number(dateS) || 0; + if (d > (rec.p || 0)) rec.p = d; + return rec; +} + +/** + * The shadow interval for one show. + * @param {object} rec show record + * @param {string} id show ID (jitter seed) + * @param {{ nowS: number, apple?: { releaseDate: number, verifiedAt: number } | null }} ctx + * @returns {{ ms: number, reason: 'news'|'dormant'|'pace'|'tier' }} + */ +function intervalFor(rec, id, { nowS, apple = null }) { + const jitter = staleness.jitterFactor(id); + if (rec.n === 1) return { ms: NEWS_MS * jitter, reason: 'news' }; + const newest = rec.p || 0; + const appleQuiet = apple && nowS - (apple.verifiedAt || 0) <= APPLE_FRESH_S && + (apple.releaseDate || 0) > 0 && nowS - apple.releaseDate > DORMANT_AFTER_S; + if (newest > 0 && nowS - newest > DORMANT_AFTER_S && appleQuiet) { + return { ms: DORMANT_MS * jitter, reason: 'dormant' }; + } + if (rec.g != null) { + const ms = Math.min(MAX_MS, Math.max(MIN_MS, (rec.g * HOUR_MS) / 4)); + return { ms: ms * jitter, reason: 'pace' }; + } + return { ms: staleness.thresholdMs(rec, id), reason: 'tier' }; +} + +/** True when Apple lists a newer episode than we know and we haven't checked well past its date. */ +function appleForces(rec, apple) { + if (!apple || !(apple.releaseDate > 0)) return false; + const lastCheckS = (rec.c || 0) / 1000; + return apple.releaseDate > (rec.p || 0) + 3600 && lastCheckS < apple.releaseDate + APPLE_FORCE_WINDOW_S; +} + +/** + * What the cadence would check now, over every candidate (no per-run cap). + * @param {string[]} candidateIds + * @param {object} shows state.shows + * @param {{ now?: number, appleOf?: (itunesId: number) => ({ releaseDate: number, verifiedAt: number } | null) }} [opts] + */ +function shadowPlan(candidateIds, shows, { now = Date.now(), appleOf = () => null } = {}) { + const nowS = now / 1000; + const due = new Set(); + const dueAt = new Map(); + const counts = { never: 0, appleForced: 0, news: 0, dormant: 0, pace: 0, tier: 0 }; + let perDay = 0; + let perDayNow = 0; + for (const id of candidateIds || []) { + const rec = shows[id] || {}; + const apple = rec.i ? appleOf(Number(rec.i)) : null; + const { ms, reason } = intervalFor(rec, id, { nowS, apple }); + counts[reason]++; + // Checks a day with hourly runs (no show more than once an hour), against today's tiers. + perDay += 24 / Math.max(1, ms / HOUR_MS); + perDayNow += 24 / (staleness.thresholdMs(rec, id) / HOUR_MS); + if (!rec.c) { + counts.never++; + due.add(id); + continue; + } + if (appleForces(rec, apple)) { + counts.appleForced++; + due.add(id); + continue; + } + const at = rec.c + ms; + dueAt.set(id, at); + if (now >= at) due.add(id); + } + return { due, dueAt, counts, perDay: Math.round(perDay), perDayNow: Math.round(perDayNow) }; +} + +/** + * Shadow against what this run did. + * @param {ReturnType} plan + * @param {string[]} currentDue every show today's tiers made due (before the per-run cap) + * @param {string[]} newTipIds shows whose tip changed this run + */ +function compare(plan, currentDue, newTipIds, now = Date.now()) { + const current = new Set(currentDue.map(String)); + let skipped = 0; + for (const id of current) if (!plan.due.has(id)) skipped++; + let earlier = 0; + for (const id of plan.due) if (!current.has(id)) earlier++; + const late = []; + for (const id of new Set(newTipIds.map(String))) { + if (plan.due.has(id)) continue; + const at = plan.dueAt.get(id); + late.push(at == null ? 0 : Math.max(0, (at - now) / HOUR_MS)); + } + late.sort((a, b) => a - b); + return { + wouldCheck: plan.due.size, + current: current.size, + skipped, + earlier, + late: late.length, + lateMedianH: late.length ? late[late.length >> 1] : 0, + lateMaxH: late.length ? late[late.length - 1] : 0, + }; +} + +/** Pace per show from the book's newest 11 dated episodes (one aggregate read, ~2 s for 52k shows). */ +const BOOK_PACE_SQL = ` + SELECT sid, COUNT(*), MIN(published_at), MAX(published_at) + FROM (SELECT sid, published_at, + ROW_NUMBER() OVER (PARTITION BY sid ORDER BY published_at DESC) AS rn + FROM episode_book WHERE published_at > 0) + WHERE rn <= ${PACE_DATES} + GROUP BY sid`; + +/** + * Seeds g and p for candidate shows from the book (stage 3's saved windows and the bootstrap), so + * the shadow has a pace for shows whose tip hasn't changed since it started. + * @returns {number} shows seeded + */ +async function seedFromBook(book, shows, candidateIds) { + const wanted = new Set((candidateIds || []).map(String)); + let seeded = 0; + for (const [sid, count, oldest, newest] of await book.query(BOOK_PACE_SQL)) { + const id = String(sid); + if (!wanted.has(id)) continue; + const rec = shows[id] || (shows[id] = {}); + const g = paceFromSpan(count, oldest, newest); + if (g != null) rec.g = g; + noteNewest(rec, newest); + seeded++; + } + return seeded; +} + +/** + * Apple's latest-episode snapshot (applefeeds `apple_latest.db`), read-only. Returns a lookup, or + * null with the reason when the file can't be opened (the shadow then runs without Apple). + */ +function openApple(path) { + try { + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(path, { readOnly: true }); + const st = db.prepare('SELECT release_date, verified_at FROM apple_latest WHERE itunes_id = ?'); + return { + lookup: (itunesId) => { + const row = st.get(itunesId); + return row ? { releaseDate: Number(row.release_date) || 0, verifiedAt: Number(row.verified_at) || 0 } : null; + }, + close: () => db.close(), + error: null, + }; + } catch (e) { + return { lookup: () => null, close: () => {}, error: e.message }; + } +} + +const fmtH = (h) => (h >= 10 ? String(Math.round(h)) : h.toFixed(1)); + +/** One line for the run report. */ +function summary(mode, plan, cmp, { appleError = null, paceShows = 0, candidates = 0 } = {}) { + if (mode !== 'shadow' || !plan) return 'cadence off'; + const c = plan.counts; + const pct = candidates ? Math.round((paceShows / candidates) * 100) : 0; + return `cadence shadow: would check ${cmp.wouldCheck} (today ${cmp.current}; ${cmp.skipped} skipped, ${cmp.earlier} earlier, ` + + `${c.appleForced} Apple-forced, ${c.never} never checked) · ${cmp.late} new tips not yet due` + + (cmp.late ? ` (median ${fmtH(cmp.lateMedianH)} h, max ${fmtH(cmp.lateMaxH)} h late)` : '') + + ` · checks/day ${plan.perDay} vs ${plan.perDayNow} · news ${c.news}, dormant ${c.dormant}, pace ${c.pace}, tier ${c.tier}` + + ` · pace known ${pct}%` + (appleError ? ` · Apple unread (${appleError})` : ''); +} + +module.exports = { + paceFromDates, + paceFromSpan, + notePace, + noteNewest, + intervalFor, + appleForces, + shadowPlan, + compare, + seedFromBook, + openApple, + summary, + BOOK_PACE_SQL, + HOUR_MS, +}; diff --git a/scripts/sync/lib/cadence.test.js b/scripts/sync/lib/cadence.test.js new file mode 100644 index 000000000..16ba2fe45 --- /dev/null +++ b/scripts/sync/lib/cadence.test.js @@ -0,0 +1,143 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const staleness = require('./staleness'); +const { + paceFromDates, paceFromSpan, notePace, noteNewest, intervalFor, appleForces, + shadowPlan, compare, seedFromBook, openApple, summary, HOUR_MS, +} = require('./cadence'); + +const NOW = 1_792_700_000_000; +const NOW_S = NOW / 1000; +const DAY_S = 86_400; +const jit = (id) => staleness.jitterFactor(id); + +test('pace: the average gap over the newest 11 dated episodes; null under 3', () => { + const weekly = Array.from({ length: 20 }, (_, i) => NOW_S - i * 7 * DAY_S); + assert.equal(paceFromDates(weekly), 168); + // Only the newest 11 count: an old daily stretch doesn't pull a weekly show down. + const mixed = [...weekly.slice(0, 11), ...Array.from({ length: 30 }, (_, i) => NOW_S - 80 * 7 * DAY_S - i * DAY_S)]; + assert.equal(paceFromDates(mixed), 168); + assert.equal(paceFromDates([NOW_S, NOW_S - DAY_S, 0, null]), null, 'undated episodes are ignored'); + assert.equal(paceFromDates([NOW_S, NOW_S - DAY_S, NOW_S - 2 * DAY_S]), 24); + assert.equal(paceFromDates(null), null); + assert.equal(paceFromSpan(11, NOW_S - 10 * DAY_S, NOW_S), 24, 'the book seed uses the same rule'); + assert.equal(paceFromSpan(2, 0, NOW_S), null); +}); + +test('notePace and noteNewest keep the newest date we know', () => { + const rec = notePace({ p: NOW_S + 5 }, [NOW_S, NOW_S - DAY_S, NOW_S - 2 * DAY_S]); + assert.deepEqual(rec, { p: NOW_S + 5, g: 24 }); + assert.equal(noteNewest({}, String(NOW_S)).p, NOW_S); + assert.deepEqual(noteNewest({ p: 5 }, undefined), { p: 5 }); + assert.deepEqual(notePace({}, [NOW_S]), { p: NOW_S }, 'one episode: date but no pace'); +}); + +test('interval: news hourly, a quarter of the gap within 3–48 h, tiers when the pace is unknown', () => { + const ctx = { nowS: NOW_S }; + assert.deepEqual(intervalFor({ n: 1, g: 500 }, 'a', ctx), { ms: HOUR_MS * jit('a'), reason: 'news' }); + assert.deepEqual(intervalFor({ g: 24 }, 'b', ctx), { ms: 6 * HOUR_MS * jit('b'), reason: 'pace' }); + assert.equal(intervalFor({ g: 2 }, 'c', ctx).ms, 3 * HOUR_MS * jit('c'), 'never under 3 h'); + assert.equal(intervalFor({ g: 24 * 30 }, 'd', ctx).ms, 48 * HOUR_MS * jit('d'), 'never over 48 h while active'); + assert.equal(intervalFor({ g: 0 }, 'e', ctx).ms, 3 * HOUR_MS * jit('e'), 'a burst show is still a pace'); + assert.deepEqual(intervalFor({ x: 1 }, 'f', ctx), { ms: staleness.thresholdMs({ x: 1 }, 'f'), reason: 'tier' }); +}); + +test('dormant: weekly only when both our newest date and a fresh Apple answer are over 180 days old', () => { + const rec = { g: 24 * 30, p: NOW_S - 200 * DAY_S }; + const quiet = { releaseDate: NOW_S - 190 * DAY_S, verifiedAt: NOW_S - 2 * DAY_S }; + assert.deepEqual(intervalFor(rec, 'g', { nowS: NOW_S, apple: quiet }), { ms: 7 * 24 * HOUR_MS * jit('g'), reason: 'dormant' }); + assert.equal(intervalFor(rec, 'g', { nowS: NOW_S }).reason, 'pace', 'no Apple answer: not dormant'); + assert.equal(intervalFor(rec, 'g', { nowS: NOW_S, apple: { ...quiet, verifiedAt: NOW_S - 40 * DAY_S } }).reason, 'pace', 'stale Apple answer'); + assert.equal(intervalFor(rec, 'g', { nowS: NOW_S, apple: { ...quiet, releaseDate: NOW_S - 3 * DAY_S } }).reason, 'pace', 'Apple has a recent episode'); + assert.equal(intervalFor({ ...rec, p: NOW_S - 100 * DAY_S }, 'g', { nowS: NOW_S, apple: quiet }).reason, 'pace'); + assert.equal(intervalFor({ g: 24 * 30 }, 'g', { nowS: NOW_S, apple: quiet }).reason, 'pace', 'no newest date: not dormant'); +}); + +test('Apple forces a check for a newer episode until we have checked hours past its date', () => { + const apple = { releaseDate: NOW_S - 2 * 3600, verifiedAt: NOW_S }; + assert.equal(appleForces({ p: NOW_S - 7 * DAY_S, c: NOW - 3 * HOUR_MS }, apple), true); + assert.equal(appleForces({ p: NOW_S - 7 * DAY_S, c: (apple.releaseDate + 7 * 3600) * 1000 }, apple), false, 'checked well after it'); + assert.equal(appleForces({ p: apple.releaseDate - 600, c: 0 }, apple), false, 'within an hour of ours: same episode'); + assert.equal(appleForces({ p: NOW_S, c: 0 }, apple), false); + assert.equal(appleForces({ c: 0 }, null), false); + assert.equal(appleForces({ c: 0 }, { releaseDate: 0 }), false); +}); + +test('shadow plan: never-checked, Apple-forced and interval-due shows; daily check estimate', () => { + const shows = { + news: { n: 1, c: NOW - 2 * HOUR_MS }, + daily: { g: 24, c: NOW - 12 * HOUR_MS, p: NOW_S - DAY_S }, + weekly: { g: 168, c: NOW - 12 * HOUR_MS, p: NOW_S - DAY_S, i: 11 }, + forced: { g: 168, c: NOW - 12 * HOUR_MS, p: NOW_S - 7 * DAY_S, i: 22 }, + fresh: {}, + }; + const apple = { 11: { releaseDate: NOW_S - DAY_S, verifiedAt: NOW_S }, 22: { releaseDate: NOW_S - 3600, verifiedAt: NOW_S } }; + const plan = shadowPlan(Object.keys(shows), shows, { now: NOW, appleOf: (i) => apple[i] || null }); + assert.deepEqual([...plan.due].sort(), ['daily', 'forced', 'fresh', 'news']); + assert.deepEqual(plan.counts, { never: 1, appleForced: 1, news: 1, dormant: 0, pace: 3, tier: 1 }); + assert.equal(plan.dueAt.get('weekly'), shows.weekly.c + 42 * HOUR_MS * jit('weekly')); + assert.equal(plan.dueAt.has('fresh'), false); + const perDay = 24 / jit('news') + 24 / (6 * jit('daily')) + 24 / (42 * jit('weekly')) + 24 / (42 * jit('forced')) + + 24 / (staleness.thresholdMs({}, 'fresh') / HOUR_MS); + assert.equal(plan.perDay, Math.round(perDay)); + assert.ok(plan.perDayNow > 0); + assert.deepEqual(shadowPlan(null, {}).due.size, 0); +}); + +test('compare: skipped, earlier, and new tips the cadence would have found later', () => { + const plan = { + due: new Set(['a', 'b', 'c']), + dueAt: new Map([['d', NOW + 5 * HOUR_MS], ['e', NOW + 1 * HOUR_MS], ['f', NOW + 30 * HOUR_MS]]), + }; + const cmp = compare(plan, ['a', 'd', 'e', 'f', 'g'], ['a', 'd', 'e', 'f', 'f'], NOW); + assert.deepEqual(cmp, { wouldCheck: 3, current: 5, skipped: 4, earlier: 2, late: 3, lateMedianH: 5, lateMaxH: 30 }); + assert.deepEqual(compare(plan, [], [], NOW), { wouldCheck: 3, current: 0, skipped: 0, earlier: 3, late: 0, lateMedianH: 0, lateMaxH: 0 }); +}); + +test('seed from the book: candidate shows get pace and newest date; others are left alone', async () => { + const book = { query: async (sql) => { + assert.match(sql, /ROW_NUMBER\(\) OVER \(PARTITION BY sid ORDER BY published_at DESC\)/); + return [['1', '11', String(NOW_S - 10 * DAY_S), String(NOW_S)], ['2', '2', '100', '200'], ['3', '11', '0', '1']]; + } }; + const shows = { 2: { p: 500, g: 7 } }; + assert.equal(await seedFromBook(book, shows, ['1', '2']), 2); + assert.deepEqual(shows, { 1: { g: 24, p: NOW_S }, 2: { p: 500, g: 7 } }, 'too few episodes keep the old pace; dates never go back'); +}); + +test('Apple snapshot: read-only lookups; a missing file means no Apple, not a failure', () => { + const missing = openApple(path.join(os.tmpdir(), 'no-such-dir', 'apple_latest.db')); + assert.ok(missing.error); + assert.equal(missing.lookup(1), null); + missing.close(); + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'cadence-')); + const file = path.join(dir, 'apple_latest.db'); + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(file); + db.exec('CREATE TABLE apple_latest (itunes_id INTEGER PRIMARY KEY, release_date INTEGER, track_count INTEGER, verified_at INTEGER)'); + db.prepare('INSERT INTO apple_latest VALUES (?, ?, ?, ?)').run(1200361736, 1792403100, 2754, 1792428098); + db.close(); + const apple = openApple(file); + assert.equal(apple.error, null); + assert.deepEqual(apple.lookup(1200361736), { releaseDate: 1792403100, verifiedAt: 1792428098 }); + assert.equal(apple.lookup(5), null); + assert.throws(() => new DatabaseSync(file, { readOnly: true }).exec('DELETE FROM apple_latest')); + apple.close(); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +test('summary: one report line', () => { + const plan = { counts: { never: 2, appleForced: 3, news: 4, dormant: 5, pace: 6, tier: 7 }, perDay: 900, perDayNow: 1000 }; + const cmp = { wouldCheck: 10, current: 12, skipped: 4, earlier: 2, late: 2, lateMedianH: 2.25, lateMaxH: 30.4 }; + assert.equal(summary('shadow', plan, cmp, { paceShows: 6, candidates: 24 }), + 'cadence shadow: would check 10 (today 12; 4 skipped, 2 earlier, 3 Apple-forced, 2 never checked) · ' + + '2 new tips not yet due (median 2.3 h, max 30 h late) · checks/day 900 vs 1000 · ' + + 'news 4, dormant 5, pace 6, tier 7 · pace known 25%'); + assert.match(summary('shadow', plan, { ...cmp, late: 0 }, { appleError: 'gone' }), /0 new tips not yet due · .* · Apple unread \(gone\)$/); + assert.equal(summary('off', null, null), 'cadence off'); +}); diff --git a/scripts/sync/lib/chart-countries.js b/scripts/sync/lib/chart-countries.js index 4cdce62f2..709de9288 100644 --- a/scripts/sync/lib/chart-countries.js +++ b/scripts/sync/lib/chart-countries.js @@ -133,6 +133,7 @@ function mergePodcastRowsWithCountries(podcastRows, countriesByItunes) { latestEpId, categories, medium, + itunesId: Number(itunesId), countryCsv, }); } diff --git a/scripts/sync/lib/chart-countries.test.js b/scripts/sync/lib/chart-countries.test.js index 2d438eced..f3467f27d 100644 --- a/scripts/sync/lib/chart-countries.test.js +++ b/scripts/sync/lib/chart-countries.test.js @@ -34,6 +34,7 @@ describe('chart-countries', () => { latestEpId: 'e1', categories: 'News', medium: 'podcast', + itunesId: 111, countryCsv: 'us,gb', }, { @@ -41,6 +42,7 @@ describe('chart-countries', () => { latestEpId: null, categories: '', medium: 'podcast', + itunesId: 222, countryCsv: 'de', }, ]); diff --git a/scripts/sync/lib/config.js b/scripts/sync/lib/config.js index 3b44c5060..cd5214209 100644 --- a/scripts/sync/lib/config.js +++ b/scripts/sync/lib/config.js @@ -157,6 +157,10 @@ module.exports = { // records, stage 4 compares) | on (stage 4 reads the saved list instead of fetching it again). WINDOWS_MODE: ['off', 'shadow', 'on'].includes(process.env.WINDOWS_MODE) ? process.env.WINDOWS_MODE : 'off', REGISTRY_DB: process.env.REGISTRY_DB || '/var/lib/boxlore-registry/published/registry.db', + // Per-show check cadence (own-episode-IDs P1b, lib/cadence.js): off | shadow (stage 3 reports + // what the cadence would check; checks still follow the tiers above). P3b switches it on. + CADENCE_MODE: process.env.CADENCE_MODE === 'off' ? 'off' : 'shadow', + APPLE_LATEST_DB: process.env.APPLE_LATEST_DB || '/var/lib/boxlore-applefeeds/published/apple_latest.db', // --- Cleanup --- CLEANUP_GRACE_DAYS: 7, diff --git a/scripts/sync/lib/stale-tips.js b/scripts/sync/lib/stale-tips.js index 4d05c4515..8d09b9ad8 100644 --- a/scripts/sync/lib/stale-tips.js +++ b/scripts/sync/lib/stale-tips.js @@ -104,7 +104,7 @@ async function runStaleTips(opts) { const { mode, due, db, feedsUrl, registryPath, fetchImpl = fetch, warn = () => {} } = opts; const mediumOf = opts.mediumOf || (() => 'podcast'); const now = opts.now || Date.now; - const out = { mode, stale: 0, answered: 0, newer: 0, updated: 0, noItem: 0, failedChunks: 0, error: null, updatedIds: [] }; + const out = { mode, stale: 0, answered: 0, newer: 0, updated: 0, noItem: 0, failedChunks: 0, error: null, updatedIds: [], tipDates: {} }; if (mode !== 'shadow' && mode !== 'on') return out; let flagged; @@ -156,6 +156,7 @@ async function runStaleTips(opts) { await db.execute(TIP_SQL, tipUpdateArgs(r.item, mediumOf(showId), now(), showId, feedIdentity(showId, r.item))); out.updated++; out.updatedIds.push(showId); + out.tipDates[showId] = Number(r.item.datePublished) || 0; } catch (e) { warn(`[R5] show ${showId}: tip write failed (${e.message})`); } diff --git a/scripts/sync/lib/stale-tips.test.js b/scripts/sync/lib/stale-tips.test.js index 6be0abb0b..7de8479da 100644 --- a/scripts/sync/lib/stale-tips.test.js +++ b/scripts/sync/lib/stale-tips.test.js @@ -79,6 +79,7 @@ describe('stale tips (R5)', () => { assert.equal(args[16], 7); assert.match(summary(r), /R5 on: 3 stale due, 3 answered, 1 tips from the feed, 1 without a served ID yet/); assert.deepEqual(r.updatedIds, [7], 'stage 3 saves these shows\' feed windows'); + assert.deepEqual(r.tipDates, { 7: NEW }, 'the check cadence keeps the new tip\'s date'); }); it('shadow: counts newer feed episodes from the date summary, never asks for IDs, never writes', async () => { diff --git a/scripts/sync/lib/state.js b/scripts/sync/lib/state.js index 2290edf0e..a47167566 100644 --- a/scripts/sync/lib/state.js +++ b/scripts/sync/lib/state.js @@ -16,7 +16,9 @@ * "x": 1 (present only when show is on relaxed-check countries only — 48h), * "s": , * "k": , - * "d": + * "d": , + * "g": , "p": , "i": + * (check cadence, lib/cadence.js) * } * } * } From 5429a560a7dcbf903129372a0311d0a4e5d57585 Mon Sep 17 00:00:00 2001 From: Aswin C Date: Fri, 23 Oct 2026 01:39:42 +0530 Subject: [PATCH 2/2] fix(sync): read cadence pace from the book in bounded chunks; require Node 22.13 --- CONTRIBUTING.md | 2 +- scripts/README.md | 6 +++--- scripts/package.json | 2 +- scripts/sync/lib/cadence.js | 32 +++++++++++++++++++------------- scripts/sync/lib/cadence.test.js | 21 ++++++++++++++++----- 5 files changed, 40 insertions(+), 23 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 53bed32f8..4a1604f11 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,7 +47,7 @@ In accordance with the [PolyForm Strict License 1.0.0](LICENSE), you are welcome | Java | JDK 17; set `JAVA_HOME` and the Android Studio Gradle JDK to this installation | | Android SDK | Platform 36, Platform-Tools, and the Build-Tools version requested by the Android Gradle plugin | | Gradle | Use the checked-in `./gradlew` wrapper; no separate Gradle installation is needed | -| Node.js | Version 20 and npm for scripts; not needed for ordinary Android builds | +| Node.js | Version 22.13 or newer and npm for scripts; not needed for ordinary Android builds | | Python | Python 3 for local configuration and release-tooling tests | | Device | Android 12 / API 31 or newer when running the app | diff --git a/scripts/README.md b/scripts/README.md index c59850b98..af3c2e7e4 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -6,7 +6,7 @@ Development guidance for catalog synchronization, notification checks, feed repa Catalog sync refreshes charts, imports podcasts, syncs episodes, and vectorizes content. It runs outside GitHub Actions; the former `sync-pi-data` workflow has been removed. Adding a replacement GitHub sync workflow requires explicit maintainer approval. -Scripts need Node.js 22.5 or newer (`engines` in `scripts/package.json`): stage 3 reads the feed-address registry with `node:sqlite`. The runner uses Node 22, and so do the workflows that install `scripts/`. +Scripts need Node.js 22.13 or newer (`engines` in `scripts/package.json`): stage 3 reads the feed-address registry and Apple's snapshot with `node:sqlite`, which needs a flag before 22.13. The runner uses Node 22, and so do the workflows that install `scripts/`. A Git push does not deploy this pipeline. Changes to `scripts/sync/` or `scripts/package.json` require a separate deployment and verification that the active runner uses the updated code and configuration. Read the production runbook in root `AGENTS.internal.md` before deployment. If it is unavailable, obtain it from the maintainer before performing production operations. @@ -33,7 +33,7 @@ A Git push does not deploy this pipeline. Changes to `scripts/sync/` or `scripts | [`scripts/sync/lib/catalog-rows.js`](sync/lib/catalog-rows.js) | Scrubbed values for stage 2's API import and stage 3's latest-episode update: an odd optional field from Podcast Index becomes empty instead of failing the write. The latest-episode update stores the episode's served ID in `latest_ep_id` (what Home, trending and bootstrap send), plus `latest_ep_key` and `latest_ep_eid`, from `tipIdentity` in `lib/episode-identity.js` (by catalog key, else Podcast Index's ID as an alias in the same show, else that ID). Change detection and the tip queue keep Podcast Index's own ID | | [`scripts/sync/lib/stale-tips.js`](sync/lib/stale-tips.js) | Stage 3's tips for stale shows (`R5_MODE`: `off` by default, `shadow`, `on`). After the Podcast Index pass, checked chart shows that the published feed-address registry (`REGISTRY_DB`) flags `episode_stale` ask the local feed reader (`FEEDS_URL`, `POST /feeds/latest`, 40 shows per call) for their newest episode. When it is playable, a different episode and newer than the stored tip, it becomes the tip under its served ID, catalog key and recipe ID. Only the tip columns change: `qdrant_vectorized` and the tip queue stay with the Podcast Index pass. `shadow` reads only the feed's dates and never writes. A registry or feed-reader failure skips it, and the Podcast Index tips stay | | [`scripts/sync/lib/show-window.js`](sync/lib/show-window.js) | Saved episode windows (`WINDOWS_MODE`: `off` by default, `shadow`, `on`). Stage 3 records each full list it fetches as episode-book rows, with descriptions, and a `show_window` row: the book row IDs in list order. It fetches at the cap stage 4 enforces, chart or earned. Shows whose R5 tip came from the feed also save the feed reader's merged list (`GET /feeds/episodes`), so their feed-only episodes get vectors. With `on`, stage 4 reads a saved window instead of fetching the list from Podcast Index, when it is usable, fresh (3 days) and long enough. `shadow` records windows and has stage 4 compare them with its own fetch. A window with an episode that has no catalog key, or that another show serves, is unusable, and stage 4 takes the fetch path. Tip-queue rows carry `gen`, and stage 4 clears only rows enqueued before it started | -| [`scripts/sync/lib/cadence.js`](sync/lib/cadence.js) | Per-show check schedule for stage 3, in shadow (`CADENCE_MODE`: `shadow` by default, `off`). Checks still follow the news/core/relaxed tiers; the run report adds what the schedule would have checked. Each show's interval follows its pace, the average gap between its newest 11 episodes: news shows hourly, others a quarter of the gap within 3 to 48 hours, and dormant shows weekly when both the newest episode and a fresh Apple answer are over 180 days old. A show with no known pace keeps its tier. A show Apple lists a newer episode for (`APPLE_LATEST_DB`, the applefeeds snapshot) is due until it has been checked 6 hours past that date. Pace and the newest date come from each fetched list and from the episode book about daily; show records gain `g`, `p` and `i` (iTunes ID). The report line counts shows the schedule would skip or check earlier, new tips it would have found later and by how much, and checks a day against today's tiers. A book or snapshot failure only skips that part | +| [`scripts/sync/lib/cadence.js`](sync/lib/cadence.js) | Per-show check schedule for stage 3, in shadow (`CADENCE_MODE`: `shadow` by default, `off`). Checks still follow the news/core/relaxed tiers; the run report adds what the schedule would have checked. Each show's interval follows its pace, the average gap between its newest 11 episodes: news shows hourly, others a quarter of the gap within 3 to 48 hours, and dormant shows weekly when both the newest episode and a fresh Apple answer are over 180 days old. A show with no known pace keeps its tier. A show Apple lists a newer episode for (`APPLE_LATEST_DB`, the applefeeds snapshot) is due until it has been checked 6 hours past that date. Pace and the newest date come from each fetched list and from the episode book about daily (400 shows per read); show records gain `g`, `p` and `i` (iTunes ID). The report line counts shows the schedule would skip or check earlier, new tips it would have found later and by how much, and checks a day against today's tiers. A book or snapshot failure only skips that part | | [`scripts/sync/lib/attribution.js`](sync/lib/attribution.js) | Vector attribution log: one JSON line per episode point stored or deleted (prune, cap trim), show vector stored, and show imported or deleted. Written to `SYNC_ATTRIBUTION_DIR` (default `scripts/data/attribution/`, gitignored) as daily files, gzipped after the day ends and kept 60 days. Best effort: a write failure never fails a stage. Stage 4 adds `stored`, `pruned` and `reasons …` to its run line | | [`scripts/sync/attribution-report.js`](sync/attribution-report.js) | Read-only report over that log: embeddings by reason (new episode, first coverage, returning show, re-embed after cap trim or prune, Podcast Index ID change, backfill), deletions, and how long chart entrants stay. `--days N`, `--json` | | [`scripts/sync/lib/episode-book.js`](sync/lib/episode-book.js) + [`episode-book-cli.js`](sync/episode-book-cli.js) | The episode book in sqld: one permanent row per (show, catalog key) with its recipe ID and served ID, plus Podcast Index aliases, `served_show` and `show_window`. Every book write also marks the show in `served_show` when one of its episodes is listed by Podcast Index under an ID other than its served ID (renumbered, duplicated or recipe-served), so the Worker's v1 overlay translates that ID back. That check reaches a show's aliases through `episode_pi_alias_rid` (an index on `rid`); without it each check scanned every alias inside the write. `ensure` creates the tables, indexes and columns; `stats` counts. Writes go through `turso.atomic()` (one Hrana batch: every step runs only after the previous one succeeded, then COMMIT, else ROLLBACK). Rules: [`docs/EPISODE_IDS.md`](../docs/EPISODE_IDS.md) | @@ -51,7 +51,7 @@ The runner orchestrator lives outside this repository. It invokes the stages in - Preserve the chart index: `charts.itunes_id` is TEXT, so casting that column to INTEGER disables the index and inflates `rows_read`. Use `c.itunes_id = CAST(p.itunes_id AS TEXT)`, [`sync/lib/chart-countries.js`](sync/lib/chart-countries.js) (`loadCountriesByItunesId` plus a JavaScript filter), or page podcasts by `id` and normalize IDs in JavaScript. This applies to stages 2, 3, 4, 5, and remediation. - Keep credentials and environment files out of Git, including `.env`, Podcast Index keys, Turso tokens, and Telegram secrets. -Use Node.js 20 to match the script workflows. From the repository root: +Use Node.js 22.13 or newer to match the runner and the script workflows. From the repository root: ```bash npm ci --prefix scripts diff --git a/scripts/package.json b/scripts/package.json index 62d4b13d4..e95ff76fb 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -29,6 +29,6 @@ "test:check-new-episodes": "node --test check-new-episodes-lib.test.js" }, "engines": { - "node": ">=22.5" + "node": ">=22.13" } } diff --git a/scripts/sync/lib/cadence.js b/scripts/sync/lib/cadence.js index 014d34f65..fb8ef664d 100644 --- a/scripts/sync/lib/cadence.js +++ b/scripts/sync/lib/cadence.js @@ -164,14 +164,19 @@ function compare(plan, currentDue, newTipIds, now = Date.now()) { }; } -/** Pace per show from the book's newest 11 dated episodes (one aggregate read, ~2 s for 52k shows). */ -const BOOK_PACE_SQL = ` +/** Shows per book read: each answer is at most this many rows (sqld's RESPONSE_TOO_LARGE). */ +const BOOK_CHUNK = 400; + +/** Pace for a chunk of shows from their newest 11 dated book episodes (index `episode_book_sid_date`). */ +function bookPaceSql(n) { + return ` SELECT sid, COUNT(*), MIN(published_at), MAX(published_at) FROM (SELECT sid, published_at, ROW_NUMBER() OVER (PARTITION BY sid ORDER BY published_at DESC) AS rn - FROM episode_book WHERE published_at > 0) + FROM episode_book WHERE sid IN (${Array(n).fill('?').join(',')}) AND published_at > 0) WHERE rn <= ${PACE_DATES} GROUP BY sid`; +} /** * Seeds g and p for candidate shows from the book (stage 3's saved windows and the bootstrap), so @@ -179,16 +184,17 @@ const BOOK_PACE_SQL = ` * @returns {number} shows seeded */ async function seedFromBook(book, shows, candidateIds) { - const wanted = new Set((candidateIds || []).map(String)); + const ids = [...new Set((candidateIds || []).map(String))].filter((id) => /^[1-9][0-9]*$/.test(id)); let seeded = 0; - for (const [sid, count, oldest, newest] of await book.query(BOOK_PACE_SQL)) { - const id = String(sid); - if (!wanted.has(id)) continue; - const rec = shows[id] || (shows[id] = {}); - const g = paceFromSpan(count, oldest, newest); - if (g != null) rec.g = g; - noteNewest(rec, newest); - seeded++; + for (let i = 0; i < ids.length; i += BOOK_CHUNK) { + const chunk = ids.slice(i, i + BOOK_CHUNK); + for (const [sid, count, oldest, newest] of await book.query(bookPaceSql(chunk.length), chunk.map(Number))) { + const rec = shows[String(sid)] || (shows[String(sid)] = {}); + const g = paceFromSpan(count, oldest, newest); + if (g != null) rec.g = g; + noteNewest(rec, newest); + seeded++; + } } return seeded; } @@ -241,6 +247,6 @@ module.exports = { seedFromBook, openApple, summary, - BOOK_PACE_SQL, + BOOK_CHUNK, HOUR_MS, }; diff --git a/scripts/sync/lib/cadence.test.js b/scripts/sync/lib/cadence.test.js index 16ba2fe45..63f5c8021 100644 --- a/scripts/sync/lib/cadence.test.js +++ b/scripts/sync/lib/cadence.test.js @@ -8,7 +8,7 @@ const path = require('node:path'); const staleness = require('./staleness'); const { paceFromDates, paceFromSpan, notePace, noteNewest, intervalFor, appleForces, - shadowPlan, compare, seedFromBook, openApple, summary, HOUR_MS, + shadowPlan, compare, seedFromBook, openApple, summary, BOOK_CHUNK, HOUR_MS, } = require('./cadence'); const NOW = 1_792_700_000_000; @@ -99,14 +99,25 @@ test('compare: skipped, earlier, and new tips the cadence would have found later assert.deepEqual(compare(plan, [], [], NOW), { wouldCheck: 3, current: 0, skipped: 0, earlier: 3, late: 0, lateMedianH: 0, lateMaxH: 0 }); }); -test('seed from the book: candidate shows get pace and newest date; others are left alone', async () => { - const book = { query: async (sql) => { +test('seed from the book: candidate shows get pace and newest date, read in bounded chunks', async () => { + const rows = { 1: ['1', '11', String(NOW_S - 10 * DAY_S), String(NOW_S)], 2: ['2', '2', '100', '200'] }; + const calls = []; + const book = { query: async (sql, args) => { assert.match(sql, /ROW_NUMBER\(\) OVER \(PARTITION BY sid ORDER BY published_at DESC\)/); - return [['1', '11', String(NOW_S - 10 * DAY_S), String(NOW_S)], ['2', '2', '100', '200'], ['3', '11', '0', '1']]; + assert.equal((sql.match(/\?/g) || []).length, args.length, 'one placeholder per show'); + calls.push(args); + return args.filter((id) => rows[id]).map((id) => rows[id]); } }; const shows = { 2: { p: 500, g: 7 } }; - assert.equal(await seedFromBook(book, shows, ['1', '2']), 2); + assert.equal(await seedFromBook(book, shows, ['1', '2', '2', 'x']), 2); + assert.deepEqual(calls, [[1, 2]], 'deduplicated numeric IDs'); assert.deepEqual(shows, { 1: { g: 24, p: NOW_S }, 2: { p: 500, g: 7 } }, 'too few episodes keep the old pace; dates never go back'); + + calls.length = 0; + const many = Array.from({ length: BOOK_CHUNK * 2 + 5 }, (_, i) => String(i + 1)); + await seedFromBook(book, {}, many); + assert.deepEqual(calls.map((c) => c.length), [BOOK_CHUNK, BOOK_CHUNK, 5]); + assert.equal(await seedFromBook(book, {}, []), 0); }); test('Apple snapshot: read-only lookups; a missing file means no Apple, not a failure', () => {