Adaptive show crawler + Refresh button (0.5.0)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "p0dcasters",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "private": true,
   "type": "module",
   "scripts": {
@@ -18,7 +18,7 @@
     "@profullstack/openprofile": "^0.2.0",
     "@profullstack/player": "0.16.0",
     "@profullstack/profile": "0.1.6",
-    "@profullstack/scraper-daemon": "0.2.2",
+    "@profullstack/scraper-daemon": "0.3.0",
     "@profullstack/submit-feed": "^0.1.3",
     "@profullstack/throttle": "^0.4.1",
     "@profullstack/x402-gateway": "^0.16.1",

scripts/daemon-lib.mjs

@@ -78,6 +78,7 @@ export async function headStamp(fetchImpl, url = DUMP_URL, { signal, timeoutMs =
  * @param {{acquire(): Promise<boolean>, release(): void}} deps.lock   the shell's flock
  * @param {(s: {job: string, lastOk: number|null, lastError: string|null}) => Promise<void>} [deps.alert]
  * @param {() => Promise<object>} [deps.newsletter]  one pass of the weekly digest (src/lib/newsletter.ts)
+ * @param {object} [deps.crawl]  the show crawler (src/lib/feed-crawl.ts): { lib, q, concurrency }
  * @param {object} [deps.bin]   { node, bun, python }
  * @param {string} [deps.repo]  the checkout the scripts live in
  */
@@ -94,6 +95,7 @@ export function buildDaemon(deps) {
     lock,
     alert,
     newsletter,
+    crawl = null,
     console: out = console,
     now = Date.now,
     minRows = 10_000,
@@ -352,8 +354,9 @@ export function buildDaemon(deps) {
     redis,
     store,
     // Two: the directory pipeline runs for hours and must not hold the weekly
-    // digest back. Each task still runs one at a time on its own.
-    concurrency: 2,
+    // digest back. Each task still runs one at a time on its own. The show
+    // crawler's reads come on top, so it never waits behind either.
+    concurrency: 2 + (crawl ? crawl.concurrency + 1 : 0),
     onRun,
     onStall: async (s) => {
       out.error(`[directory] stalled: no success since ${s.lastOk ? new Date(s.lastOk).toISOString() : "start"}${s.lastError ? `: ${s.lastError}` : ""}`);
@@ -380,5 +383,52 @@ export function buildDaemon(deps) {
       },
     });
   }
+  if (crawl) addShowCrawler(daemon, crawl, out);
   return daemon;
 }
+
+/** Feeds read per tier at once from one host: anchor.fm alone is 116k shows. */
+export const SHOWS_PER_HOST = 4;
+export const SHOWS_JOB = "shows";
+
+/**
+ * The show crawler: every listed show's feed on its own rhythm (see
+ * src/lib/feed-crawl.ts), plus a job that keeps feed_state in step with the
+ * podcasts table the dump reload swaps in.
+ *
+ * @param {ReturnType<import('@profullstack/scraper-daemon').createDaemon>} daemon
+ * @param {{ lib: typeof import('../src/lib/feed-crawl.ts'), q: Function, concurrency: number, tiers?: string[] }} crawl
+ */
+export function addShowCrawler(daemon, { lib, q, concurrency, tiers }, out = console) {
+  const TIERS = tiers ?? ["hot", "active", "slow", "dormant", "archived"];
+  daemon.job({
+    name: "shows-seed",
+    every: "30m",
+    timeout: "20m",
+    async run() {
+      const seeded = await lib.seedFeedState(q);
+      // After a reload the fresh rows carry the dump's dates; put back anything newer we read.
+      const reapplied = await lib.applyToPodcasts(q);
+      return { ...seeded, reapplied };
+    },
+  });
+  daemon.feeds({
+    name: SHOWS_JOB,
+    concurrency,
+    perHost: SHOWS_PER_HOST,
+    timeout: "1m",
+    staleAfter: "2h",
+    due: (tier, limit) => lib.dueFeeds(q, TIERS.indexOf(tier), limit),
+    run: async (feed, ctx) => {
+      const r = await lib.readFeed(feed, ctx.signal);
+      return { dates: r.dates, changed: !r.notModified, crawl: r };
+    },
+    save: (feed, plan, result) => lib.saveCrawl(q, feed, plan, result.crawl),
+    fail: (feed, f) => lib.saveFailure(q, feed, f),
+    previous: (feed) => feed.minutes,
+    fails: (feed) => feed.fails,
+    // Ten failures in a row and a show is read once a quarter, like an archived one.
+    retireAfter: 10,
+  });
+  out.log?.(`[p0dcasters.com] show crawler on: ${concurrency} at once, ${SHOWS_PER_HOST} per host`);
+}

scripts/daemon.mjs

@@ -29,6 +29,7 @@ import { buildDaemon, pathsFor } from "./daemon-lib.mjs";
 // Bun runs the TypeScript directly: the same sender the web app's admin route uses.
 import { runWeeklyNewsletter, slotFor, slotFrom, sqlStore } from "../src/lib/newsletter.ts";
 import { sendMail } from "../src/lib/auth/mail.ts";
+import * as feedCrawl from "../src/lib/feed-crawl.ts";
 
 const REPO = join(dirname(fileURLToPath(import.meta.url)), "..");
 const P = pathsFor();
@@ -234,7 +235,9 @@ if (!c) {
   console.error("p0dcasters daemon: no working DATABASE_URL; exiting (systemd retries)");
   process.exit(1);
 }
-const sql = new SQL(c.url, { max: 2 });
+// Two for the directory and the digest, plus room for the show crawler's writes.
+const CRAWL_CONCURRENCY = Number(process.env.CRAWL_CONCURRENCY ?? 24);
+const sql = new SQL(c.url, { max: CRAWL_CONCURRENCY > 0 ? 8 : 2 });
 // scraper-daemon 0.1.1+ passes state as text both ways, so Bun's SQL needs no help here.
 const store = pgStore((text, params) => sql.unsafe(text, params));
 
@@ -298,6 +301,11 @@ if (process.argv.includes("--status")) {
   process.exit(0);
 }
 
+/* ---------- the show crawler (src/lib/feed-crawl.ts) ---------- */
+
+const crawlQuery = (text, params = []) => sql.unsafe(text, params);
+if (CRAWL_CONCURRENCY > 0) await feedCrawl.ensureFeedState(crawlQuery);
+
 const daemon = buildDaemon({
   createDaemon,
   store,
@@ -310,6 +318,7 @@ const daemon = buildDaemon({
   lock,
   alert,
   newsletter: () => newsletterPass(),
+  crawl: CRAWL_CONCURRENCY > 0 ? { lib: feedCrawl, q: crawlQuery, concurrency: CRAWL_CONCURRENCY } : null,
   repo: REPO,
 });
 

src/app/api/episodes/[slug]/refresh/route.ts

@@ -0,0 +1,48 @@
+import { revalidatePath } from "next/cache";
+import { db, one } from "@/lib/db";
+import { episodeCache, fetchEpisodes } from "@/lib/feed";
+
+export const dynamic = "force-dynamic";
+
+/** A show read this recently by a refresh is answered from what we have. */
+const RECENT_MS = 2 * 60_000;
+const lastRefresh = new Map<string, number>();
+
+/**
+ * Read a show's feed now: the Refresh button on a show page.
+ *
+ * The page's episodes come straight from the feed, so dropping the cached copy
+ * and reading it again is the whole refresh as far as the reader can see. The
+ * show is also moved to the front of the crawler's queue (feed_state), so the
+ * directory row every list sorts by is rewritten by the daemon within seconds.
+ */
+export async function POST(_req: Request, { params }: { params: Promise<{ slug: string }> }) {
+  const { slug } = await params;
+  const show = await one<{ slug: string; feed_url: string }>("SELECT slug, feed_url FROM podcasts WHERE slug = ?", [slug]);
+  if (!show) return Response.json({ error: "not found" }, { status: 404 });
+
+  const last = lastRefresh.get(show.feed_url) ?? 0;
+  if (Date.now() - last < RECENT_MS) {
+    return Response.json({ ok: true, state: "recent" }, { headers: { "cache-control": "no-store" } });
+  }
+  lastRefresh.set(show.feed_url, Date.now());
+  if (lastRefresh.size > 5000) lastRefresh.clear();
+
+  const before = episodeCache.get(show.feed_url);
+  episodeCache.delete(show.feed_url);
+  const episodes = await fetchEpisodes(show.feed_url);
+  const newest = (list: { pubdate: number }[] | undefined) => (list ?? []).reduce((m, e) => Math.max(m, e.pubdate), 0);
+  const fresh = before ? episodes.filter((e) => e.pubdate > newest(before)).length : null;
+
+  try {
+    await db().execute({ sql: "UPDATE feed_state SET next_at = '1970-01-01T00:00:00Z' WHERE feed_url = ?", args: [show.feed_url] });
+  } catch {
+    // No crawler state yet (first deploy): the page is still current.
+  }
+  revalidatePath(`/podcast/${show.slug}`);
+
+  return Response.json(
+    { ok: episodes.length > 0, state: episodes.length ? "read" : "failed", episodes: episodes.length, newEpisodes: fresh, newest: newest(episodes) || null },
+    { headers: { "cache-control": "no-store" } },
+  );
+}

src/app/podcast/[slug]/page.tsx

@@ -12,6 +12,7 @@ import FollowButton from "@/components/FollowButton";
 import { LatestButton, ShowEpisodes } from "@/components/ShowEpisodes";
 import PlaylistButtons from "@/components/PlaylistButtons";
 import ShareButton from "@/components/ShareButton";
+import RefreshButton from "@/components/RefreshButton";
 import { nixampUrl, playlistUrl } from "@/lib/playlist";
 import { profileUrl } from "@/lib/openprofile/generate";
 import { cadence, languageName, titleCase, clamp, safeImage, normalizeLang } from "@/lib/format";
@@ -192,6 +193,7 @@ export default async function Show({ params }: { params: Promise<{ slug: string
               <LatestButton feedUrl={p.feed_url} show={show} />
             </Suspense>
             <FollowButton slug={p.slug} />
+            <RefreshButton slug={p.slug} />
             <a className="btn" href={p.feed_url} rel="noopener nofollow">
               RSS feed
             </a>

src/components/RefreshButton.tsx

@@ -0,0 +1,35 @@
+"use client";
+
+import { useRouter } from "next/navigation";
+import { useState } from "react";
+
+/**
+ * Read the show's feed now instead of waiting for the crawler: POSTs to
+ * /api/episodes/<slug>/refresh, then re-renders the page with what came back.
+ */
+export default function RefreshButton({ slug }: { slug: string }) {
+  const router = useRouter();
+  const [state, setState] = useState<"idle" | "busy" | "done" | "same" | "failed">("idle");
+
+  const refresh = async () => {
+    setState("busy");
+    try {
+      const res = await fetch(`/api/episodes/${encodeURIComponent(slug)}/refresh`, { method: "POST" });
+      const body = (await res.json().catch(() => ({}))) as { ok?: boolean; state?: string; newEpisodes?: number | null };
+      if (!body.ok) return setState("failed");
+      setState(body.newEpisodes ? "done" : "same");
+      router.refresh();
+    } catch {
+      setState("failed");
+    } finally {
+      window.setTimeout(() => setState("idle"), 2500);
+    }
+  };
+
+  const label = { idle: "↻ Refresh", busy: "Reading…", done: "New episodes", same: "Up to date", failed: "Feed did not answer" }[state];
+  return (
+    <button type="button" className="btn" onClick={refresh} disabled={state === "busy"} aria-live="polite" title="Read this show's feed now">
+      {label}
+    </button>
+  );
+}