Auto-approve submissions from our own crawlers (0.3.1)

README.md

@@ -87,6 +87,7 @@ would silently repoint everyone's follows at other shows.
 | `RESEND_API_KEY` | sending the magic link. Without it, outside production, the link is printed to the console instead |
 | `SUBMIT_FORWARD_TO` | where `/api/submit` also sends a listed feed. Defaults to `https://rssamplifier.com` in production; set it empty to stop forwarding |
 | `ADMIN_EMAILS` | comma-separated addresses that may review submissions at `/admin/submissions` and get an email per submission. Defaults to the operator's address in `src/lib/review.ts` |
+| `CRAWLER_TOKENS` | comma-separated `name:token` pairs for our own crawlers. A `POST /api/submit` with header `x-crawler-token: <token>` is recorded as `submitted_by = crawler:<name>`, skips the hourly per-address limit, and a feed that passes the rules is listed at once (no review queue, no admin email). Everything else still waits for review |
 | `COINPAY_X402_KEY`, `CRAWL_PAY_TO` | selling crawl access to AI training crawlers (see below). Until both are set, training crawlers get `402` with an empty offer: nothing is sold, nothing is given away |
 
 ## Rebuilding the data

package.json

@@ -1,6 +1,6 @@
 {
   "name": "p0dcasters",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "private": true,
   "type": "module",
   "scripts": {

src/app/api/submit/route.ts

@@ -3,6 +3,8 @@ import { parseSubmission, submitReply, wantsHtml } from "@profullstack/submit-fe
 import type { AcceptedFeed, RejectedFeed } from "@profullstack/submit-feed";
 import { currentUser, origin } from "@/lib/auth/session";
 import { bearerPrincipal, SCOPE_SUBMIT } from "@/lib/openaccess";
+import { CRAWLER_HEADER, crawlerSource } from "@/lib/crawler-submit";
+import { env } from "@/lib/auth/crypto";
 import {
   MAX_URLS,
   RATE_LIMIT,
@@ -34,6 +36,8 @@ export const maxDuration = 60;
  * No account is needed; the budget is twenty requests an hour per address.
  * A signed-in listener, or an OpenAccess principal holding podcasts:submit,
  * is remembered on the row as the submitter, which the reviewer sees.
+ * A request carrying a valid x-crawler-token is one of our own crawlers: its
+ * row's source is `crawler:<name>` and a passing feed is listed at once.
  */
 export async function POST(req: Request) {
   const html = wantsHtml(req.headers.get("accept"));
@@ -61,7 +65,10 @@ export async function POST(req: Request) {
   }
 
   const ipHash = hashIp(clientIp(req));
-  if ((await requestsThisHour(ipHash)) >= RATE_LIMIT) {
+  // One of our own crawlers (src/lib/crawler-submit.ts): its feeds are listed
+  // without review and it is not held to the per-address hourly budget.
+  const crawler = crawlerSource(req.headers.get(CRAWLER_HEADER), env("CRAWLER_TOKENS"));
+  if (!crawler && (await requestsThisHour(ipHash)) >= RATE_LIMIT) {
     return html
       ? back("rate-limited")
       : Response.json(submitReply({ ok: false, error: "rate-limited", retryAfterSeconds: 3600 }), {
@@ -71,7 +78,7 @@ export async function POST(req: Request) {
   }
 
   const [user, principal] = await Promise.all([currentUser(), bearerPrincipal(req)]);
-  const submittedBy = user?.email ?? (principal?.scopes.includes(SCOPE_SUBMIT) ? principal.sub : null);
+  const submittedBy = crawler ?? user?.email ?? (principal?.scopes.includes(SCOPE_SUBMIT) ? principal.sub : null);
   const { batchId, ids } = await createBatch(s.urls, { userId: user?.id ?? null, ipHash, submittedBy });
   const statusUrl = `${site}/submissions/${batchId}`;
   const rejected: RejectedFeed[] = s.invalid.map((url) => ({ url, error: "invalid-url" }));

src/lib/crawler-submit.ts

@@ -0,0 +1,58 @@
+import { timingSafeEqual } from "node:crypto";
+
+/**
+ * Submissions that come from our own crawlers rather than from a person.
+ *
+ * The review queue (src/lib/review.ts) exists so a stranger cannot list a show
+ * without someone looking at it. A feed our own pipeline found (a bulk import
+ * of a publisher list, a discovery crawl) has already been chosen by us, so it
+ * should be listed as soon as it passes the directory's rules, with no admin
+ * email. Anthony, 2026-10-28: "auto approve this unless it's a manual
+ * submission not our crawler".
+ *
+ * A crawler proves it is ours with a shared secret in the `x-crawler-token`
+ * header. CRAWLER_TOKENS holds one `name:token` pair per crawler, comma
+ * separated; the name is recorded as the submission's source
+ * (`submitted_by = crawler:<name>`), which is what resolveSubmission keys
+ * auto-approval on, so a batch drained later (after a restart) is still
+ * approved. A user agent alone proves nothing and earns nothing here.
+ *
+ * Pure apart from node:crypto, so a test can import it.
+ */
+
+export const CRAWLER_PREFIX = "crawler:";
+export const CRAWLER_HEADER = "x-crawler-token";
+
+/** `name:token,name2:token2` -> [[name, token], ...]; malformed pairs are dropped. */
+export function parseCrawlerTokens(raw: string | null | undefined): Array<[string, string]> {
+  const out: Array<[string, string]> = [];
+  for (const part of (raw ?? "").split(",")) {
+    const i = part.indexOf(":");
+    if (i <= 0) continue;
+    const name = part.slice(0, i).trim().toLowerCase().replace(/[^a-z0-9._-]/g, "");
+    const token = part.slice(i + 1).trim();
+    if (name && token.length >= 16) out.push([name, token]);
+  }
+  return out;
+}
+
+function same(a: string, b: string): boolean {
+  const x = Buffer.from(a);
+  const y = Buffer.from(b);
+  return x.length === y.length && timingSafeEqual(x, y);
+}
+
+/** The crawler's source label (`crawler:<name>`) when the token matches one, else null. */
+export function crawlerSource(token: string | null | undefined, raw: string | null | undefined): string | null {
+  const t = (token ?? "").trim();
+  if (!t) return null;
+  for (const [name, secret] of parseCrawlerTokens(raw)) {
+    if (same(t, secret)) return `${CRAWLER_PREFIX}${name}`;
+  }
+  return null;
+}
+
+/** True when a submission row's source is one of our crawlers. */
+export function isCrawlerSource(submittedBy: string | null | undefined): boolean {
+  return Boolean(submittedBy && submittedBy.startsWith(CRAWLER_PREFIX) && submittedBy.length > CRAWLER_PREFIX.length);
+}

src/lib/submit.ts

@@ -5,6 +5,7 @@ import type { Podcast } from "@/lib/db";
 import { env, now, secret, sha256, token } from "@/lib/auth/crypto";
 import { normalizeLang } from "@/lib/format";
 import { notifyReviewers } from "@/lib/review";
+import { isCrawlerSource } from "@/lib/crawler-submit";
 
 /**
  * Feed submission, the site-specific half.
@@ -187,6 +188,17 @@ export async function resolveSubmission(id: string, opts: { notify?: boolean } =
   });
 
   if (outcome.status === "listed") void forward(outcome.feedUrl);
+
+  // One of our own crawlers found this feed (src/lib/crawler-submit.ts): it
+  // passed the rules, so list it now instead of queueing it for a person.
+  if (outcome.status === "review" && isCrawlerSource(row.submitted_by)) {
+    const d = await decideSubmission(id, "approve", `auto:${row.submitted_by}`, "auto-approved: crawler submission", { reread: false });
+    if (d.ok && d.status !== "rejected" && d.slug) {
+      return { status: d.status, slug: d.slug, title: d.title ?? outcome.title, feedUrl: outcome.feedUrl };
+    }
+    return outcome;
+  }
+
   if (outcome.status === "review" && opts.notify !== false) {
     void notifyReviewers({
       title: outcome.title,
@@ -236,7 +248,13 @@ export type Decision =
  * Reject: the row records who said no and why, and the same feed can be
  * submitted again once whatever it was is fixed.
  */
-export async function decideSubmission(id: string, decision: "approve" | "reject", by: string, reason: string | null): Promise<Decision> {
+export async function decideSubmission(
+  id: string,
+  decision: "approve" | "reject",
+  by: string,
+  reason: string | null,
+  opts: { reread?: boolean } = {},
+): Promise<Decision> {
   const row = await submissionRow(id);
   if (!row) return { ok: false, error: "no such submission", status: 404 };
   if (row.status !== "review") return { ok: false, error: `submission is ${row.status}, not waiting for review`, status: 409 };
@@ -266,11 +284,14 @@ export async function decideSubmission(id: string, decision: "approve" | "reject
     return { ok: false, error: "the submission carries no show to list; submit the feed again", status: 409 };
   }
   let fresh: Omit<Podcast, "id" | "slug"> | null = null;
-  try {
-    const r = await resolveFeed(feedUrl, { kind: "podcast", timeoutMs: 10_000, userAgent: USER_AGENT, maxCandidates: 0 });
-    if (r.ok && r.feed.title) fresh = rowFromFeed(r.feed, feedUrl, stored.host);
-  } catch {
-    /* the snapshot stands */
+  // reread: false when the snapshot was taken moments ago (auto-approval).
+  if (opts.reread !== false) {
+    try {
+      const r = await resolveFeed(feedUrl, { kind: "podcast", timeoutMs: 10_000, userAgent: USER_AGENT, maxCandidates: 0 });
+      if (r.ok && r.feed.title) fresh = rowFromFeed(r.feed, feedUrl, stored.host);
+    } catch {
+      /* the snapshot stands */
+    }
   }
   const toList = fresh ?? stored;
   const slug = await claimSlug(toList.title, feedUrl);

test/crawler-submit.test.mjs

@@ -0,0 +1,37 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { parseCrawlerTokens, crawlerSource, isCrawlerSource } from '../src/lib/crawler-submit.ts';
+
+const RAW = 'podcorn:0123456789abcdef0123, Discovery : fedcba9876543210fedc ,bad,short:abc,:nonamexxxxxxxxxxxxxx';
+
+test('parses name:token pairs and drops malformed or short ones', () => {
+  assert.deepEqual(parseCrawlerTokens(RAW), [
+    ['podcorn', '0123456789abcdef0123'],
+    ['discovery', 'fedcba9876543210fedc'],
+  ]);
+  assert.deepEqual(parseCrawlerTokens(undefined), []);
+  assert.deepEqual(parseCrawlerTokens(''), []);
+});
+
+test('a matching token names the crawler as the source', () => {
+  assert.equal(crawlerSource('0123456789abcdef0123', RAW), 'crawler:podcorn');
+  assert.equal(crawlerSource(' fedcba9876543210fedc ', RAW), 'crawler:discovery');
+});
+
+test('no token, a wrong token, or no configured tokens is a manual submission', () => {
+  assert.equal(crawlerSource(null, RAW), null);
+  assert.equal(crawlerSource('', RAW), null);
+  assert.equal(crawlerSource('0123456789abcdef0124', RAW), null);
+  assert.equal(crawlerSource('0123456789abcdef012', RAW), null);
+  assert.equal(crawlerSource('0123456789abcdef0123', undefined), null);
+  assert.equal(crawlerSource('feed-submit/1.0', RAW), null);
+});
+
+test('only crawler:<name> sources auto-approve', () => {
+  assert.equal(isCrawlerSource('crawler:podcorn'), true);
+  assert.equal(isCrawlerSource('crawler:'), false);
+  assert.equal(isCrawlerSource(null), false);
+  assert.equal(isCrawlerSource(''), false);
+  assert.equal(isCrawlerSource('anthony@profullstack.com'), false);
+  assert.equal(isCrawlerSource('network:bbc'), false);
+});