From e2c05da7a2f1d172f39f33cb7f178499247041cc Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 28 Oct 2026 22:41:09 +0000 Subject: [PATCH] Auto-approve submissions from our own crawlers (0.3.1) A POST /api/submit carrying x-crawler-token that matches CRAWLER_TOKENS (name:token pairs) is recorded as submitted_by = crawler:, skips the hourly per-address limit, and a feed that passes the rules is listed at once by resolveSubmission (reviewed_by auto:crawler:), with no admin email. Manual submissions (form, MCP, signed-in, OpenAccess) still wait for review. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + package.json | 2 +- src/app/api/submit/route.ts | 11 +++++-- src/lib/crawler-submit.ts | 58 ++++++++++++++++++++++++++++++++++++ src/lib/submit.ts | 33 ++++++++++++++++---- test/crawler-submit.test.mjs | 37 +++++++++++++++++++++++ 6 files changed, 133 insertions(+), 9 deletions(-) create mode 100644 src/lib/crawler-submit.ts create mode 100644 test/crawler-submit.test.mjs diff --git a/README.md b/README.md index b601a2f..6cb20f5 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ would silently repoint everyone's follows at other shows. | `RESEND_API_KEY` | sending the magic link. Without it, outside production, the link is printed to the console instead | | `SUBMIT_FORWARD_TO` | where `/api/submit` also sends a listed feed. Defaults to `https://rssamplifier.com` in production; set it empty to stop forwarding | | `ADMIN_EMAILS` | comma-separated addresses that may review submissions at `/admin/submissions` and get an email per submission. Defaults to the operator's address in `src/lib/review.ts` | +| `CRAWLER_TOKENS` | comma-separated `name:token` pairs for our own crawlers. A `POST /api/submit` with header `x-crawler-token: ` is recorded as `submitted_by = crawler:`, skips the hourly per-address limit, and a feed that passes the rules is listed at once (no review queue, no admin email). Everything else still waits for review | | `COINPAY_X402_KEY`, `CRAWL_PAY_TO` | selling crawl access to AI training crawlers (see below). Until both are set, training crawlers get `402` with an empty offer: nothing is sold, nothing is given away | ## Rebuilding the data diff --git a/package.json b/package.json index 533a5cb..36e5642 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "p0dcasters", - "version": "0.3.0", + "version": "0.3.1", "private": true, "type": "module", "scripts": { diff --git a/src/app/api/submit/route.ts b/src/app/api/submit/route.ts index 8a08b54..129ab1a 100644 --- a/src/app/api/submit/route.ts +++ b/src/app/api/submit/route.ts @@ -3,6 +3,8 @@ import { parseSubmission, submitReply, wantsHtml } from "@profullstack/submit-fe import type { AcceptedFeed, RejectedFeed } from "@profullstack/submit-feed"; import { currentUser, origin } from "@/lib/auth/session"; import { bearerPrincipal, SCOPE_SUBMIT } from "@/lib/openaccess"; +import { CRAWLER_HEADER, crawlerSource } from "@/lib/crawler-submit"; +import { env } from "@/lib/auth/crypto"; import { MAX_URLS, RATE_LIMIT, @@ -34,6 +36,8 @@ export const maxDuration = 60; * No account is needed; the budget is twenty requests an hour per address. * A signed-in listener, or an OpenAccess principal holding podcasts:submit, * is remembered on the row as the submitter, which the reviewer sees. + * A request carrying a valid x-crawler-token is one of our own crawlers: its + * row's source is `crawler:` and a passing feed is listed at once. */ export async function POST(req: Request) { const html = wantsHtml(req.headers.get("accept")); @@ -61,7 +65,10 @@ export async function POST(req: Request) { } const ipHash = hashIp(clientIp(req)); - if ((await requestsThisHour(ipHash)) >= RATE_LIMIT) { + // One of our own crawlers (src/lib/crawler-submit.ts): its feeds are listed + // without review and it is not held to the per-address hourly budget. + const crawler = crawlerSource(req.headers.get(CRAWLER_HEADER), env("CRAWLER_TOKENS")); + if (!crawler && (await requestsThisHour(ipHash)) >= RATE_LIMIT) { return html ? back("rate-limited") : Response.json(submitReply({ ok: false, error: "rate-limited", retryAfterSeconds: 3600 }), { @@ -71,7 +78,7 @@ export async function POST(req: Request) { } const [user, principal] = await Promise.all([currentUser(), bearerPrincipal(req)]); - const submittedBy = user?.email ?? (principal?.scopes.includes(SCOPE_SUBMIT) ? principal.sub : null); + const submittedBy = crawler ?? user?.email ?? (principal?.scopes.includes(SCOPE_SUBMIT) ? principal.sub : null); const { batchId, ids } = await createBatch(s.urls, { userId: user?.id ?? null, ipHash, submittedBy }); const statusUrl = `${site}/submissions/${batchId}`; const rejected: RejectedFeed[] = s.invalid.map((url) => ({ url, error: "invalid-url" })); diff --git a/src/lib/crawler-submit.ts b/src/lib/crawler-submit.ts new file mode 100644 index 0000000..7019b1b --- /dev/null +++ b/src/lib/crawler-submit.ts @@ -0,0 +1,58 @@ +import { timingSafeEqual } from "node:crypto"; + +/** + * Submissions that come from our own crawlers rather than from a person. + * + * The review queue (src/lib/review.ts) exists so a stranger cannot list a show + * without someone looking at it. A feed our own pipeline found (a bulk import + * of a publisher list, a discovery crawl) has already been chosen by us, so it + * should be listed as soon as it passes the directory's rules, with no admin + * email. Anthony, 2026-10-28: "auto approve this unless it's a manual + * submission not our crawler". + * + * A crawler proves it is ours with a shared secret in the `x-crawler-token` + * header. CRAWLER_TOKENS holds one `name:token` pair per crawler, comma + * separated; the name is recorded as the submission's source + * (`submitted_by = crawler:`), which is what resolveSubmission keys + * auto-approval on, so a batch drained later (after a restart) is still + * approved. A user agent alone proves nothing and earns nothing here. + * + * Pure apart from node:crypto, so a test can import it. + */ + +export const CRAWLER_PREFIX = "crawler:"; +export const CRAWLER_HEADER = "x-crawler-token"; + +/** `name:token,name2:token2` -> [[name, token], ...]; malformed pairs are dropped. */ +export function parseCrawlerTokens(raw: string | null | undefined): Array<[string, string]> { + const out: Array<[string, string]> = []; + for (const part of (raw ?? "").split(",")) { + const i = part.indexOf(":"); + if (i <= 0) continue; + const name = part.slice(0, i).trim().toLowerCase().replace(/[^a-z0-9._-]/g, ""); + const token = part.slice(i + 1).trim(); + if (name && token.length >= 16) out.push([name, token]); + } + return out; +} + +function same(a: string, b: string): boolean { + const x = Buffer.from(a); + const y = Buffer.from(b); + return x.length === y.length && timingSafeEqual(x, y); +} + +/** The crawler's source label (`crawler:`) when the token matches one, else null. */ +export function crawlerSource(token: string | null | undefined, raw: string | null | undefined): string | null { + const t = (token ?? "").trim(); + if (!t) return null; + for (const [name, secret] of parseCrawlerTokens(raw)) { + if (same(t, secret)) return `${CRAWLER_PREFIX}${name}`; + } + return null; +} + +/** True when a submission row's source is one of our crawlers. */ +export function isCrawlerSource(submittedBy: string | null | undefined): boolean { + return Boolean(submittedBy && submittedBy.startsWith(CRAWLER_PREFIX) && submittedBy.length > CRAWLER_PREFIX.length); +} diff --git a/src/lib/submit.ts b/src/lib/submit.ts index 74761a0..e8b1897 100644 --- a/src/lib/submit.ts +++ b/src/lib/submit.ts @@ -5,6 +5,7 @@ import type { Podcast } from "@/lib/db"; import { env, now, secret, sha256, token } from "@/lib/auth/crypto"; import { normalizeLang } from "@/lib/format"; import { notifyReviewers } from "@/lib/review"; +import { isCrawlerSource } from "@/lib/crawler-submit"; /** * Feed submission, the site-specific half. @@ -187,6 +188,17 @@ export async function resolveSubmission(id: string, opts: { notify?: boolean } = }); if (outcome.status === "listed") void forward(outcome.feedUrl); + + // One of our own crawlers found this feed (src/lib/crawler-submit.ts): it + // passed the rules, so list it now instead of queueing it for a person. + if (outcome.status === "review" && isCrawlerSource(row.submitted_by)) { + const d = await decideSubmission(id, "approve", `auto:${row.submitted_by}`, "auto-approved: crawler submission", { reread: false }); + if (d.ok && d.status !== "rejected" && d.slug) { + return { status: d.status, slug: d.slug, title: d.title ?? outcome.title, feedUrl: outcome.feedUrl }; + } + return outcome; + } + if (outcome.status === "review" && opts.notify !== false) { void notifyReviewers({ title: outcome.title, @@ -236,7 +248,13 @@ export type Decision = * Reject: the row records who said no and why, and the same feed can be * submitted again once whatever it was is fixed. */ -export async function decideSubmission(id: string, decision: "approve" | "reject", by: string, reason: string | null): Promise { +export async function decideSubmission( + id: string, + decision: "approve" | "reject", + by: string, + reason: string | null, + opts: { reread?: boolean } = {}, +): Promise { const row = await submissionRow(id); if (!row) return { ok: false, error: "no such submission", status: 404 }; if (row.status !== "review") return { ok: false, error: `submission is ${row.status}, not waiting for review`, status: 409 }; @@ -266,11 +284,14 @@ export async function decideSubmission(id: string, decision: "approve" | "reject return { ok: false, error: "the submission carries no show to list; submit the feed again", status: 409 }; } let fresh: Omit | null = null; - try { - const r = await resolveFeed(feedUrl, { kind: "podcast", timeoutMs: 10_000, userAgent: USER_AGENT, maxCandidates: 0 }); - if (r.ok && r.feed.title) fresh = rowFromFeed(r.feed, feedUrl, stored.host); - } catch { - /* the snapshot stands */ + // reread: false when the snapshot was taken moments ago (auto-approval). + if (opts.reread !== false) { + try { + const r = await resolveFeed(feedUrl, { kind: "podcast", timeoutMs: 10_000, userAgent: USER_AGENT, maxCandidates: 0 }); + if (r.ok && r.feed.title) fresh = rowFromFeed(r.feed, feedUrl, stored.host); + } catch { + /* the snapshot stands */ + } } const toList = fresh ?? stored; const slug = await claimSlug(toList.title, feedUrl); diff --git a/test/crawler-submit.test.mjs b/test/crawler-submit.test.mjs new file mode 100644 index 0000000..6a852a5 --- /dev/null +++ b/test/crawler-submit.test.mjs @@ -0,0 +1,37 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { parseCrawlerTokens, crawlerSource, isCrawlerSource } from '../src/lib/crawler-submit.ts'; + +const RAW = 'podcorn:0123456789abcdef0123, Discovery : fedcba9876543210fedc ,bad,short:abc,:nonamexxxxxxxxxxxxxx'; + +test('parses name:token pairs and drops malformed or short ones', () => { + assert.deepEqual(parseCrawlerTokens(RAW), [ + ['podcorn', '0123456789abcdef0123'], + ['discovery', 'fedcba9876543210fedc'], + ]); + assert.deepEqual(parseCrawlerTokens(undefined), []); + assert.deepEqual(parseCrawlerTokens(''), []); +}); + +test('a matching token names the crawler as the source', () => { + assert.equal(crawlerSource('0123456789abcdef0123', RAW), 'crawler:podcorn'); + assert.equal(crawlerSource(' fedcba9876543210fedc ', RAW), 'crawler:discovery'); +}); + +test('no token, a wrong token, or no configured tokens is a manual submission', () => { + assert.equal(crawlerSource(null, RAW), null); + assert.equal(crawlerSource('', RAW), null); + assert.equal(crawlerSource('0123456789abcdef0124', RAW), null); + assert.equal(crawlerSource('0123456789abcdef012', RAW), null); + assert.equal(crawlerSource('0123456789abcdef0123', undefined), null); + assert.equal(crawlerSource('feed-submit/1.0', RAW), null); +}); + +test('only crawler: sources auto-approve', () => { + assert.equal(isCrawlerSource('crawler:podcorn'), true); + assert.equal(isCrawlerSource('crawler:'), false); + assert.equal(isCrawlerSource(null), false); + assert.equal(isCrawlerSource(''), false); + assert.equal(isCrawlerSource('anthony@profullstack.com'), false); + assert.equal(isCrawlerSource('network:bbc'), false); +});