Charge AI training crawlers a day pass (x402 crawl gateway 0.11)

Caddyfile

@@ -17,6 +17,20 @@
 	@api path /api/* /api
 	reverse_proxy @api localhost:8090
 
+	# x402 crawl gateway (services/api/src/crawl-gate.ts). /crawl is the day-pass
+	# sales page and checkout, answered by the API. Every other page read is
+	# checked there first: AI training crawlers get the 402 offer, everyone else
+	# (people, search crawlers, AI chat fetchers) gets the page.
+	@crawl path /crawl /crawl/*
+	reverse_proxy @crawl localhost:8090
+	@pages {
+		method GET HEAD
+		not path /api/* /api /mcp/* /mcp /crawl /crawl/*
+	}
+	forward_auth @pages localhost:8090 {
+		uri /api/crawl-gate
+	}
+
 	# OpenMCP relay (tronbrowser.dev/mcp/tron): same API process. Its descriptor
 	# is the static /.well-known/openmcp.json below, which is what makes the
 	# listing "verified" on a catalog (served from this origin, not registered).
@@ -34,10 +48,14 @@
 		X-Frame-Options "DENY"
 		Referrer-Policy "strict-origin-when-cross-origin"
 		Permissions-Policy "camera=(), microphone=(), geolocation=(), browsing-topics=()"
-		Content-Security-Policy "default-src 'self'; script-src 'self' https://crawlproof.com https://cdn.jsdelivr.net; connect-src 'self' https://crawlproof.com https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; frame-src 'self' https://crawlproof.com; base-uri 'self'; form-action 'self'; frame-ancestors 'none'"
 		-Server
 	}
 
+	# The crawl pass pages get no CSP: their Buy button posts to /crawl/checkout,
+	# which redirects to coinpayportal.com, and form-action 'self' would block it.
+	@notcrawl not path /crawl /crawl/*
+	header @notcrawl Content-Security-Policy "default-src 'self'; script-src 'self' https://crawlproof.com https://cdn.jsdelivr.net; connect-src 'self' https://crawlproof.com https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; frame-src 'self' https://crawlproof.com; base-uri 'self'; form-action 'self'; frame-ancestors 'none'"
+
 	# Correct content types for machine-readable files.
 	@plain path /robots.txt /llms.txt /llms-full.txt /.well-known/security.txt
 	header @plain Content-Type "text/plain; charset=utf-8"

services/api/package.json

@@ -22,6 +22,7 @@
     "@logicsrc/agentswarm": "^0.1.0",
     "@profullstack/coinpay-onboard": "0.2.0",
     "@profullstack/libsql-pg": "^0.1.1",
+    "@profullstack/x402-gateway": "^0.11.0",
     "@tronbrowser/sdk": "workspace:*",
     "deepagents": "^1.10.5",
     "fflate": "^0.8.3",

services/api/src/crawl-gate.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, it } from 'vitest';
+import { crawlCheck, createCrawlGate } from './crawl-gate.js';
+
+const GPT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)';
+const CHROME = {
+  'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+  accept: 'text/html',
+  'sec-fetch-mode': 'navigate',
+  'sec-fetch-site': 'none',
+  'sec-fetch-dest': 'document',
+};
+const ORIGIN = 'https://tronbrowser.dev';
+const gate = createCrawlGate({ COINPAY_X402_KEY: 'cp_live_test', CRAWL_PAY_TO: '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5' })!;
+const check = (uri: string, headers: Record<string, string>) =>
+  crawlCheck(gate, new Request('http://localhost:8090/api/crawl-gate', { headers: { ...headers, 'x-forwarded-uri': uri, 'x-forwarded-method': 'GET' } }), ORIGIN);
+
+describe('crawl gate', () => {
+  it('is off without a key, and then lets every page through', async () => {
+    expect(createCrawlGate({})).toBeNull();
+    expect((await crawlCheck(null, new Request('http://localhost/api/crawl-gate'), ORIGIN)).status).toBe(204);
+  });
+  it('answers Caddy with a 402 offer for GPTBot', async () => {
+    const res = await check('/', { 'user-agent': GPT });
+    expect(res.status).toBe(402);
+    expect(res.headers.get('payment-required')).toBeTruthy();
+  });
+  it('sells a $1 day pass on /crawl', async () => {
+    const res = (await gate.handle(new Request(`${ORIGIN}/crawl`, { headers: { accept: 'application/json' } })))!;
+    expect(res.status).toBe(402);
+    expect(String(((await res.json()) as { accepts: Array<{ amount: string }> }).accepts[0]!.amount)).toBe('1000000');
+  });
+  it('lets browsers, signed-in users and the API through', async () => {
+    expect((await check('/', CHROME)).status).toBe(204);
+    expect((await check('/', { 'user-agent': GPT, cookie: 'tb_session=x' })).status).toBe(204);
+    expect((await check('/api/healthz', { 'user-agent': GPT })).status).toBe(204);
+  });
+});

services/api/src/crawl-gate.ts

@@ -0,0 +1,48 @@
+/**
+ * The crawl door: AI training crawlers, headless browsers and SEO scrapers buy a
+ * day pass ($1 a day over x402, or any coin / PayPal through CoinPay's hosted
+ * checkout) on /crawl, and get a 402 on the site's pages. People, verified search
+ * crawlers, AI chat/search fetchers and signed-in users are never charged, and
+ * the API and MCP relay are never metered. Built only with a scoped key and a
+ * payee, so a local or self-hosted server never sells.
+ *
+ * Caddy serves the pages, so it asks here first: `forward_auth` sends every page
+ * request to /api/crawl-gate with the original method and path in
+ * X-Forwarded-Method / X-Forwarded-Uri. A 2xx lets Caddy serve the page; any
+ * other answer (the 402 offer) is copied back to the client as it is.
+ */
+import { createGateway } from '@profullstack/x402-gateway';
+
+export type CrawlGate = { handle: (request: Request) => Promise<Response | null> };
+
+export function createCrawlGate(env: Record<string, string | undefined> = process.env): CrawlGate | null {
+  const apiKey = env.COINPAY_X402_KEY ?? '';
+  const payTo = env.CRAWL_PAY_TO ?? '';
+  if (!apiKey || !payTo) return null;
+  return createGateway({
+    siteUrl: (env.APP_URL || 'https://tronbrowser.dev').replace(/\/+$/, ''),
+    siteName: 'TronBrowser',
+    coinpay: { apiKey },
+    payTo,
+    contact: 'mailto:anthony@profullstack.com',
+    openPaths: ['/api/', '/mcp/', '/llms.txt', '/llms-full.txt', '/skill.md', '/install.sh'],
+    exempt: (request: Request) => /(?:^|;\s*)tb_session=/.test(request.headers.get('cookie') ?? ''),
+  });
+}
+
+/** The page request Caddy is asking about, rebuilt from its forward_auth headers. */
+export function forwardedRequest(request: Request, origin: string): Request {
+  const uri = request.headers.get('x-forwarded-uri') || '/';
+  const method = (request.headers.get('x-forwarded-method') || 'GET').toUpperCase();
+  const headers = new Headers(request.headers);
+  for (const name of ['x-forwarded-uri', 'x-forwarded-method', 'x-forwarded-host', 'x-forwarded-proto', 'content-length']) {
+    headers.delete(name);
+  }
+  return new Request(new URL(uri.startsWith('/') ? uri : '/', origin), { method: method === 'HEAD' ? 'HEAD' : 'GET', headers });
+}
+
+/** forward_auth answer: 204 to serve the page, or the gateway's own answer. */
+export async function crawlCheck(gate: CrawlGate | null, request: Request, origin: string): Promise<Response> {
+  if (!gate) return new Response(null, { status: 204 });
+  return (await gate.handle(forwardedRequest(request, origin))) ?? new Response(null, { status: 204 });
+}

services/api/src/index.ts

@@ -18,6 +18,7 @@ import { safeRedirect } from './redirect.js';
 import { extLoginTarget } from './ext-login.js';
 import { assertDatabaseUrl, db } from './db.js';
 import { pushService } from './push/routes.js';
+import { createCrawlGate, crawlCheck } from './crawl-gate.js';
 
 // Fail fast: a missing or non-Postgres DATABASE_URL is a deploy error, not a
 // condition to limp along under (there is no file-database fallback).
@@ -59,6 +60,12 @@ async function startSession(c: any, userId: string, redirect?: string) {
 
 app.get('/api/healthz', (c) => c.json({ ok: true }));
 
+/* ---------- x402 crawl gateway: /crawl sales page + Caddy forward_auth check ---------- */
+const crawlGate = createCrawlGate();
+app.all('/api/crawl-gate', (c) => crawlCheck(crawlGate, c.req.raw, APP_URL));
+app.all('/crawl', async (c) => (crawlGate && (await crawlGate.handle(c.req.raw))) || c.notFound());
+app.all('/crawl/*', async (c) => (crawlGate && (await crawlGate.handle(c.req.raw))) || c.notFound());
+
 /* ---------- Web Push service: tronbrowser.dev/api/1/push ---------- */
 // ungoogled-chromium has no push service; the bundled extension registers
 // here instead (the push service is a TronBrowser setting, this is the default).