Charge AI training crawlers a day pass (x402 crawl gateway 0.11)

package.json

@@ -52,6 +52,7 @@
     "@libsql/client": "0.18.0",
     "@modelcontextprotocol/sdk": "1.32.1",
     "@profullstack/footer": "^0.2.4",
+    "@profullstack/x402-gateway": "^0.11.0",
     "@simplewebauthn/server": "^13.3.3",
     "hono": "4.13.13",
     "undici": "^7.30.0",

pnpm-workspace.yaml

@@ -1,2 +1,4 @@
 allowBuilds:
   esbuild: true
+minimumReleaseAgeExclude:
+  - '@profullstack/x402-gateway@0.11.0'

src/crawl-gate.ts

@@ -0,0 +1,28 @@
+/**
+ * The crawl door: AI training crawlers, headless browsers and SEO scrapers buy a
+ * day pass ($1 a day over x402, or any coin / PayPal through CoinPay's hosted
+ * checkout) on /crawl, and get a 402 everywhere else. People, verified search
+ * crawlers, AI chat/search fetchers and signed-in users are never charged, and
+ * the API, MCP and OAuth endpoints are never metered. Built only with a scoped
+ * key and a payee, so a local or self-hosted server never sells.
+ */
+import { createGateway } from '@profullstack/x402-gateway';
+import { SESSION_COOKIE } from './auth.js';
+
+export type CrawlGate = ReturnType<typeof createGateway>;
+
+export function createCrawlGate(env: Record<string, string | undefined> = process.env): CrawlGate | null {
+  const apiKey = env.COINPAY_X402_KEY ?? '';
+  const payTo = env.CRAWL_PAY_TO ?? '';
+  if (!apiKey || !payTo) return null;
+  const session = new RegExp(`(?:^|;\\s*)${SESSION_COOKIE}=`);
+  return createGateway({
+    siteUrl: (env.SUBMITFLEET_PUBLIC_ORIGIN || 'https://submitfleet.com').replace(/\/+$/, ''),
+    siteName: 'SubmitFleet',
+    coinpay: { apiKey },
+    payTo,
+    contact: 'mailto:anthony@profullstack.com',
+    openPaths: ['/healthz', '/readyz', '/install.sh', '/llms.txt', '/api/', '/mcp', '/oauth/'],
+    exempt: (request: Request) => session.test(request.headers.get('cookie') ?? ''),
+  });
+}

src/http.ts

@@ -6,6 +6,8 @@ import { join } from 'node:path';
 import type { Mailer } from './mail.js';
 import { mountWeb, resolveActor, sameOrigin, WEB_ROOT } from './web.js';
 import { buildMcpServer } from './mcp.js';
+import { createCrawlGate, type CrawlGate } from './crawl-gate.js';
+import { x402Gateway } from '@profullstack/x402-gateway/hono';
 import type { SubmitFleet } from './service.js';
 import { SubmitFleetError } from './util.js';
 import type { Actor } from './workspaces.js';
@@ -89,7 +91,7 @@ function bearer(c: Context): string | null {
   return h.startsWith('Bearer ') ? h.slice(7).trim() : null;
 }
 
-export function buildApp(fleet: SubmitFleet, opts: { allowedOrigins?: string[]; mailer?: Mailer; publicOrigin?: string } = {}): Hono<Env> {
+export function buildApp(fleet: SubmitFleet, opts: { allowedOrigins?: string[]; mailer?: Mailer; publicOrigin?: string; crawlGate?: CrawlGate | null } = {}): Hono<Env> {
   const app = new Hono<Env>();
   const allowed = new Set(opts.allowedOrigins ?? []);
 
@@ -109,6 +111,10 @@ export function buildApp(fleet: SubmitFleet, opts: { allowedOrigins?: string[];
     c.header('x-frame-options', 'DENY');
   });
 
+  // Training crawlers buy a day pass before anything else is done for them.
+  const crawlGate = opts.crawlGate !== undefined ? opts.crawlGate : createCrawlGate();
+  if (crawlGate) app.use('*', x402Gateway(crawlGate));
+
   app.get('/healthz', (c) => c.json({ ok: true }));
   app.get('/readyz', async (c) => {
     const checks = await fleet.doctor();

test/crawl-gate.test.ts

@@ -0,0 +1,46 @@
+import { afterEach, describe, expect, it } from 'vitest';
+import { createCrawlGate } from '../src/crawl-gate.js';
+import { buildApp } from '../src/http.js';
+import { harness, type Harness } from './helpers.js';
+
+process.env.PROFULLSTACK_FOOTER_OFFLINE ??= '1';
+
+const GPT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)';
+const CHROME = {
+  'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+  accept: 'text/html',
+  'sec-fetch-mode': 'navigate',
+  'sec-fetch-site': 'none',
+  'sec-fetch-dest': 'document',
+};
+
+let h: Harness;
+afterEach(() => h?.cleanup());
+
+async function app() {
+  h = await harness();
+  const crawlGate = createCrawlGate({ COINPAY_X402_KEY: 'cp_live_test', CRAWL_PAY_TO: '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5' });
+  return buildApp(h.fleet, { crawlGate });
+}
+
+describe('crawl gateway', () => {
+  it('is off without a key', () => {
+    expect(createCrawlGate({})).toBeNull();
+  });
+  it('charges GPTBot with a PAYMENT-REQUIRED offer', async () => {
+    const res = await (await app()).request('/', { headers: { 'user-agent': GPT } });
+    expect(res.status).toBe(402);
+    expect(res.headers.get('payment-required')).toBeTruthy();
+  });
+  it('sells a $1 day pass on /crawl', async () => {
+    const res = await (await app()).request('/crawl', { headers: { accept: 'application/json' } });
+    expect(res.status).toBe(402);
+    expect(String(((await res.json()) as { accepts: Array<{ amount: string }> }).accepts[0]!.amount)).toBe('1000000');
+  });
+  it('lets a browser, the API and the health check through', async () => {
+    const a = await app();
+    expect((await a.request('/', { headers: CHROME })).status).not.toBe(402);
+    expect((await a.request('/healthz', { headers: { 'user-agent': GPT } })).status).toBe(200);
+    expect((await a.request('/api/v1/products', { headers: { 'user-agent': GPT } })).status).toBe(401);
+  });
+});