diff --git a/package.json b/package.json index 930731b..5c441b5 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "@libsql/client": "0.18.0", "@modelcontextprotocol/sdk": "1.32.1", "@profullstack/footer": "^0.2.4", + "@profullstack/x402-gateway": "^0.11.0", "@simplewebauthn/server": "^13.3.3", "hono": "4.13.13", "undici": "^7.30.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4d044a8..011c97c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@profullstack/footer': specifier: ^0.2.4 version: 0.2.4 + '@profullstack/x402-gateway': + specifier: ^0.11.0 + version: 0.11.0 '@simplewebauthn/server': specifier: ^13.3.3 version: 13.3.3 @@ -360,6 +363,14 @@ packages: react: optional: true + '@profullstack/keys@0.1.0': + resolution: {integrity: sha512-Vys47LvS5ivhdZu4SYOxh9Le3DEmWhjP0+CtrU45RhXAbY0ZFG/2Dk9V1bIIUGDp8UWMJqctvUD21JrNjfh04w==} + engines: {node: '>=20.11'} + + '@profullstack/x402-gateway@0.11.0': + resolution: {integrity: sha512-Lfrz+SRTwQZfcoCaVtaaMNPJx74Lp/5oyS7vJJAEuJmH1TjBzhE7dWquISkj1p42oc+VW5I9xH9p7YhEKqmr4Q==} + engines: {node: '>=20.11'} + '@rolldown/binding-android-arm-eabi@1.2.13': resolution: {integrity: sha512-Vu/NICrgdRvUOeaJZIyB2FP63s6fO/4ud5l8dXmxDx9yDyn+mN0y+gsvsVviSli+T361Aqo0h46EuMoobrwawA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -1568,6 +1579,12 @@ snapshots: '@profullstack/footer@0.2.4': {} + '@profullstack/keys@0.1.0': {} + + '@profullstack/x402-gateway@0.11.0': + dependencies: + '@profullstack/keys': 0.1.0 + '@rolldown/binding-android-arm-eabi@1.2.13': optional: true diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5ed0b5a..940cde8 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,2 +1,4 @@ allowBuilds: esbuild: true +minimumReleaseAgeExclude: + - '@profullstack/x402-gateway@0.11.0' diff --git a/src/crawl-gate.ts b/src/crawl-gate.ts new file mode 100644 index 0000000..ebf8350 --- /dev/null +++ b/src/crawl-gate.ts @@ -0,0 +1,28 @@ +/** + * The crawl door: AI training crawlers, headless browsers and SEO scrapers buy a + * day pass ($1 a day over x402, or any coin / PayPal through CoinPay's hosted + * checkout) on /crawl, and get a 402 everywhere else. People, verified search + * crawlers, AI chat/search fetchers and signed-in users are never charged, and + * the API, MCP and OAuth endpoints are never metered. Built only with a scoped + * key and a payee, so a local or self-hosted server never sells. + */ +import { createGateway } from '@profullstack/x402-gateway'; +import { SESSION_COOKIE } from './auth.js'; + +export type CrawlGate = ReturnType; + +export function createCrawlGate(env: Record = process.env): CrawlGate | null { + const apiKey = env.COINPAY_X402_KEY ?? ''; + const payTo = env.CRAWL_PAY_TO ?? ''; + if (!apiKey || !payTo) return null; + const session = new RegExp(`(?:^|;\\s*)${SESSION_COOKIE}=`); + return createGateway({ + siteUrl: (env.SUBMITFLEET_PUBLIC_ORIGIN || 'https://submitfleet.com').replace(/\/+$/, ''), + siteName: 'SubmitFleet', + coinpay: { apiKey }, + payTo, + contact: 'mailto:anthony@profullstack.com', + openPaths: ['/healthz', '/readyz', '/install.sh', '/llms.txt', '/api/', '/mcp', '/oauth/'], + exempt: (request: Request) => session.test(request.headers.get('cookie') ?? ''), + }); +} diff --git a/src/http.ts b/src/http.ts index 72bd8de..2a151a3 100644 --- a/src/http.ts +++ b/src/http.ts @@ -6,6 +6,8 @@ import { join } from 'node:path'; import type { Mailer } from './mail.js'; import { mountWeb, resolveActor, sameOrigin, WEB_ROOT } from './web.js'; import { buildMcpServer } from './mcp.js'; +import { createCrawlGate, type CrawlGate } from './crawl-gate.js'; +import { x402Gateway } from '@profullstack/x402-gateway/hono'; import type { SubmitFleet } from './service.js'; import { SubmitFleetError } from './util.js'; import type { Actor } from './workspaces.js'; @@ -89,7 +91,7 @@ function bearer(c: Context): string | null { return h.startsWith('Bearer ') ? h.slice(7).trim() : null; } -export function buildApp(fleet: SubmitFleet, opts: { allowedOrigins?: string[]; mailer?: Mailer; publicOrigin?: string } = {}): Hono { +export function buildApp(fleet: SubmitFleet, opts: { allowedOrigins?: string[]; mailer?: Mailer; publicOrigin?: string; crawlGate?: CrawlGate | null } = {}): Hono { const app = new Hono(); const allowed = new Set(opts.allowedOrigins ?? []); @@ -109,6 +111,10 @@ export function buildApp(fleet: SubmitFleet, opts: { allowedOrigins?: string[]; c.header('x-frame-options', 'DENY'); }); + // Training crawlers buy a day pass before anything else is done for them. + const crawlGate = opts.crawlGate !== undefined ? opts.crawlGate : createCrawlGate(); + if (crawlGate) app.use('*', x402Gateway(crawlGate)); + app.get('/healthz', (c) => c.json({ ok: true })); app.get('/readyz', async (c) => { const checks = await fleet.doctor(); diff --git a/test/crawl-gate.test.ts b/test/crawl-gate.test.ts new file mode 100644 index 0000000..9489e8c --- /dev/null +++ b/test/crawl-gate.test.ts @@ -0,0 +1,46 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { createCrawlGate } from '../src/crawl-gate.js'; +import { buildApp } from '../src/http.js'; +import { harness, type Harness } from './helpers.js'; + +process.env.PROFULLSTACK_FOOTER_OFFLINE ??= '1'; + +const GPT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)'; +const CHROME = { + 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', + accept: 'text/html', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-dest': 'document', +}; + +let h: Harness; +afterEach(() => h?.cleanup()); + +async function app() { + h = await harness(); + const crawlGate = createCrawlGate({ COINPAY_X402_KEY: 'cp_live_test', CRAWL_PAY_TO: '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5' }); + return buildApp(h.fleet, { crawlGate }); +} + +describe('crawl gateway', () => { + it('is off without a key', () => { + expect(createCrawlGate({})).toBeNull(); + }); + it('charges GPTBot with a PAYMENT-REQUIRED offer', async () => { + const res = await (await app()).request('/', { headers: { 'user-agent': GPT } }); + expect(res.status).toBe(402); + expect(res.headers.get('payment-required')).toBeTruthy(); + }); + it('sells a $1 day pass on /crawl', async () => { + const res = await (await app()).request('/crawl', { headers: { accept: 'application/json' } }); + expect(res.status).toBe(402); + expect(String(((await res.json()) as { accepts: Array<{ amount: string }> }).accepts[0]!.amount)).toBe('1000000'); + }); + it('lets a browser, the API and the health check through', async () => { + const a = await app(); + expect((await a.request('/', { headers: CHROME })).status).not.toBe(402); + expect((await a.request('/healthz', { headers: { 'user-agent': GPT } })).status).toBe(200); + expect((await a.request('/api/v1/products', { headers: { 'user-agent': GPT } })).status).toBe(401); + }); +});