diff --git a/Caddyfile b/Caddyfile index b66634b..dec38b9 100644 --- a/Caddyfile +++ b/Caddyfile @@ -17,6 +17,20 @@ @api path /api/* /api reverse_proxy @api localhost:8090 + # x402 crawl gateway (services/api/src/crawl-gate.ts). /crawl is the day-pass + # sales page and checkout, answered by the API. Every other page read is + # checked there first: AI training crawlers get the 402 offer, everyone else + # (people, search crawlers, AI chat fetchers) gets the page. + @crawl path /crawl /crawl/* + reverse_proxy @crawl localhost:8090 + @pages { + method GET HEAD + not path /api/* /api /mcp/* /mcp /crawl /crawl/* + } + forward_auth @pages localhost:8090 { + uri /api/crawl-gate + } + # OpenMCP relay (tronbrowser.dev/mcp/tron): same API process. Its descriptor # is the static /.well-known/openmcp.json below, which is what makes the # listing "verified" on a catalog (served from this origin, not registered). @@ -34,10 +48,14 @@ X-Frame-Options "DENY" Referrer-Policy "strict-origin-when-cross-origin" Permissions-Policy "camera=(), microphone=(), geolocation=(), browsing-topics=()" - Content-Security-Policy "default-src 'self'; script-src 'self' https://crawlproof.com https://cdn.jsdelivr.net; connect-src 'self' https://crawlproof.com https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; frame-src 'self' https://crawlproof.com; base-uri 'self'; form-action 'self'; frame-ancestors 'none'" -Server } + # The crawl pass pages get no CSP: their Buy button posts to /crawl/checkout, + # which redirects to coinpayportal.com, and form-action 'self' would block it. + @notcrawl not path /crawl /crawl/* + header @notcrawl Content-Security-Policy "default-src 'self'; script-src 'self' https://crawlproof.com https://cdn.jsdelivr.net; connect-src 'self' https://crawlproof.com https://cdn.jsdelivr.net; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'; frame-src 'self' https://crawlproof.com; base-uri 'self'; form-action 'self'; frame-ancestors 'none'" + # Correct content types for machine-readable files. @plain path /robots.txt /llms.txt /llms-full.txt /.well-known/security.txt header @plain Content-Type "text/plain; charset=utf-8" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f5d87b9..a86a9ed 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -303,6 +303,9 @@ importers: '@profullstack/libsql-pg': specifier: ^0.1.1 version: 0.1.1(@libsql/client@0.14.0) + '@profullstack/x402-gateway': + specifier: ^0.11.0 + version: 0.11.0 '@tronbrowser/sdk': specifier: workspace:* version: link:../../packages/sdk @@ -1264,6 +1267,10 @@ packages: '@profullstack/emailer@1.0.1': resolution: {integrity: sha512-/uhHJJGH+1xSSz3mJn6X+m6aruYjMD3JOaRp/d4R/YWlzpy07H9z0/JUleIyRyBPNmaANSIwjTZ7aVjaukOEpg==} + '@profullstack/keys@0.1.0': + resolution: {integrity: sha512-Vys47LvS5ivhdZu4SYOxh9Le3DEmWhjP0+CtrU45RhXAbY0ZFG/2Dk9V1bIIUGDp8UWMJqctvUD21JrNjfh04w==} + engines: {node: '>=20.11'} + '@profullstack/libsql-pg@0.1.1': resolution: {integrity: sha512-KD4+bCDM31hwheWYxOyn6zxRw1hTqhEqAl/LXfgoQNbZ88ODbDNJViDtxPWCCv7IPgXAMWsh2kbkrjRiT7Sn1Q==} engines: {node: '>=20'} @@ -1302,6 +1309,10 @@ packages: react: optional: true + '@profullstack/x402-gateway@0.11.0': + resolution: {integrity: sha512-Lfrz+SRTwQZfcoCaVtaaMNPJx74Lp/5oyS7vJJAEuJmH1TjBzhE7dWquISkj1p42oc+VW5I9xH9p7YhEKqmr4Q==} + engines: {node: '>=20.11'} + '@react-native/assets-registry@0.86.2': resolution: {integrity: sha512-vcX/mBjWAVnWofu7KecotquI2unZ/tITwA7OGdq/mdY/zmGXIEvYhfEYyOQij/LRqi9WAL+iizInTBWnxDhK/Q==} engines: {node: ^20.19.4 || ^22.13.0 || ^24.3.0 || >= 25.0.0} @@ -5023,6 +5034,8 @@ snapshots: '@profullstack/emailer@1.0.1': {} + '@profullstack/keys@0.1.0': {} + '@profullstack/libsql-pg@0.1.1(@libsql/client@0.14.0)': dependencies: pg: 8.23.0 @@ -5044,6 +5057,10 @@ snapshots: optionalDependencies: react: 19.2.7 + '@profullstack/x402-gateway@0.11.0': + dependencies: + '@profullstack/keys': 0.1.0 + '@react-native/assets-registry@0.86.2': {} '@react-native/babel-plugin-codegen@0.86.2(@babel/core@7.29.7)': diff --git a/services/api/package.json b/services/api/package.json index 859ead5..4a1b1ea 100644 --- a/services/api/package.json +++ b/services/api/package.json @@ -22,6 +22,7 @@ "@logicsrc/agentswarm": "^0.1.0", "@profullstack/coinpay-onboard": "0.2.0", "@profullstack/libsql-pg": "^0.1.1", + "@profullstack/x402-gateway": "^0.11.0", "@tronbrowser/sdk": "workspace:*", "deepagents": "^1.10.5", "fflate": "^0.8.3", diff --git a/services/api/src/crawl-gate.test.ts b/services/api/src/crawl-gate.test.ts new file mode 100644 index 0000000..1d11e8c --- /dev/null +++ b/services/api/src/crawl-gate.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest'; +import { crawlCheck, createCrawlGate } from './crawl-gate.js'; + +const GPT = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)'; +const CHROME = { + 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', + accept: 'text/html', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-dest': 'document', +}; +const ORIGIN = 'https://tronbrowser.dev'; +const gate = createCrawlGate({ COINPAY_X402_KEY: 'cp_live_test', CRAWL_PAY_TO: '0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5' })!; +const check = (uri: string, headers: Record) => + crawlCheck(gate, new Request('http://localhost:8090/api/crawl-gate', { headers: { ...headers, 'x-forwarded-uri': uri, 'x-forwarded-method': 'GET' } }), ORIGIN); + +describe('crawl gate', () => { + it('is off without a key, and then lets every page through', async () => { + expect(createCrawlGate({})).toBeNull(); + expect((await crawlCheck(null, new Request('http://localhost/api/crawl-gate'), ORIGIN)).status).toBe(204); + }); + it('answers Caddy with a 402 offer for GPTBot', async () => { + const res = await check('/', { 'user-agent': GPT }); + expect(res.status).toBe(402); + expect(res.headers.get('payment-required')).toBeTruthy(); + }); + it('sells a $1 day pass on /crawl', async () => { + const res = (await gate.handle(new Request(`${ORIGIN}/crawl`, { headers: { accept: 'application/json' } })))!; + expect(res.status).toBe(402); + expect(String(((await res.json()) as { accepts: Array<{ amount: string }> }).accepts[0]!.amount)).toBe('1000000'); + }); + it('lets browsers, signed-in users and the API through', async () => { + expect((await check('/', CHROME)).status).toBe(204); + expect((await check('/', { 'user-agent': GPT, cookie: 'tb_session=x' })).status).toBe(204); + expect((await check('/api/healthz', { 'user-agent': GPT })).status).toBe(204); + }); +}); diff --git a/services/api/src/crawl-gate.ts b/services/api/src/crawl-gate.ts new file mode 100644 index 0000000..9e810d5 --- /dev/null +++ b/services/api/src/crawl-gate.ts @@ -0,0 +1,48 @@ +/** + * The crawl door: AI training crawlers, headless browsers and SEO scrapers buy a + * day pass ($1 a day over x402, or any coin / PayPal through CoinPay's hosted + * checkout) on /crawl, and get a 402 on the site's pages. People, verified search + * crawlers, AI chat/search fetchers and signed-in users are never charged, and + * the API and MCP relay are never metered. Built only with a scoped key and a + * payee, so a local or self-hosted server never sells. + * + * Caddy serves the pages, so it asks here first: `forward_auth` sends every page + * request to /api/crawl-gate with the original method and path in + * X-Forwarded-Method / X-Forwarded-Uri. A 2xx lets Caddy serve the page; any + * other answer (the 402 offer) is copied back to the client as it is. + */ +import { createGateway } from '@profullstack/x402-gateway'; + +export type CrawlGate = { handle: (request: Request) => Promise }; + +export function createCrawlGate(env: Record = process.env): CrawlGate | null { + const apiKey = env.COINPAY_X402_KEY ?? ''; + const payTo = env.CRAWL_PAY_TO ?? ''; + if (!apiKey || !payTo) return null; + return createGateway({ + siteUrl: (env.APP_URL || 'https://tronbrowser.dev').replace(/\/+$/, ''), + siteName: 'TronBrowser', + coinpay: { apiKey }, + payTo, + contact: 'mailto:anthony@profullstack.com', + openPaths: ['/api/', '/mcp/', '/llms.txt', '/llms-full.txt', '/skill.md', '/install.sh'], + exempt: (request: Request) => /(?:^|;\s*)tb_session=/.test(request.headers.get('cookie') ?? ''), + }); +} + +/** The page request Caddy is asking about, rebuilt from its forward_auth headers. */ +export function forwardedRequest(request: Request, origin: string): Request { + const uri = request.headers.get('x-forwarded-uri') || '/'; + const method = (request.headers.get('x-forwarded-method') || 'GET').toUpperCase(); + const headers = new Headers(request.headers); + for (const name of ['x-forwarded-uri', 'x-forwarded-method', 'x-forwarded-host', 'x-forwarded-proto', 'content-length']) { + headers.delete(name); + } + return new Request(new URL(uri.startsWith('/') ? uri : '/', origin), { method: method === 'HEAD' ? 'HEAD' : 'GET', headers }); +} + +/** forward_auth answer: 204 to serve the page, or the gateway's own answer. */ +export async function crawlCheck(gate: CrawlGate | null, request: Request, origin: string): Promise { + if (!gate) return new Response(null, { status: 204 }); + return (await gate.handle(forwardedRequest(request, origin))) ?? new Response(null, { status: 204 }); +} diff --git a/services/api/src/index.ts b/services/api/src/index.ts index a7908fc..e5100eb 100644 --- a/services/api/src/index.ts +++ b/services/api/src/index.ts @@ -18,6 +18,7 @@ import { safeRedirect } from './redirect.js'; import { extLoginTarget } from './ext-login.js'; import { assertDatabaseUrl, db } from './db.js'; import { pushService } from './push/routes.js'; +import { createCrawlGate, crawlCheck } from './crawl-gate.js'; // Fail fast: a missing or non-Postgres DATABASE_URL is a deploy error, not a // condition to limp along under (there is no file-database fallback). @@ -59,6 +60,12 @@ async function startSession(c: any, userId: string, redirect?: string) { app.get('/api/healthz', (c) => c.json({ ok: true })); +/* ---------- x402 crawl gateway: /crawl sales page + Caddy forward_auth check ---------- */ +const crawlGate = createCrawlGate(); +app.all('/api/crawl-gate', (c) => crawlCheck(crawlGate, c.req.raw, APP_URL)); +app.all('/crawl', async (c) => (crawlGate && (await crawlGate.handle(c.req.raw))) || c.notFound()); +app.all('/crawl/*', async (c) => (crawlGate && (await crawlGate.handle(c.req.raw))) || c.notFound()); + /* ---------- Web Push service: tronbrowser.dev/api/1/push ---------- */ // ungoogled-chromium has no push service; the bundled extension registers // here instead (the push service is a TronBrowser setting, this is the default).