diff --git a/apps/web/lib/crawl-gateway.ts b/apps/web/lib/crawl-gateway.ts new file mode 100644 index 0000000..4dc3814 --- /dev/null +++ b/apps/web/lib/crawl-gateway.ts @@ -0,0 +1,28 @@ +import { createGateway } from "@profullstack/x402-gateway"; +import { x402Proxy } from "@profullstack/x402-gateway/next"; + +/** + * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, ...) + * by the day over x402, settled by CoinPay in USDC, with a CoinPay checkout + * for people at /crawl. Browsers, Googlebot and AI chat/search fetchers pass + * through untouched. + * + * Runs inside the proxy, so nothing here may import Node-only modules. The env + * is read through a non-literal key on purpose: Next inlines `process.env.NAME` + * at build time, and these are runtime secrets. + */ +const env = (name: string) => process.env[name]; + +export const gateway = createGateway({ + siteUrl: (env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || "https://hqtui.com").replace(/\/+$/, ""), + siteName: "HQTUI", + coinpay: { apiKey: env("COINPAY_X402_KEY") }, + payTo: env("CRAWL_PAY_TO"), + contact: "mailto:anthony@profullstack.com", + // Health checks, the API (votes, book checkout and its CoinPay webhook, the + // paid download) and the scripts people curl are never charged. + openPaths: ["/api/", "/install.sh", "/demo.sh", "/robots.txt", "/sitemap.xml", "/llms.txt", "/.well-known/"], +}); + +/** Resolves to a Response for a refused crawler, or undefined to carry on. */ +export const gate = x402Proxy(gateway); diff --git a/apps/web/next.config.ts b/apps/web/next.config.ts index 6c8e27b..077e297 100644 --- a/apps/web/next.config.ts +++ b/apps/web/next.config.ts @@ -23,7 +23,8 @@ const CSP = [ "frame-src 'self' https://crawlproof.com", "frame-ancestors 'self'", "base-uri 'self'", - "form-action 'self'", + // The crawl pass checkout (/crawl/checkout) redirects to CoinPay. + "form-action 'self' https://coinpayportal.com", "object-src 'none'", "upgrade-insecure-requests", ].join("; "); diff --git a/apps/web/package.json b/apps/web/package.json index 245a6ad..3e24cd6 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -12,6 +12,7 @@ "@profullstack/footer": "latest", "@profullstack/hqtui": "^0.10.3", "@profullstack/libsql-pg": "^0.1.3", + "@profullstack/x402-gateway": "^0.11.0", "class-variance-authority": "0.7.1", "clsx": "2.1.1", "lucide-react": "1.37.0", diff --git a/apps/web/proxy.ts b/apps/web/proxy.ts new file mode 100644 index 0000000..de78772 --- /dev/null +++ b/apps/web/proxy.ts @@ -0,0 +1,18 @@ +import type { NextRequest } from "next/server"; +import { gate } from "@/lib/crawl-gateway"; + +/** + * Crawl gateway first: AI training crawlers get 402 Payment Required (or the + * pass page at /crawl) unless they present a paid pass. Everyone else carries + * on to the app. + */ +export async function proxy(request: NextRequest) { + const answer = await gate(request); + if (answer) return answer; +} + +export const config = { + matcher: [ + "/((?!_next/static|_next/image|favicon.ico|.*\\.(?:svg|png|jpg|jpeg|gif|webp|avif|ico|woff2?|css|js|map|pdf)$).*)", + ], +}; diff --git a/apps/web/test/crawl-gateway.test.ts b/apps/web/test/crawl-gateway.test.ts new file mode 100644 index 0000000..ce80a1d --- /dev/null +++ b/apps/web/test/crawl-gateway.test.ts @@ -0,0 +1,36 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +process.env.COINPAY_X402_KEY = "cp_live_test"; +process.env.CRAWL_PAY_TO = "0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5"; +const { gateway } = await import("../lib/crawl-gateway.ts"); + +const GPTBOT = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)"; +const HUMAN = { + "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36", + accept: "text/html,application/xhtml+xml", + "accept-language": "en-US", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "none", + "sec-fetch-dest": "document", + "x-real-ip": "198.51.100.7", +}; + +test("a training crawler is answered 402 with PAYMENT-REQUIRED", async () => { + const res = await gateway.handle(new Request("https://hqtui.com/", { headers: { "user-agent": GPTBOT, "x-real-ip": "203.0.113.9" } })); + assert.equal(res?.status, 402); + assert.ok(res?.headers.get("payment-required")); +}); + +test("/crawl offers a $1 day pass", async () => { + const res = await gateway.handle(new Request("https://hqtui.com/crawl", { headers: { accept: "application/json" } })); + assert.equal(res?.status, 402); + assert.ok(res?.headers.get("payment-required")); + const body = (await res!.json()) as { accepts: { amount: string }[] }; + assert.equal(String(body.accepts[0]?.amount), "1000000"); +}); + +test("a browser and the API pass through", async () => { + assert.equal(await gateway.handle(new Request("https://hqtui.com/", { headers: HUMAN })), null); + assert.equal(await gateway.handle(new Request("https://hqtui.com/api/health", { headers: { "user-agent": GPTBOT } })), null); +}); diff --git a/bun.lock b/bun.lock index 29ff304..d9d7a95 100644 --- a/bun.lock +++ b/bun.lock @@ -37,6 +37,7 @@ "@profullstack/footer": "latest", "@profullstack/hqtui": "^0.10.3", "@profullstack/libsql-pg": "^0.1.3", + "@profullstack/x402-gateway": "^0.11.0", "class-variance-authority": "0.7.1", "clsx": "2.1.1", "lucide-react": "1.37.0", @@ -282,7 +283,7 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], - "@profullstack/footer": ["@profullstack/footer@0.2.1", "", { "peerDependencies": { "react": ">=18" }, "optionalPeers": ["react"], "bin": { "profullstack-footer": "bin/profullstack-footer.js" } }, "sha512-KHmKVwrlzCBrSAAJ4zobNFCxTg8VF0gvNMeAut00dX3+gbWCtk6eOwIR2AaqEC7smj6Aio6cIexzt/dUnv1McA=="], + "@profullstack/footer": ["@profullstack/footer@0.2.5", "", { "peerDependencies": { "react": ">=18" }, "optionalPeers": ["react"], "bin": { "profullstack-footer": "bin/profullstack-footer.js" } }, "sha512-/CKwFP3m+qM0ZFTC9Rj11fgClutJzeYliX44Qp4qwrv2JgA/1k4tVOB3mhkPhaeckUhnfg5UyESDI1uoWr9c2w=="], "@profullstack/hqtui": ["@profullstack/hqtui@workspace:packages/hqtui"], @@ -296,8 +297,12 @@ "@profullstack/hqtui-web": ["@profullstack/hqtui-web@workspace:apps/web"], + "@profullstack/keys": ["@profullstack/keys@0.1.0", "", {}, "sha512-Vys47LvS5ivhdZu4SYOxh9Le3DEmWhjP0+CtrU45RhXAbY0ZFG/2Dk9V1bIIUGDp8UWMJqctvUD21JrNjfh04w=="], + "@profullstack/libsql-pg": ["@profullstack/libsql-pg@0.1.3", "", { "dependencies": { "@profullstack/libsql-pg": "^0.1.1", "pg": "^8.13.0" }, "optionalDependencies": { "@libsql/client": "^0.15.0" }, "bin": { "libsql-pg": "bin/libsql-pg.js" } }, "sha512-9VuaWQKDxj5gy304DQlailRJ9M76vqC1Pv+z+Vjm/FiHrJ/VwYI3ycnL0GgjUF4971ZEUXY6Dw2EISdU1FtKjQ=="], + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.11.0", "", { "dependencies": { "@profullstack/keys": "^0.1.0" } }, "sha512-Lfrz+SRTwQZfcoCaVtaaMNPJx74Lp/5oyS7vJJAEuJmH1TjBzhE7dWquISkj1p42oc+VW5I9xH9p7YhEKqmr4Q=="], + "@sec-ant/readable-stream": ["@sec-ant/readable-stream@0.4.1", "", {}, "sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg=="], "@sindresorhus/merge-streams": ["@sindresorhus/merge-streams@4.0.0", "", {}, "sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ=="],