feat(web): x402 crawl gateway for training crawlers

apps/web/lib/crawl-gateway.ts

@@ -0,0 +1,28 @@
+import { createGateway } from "@profullstack/x402-gateway";
+import { x402Proxy } from "@profullstack/x402-gateway/next";
+
+/**
+ * Sells crawl access to AI training crawlers (GPTBot, ClaudeBot, CCBot, ...)
+ * by the day over x402, settled by CoinPay in USDC, with a CoinPay checkout
+ * for people at /crawl. Browsers, Googlebot and AI chat/search fetchers pass
+ * through untouched.
+ *
+ * Runs inside the proxy, so nothing here may import Node-only modules. The env
+ * is read through a non-literal key on purpose: Next inlines `process.env.NAME`
+ * at build time, and these are runtime secrets.
+ */
+const env = (name: string) => process.env[name];
+
+export const gateway = createGateway({
+  siteUrl: (env("SITE_URL") || env("NEXT_PUBLIC_SITE_URL") || "https://hqtui.com").replace(/\/+$/, ""),
+  siteName: "HQTUI",
+  coinpay: { apiKey: env("COINPAY_X402_KEY") },
+  payTo: env("CRAWL_PAY_TO"),
+  contact: "mailto:anthony@profullstack.com",
+  // Health checks, the API (votes, book checkout and its CoinPay webhook, the
+  // paid download) and the scripts people curl are never charged.
+  openPaths: ["/api/", "/install.sh", "/demo.sh", "/robots.txt", "/sitemap.xml", "/llms.txt", "/.well-known/"],
+});
+
+/** Resolves to a Response for a refused crawler, or undefined to carry on. */
+export const gate = x402Proxy(gateway);

apps/web/next.config.ts

@@ -23,7 +23,8 @@ const CSP = [
   "frame-src 'self' https://crawlproof.com",
   "frame-ancestors 'self'",
   "base-uri 'self'",
-  "form-action 'self'",
+  // The crawl pass checkout (/crawl/checkout) redirects to CoinPay.
+  "form-action 'self' https://coinpayportal.com",
   "object-src 'none'",
   "upgrade-insecure-requests",
 ].join("; ");

apps/web/package.json

@@ -12,6 +12,7 @@
     "@profullstack/footer": "latest",
     "@profullstack/hqtui": "^0.10.3",
     "@profullstack/libsql-pg": "^0.1.3",
+    "@profullstack/x402-gateway": "^0.11.0",
     "class-variance-authority": "0.7.1",
     "clsx": "2.1.1",
     "lucide-react": "1.37.0",

apps/web/proxy.ts

@@ -0,0 +1,18 @@
+import type { NextRequest } from "next/server";
+import { gate } from "@/lib/crawl-gateway";
+
+/**
+ * Crawl gateway first: AI training crawlers get 402 Payment Required (or the
+ * pass page at /crawl) unless they present a paid pass. Everyone else carries
+ * on to the app.
+ */
+export async function proxy(request: NextRequest) {
+  const answer = await gate(request);
+  if (answer) return answer;
+}
+
+export const config = {
+  matcher: [
+    "/((?!_next/static|_next/image|favicon.ico|.*\\.(?:svg|png|jpg|jpeg|gif|webp|avif|ico|woff2?|css|js|map|pdf)$).*)",
+  ],
+};

apps/web/test/crawl-gateway.test.ts

@@ -0,0 +1,36 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+
+process.env.COINPAY_X402_KEY = "cp_live_test";
+process.env.CRAWL_PAY_TO = "0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5";
+const { gateway } = await import("../lib/crawl-gateway.ts");
+
+const GPTBOT = "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)";
+const HUMAN = {
+  "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
+  accept: "text/html,application/xhtml+xml",
+  "accept-language": "en-US",
+  "sec-fetch-mode": "navigate",
+  "sec-fetch-site": "none",
+  "sec-fetch-dest": "document",
+  "x-real-ip": "198.51.100.7",
+};
+
+test("a training crawler is answered 402 with PAYMENT-REQUIRED", async () => {
+  const res = await gateway.handle(new Request("https://hqtui.com/", { headers: { "user-agent": GPTBOT, "x-real-ip": "203.0.113.9" } }));
+  assert.equal(res?.status, 402);
+  assert.ok(res?.headers.get("payment-required"));
+});
+
+test("/crawl offers a $1 day pass", async () => {
+  const res = await gateway.handle(new Request("https://hqtui.com/crawl", { headers: { accept: "application/json" } }));
+  assert.equal(res?.status, 402);
+  assert.ok(res?.headers.get("payment-required"));
+  const body = (await res!.json()) as { accepts: { amount: string }[] };
+  assert.equal(String(body.accepts[0]?.amount), "1000000");
+});
+
+test("a browser and the API pass through", async () => {
+  assert.equal(await gateway.handle(new Request("https://hqtui.com/", { headers: HUMAN })), null);
+  assert.equal(await gateway.handle(new Request("https://hqtui.com/api/health", { headers: { "user-agent": GPTBOT } })), null);
+});