feat(gateway): x402 crawl gateway for training crawlers

services/gateway/package.json

@@ -12,6 +12,7 @@
   },
   "dependencies": {
     "@nightcell7/observability": "workspace:*",
+    "@profullstack/x402-gateway": "^0.11.0",
     "zod": "^4.1.13"
   }
 }

services/gateway/src/crawl.test.ts

@@ -0,0 +1,95 @@
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { HealthReporter, createLogger } from "@nightcell7/observability";
+import { createCrawlGate } from "./crawl";
+import { createGatewayServer } from "./server";
+
+const env = {
+  COINPAY_X402_KEY: "cp_live_test",
+  CRAWL_PAY_TO: "0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5",
+} as NodeJS.ProcessEnv;
+
+const GPTBOT =
+  "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)";
+const CHROME =
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36";
+const HUMAN = {
+  "user-agent": CHROME,
+  accept: "text/html,application/xhtml+xml",
+  "accept-language": "en-US",
+  "sec-fetch-mode": "navigate",
+  "sec-fetch-site": "none",
+  "sec-fetch-dest": "document",
+  "x-real-ip": "198.51.100.7",
+};
+
+let upstream: http.Server;
+let gateway: http.Server;
+let origin = "";
+
+function listen(server: http.Server): Promise<number> {
+  return new Promise((resolve) =>
+    server.listen(0, "127.0.0.1", () => resolve((server.address() as AddressInfo).port)),
+  );
+}
+
+beforeAll(async () => {
+  upstream = http.createServer((_req, res) => {
+    res.writeHead(200, { "content-type": "text/html" });
+    res.end("<h1>site</h1>");
+  });
+  const port = await listen(upstream);
+  const target = new URL(`http://127.0.0.1:${port}`);
+  gateway = createGatewayServer({
+    upstreams: { site: target, game: target, api: target, multiplayer: target },
+    isProduction: false,
+    maintenance: false,
+    logger: createLogger({ service: "test", level: "error", sink: () => {} }),
+    health: new HealthReporter("gateway", "test"),
+    crawlGate: createCrawlGate(env),
+  });
+  origin = `http://127.0.0.1:${await listen(gateway)}`;
+});
+
+afterAll(async () => {
+  await new Promise((resolve) => gateway.close(resolve));
+  await new Promise((resolve) => upstream.close(resolve));
+});
+
+describe("crawl gate", () => {
+  it("is off without a key", () => {
+    expect(createCrawlGate({} as NodeJS.ProcessEnv)).toBeNull();
+  });
+
+  it("answers a training crawler 402 with PAYMENT-REQUIRED", async () => {
+    const res = await fetch(`${origin}/`, {
+      headers: { "user-agent": GPTBOT, accept: "application/json", "x-real-ip": "203.0.113.9" },
+    });
+    expect(res.status).toBe(402);
+    expect(res.headers.get("payment-required")).toBeTruthy();
+  });
+
+  it("sells a day pass at /crawl", async () => {
+    const res = await fetch(`${origin}/crawl`, {
+      headers: { "user-agent": "x402-test/1.0", accept: "application/json" },
+    });
+    expect(res.status).toBe(402);
+    expect(res.headers.get("payment-required")).toBeTruthy();
+    const body = (await res.json()) as { accepts: { amount: string }[] };
+    expect(String(body.accepts[0]?.amount)).toBe("1000000");
+  });
+
+  it("lets a browser through to the site", async () => {
+    const res = await fetch(`${origin}/`, { headers: HUMAN });
+    expect(res.status).toBe(200);
+    expect(await res.text()).toContain("site");
+  });
+
+  it("never charges the health checks or the API", async () => {
+    const ready = await fetch(`${origin}/health/ready`, { headers: { "user-agent": GPTBOT } });
+    expect(ready.status).not.toBe(402);
+    const api = await fetch(`${origin}/api/v1/catalog`, { headers: { "user-agent": GPTBOT } });
+    expect(api.status).toBe(200);
+  });
+});

services/gateway/src/crawl.ts

@@ -0,0 +1,98 @@
+import type http from "node:http";
+import { createGateway } from "@profullstack/x402-gateway";
+
+/**
+ * The crawl door: training crawlers (GPTBot, CCBot, ...) are answered 402 with
+ * an x402 offer for a $1 day pass, and /crawl sells that pass (USDC by x402 or
+ * a CoinPay checkout for humans). Browsers, verified search crawlers and AI
+ * chat fetchers pass straight through; that is the gateway's default.
+ *
+ * Off unless both COINPAY_X402_KEY and CRAWL_PAY_TO are set, so local dev and
+ * tests that do not ask for it never see it.
+ */
+export interface CrawlGate {
+  handle: (request: Request) => Promise<Response | null>;
+}
+
+const SESSION_COOKIE = /(?:^|;\s*)nc7_session=/;
+
+export function createCrawlGate(env: NodeJS.ProcessEnv = process.env): CrawlGate | null {
+  const apiKey = env["COINPAY_X402_KEY"] ?? "";
+  const payTo = env["CRAWL_PAY_TO"] ?? "";
+  if (!apiKey || !payTo) return null;
+  const siteUrl = (env["SITE_URL"] || "https://nightcell7.com").replace(/\/+$/, "");
+  return createGateway({
+    siteUrl,
+    siteName: "NIGHTCELL 7",
+    coinpay: { apiKey },
+    payTo,
+    contact: "mailto:anthony@profullstack.com",
+    // Health checks, the API (game clients, CoinPay webhooks, matchmaking),
+    // static build assets and the crawl manifests are never charged.
+    openPaths: [
+      "/health/live",
+      "/health/ready",
+      "/api/v1",
+      "/api/v1/",
+      "/_next/",
+      "/robots.txt",
+      "/sitemap.xml",
+      "/llms.txt",
+      "/.well-known/",
+      "/favicon.ico",
+    ],
+    exempt: (request: Request) => SESSION_COOKIE.test(request.headers.get("cookie") ?? ""),
+  });
+}
+
+/** Does the gateway need the request body to answer this? Only the checkout POST. */
+export function needsBody(method: string | undefined, pathname: string): boolean {
+  return method === "POST" && pathname.startsWith("/crawl");
+}
+
+/** A fetch Request built from the raw inbound Node request (before header sanitising). */
+export function requestFor(req: http.IncomingMessage, body?: Buffer): Request {
+  const headers = new Headers();
+  for (const [name, value] of Object.entries(req.headers)) {
+    if (typeof value === "string") headers.set(name, value);
+    else if (Array.isArray(value)) headers.set(name, value.join(", "));
+  }
+  if (!headers.has("x-real-ip") && !headers.has("x-forwarded-for") && req.socket?.remoteAddress) {
+    headers.set("x-forwarded-for", req.socket.remoteAddress);
+  }
+  const method = req.method ?? "GET";
+  const init: RequestInit = { method, headers };
+  if (body && method !== "GET" && method !== "HEAD") init.body = new Uint8Array(body);
+  return new Request(`http://localhost${req.url ?? "/"}`, init);
+}
+
+export function readBody(req: http.IncomingMessage, limit = 64 * 1024): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    let size = 0;
+    req.on("data", (chunk: Buffer) => {
+      size += chunk.length;
+      if (size > limit) {
+        req.destroy();
+        reject(new Error("body too large"));
+        return;
+      }
+      chunks.push(chunk);
+    });
+    req.on("end", () => resolve(Buffer.concat(chunks)));
+    req.on("error", reject);
+  });
+}
+
+/** Write the gateway's answer back through the Node response. */
+export async function answerWith(res: http.ServerResponse, answer: Response): Promise<void> {
+  const headers: Record<string, string | string[]> = {};
+  answer.headers.forEach((value, name) => {
+    if (name === "set-cookie") return;
+    headers[name] = value;
+  });
+  const cookies = answer.headers.getSetCookie?.() ?? [];
+  if (cookies.length) headers["set-cookie"] = cookies;
+  res.writeHead(answer.status, headers);
+  res.end(Buffer.from(await answer.arrayBuffer()));
+}

services/gateway/src/index.ts

@@ -7,6 +7,7 @@ import {
   parseEnv,
 } from "@nightcell7/observability";
 import { UPSTREAM } from "./routes";
+import { createCrawlGate } from "./crawl";
 import { createGatewayServer } from "./server";
 
 /**
@@ -60,6 +61,7 @@ const server = createGatewayServer({
   maintenance: env.MAINTENANCE_MODE,
   logger,
   health,
+  crawlGate: createCrawlGate(process.env),
 });
 
 server.listen(env.GATEWAY_PORT, () => {

services/gateway/src/server.ts

@@ -11,13 +11,16 @@ import {
   securityHeaders,
   type UpstreamName,
 } from "./routes";
+import { answerWith, needsBody, readBody, requestFor, type CrawlGate } from "./crawl";
 
 export interface GatewayOptions {
   upstreams: Record<UpstreamName, URL>;
   isProduction: boolean;
   maintenance: boolean;
   logger: Logger;
   health: HealthReporter;
+  /** The crawl door (crawl.ts): 402 for training crawlers, the /crawl shop. */
+  crawlGate?: CrawlGate | null;
 }
 
 /**
@@ -26,7 +29,7 @@ export interface GatewayOptions {
  * be exercised end to end in a test.
  */
 export function createGatewayServer(options: GatewayOptions): http.Server {
-  const { upstreams, isProduction, maintenance, logger, health } = options;
+  const { upstreams, isProduction, maintenance, logger, health, crawlGate } = options;
 
   const server = http.createServer((req, res) => {
     const correlationId = (req.headers[CORRELATION_HEADER] as string) || newCorrelationId();
@@ -47,6 +50,47 @@ export function createGatewayServer(options: GatewayOptions): http.Server {
       return;
     }
 
+    if (crawlGate) {
+      void sellOrProxy(req, res, pathname, correlationId);
+      return;
+    }
+
+    route_(req, res, pathname, correlationId);
+  });
+
+  async function sellOrProxy(
+    req: http.IncomingMessage,
+    res: http.ServerResponse,
+    pathname: string,
+    correlationId: string,
+  ): Promise<void> {
+    try {
+      const body = needsBody(req.method, pathname) ? await readBody(req) : undefined;
+      const answer = await crawlGate!.handle(requestFor(req, body));
+      if (answer) {
+        await answerWith(res, answer);
+        return;
+      }
+      if (body) {
+        // Nothing upstream serves /crawl POSTs; the gateway always answers those.
+        res.writeHead(404, { "content-type": "application/json", "cache-control": "no-store" });
+        res.end(JSON.stringify({ error: "not_found" }));
+        return;
+      }
+    } catch (error) {
+      // A failing crawl door must never take the site down: fall through.
+      logger.warn("crawl gate failed", { correlationId, error: (error as Error).message });
+      if (res.headersSent || res.writableEnded) return;
+    }
+    route_(req, res, pathname, correlationId);
+  }
+
+  function route_(
+    req: http.IncomingMessage,
+    res: http.ServerResponse,
+    pathname: string,
+    correlationId: string,
+  ): void {
     const route = resolveRoute(pathname);
 
     // Maintenance mode still serves the marketing site so players see a real
@@ -60,7 +104,7 @@ export function createGatewayServer(options: GatewayOptions): http.Server {
     if (!route.cacheable) res.setHeader("cache-control", "no-store");
 
     proxyHttp(req, res, route.upstream, correlationId);
-  });
+  }
 
   function proxyHttp(
     req: http.IncomingMessage,