diff --git a/bun.lock b/bun.lock index 3898e47..73c799c 100644 --- a/bun.lock +++ b/bun.lock @@ -216,6 +216,7 @@ "version": "0.5.6", "dependencies": { "@nightcell7/observability": "workspace:*", + "@profullstack/x402-gateway": "^0.11.0", "zod": "^4.1.13", }, }, @@ -866,6 +867,10 @@ "@profullstack/footer": ["@profullstack/footer@0.2.5", "", { "peerDependencies": { "react": ">=18" }, "optionalPeers": ["react"], "bin": { "profullstack-footer": "bin/profullstack-footer.js" } }, "sha512-/CKwFP3m+qM0ZFTC9Rj11fgClutJzeYliX44Qp4qwrv2JgA/1k4tVOB3mhkPhaeckUhnfg5UyESDI1uoWr9c2w=="], + "@profullstack/keys": ["@profullstack/keys@0.1.0", "", {}, "sha512-Vys47LvS5ivhdZu4SYOxh9Le3DEmWhjP0+CtrU45RhXAbY0ZFG/2Dk9V1bIIUGDp8UWMJqctvUD21JrNjfh04w=="], + + "@profullstack/x402-gateway": ["@profullstack/x402-gateway@0.11.0", "", { "dependencies": { "@profullstack/keys": "^0.1.0" } }, "sha512-Lfrz+SRTwQZfcoCaVtaaMNPJx74Lp/5oyS7vJJAEuJmH1TjBzhE7dWquISkj1p42oc+VW5I9xH9p7YhEKqmr4Q=="], + "@react-email/render": ["@react-email/render@1.1.2", "", { "dependencies": { "html-to-text": "9.0.5", "prettier": "3.9.6", "react-promise-suspense": "0.3.4" }, "peerDependencies": { "react": "^18.0 || ^19.0 || ^19.0.0-rc", "react-dom": "^18.0 || ^19.0 || ^19.0.0-rc" } }, "sha512-RnRehYN3v9gVlNMehHPHhyp2RQo7+pSkHDtXPvg3s0GbzM9SQMW4Qrf8GRNvtpLC4gsI+Wt0VatNRUFqjvevbw=="], "@rollup/plugin-babel": ["@rollup/plugin-babel@6.1.0", "", { "dependencies": { "@babel/helper-module-imports": "7.29.7", "@rollup/pluginutils": "5.4.0" }, "peerDependencies": { "@babel/core": "^7.0.0", "@types/babel__core": "^7.1.9", "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" }, "optionalPeers": ["@types/babel__core", "rollup"] }, "sha512-dFZNuFD2YRcoomP4oYf+DvQNSUA9ih+A3vUqopQx5EdtPGo3WBnQcI/S8pwpz91UsGfL0HsMSOlaMld8HrbubA=="], diff --git a/services/gateway/package.json b/services/gateway/package.json index fb1f82e..df25a00 100644 --- a/services/gateway/package.json +++ b/services/gateway/package.json @@ -12,6 +12,7 @@ }, "dependencies": { "@nightcell7/observability": "workspace:*", + "@profullstack/x402-gateway": "^0.11.0", "zod": "^4.1.13" } } diff --git a/services/gateway/src/crawl.test.ts b/services/gateway/src/crawl.test.ts new file mode 100644 index 0000000..4c0511d --- /dev/null +++ b/services/gateway/src/crawl.test.ts @@ -0,0 +1,95 @@ +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { HealthReporter, createLogger } from "@nightcell7/observability"; +import { createCrawlGate } from "./crawl"; +import { createGatewayServer } from "./server"; + +const env = { + COINPAY_X402_KEY: "cp_live_test", + CRAWL_PAY_TO: "0xCC3b072391AE7A8d10cF00DdC5F61DB2cA5541E5", +} as NodeJS.ProcessEnv; + +const GPTBOT = + "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)"; +const CHROME = + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"; +const HUMAN = { + "user-agent": CHROME, + accept: "text/html,application/xhtml+xml", + "accept-language": "en-US", + "sec-fetch-mode": "navigate", + "sec-fetch-site": "none", + "sec-fetch-dest": "document", + "x-real-ip": "198.51.100.7", +}; + +let upstream: http.Server; +let gateway: http.Server; +let origin = ""; + +function listen(server: http.Server): Promise { + return new Promise((resolve) => + server.listen(0, "127.0.0.1", () => resolve((server.address() as AddressInfo).port)), + ); +} + +beforeAll(async () => { + upstream = http.createServer((_req, res) => { + res.writeHead(200, { "content-type": "text/html" }); + res.end("

site

"); + }); + const port = await listen(upstream); + const target = new URL(`http://127.0.0.1:${port}`); + gateway = createGatewayServer({ + upstreams: { site: target, game: target, api: target, multiplayer: target }, + isProduction: false, + maintenance: false, + logger: createLogger({ service: "test", level: "error", sink: () => {} }), + health: new HealthReporter("gateway", "test"), + crawlGate: createCrawlGate(env), + }); + origin = `http://127.0.0.1:${await listen(gateway)}`; +}); + +afterAll(async () => { + await new Promise((resolve) => gateway.close(resolve)); + await new Promise((resolve) => upstream.close(resolve)); +}); + +describe("crawl gate", () => { + it("is off without a key", () => { + expect(createCrawlGate({} as NodeJS.ProcessEnv)).toBeNull(); + }); + + it("answers a training crawler 402 with PAYMENT-REQUIRED", async () => { + const res = await fetch(`${origin}/`, { + headers: { "user-agent": GPTBOT, accept: "application/json", "x-real-ip": "203.0.113.9" }, + }); + expect(res.status).toBe(402); + expect(res.headers.get("payment-required")).toBeTruthy(); + }); + + it("sells a day pass at /crawl", async () => { + const res = await fetch(`${origin}/crawl`, { + headers: { "user-agent": "x402-test/1.0", accept: "application/json" }, + }); + expect(res.status).toBe(402); + expect(res.headers.get("payment-required")).toBeTruthy(); + const body = (await res.json()) as { accepts: { amount: string }[] }; + expect(String(body.accepts[0]?.amount)).toBe("1000000"); + }); + + it("lets a browser through to the site", async () => { + const res = await fetch(`${origin}/`, { headers: HUMAN }); + expect(res.status).toBe(200); + expect(await res.text()).toContain("site"); + }); + + it("never charges the health checks or the API", async () => { + const ready = await fetch(`${origin}/health/ready`, { headers: { "user-agent": GPTBOT } }); + expect(ready.status).not.toBe(402); + const api = await fetch(`${origin}/api/v1/catalog`, { headers: { "user-agent": GPTBOT } }); + expect(api.status).toBe(200); + }); +}); diff --git a/services/gateway/src/crawl.ts b/services/gateway/src/crawl.ts new file mode 100644 index 0000000..91cfb44 --- /dev/null +++ b/services/gateway/src/crawl.ts @@ -0,0 +1,98 @@ +import type http from "node:http"; +import { createGateway } from "@profullstack/x402-gateway"; + +/** + * The crawl door: training crawlers (GPTBot, CCBot, ...) are answered 402 with + * an x402 offer for a $1 day pass, and /crawl sells that pass (USDC by x402 or + * a CoinPay checkout for humans). Browsers, verified search crawlers and AI + * chat fetchers pass straight through; that is the gateway's default. + * + * Off unless both COINPAY_X402_KEY and CRAWL_PAY_TO are set, so local dev and + * tests that do not ask for it never see it. + */ +export interface CrawlGate { + handle: (request: Request) => Promise; +} + +const SESSION_COOKIE = /(?:^|;\s*)nc7_session=/; + +export function createCrawlGate(env: NodeJS.ProcessEnv = process.env): CrawlGate | null { + const apiKey = env["COINPAY_X402_KEY"] ?? ""; + const payTo = env["CRAWL_PAY_TO"] ?? ""; + if (!apiKey || !payTo) return null; + const siteUrl = (env["SITE_URL"] || "https://nightcell7.com").replace(/\/+$/, ""); + return createGateway({ + siteUrl, + siteName: "NIGHTCELL 7", + coinpay: { apiKey }, + payTo, + contact: "mailto:anthony@profullstack.com", + // Health checks, the API (game clients, CoinPay webhooks, matchmaking), + // static build assets and the crawl manifests are never charged. + openPaths: [ + "/health/live", + "/health/ready", + "/api/v1", + "/api/v1/", + "/_next/", + "/robots.txt", + "/sitemap.xml", + "/llms.txt", + "/.well-known/", + "/favicon.ico", + ], + exempt: (request: Request) => SESSION_COOKIE.test(request.headers.get("cookie") ?? ""), + }); +} + +/** Does the gateway need the request body to answer this? Only the checkout POST. */ +export function needsBody(method: string | undefined, pathname: string): boolean { + return method === "POST" && pathname.startsWith("/crawl"); +} + +/** A fetch Request built from the raw inbound Node request (before header sanitising). */ +export function requestFor(req: http.IncomingMessage, body?: Buffer): Request { + const headers = new Headers(); + for (const [name, value] of Object.entries(req.headers)) { + if (typeof value === "string") headers.set(name, value); + else if (Array.isArray(value)) headers.set(name, value.join(", ")); + } + if (!headers.has("x-real-ip") && !headers.has("x-forwarded-for") && req.socket?.remoteAddress) { + headers.set("x-forwarded-for", req.socket.remoteAddress); + } + const method = req.method ?? "GET"; + const init: RequestInit = { method, headers }; + if (body && method !== "GET" && method !== "HEAD") init.body = new Uint8Array(body); + return new Request(`http://localhost${req.url ?? "/"}`, init); +} + +export function readBody(req: http.IncomingMessage, limit = 64 * 1024): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let size = 0; + req.on("data", (chunk: Buffer) => { + size += chunk.length; + if (size > limit) { + req.destroy(); + reject(new Error("body too large")); + return; + } + chunks.push(chunk); + }); + req.on("end", () => resolve(Buffer.concat(chunks))); + req.on("error", reject); + }); +} + +/** Write the gateway's answer back through the Node response. */ +export async function answerWith(res: http.ServerResponse, answer: Response): Promise { + const headers: Record = {}; + answer.headers.forEach((value, name) => { + if (name === "set-cookie") return; + headers[name] = value; + }); + const cookies = answer.headers.getSetCookie?.() ?? []; + if (cookies.length) headers["set-cookie"] = cookies; + res.writeHead(answer.status, headers); + res.end(Buffer.from(await answer.arrayBuffer())); +} diff --git a/services/gateway/src/index.ts b/services/gateway/src/index.ts index 7cbd7e6..c98aa03 100644 --- a/services/gateway/src/index.ts +++ b/services/gateway/src/index.ts @@ -7,6 +7,7 @@ import { parseEnv, } from "@nightcell7/observability"; import { UPSTREAM } from "./routes"; +import { createCrawlGate } from "./crawl"; import { createGatewayServer } from "./server"; /** @@ -60,6 +61,7 @@ const server = createGatewayServer({ maintenance: env.MAINTENANCE_MODE, logger, health, + crawlGate: createCrawlGate(process.env), }); server.listen(env.GATEWAY_PORT, () => { diff --git a/services/gateway/src/server.ts b/services/gateway/src/server.ts index 0cd19d5..4512303 100644 --- a/services/gateway/src/server.ts +++ b/services/gateway/src/server.ts @@ -11,6 +11,7 @@ import { securityHeaders, type UpstreamName, } from "./routes"; +import { answerWith, needsBody, readBody, requestFor, type CrawlGate } from "./crawl"; export interface GatewayOptions { upstreams: Record; @@ -18,6 +19,8 @@ export interface GatewayOptions { maintenance: boolean; logger: Logger; health: HealthReporter; + /** The crawl door (crawl.ts): 402 for training crawlers, the /crawl shop. */ + crawlGate?: CrawlGate | null; } /** @@ -26,7 +29,7 @@ export interface GatewayOptions { * be exercised end to end in a test. */ export function createGatewayServer(options: GatewayOptions): http.Server { - const { upstreams, isProduction, maintenance, logger, health } = options; + const { upstreams, isProduction, maintenance, logger, health, crawlGate } = options; const server = http.createServer((req, res) => { const correlationId = (req.headers[CORRELATION_HEADER] as string) || newCorrelationId(); @@ -47,6 +50,47 @@ export function createGatewayServer(options: GatewayOptions): http.Server { return; } + if (crawlGate) { + void sellOrProxy(req, res, pathname, correlationId); + return; + } + + route_(req, res, pathname, correlationId); + }); + + async function sellOrProxy( + req: http.IncomingMessage, + res: http.ServerResponse, + pathname: string, + correlationId: string, + ): Promise { + try { + const body = needsBody(req.method, pathname) ? await readBody(req) : undefined; + const answer = await crawlGate!.handle(requestFor(req, body)); + if (answer) { + await answerWith(res, answer); + return; + } + if (body) { + // Nothing upstream serves /crawl POSTs; the gateway always answers those. + res.writeHead(404, { "content-type": "application/json", "cache-control": "no-store" }); + res.end(JSON.stringify({ error: "not_found" })); + return; + } + } catch (error) { + // A failing crawl door must never take the site down: fall through. + logger.warn("crawl gate failed", { correlationId, error: (error as Error).message }); + if (res.headersSent || res.writableEnded) return; + } + route_(req, res, pathname, correlationId); + } + + function route_( + req: http.IncomingMessage, + res: http.ServerResponse, + pathname: string, + correlationId: string, + ): void { const route = resolveRoute(pathname); // Maintenance mode still serves the marketing site so players see a real @@ -60,7 +104,7 @@ export function createGatewayServer(options: GatewayOptions): http.Server { if (!route.cacheable) res.setHeader("cache-control", "no-store"); proxyHttp(req, res, route.upstream, correlationId); - }); + } function proxyHttp( req: http.IncomingMessage,