From 3bcb3d117dade0d639ce47f58a64958458eeab58 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 7 Sep 2026 04:16:48 -0700 Subject: [PATCH 1/6] feat: add opt-in HTTP context for Node and Rust --- docs/code-review.md | 6 + docs/feature-mapping.md | 50 +++++ src/app.ts | 8 + src/cli.ts | 19 +- src/http-relations.test.ts | 304 +++++++++++++++++++++++++++++ src/http-relations.ts | 378 +++++++++++++++++++++++++++++++++++++ src/review.ts | 15 +- 7 files changed, 778 insertions(+), 2 deletions(-) create mode 100644 src/http-relations.test.ts create mode 100644 src/http-relations.ts diff --git a/docs/code-review.md b/docs/code-review.md index 1c6fac9..3d33f73 100644 --- a/docs/code-review.md +++ b/docs/code-review.md @@ -187,3 +187,9 @@ coordinates to the public npm registry. Enable it explicitly with deduplicates registry calls per `(name, version)`. [slop-paper]: https://arxiv.org/abs/2406.10279 + +For a Node frontend and Rust backend sharing one HTTP path namespace, +`review --link-http frontend:backend` adds bounded, literal HTTP candidate context +only for that review. Existing feature records and default review behavior are +unchanged. See [Optional HTTP relations](feature-mapping.md#optional-http-relations) +for the root-pair assertion, supported syntax, ambiguity rules, and limits. diff --git a/docs/feature-mapping.md b/docs/feature-mapping.md index dab0364..9fb2391 100644 --- a/docs/feature-mapping.md +++ b/docs/feature-mapping.md @@ -195,3 +195,53 @@ Known gaps: or runtime route conventions - no import graph expansion beyond nearby tests yet - agent mapping depends on provider quality and validates paths but not semantic intent + +## Optional HTTP relations + +For a Node/TypeScript frontend and Rust backend that share one HTTP path +namespace, opt in with explicit, non-overlapping repository-relative roots: + +```sh +clawpatch map --link-http frontend:backend --json +clawpatch review --link-http frontend:backend --limit 3 +``` + +The root pair asserts which client and service belong together. Clawpatch does +not discover deployment origins, proxy rules, or service topology. The output +contains **candidate** HTTP relations, not proof of runtime connectivity. +Verify routing before relying on a relation in a finding. + +The first version matches literal `fetch("/path")` (GET) and +`fetch("/path", { method: "POST" })` calls to Rust `#[get("/path")]`, +`#[post("/path")]`, `put`, `patch`, `delete`, `head`, or `options` attributes. +Caller scanning supports `.js`, `.ts`, `.mjs`, `.cjs`, `.mts`, and `.cts`; JSX/TSX +files are skipped. Methods and paths must match exactly. Additional fetch options, computed values, +template literals, query strings, absolute URLs, parameters, wildcard paths, +Axios, and other handler syntaxes are unsupported. Comments and string contents +are skipped. Complex template interpolations containing division or regular +expressions conservatively end scanning of that caller file. Actix-shaped `web::scope(...)` and any Rust `.mount(...)` call disable the pass +because their prefixes are unresolved, including mounts in helpers whose server +was constructed elsewhere. External prefixes and macro-generated +routes remain outside this heuristic; the supplied roots must use the same path +namespace. Multiple recognized handlers for the same method/path are ambiguous +and produce no link, even when declared in one file. + +Mapping returns an `http` object containing `relations`, `omitted`, and +`skippedReason`; `--dry-run` returns it too. Each relation identifies the HTTP +method/path, caller and handler files/lines, and the features owning those files. +Default mapping output and stored feature slices remain unchanged. Reviews with +this flag recompute relations from current source, then append up to three +counterpart files to an ephemeral prompt copy after existing context. Existing +context-file and per-file prompt limits still apply, including omission reporting. +Review without the flag never adds HTTP context. Mapping alone does not enable +it for later reviews, fixes, revalidation, or `ci` runs. + +The scan honors configured include/exclude filters and normal mapper directory +exclusions, skips symlinks, and links only files owned by active features. It +scans at most 500 source files, 256,000 bytes per file, and 8,000,000 bytes total; +exceeding a scan budget returns no relations with a reason rather than matching +against an incomplete inventory. Output is sorted by source path and declaration +order and limited to 200 relations; `omitted` counts links dropped by that output +limit. The three-file review context limit is applied independently for each +feature, so a broad co-owner cannot suppress context for a narrower feature. There is no graph storage +or feature schema migration. diff --git a/src/app.ts b/src/app.ts index 757cda8..14576e9 100644 --- a/src/app.ts +++ b/src/app.ts @@ -8,6 +8,7 @@ import { nowIso, writeJson } from "./fs.js"; import { discoverGit } from "./git.js"; import { mapWithSource } from "./agent-mapper.js"; import { mapFeatures } from "./mapper.js"; +import { findHttpRelations } from "./http-relations.js"; import { emitProgress } from "./progress.js"; import { providerByName } from "./provider.js"; import { @@ -112,6 +113,11 @@ export async function mapCommand( emitProgress(context, "map", event, fields); }, }); + const linkHttp = stringFlag(flags, "linkHttp"); + const http = + linkHttp === undefined + ? {} + : { http: await findHttpRelations(loaded.root, result.features, linkHttp, filters) }; const activeFeatureIds = new Set(result.features.map((feature) => feature.featureId)); if (flags["dryRun"] === true) { emitProgress(context, "map", "done", { @@ -120,6 +126,7 @@ export async function mapCommand( elapsed: `${Math.round((Date.now() - started) / 1000)}s`, }); return { + ...http, dryRun: true, features: result.features.length, new: result.created, @@ -152,6 +159,7 @@ export async function mapCommand( elapsed: `${Math.round((Date.now() - started) / 1000)}s`, }); return { + ...http, features: result.features.length, new: result.created, changed: result.changed, diff --git a/src/cli.ts b/src/cli.ts index a6da8f0..d129152 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -21,6 +21,7 @@ import { } from "./app.js"; import { ClawpatchError } from "./errors.js"; import { GlobalOptions } from "./config.js"; +import { httpRoots } from "./http-relations.js"; const moduleRequire = createRequire(import.meta.url); @@ -112,6 +113,7 @@ export function parseArgs(argv: string[]): ParsedArgs { command = "status"; } validateCommandFlags(command, flags); + if (typeof flags["linkHttp"] === "string") httpRoots(flags["linkHttp"]); validateCommandRequirements(command, flags); return { command, flags, global, help: false, version: false }; } @@ -131,7 +133,15 @@ type CommandSpec = { const commandSpecs = { init: { flags: ["force"], usage: ["clawpatch init [flags]"], run: initCommand }, map: { - flags: ["dryRun", "source", "provider", "model", "reasoningEffort", "skipGitRepoCheck"], + flags: [ + "dryRun", + "source", + "provider", + "model", + "reasoningEffort", + "skipGitRepoCheck", + "linkHttp", + ], usage: ["clawpatch map [flags]"], run: mapCommand, }, @@ -140,6 +150,7 @@ const commandSpecs = { flags: [ "feature", "featureList", + "linkHttp", "project", "limit", "since", @@ -316,6 +327,12 @@ const optionSpecs: Record = { target: "command", help: " --rate-limit-per-minute cap provider calls per 60s window (env: CLAWPATCH_RPM)", }, + "link-http": { + name: "linkHttp", + kind: "value", + target: "command", + help: " --link-http opt-in HTTP candidate context between directory roots", + }, source: { name: "source", kind: "value", diff --git a/src/http-relations.test.ts b/src/http-relations.test.ts new file mode 100644 index 0000000..b5068e9 --- /dev/null +++ b/src/http-relations.test.ts @@ -0,0 +1,304 @@ +import { mkdtemp, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { initCommand, makeContext, mapCommand, reviewCommand } from "./app.js"; +import { parseArgs } from "./cli.js"; +import { defaultConfig } from "./config.js"; +import { findHttpRelations, httpEndpoints, httpRoots, withHttpContext } from "./http-relations.js"; +import { buildReviewPromptBundle } from "./prompt.js"; +import { readFeatures, readProject, statePaths } from "./state.js"; +import { testOptions, writeFixture } from "./test-helpers.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); +}); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "clawpatch-http-")); + roots.push(root); + await writeFixture( + root, + "package.json", + JSON.stringify({ name: "http-test", workspaces: ["frontend"] }), + ); + await writeFixture( + root, + "frontend/package.json", + JSON.stringify({ name: "frontend", type: "module" }), + ); + await writeFixture(root, "frontend/src/app.ts", 'await fetch("/api/login");\n'); + await writeFixture(root, "Cargo.toml", '[workspace]\nmembers = ["backend"]\nresolver = "2"\n'); + await writeFixture( + root, + "backend/Cargo.toml", + '[package]\nname = "backend"\nversion = "0.1.0"\nedition = "2021"\n', + ); + await writeFixture( + root, + "backend/src/main.rs", + '#[get("/api/login")]\nasync fn login() {}\nfn main() {}\n', + ); + const context = await makeContext({ ...testOptions(root), quiet: true }); + await initCommand(context, {}); + const baseline = await mapCommand(context, {}); + const paths = statePaths(join(root, ".clawpatch")); + const features = await readFeatures(paths); + const scan = () => + findHttpRelations(root, features, "frontend:backend", { include: ["**"], exclude: [] }); + return { root, context, baseline, paths, features, scan }; +} + +describe("HTTP candidate context", () => { + it("preserves mapping records and adds current counterpart files only to a bounded prompt copy", async () => { + const setup = await fixture(); + expect(setup.baseline).not.toHaveProperty("http"); + const before = await readFeatures(setup.paths); + const mapped = (await mapCommand(setup.context, { + linkHttp: "frontend:backend", + dryRun: true, + })) as { http: Awaited> }; + expect(mapped.http.relations).toHaveLength(1); + expect(mapped.http.relations[0]).toMatchObject({ + method: "GET", + path: "/api/login", + caller: { file: "frontend/src/app.ts", line: 1 }, + handler: { file: "backend/src/main.rs", line: 1 }, + }); + expect(await readFeatures(setup.paths)).toEqual(before); + const relation = mapped.http.relations[0]!; + const backend = setup.features.find((f) => f.featureId === relation.handler.featureIds[0])!; + const enriched = withHttpContext(backend, mapped.http.relations); + expect(enriched.contextFiles).toContainEqual({ + path: "frontend/src/app.ts", + reason: "candidate HTTP GET /api/login; verify runtime routing", + }); + expect(backend.contextFiles).not.toContainEqual( + expect.objectContaining({ path: "frontend/src/app.ts" }), + ); + const project = (await readProject(setup.paths))!; + const prompt = await buildReviewPromptBundle(setup.root, project, enriched, defaultConfig()); + expect(prompt.manifest.includedFiles).toContainEqual( + expect.objectContaining({ path: "frontend/src/app.ts", readable: true, role: "context" }), + ); + expect(prompt.prompt).toContain('await fetch("/api/login")'); + const config = defaultConfig(); + config.review.maxContextFiles = 1; + const limited = await buildReviewPromptBundle(setup.root, project, enriched, config); + expect(limited.manifest.omittedFiles).toContainEqual({ + path: "frontend/src/app.ts", + role: "context", + reason: "maxContextFiles", + }); + await writeFixture( + setup.root, + "backend/src/main.rs", + '#[post("/api/login")]\nasync fn login() {}\nfn main() {}\n', + ); + expect((await setup.scan()).relations).toEqual([]); + }); + + it("uses HTTP context during review without storing it or enabling later reviews", async () => { + const setup = await fixture(); + await writeFixture(setup.root, "frontend/src/app.ts", 'fetch("/api/login"); // TODO_BUG\n'); + const backend = setup.features.find((f) => f.source === "rust-command")!; + const flags = { feature: backend.featureId, provider: "mock" }; + expect(await reviewCommand(setup.context, flags)).toMatchObject({ findings: 0 }); + expect( + await reviewCommand(setup.context, { ...flags, linkHttp: "frontend:backend" }), + ).toMatchObject({ findings: 1 }); + const saved = (await readFeatures(setup.paths)).find((f) => f.featureId === backend.featureId)!; + expect(saved.contextFiles).toEqual(backend.contextFiles); + expect(await reviewCommand(setup.context, flags)).toMatchObject({ findings: 0 }); + }); + + it("rejects ambiguity in one file and unresolved mount prefixes", async () => { + const setup = await fixture(); + await writeFixture( + setup.root, + "backend/src/main.rs", + '#[get("/api/login")]\nfn a() {}\n#[get("/api/login")]\nfn b() {}\n', + ); + expect((await setup.scan()).relations).toEqual([]); + await writeFixture( + setup.root, + "backend/src/main.rs", + '#[get("/api/login")]\nfn a() {}\nweb::scope("/v2");\n', + ); + expect((await setup.scan()).skippedReason).toContain("prefixes"); + await writeFixture( + setup.root, + "backend/src/main.rs", + '#[get("/api/login")]\nfn login() {}\nfn main() {}\n', + ); + await writeFixture( + setup.root, + "backend/src/config.rs", + 'fn configure(server: Rocket) -> Rocket { server.mount("/v2", routes![login]) }\n', + ); + expect((await setup.scan()).skippedReason).toContain("prefixes"); + await writeFixture( + setup.root, + "backend/src/config.rs", + 'fn configure(server: Rocket) { server./* prefix */mount("/v2", routes![login]); }\n', + ); + expect((await setup.scan()).skippedReason).toContain("prefixes"); + await writeFixture( + setup.root, + "backend/src/config.rs", + 'fn configure() { web/* prefix */::scope("/v2"); }\n', + ); + expect((await setup.scan()).skippedReason).toContain("prefixes"); + }); + + it("honors filters and refuses partial scans, missing sources, and symlinked roots", async () => { + const setup = await fixture(); + expect( + ( + await findHttpRelations(setup.root, setup.features, "frontend:backend", { + include: ["**"], + exclude: ["backend/**"], + }) + ).relations, + ).toEqual([]); + await writeFixture(setup.root, "backend/src/oversized.rs", " ".repeat(256_001)); + expect(await setup.scan()).toMatchObject({ + relations: [], + skippedReason: expect.stringContaining("byte budget"), + }); + await symlink(join(setup.root, "frontend"), join(setup.root, "alias"), "dir"); + await expect( + findHttpRelations(setup.root, setup.features, "frontend:alias", { + include: ["**"], + exclude: [], + }), + ).rejects.toThrow("must not overlap"); + const outside = await mkdtemp(join(tmpdir(), "clawpatch-http-outside-")); + roots.push(outside); + await symlink(outside, join(setup.root, "external"), "dir"); + await expect( + findHttpRelations(setup.root, setup.features, "frontend:external", { + include: ["**"], + exclude: [], + }), + ).rejects.toThrow("invalid HTTP relation root"); + await expect( + findHttpRelations(setup.root, setup.features, "frontend:missing", { + include: ["**"], + exclude: [], + }), + ).rejects.toThrow("invalid HTTP relation root"); + }); + + it("ignores unsupported files before budgeting and unrelated scope text", async () => { + const setup = await fixture(); + await Promise.all( + Array.from({ length: 501 }, (_, i) => + writeFixture(setup.root, `frontend/ui/part${i}.tsx`, ""), + ), + ); + await writeFixture(setup.root, "frontend/ui/large.tsx", " ".repeat(256_001)); + await writeFixture( + setup.root, + "backend/src/main.rs", + '#[get("/api/login")]\nfn login() {}\n// web::scope("/ignored")\nlet example = "web::scope(";\nstd::thread::scope(|s| {});\n', + ); + expect((await setup.scan()).relations).toHaveLength(1); + }); + + it("caps review context independently from the mapped relations", async () => { + const setup = await fixture(); + for (let i = 0; i < 5; i++) + await writeFixture(setup.root, `frontend/src/client${i}.ts`, 'fetch("/api/login");\n'); + await mapCommand(setup.context, {}); + const features = await readFeatures(setup.paths); + const result = await findHttpRelations(setup.root, features, "frontend:backend", { + include: ["**"], + exclude: [], + }); + expect(result.relations).toHaveLength(6); + expect(result.omitted).toBe(0); + const backend = features.find((f) => f.source === "rust-command")!; + const context = withHttpContext(backend, result.relations).contextFiles.filter((ref) => + ref.reason.startsWith("candidate HTTP"), + ); + expect(context.map((ref) => ref.path)).toEqual([ + "frontend/src/app.ts", + "frontend/src/client0.ts", + "frontend/src/client1.ts", + ]); + }); +}); + +describe("literal HTTP syntax", () => { + it("matches methods exactly and rejects options, members, templates, absolute/dynamic paths", () => { + const source = `fetch("/get"); fetch('/post', { method: "POST" }); fetch("/no", options); fetch("/no", { ...options }); obj.fetch("/no"); fetch(\`/no\`); fetch("https://host/no"); fetch("//host/no"); fetch("/no?q=x"); fetch("/users/:id"); fetch("/get", { method: verb });`; + expect( + httpEndpoints(source, "client.ts", "caller").map(({ method, path }) => [method, path]), + ).toEqual([ + ["GET", "/get"], + ["POST", "/post"], + ]); + }); + + it("ignores comments, regexes, templates, strings, Rust raw strings and nested comments", () => { + expect( + httpEndpoints( + '// fetch("/no")\n/* fetch("/no") */\nconst x = \'fetch("/no")\';\nconst t = `fetch("/no")`;\nconst regex = /fetch(.+)/;\nfetch("/yes")', + "client.ts", + "caller", + ).map((r) => r.path), + ).toEqual(["/yes"]); + const source = + '// #[get("/no")]\n/* /* nested */ #[get("/no")] */\nlet s = r###" #[get("/no")] "###;\n#[post("/yes")]\nfn yes() {}'; + expect( + httpEndpoints(source, "main.rs", "handler").map(({ method, path }) => [method, path]), + ).toEqual([["POST", "/yes"]]); + }); + + it("does not confuse nested templates, JSX text, or spaced members with fetch calls", () => { + const nested = 'const x = `outer ${`fetch("/fake")`} end`; fetch("/real");'; + expect(httpEndpoints(nested, "client.ts", "caller").map((r) => r.path)).toEqual(["/real"]); + expect( + httpEndpoints('object. fetch("/fake"); fetch("/real");', "client.ts", "caller").map( + (r) => r.path, + ), + ).toEqual(["/real"]); + expect( + httpEndpoints( + 'client. /* wrapper */ fetch("/fake"); fetch("/real");', + "client.ts", + "caller", + ).map((r) => r.path), + ).toEqual(["/real"]); + expect(httpEndpoints('
fetch("/fake")
', "client.tsx", "caller")).toEqual([]); + }); + + it("tracks line numbers across a dense supported source file", () => { + const endpoints = httpEndpoints('fetch("/");\n'.repeat(20_000), "client.ts", "caller"); + expect(endpoints).toHaveLength(20_000); + expect(endpoints[0]?.line).toBe(1); + expect(endpoints.at(-1)?.line).toBe(20_000); + }); + + it("validates the explicit service pair and exposes it only on map/review", () => { + for (const value of [ + "", + "frontend", + ".:backend", + "../frontend:backend", + "/frontend:backend", + "front:front/back", + "front:front", + "front:back:third", + ]) + expect(() => httpRoots(value)).toThrow(); + expect(httpRoots("apps/web:services/api")).toEqual(["apps/web", "services/api"]); + for (const command of ["map", "review"]) + expect(parseArgs([command, "--link-http", "frontend:backend"]).flags["linkHttp"]).toBe( + "frontend:backend", + ); + expect(() => parseArgs(["ci", "--link-http", "frontend:backend"])).toThrow(); + }); +}); diff --git a/src/http-relations.ts b/src/http-relations.ts new file mode 100644 index 0000000..882a3de --- /dev/null +++ b/src/http-relations.ts @@ -0,0 +1,378 @@ +import { open, realpath, stat } from "node:fs/promises"; +import { isAbsolute, relative, resolve, sep } from "node:path"; +import { ClawpatchError } from "./errors.js"; +import { pathMatchesFilters, walk, type PathFilters } from "./mappers/shared.js"; +import type { FeatureRecord } from "./types.js"; + +export type HttpRelation = { + method: string; + path: string; + caller: { file: string; line: number; featureIds: string[] }; + handler: { file: string; line: number; featureIds: string[] }; +}; +export type HttpRelations = { + relations: HttpRelation[]; + omitted: number; + skippedReason: string | null; +}; +type Endpoint = { method: string; path: string; file: string; line: number }; +const sourceLimit = 256_000; +const maxFiles = 500; +const totalLimit = 8_000_000; +const counterpartLimit = 3; +const methods = "get|post|put|patch|delete|head|options"; + +export function httpRoots(value: string): [string, string] { + const parts = value.split(":"); + if ( + parts.length !== 2 || + parts.some( + (part) => + !part || + part.split("/").some((p) => !p || p === "." || p === "..") || + /[\\]/u.test(part) || + isAbsolute(part), + ) + ) { + throw new ClawpatchError( + "--link-http requires caller:backend repository-relative directory roots", + 2, + "invalid-usage", + ); + } + const [caller, handler] = parts as [string, string]; + if (within(caller, handler) || within(handler, caller)) { + throw new ClawpatchError("--link-http roots must not overlap", 2, "invalid-usage"); + } + return [caller, handler]; +} + +export async function findHttpRelations( + root: string, + features: FeatureRecord[], + value: string, + filters: PathFilters, +): Promise { + const [callerRoot, handlerRoot] = httpRoots(value); + const realRoot = await realpath(root); + const canonicalRoots: string[] = []; + for (const scope of [callerRoot, handlerRoot]) { + const full = resolve(root, scope); + const actual = await realpath(full).catch(() => null); + if (actual === null || !inside(realRoot, actual) || !(await stat(actual)).isDirectory()) { + throw new ClawpatchError(`invalid HTTP relation root: ${scope}`, 2, "invalid-usage"); + } + canonicalRoots.push(actual); + } + if ( + inside(canonicalRoots[0]!, canonicalRoots[1]!) || + inside(canonicalRoots[1]!, canonicalRoots[0]!) + ) { + throw new ClawpatchError("--link-http roots must not overlap", 2, "invalid-usage"); + } + const files = (await walk(root, [callerRoot, handlerRoot])) + .filter( + (file) => + pathMatchesFilters(file, filters) && + ((within(file, callerRoot) && /\.[cm]?[jt]s$/u.test(file)) || + (within(file, handlerRoot) && file.endsWith(".rs"))), + ) + .toSorted(); + if (files.length > maxFiles) return skipped("HTTP relation scan exceeds 500 source files"); + const callers: Endpoint[] = []; + const handlers: Endpoint[] = []; + let bytes = 0; + for (const file of files) { + const full = resolve(root, file); + const actual = await realpath(full).catch(() => null); + if (actual === null || !inside(realRoot, actual)) + return skipped("HTTP relation source is missing or outside the repository"); + const handle = await open(actual, "r"); + let source: string; + try { + const buffer = Buffer.alloc(sourceLimit + 1); + let bytesRead = 0; + while (bytesRead < buffer.length) { + const read = await handle.read(buffer, bytesRead, buffer.length - bytesRead, bytesRead); + if (read.bytesRead === 0) break; + bytesRead += read.bytesRead; + } + bytes += bytesRead; + if (bytesRead > sourceLimit || bytes > totalLimit) + return skipped("HTTP relation scan exceeds its source byte budget"); + source = buffer.subarray(0, bytesRead).toString("utf8"); + } finally { + await handle.close(); + } + if (within(file, callerRoot) && !file.endsWith(".rs")) + callers.push(...httpEndpoints(source, file, "caller")); + if (within(file, handlerRoot) && file.endsWith(".rs")) { + // Mounting changes route paths; do not guess prefixes from local declarations. + if (hasRouteMount(source)) + return skipped("HTTP backend has unresolved scope or mount prefixes"); + handlers.push(...httpEndpoints(source, file, "handler")); + } + } + const ownersByFile = new Map>(); + for (const feature of features) { + if (feature.status === "skipped") continue; + for (const ref of feature.ownedFiles) { + const ids = ownersByFile.get(ref.path) ?? new Set(); + ids.add(feature.featureId); + ownersByFile.set(ref.path, ids); + } + } + const ownerIds = new Map([...ownersByFile].map(([file, ids]) => [file, [...ids].toSorted()])); + const byRoute = new Map(); + for (const handler of handlers) { + const key = `${handler.method} ${handler.path}`; + byRoute.set(key, [...(byRoute.get(key) ?? []), handler]); + } + const relations: HttpRelation[] = []; + const seen = new Set(); + let omitted = 0; + for (const caller of callers) { + const matches = byRoute.get(`${caller.method} ${caller.path}`) ?? []; + if (matches.length !== 1) continue; + const handler = matches[0]!; + const callerIds = ownerIds.get(caller.file) ?? []; + const handlerIds = ownerIds.get(handler.file) ?? []; + if (!callerIds.length || !handlerIds.length) continue; + const key = `${caller.file}:${caller.method}:${caller.path}:${handler.file}`; + if (seen.has(key)) continue; + seen.add(key); + if (relations.length >= 200) { + omitted += 1; + continue; + } + relations.push({ + method: caller.method, + path: caller.path, + caller: { file: caller.file, line: caller.line, featureIds: callerIds }, + handler: { file: handler.file, line: handler.line, featureIds: handlerIds }, + }); + } + return { relations, omitted, skippedReason: null }; +} + +function hasRouteMount(source: string): boolean { + const tokens = codeSource(source, codeMask(source, true)); + return /\bweb\s*::\s*scope\s*\(/u.test(tokens) || /\.\s*mount\s*\(/u.test(tokens); +} + +function codeSource(source: string, mask: Uint8Array): string { + return source + .split("") + .map((char, index) => (mask[index] === 1 ? char : " ")) + .join(""); +} + +export function withHttpContext(feature: FeatureRecord, relations: HttpRelation[]): FeatureRecord { + const refs = new Map(); + for (const relation of relations) { + const counterpart = relation.caller.featureIds.includes(feature.featureId) + ? relation.handler + : relation.handler.featureIds.includes(feature.featureId) + ? relation.caller + : null; + if (counterpart !== null) + refs.set( + counterpart.file, + `candidate HTTP ${relation.method} ${relation.path}; verify runtime routing`, + ); + } + return { + ...feature, + contextFiles: [ + ...feature.contextFiles, + ...[...refs].slice(0, counterpartLimit).map(([path, reason]) => ({ path, reason })), + ], + }; +} + +export function httpEndpoints( + source: string, + file: string, + role: "caller" | "handler", +): Endpoint[] { + if (role === "caller" && /\.[jt]sx$/u.test(file)) return []; + const code = codeMask(source, role === "handler"); + const tokens = codeSource(source, code); + const literal = String.raw`(["'])(\/[A-Za-z0-9_./~-]*)\1`; + const pattern = + role === "caller" + ? new RegExp( + String.raw`\bfetch\s*\(\s*${literal}\s*(?:,\s*\{\s*method\s*:\s*["'](${methods.toUpperCase()})["']\s*\}\s*)?\)`, + "gu", + ) + : new RegExp(String.raw`#\[\s*(${methods})\s*\(\s*"(\/[A-Za-z0-9_./~-]*)"\s*\)\s*\]`, "gu"); + const endpoints: Endpoint[] = []; + let line = 1; + let lineCursor = 0; + for (const match of source.matchAll(pattern)) { + while (lineCursor < match.index) { + if (source[lineCursor++] === "\n") line += 1; + } + if ( + !code[match.index] || + (role === "caller" && + (/[\w$]/u.test(source[match.index - 1] ?? "") || + previousCodeChar(tokens, match.index) === ".")) + ) + continue; + const method = role === "caller" ? (match[3] ?? "GET") : match[1]!.toUpperCase(); + const path = match[2]!; + if (path.startsWith("//") || path.split("/").some((part) => part === "." || part === "..")) + continue; + endpoints.push({ method, path, file, line }); + } + return endpoints; +} + +function previousCodeChar(tokens: string, index: number): string | undefined { + for (let cursor = index - 1; cursor >= 0; cursor -= 1) { + const char = tokens[cursor]!; + if (!/\s/u.test(char)) return char; + } + return undefined; +} + +// Mask literals/comments before matching. Unsupported JS regex/template syntax is +// deliberately skipped rather than evaluating source or interpolations. +function codeMask(source: string, rust: boolean): Uint8Array { + const mask = new Uint8Array(source.length); + let i = 0; + while (i < source.length) { + if (source.startsWith("//", i)) { + const end = source.indexOf("\n", i); + i = end < 0 ? source.length : end; + continue; + } + if (source.startsWith("/*", i)) { + let depth = 1; + i += 2; + while (i < source.length && depth) { + if (rust && source.startsWith("/*", i)) { + depth += 1; + i += 2; + } else if (source.startsWith("*/", i)) { + depth -= 1; + i += 2; + } else i += 1; + } + continue; + } + const raw = rust ? /^r(#+)?"/u.exec(source.slice(i)) : null; + if (raw !== null) { + const end = source.indexOf(`"${raw[1] ?? ""}`, i + raw[0].length); + i = end < 0 ? source.length : end + 1 + (raw[1]?.length ?? 0); + continue; + } + const char = source[i]!; + // Rust lifetimes are identifiers, not unterminated character literals. + if ( + rust && + char === "'" && + /^'[A-Za-z_]\w*(?![\w'])/u.test(source.slice(i)) && + !/^'[^'\n]+'/u.test(source.slice(i)) + ) { + mask[i++] = 1; + continue; + } + if (!rust && char === "`") { + i = templateEnd(source, i); + continue; + } + if (char === '"' || char === "'") { + i += 1; + while (i < source.length) { + if (source[i] === "\\") i += 2; + else if (source[i++] === char) break; + } + continue; + } + if (!rust && char === "/") { + const before = source.slice(0, i).trimEnd(); + if ( + !before || + /[([{=,:;!&|?*~^]$/u.test(before) || + /(?:return|throw|yield|=>)$/u.test(before) + ) { + i += 1; + let characterClass = false; + while (i < source.length) { + const current = source[i++]; + if (current === "\\") i += 1; + else if (current === "[") characterClass = true; + else if (current === "]") characterClass = false; + else if (current === "/" && !characterClass) break; + } + continue; + } + } + mask[i++] = 1; + } + return mask; +} + +function templateEnd(source: string, start: number): number { + let depth = 0; + let i = start + 1; + while (i < source.length) { + const char = source[i]!; + if (char === "\\") { + i += 2; + continue; + } + if (depth === 0) { + if (char === "`") return i + 1; + if (source.startsWith("${", i)) { + depth = 1; + i += 2; + continue; + } + } else { + if (char === "`") { + i = templateEnd(source, i); + continue; + } + if (char === '"' || char === "'") { + i += 1; + while (i < source.length) { + if (source[i] === "\\") i += 2; + else if (source[i++] === char) break; + } + continue; + } + if (source.startsWith("//", i)) { + const end = source.indexOf("\n", i); + i = end < 0 ? source.length : end; + continue; + } + if (source.startsWith("/*", i)) { + const end = source.indexOf("*/", i + 2); + i = end < 0 ? source.length : end + 2; + continue; + } + // A slash in an interpolation could be division or a regex containing braces. + // Leave the rest unscanned rather than guessing where the template ends. + if (char === "/") return source.length; + if (char === "{") depth += 1; + if (char === "}") depth -= 1; + } + i += 1; + } + return source.length; +} + +function within(file: string, scope: string): boolean { + return file === scope || file.startsWith(`${scope}/`); +} +function inside(root: string, file: string): boolean { + const path = relative(root, file); + return !isAbsolute(path) && path !== ".." && !path.startsWith(`..${sep}`); +} + +function skipped(skippedReason: string): HttpRelations { + return { relations: [], omitted: 0, skippedReason }; +} diff --git a/src/review.ts b/src/review.ts index b2459d9..b05c3ec 100644 --- a/src/review.ts +++ b/src/review.ts @@ -9,6 +9,7 @@ import { findingFromOutput, mergeFinding } from "./findings.js"; import { nowIso } from "./fs.js"; import { discoverGit } from "./git.js"; import { runId } from "./id.js"; +import { findHttpRelations, withHttpContext, type HttpRelation } from "./http-relations.js"; import { emitProgress } from "./progress.js"; import { providerByName } from "./provider.js"; import type { DroppedFinding } from "./provider-types.js"; @@ -50,6 +51,14 @@ export async function reviewCommand( const mode = reviewMode(flags); const customPrompt = await loadCustomReviewPrompt(flags); const features = await selectReviewFeatures(loaded, flags); + const linkHttp = stringFlag(flags, "linkHttp"); + const http = + linkHttp === undefined + ? undefined + : await findHttpRelations(loaded.root, await readFeatures(loaded.paths), linkHttp, { + include: config.include, + exclude: config.exclude, + }); if (features.length === 0 && hasFileFilter(flags)) { if (flags["dryRun"] === true) { return { next: "no features touched by diff" }; @@ -69,6 +78,7 @@ export async function reviewCommand( if (flags["dryRun"] === true) { return { dryRun: true, + ...(http === undefined ? {} : { http }), wouldReview: features.length, mode, jobs: reviewJobs(flags), @@ -122,6 +132,7 @@ export async function reviewCommand( customPrompt, limiter, registryPostValidator, + httpRelations: http?.relations ?? [], allowNonPendingFeatureReview: stringFlag(flags, "feature") !== undefined || stringFlag(flags, "featureList") !== undefined || @@ -200,6 +211,7 @@ export async function reviewCommand( config.provider.name, ); return { + ...(http === undefined ? {} : { http }), run: currentRunId, reviewed: features.length, findings: findingIds.length, @@ -296,6 +308,7 @@ type ReviewFeatureOptions = { limiter: RpmLimiter; registryPostValidator: FindingPostValidator | undefined; allowNonPendingFeatureReview: boolean; + httpRelations: HttpRelation[]; }; async function reviewFeature( @@ -337,7 +350,7 @@ async function reviewFeature( const reviewPrompt = await buildReviewPromptBundle( loaded.root, loaded.project, - lockedFeature, + withHttpContext(lockedFeature, options.httpRelations), config, mode, customPrompt, From 50475edabf5079d81499be8d31d7f989e8e7b2cc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 7 Sep 2026 04:24:18 -0700 Subject: [PATCH 2/6] fix: recognize generic mounts and literal HTTP punctuation --- docs/feature-mapping.md | 5 +++-- src/http-relations.test.ts | 21 +++++++++++++++++++++ src/http-relations.ts | 14 ++++++++++---- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/docs/feature-mapping.md b/docs/feature-mapping.md index 9fb2391..e225bf2 100644 --- a/docs/feature-mapping.md +++ b/docs/feature-mapping.md @@ -211,12 +211,13 @@ not discover deployment origins, proxy rules, or service topology. The output contains **candidate** HTTP relations, not proof of runtime connectivity. Verify routing before relying on a relation in a finding. -The first version matches literal `fetch("/path")` (GET) and +The first version matches unescaped literal `fetch("/path")` (GET) and `fetch("/path", { method: "POST" })` calls to Rust `#[get("/path")]`, `#[post("/path")]`, `put`, `patch`, `delete`, `head`, or `options` attributes. Caller scanning supports `.js`, `.ts`, `.mjs`, `.cjs`, `.mts`, and `.cts`; JSX/TSX files are skipped. Methods and paths must match exactly. Additional fetch options, computed values, -template literals, query strings, absolute URLs, parameters, wildcard paths, +template literals, escaped literals, whitespace in paths, query strings, fragments, +absolute URLs, parameters, wildcard paths, Axios, and other handler syntaxes are unsupported. Comments and string contents are skipped. Complex template interpolations containing division or regular expressions conservatively end scanning of that caller file. Actix-shaped `web::scope(...)` and any Rust `.mount(...)` call disable the pass diff --git a/src/http-relations.test.ts b/src/http-relations.test.ts index b5068e9..a21a1f9 100644 --- a/src/http-relations.test.ts +++ b/src/http-relations.test.ts @@ -150,6 +150,12 @@ describe("HTTP candidate context", () => { 'fn configure() { web/* prefix */::scope("/v2"); }\n', ); expect((await setup.scan()).skippedReason).toContain("prefixes"); + await writeFixture( + setup.root, + "backend/src/config.rs", + 'fn configure(server: Rocket) { server.mount::<_, _>("/v2", routes![login]); }\n', + ); + expect((await setup.scan()).skippedReason).toContain("prefixes"); }); it("honors filters and refuses partial scans, missing sources, and symlinked roots", async () => { @@ -275,6 +281,21 @@ describe("literal HTTP syntax", () => { expect(httpEndpoints('
fetch("/fake")
', "client.tsx", "caller")).toEqual([]); }); + it("matches unescaped static path punctuation and Unicode", () => { + for (const path of [ + "/users/@me", + "/search/a+b", + "/encoded/%2F", + "/Über", + "/authors/O'Reilly", + "/time/12:00", + ]) { + const literal = JSON.stringify(path); + expect(httpEndpoints(`fetch(${literal})`, "client.ts", "caller")[0]?.path).toBe(path); + expect(httpEndpoints(`#[get(${literal})]`, "main.rs", "handler")[0]?.path).toBe(path); + } + }); + it("tracks line numbers across a dense supported source file", () => { const endpoints = httpEndpoints('fetch("/");\n'.repeat(20_000), "client.ts", "caller"); expect(endpoints).toHaveLength(20_000); diff --git a/src/http-relations.ts b/src/http-relations.ts index 882a3de..b075028 100644 --- a/src/http-relations.ts +++ b/src/http-relations.ts @@ -157,7 +157,9 @@ export async function findHttpRelations( function hasRouteMount(source: string): boolean { const tokens = codeSource(source, codeMask(source, true)); - return /\bweb\s*::\s*scope\s*\(/u.test(tokens) || /\.\s*mount\s*\(/u.test(tokens); + return ( + /\bweb\s*::\s*scope\s*(?:\(|::\s*<)/u.test(tokens) || /\.\s*mount\s*(?:\(|::\s*<)/u.test(tokens) + ); } function codeSource(source: string, mask: Uint8Array): string { @@ -198,14 +200,14 @@ export function httpEndpoints( if (role === "caller" && /\.[jt]sx$/u.test(file)) return []; const code = codeMask(source, role === "handler"); const tokens = codeSource(source, code); - const literal = String.raw`(["'])(\/[A-Za-z0-9_./~-]*)\1`; + const literal = String.raw`(["'])(\/(?:(?!\1)[^\\\r\n])*)\1`; const pattern = role === "caller" ? new RegExp( String.raw`\bfetch\s*\(\s*${literal}\s*(?:,\s*\{\s*method\s*:\s*["'](${methods.toUpperCase()})["']\s*\}\s*)?\)`, "gu", ) - : new RegExp(String.raw`#\[\s*(${methods})\s*\(\s*"(\/[A-Za-z0-9_./~-]*)"\s*\)\s*\]`, "gu"); + : new RegExp(String.raw`#\[\s*(${methods})\s*\(\s*"(\/[^"\\\r\n]*)"\s*\)\s*\]`, "gu"); const endpoints: Endpoint[] = []; let line = 1; let lineCursor = 0; @@ -222,7 +224,11 @@ export function httpEndpoints( continue; const method = role === "caller" ? (match[3] ?? "GET") : match[1]!.toUpperCase(); const path = match[2]!; - if (path.startsWith("//") || path.split("/").some((part) => part === "." || part === "..")) + if ( + path.startsWith("//") || + /[?#*{}<>\s]/u.test(path) || + path.split("/").some((part) => part === "." || part === ".." || part.startsWith(":")) + ) continue; endpoints.push({ method, path, file, line }); } From 400cf8f38c9d27e25c98e84685f3774ab2917ab6 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 7 Sep 2026 04:33:07 -0700 Subject: [PATCH 3/6] perf: bound duplicate HTTP handler indexing --- src/http-relations.test.ts | 21 +++++++++++++++++++++ src/http-relations.ts | 10 +++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/http-relations.test.ts b/src/http-relations.test.ts index a21a1f9..615e3cf 100644 --- a/src/http-relations.test.ts +++ b/src/http-relations.test.ts @@ -296,6 +296,27 @@ describe("literal HTTP syntax", () => { } }); + it("keeps unique routes usable in a dense duplicate-handler inventory", async () => { + const setup = await fixture(); + const handlers = Array.from( + { length: 4_000 }, + (_, i) => `#[get("/api/login")]\nfn route_${i}() {}\n`, + ).join(""); + await writeFixture( + setup.root, + "backend/src/main.rs", + handlers + '#[get("/health")]\nfn health() {}\nfn main() {}\n', + ); + await writeFixture( + setup.root, + "frontend/src/app.ts", + 'fetch("/api/login");\nfetch("/health");\n', + ); + const result = await setup.scan(); + expect(result.skippedReason).toBeNull(); + expect(result.relations.map((relation) => relation.path)).toEqual(["/health"]); + }); + it("tracks line numbers across a dense supported source file", () => { const endpoints = httpEndpoints('fetch("/");\n'.repeat(20_000), "client.ts", "caller"); expect(endpoints).toHaveLength(20_000); diff --git a/src/http-relations.ts b/src/http-relations.ts index b075028..c8c45f2 100644 --- a/src/http-relations.ts +++ b/src/http-relations.ts @@ -123,18 +123,18 @@ export async function findHttpRelations( } } const ownerIds = new Map([...ownersByFile].map(([file, ids]) => [file, [...ids].toSorted()])); - const byRoute = new Map(); + // Once a route is ambiguous, retaining more endpoints only wastes work. + const byRoute = new Map(); for (const handler of handlers) { const key = `${handler.method} ${handler.path}`; - byRoute.set(key, [...(byRoute.get(key) ?? []), handler]); + byRoute.set(key, byRoute.has(key) ? null : handler); } const relations: HttpRelation[] = []; const seen = new Set(); let omitted = 0; for (const caller of callers) { - const matches = byRoute.get(`${caller.method} ${caller.path}`) ?? []; - if (matches.length !== 1) continue; - const handler = matches[0]!; + const handler = byRoute.get(`${caller.method} ${caller.path}`); + if (handler === undefined || handler === null) continue; const callerIds = ownerIds.get(caller.file) ?? []; const handlerIds = ownerIds.get(handler.file) ?? []; if (!callerIds.length || !handlerIds.length) continue; From c4bf93e85f517e8b3620e83006dc98f6dc1b3702 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 7 Sep 2026 04:39:55 -0700 Subject: [PATCH 4/6] fix: scan nested HTTP templates without recursion --- src/http-relations.test.ts | 8 ++++++++ src/http-relations.ts | 18 ++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/http-relations.test.ts b/src/http-relations.test.ts index 615e3cf..564e62e 100644 --- a/src/http-relations.test.ts +++ b/src/http-relations.test.ts @@ -281,6 +281,14 @@ describe("literal HTTP syntax", () => { expect(httpEndpoints('
fetch("/fake")
', "client.tsx", "caller")).toEqual([]); }); + it("skips deeply nested templates without consuming the call stack", () => { + const source = + "const nested = " + "`x${".repeat(10_000) + "0" + "}`".repeat(10_000) + '; fetch("/real");'; + expect(httpEndpoints(source, "client.ts", "caller").map((endpoint) => endpoint.path)).toEqual([ + "/real", + ]); + }); + it("matches unescaped static path punctuation and Unicode", () => { for (const path of [ "/users/@me", diff --git a/src/http-relations.ts b/src/http-relations.ts index c8c45f2..0aa9902 100644 --- a/src/http-relations.ts +++ b/src/http-relations.ts @@ -322,24 +322,30 @@ function codeMask(source: string, rust: boolean): Uint8Array { } function templateEnd(source: string, start: number): number { - let depth = 0; + const depths = [0]; let i = start + 1; while (i < source.length) { + const frame = depths.length - 1; + const depth = depths[frame]!; const char = source[i]!; if (char === "\\") { i += 2; continue; } if (depth === 0) { - if (char === "`") return i + 1; + if (char === "`") { + depths.pop(); + if (depths.length === 0) return i + 1; + } if (source.startsWith("${", i)) { - depth = 1; + depths[frame] = 1; i += 2; continue; } } else { if (char === "`") { - i = templateEnd(source, i); + depths.push(0); + i += 1; continue; } if (char === '"' || char === "'") { @@ -363,8 +369,8 @@ function templateEnd(source: string, start: number): number { // A slash in an interpolation could be division or a regex containing braces. // Leave the rest unscanned rather than guessing where the template ends. if (char === "/") return source.length; - if (char === "{") depth += 1; - if (char === "}") depth -= 1; + if (char === "{") depths[frame] = depth + 1; + if (char === "}") depths[frame] = depth - 1; } i += 1; } From b6e0701257f344ba64f308806f2cda538d527389 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 7 Sep 2026 04:58:06 -0700 Subject: [PATCH 5/6] refactor: tokenize HTTP callers with js-tokens --- docs/feature-mapping.md | 4 +- package.json | 1 + pnpm-lock.yaml | 8 +++ scripts/package-smoke.mjs | 19 +++++- src/http-relations.test.ts | 16 +++++ src/http-relations.ts | 126 ++++++++++++------------------------- src/package-smoke.test.ts | 9 ++- 7 files changed, 90 insertions(+), 93 deletions(-) diff --git a/docs/feature-mapping.md b/docs/feature-mapping.md index e225bf2..2e15117 100644 --- a/docs/feature-mapping.md +++ b/docs/feature-mapping.md @@ -219,8 +219,8 @@ files are skipped. Methods and paths must match exactly. Additional fetch option template literals, escaped literals, whitespace in paths, query strings, fragments, absolute URLs, parameters, wildcard paths, Axios, and other handler syntaxes are unsupported. Comments and string contents -are skipped. Complex template interpolations containing division or regular -expressions conservatively end scanning of that caller file. Actix-shaped `web::scope(...)` and any Rust `.mount(...)` call disable the pass +are skipped. JavaScript tokenization follows [js-tokens](https://github.com/lydell/js-tokens) +lexical coverage; files that exceed tokenizer limits are skipped. Actix-shaped `web::scope(...)` and any Rust `.mount(...)` call disable the pass because their prefixes are unresolved, including mounts in helpers whose server was constructed elsewhere. External prefixes and macro-generated routes remain outside this heuristic; the supplied roots must use the same path diff --git a/package.json b/package.json index 11ea594..424a564 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "crabbox:warmup": "crabbox warmup" }, "dependencies": { + "js-tokens": "^10.0.0", "proper-lockfile": "^4.1.2", "zod": "^4.5.4" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 563e0be..5661659 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,9 @@ importers: .: dependencies: + js-tokens: + specifier: ^10.0.0 + version: 10.0.0 proper-lockfile: specifier: ^4.1.2 version: 4.1.2 @@ -587,6 +590,9 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + js-tokens@10.0.0: + resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==} + lightningcss-android-arm64@1.33.0: resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} engines: {node: '>= 12.0.0'} @@ -1141,6 +1147,8 @@ snapshots: graceful-fs@4.2.11: {} + js-tokens@10.0.0: {} + lightningcss-android-arm64@1.33.0: optional: true diff --git a/scripts/package-smoke.mjs b/scripts/package-smoke.mjs index e24e3b4..a23390b 100644 --- a/scripts/package-smoke.mjs +++ b/scripts/package-smoke.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node import { execFileSync } from "node:child_process"; -import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { dirname, isAbsolute, join } from "node:path"; @@ -255,7 +255,7 @@ function runtimeDependencyPaths(rootPath = root) { function collect(packageJsonPath, packageRequire) { const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")); for (const name of runtimeDependencyNames(packageJson)) { - const dependencyPackageJson = packageRequire.resolve(`${name}/package.json`); + const dependencyPackageJson = runtimeDependencyManifest(packageRequire, name); const dependencyPath = dirname(dependencyPackageJson); if (dependencyPaths.has(dependencyPath)) { continue; @@ -270,6 +270,19 @@ function runtimeDependencyPaths(rootPath = root) { return [...dependencyPaths.values()]; } +function runtimeDependencyManifest(packageRequire, name) { + let directory = dirname(packageRequire.resolve(name)); + for (;;) { + const candidate = join(directory, "package.json"); + if (existsSync(candidate) && JSON.parse(readFileSync(candidate, "utf8")).name === name) { + return candidate; + } + const parent = dirname(directory); + if (parent === directory) throw new Error(`package metadata not found for ${name}`); + directory = parent; + } +} + function runtimeDependencyNames(packageJson) { return Object.keys(packageJson.dependencies ?? {}); } @@ -329,7 +342,7 @@ function verifyRuntimeDependencies(context) { for (const name of runtimeDependencyNames(packageJson)) { run(context, "node", [ "-e", - "require.resolve(`${process.argv[1]}/package.json`, { paths: [process.argv[2]] })", + "require.resolve(process.argv[1], { paths: [process.argv[2]] })", name, packageRoot, ]); diff --git a/src/http-relations.test.ts b/src/http-relations.test.ts index 564e62e..3931282 100644 --- a/src/http-relations.test.ts +++ b/src/http-relations.test.ts @@ -289,6 +289,22 @@ describe("literal HTTP syntax", () => { ]); }); + it("keeps calls after regex literals in control-flow bodies and TypeScript", () => { + for (const prefix of [ + "if (enabled)", + "while (enabled)", + "for (;enabled;)", + "if (enabled && check())", + ]) { + const source = prefix + ` /["']/.test(value); fetch("/api/login");`; + expect(httpEndpoints(source, "client.ts", "caller").map((r) => r.path)).toEqual([ + "/api/login", + ]); + } + const source = `@decorator class Client { run(value: string): void { if (enabled) /["']/.test(value); fetch("/api/login"); } }`; + expect(httpEndpoints(source, "client.ts", "caller").map((r) => r.path)).toEqual(["/api/login"]); + }); + it("matches unescaped static path punctuation and Unicode", () => { for (const path of [ "/users/@me", diff --git a/src/http-relations.ts b/src/http-relations.ts index 0aa9902..8b6acef 100644 --- a/src/http-relations.ts +++ b/src/http-relations.ts @@ -1,4 +1,5 @@ import { open, realpath, stat } from "node:fs/promises"; +import jsTokens from "js-tokens"; import { isAbsolute, relative, resolve, sep } from "node:path"; import { ClawpatchError } from "./errors.js"; import { pathMatchesFilters, walk, type PathFilters } from "./mappers/shared.js"; @@ -156,7 +157,7 @@ export async function findHttpRelations( } function hasRouteMount(source: string): boolean { - const tokens = codeSource(source, codeMask(source, true)); + const tokens = codeSource(source, rustCodeMask(source)); return ( /\bweb\s*::\s*scope\s*(?:\(|::\s*<)/u.test(tokens) || /\.\s*mount\s*(?:\(|::\s*<)/u.test(tokens) ); @@ -198,7 +199,7 @@ export function httpEndpoints( role: "caller" | "handler", ): Endpoint[] { if (role === "caller" && /\.[jt]sx$/u.test(file)) return []; - const code = codeMask(source, role === "handler"); + const code = role === "handler" ? rustCodeMask(source) : javascriptCodeMask(source); const tokens = codeSource(source, code); const literal = String.raw`(["'])(\/(?:(?!\1)[^\\\r\n])*)\1`; const pattern = @@ -243,9 +244,40 @@ function previousCodeChar(tokens: string, index: number): string | undefined { return undefined; } -// Mask literals/comments before matching. Unsupported JS regex/template syntax is -// deliberately skipped rather than evaluating source or interpolations. -function codeMask(source: string, rust: boolean): Uint8Array { +function javascriptCodeMask(source: string): Uint8Array { + const mask = new Uint8Array(source.length); + let offset = 0; + let templateDepth = 0; + try { + for (const token of jsTokens(source)) { + const end = offset + token.value.length; + if (token.type === "TemplateHead") templateDepth += 1; + if ( + templateDepth === 0 && + [ + "IdentifierName", + "PrivateIdentifier", + "NumericLiteral", + "Punctuator", + "WhiteSpace", + "LineTerminatorSequence", + "Invalid", + ].includes(token.type) + ) { + mask.fill(1, offset, end); + } + if (token.type === "TemplateTail") templateDepth -= 1; + offset = end; + } + } catch (error) { + // A failed tokenization must never expose part of a string as caller code. + if (error instanceof RangeError) return new Uint8Array(source.length); + throw error; + } + return mask; +} + +function rustCodeMask(source: string): Uint8Array { const mask = new Uint8Array(source.length); let i = 0; while (i < source.length) { @@ -258,7 +290,7 @@ function codeMask(source: string, rust: boolean): Uint8Array { let depth = 1; i += 2; while (i < source.length && depth) { - if (rust && source.startsWith("/*", i)) { + if (source.startsWith("/*", i)) { depth += 1; i += 2; } else if (source.startsWith("*/", i)) { @@ -268,7 +300,7 @@ function codeMask(source: string, rust: boolean): Uint8Array { } continue; } - const raw = rust ? /^r(#+)?"/u.exec(source.slice(i)) : null; + const raw = /^r(#+)?"/u.exec(source.slice(i)); if (raw !== null) { const end = source.indexOf(`"${raw[1] ?? ""}`, i + raw[0].length); i = end < 0 ? source.length : end + 1 + (raw[1]?.length ?? 0); @@ -277,7 +309,6 @@ function codeMask(source: string, rust: boolean): Uint8Array { const char = source[i]!; // Rust lifetimes are identifiers, not unterminated character literals. if ( - rust && char === "'" && /^'[A-Za-z_]\w*(?![\w'])/u.test(source.slice(i)) && !/^'[^'\n]+'/u.test(source.slice(i)) @@ -285,10 +316,6 @@ function codeMask(source: string, rust: boolean): Uint8Array { mask[i++] = 1; continue; } - if (!rust && char === "`") { - i = templateEnd(source, i); - continue; - } if (char === '"' || char === "'") { i += 1; while (i < source.length) { @@ -297,86 +324,11 @@ function codeMask(source: string, rust: boolean): Uint8Array { } continue; } - if (!rust && char === "/") { - const before = source.slice(0, i).trimEnd(); - if ( - !before || - /[([{=,:;!&|?*~^]$/u.test(before) || - /(?:return|throw|yield|=>)$/u.test(before) - ) { - i += 1; - let characterClass = false; - while (i < source.length) { - const current = source[i++]; - if (current === "\\") i += 1; - else if (current === "[") characterClass = true; - else if (current === "]") characterClass = false; - else if (current === "/" && !characterClass) break; - } - continue; - } - } mask[i++] = 1; } return mask; } -function templateEnd(source: string, start: number): number { - const depths = [0]; - let i = start + 1; - while (i < source.length) { - const frame = depths.length - 1; - const depth = depths[frame]!; - const char = source[i]!; - if (char === "\\") { - i += 2; - continue; - } - if (depth === 0) { - if (char === "`") { - depths.pop(); - if (depths.length === 0) return i + 1; - } - if (source.startsWith("${", i)) { - depths[frame] = 1; - i += 2; - continue; - } - } else { - if (char === "`") { - depths.push(0); - i += 1; - continue; - } - if (char === '"' || char === "'") { - i += 1; - while (i < source.length) { - if (source[i] === "\\") i += 2; - else if (source[i++] === char) break; - } - continue; - } - if (source.startsWith("//", i)) { - const end = source.indexOf("\n", i); - i = end < 0 ? source.length : end; - continue; - } - if (source.startsWith("/*", i)) { - const end = source.indexOf("*/", i + 2); - i = end < 0 ? source.length : end + 2; - continue; - } - // A slash in an interpolation could be division or a regex containing braces. - // Leave the rest unscanned rather than guessing where the template ends. - if (char === "/") return source.length; - if (char === "{") depths[frame] = depth + 1; - if (char === "}") depths[frame] = depth - 1; - } - i += 1; - } - return source.length; -} - function within(file: string, scope: string): boolean { return file === scope || file.startsWith(`${scope}/`); } diff --git a/src/package-smoke.test.ts b/src/package-smoke.test.ts index dcb6518..81ccc82 100644 --- a/src/package-smoke.test.ts +++ b/src/package-smoke.test.ts @@ -46,7 +46,14 @@ describe("package smoke harness", () => { return packageJson.name; }); expect(packedDependencyNames).toEqual( - expect.arrayContaining(["proper-lockfile", "graceful-fs", "retry", "signal-exit", "zod"]), + expect.arrayContaining([ + "js-tokens", + "proper-lockfile", + "graceful-fs", + "retry", + "signal-exit", + "zod", + ]), ); const packArgs = smoke.packDependencyArgs({ From 8835259ef6b224514f1c9c4fd1d7c0b4136e86a3 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 7 Sep 2026 05:12:14 -0700 Subject: [PATCH 6/6] fix: match HTTP callers at token boundaries --- docs/feature-mapping.md | 2 +- src/http-relations.test.ts | 23 ++++++ src/http-relations.ts | 144 ++++++++++++++++++++++--------------- 3 files changed, 112 insertions(+), 57 deletions(-) diff --git a/docs/feature-mapping.md b/docs/feature-mapping.md index 2e15117..2cf6cae 100644 --- a/docs/feature-mapping.md +++ b/docs/feature-mapping.md @@ -215,7 +215,7 @@ The first version matches unescaped literal `fetch("/path")` (GET) and `fetch("/path", { method: "POST" })` calls to Rust `#[get("/path")]`, `#[post("/path")]`, `put`, `patch`, `delete`, `head`, or `options` attributes. Caller scanning supports `.js`, `.ts`, `.mjs`, `.cjs`, `.mts`, and `.cts`; JSX/TSX -files are skipped. Methods and paths must match exactly. Additional fetch options, computed values, +files are skipped. Matching uses the standard HTTP method and exact literal path. Additional fetch options, computed values, template literals, escaped literals, whitespace in paths, query strings, fragments, absolute URLs, parameters, wildcard paths, Axios, and other handler syntaxes are unsupported. Comments and string contents diff --git a/src/http-relations.test.ts b/src/http-relations.test.ts index 3931282..884ac80 100644 --- a/src/http-relations.test.ts +++ b/src/http-relations.test.ts @@ -246,6 +246,12 @@ describe("literal HTTP syntax", () => { ["GET", "/get"], ["POST", "/post"], ]); + expect(httpEndpoints('fetch("/patch", { method: "patch" });', "client.ts", "caller")).toEqual( + [], + ); + expect( + httpEndpoints('fetch("/patch", { method: "PATCH" });', "client.ts", "caller")[0]?.method, + ).toBe("PATCH"); }); it("ignores comments, regexes, templates, strings, Rust raw strings and nested comments", () => { @@ -305,6 +311,23 @@ describe("literal HTTP syntax", () => { expect(httpEndpoints(source, "client.ts", "caller").map((r) => r.path)).toEqual(["/api/login"]); }); + it("matches identifier boundaries without accepting private methods or Unicode prefixes", () => { + const source = + 'class Client { #fetch() {} run() { this.#fetch("/private"); } } πfetch("/unicode"); const item = object.member\nfetch("/real");'; + expect(httpEndpoints(source, "client.ts", "caller").map((r) => r.path)).toEqual(["/real"]); + }); + + it("recognizes literal calls across comments and trailing commas", () => { + const source = + 'fetch /* route */ ("/get",);\nfetch("/post", { "method": /* verb */ "post", },);'; + expect( + httpEndpoints(source, "client.ts", "caller").map(({ method, path }) => [method, path]), + ).toEqual([ + ["GET", "/get"], + ["POST", "/post"], + ]); + }); + it("matches unescaped static path punctuation and Unicode", () => { for (const path of [ "/users/@me", diff --git a/src/http-relations.ts b/src/http-relations.ts index 8b6acef..3a1fe62 100644 --- a/src/http-relations.ts +++ b/src/http-relations.ts @@ -1,5 +1,5 @@ import { open, realpath, stat } from "node:fs/promises"; -import jsTokens from "js-tokens"; +import jsTokens, { type Token } from "js-tokens"; import { isAbsolute, relative, resolve, sep } from "node:path"; import { ClawpatchError } from "./errors.js"; import { pathMatchesFilters, walk, type PathFilters } from "./mappers/shared.js"; @@ -22,6 +22,8 @@ const maxFiles = 500; const totalLimit = 8_000_000; const counterpartLimit = 3; const methods = "get|post|put|patch|delete|head|options"; +const methodNames = new Set(methods.toUpperCase().split("|")); +const fetchNormalizedMethods = new Set(["DELETE", "GET", "HEAD", "OPTIONS", "POST", "PUT"]); export function httpRoots(value: string): [string, string] { const parts = value.split(":"); @@ -198,17 +200,12 @@ export function httpEndpoints( file: string, role: "caller" | "handler", ): Endpoint[] { - if (role === "caller" && /\.[jt]sx$/u.test(file)) return []; - const code = role === "handler" ? rustCodeMask(source) : javascriptCodeMask(source); - const tokens = codeSource(source, code); - const literal = String.raw`(["'])(\/(?:(?!\1)[^\\\r\n])*)\1`; - const pattern = - role === "caller" - ? new RegExp( - String.raw`\bfetch\s*\(\s*${literal}\s*(?:,\s*\{\s*method\s*:\s*["'](${methods.toUpperCase()})["']\s*\}\s*)?\)`, - "gu", - ) - : new RegExp(String.raw`#\[\s*(${methods})\s*\(\s*"(\/[^"\\\r\n]*)"\s*\)\s*\]`, "gu"); + if (role === "caller") return /\.[jt]sx$/u.test(file) ? [] : javascriptEndpoints(source, file); + const code = rustCodeMask(source); + const pattern = new RegExp( + String.raw`#\[\s*(${methods})\s*\(\s*"(\/[^"\\\r\n]*)"\s*\)\s*\]`, + "gu", + ); const endpoints: Endpoint[] = []; let line = 1; let lineCursor = 0; @@ -216,65 +213,100 @@ export function httpEndpoints( while (lineCursor < match.index) { if (source[lineCursor++] === "\n") line += 1; } - if ( - !code[match.index] || - (role === "caller" && - (/[\w$]/u.test(source[match.index - 1] ?? "") || - previousCodeChar(tokens, match.index) === ".")) - ) - continue; - const method = role === "caller" ? (match[3] ?? "GET") : match[1]!.toUpperCase(); - const path = match[2]!; - if ( - path.startsWith("//") || - /[?#*{}<>\s]/u.test(path) || - path.split("/").some((part) => part === "." || part === ".." || part.startsWith(":")) - ) - continue; - endpoints.push({ method, path, file, line }); + if (code[match.index] && isHttpPath(match[2]!)) + endpoints.push({ method: match[1]!.toUpperCase(), path: match[2]!, file, line }); } return endpoints; } -function previousCodeChar(tokens: string, index: number): string | undefined { - for (let cursor = index - 1; cursor >= 0; cursor -= 1) { - const char = tokens[cursor]!; - if (!/\s/u.test(char)) return char; - } - return undefined; -} +type LocatedToken = { token: Token; line: number; inTemplate: boolean }; -function javascriptCodeMask(source: string): Uint8Array { - const mask = new Uint8Array(source.length); - let offset = 0; +function javascriptEndpoints(source: string, file: string): Endpoint[] { + const tokens: LocatedToken[] = []; + let line = 1; let templateDepth = 0; try { for (const token of jsTokens(source)) { - const end = offset + token.value.length; if (token.type === "TemplateHead") templateDepth += 1; if ( - templateDepth === 0 && - [ - "IdentifierName", - "PrivateIdentifier", - "NumericLiteral", - "Punctuator", - "WhiteSpace", - "LineTerminatorSequence", - "Invalid", - ].includes(token.type) - ) { - mask.fill(1, offset, end); - } + token.type !== "WhiteSpace" && + token.type !== "LineTerminatorSequence" && + !token.type.endsWith("Comment") + ) + tokens.push({ token, line, inTemplate: templateDepth > 0 }); if (token.type === "TemplateTail") templateDepth -= 1; - offset = end; + for (const char of token.value) if (char === "\n") line += 1; } } catch (error) { - // A failed tokenization must never expose part of a string as caller code. - if (error instanceof RangeError) return new Uint8Array(source.length); + // Do not emit partial relations if the tokenizer exceeds its own limits. + if (error instanceof RangeError) return []; throw error; } - return mask; + const endpoints: Endpoint[] = []; + for (let index = 0; index < tokens.length; index += 1) { + const current = tokens[index]!; + if ( + current.inTemplate || + current.token.type !== "IdentifierName" || + current.token.value !== "fetch" + ) + continue; + const previous = tokens[index - 1]?.token.value; + if (previous === "." || previous === "?.") continue; + const call = literalFetchCall(tokens, index); + if (call !== null) endpoints.push({ ...call, file, line: current.line }); + } + return endpoints; +} + +function literalFetchCall( + tokens: LocatedToken[], + start: number, +): { method: string; path: string } | null { + let index = start + 1; + if (tokens[index++]?.token.value !== "(") return null; + const path = unescapedString(tokens[index++]?.token); + if (path === null || !isHttpPath(path)) return null; + let method = "GET"; + if (tokens[index]?.token.value === ",") { + index += 1; + if (tokens[index]?.token.value !== ")") { + if (tokens[index++]?.token.value !== "{") return null; + const key = tokens[index++]?.token; + if ( + !(key?.type === "IdentifierName" && key.value === "method") && + unescapedString(key) !== "method" + ) + return null; + if (tokens[index++]?.token.value !== ":") return null; + const raw = unescapedString(tokens[index++]?.token); + if (raw === null) return null; + const upper = raw.toUpperCase(); + // Fetch normalizes these six verbs; PATCH remains case-sensitive. + const value = fetchNormalizedMethods.has(upper) ? upper : raw; + if (!methodNames.has(value)) return null; + method = value; + if (tokens[index]?.token.value === ",") index += 1; + if (tokens[index++]?.token.value !== "}") return null; + if (tokens[index]?.token.value === ",") index += 1; + } + } + return tokens[index]?.token.value === ")" ? { method, path } : null; +} + +function unescapedString(token: Token | undefined): string | null { + return token?.type === "StringLiteral" && token.closed && !token.value.includes("\\") + ? token.value.slice(1, -1) + : null; +} + +function isHttpPath(path: string): boolean { + return ( + path.startsWith("/") && + !path.startsWith("//") && + !/[?#*{}<>\s]/u.test(path) && + !path.split("/").some((part) => part === "." || part === ".." || part.startsWith(":")) + ); } function rustCodeMask(source: string): Uint8Array {