diff --git a/docs/code-review.md b/docs/code-review.md index 1c6fac9..3d33f73 100644 --- a/docs/code-review.md +++ b/docs/code-review.md @@ -187,3 +187,9 @@ coordinates to the public npm registry. Enable it explicitly with deduplicates registry calls per `(name, version)`. [slop-paper]: https://arxiv.org/abs/2406.10279 + +For a Node frontend and Rust backend sharing one HTTP path namespace, +`review --link-http frontend:backend` adds bounded, literal HTTP candidate context +only for that review. Existing feature records and default review behavior are +unchanged. See [Optional HTTP relations](feature-mapping.md#optional-http-relations) +for the root-pair assertion, supported syntax, ambiguity rules, and limits. diff --git a/docs/feature-mapping.md b/docs/feature-mapping.md index dab0364..2cf6cae 100644 --- a/docs/feature-mapping.md +++ b/docs/feature-mapping.md @@ -195,3 +195,54 @@ Known gaps: or runtime route conventions - no import graph expansion beyond nearby tests yet - agent mapping depends on provider quality and validates paths but not semantic intent + +## Optional HTTP relations + +For a Node/TypeScript frontend and Rust backend that share one HTTP path +namespace, opt in with explicit, non-overlapping repository-relative roots: + +```sh +clawpatch map --link-http frontend:backend --json +clawpatch review --link-http frontend:backend --limit 3 +``` + +The root pair asserts which client and service belong together. Clawpatch does +not discover deployment origins, proxy rules, or service topology. The output +contains **candidate** HTTP relations, not proof of runtime connectivity. +Verify routing before relying on a relation in a finding. + +The first version matches unescaped literal `fetch("/path")` (GET) and +`fetch("/path", { method: "POST" })` calls to Rust `#[get("/path")]`, +`#[post("/path")]`, `put`, `patch`, `delete`, `head`, or `options` attributes. +Caller scanning supports `.js`, `.ts`, `.mjs`, `.cjs`, `.mts`, and `.cts`; JSX/TSX +files are skipped. Matching uses the standard HTTP method and exact literal path. Additional fetch options, computed values, +template literals, escaped literals, whitespace in paths, query strings, fragments, +absolute URLs, parameters, wildcard paths, +Axios, and other handler syntaxes are unsupported. Comments and string contents +are skipped. JavaScript tokenization follows [js-tokens](https://github.com/lydell/js-tokens) +lexical coverage; files that exceed tokenizer limits are skipped. Actix-shaped `web::scope(...)` and any Rust `.mount(...)` call disable the pass +because their prefixes are unresolved, including mounts in helpers whose server +was constructed elsewhere. External prefixes and macro-generated +routes remain outside this heuristic; the supplied roots must use the same path +namespace. Multiple recognized handlers for the same method/path are ambiguous +and produce no link, even when declared in one file. + +Mapping returns an `http` object containing `relations`, `omitted`, and +`skippedReason`; `--dry-run` returns it too. Each relation identifies the HTTP +method/path, caller and handler files/lines, and the features owning those files. +Default mapping output and stored feature slices remain unchanged. Reviews with +this flag recompute relations from current source, then append up to three +counterpart files to an ephemeral prompt copy after existing context. Existing +context-file and per-file prompt limits still apply, including omission reporting. +Review without the flag never adds HTTP context. Mapping alone does not enable +it for later reviews, fixes, revalidation, or `ci` runs. + +The scan honors configured include/exclude filters and normal mapper directory +exclusions, skips symlinks, and links only files owned by active features. It +scans at most 500 source files, 256,000 bytes per file, and 8,000,000 bytes total; +exceeding a scan budget returns no relations with a reason rather than matching +against an incomplete inventory. Output is sorted by source path and declaration +order and limited to 200 relations; `omitted` counts links dropped by that output +limit. The three-file review context limit is applied independently for each +feature, so a broad co-owner cannot suppress context for a narrower feature. There is no graph storage +or feature schema migration. diff --git a/package.json b/package.json index 11ea594..424a564 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "crabbox:warmup": "crabbox warmup" }, "dependencies": { + "js-tokens": "^10.0.0", "proper-lockfile": "^4.1.2", "zod": "^4.5.4" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 563e0be..5661659 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,9 @@ importers: .: dependencies: + js-tokens: + specifier: ^10.0.0 + version: 10.0.0 proper-lockfile: specifier: ^4.1.2 version: 4.1.2 @@ -587,6 +590,9 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + js-tokens@10.0.0: + resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==} + lightningcss-android-arm64@1.33.0: resolution: {integrity: sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==} engines: {node: '>= 12.0.0'} @@ -1141,6 +1147,8 @@ snapshots: graceful-fs@4.2.11: {} + js-tokens@10.0.0: {} + lightningcss-android-arm64@1.33.0: optional: true diff --git a/scripts/package-smoke.mjs b/scripts/package-smoke.mjs index e24e3b4..a23390b 100644 --- a/scripts/package-smoke.mjs +++ b/scripts/package-smoke.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node import { execFileSync } from "node:child_process"; -import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { dirname, isAbsolute, join } from "node:path"; @@ -255,7 +255,7 @@ function runtimeDependencyPaths(rootPath = root) { function collect(packageJsonPath, packageRequire) { const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")); for (const name of runtimeDependencyNames(packageJson)) { - const dependencyPackageJson = packageRequire.resolve(`${name}/package.json`); + const dependencyPackageJson = runtimeDependencyManifest(packageRequire, name); const dependencyPath = dirname(dependencyPackageJson); if (dependencyPaths.has(dependencyPath)) { continue; @@ -270,6 +270,19 @@ function runtimeDependencyPaths(rootPath = root) { return [...dependencyPaths.values()]; } +function runtimeDependencyManifest(packageRequire, name) { + let directory = dirname(packageRequire.resolve(name)); + for (;;) { + const candidate = join(directory, "package.json"); + if (existsSync(candidate) && JSON.parse(readFileSync(candidate, "utf8")).name === name) { + return candidate; + } + const parent = dirname(directory); + if (parent === directory) throw new Error(`package metadata not found for ${name}`); + directory = parent; + } +} + function runtimeDependencyNames(packageJson) { return Object.keys(packageJson.dependencies ?? {}); } @@ -329,7 +342,7 @@ function verifyRuntimeDependencies(context) { for (const name of runtimeDependencyNames(packageJson)) { run(context, "node", [ "-e", - "require.resolve(`${process.argv[1]}/package.json`, { paths: [process.argv[2]] })", + "require.resolve(process.argv[1], { paths: [process.argv[2]] })", name, packageRoot, ]); diff --git a/src/app.ts b/src/app.ts index 757cda8..14576e9 100644 --- a/src/app.ts +++ b/src/app.ts @@ -8,6 +8,7 @@ import { nowIso, writeJson } from "./fs.js"; import { discoverGit } from "./git.js"; import { mapWithSource } from "./agent-mapper.js"; import { mapFeatures } from "./mapper.js"; +import { findHttpRelations } from "./http-relations.js"; import { emitProgress } from "./progress.js"; import { providerByName } from "./provider.js"; import { @@ -112,6 +113,11 @@ export async function mapCommand( emitProgress(context, "map", event, fields); }, }); + const linkHttp = stringFlag(flags, "linkHttp"); + const http = + linkHttp === undefined + ? {} + : { http: await findHttpRelations(loaded.root, result.features, linkHttp, filters) }; const activeFeatureIds = new Set(result.features.map((feature) => feature.featureId)); if (flags["dryRun"] === true) { emitProgress(context, "map", "done", { @@ -120,6 +126,7 @@ export async function mapCommand( elapsed: `${Math.round((Date.now() - started) / 1000)}s`, }); return { + ...http, dryRun: true, features: result.features.length, new: result.created, @@ -152,6 +159,7 @@ export async function mapCommand( elapsed: `${Math.round((Date.now() - started) / 1000)}s`, }); return { + ...http, features: result.features.length, new: result.created, changed: result.changed, diff --git a/src/cli.ts b/src/cli.ts index a6da8f0..d129152 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -21,6 +21,7 @@ import { } from "./app.js"; import { ClawpatchError } from "./errors.js"; import { GlobalOptions } from "./config.js"; +import { httpRoots } from "./http-relations.js"; const moduleRequire = createRequire(import.meta.url); @@ -112,6 +113,7 @@ export function parseArgs(argv: string[]): ParsedArgs { command = "status"; } validateCommandFlags(command, flags); + if (typeof flags["linkHttp"] === "string") httpRoots(flags["linkHttp"]); validateCommandRequirements(command, flags); return { command, flags, global, help: false, version: false }; } @@ -131,7 +133,15 @@ type CommandSpec = { const commandSpecs = { init: { flags: ["force"], usage: ["clawpatch init [flags]"], run: initCommand }, map: { - flags: ["dryRun", "source", "provider", "model", "reasoningEffort", "skipGitRepoCheck"], + flags: [ + "dryRun", + "source", + "provider", + "model", + "reasoningEffort", + "skipGitRepoCheck", + "linkHttp", + ], usage: ["clawpatch map [flags]"], run: mapCommand, }, @@ -140,6 +150,7 @@ const commandSpecs = { flags: [ "feature", "featureList", + "linkHttp", "project", "limit", "since", @@ -316,6 +327,12 @@ const optionSpecs: Record = { target: "command", help: " --rate-limit-per-minute cap provider calls per 60s window (env: CLAWPATCH_RPM)", }, + "link-http": { + name: "linkHttp", + kind: "value", + target: "command", + help: " --link-http opt-in HTTP candidate context between directory roots", + }, source: { name: "source", kind: "value", diff --git a/src/http-relations.test.ts b/src/http-relations.test.ts new file mode 100644 index 0000000..884ac80 --- /dev/null +++ b/src/http-relations.test.ts @@ -0,0 +1,393 @@ +import { mkdtemp, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { initCommand, makeContext, mapCommand, reviewCommand } from "./app.js"; +import { parseArgs } from "./cli.js"; +import { defaultConfig } from "./config.js"; +import { findHttpRelations, httpEndpoints, httpRoots, withHttpContext } from "./http-relations.js"; +import { buildReviewPromptBundle } from "./prompt.js"; +import { readFeatures, readProject, statePaths } from "./state.js"; +import { testOptions, writeFixture } from "./test-helpers.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); +}); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "clawpatch-http-")); + roots.push(root); + await writeFixture( + root, + "package.json", + JSON.stringify({ name: "http-test", workspaces: ["frontend"] }), + ); + await writeFixture( + root, + "frontend/package.json", + JSON.stringify({ name: "frontend", type: "module" }), + ); + await writeFixture(root, "frontend/src/app.ts", 'await fetch("/api/login");\n'); + await writeFixture(root, "Cargo.toml", '[workspace]\nmembers = ["backend"]\nresolver = "2"\n'); + await writeFixture( + root, + "backend/Cargo.toml", + '[package]\nname = "backend"\nversion = "0.1.0"\nedition = "2021"\n', + ); + await writeFixture( + root, + "backend/src/main.rs", + '#[get("/api/login")]\nasync fn login() {}\nfn main() {}\n', + ); + const context = await makeContext({ ...testOptions(root), quiet: true }); + await initCommand(context, {}); + const baseline = await mapCommand(context, {}); + const paths = statePaths(join(root, ".clawpatch")); + const features = await readFeatures(paths); + const scan = () => + findHttpRelations(root, features, "frontend:backend", { include: ["**"], exclude: [] }); + return { root, context, baseline, paths, features, scan }; +} + +describe("HTTP candidate context", () => { + it("preserves mapping records and adds current counterpart files only to a bounded prompt copy", async () => { + const setup = await fixture(); + expect(setup.baseline).not.toHaveProperty("http"); + const before = await readFeatures(setup.paths); + const mapped = (await mapCommand(setup.context, { + linkHttp: "frontend:backend", + dryRun: true, + })) as { http: Awaited> }; + expect(mapped.http.relations).toHaveLength(1); + expect(mapped.http.relations[0]).toMatchObject({ + method: "GET", + path: "/api/login", + caller: { file: "frontend/src/app.ts", line: 1 }, + handler: { file: "backend/src/main.rs", line: 1 }, + }); + expect(await readFeatures(setup.paths)).toEqual(before); + const relation = mapped.http.relations[0]!; + const backend = setup.features.find((f) => f.featureId === relation.handler.featureIds[0])!; + const enriched = withHttpContext(backend, mapped.http.relations); + expect(enriched.contextFiles).toContainEqual({ + path: "frontend/src/app.ts", + reason: "candidate HTTP GET /api/login; verify runtime routing", + }); + expect(backend.contextFiles).not.toContainEqual( + expect.objectContaining({ path: "frontend/src/app.ts" }), + ); + const project = (await readProject(setup.paths))!; + const prompt = await buildReviewPromptBundle(setup.root, project, enriched, defaultConfig()); + expect(prompt.manifest.includedFiles).toContainEqual( + expect.objectContaining({ path: "frontend/src/app.ts", readable: true, role: "context" }), + ); + expect(prompt.prompt).toContain('await fetch("/api/login")'); + const config = defaultConfig(); + config.review.maxContextFiles = 1; + const limited = await buildReviewPromptBundle(setup.root, project, enriched, config); + expect(limited.manifest.omittedFiles).toContainEqual({ + path: "frontend/src/app.ts", + role: "context", + reason: "maxContextFiles", + }); + await writeFixture( + setup.root, + "backend/src/main.rs", + '#[post("/api/login")]\nasync fn login() {}\nfn main() {}\n', + ); + expect((await setup.scan()).relations).toEqual([]); + }); + + it("uses HTTP context during review without storing it or enabling later reviews", async () => { + const setup = await fixture(); + await writeFixture(setup.root, "frontend/src/app.ts", 'fetch("/api/login"); // TODO_BUG\n'); + const backend = setup.features.find((f) => f.source === "rust-command")!; + const flags = { feature: backend.featureId, provider: "mock" }; + expect(await reviewCommand(setup.context, flags)).toMatchObject({ findings: 0 }); + expect( + await reviewCommand(setup.context, { ...flags, linkHttp: "frontend:backend" }), + ).toMatchObject({ findings: 1 }); + const saved = (await readFeatures(setup.paths)).find((f) => f.featureId === backend.featureId)!; + expect(saved.contextFiles).toEqual(backend.contextFiles); + expect(await reviewCommand(setup.context, flags)).toMatchObject({ findings: 0 }); + }); + + it("rejects ambiguity in one file and unresolved mount prefixes", async () => { + const setup = await fixture(); + await writeFixture( + setup.root, + "backend/src/main.rs", + '#[get("/api/login")]\nfn a() {}\n#[get("/api/login")]\nfn b() {}\n', + ); + expect((await setup.scan()).relations).toEqual([]); + await writeFixture( + setup.root, + "backend/src/main.rs", + '#[get("/api/login")]\nfn a() {}\nweb::scope("/v2");\n', + ); + expect((await setup.scan()).skippedReason).toContain("prefixes"); + await writeFixture( + setup.root, + "backend/src/main.rs", + '#[get("/api/login")]\nfn login() {}\nfn main() {}\n', + ); + await writeFixture( + setup.root, + "backend/src/config.rs", + 'fn configure(server: Rocket) -> Rocket { server.mount("/v2", routes![login]) }\n', + ); + expect((await setup.scan()).skippedReason).toContain("prefixes"); + await writeFixture( + setup.root, + "backend/src/config.rs", + 'fn configure(server: Rocket) { server./* prefix */mount("/v2", routes![login]); }\n', + ); + expect((await setup.scan()).skippedReason).toContain("prefixes"); + await writeFixture( + setup.root, + "backend/src/config.rs", + 'fn configure() { web/* prefix */::scope("/v2"); }\n', + ); + expect((await setup.scan()).skippedReason).toContain("prefixes"); + await writeFixture( + setup.root, + "backend/src/config.rs", + 'fn configure(server: Rocket) { server.mount::<_, _>("/v2", routes![login]); }\n', + ); + expect((await setup.scan()).skippedReason).toContain("prefixes"); + }); + + it("honors filters and refuses partial scans, missing sources, and symlinked roots", async () => { + const setup = await fixture(); + expect( + ( + await findHttpRelations(setup.root, setup.features, "frontend:backend", { + include: ["**"], + exclude: ["backend/**"], + }) + ).relations, + ).toEqual([]); + await writeFixture(setup.root, "backend/src/oversized.rs", " ".repeat(256_001)); + expect(await setup.scan()).toMatchObject({ + relations: [], + skippedReason: expect.stringContaining("byte budget"), + }); + await symlink(join(setup.root, "frontend"), join(setup.root, "alias"), "dir"); + await expect( + findHttpRelations(setup.root, setup.features, "frontend:alias", { + include: ["**"], + exclude: [], + }), + ).rejects.toThrow("must not overlap"); + const outside = await mkdtemp(join(tmpdir(), "clawpatch-http-outside-")); + roots.push(outside); + await symlink(outside, join(setup.root, "external"), "dir"); + await expect( + findHttpRelations(setup.root, setup.features, "frontend:external", { + include: ["**"], + exclude: [], + }), + ).rejects.toThrow("invalid HTTP relation root"); + await expect( + findHttpRelations(setup.root, setup.features, "frontend:missing", { + include: ["**"], + exclude: [], + }), + ).rejects.toThrow("invalid HTTP relation root"); + }); + + it("ignores unsupported files before budgeting and unrelated scope text", async () => { + const setup = await fixture(); + await Promise.all( + Array.from({ length: 501 }, (_, i) => + writeFixture(setup.root, `frontend/ui/part${i}.tsx`, ""), + ), + ); + await writeFixture(setup.root, "frontend/ui/large.tsx", " ".repeat(256_001)); + await writeFixture( + setup.root, + "backend/src/main.rs", + '#[get("/api/login")]\nfn login() {}\n// web::scope("/ignored")\nlet example = "web::scope(";\nstd::thread::scope(|s| {});\n', + ); + expect((await setup.scan()).relations).toHaveLength(1); + }); + + it("caps review context independently from the mapped relations", async () => { + const setup = await fixture(); + for (let i = 0; i < 5; i++) + await writeFixture(setup.root, `frontend/src/client${i}.ts`, 'fetch("/api/login");\n'); + await mapCommand(setup.context, {}); + const features = await readFeatures(setup.paths); + const result = await findHttpRelations(setup.root, features, "frontend:backend", { + include: ["**"], + exclude: [], + }); + expect(result.relations).toHaveLength(6); + expect(result.omitted).toBe(0); + const backend = features.find((f) => f.source === "rust-command")!; + const context = withHttpContext(backend, result.relations).contextFiles.filter((ref) => + ref.reason.startsWith("candidate HTTP"), + ); + expect(context.map((ref) => ref.path)).toEqual([ + "frontend/src/app.ts", + "frontend/src/client0.ts", + "frontend/src/client1.ts", + ]); + }); +}); + +describe("literal HTTP syntax", () => { + it("matches methods exactly and rejects options, members, templates, absolute/dynamic paths", () => { + const source = `fetch("/get"); fetch('/post', { method: "POST" }); fetch("/no", options); fetch("/no", { ...options }); obj.fetch("/no"); fetch(\`/no\`); fetch("https://host/no"); fetch("//host/no"); fetch("/no?q=x"); fetch("/users/:id"); fetch("/get", { method: verb });`; + expect( + httpEndpoints(source, "client.ts", "caller").map(({ method, path }) => [method, path]), + ).toEqual([ + ["GET", "/get"], + ["POST", "/post"], + ]); + expect(httpEndpoints('fetch("/patch", { method: "patch" });', "client.ts", "caller")).toEqual( + [], + ); + expect( + httpEndpoints('fetch("/patch", { method: "PATCH" });', "client.ts", "caller")[0]?.method, + ).toBe("PATCH"); + }); + + it("ignores comments, regexes, templates, strings, Rust raw strings and nested comments", () => { + expect( + httpEndpoints( + '// fetch("/no")\n/* fetch("/no") */\nconst x = \'fetch("/no")\';\nconst t = `fetch("/no")`;\nconst regex = /fetch(.+)/;\nfetch("/yes")', + "client.ts", + "caller", + ).map((r) => r.path), + ).toEqual(["/yes"]); + const source = + '// #[get("/no")]\n/* /* nested */ #[get("/no")] */\nlet s = r###" #[get("/no")] "###;\n#[post("/yes")]\nfn yes() {}'; + expect( + httpEndpoints(source, "main.rs", "handler").map(({ method, path }) => [method, path]), + ).toEqual([["POST", "/yes"]]); + }); + + it("does not confuse nested templates, JSX text, or spaced members with fetch calls", () => { + const nested = 'const x = `outer ${`fetch("/fake")`} end`; fetch("/real");'; + expect(httpEndpoints(nested, "client.ts", "caller").map((r) => r.path)).toEqual(["/real"]); + expect( + httpEndpoints('object. fetch("/fake"); fetch("/real");', "client.ts", "caller").map( + (r) => r.path, + ), + ).toEqual(["/real"]); + expect( + httpEndpoints( + 'client. /* wrapper */ fetch("/fake"); fetch("/real");', + "client.ts", + "caller", + ).map((r) => r.path), + ).toEqual(["/real"]); + expect(httpEndpoints('
fetch("/fake")
', "client.tsx", "caller")).toEqual([]); + }); + + it("skips deeply nested templates without consuming the call stack", () => { + const source = + "const nested = " + "`x${".repeat(10_000) + "0" + "}`".repeat(10_000) + '; fetch("/real");'; + expect(httpEndpoints(source, "client.ts", "caller").map((endpoint) => endpoint.path)).toEqual([ + "/real", + ]); + }); + + it("keeps calls after regex literals in control-flow bodies and TypeScript", () => { + for (const prefix of [ + "if (enabled)", + "while (enabled)", + "for (;enabled;)", + "if (enabled && check())", + ]) { + const source = prefix + ` /["']/.test(value); fetch("/api/login");`; + expect(httpEndpoints(source, "client.ts", "caller").map((r) => r.path)).toEqual([ + "/api/login", + ]); + } + const source = `@decorator class Client { run(value: string): void { if (enabled) /["']/.test(value); fetch("/api/login"); } }`; + expect(httpEndpoints(source, "client.ts", "caller").map((r) => r.path)).toEqual(["/api/login"]); + }); + + it("matches identifier boundaries without accepting private methods or Unicode prefixes", () => { + const source = + 'class Client { #fetch() {} run() { this.#fetch("/private"); } } πfetch("/unicode"); const item = object.member\nfetch("/real");'; + expect(httpEndpoints(source, "client.ts", "caller").map((r) => r.path)).toEqual(["/real"]); + }); + + it("recognizes literal calls across comments and trailing commas", () => { + const source = + 'fetch /* route */ ("/get",);\nfetch("/post", { "method": /* verb */ "post", },);'; + expect( + httpEndpoints(source, "client.ts", "caller").map(({ method, path }) => [method, path]), + ).toEqual([ + ["GET", "/get"], + ["POST", "/post"], + ]); + }); + + it("matches unescaped static path punctuation and Unicode", () => { + for (const path of [ + "/users/@me", + "/search/a+b", + "/encoded/%2F", + "/Über", + "/authors/O'Reilly", + "/time/12:00", + ]) { + const literal = JSON.stringify(path); + expect(httpEndpoints(`fetch(${literal})`, "client.ts", "caller")[0]?.path).toBe(path); + expect(httpEndpoints(`#[get(${literal})]`, "main.rs", "handler")[0]?.path).toBe(path); + } + }); + + it("keeps unique routes usable in a dense duplicate-handler inventory", async () => { + const setup = await fixture(); + const handlers = Array.from( + { length: 4_000 }, + (_, i) => `#[get("/api/login")]\nfn route_${i}() {}\n`, + ).join(""); + await writeFixture( + setup.root, + "backend/src/main.rs", + handlers + '#[get("/health")]\nfn health() {}\nfn main() {}\n', + ); + await writeFixture( + setup.root, + "frontend/src/app.ts", + 'fetch("/api/login");\nfetch("/health");\n', + ); + const result = await setup.scan(); + expect(result.skippedReason).toBeNull(); + expect(result.relations.map((relation) => relation.path)).toEqual(["/health"]); + }); + + it("tracks line numbers across a dense supported source file", () => { + const endpoints = httpEndpoints('fetch("/");\n'.repeat(20_000), "client.ts", "caller"); + expect(endpoints).toHaveLength(20_000); + expect(endpoints[0]?.line).toBe(1); + expect(endpoints.at(-1)?.line).toBe(20_000); + }); + + it("validates the explicit service pair and exposes it only on map/review", () => { + for (const value of [ + "", + "frontend", + ".:backend", + "../frontend:backend", + "/frontend:backend", + "front:front/back", + "front:front", + "front:back:third", + ]) + expect(() => httpRoots(value)).toThrow(); + expect(httpRoots("apps/web:services/api")).toEqual(["apps/web", "services/api"]); + for (const command of ["map", "review"]) + expect(parseArgs([command, "--link-http", "frontend:backend"]).flags["linkHttp"]).toBe( + "frontend:backend", + ); + expect(() => parseArgs(["ci", "--link-http", "frontend:backend"])).toThrow(); + }); +}); diff --git a/src/http-relations.ts b/src/http-relations.ts new file mode 100644 index 0000000..3a1fe62 --- /dev/null +++ b/src/http-relations.ts @@ -0,0 +1,374 @@ +import { open, realpath, stat } from "node:fs/promises"; +import jsTokens, { type Token } from "js-tokens"; +import { isAbsolute, relative, resolve, sep } from "node:path"; +import { ClawpatchError } from "./errors.js"; +import { pathMatchesFilters, walk, type PathFilters } from "./mappers/shared.js"; +import type { FeatureRecord } from "./types.js"; + +export type HttpRelation = { + method: string; + path: string; + caller: { file: string; line: number; featureIds: string[] }; + handler: { file: string; line: number; featureIds: string[] }; +}; +export type HttpRelations = { + relations: HttpRelation[]; + omitted: number; + skippedReason: string | null; +}; +type Endpoint = { method: string; path: string; file: string; line: number }; +const sourceLimit = 256_000; +const maxFiles = 500; +const totalLimit = 8_000_000; +const counterpartLimit = 3; +const methods = "get|post|put|patch|delete|head|options"; +const methodNames = new Set(methods.toUpperCase().split("|")); +const fetchNormalizedMethods = new Set(["DELETE", "GET", "HEAD", "OPTIONS", "POST", "PUT"]); + +export function httpRoots(value: string): [string, string] { + const parts = value.split(":"); + if ( + parts.length !== 2 || + parts.some( + (part) => + !part || + part.split("/").some((p) => !p || p === "." || p === "..") || + /[\\]/u.test(part) || + isAbsolute(part), + ) + ) { + throw new ClawpatchError( + "--link-http requires caller:backend repository-relative directory roots", + 2, + "invalid-usage", + ); + } + const [caller, handler] = parts as [string, string]; + if (within(caller, handler) || within(handler, caller)) { + throw new ClawpatchError("--link-http roots must not overlap", 2, "invalid-usage"); + } + return [caller, handler]; +} + +export async function findHttpRelations( + root: string, + features: FeatureRecord[], + value: string, + filters: PathFilters, +): Promise { + const [callerRoot, handlerRoot] = httpRoots(value); + const realRoot = await realpath(root); + const canonicalRoots: string[] = []; + for (const scope of [callerRoot, handlerRoot]) { + const full = resolve(root, scope); + const actual = await realpath(full).catch(() => null); + if (actual === null || !inside(realRoot, actual) || !(await stat(actual)).isDirectory()) { + throw new ClawpatchError(`invalid HTTP relation root: ${scope}`, 2, "invalid-usage"); + } + canonicalRoots.push(actual); + } + if ( + inside(canonicalRoots[0]!, canonicalRoots[1]!) || + inside(canonicalRoots[1]!, canonicalRoots[0]!) + ) { + throw new ClawpatchError("--link-http roots must not overlap", 2, "invalid-usage"); + } + const files = (await walk(root, [callerRoot, handlerRoot])) + .filter( + (file) => + pathMatchesFilters(file, filters) && + ((within(file, callerRoot) && /\.[cm]?[jt]s$/u.test(file)) || + (within(file, handlerRoot) && file.endsWith(".rs"))), + ) + .toSorted(); + if (files.length > maxFiles) return skipped("HTTP relation scan exceeds 500 source files"); + const callers: Endpoint[] = []; + const handlers: Endpoint[] = []; + let bytes = 0; + for (const file of files) { + const full = resolve(root, file); + const actual = await realpath(full).catch(() => null); + if (actual === null || !inside(realRoot, actual)) + return skipped("HTTP relation source is missing or outside the repository"); + const handle = await open(actual, "r"); + let source: string; + try { + const buffer = Buffer.alloc(sourceLimit + 1); + let bytesRead = 0; + while (bytesRead < buffer.length) { + const read = await handle.read(buffer, bytesRead, buffer.length - bytesRead, bytesRead); + if (read.bytesRead === 0) break; + bytesRead += read.bytesRead; + } + bytes += bytesRead; + if (bytesRead > sourceLimit || bytes > totalLimit) + return skipped("HTTP relation scan exceeds its source byte budget"); + source = buffer.subarray(0, bytesRead).toString("utf8"); + } finally { + await handle.close(); + } + if (within(file, callerRoot) && !file.endsWith(".rs")) + callers.push(...httpEndpoints(source, file, "caller")); + if (within(file, handlerRoot) && file.endsWith(".rs")) { + // Mounting changes route paths; do not guess prefixes from local declarations. + if (hasRouteMount(source)) + return skipped("HTTP backend has unresolved scope or mount prefixes"); + handlers.push(...httpEndpoints(source, file, "handler")); + } + } + const ownersByFile = new Map>(); + for (const feature of features) { + if (feature.status === "skipped") continue; + for (const ref of feature.ownedFiles) { + const ids = ownersByFile.get(ref.path) ?? new Set(); + ids.add(feature.featureId); + ownersByFile.set(ref.path, ids); + } + } + const ownerIds = new Map([...ownersByFile].map(([file, ids]) => [file, [...ids].toSorted()])); + // Once a route is ambiguous, retaining more endpoints only wastes work. + const byRoute = new Map(); + for (const handler of handlers) { + const key = `${handler.method} ${handler.path}`; + byRoute.set(key, byRoute.has(key) ? null : handler); + } + const relations: HttpRelation[] = []; + const seen = new Set(); + let omitted = 0; + for (const caller of callers) { + const handler = byRoute.get(`${caller.method} ${caller.path}`); + if (handler === undefined || handler === null) continue; + const callerIds = ownerIds.get(caller.file) ?? []; + const handlerIds = ownerIds.get(handler.file) ?? []; + if (!callerIds.length || !handlerIds.length) continue; + const key = `${caller.file}:${caller.method}:${caller.path}:${handler.file}`; + if (seen.has(key)) continue; + seen.add(key); + if (relations.length >= 200) { + omitted += 1; + continue; + } + relations.push({ + method: caller.method, + path: caller.path, + caller: { file: caller.file, line: caller.line, featureIds: callerIds }, + handler: { file: handler.file, line: handler.line, featureIds: handlerIds }, + }); + } + return { relations, omitted, skippedReason: null }; +} + +function hasRouteMount(source: string): boolean { + const tokens = codeSource(source, rustCodeMask(source)); + return ( + /\bweb\s*::\s*scope\s*(?:\(|::\s*<)/u.test(tokens) || /\.\s*mount\s*(?:\(|::\s*<)/u.test(tokens) + ); +} + +function codeSource(source: string, mask: Uint8Array): string { + return source + .split("") + .map((char, index) => (mask[index] === 1 ? char : " ")) + .join(""); +} + +export function withHttpContext(feature: FeatureRecord, relations: HttpRelation[]): FeatureRecord { + const refs = new Map(); + for (const relation of relations) { + const counterpart = relation.caller.featureIds.includes(feature.featureId) + ? relation.handler + : relation.handler.featureIds.includes(feature.featureId) + ? relation.caller + : null; + if (counterpart !== null) + refs.set( + counterpart.file, + `candidate HTTP ${relation.method} ${relation.path}; verify runtime routing`, + ); + } + return { + ...feature, + contextFiles: [ + ...feature.contextFiles, + ...[...refs].slice(0, counterpartLimit).map(([path, reason]) => ({ path, reason })), + ], + }; +} + +export function httpEndpoints( + source: string, + file: string, + role: "caller" | "handler", +): Endpoint[] { + if (role === "caller") return /\.[jt]sx$/u.test(file) ? [] : javascriptEndpoints(source, file); + const code = rustCodeMask(source); + const pattern = new RegExp( + String.raw`#\[\s*(${methods})\s*\(\s*"(\/[^"\\\r\n]*)"\s*\)\s*\]`, + "gu", + ); + const endpoints: Endpoint[] = []; + let line = 1; + let lineCursor = 0; + for (const match of source.matchAll(pattern)) { + while (lineCursor < match.index) { + if (source[lineCursor++] === "\n") line += 1; + } + if (code[match.index] && isHttpPath(match[2]!)) + endpoints.push({ method: match[1]!.toUpperCase(), path: match[2]!, file, line }); + } + return endpoints; +} + +type LocatedToken = { token: Token; line: number; inTemplate: boolean }; + +function javascriptEndpoints(source: string, file: string): Endpoint[] { + const tokens: LocatedToken[] = []; + let line = 1; + let templateDepth = 0; + try { + for (const token of jsTokens(source)) { + if (token.type === "TemplateHead") templateDepth += 1; + if ( + token.type !== "WhiteSpace" && + token.type !== "LineTerminatorSequence" && + !token.type.endsWith("Comment") + ) + tokens.push({ token, line, inTemplate: templateDepth > 0 }); + if (token.type === "TemplateTail") templateDepth -= 1; + for (const char of token.value) if (char === "\n") line += 1; + } + } catch (error) { + // Do not emit partial relations if the tokenizer exceeds its own limits. + if (error instanceof RangeError) return []; + throw error; + } + const endpoints: Endpoint[] = []; + for (let index = 0; index < tokens.length; index += 1) { + const current = tokens[index]!; + if ( + current.inTemplate || + current.token.type !== "IdentifierName" || + current.token.value !== "fetch" + ) + continue; + const previous = tokens[index - 1]?.token.value; + if (previous === "." || previous === "?.") continue; + const call = literalFetchCall(tokens, index); + if (call !== null) endpoints.push({ ...call, file, line: current.line }); + } + return endpoints; +} + +function literalFetchCall( + tokens: LocatedToken[], + start: number, +): { method: string; path: string } | null { + let index = start + 1; + if (tokens[index++]?.token.value !== "(") return null; + const path = unescapedString(tokens[index++]?.token); + if (path === null || !isHttpPath(path)) return null; + let method = "GET"; + if (tokens[index]?.token.value === ",") { + index += 1; + if (tokens[index]?.token.value !== ")") { + if (tokens[index++]?.token.value !== "{") return null; + const key = tokens[index++]?.token; + if ( + !(key?.type === "IdentifierName" && key.value === "method") && + unescapedString(key) !== "method" + ) + return null; + if (tokens[index++]?.token.value !== ":") return null; + const raw = unescapedString(tokens[index++]?.token); + if (raw === null) return null; + const upper = raw.toUpperCase(); + // Fetch normalizes these six verbs; PATCH remains case-sensitive. + const value = fetchNormalizedMethods.has(upper) ? upper : raw; + if (!methodNames.has(value)) return null; + method = value; + if (tokens[index]?.token.value === ",") index += 1; + if (tokens[index++]?.token.value !== "}") return null; + if (tokens[index]?.token.value === ",") index += 1; + } + } + return tokens[index]?.token.value === ")" ? { method, path } : null; +} + +function unescapedString(token: Token | undefined): string | null { + return token?.type === "StringLiteral" && token.closed && !token.value.includes("\\") + ? token.value.slice(1, -1) + : null; +} + +function isHttpPath(path: string): boolean { + return ( + path.startsWith("/") && + !path.startsWith("//") && + !/[?#*{}<>\s]/u.test(path) && + !path.split("/").some((part) => part === "." || part === ".." || part.startsWith(":")) + ); +} + +function rustCodeMask(source: string): Uint8Array { + const mask = new Uint8Array(source.length); + let i = 0; + while (i < source.length) { + if (source.startsWith("//", i)) { + const end = source.indexOf("\n", i); + i = end < 0 ? source.length : end; + continue; + } + if (source.startsWith("/*", i)) { + let depth = 1; + i += 2; + while (i < source.length && depth) { + if (source.startsWith("/*", i)) { + depth += 1; + i += 2; + } else if (source.startsWith("*/", i)) { + depth -= 1; + i += 2; + } else i += 1; + } + continue; + } + const raw = /^r(#+)?"/u.exec(source.slice(i)); + if (raw !== null) { + const end = source.indexOf(`"${raw[1] ?? ""}`, i + raw[0].length); + i = end < 0 ? source.length : end + 1 + (raw[1]?.length ?? 0); + continue; + } + const char = source[i]!; + // Rust lifetimes are identifiers, not unterminated character literals. + if ( + char === "'" && + /^'[A-Za-z_]\w*(?![\w'])/u.test(source.slice(i)) && + !/^'[^'\n]+'/u.test(source.slice(i)) + ) { + mask[i++] = 1; + continue; + } + if (char === '"' || char === "'") { + i += 1; + while (i < source.length) { + if (source[i] === "\\") i += 2; + else if (source[i++] === char) break; + } + continue; + } + mask[i++] = 1; + } + return mask; +} + +function within(file: string, scope: string): boolean { + return file === scope || file.startsWith(`${scope}/`); +} +function inside(root: string, file: string): boolean { + const path = relative(root, file); + return !isAbsolute(path) && path !== ".." && !path.startsWith(`..${sep}`); +} + +function skipped(skippedReason: string): HttpRelations { + return { relations: [], omitted: 0, skippedReason }; +} diff --git a/src/package-smoke.test.ts b/src/package-smoke.test.ts index dcb6518..81ccc82 100644 --- a/src/package-smoke.test.ts +++ b/src/package-smoke.test.ts @@ -46,7 +46,14 @@ describe("package smoke harness", () => { return packageJson.name; }); expect(packedDependencyNames).toEqual( - expect.arrayContaining(["proper-lockfile", "graceful-fs", "retry", "signal-exit", "zod"]), + expect.arrayContaining([ + "js-tokens", + "proper-lockfile", + "graceful-fs", + "retry", + "signal-exit", + "zod", + ]), ); const packArgs = smoke.packDependencyArgs({ diff --git a/src/review.ts b/src/review.ts index b2459d9..b05c3ec 100644 --- a/src/review.ts +++ b/src/review.ts @@ -9,6 +9,7 @@ import { findingFromOutput, mergeFinding } from "./findings.js"; import { nowIso } from "./fs.js"; import { discoverGit } from "./git.js"; import { runId } from "./id.js"; +import { findHttpRelations, withHttpContext, type HttpRelation } from "./http-relations.js"; import { emitProgress } from "./progress.js"; import { providerByName } from "./provider.js"; import type { DroppedFinding } from "./provider-types.js"; @@ -50,6 +51,14 @@ export async function reviewCommand( const mode = reviewMode(flags); const customPrompt = await loadCustomReviewPrompt(flags); const features = await selectReviewFeatures(loaded, flags); + const linkHttp = stringFlag(flags, "linkHttp"); + const http = + linkHttp === undefined + ? undefined + : await findHttpRelations(loaded.root, await readFeatures(loaded.paths), linkHttp, { + include: config.include, + exclude: config.exclude, + }); if (features.length === 0 && hasFileFilter(flags)) { if (flags["dryRun"] === true) { return { next: "no features touched by diff" }; @@ -69,6 +78,7 @@ export async function reviewCommand( if (flags["dryRun"] === true) { return { dryRun: true, + ...(http === undefined ? {} : { http }), wouldReview: features.length, mode, jobs: reviewJobs(flags), @@ -122,6 +132,7 @@ export async function reviewCommand( customPrompt, limiter, registryPostValidator, + httpRelations: http?.relations ?? [], allowNonPendingFeatureReview: stringFlag(flags, "feature") !== undefined || stringFlag(flags, "featureList") !== undefined || @@ -200,6 +211,7 @@ export async function reviewCommand( config.provider.name, ); return { + ...(http === undefined ? {} : { http }), run: currentRunId, reviewed: features.length, findings: findingIds.length, @@ -296,6 +308,7 @@ type ReviewFeatureOptions = { limiter: RpmLimiter; registryPostValidator: FindingPostValidator | undefined; allowNonPendingFeatureReview: boolean; + httpRelations: HttpRelation[]; }; async function reviewFeature( @@ -337,7 +350,7 @@ async function reviewFeature( const reviewPrompt = await buildReviewPromptBundle( loaded.root, loaded.project, - lockedFeature, + withHttpContext(lockedFeature, options.httpRelations), config, mode, customPrompt,