From bee01b165808eae37866fb4e51d8023c5de0b50f Mon Sep 17 00:00:00 2001 From: anirudhkumar-nanonets Date: Thu, 15 Oct 2026 15:13:13 +0530 Subject: [PATCH 1/2] chore(app): remove the brain build that posted to assign --- docs/github-app.md | 50 --- src/app/brain-build-process.ts | 151 -------- src/app/brain-build-worker.ts | 82 ----- src/app/brain-build.ts | 427 ----------------------- src/app/checkout.ts | 75 ---- src/app/history.ts | 437 ------------------------ src/app/identity.ts | 155 --------- src/app/main.ts | 8 - src/app/server.ts | 133 -------- src/app/sources.ts | 492 --------------------------- test/app-brain-build-process.test.ts | 39 --- test/app-sources-context.test.ts | 81 ----- 12 files changed, 2130 deletions(-) delete mode 100644 src/app/brain-build-process.ts delete mode 100644 src/app/brain-build-worker.ts delete mode 100644 src/app/brain-build.ts delete mode 100644 src/app/history.ts delete mode 100644 src/app/sources.ts delete mode 100644 test/app-brain-build-process.test.ts delete mode 100644 test/app-sources-context.test.ts diff --git a/docs/github-app.md b/docs/github-app.md index a8df07b0d..e3007a216 100644 --- a/docs/github-app.md +++ b/docs/github-app.md @@ -85,52 +85,6 @@ App settings → Install App → pick the repos. **Installing requires admin on repository** (or org-owner for an org-wide install) — the one thing an App does not get you around. -## Reading a repository into a Trail brain - -The App has a second, optional job: reading one repository's own history so -Trail can mine rules out of it. That is what backs "paste a repo URL" during -brain onboarding, and it lives here because this is the only service holding the -GitHub App credentials and the only one that can build a symbol graph. - -`POST /brain/build`, off unless `GRAFT_BRAIN_BUILD_SECRET` is set: - -```bash -curl -X POST https:///brain/build \ - -H "authorization: Bearer $GRAFT_BRAIN_BUILD_SECRET" \ - -H 'content-type: application/json' \ - -d '{"owner":"acme","repo":"api"}' -``` - -It resolves the installation for `acme/api` (`GET /repos/{owner}/{repo}/installation`, -App-JWT authed), clones it shallow, builds the graph, and reads: - -- **commit subjects and bodies**, `--no-merges --reverse`, up to 1000 -- **closed pull requests and their discussion**, most-discussed first, bots dropped -- **exported symbols**, with the body hash that later tells Trail whether a rule - still describes the code it was mined from - -What comes back is a digest of that — **messages, titles, comments, symbol ids -and hashes. No source code.** Two modes: - -| Request | Response | -| --- | --- | -| `owner` + `repo` | `202` with the digest, for the caller to ingest itself | -| plus `brainId` + `brainToken` | the digest is POSTed to the brain; `202` with its job id | - -The platform uses the first: it already holds the user's session and writes to -the brain directly, so handing this service a workspace key just to have it call -back would mean minting a credential per build for nothing. - -`404` with an `error` means the App is not installed on the repository — the -expected answer for "someone pasted a repo we cannot see", not a failure. The -caller turns it into a choice: install the App, or run `graft init --trail` -locally, where the code never leaves the machine. - -| Variable | Meaning | -| --- | --- | -| `GRAFT_BRAIN_BUILD_SECRET` | Bearer secret for the route. Unset disables it entirely. | -| `GRAFT_BRAIN_URL` | Platform base URL the digest is posted to, when a `brainToken` is sent. Overrides anything a request supplies, so a caller cannot redirect a repository's history elsewhere. | - ## Security The App clones code written by strangers on every fork PR while holding a token @@ -146,10 +100,6 @@ for the base repository, so: - **Pages are capabilities, not public URLs.** `/p/?t=` — an unknown page and a bad token are both `404`, so the endpoint cannot be used to discover which pull requests exist. Links expire with the page they point at. -- **`/brain/build` sends no source.** Its digest is commit messages, pull-request - discussion, symbol ids and hashes; the checkout is deleted in a `finally`. The - route is off until `GRAFT_BRAIN_BUILD_SECRET` is set, and the secret is - compared in constant time — it is long-lived, unlike a per-payload signature. ## What is not built yet diff --git a/src/app/brain-build-process.ts b/src/app/brain-build-process.ts deleted file mode 100644 index 501e19931..000000000 --- a/src/app/brain-build-process.ts +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Running a repository read somewhere it cannot stop the App answering. - * - * The same problem the reviewer had, and the same fix — see review-process.ts - * for the full reasoning and the production measurements behind it. In short: - * every expensive step of a read is synchronous (git through `spawnSync`, the - * parse a loop of native tree-sitter calls, the graph written with a blocking - * stringify), so in-process one read holds the event loop for its whole - * duration. - * - * Onboarding made that visible in a way reviews did not. Measured on - * NanoNets/assign: 0.7s to check access, 7s to clone, 81s to build the symbol - * graph, 21s to walk the pull requests. During those 81 seconds the App - * answered nothing at all — including the one-second question "can we see this - * other repository", asked by the next person to paste a URL, who then waited - * out someone else's clone before being told their repo is private. At a - * hundred signups there is nearly always a read in flight, so that is not an - * edge case, it is the normal case. - * - * The split is where the credential is: the parent resolves access and mints - * the token (network, never blocking), and the child does the work with it. - */ -import { fork, type ChildProcess } from "node:child_process"; -import { existsSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { resolveRepoRead, type BrainBuildDeps, type BrainBuildJob, type BrainBuildResult } from "./brain-build.js"; -import type { FromChild, StartMessage } from "./brain-build-worker.js"; - -/** - * How long one read may take before the process running it is killed. - * - * Well above the worst honest read seen (110s on a large monorepo) plus the - * clone budget checkout.ts allows itself, because the cost of being wrong is a - * read that never happens. It bounds a lost slot, not responsiveness — the - * server stopped caring how long a read takes the moment it left this process. - */ -const DEFAULT_TIMEOUT_MS = 15 * 60 * 1000; - -export interface ChildBuilderOptions { - /** The module the child runs. Only tests pass this. */ - entry?: string; - timeoutMs?: number; -} - -/** - * The module a read runs in. `dist/app/brain-build-worker.js` when installed or - * containerised; the TypeScript source next door when run from a checkout, - * since `fork` inherits `process.execArgv` and a parent started through tsx - * hands the child the same loader. - */ -export function brainBuildWorkerEntry(): string { - const js = fileURLToPath(new URL("./brain-build-worker.js", import.meta.url)); - if (existsSync(js)) return js; - const ts = js.replace(/\.js$/, ".ts"); - return existsSync(ts) ? ts : js; -} - -/** Reads in flight, so a shutdown does not leave a clone made with a token. */ -const live = new Set(); -let hooked = false; - -function track(child: ChildProcess): void { - live.add(child); - if (hooked) return; - hooked = true; - process.on("exit", () => { - for (const c of live) c.kill("SIGKILL"); - }); -} - -/** - * A builder with buildRepoIntoBrain's signature that runs the heavy half in a - * child process. Same shape on purpose: server.ts swaps one for the other in a - * single line, and a test injecting its own builder keeps running in-process. - */ -export function childBuilder(opts: ChildBuilderOptions = {}): (job: BrainBuildJob, deps: BrainBuildDeps) => Promise { - const entry = opts.entry ?? brainBuildWorkerEntry(); - const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; - return async (job, deps) => { - // In the parent, deliberately: it is pure network, it is what decides - // whether there is anything to fork for, and a RepoNotAccessibleError has - // to reach the caller as itself rather than as a dead child process. - const auth = await resolveRepoRead(job, deps); - return runInChild(job, deps, auth, entry, timeoutMs); - }; -} - -async function runInChild( - job: BrainBuildJob, - deps: BrainBuildDeps, - auth: Awaited>, - entry: string, - timeoutMs: number, -): Promise { - const log = deps.log ?? ((): void => {}); - const tag = `${job.owner}/${job.repo}`; - // The repository as argv makes `ps` in a container say which read each - // process is; the token stays out of it. - const child = fork(entry, [tag], { stdio: ["ignore", "inherit", "inherit", "ipc"] }); - track(child); - - try { - return await new Promise((resolve, reject) => { - let settled = false; - const settle = (fn: () => void): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - fn(); - }; - - const timer = setTimeout(() => { - settle(() => reject(new Error(`read of ${tag} exceeded ${timeoutMs}ms and was killed`))); - }, timeoutMs); - timer.unref(); - - child.on("message", (raw) => { - const msg = raw as FromChild; - if (msg.t === "log") { - log(msg.msg); - return; - } - if (msg.t === "done") { - settle(() => resolve(msg.result)); - return; - } - settle(() => reject(new Error(msg.message))); - }); - - child.on("error", (err) => settle(() => reject(err))); - - // The case that used to be an outage: a child that dies without - // reporting. An OOM kill on a large repository, a grammar segfault on a - // stranger's source. Now it costs one read, named, with the signal. - child.on("exit", (code, signal) => - settle(() => - reject(new Error(`read of ${tag} exited ${signal ? `on ${signal}` : `with code ${code}`} before reporting a result`)), - ), - ); - - const start: StartMessage = { t: "start", job, auth, api: deps.api, githubHost: deps.githubHost }; - if (child.connected) child.send(start); - }); - } finally { - live.delete(child); - if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); - } -} - -/** The App's default builder. */ -export const buildRepoInChildProcess = childBuilder(); diff --git a/src/app/brain-build-worker.ts b/src/app/brain-build-worker.ts deleted file mode 100644 index d4ee89bfa..000000000 --- a/src/app/brain-build-worker.ts +++ /dev/null @@ -1,82 +0,0 @@ -/** - * The process one repository read runs in. - * - * Forked by brain-build-process.ts, one per read, and it exits when the read - * does. It receives the credential it needs rather than the App's private key, - * for the same reason the reviewer's worker does: `ps` and `/proc//environ` - * are readable, and a key that never crosses the boundary cannot leak across it. - * - * Everything expensive happens here — the clone, the tree-sitter parse, the - * pull-request walk — and none of it can reach the server's event loop. - */ -import { readRepository, type BrainBuildJob, type RepoReadAuth } from "./brain-build.js"; - -/** Parent → child, once, immediately after the fork. */ -export interface StartMessage { - t: "start"; - job: BrainBuildJob; - auth: RepoReadAuth; - api?: string; - githubHost?: string; -} - -/** Child → parent: a line for the App's log, so a read is still traceable. */ -export interface LogMessage { - t: "log"; - msg: string; -} - -export interface DoneMessage { - t: "done"; - result: Awaited>; -} - -export interface FailedMessage { - t: "failed"; - message: string; -} - -export type FromChild = LogMessage | DoneMessage | FailedMessage; - -/** - * Send, and tell me when it has actually gone. - * - * `process.send` is asynchronous, and a digest of a large repository is several - * megabytes — a thousand commits, four thousand symbols. Disconnecting on the - * next line tears the channel down mid-write: the child exits 0, having done - * every bit of the work, and the parent reports "exited before reporting a - * result". It only shows on big repositories, because a small payload clears - * the pipe in one write and survives the race. - */ -function send(msg: FromChild, done?: () => void): void { - if (!process.send) { - done?.(); - return; - } - try { - process.send(msg, undefined, undefined, () => done?.()); - } catch { - /* parent went away; the exit is the report */ - done?.(); - } -} - -process.on("message", (raw) => { - const msg = raw as StartMessage; - if (msg.t !== "start") return; - void readRepository(msg.job, { - // No creds: this half needs none. resolveRepoRead already did every call - // that requires the App's identity, and its answer is in `auth`. - creds: { appId: "", privateKey: "" }, - fetch: globalThis.fetch, - api: msg.api, - githubHost: msg.githubHost, - log: (line: string) => send({ t: "log", msg: line }), - }, msg.auth) - .then((result) => { - send({ t: "done", result }, () => process.disconnect?.()); - }) - .catch((e: unknown) => { - send({ t: "failed", message: e instanceof Error ? e.message : String(e) }, () => process.disconnect?.()); - }); -}); diff --git a/src/app/brain-build.ts b/src/app/brain-build.ts deleted file mode 100644 index 8b9d1094a..000000000 --- a/src/app/brain-build.ts +++ /dev/null @@ -1,427 +0,0 @@ -/** - * Building one repository into a brain, on demand. - * - * The whole point of the onboarding flow this serves: someone pastes a - * repository URL, and a minute later they are looking at rules mined from their - * own history. Nothing is installed and nobody has signed up yet, so this has to - * work from the repository name alone. - * - * Runs in the app rather than in the platform because only the app has the - * GitHub App credentials, only it can clone, and only graft can build a symbol - * graph. What crosses back to the platform is a digest of messages and symbol - * ids — never source. - */ -import { buildGraph } from "../graph/build.js"; -import { contextDirFor } from "../context/node-file.js"; -import { loadGraphCached } from "../graph/load.js"; -import { checkoutRepository } from "./checkout.js"; -import { - appJwt, - ghHeaders, - installationFor, - publicReadInstallation, - repoAccessGap, - type AppCredentials, - type Fetch, - type RepoAccessGap, -} from "./identity.js"; -import { buildDigest, postDigest, readCommits, readSymbols, readThreads, type RepoDigest } from "./history.js"; -import { - budgetSources, - readAgentInstructions, - readBranchProtection, - readCodeowners, - readCodifiedRules, - readDecisionDocs, - readDeclinedIssues, - readReverts, - readTestNames, - type HistorySource, -} from "./sources.js"; - -/** One request to build a repository into a brain. */ -export interface BrainBuildJob { - owner: string; - repo: string; - /** Branch to read; empty means the repository's default. */ - ref?: string; - /** - * The brain the rules land in, and the workspace key to write it with. - * - * Both optional, and omitting them changes the mode: with them, the digest is - * posted straight to the brain and only a job id comes back; without them the - * digest is RETURNED and the caller ingests it itself. - * - * The second mode is what the platform's own proxy uses. It already holds the - * user's session and can write to the brain directly, so handing graft a - * workspace key just to have it call back would mean minting a credential per - * build for no gain. - */ - brainId?: string; - brainToken?: string; - /** Platform base URL. Defaults to production. */ - brainBaseUrl?: string; - /** File the rules as approved rather than as drafts. Onboarding sets it: - * a brain whose every rule is an invisible draft answers nothing. */ - autoApprove?: boolean; -} - -export interface BrainBuildResult { - /** Set only when the digest was posted; empty in return-the-digest mode. */ - jobId: string; - headSha: string; - commits: number; - threads: number; - symbols: number; - sources: number; - /** Set only in return-the-digest mode. */ - digest?: RepoDigest; -} - -export interface BrainBuildDeps { - creds: AppCredentials; - fetch: Fetch; - api?: string; - githubHost?: string; - log?: (msg: string) => void; - now?: () => number; - /** - * Token used for a public repository the App is not installed on. - * - * Optional. Without it the read borrows one of our own installation's tokens, - * which GitHub answers for any public resource, and falls back to anonymous - * if the App has no installations at all. Anonymous works but is 60 requests - * an hour for the whole box — one pull-request walk — so it is the last - * resort, not the plan. - */ - publicToken?: string; - /** Whose installation to borrow for those reads. Ours, so a public read - * spends our rate limit and not a customer's. */ - publicOwner?: string; -} - -/** Raised when the App cannot see the repository. Distinct because the caller - * turns it into a specific answer — "install the app, or run it locally" — and - * not into a 500. */ -export class RepoNotAccessibleError extends Error { - constructor( - message: string, - /** Which gap it is, so the caller can send the user to the right place. */ - readonly gap: RepoAccessGap = { reason: "not_installed", ownerId: null, installationId: null }, - ) { - super(message); - } -} - -/** What a repository lets us do, before any of the work is started. */ -export interface RepoAccess { - /** True when we can read it: installed on it, or it is public. */ - accessible: boolean; - isPrivate: boolean; - /** How we would read it — for the log, and for deciding nothing else. */ - via: "installation" | "public" | "none"; - /** Set only when accessible is false: what to tell the user, and where to - * send them. Same shape RepoNotAccessibleError carries. */ - reason?: string; - gap?: RepoAccessGap; -} - -/** - * Can we read this repository at all? - * - * The cheap half of a build, on its own: an installation lookup and at most - * one repository lookup, about a second in total, with no clone and no graph. - * - * It exists as its own call because the answer decides which screen the user - * sees next, and the rest of the read takes minutes. Waiting for the whole - * build to find out means either holding the person there or guessing from a - * timer, and a guess is wrong exactly when the box is busy — which is when - * two people are onboarding at once. - */ -export async function checkRepoAccess(job: { owner: string; repo: string }, deps: BrainBuildDeps): Promise { - const api = deps.api ?? "https://api.github.com"; - const installationId = await installationFor(deps.creds, job.owner, job.repo, deps.fetch, (deps.now ?? Date.now)(), api); - if (installationId !== null) { - const token = await installationToken(deps, installationId, `${job.owner}/${job.repo}`, api); - const meta = await repoMeta(job.owner, job.repo, token, deps.fetch, api); - return { accessible: true, isPrivate: meta?.isPrivate ?? true, via: "installation" }; - } - const meta = await repoMeta(job.owner, job.repo, await publicReadToken(deps, api), deps.fetch, api); - if (meta && !meta.isPrivate) return { accessible: true, isPrivate: false, via: "public" }; - - const gap = await repoAccessGap(deps.creds, job.owner, deps.fetch, (deps.now ?? Date.now)(), api); - return { - accessible: false, - isPrivate: true, - via: "none", - reason: - gap.reason === "repo_not_selected" - ? `graft is installed on ${job.owner} but ${job.owner}/${job.repo} is not in the list of repositories it can see` - : `graft is not installed on ${job.owner}`, - gap, - }; -} - -/** - * Read the repository and hand its history to the brain. - * - * An installation is the preferred way in: it is the only way into a private - * repository, and it carries a rate limit worth having. It is not required for - * a public one, though — those clone anonymously and answer the API - * anonymously — so a failed installation lookup is a reason to try the open - * door, not a reason to stop. Only a repository that is private or unreadable - * both ways is one we genuinely cannot see, and that is the case the UI turns - * into "read it on your machine". - */ -/** Everything the heavy half needs, and nothing it could mint for itself. */ -export interface RepoReadAuth { - /** Installation or public-read token, for the GitHub API calls. */ - token: string; - /** Credential for the clone. Empty for a public repository we are not - * installed on: git serves those to anyone, and a token minted for another - * account is not promised to work on this one. */ - cloneToken: string; - meta: RepoMeta; -} - -/** - * Decide how this repository gets read, and mint the credential for it. - * - * The network half of a build, kept separate from the work: an installation - * lookup, a token, a repository lookup. About a second, none of it blocking. - * - * Throws RepoNotAccessibleError when there is no way in. That is the answer the - * onboarding UI turns into a choice, which is why it is raised here — before a - * single byte is cloned — rather than discovered somewhere inside the read. - */ -export async function resolveRepoRead(job: BrainBuildJob, deps: BrainBuildDeps): Promise { - const api = deps.api ?? "https://api.github.com"; - const tag = `${job.owner}/${job.repo}`; - const installationId = await installationFor(deps.creds, job.owner, job.repo, deps.fetch, (deps.now ?? Date.now)(), api); - if (installationId !== null) { - const token = await installationToken(deps, installationId, tag, api); - const meta = await repoMeta(job.owner, job.repo, token, deps.fetch, api); - return { token, cloneToken: token, meta: meta ?? { isPrivate: true, defaultBranch: "" } }; - } - - // No installation on this repository. That rules out nothing yet: a public - // repo clones with no credential at all, and its API answers to any token we - // hold. So find a credential for the reading, then ask whether it is public. - const token = await publicReadToken(deps, api); - const meta = await repoMeta(job.owner, job.repo, token, deps.fetch, api); - if (!meta || meta.isPrivate) { - // Which of the two gaps it is decides what the UI can offer, so it is - // resolved here rather than guessed there. - const gap = await repoAccessGap(deps.creds, job.owner, deps.fetch, (deps.now ?? Date.now)(), api); - throw new RepoNotAccessibleError( - gap.reason === "repo_not_selected" - ? `graft is installed on ${job.owner} but ${tag} is not in the list of repositories it can see` - : `graft is not installed on ${job.owner}`, - gap, - ); - } - return { token, cloneToken: "", meta }; -} - -/** - * Read the repository and hand its history to the brain. - * - * An installation is the preferred way in: it is the only way into a private - * repository, and it carries a rate limit worth having. It is not required for - * a public one, though — those clone anonymously and answer the API - * anonymously — so a failed installation lookup is a reason to try the open - * door, not a reason to stop. Only a repository that is private or unreadable - * both ways is one we genuinely cannot see, and that is the case the UI turns - * into "read it on your machine". - */ -export async function buildRepoIntoBrain(job: BrainBuildJob, deps: BrainBuildDeps): Promise { - return readRepository(job, deps, await resolveRepoRead(job, deps)); -} - -/** - * The heavy half: clone, graph, history, digest. - * - * Takes its credential rather than minting one, because this is the half that - * runs somewhere else. Every expensive thing in it is synchronous — the git - * calls are spawnSync, the parse is a loop of native tree-sitter calls, the - * graph is written with a blocking stringify — so in the server process it - * holds the event loop for the whole read, and a second person pasting a URL - * waits behind the first. See brain-build-process.ts. - */ -export async function readRepository(job: BrainBuildJob, deps: BrainBuildDeps, auth: RepoReadAuth): Promise { - const rawLog = deps.log ?? ((): void => {}); - const api = deps.api ?? "https://api.github.com"; - const tag = `${job.owner}/${job.repo}`; - const { token, cloneToken, meta } = auth; - - // Every line carries how long we have been at it. A read that feels slow is - // four things in a trench coat — the clone, the graph build, the - // pull-request walk, the digest — and without the elapsed time on each, the - // only observation anyone can make is "it took two minutes". - const startedAt = (deps.now ?? Date.now)(); - const log = (msg: string): void => rawLog(`[+${(((deps.now ?? Date.now)() - startedAt) / 1000).toFixed(1)}s] ${msg}`); - const phase = (name: string): (() => void) => { - const at = (deps.now ?? Date.now)(); - return () => log(`${tag}: ${name} took ${(((deps.now ?? Date.now)() - at) / 1000).toFixed(1)}s`); - }; - - const doneClone = phase("clone"); - const checkout = checkoutRepository({ - owner: job.owner, - repo: job.repo, - ref: job.ref || meta.defaultBranch, - token: cloneToken, - api: deps.githubHost, - log, - }); - doneClone(); - try { - // `graphOnly`: the symbol ids and their hashes are all the digest needs, and - // the markdown projections cost real time on a large repo. - const doneGraph = phase("symbol graph"); - await buildGraph(checkout.dir, { graphOnly: true }); - const graph = loadGraphCached(contextDirFor(checkout.dir)); - doneGraph(); - - const commits = readCommits(checkout.dir); - const symbols = readSymbols(graph); - // The threads are the slow part (two API calls per pull request), and the - // one most likely to fail on a rate limit. A failure here degrades to a - // commits-only ingest rather than losing the whole build. - let threads: Awaited> = []; - const doneThreads = phase("pull-request discussion"); - try { - threads = await readThreads(job.owner, job.repo, token, deps.fetch, api); - } catch (e) { - log(`${tag}: pull-request discussion unavailable (${e instanceof Error ? e.message : e}); mining commits only`); - } - - doneThreads(); - - // Everything the repo already states as a rule. All best-effort: the file - // readers cannot fail the build, and the two API readers swallow their own - // errors, so a repo with none of this still gets a brain from its history. - const doneSources = phase("stated sources"); - const sources = budgetSources([ - ...readAgentInstructions(checkout.dir), - ...readDecisionDocs(checkout.dir), - ...readCodeowners(checkout.dir), - ...readCodifiedRules(checkout.dir), - ...readReverts(checkout.dir), - ...readTestNames(symbols.map((s) => ({ name: s.name, path: s.path }))), - ...(await readBranchProtection(job.owner, job.repo, meta.defaultBranch, token, deps.fetch, api)), - ...(await readDeclinedIssues(job.owner, job.repo, token, deps.fetch, api)), - ]); - - doneSources(); - - log( - `${tag}: read ${commits.length} commits, ${threads.length} threads, ${symbols.length} symbols, ${sources.length} stated sources`, - ); - - const digest = buildDigest({ - owner: job.owner, - name: job.repo, - headSha: checkout.headSha, - defaultBranch: job.ref || meta.defaultBranch, - isPrivate: meta.isPrivate, - commits, - threads, - symbols, - sources, - autoApprove: job.autoApprove ?? false, - }); - - const counts = { - headSha: checkout.headSha, - commits: commits.length, - threads: threads.length, - symbols: symbols.length, - sources: sources.length, - }; - if (!job.brainId || !job.brainToken) { - return { jobId: "", ...counts, digest }; - } - const { jobId } = await postDigest( - job.brainBaseUrl ?? "https://agents.nanonets.com", - job.brainId, - job.brainToken, - digest, - deps.fetch, - ); - return { jobId, ...counts }; - } finally { - // Always: the clone was made with an installation token and is not something - // to leave in /tmp. - checkout.cleanup(); - } -} - -/** What the repository says about itself. */ -interface RepoMeta { - isPrivate: boolean; - defaultBranch: string; -} - -/** - * The repository's visibility and default branch, in one call. - * - * Null when the call did not answer, which is not the same as private and is - * why this does not fold the two together: with an installation, an unanswered - * call is a hiccup the read can carry on past; without one, it is the whole - * decision about whether we can see this repository at all. Callers that only - * need the fields treat null as private, because saying a repo is public when - * it is not would leak its name onto a pre-signup screen. - */ -async function repoMeta(owner: string, repo: string, token: string, fetchImpl: Fetch, api: string): Promise { - try { - const res = await fetchImpl(`${api}/repos/${owner}/${repo}`, { headers: ghHeaders(token) }); - if (!res.ok) return null; - const parsed = JSON.parse(await res.text()) as { private?: boolean; default_branch?: string }; - return { - isPrivate: parsed.private !== false, - defaultBranch: (parsed.default_branch ?? "").trim(), - }; - } catch { - return null; - } -} - -/** - * A credential for reading a public repository we are not installed on. - * - * Order: a token configured for exactly this, then one of our own - * installation's, then nothing. Every step of that ladder works — the - * difference is only the rate limit, and the drop to anonymous is a factor of - * eighty, so it is worth two extra calls to avoid. - */ -async function publicReadToken(deps: BrainBuildDeps, api: string): Promise { - if (deps.publicToken) return deps.publicToken; - try { - const id = await publicReadInstallation( - deps.creds, - deps.publicOwner ?? "trailhq", - deps.fetch, - (deps.now ?? Date.now)(), - api, - ); - if (id === null) return ""; - return await installationToken(deps, id, "public read", api); - } catch { - return ""; - } -} - -/** Mint an installation token, the credential every authenticated read uses. */ -async function installationToken(deps: BrainBuildDeps, installationId: number, tag: string, api: string): Promise { - const res = await deps.fetch(`${api}/app/installations/${installationId}/access_tokens`, { - method: "POST", - headers: { - authorization: `Bearer ${appJwt(deps.creds, (deps.now ?? Date.now)())}`, - accept: "application/vnd.github+json", - "user-agent": "graft-app", - }, - }); - if (!res.ok) throw new Error(`installation token for ${tag} failed: ${res.status}`); - return (JSON.parse(await res.text()) as { token: string }).token; -} diff --git a/src/app/checkout.ts b/src/app/checkout.ts index 7d129af69..f5c40a307 100644 --- a/src/app/checkout.ts +++ b/src/app/checkout.ts @@ -184,81 +184,6 @@ export function checkoutPullRequest(req: CheckoutRequest): Checkout { return { dir, base: "refs/graft/base", ref, cleanup }; } -/** What a plain-repository checkout needs: no pull request, just a ref. */ -export interface RepoCheckoutRequest { - owner: string; - repo: string; - /** Branch to read. Empty means "whatever HEAD points at" (the default branch). */ - ref?: string; - token: string; - /** How much history to fetch. The default is deeper than a PR review needs - * because commit MESSAGES are the payload here, not a diff — 50 commits is a - * fortnight on an active repo and would mine almost nothing. */ - depth?: number; - api?: string; - log?: (msg: string) => void; -} - -/** A plain checkout: the tree, the commit it landed on, and the cleanup. */ -export interface RepoCheckout { - dir: string; - headSha: string; - /** The branch actually checked out, resolved when the caller named none. */ - branch: string; - cleanup: () => void; -} - -/** - * Clone one repository at a ref, shallow, for reading rather than reviewing. - * - * Separate from {@link checkoutPullRequest} rather than a flag on it: that - * function's whole shape — two refspecs, the merge/head fallback, the diff-base - * resolution — exists to answer "what does this pull request change", and none - * of it applies to "read this repository's history". Sharing the hardened `git` - * runner is the part worth reusing. - */ -export function checkoutRepository(req: RepoCheckoutRequest): RepoCheckout { - const host = (req.api ?? "https://github.com").replace(/\/$/, ""); - const dir = mkdtempSync(join(tmpdir(), `graft-repo-${req.owner}-${req.repo}-`)); - const cleanup = (): void => rmSync(dir, { recursive: true, force: true }); - const log = req.log ?? ((): void => {}); - const tag = `${req.owner}/${req.repo}`; - const fail = (step: string, err: string): never => { - cleanup(); - throw new Error(`${tag}: ${step} failed: ${redact(err, req.token)}`); - }; - - const init = git(dir, ["init", "--quiet", "--initial-branch=graft-tmp"]); - if (!init.ok) fail("init", init.err); - const remote = git(dir, ["remote", "add", "origin", `${host}/${req.owner}/${req.repo}.git`]); - if (!remote.ok) fail("remote", remote.err); - - const depth = req.depth ?? REPO_FETCH_DEPTH; - // A named branch is fetched by name; with none, `HEAD` resolves to whatever - // the remote's default branch is, which is what the caller means by "the - // repository" and saves an API round trip to look the name up. - const refspec = req.ref - ? `+refs/heads/${req.ref}:refs/graft/head` - : `+HEAD:refs/graft/head`; - const fetched = git( - dir, - ["fetch", "--quiet", `--depth=${depth}`, "--no-recurse-submodules", "origin", refspec], - req.token, - ); - if (!fetched.ok) fail("fetch", fetched.err); - - const co = git(dir, ["checkout", "--quiet", "--detach", "refs/graft/head"]); - if (!co.ok) fail("checkout", co.err); - - const headSha = line(dir, ["rev-parse", "HEAD"]) ?? ""; - const branch = req.ref ?? line(dir, ["rev-parse", "--abbrev-ref", "origin/HEAD"]) ?? ""; - log(`${tag}: read ${req.ref || "HEAD"} at ${short(headSha)} (${depth} commits deep)`); - return { dir, headSha, branch: branch.replace(/^origin\//, ""), cleanup }; -} - -/** Default history depth for a repository read. See RepoCheckoutRequest.depth. */ -const REPO_FETCH_DEPTH = 500; - /** The one fetch each attempt makes: the PR ref to review plus the base branch. */ function fetchArgs(req: CheckoutRequest, pull: "merge" | "head", deepen = false): string[] { return [ diff --git a/src/app/history.ts b/src/app/history.ts deleted file mode 100644 index 4cbbe530f..000000000 --- a/src/app/history.ts +++ /dev/null @@ -1,437 +0,0 @@ -/** - * Reading a repository's history into a rule-bearing digest. - * - * What a code graph cannot tell you is why the code is the way it is. That - * lives in two places: commit messages, and the discussion under pull requests. - * This module reads both, plus the symbol ids from the graph so a rule mined - * out of them can be anchored to the code it governs. - * - * It deliberately sends NO source code onward. The digest is messages, titles, - * comments, symbol ids and hashes — nothing a reader could reconstruct a private - * codebase from. - */ -import { spawnSync } from "node:child_process"; -import type { GraphV1 } from "../graph/types.js"; -import { ghHeaders, type Fetch } from "./identity.js"; -import type { HistorySource } from "./sources.js"; - -/** One commit's rule-bearing content. */ -export interface HistoryCommit { - sha: string; - subject: string; - body: string; - files: string[]; -} - -/** One pull request's discussion. */ -export interface HistoryThread { - number: number; - title: string; - body: string; - mergeSha: string; - comments: Array<{ body: string; path?: string }>; -} - -/** One symbol, as the brain needs it to anchor a rule. */ -export interface HistorySymbol { - id: string; - path: string; - name: string; - kind: string; - signature: string; - fingerprint: string; -} - -/** The whole payload posted to the brain. */ -export interface RepoDigest { - provider: "github"; - owner: string; - name: string; - head_sha: string; - default_branch: string; - is_private: boolean; - commits: Array<{ sha: string; subject: string; body: string; files: string[]; symbols: string[] }>; - threads: Array<{ - number: number; - title: string; - body: string; - merge_sha: string; - comments: Array<{ body: string; path?: string }>; - }>; - symbols: HistorySymbol[]; - /** Everything in the repo that is already a rule, or nearly one: instruction - * files, decision records, config, ownership, reverts, test names. One array - * rather than a field per kind, so adding a source is adding an entry. */ - sources: HistorySource[]; - auto_approve: boolean; - /** The agents `graft init` wired this repo for (claude, cursor, agents, …). - * Trail shows only the context files those agents read. Absent or empty - * means "not known", and Trail then shows every kind. */ - agents?: string[]; -} - -/** Field separators inside one `git log` record. Chosen for being bytes no - * commit message contains, the same trick blast/owners.ts uses. */ -const REC = "\x01"; -const FIELD = "\x02"; - -/** Most commits read out of one repository. Above the brain's own per-ingest - * cap, so the trim happens here where the ordering is known. */ -const MAX_COMMITS = 1_000; - -/** Run git in `root`, or null when git fails. */ -function git(root: string, args: string[]): string | null { - const res = spawnSync("git", ["-c", "core.quotePath=false", ...args], { - cwd: root, - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - maxBuffer: 64 * 1024 * 1024, - }); - if (res.error || res.status !== 0 || typeof res.stdout !== "string") return null; - return res.stdout; -} - -/** - * Commit subjects, bodies and touched files, oldest first. - * - * `--no-merges`: a merge commit's message is "Merge pull request #N from …", - * which establishes nothing — the decision is in the commits it brings in, and - * in the thread. `--reverse` so a later reversal reads as a reversal downstream; - * the extraction prompt relies on that ordering. - */ -export function readCommits(root: string, max = MAX_COMMITS): HistoryCommit[] { - const fmt = `${REC}%H${FIELD}%s${FIELD}%b${FIELD}`; - const out = git(root, [ - "log", - "--no-merges", - "--reverse", - `-n`, - String(max), - `--format=${fmt}`, - "--name-only", - ]); - if (!out) return []; - - const commits: HistoryCommit[] = []; - for (const record of out.split(REC)) { - if (!record.trim()) continue; - const [sha = "", subject = "", body = "", rest = ""] = record.split(FIELD); - const files = rest - .split("\n") - .map((l) => l.trim()) - .filter((l) => l !== ""); - // A commit with no subject is a broken record, not a commit worth mining. - if (!sha.trim() || !subject.trim()) continue; - commits.push({ sha: sha.trim(), subject: subject.trim(), body: body.trim(), files }); - } - return commits; -} - -/** How many pull requests to read, and how many comment pages per thread. */ -const MAX_THREADS = 200; -const MAX_COMMENT_PAGES = 3; -/** How many threads' comments are fetched at once. Two requests per pull - * request, so this is the real cost of a repository read. A pool of 24 (so at - * most 48 requests in flight) stays well under GitHub's secondary limit on - * concurrent requests, and a slow thread no longer holds up the seven that - * happened to share its batch. */ -export const THREAD_FETCH_CONCURRENCY = 24; - -/** - * What one push has already asked GitHub, shared between its reads. - * - * The early upload reads the newest pull requests and the full read reads them - * again as part of the 200. Promises rather than results, so a full read that - * starts while the early one is still in flight waits on the same request - * instead of sending a second — and threads with no discussion are remembered - * too, which a map of finished threads could not say. - */ -export interface ThreadReadCache { - /** Pages of the closed-pull-request list, by page number. */ - pages: Map>; - /** One pull request's discussion, by number; null when it had none. */ - threads: Map>; -} - -export function newThreadReadCache(): ThreadReadCache { - return { pages: new Map(), threads: new Map() }; -} - -/** Run `fn` over `items` with at most `limit` in flight, results in input order. */ -export async function mapPool(items: readonly T[], limit: number, fn: (item: T, index: number) => Promise): Promise { - const out = new Array(items.length); - let next = 0; - const worker = async () => { - for (;;) { - const i = next++; - if (i >= items.length) return; - out[i] = await fn(items[i]!, i); - } - }; - await Promise.all(Array.from({ length: Math.min(Math.max(1, limit), items.length) }, worker)); - return out; -} - -export interface PullListItem { - number?: number; - title?: string; - body?: string | null; - merge_commit_sha?: string | null; -} - -interface CommentItem { - body?: string | null; - path?: string | null; - user?: { type?: string } | null; -} - -/** - * Closed pull requests and their discussion, most-discussed first. - * - * Closed only: an open PR's discussion has not concluded, so mining a rule out - * of it would record a proposal as a decision. - * - * Comment counts have to be discovered by fetching, not by reading the list. - * GitHub's `GET /pulls` list response carries NO `comments` or - * `review_comments` field — those exist only on the single-PR GET — so an - * earlier version that pre-filtered on them read `undefined` for every pull - * request, scored them all zero and skipped the lot. The failure was silent and - * total: every repository came back with no discussion at all. So the comments - * are fetched for the most recently updated closed pull requests, and the - * ranking happens afterwards, on counts that are real. - * - * Bot comments are dropped. A CI bot posting a coverage table is the single - * largest source of text in a busy repo's threads and it establishes nothing. - */ -export async function readThreads( - owner: string, - repo: string, - token: string, - fetchImpl: Fetch, - api = "https://api.github.com", - max = MAX_THREADS, - // Threads already read (the early upload's newest ones), reused rather than - // fetched a second time. - known: ReadonlyMap = new Map(), - // Everything this push has already asked GitHub, in flight or finished. See - // ThreadReadCache; a fresh one when the caller has nothing to share. - cache: ThreadReadCache = newThreadReadCache(), -): Promise { - const headers = ghHeaders(token); - - const readPage = (page: number): Promise => { - const cached = cache.pages.get(page); - if (cached) return cached; - const pending = (async () => { - const res = await fetchImpl( - `${api}/repos/${owner}/${repo}/pulls?state=closed&sort=updated&direction=desc&per_page=100&page=${page}`, - { headers }, - ); - if (!res.ok) return null; - const batch = JSON.parse(await res.text()) as PullListItem[]; - return Array.isArray(batch) ? batch : null; - })(); - // A failed page is forgotten, so the next read asks again rather than - // inheriting one blip for the rest of the push. - pending.catch(() => cache.pages.delete(page)); - cache.pages.set(page, pending); - return pending; - }; - - const pulls: PullListItem[] = []; - // 100 per page, up to the cap. Sorted by GitHub as most-recently-updated, - // which is the right window: rules decided two years ago that still hold get - // restated in newer threads, and ones that do not are usually superseded. - for (let page = 1; pulls.length < max && page <= Math.ceil(max / 100); page++) { - const batch = await readPage(page); - if (!batch || batch.length === 0) break; - pulls.push(...batch); - } - - // Deduplicated: a pull request updated between two page reads moves up the - // list and would otherwise be read, and sent, twice. - const seenNumbers = new Set(); - const candidates = pulls - .filter((p): p is PullListItem & { number: number } => typeof p.number === "number") - .filter((p) => !seenNumbers.has(p.number) && (seenNumbers.add(p.number), true)) - .slice(0, max); - - const readThread = (p: PullListItem & { number: number }): Promise => { - const seen = known.get(p.number); - if (seen) return Promise.resolve(seen); - const cached = cache.threads.get(p.number); - if (cached) return cached; - const pending = (async () => { - const [issueComments, reviewComments] = await Promise.all([ - readComments(`${api}/repos/${owner}/${repo}/issues/${p.number}/comments`, headers, fetchImpl), - readComments(`${api}/repos/${owner}/${repo}/pulls/${p.number}/comments`, headers, fetchImpl), - ]); - const comments = [...issueComments, ...reviewComments]; - if (comments.length === 0) return null; - return { - number: p.number, - title: (p.title ?? "").trim(), - body: (p.body ?? "").trim(), - mergeSha: (p.merge_commit_sha ?? "").trim(), - comments, - } satisfies HistoryThread; - })(); - pending.catch(() => cache.threads.delete(p.number)); - cache.threads.set(p.number, pending); - return pending; - }; - - // A pool rather than one at a time: two requests per pull request over 200 - // of them is 400 sequential round trips, which is minutes of a person - // waiting on a progress screen. - const fetched = await mapPool(candidates, THREAD_FETCH_CONCURRENCY, readThread); - const threads: HistoryThread[] = fetched.filter((t): t is HistoryThread => t !== null); - - // Most-discussed first, because the digest renderer spends its budget in this - // order and a twenty-comment thread contains an argument while a one-comment - // thread contains a rubber stamp. - threads.sort((a, b) => b.comments.length - a.comments.length); - return threads; -} - -/** One comment endpoint, paginated, bots removed. */ -async function readComments( - url: string, - headers: Record, - fetchImpl: Fetch, -): Promise> { - const out: Array<{ body: string; path?: string }> = []; - for (let page = 1; page <= MAX_COMMENT_PAGES; page++) { - const res = await fetchImpl(`${url}?per_page=100&page=${page}`, { headers }); - if (!res.ok) break; - let batch: CommentItem[]; - try { - batch = JSON.parse(await res.text()) as CommentItem[]; - } catch { - break; - } - if (!Array.isArray(batch) || batch.length === 0) break; - for (const c of batch) { - const body = (c.body ?? "").trim(); - if (!body) continue; - if (c.user?.type === "Bot") continue; - out.push(c.path ? { body, path: c.path } : { body }); - } - if (batch.length < 100) break; - } - return out; -} - -/** Most symbols to send. The brain only needs enough to anchor rules against. */ -const MAX_SYMBOLS = 4_000; - -/** - * Exported symbols from the graph, with the body hash that makes an anchored - * rule self-invalidating. - * - * Exported only, and biggest first: an internal helper is not what a team's - * rules are about, and the budget is better spent on the surface a rule is - * likely to govern. - */ -export function readSymbols(graph: GraphV1 | null, max = MAX_SYMBOLS): HistorySymbol[] { - if (!graph) return []; - return graph.nodes - .filter((n) => n.exported && n.kind !== "file") - .sort((a, b) => (b.chars ?? 0) - (a.chars ?? 0)) - .slice(0, max) - .map((n) => ({ - id: n.id, - path: n.path, - name: n.name, - kind: n.kind, - signature: n.signature ?? "", - fingerprint: n.body_hash, - })); -} - -/** - * Assemble the digest. - * - * Each commit carries the symbol ids its files touched, so a rule mined from a - * commit message can be anchored even when the message names no symbol — which - * is most of the time. - */ -export function buildDigest(input: { - owner: string; - name: string; - headSha: string; - defaultBranch: string; - isPrivate: boolean; - commits: HistoryCommit[]; - threads: HistoryThread[]; - symbols: HistorySymbol[]; - sources: HistorySource[]; - autoApprove: boolean; - agents?: string[]; -}): RepoDigest { - const byPath = new Map(); - for (const s of input.symbols) { - const bucket = byPath.get(s.path); - if (bucket) bucket.push(s.id); - else byPath.set(s.path, [s.id]); - } - - return { - provider: "github", - owner: input.owner, - name: input.name, - head_sha: input.headSha, - default_branch: input.defaultBranch, - is_private: input.isPrivate, - commits: input.commits.map((c) => ({ - sha: c.sha, - subject: c.subject, - body: c.body, - files: c.files, - // Capped per commit: a sweeping refactor touches hundreds of files, and - // listing every symbol in all of them would drown the message it belongs to. - symbols: c.files.flatMap((f) => byPath.get(f) ?? []).slice(0, 20), - })), - threads: input.threads.map((t) => ({ - number: t.number, - title: t.title, - body: t.body, - merge_sha: t.mergeSha, - comments: t.comments, - })), - symbols: input.symbols, - sources: input.sources, - auto_approve: input.autoApprove, - ...(input.agents?.length ? { agents: input.agents } : {}), - }; -} - -/** - * Post the digest to a brain. - * - * Returns the job id so the caller can hand it back to whoever asked for the - * build, which is what the onboarding screen polls. - */ -export async function postDigest( - baseUrl: string, - brainId: string, - brainToken: string, - digest: RepoDigest, - fetchImpl: Fetch, -): Promise<{ jobId: string }> { - const res = await fetchImpl(`${baseUrl.replace(/\/+$/, "")}/api/brains/${encodeURIComponent(brainId)}/repo`, { - method: "POST", - headers: { - authorization: `Bearer ${brainToken}`, - "content-type": "application/json", - accept: "application/json", - }, - body: JSON.stringify(digest), - }); - const text = await res.text(); - if (!res.ok) { - throw new Error(`brain ingest failed: ${res.status} ${text.slice(0, 300)}`); - } - const parsed = JSON.parse(text) as { job_id?: string }; - return { jobId: String(parsed.job_id ?? "") }; -} diff --git a/src/app/identity.ts b/src/app/identity.ts index 526d9ad6d..06de2968b 100644 --- a/src/app/identity.ts +++ b/src/app/identity.ts @@ -66,161 +66,6 @@ interface CacheEntry { * Tokens last an hour and a busy repository can fire a dozen webhooks a minute; * without the cache every one of them spends a round trip and a signature. */ -/** - * The installation id for one repository. - * - * Every existing caller gets the id from a webhook payload, because every - * existing caller is reacting to one. A request that names a repository instead - * has to look it up, and this is the only endpoint that answers it: it is - * App-JWT authed, so it works before any installation token exists. - * - * Returns null when the App is not installed on the repository — which is the - * expected answer for "the user pasted a repo we cannot see", not an error. - */ -/** - * Headers for a GitHub REST call, with or without a token. - * - * An empty token means anonymous, and anonymous has to mean NO authorization - * header — `Bearer ` with nothing after it is a 401, not a fallback. That - * distinction is the whole reason this is a function: a public repository read - * without an installation goes down exactly this path. - */ -export function ghHeaders(token: string): Record { - const headers: Record = { - accept: "application/vnd.github+json", - "user-agent": "graft-app", - }; - if (token) headers.authorization = `Bearer ${token}`; - return headers; -} - -/** - * An installation token for reading PUBLIC repositories the App is not - * installed on. - * - * GitHub answers any public resource to any installation token, at 5,000 - * requests an hour. Anonymous answers the same resources at 60 an hour for the - * whole box, which one repository's pull-request walk spends in a minute — so - * without this, the second public repo of the hour reads as "we cannot see it". - * - * `owner` is our own account, so this borrows OUR installation's rate limit - * rather than some customer's. Falls back to the first installation, and to - * null when the App has none, in which case the caller reads anonymously. - */ -export async function publicReadInstallation( - creds: AppCredentials, - owner: string, - fetchImpl: Fetch, - nowMs: number = Date.now(), - api = "https://api.github.com", -): Promise { - const res = await fetchImpl(`${api}/app/installations?per_page=100`, { - headers: { - authorization: `Bearer ${appJwt(creds, nowMs)}`, - accept: "application/vnd.github+json", - "user-agent": "graft-app", - }, - }); - if (!res.ok) return null; - const list = JSON.parse(await res.text()) as Array<{ id?: number; account?: { login?: string } }>; - if (!Array.isArray(list) || list.length === 0) return null; - const wanted = owner.toLowerCase(); - const mine = list.find((i) => (i.account?.login ?? "").toLowerCase() === wanted); - return (mine ?? list[0]).id ?? null; -} - -export async function installationFor( - creds: AppCredentials, - owner: string, - repo: string, - fetchImpl: Fetch, - nowMs: number = Date.now(), - api = "https://api.github.com", -): Promise { - const res = await fetchImpl(`${api}/repos/${owner}/${repo}/installation`, { - headers: { - authorization: `Bearer ${appJwt(creds, nowMs)}`, - accept: "application/vnd.github+json", - "user-agent": "graft-app", - }, - }); - if (res.status === 404) return null; - if (!res.ok) { - const body = await res.text(); - throw new Error(`installation lookup for ${owner}/${repo} failed: ${res.status} ${body.slice(0, 200)}`); - } - const parsed = JSON.parse(await res.text()) as { id?: number }; - return typeof parsed.id === "number" ? parsed.id : null; -} - -/** - * Why a repository is unreachable, and where to send the user to fix it. - * - * Two failures look identical from `/repos/{o}/{r}/installation` — both 404 — - * and they need different sentences and different links. Either the App is not - * on the account at all, so they install it; or it is installed and this - * repository simply is not in its selected list, so they edit that list. Telling - * someone to install an App they already installed is the kind of dead end that - * ends the session. - * - * ownerId is the account's numeric id, which preselects the right account on - * GitHub's install screen. There is no equivalent for preselecting a repository; - * GitHub owns that checkbox list. - */ -export interface RepoAccessGap { - reason: "not_installed" | "repo_not_selected"; - ownerId: number | null; - installationId: number | null; -} - -/** Which of the two gaps applies to owner/repo. */ -export async function repoAccessGap( - creds: AppCredentials, - owner: string, - fetchImpl: Fetch, - nowMs: number = Date.now(), - api = "https://api.github.com", -): Promise { - const headers = { - authorization: `Bearer ${appJwt(creds, nowMs)}`, - accept: "application/vnd.github+json", - "user-agent": "graft-app", - }; - // An account is an org or a user and the endpoints differ, so try the org one - // and fall back rather than making the caller know which it is. - for (const path of [`/orgs/${owner}/installation`, `/users/${owner}/installation`]) { - try { - const res = await fetchImpl(`${api}${path}`, { headers }); - if (!res.ok) continue; - const parsed = JSON.parse(await res.text()) as { id?: number; account?: { id?: number } }; - return { - reason: "repo_not_selected", - ownerId: parsed.account?.id ?? null, - installationId: parsed.id ?? null, - }; - } catch { - // Try the other shape; a network failure here only costs a less specific - // message, never the whole answer. - } - } - return { reason: "not_installed", ownerId: await ownerIdFor(owner, fetchImpl, api), installationId: null }; -} - -/** The account's numeric id, for preselecting it on the install screen. */ -async function ownerIdFor(owner: string, fetchImpl: Fetch, api: string): Promise { - try { - // Unauthenticated: a public account's id is public, and this runs on a path - // where the App has no installation token to use anyway. - const res = await fetchImpl(`${api}/users/${owner}`, { - headers: { accept: "application/vnd.github+json", "user-agent": "graft-app" }, - }); - if (!res.ok) return null; - return (JSON.parse(await res.text()) as { id?: number }).id ?? null; - } catch { - return null; - } -} - export class InstallationTokens { private readonly cache = new Map(); diff --git a/src/app/main.ts b/src/app/main.ts index 745f3f03d..381bb7ac4 100644 --- a/src/app/main.ts +++ b/src/app/main.ts @@ -46,14 +46,6 @@ const { server, queue } = createApp({ // server that will not start reviews nothing at all, which is strictly worse // than one whose old links go stale. pageDir: process.env.GRAFT_PAGE_DIR, - // Optional for the same reason as pageDir: unset simply disables - // `POST /brain/build`, and a deployment that only reviews pull requests wants - // exactly that. The route is the one place the app writes to the platform, so - // it stays off until someone deliberately turns it on. - brainBuildSecret: process.env.GRAFT_BRAIN_BUILD_SECRET, - brainBaseUrl: process.env.GRAFT_BRAIN_URL, - publicToken: process.env.GRAFT_PUBLIC_GITHUB_TOKEN || process.env.GITHUB_TOKEN, - publicOwner: process.env.GRAFT_PUBLIC_INSTALLATION_OWNER, }); /** diff --git a/src/app/server.ts b/src/app/server.ts index b23553bff..3ddec8ede 100644 --- a/src/app/server.ts +++ b/src/app/server.ts @@ -17,11 +17,8 @@ * everything in here is I/O again. */ import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; -import { timingSafeEqual } from "node:crypto"; import { jobKey, reviewJobFor, type ReviewJob } from "./events.js"; import { InstallationTokens, verifySignature, type AppCredentials, type Fetch } from "./identity.js"; -import { buildRepoIntoBrain, checkRepoAccess, RepoNotAccessibleError, type BrainBuildJob } from "./brain-build.js"; -import { buildRepoInChildProcess } from "./brain-build-process.js"; import { PageStore } from "./pages.js"; import { WorkQueue } from "./queue.js"; import { reviewInChildProcess } from "./review-process.js"; @@ -33,12 +30,6 @@ const MAX_BODY_BYTES = 2 * 1024 * 1024; /** Seams for tests: nothing here has a default that touches the network. */ export interface AppSeams { fetch?: Fetch; - /** Swapped out so a test can assert what the route handed the builder without - * cloning a repository or reaching GitHub. */ - brainBuild?: typeof buildRepoIntoBrain; - /** Swapped out so a test can assert the access route's answers without - * reaching GitHub. */ - brainAccess?: typeof checkRepoAccess; /** Swapped out to assert what the queue was handed, without a clone — and, being * called in-process, without the fork the default reviewer does. */ review?: typeof reviewPullRequest; @@ -47,19 +38,6 @@ export interface AppSeams { export interface AppConfig extends AppCredentials { webhookSecret: string; - /** Shared secret for `POST /brain/build`. That route is called by the - * platform, not by GitHub, so it cannot use the webhook's payload signature — - * it takes a bearer token instead. Unset disables the route entirely, which - * is the right default for a deployment that only reviews pull requests. */ - brainBuildSecret?: string; - /** Platform base URL the digest is posted to. Defaults to production. */ - brainBaseUrl?: string; - /** Token for reading public repositories the App is not installed on. - * Optional — without it the read borrows one of our own installation's - * tokens, and only falls back to anonymous if the App has none. */ - publicToken?: string; - /** Whose installation to borrow for that. Defaults to ours. */ - publicOwner?: string; /** Public origin, for the links put in comments, e.g. https://graft.example.com */ publicUrl: string; /** Where pages are kept, so a restart does not strand the links already posted. */ @@ -150,105 +128,6 @@ export function createApp( return send(res, 202, "application/json", JSON.stringify({ queued: true })); } - // Can we read this repository? The cheap question, answered on its own. - // - // Same secret as /brain/build, because it is the same caller and the same - // trust. Kept separate from it because it costs a second where the build - // costs minutes, and the platform needs the answer before it can decide - // which screen to show. - if (req.method === "POST" && url.pathname === "/brain/access") { - if (!config.brainBuildSecret) return send(res, 404, "text/plain", "not found"); - if (!bearerMatches(header(req, "authorization"), config.brainBuildSecret)) { - return send(res, 401, "text/plain", "unauthorized"); - } - const body = await readBody(req); - if (body === null) return send(res, 413, "text/plain", "payload too large"); - let ask: { owner?: string; repo?: string }; - try { - ask = JSON.parse(body) as { owner?: string; repo?: string }; - } catch { - return send(res, 400, "text/plain", "bad json"); - } - if (!ask.owner || !ask.repo) { - return send(res, 400, "application/json", JSON.stringify({ error: "owner and repo are required" })); - } - try { - const access = await (seams.brainAccess ?? checkRepoAccess)({ owner: ask.owner, repo: ask.repo }, { - creds: config, - fetch: fetchImpl, - api: config.api, - publicToken: config.publicToken, - publicOwner: config.publicOwner, - log, - now: seams.now, - }); - // 200 either way: "we cannot see it" is an answer, not a failure, and - // the platform turns it into a choice rather than an error. - return send(res, 200, "application/json", JSON.stringify(access)); - } catch (e) { - const msg = e instanceof Error ? e.message : String(e); - log(`brain access ${ask.owner}/${ask.repo} failed: ${msg}`); - return send(res, 502, "application/json", JSON.stringify({ error: msg })); - } - } - - // Build one repository into a brain, on demand. Called by the platform when - // someone pastes a repository URL during onboarding — there is no webhook - // behind it, so it authenticates with a bearer token rather than a payload - // signature. - if (req.method === "POST" && url.pathname === "/brain/build") { - if (!config.brainBuildSecret) return send(res, 404, "text/plain", "not found"); - // Compared in constant time for the same reason verifySignature is: this - // is a long-lived shared secret, and a timing oracle on it is worth more - // to an attacker than one on a per-payload signature. - if (!bearerMatches(header(req, "authorization"), config.brainBuildSecret)) { - return send(res, 401, "text/plain", "unauthorized"); - } - const body = await readBody(req); - if (body === null) return send(res, 413, "text/plain", "payload too large"); - let job: BrainBuildJob; - try { - job = JSON.parse(body) as BrainBuildJob; - } catch { - return send(res, 400, "text/plain", "bad json"); - } - // brainId/brainToken are optional: without them the digest comes back in - // the response for the caller to ingest itself. See BrainBuildJob. - if (!job.owner || !job.repo) { - return send(res, 400, "application/json", JSON.stringify({ error: "owner and repo are required" })); - } - // Synchronous on purpose: the caller is a person waiting on a screen, and - // the platform's own import job is what makes the SLOW half (extraction and - // placement) asynchronous. This half is a shallow clone and a graph build. - try { - // The deployment's platform URL wins over anything a caller sends, so a - // request cannot redirect a repository's history to another host. - if (config.brainBaseUrl) job.brainBaseUrl = config.brainBaseUrl; - // Out of this process by default. A read is minutes of blocking work - // and the App has to keep answering — including the one-second access - // question the next person's onboarding is waiting on. A test that - // injects its own builder still runs in-process, which is what a test - // wants. - const built = await (seams.brainBuild ?? buildRepoInChildProcess)(job, { - creds: config, - fetch: fetchImpl, - api: config.api, - publicToken: config.publicToken, - publicOwner: config.publicOwner, - log, - now: seams.now, - }); - return send(res, 202, "application/json", JSON.stringify(built)); - } catch (e) { - if (e instanceof RepoNotAccessibleError) { - return send(res, 404, "application/json", JSON.stringify({ error: e.message, ...e.gap })); - } - const msg = e instanceof Error ? e.message : String(e); - log(`brain build ${job.owner}/${job.repo} failed: ${msg}`); - return send(res, 502, "application/json", JSON.stringify({ error: msg })); - } - } - return send(res, 404, "text/plain", "not found"); } @@ -261,18 +140,6 @@ const header = (req: IncomingMessage, name: string): string | undefined => { return Array.isArray(v) ? v[0] : v; }; -/** Constant-time `Authorization: Bearer ` check. */ -function bearerMatches(headerValue: string | undefined, secret: string): boolean { - const prefix = "Bearer "; - if (!headerValue || !headerValue.startsWith(prefix)) return false; - const given = Buffer.from(headerValue.slice(prefix.length)); - const want = Buffer.from(secret); - // timingSafeEqual throws on a length mismatch, which is itself a leak of the - // secret's length — but one byte of length is not worth a branch that skips - // the comparison, and the lengths here are fixed by deployment config. - return given.length === want.length && timingSafeEqual(given, want); -} - function send(res: ServerResponse, status: number, type: string, body: string, extra: Record = {}): void { res.writeHead(status, { "content-type": type, ...extra }); res.end(body); diff --git a/src/app/sources.ts b/src/app/sources.ts deleted file mode 100644 index b085145d2..000000000 --- a/src/app/sources.ts +++ /dev/null @@ -1,492 +0,0 @@ -/** - * Everything in a repository that is already a rule, or nearly one. - * - * Commit messages and pull-request discussion say what a team decided in - * passing. These are the places where they wrote it down on purpose: the - * instruction files they hand their own agents, the decision records, the linter - * config, the ownership rules. Higher precision per byte than any amount of - * history, and most of it is a file read. - * - * Every reader returns text, never source code, and every one is best-effort: - * a repo with none of these still builds a brain from its history. - */ -import { readFileSync, existsSync, readdirSync, statSync } from "node:fs"; -import { join, relative } from "node:path"; -import { spawnSync } from "node:child_process"; -import { ghHeaders, type Fetch } from "./identity.js"; - -/** What kind of thing a source is, so extraction can weigh it. */ -export type SourceKind = - | "agent_instructions" - | "decision_doc" - | "lint_config" - | "ci_config" - | "codeowners" - | "branch_protection" - | "revert" - | "test_name" - | "declined_issue"; - -/** One rule-bearing artefact. */ -export interface HistorySource { - kind: SourceKind; - /** Repo-relative path, or an API-ish label like "branch protection: main". */ - path: string; - text: string; - /** Deep link, where one exists (an issue, a reverted commit). */ - url?: string; -} - -/** Per-file and total caps, so one enormous doc cannot crowd out everything. */ -const MAX_FILE_CHARS = 24_000; -const MAX_TOTAL_CHARS = 300_000; -const MAX_SOURCES = 120; - -function readText(root: string, rel: string): string | null { - try { - const p = join(root, rel); - if (!existsSync(p) || !statSync(p).isFile()) return null; - const raw = readFileSync(p, "utf8"); - const trimmed = raw.trim(); - if (!trimmed) return null; - return trimmed.length > MAX_FILE_CHARS ? `${trimmed.slice(0, MAX_FILE_CHARS)}\n[…truncated]` : trimmed; - } catch { - return null; - } -} - -/** Files directly under `rel`, one level deep, matching `test`. */ -function filesIn(root: string, rel: string, test: (name: string) => boolean): string[] { - try { - const dir = join(root, rel); - if (!existsSync(dir) || !statSync(dir).isDirectory()) return []; - return readdirSync(dir) - .filter(test) - .map((n) => join(rel, n)) - .sort(); - } catch { - return []; - } -} - -/** - * Instruction files the team already wrote for coding agents. - * - * The single highest-precision source in a repository: someone sat down and - * wrote imperative rules for a machine to follow. Nothing has to be inferred. - * - * Graft's own managed blocks are stripped first. Without that the brain reads - * back graft's instructions about using graft, and then graft carries them into - * the same file — a loop that fills the rulebook with itself. - */ -export function readAgentInstructions(root: string): HistorySource[] { - const candidates = [ - "CLAUDE.md", - "AGENTS.md", - "GEMINI.md", - ".github/copilot-instructions.md", - ".windsurf/rules/graft.md", - ...filesIn(root, ".cursor/rules", (n) => n.endsWith(".mdc") || n.endsWith(".md")), - // The same files one folder down: a monorepo keeps its frontend's rules in - // web/CLAUDE.md, and those are as deliberate as the root's. - ...folderInstructionFiles(root), - // Skills a team wrote for its agents. Graft's own is left out for the same - // reason its managed blocks are: the brain must not learn graft from graft. - ...skillFiles(root), - ]; - const out: HistorySource[] = []; - const seen = new Set(); - for (const rel of candidates) { - const key = rel.replace(/\\/g, "/"); - if (seen.has(key)) continue; - seen.add(key); - const text = stripManagedBlocks(readText(root, rel) ?? ""); - if (text.trim()) out.push({ kind: "agent_instructions", path: key, text }); - } - return out; -} - -/** Instruction files below the root that are worth reading as instructions. */ -const FOLDER_INSTRUCTION_NAMES = new Set(["CLAUDE.md", "AGENTS.md"]); -/** How deep below the root a folder instruction file may sit (a/b/c/d/CLAUDE.md). */ -const MAX_FOLDER_DEPTH = 4; -/** Most folder instruction files, and most skills, carried per push. */ -export const MAX_FOLDER_FILES = 30; -export const MAX_SKILL_FILES = 30; - -/** Directories never searched for instruction files: dependencies, build - * output, graft's own cache. Dot-directories are skipped as a class. */ -const SKIP_INSTRUCTION_DIRS = new Set([ - "node_modules", - "dist", - "build", - "out", - "vendor", - "graft", - "target", - "coverage", - "__pycache__", - "venv", - "bower_components", -]); - -function skippedSegment(seg: string): boolean { - return seg.startsWith(".") || SKIP_INSTRUCTION_DIRS.has(seg); -} - -/** - * `**\/CLAUDE.md` and `**\/AGENTS.md` below the root, shallowest first. - * - * Asked of git first, because git already knows what the repository ignores: - * a CLAUDE.md inside an ignored folder is someone's scratch, not the team's. - * The walk is the fallback for a checkout git cannot answer for. - */ -export function folderInstructionFiles(root: string, max = MAX_FOLDER_FILES): string[] { - const keep = (rel: string): boolean => { - const parts = rel.split("/"); - const name = parts[parts.length - 1] ?? ""; - const dirs = parts.slice(0, -1); - if (!FOLDER_INSTRUCTION_NAMES.has(name)) return false; - if (dirs.length === 0 || dirs.length > MAX_FOLDER_DEPTH) return false; - return !dirs.some(skippedSegment); - }; - let found: string[] | null = null; - const listed = git(root, ["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", "*CLAUDE.md", "*AGENTS.md"]); - if (listed !== null) found = [...new Set(listed.split("\0").filter(Boolean))].filter(keep); - if (found === null) found = walkFor(root, keep); - const depth = (p: string) => p.split("/").length; - return found - .filter((rel) => existsSync(join(root, rel))) - .sort((a, b) => depth(a) - depth(b) || a.localeCompare(b)) - .slice(0, max); -} - -/** The filesystem walk behind folderInstructionFiles, for a tree git cannot list. */ -function walkFor(root: string, keep: (rel: string) => boolean): string[] { - const out: string[] = []; - const visit = (rel: string, depth: number) => { - if (depth > MAX_FOLDER_DEPTH) return; - let names: string[]; - try { - names = readdirSync(join(root, rel)); - } catch { - return; - } - for (const name of names) { - const child = rel ? `${rel}/${name}` : name; - let isDir = false; - try { - isDir = statSync(join(root, child)).isDirectory(); - } catch { - continue; - } - if (isDir) { - if (!skippedSegment(name)) visit(child, depth + 1); - } else if (depth > 0 && keep(child)) { - out.push(child); - } - } - }; - visit("", 0); - return out; -} - -/** `.claude/skills//SKILL.md`, graft's own skill excluded. */ -export function skillFiles(root: string, max = MAX_SKILL_FILES): string[] { - const dir = join(root, ".claude", "skills"); - let names: string[]; - try { - names = readdirSync(dir).sort(); - } catch { - return []; - } - const out: string[] = []; - for (const name of names) { - if (name === "graft" || name.startsWith(".")) continue; - const rel = `.claude/skills/${name}/SKILL.md`; - try { - if (!statSync(join(root, rel)).isFile()) continue; - } catch { - continue; - } - out.push(rel); - if (out.length >= max) break; - } - return out; -} - -/** Remove graft's own fenced regions (trail's too), so the brain never learns graft from graft. */ -function stripManagedBlocks(text: string): string { - return text - .replace(/[\s\S]*?/g, "") - .trim(); -} - -/** Architecture decision records and the docs that read like them. */ -export function readDecisionDocs(root: string): HistorySource[] { - const mdIn = (dir: string) => filesIn(root, dir, (n) => n.endsWith(".md")); - const candidates = [ - "ARCHITECTURE.md", - "CONTRIBUTING.md", - "docs/ARCHITECTURE.md", - "docs/CONTRIBUTING.md", - ...mdIn("docs/adr"), - ...mdIn("docs/decisions"), - ...mdIn("adr"), - ]; - const out: HistorySource[] = []; - for (const rel of candidates) { - const text = readText(root, rel); - if (text) out.push({ kind: "decision_doc", path: rel, text }); - } - return out; -} - -/** - * Rules the team already encoded for a machine: linters, formatters, and the - * checks CI insists on. Nothing to infer — extraction only has to phrase them. - */ -export function readCodifiedRules(root: string): HistorySource[] { - const lint = [ - ".golangci.yml", - ".golangci.yaml", - ".eslintrc.json", - ".eslintrc.js", - "eslint.config.js", - "eslint.config.mjs", - "ruff.toml", - ".ruff.toml", - ".prettierrc", - ".prettierrc.json", - "commitlint.config.cjs", - "commitlint.config.js", - ".editorconfig", - ]; - const out: HistorySource[] = []; - for (const rel of lint) { - const text = readText(root, rel); - if (text) out.push({ kind: "lint_config", path: rel, text }); - } - for (const rel of filesIn(root, ".github/workflows", (n) => n.endsWith(".yml") || n.endsWith(".yaml"))) { - const text = readText(root, rel); - if (text) out.push({ kind: "ci_config", path: rel, text }); - } - return out; -} - -/** Who must approve what. A hard constraint, stated as a file. */ -export function readCodeowners(root: string): HistorySource[] { - for (const rel of [".github/CODEOWNERS", "CODEOWNERS", "docs/CODEOWNERS"]) { - const text = readText(root, rel); - if (text) return [{ kind: "codeowners", path: rel, text }]; - } - return []; -} - -/** Run git in `root`, or null when it fails. */ -function git(root: string, args: string[]): string | null { - const res = spawnSync("git", ["-c", "core.quotePath=false", ...args], { - cwd: root, - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - maxBuffer: 64 * 1024 * 1024, - }); - if (res.error || res.status !== 0 || typeof res.stdout !== "string") return null; - return res.stdout; -} - -/** How many reverts to carry. */ -const MAX_REVERTS = 40; - -/** - * Commits that undid another commit. - * - * "We tried this and took it back" is a rule available from nowhere else, and - * it is the one kind of history that says what NOT to do. The body of a revert - * usually names the commit it undoes and, when the author bothered, why. - */ -export function readReverts(root: string, max = MAX_REVERTS): HistorySource[] { - const out = git(root, ["log", "--grep=^Revert", "-n", String(max), "--format=%H%x02%s%x02%b%x01"]); - if (!out) return []; - const sources: HistorySource[] = []; - for (const record of out.split("\x01")) { - if (!record.trim()) continue; - const [sha = "", subject = "", body = ""] = record.split("\x02"); - if (!sha.trim() || !subject.trim()) continue; - sources.push({ - kind: "revert", - path: `revert ${sha.trim().slice(0, 12)}`, - text: [subject.trim(), body.trim()].filter(Boolean).join("\n"), - }); - } - return sources; -} - -/** How many test names to carry. */ -const MAX_TEST_NAMES = 300; - -/** - * Test names, which are invariants someone wrote deliberately. - * - * `never retries on a card decline` is a rule stated as an assertion. Read from - * the graph's own symbols rather than by re-parsing, so this costs nothing: - * a test function is already a node. - * - * Only names that read like a sentence are kept. `TestFoo` is a label; - * `rejects an expired token` is a claim about how the system must behave. - */ -export function readTestNames(symbolNames: Array<{ name: string; path: string }>, max = MAX_TEST_NAMES): HistorySource[] { - const out: HistorySource[] = []; - for (const s of symbolNames) { - if (!/(^|[/.])(test|spec)[s]?[/.]|_test\.|\.test\.|\.spec\./i.test(s.path)) continue; - const sentence = humanizeTestName(s.name); - if (!sentence) continue; - out.push({ kind: "test_name", path: s.path, text: sentence }); - if (out.length === max) break; - } - return out; -} - -/** - * A test's name as a claim, or null when it is only a label. - * - * Requires three words after the prefix is stripped: fewer than that - * ("TestParse", "handles errors") states no invariant worth a rule, and letting - * those through fills the brain with noise that reads like content. - */ -function humanizeTestName(name: string): string | null { - let s = name.replace(/^(Test|it|test|should|describe)[_\s]*/i, ""); - // TestRetriesThreeTimes → Retries Three Times - if (/^[A-Za-z]+$/.test(s) && /[a-z][A-Z]/.test(s)) s = s.replace(/([a-z0-9])([A-Z])/g, "$1 $2"); - s = s.replace(/[_-]+/g, " ").replace(/\s+/g, " ").trim(); - if (s.split(" ").length < 3) return null; - return s.toLowerCase(); -} - -/** How many declined issues to carry. */ -const MAX_DECLINED = 40; - -/** - * Issues closed as not planned. - * - * A decision NOT to build something, which is half of what a codebase's rules - * are and is invisible everywhere else — nothing was committed, so no commit - * message records it. - */ -export async function readDeclinedIssues( - owner: string, - repo: string, - token: string, - fetchImpl: Fetch, - api = "https://api.github.com", - max = MAX_DECLINED, -): Promise { - try { - const res = await fetchImpl(`${api}/repos/${owner}/${repo}/issues?state=closed&per_page=100&sort=updated`, { - headers: ghHeaders(token), - }); - if (!res.ok) return []; - const items = JSON.parse(await res.text()) as Array<{ - number?: number; - title?: string; - body?: string | null; - html_url?: string; - state_reason?: string | null; - pull_request?: unknown; - }>; - if (!Array.isArray(items)) return []; - const out: HistorySource[] = []; - for (const i of items) { - // The issues endpoint returns pull requests too; they are already mined as - // threads and would be counted twice. - if (i.pull_request) continue; - if (i.state_reason !== "not_planned") continue; - if (!i.title?.trim()) continue; - out.push({ - kind: "declined_issue", - path: `issue #${i.number}`, - text: [i.title.trim(), (i.body ?? "").trim()].filter(Boolean).join("\n").slice(0, MAX_FILE_CHARS), - url: i.html_url, - }); - if (out.length === max) break; - } - return out; - } catch { - return []; - } -} - -/** Required checks and reviewers on the default branch. */ -export async function readBranchProtection( - owner: string, - repo: string, - branch: string, - token: string, - fetchImpl: Fetch, - api = "https://api.github.com", -): Promise { - if (!branch) return []; - try { - const res = await fetchImpl(`${api}/repos/${owner}/${repo}/branches/${encodeURIComponent(branch)}/protection`, { - headers: ghHeaders(token), - }); - // 404 simply means the branch is not protected, which is not a failure. - if (!res.ok) return []; - const p = JSON.parse(await res.text()) as { - required_status_checks?: { contexts?: string[] }; - required_pull_request_reviews?: { required_approving_review_count?: number; require_code_owner_reviews?: boolean }; - allow_force_pushes?: { enabled?: boolean }; - }; - const lines: string[] = []; - const checks = p.required_status_checks?.contexts ?? []; - if (checks.length) lines.push(`Checks that must pass before merging to ${branch}: ${checks.join(", ")}.`); - const reviews = p.required_pull_request_reviews; - if (reviews?.required_approving_review_count) { - lines.push(`${reviews.required_approving_review_count} approving review(s) required to merge to ${branch}.`); - } - if (reviews?.require_code_owner_reviews) lines.push(`A code owner must approve changes to ${branch}.`); - if (p.allow_force_pushes?.enabled === false) lines.push(`Force pushes to ${branch} are not allowed.`); - if (!lines.length) return []; - return [{ kind: "branch_protection", path: `branch protection: ${branch}`, text: lines.join("\n") }]; - } catch { - return []; - } -} - -/** - * Trim the collected sources to a budget, best kinds first. - * - * The order is the argument of this whole module: a file the team wrote to - * instruct an agent outranks a decision doc, which outranks a config, which - * outranks anything inferred. When the budget runs out it should run out on the - * weakest evidence. - */ -const KIND_RANK: SourceKind[] = [ - "agent_instructions", - "decision_doc", - "codeowners", - "branch_protection", - "lint_config", - "ci_config", - "revert", - "declined_issue", - "test_name", -]; - -export function budgetSources(sources: HistorySource[], maxChars = MAX_TOTAL_CHARS, maxCount = MAX_SOURCES): HistorySource[] { - const ranked = [...sources].sort((a, b) => KIND_RANK.indexOf(a.kind) - KIND_RANK.indexOf(b.kind)); - const out: HistorySource[] = []; - let spent = 0; - for (const s of ranked) { - if (out.length >= maxCount) break; - if (spent + s.text.length > maxChars) continue; - out.push(s); - spent += s.text.length; - } - return out; -} - -/** Repo-relative path, for readers handed an absolute one. */ -export function relPath(root: string, abs: string): string { - return relative(root, abs) || abs; -} diff --git a/test/app-brain-build-process.test.ts b/test/app-brain-build-process.test.ts deleted file mode 100644 index 16b21122b..000000000 --- a/test/app-brain-build-process.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -/** - * The one failure in forking a read that no protocol test would catch: the - * entry path being wrong, or the read stack failing to load in a child. - * - * Everything else about the child protocol is the reviewer's, tested next door - * in app-review-process.test.ts. What is specific here is that a deployed App - * forks THIS module, and that importing it pulls in the same nine tree-sitter - * native addons at graph/extract.ts's module scope. A grammar that cannot load - * in a forked child dies during import — silently, at runtime, on every read. - */ -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { existsSync } from "node:fs"; -import { fork } from "node:child_process"; -import { brainBuildWorkerEntry } from "../src/app/brain-build-process.js"; - -const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); - -test("the brain build worker is where a deployed App looks for it, and it comes up", async () => { - const entry = brainBuildWorkerEntry(); - assert.ok(existsSync(entry), `${entry} must exist — it is what a deployed App forks`); - - const child = fork(entry, ["load-probe"], { stdio: ["ignore", "pipe", "pipe", "ipc"] }); - let stderr = ""; - child.stderr?.on("data", (c) => { - stderr += String(c); - }); - const ended = new Promise((resolve) => { - child.on("exit", (code, signal) => resolve(signal ? `signal ${signal}` : `code ${code}`)); - }); - try { - // No read runs — a read wants a clone and a token. Still waiting for work - // after this long means the stack imported cleanly, which is the assertion. - const outcome = await Promise.race([ended, sleep(2500).then(() => "still waiting for work")]); - assert.equal(outcome, "still waiting for work", `the brain build worker did not come up: ${stderr}`); - } finally { - child.kill("SIGKILL"); - } -}); diff --git a/test/app-sources-context.test.ts b/test/app-sources-context.test.ts deleted file mode 100644 index bc04759bc..000000000 --- a/test/app-sources-context.test.ts +++ /dev/null @@ -1,81 +0,0 @@ -/** - * The early upload carries every file a team wrote for its agents, not only - * the root ones: folder CLAUDE.md / AGENTS.md files and Claude skills. What - * matters is which ones — shallow enough, outside dependencies and build - * output, not ignored, never graft's own — and that there is a cap. - */ -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { mkdirSync, writeFileSync } from 'node:fs'; -import { spawnSync } from 'node:child_process'; -import { dirname, join } from 'node:path'; -import { folderInstructionFiles, readAgentInstructions, skillFiles } from '../src/app/sources.js'; -import { tmpRepo } from './helpers.js'; - -function put(root: string, rel: string, text: string): void { - mkdirSync(dirname(join(root, rel)), { recursive: true }); - writeFileSync(join(root, rel), text); -} - -function fixture(): string { - const root = tmpRepo('sources'); - put(root, 'CLAUDE.md', '# root\n\nUse pnpm.\n'); - put(root, 'web/CLAUDE.md', '# web\n\nComponents are PascalCase.\n'); - put(root, 'services/api/AGENTS.md', '# api\n\nHandlers stay thin.\n'); - put(root, 'a/b/c/d/CLAUDE.md', '# depth four\n\nStill read.\n'); - put(root, 'a/b/c/d/e/CLAUDE.md', '# depth five\n\nToo deep.\n'); - put(root, 'node_modules/pkg/CLAUDE.md', '# dependency\n'); - put(root, 'dist/CLAUDE.md', '# build output\n'); - put(root, 'graft/CLAUDE.md', '# graft cache\n'); - put(root, '.hidden/CLAUDE.md', '# dot dir\n'); - put(root, 'scratch/CLAUDE.md', '# ignored by the repo\n'); - put(root, '.gitignore', 'scratch/\n'); - put(root, '.claude/skills/deploy/SKILL.md', '---\nname: deploy\n---\nAlways deploy from main.\n'); - put(root, '.claude/skills/graft/SKILL.md', '---\nname: graft\n---\ngraft teaches graft\n'); - put(root, '.claude/skills/empty/README.md', 'no skill file here\n'); - put(root, 'web/AGENTS.md', 'before\n\ngraft block\n\nafter\n\ntrail block\n\n'); - return root; -} - -test('folder instruction files: shallow ones, outside ignored and generated dirs, shallowest first', () => { - const root = fixture(); - spawnSync('git', ['init', '-q'], { cwd: root }); - const got = folderInstructionFiles(root); - assert.deepEqual(got, ['web/AGENTS.md', 'web/CLAUDE.md', 'services/api/AGENTS.md', 'a/b/c/d/CLAUDE.md']); -}); - -test('folder instruction files are found without git too', () => { - const root = fixture(); - const got = folderInstructionFiles(root); - // No git to read .gitignore, so scratch/ is read; everything else is the same. - assert.ok(got.includes('web/CLAUDE.md') && got.includes('a/b/c/d/CLAUDE.md')); - assert.ok(!got.some((p) => /node_modules|dist|graft\/|\.hidden|\/e\//.test(p)), got.join(', ')); -}); - -test('folder instruction files are capped', () => { - const root = tmpRepo('sources-cap'); - for (let i = 0; i < 40; i++) put(root, `pkg${String(i).padStart(2, '0')}/CLAUDE.md`, `# ${i}\n\nrule ${i}\n`); - assert.equal(folderInstructionFiles(root).length, 30); - assert.equal(folderInstructionFiles(root, 5).length, 5); -}); - -test('skills: every SKILL.md but graft\'s own', () => { - const root = fixture(); - assert.deepEqual(skillFiles(root), ['.claude/skills/deploy/SKILL.md']); -}); - -test('readAgentInstructions carries them all as agent_instructions, graft blocks stripped', () => { - const root = fixture(); - spawnSync('git', ['init', '-q'], { cwd: root }); - const got = readAgentInstructions(root); - const paths = got.map((s) => s.path); - for (const p of ['CLAUDE.md', 'web/CLAUDE.md', 'services/api/AGENTS.md', '.claude/skills/deploy/SKILL.md']) { - assert.ok(paths.includes(p), `${p} missing from ${paths.join(', ')}`); - } - assert.ok(got.every((s) => s.kind === 'agent_instructions')); - const web = got.find((s) => s.path === 'web/AGENTS.md')!; - assert.doesNotMatch(web.text, /graft block/); - assert.doesNotMatch(web.text, /trail block/, 'the block trail init writes is stripped too'); - assert.match(web.text, /before[\s\S]*after/); - assert.ok(!paths.includes('.claude/skills/graft/SKILL.md')); -}); From c43f14565119e336829ef579d7b6a3f797b21459 Mon Sep 17 00:00:00 2001 From: anirudhkumar-nanonets Date: Thu, 15 Oct 2026 15:25:47 +0530 Subject: [PATCH 2/2] fix(ci): pin CodeQL init to v4.38.1, matching analyze --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7760095b2..888512d27 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -26,7 +26,7 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: languages: javascript-typescript