From 03814c1fd55aeb8fb4152db3c76aa457c0527b7e Mon Sep 17 00:00:00 2001 From: Suleiman Shahbari Date: Fri, 21 Aug 2026 23:46:46 +0300 Subject: [PATCH] The adoption pass reads the runs inside its window, not the whole archive (#1607) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cloud-work pass wants the settled web runs of the last 48 hours. It asked the store for every archived run the project has ever had, parsed each one, and then threw away everything outside the window — every ten minutes, forever, while the archive only grows. An archived run is filed as `.json` and an id is the run's start time, so the filename already dates the record. `listAgents` now takes an optional `since` and answers it from the directory listing: a name that parses as one of our ids and is older than the cutoff is skipped before it is opened. On this repo today that is 162 reads per pass down to 4. A name that does not parse as one of our ids is read as before — an id handed in from outside is not a date, and rejecting it unread would hide it from every caller that passes a window. The id is allocated when a run is spawned and `startedAt` written when it first opens its store, so the name can be the older of the two by the length of a spawn; a caller filtering on `startedAt` sees a record leave the window at most that much early, against a window measured in days. The history list, the boot reconcile and every by-id lookup pass no `since` and read the archive whole, as they must. Closes #1607. Co-authored-by: Claude Opus 5 (1M context) --- packages/the-framework/src/cloud-work.test.ts | 12 +++++++ packages/the-framework/src/cloud-work.ts | 12 ++++--- .../src/store/agent-store.test.ts | 27 ++++++++++++++++ .../the-framework/src/store/agent-store.ts | 32 ++++++++++++++++--- 4 files changed, 73 insertions(+), 10 deletions(-) diff --git a/packages/the-framework/src/cloud-work.test.ts b/packages/the-framework/src/cloud-work.test.ts index 2c855278b..9ec578642 100644 --- a/packages/the-framework/src/cloud-work.test.ts +++ b/packages/the-framework/src/cloud-work.test.ts @@ -153,6 +153,18 @@ test('runs outside the pass: non-web, still running, no anchor, already adopted, assert.deepEqual(git.calls, [], 'nothing waiting means not even a fetch') }) +test('the archive is only asked for the window, so an old history costs no reads (#1607)', async () => { + // The pass runs on a cadence forever while the archive only grows, so the window has to reach + // the store rather than being applied after every record has already been parsed. + const asked: number[] = [] + const { d } = deps([webRun()]) + await adoptCloudWork(CWD, { + ...d, + agents: async (_cwd, since) => (asked.push(since), [webRun()]), + }) + assert.deepEqual(asked, [NOW - CLOUD_ADOPTION_WINDOW_MS]) +}) + test('a run already adopted but still owed its armed PR keeps being asked about (#1601)', async () => { // The session pushed its branch but had not opened a PR when the branch was adopted; a later // pass finds the PR (or opens the armed draft) without re-recording the branch. diff --git a/packages/the-framework/src/cloud-work.ts b/packages/the-framework/src/cloud-work.ts index 1abf11e94..6b59193b8 100644 --- a/packages/the-framework/src/cloud-work.ts +++ b/packages/the-framework/src/cloud-work.ts @@ -2,7 +2,7 @@ import { nodeGitRunner, type GitRunner } from './project.js' import { ghPrsForBranchOrThrow, pickAgentPr, type LinkedPr } from './dashboard/gh.js' import { openRemoteBranchPullRequest, type HandoffResult } from './dashboard/agent-handoff.js' import { agentBranchName } from './branch-names.js' -import { listAgents, startedAtFromAgentId, type AgentMeta, type ArchivePatch } from './store/index.js' +import { listAgents, nodeStoreFs, startedAtFromAgentId, type AgentMeta, type ArchivePatch } from './store/index.js' import { patchArchivedAgentOnDataBranch } from './archived-agent-patch.js' import { errorMessage } from './error-message.js' @@ -51,8 +51,8 @@ export interface CloudWorkDeps { git?: GitRunner /** The branch's full PR history; a listing that fails must throw (default {@link ghPrsForBranchOrThrow}). */ prs?: (cwd: string, branch: string) => Promise - /** The project's run records (default {@link listAgents}). */ - agents?: (cwd: string) => Promise + /** The project's run records, none older than `since` in epoch ms (default {@link listAgents}). */ + agents?: (cwd: string, since: number) => Promise /** Record the adopted branch and PR on the run's archive (default {@link patchArchivedAgentOnDataBranch}). */ patch?: (cwd: string, agentId: string, patch: ArchivePatch, message: string) => Promise /** Open the armed draft PR for a remote-only branch (default {@link openRemoteBranchPullRequest}). */ @@ -146,13 +146,15 @@ async function headsDescendingFrom(git: GitRunner, cwd: string, anchor: string): export async function adoptCloudWork(cwd: string, deps: CloudWorkDeps = {}): Promise { const git = deps.git ?? nodeGitRunner() const prs = deps.prs ?? ghPrsForBranchOrThrow - const agents = deps.agents ?? listAgents + const agents = deps.agents ?? ((project: string, since: number) => listAgents(project, nodeStoreFs(), since)) const patchArchive = deps.patch ?? patchArchivedAgentOnDataBranch const openPr = deps.openPr ?? openRemoteBranchPullRequest const now = deps.now ? deps.now() : Date.now() const result: CloudWorkResult = { adopted: [], failed: [] } - const waiting = waitingRuns(await agents(cwd).catch((): AgentMeta[] => []), now) + // Only the window's runs are asked for: an archive grows without bound, and everything older + // than the window is a record `waitingRuns` would drop anyway — so it is never read (#1607). + const waiting = waitingRuns(await agents(cwd, now - CLOUD_ADOPTION_WINDOW_MS).catch((): AgentMeta[] => []), now) if (waiting.length === 0) return result try { diff --git a/packages/the-framework/src/store/agent-store.test.ts b/packages/the-framework/src/store/agent-store.test.ts index c1c54dbad..f69240436 100644 --- a/packages/the-framework/src/store/agent-store.test.ts +++ b/packages/the-framework/src/store/agent-store.test.ts @@ -724,6 +724,33 @@ test('the history lists every user, and the runs archived before this shipped (# assert.deepEqual((await listAgents(CWD, fs)).map(agent => agent.id), ['r3', 'r2', 'r1']) }) +test('a `since` skips the older runs by filename, without reading them (#1607)', async () => { + // The cost this exists to remove: the adoption poll wants the last 48h and an archive holds + // years, so the read loop spent every pass parsing records its caller was about to discard. + const at = (iso: string) => JSON.stringify({ version: 1, status: 'done', id: agentIdFromStartedAt(iso), startedAt: iso, updatedAt: iso }) + const old = agentIdFromStartedAt('2026-07-04T00:00:00.000Z') + const recent = agentIdFromStartedAt('2026-07-06T00:00:00.000Z') + const fs = memFs({ + [archiveAt(old, 'json')]: at('2026-07-04T00:00:00.000Z'), + [archiveAt(recent, 'json')]: at('2026-07-06T00:00:00.000Z'), + [archiveAt('hand-picked', 'json')]: JSON.stringify({ version: 1, status: 'done', id: 'hand-picked', startedAt: AT, updatedAt: AT }), + }) + const read: string[] = [] + const spy = { ...fs, read: async (path: string) => (read.push(path), fs.read(path)) } + + const listed = await listAgents(CWD, spy, Date.parse('2026-07-05T00:00:00.000Z')) + + assert.deepEqual(listed.map(agent => agent.id).sort(), ['hand-picked', recent].sort()) + assert.equal( + read.some(path => path.includes(old)), + false, + 'the run outside the window is never opened', + ) + // An id that is not one of our timestamps cannot be dated from its name, so it is still read: + // rejecting it unread would hide it from every caller that passes a window. + assert.equal(read.some(path => path.includes('hand-picked')), true) +}) + test('a run archived under both schemes is listed once (#1179)', async () => { // An agent archived before #1179 and re-archived after exists in both places; the history is a list // of sessions, not of files. diff --git a/packages/the-framework/src/store/agent-store.ts b/packages/the-framework/src/store/agent-store.ts index 2c6c3db61..7f5e89f5a 100644 --- a/packages/the-framework/src/store/agent-store.ts +++ b/packages/the-framework/src/store/agent-store.ts @@ -864,15 +864,34 @@ export async function archivedAgentPaths(cwd: string, agentId: string, fs: Store /** Newest run first: an id sorts chronologically, so the id order IS the time order (no parse). */ const byIdDesc = (a: { id: string }, b: { id: string }): number => (a.id < b.id ? 1 : a.id > b.id ? -1 : 0) +/** + * Whether an `.json` archive entry is older than `since` going by its *name* — an id is the + * run's start time, so the filename dates the record and most of a long history can be rejected + * before it is ever read (#1607). + * + * Only a name that parses as one of our ids can reject: an id handed in from outside is not a + * date, and is read like any other. The id is allocated when the run is spawned and `startedAt` + * written when it first opens its store, so the name can be the older of the two by the length of + * a spawn — a caller filtering on `startedAt` sees a record drop out at most that much early. + */ +function namedBefore(name: string, since: number): boolean { + const startedAt = startedAtFromAgentId(name.slice(0, -'.json'.length)) + return startedAt !== undefined && Date.parse(startedAt) < since +} + /** * Every `agents/*.json` archived meta with the path it was read from, torn/half-written entries * skipped. The one home of the archived-history read loop, shared by {@link listAgents} and the * boot reconcile. A missing/unreadable dir throws to the caller, as both callers always let it. + * + * `since` (epoch ms) drops the runs that started before it without reading them; see + * {@link namedBefore}. */ -async function readArchivedMetaEntries(fs: StoreFs, agentsDir: string): Promise> { +async function readArchivedMetaEntries(fs: StoreFs, agentsDir: string, since?: number): Promise> { const entries: Array<{ path: string; meta: AgentMeta }> = [] for (const name of await fs.readdir(agentsDir)) { if (!name.endsWith('.json')) continue + if (since !== undefined && namedBefore(name, since)) continue const path = join(agentsDir, name) try { entries.push({ path, meta: JSON.parse(await fs.read(path)) as AgentMeta }) @@ -898,11 +917,11 @@ function isDeadRunningAgent(meta: AgentMeta | undefined, isAlive: (pid: number) * `agents/` and the close into the user's committed one, so an agent can sit in both places and the * history must show it once. The user directories are searched first, so the committed copy wins. */ -async function readAllArchivedMetaEntries(fs: StoreFs, cwd: string): Promise> { +async function readAllArchivedMetaEntries(fs: StoreFs, cwd: string, since?: number): Promise> { const seen = new Set() const entries: Array<{ path: string; meta: AgentMeta }> = [] for (const agentsDir of await archiveDirs(fs, cwd)) { - for (const entry of await readArchivedMetaEntries(fs, agentsDir).catch(() => [])) { + for (const entry of await readArchivedMetaEntries(fs, agentsDir, since).catch(() => [])) { if (seen.has(entry.meta.id)) continue seen.add(entry.meta.id) entries.push(entry) @@ -915,9 +934,12 @@ async function readAllArchivedMetaEntries(fs: StoreFs, cwd: string): Promise { - const entries = await readAllArchivedMetaEntries(fs, cwd) +export async function listAgents(cwd: string, fs: StoreFs = nodeStoreFs(), since?: number): Promise { + const entries = await readAllArchivedMetaEntries(fs, cwd, since) return entries.map(entry => entry.meta).sort(byIdDesc) }