From 008dd1116f72821d117bfd431137cf96f1d6b0d3 Mon Sep 17 00:00:00 2001 From: Heewon Oh Date: Sun, 25 Oct 2026 18:13:12 +0900 Subject: [PATCH] feat(verify): log sandbox disk usage --- src/verify/diskUsage.ts | 48 +++++++++++++++++++++++ src/verify/git.ts | 64 +++++++++++++++++++++++++++++++ src/verify/runner.test.ts | 13 +++++++ src/verify/runner.ts | 80 +++++++++------------------------------ 4 files changed, 143 insertions(+), 62 deletions(-) create mode 100644 src/verify/diskUsage.ts create mode 100644 src/verify/git.ts diff --git a/src/verify/diskUsage.ts b/src/verify/diskUsage.ts new file mode 100644 index 00000000..0f12f8eb --- /dev/null +++ b/src/verify/diskUsage.ts @@ -0,0 +1,48 @@ +import { execFile } from 'node:child_process'; +import { lstat, readdir } from 'node:fs/promises'; +import { homedir, tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +async function directoryBytes(path: string): Promise { + let entries; + try { entries = await readdir(path); } catch { return 0; } + let bytes = 0; + for (const entry of entries) { + const child = join(path, entry); + try { + const info = await lstat(child); + if (info.isDirectory()) bytes += await directoryBytes(child); + else if (info.isFile()) bytes += info.size; + } catch { /* A concurrent cleanup is expected during a usage sample. */ } + } + return bytes; +} + +export async function logVerifySandboxUsage(root: string, phase: 'create' | 'remove'): Promise { + const [bytes, entries] = await Promise.all([ + directoryBytes(root), readdir(dirname(root)).catch(() => []), + ]); + const live = entries.filter((entry) => entry.startsWith('.openswarm-verify-')).length; + console.log(`[Verify] sandbox usage phase=${phase} bytes=${bytes} live=${live} root=${root}`); +} + +export async function logPeriodicDiskUsage(projectPath: string, scratchRoot?: string): Promise { + const { stdout = '' } = await execFileAsync('git', ['-C', projectPath, 'worktree', 'list', '--porcelain']) + .catch(() => ({ stdout: '' })); + const worktrees = stdout.split('\n').filter((line) => line.startsWith('worktree ')).length; + const [sessionsBytes, verifyBytes] = await Promise.all([ + directoryBytes(join(homedir(), '.openswarm', 'sessions')), + (async () => { + const root = scratchRoot ?? tmpdir(); + let entries: string[]; + try { entries = await readdir(root); } catch { return 0; } + const verifyRoots = entries.filter((entry) => entry.startsWith('.openswarm-verify-')); + return (await Promise.all(verifyRoots.map((entry) => directoryBytes(join(root, entry))))) + .reduce((total, bytes) => total + bytes, 0); + })(), + ]); + console.log(`[Verify] disk usage period=5m repo=${basename(projectPath)} worker_worktrees=${worktrees} sessions_bytes=${sessionsBytes} verify_tmp_bytes=${verifyBytes}`); +} diff --git a/src/verify/git.ts b/src/verify/git.ts new file mode 100644 index 00000000..b2688909 --- /dev/null +++ b/src/verify/git.ts @@ -0,0 +1,64 @@ +import { spawn } from 'node:child_process'; + +export const GIT_TIMEOUT_MS = 30_000; +/** + * The head sandbox is a `--no-hardlinks` clone: a full copy of the object store, + * because the sandbox runs the repository's own test commands and hardlinked + * packs would share inodes with the real checkout. That copy is bounded by disk + * throughput, not by git, and on a loaded host it outgrows the 30 s every other + * git call gets — cgf-portal's 135 MB took 33 s at load 55, and the runner fell + * back to the LLM tester 29 times in one day, which is how `ruff`-red code shipped + * as ready PRs (AGT-4416). The clone gets its own budget. + */ +export const CLONE_TIMEOUT_MS = 10 * 60_000; + +/** The budget a git invocation gets: the sandbox clone's own, everything else the short default. */ +export function gitTimeoutMsFor(args: readonly string[]): number { + return args[0] === 'clone' ? CLONE_TIMEOUT_MS : GIT_TIMEOUT_MS; +} +export async function git(projectPath: string, args: string[]): Promise { + const timeoutMs = gitTimeoutMsFor(args); + return await new Promise((resolveResult, reject) => { + const maxOutputBytes = 4 * 1024 * 1024; + // Checkout hooks belong to the live developer environment. Running them in + // a verification worktree can restore external data/secrets after filtering. + const child = spawn('git', ['-C', projectPath, '-c', 'core.hooksPath=/dev/null', ...args], { + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + }); + let stdout = ''; + let stderr = ''; + let outputBytes = 0; + let settled = false; + let timer: ReturnType; + const fail = (error: Error) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (child.pid && process.platform !== 'win32') { + try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); } + } else child.kill('SIGKILL'); + reject(error); + }; + timer = setTimeout(() => fail(new Error(`git ${args[0] ?? ''} timed out after ${timeoutMs}ms`)), timeoutMs); + const append = (target: 'stdout' | 'stderr', chunk: Buffer) => { + outputBytes += chunk.length; + if (outputBytes > maxOutputBytes) { + fail(new Error(`git ${args[0] ?? ''} output exceeded ${maxOutputBytes} bytes`)); + return; + } + if (target === 'stdout') stdout += chunk.toString('utf8'); + else stderr += chunk.toString('utf8'); + }; + child.stdout.on('data', (chunk: Buffer) => append('stdout', chunk)); + child.stderr.on('data', (chunk: Buffer) => append('stderr', chunk)); + child.on('error', fail); + child.on('close', (code) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (code === 0) resolveResult(stdout.trim()); + else reject(new Error(`git exited with code ${code}: ${stderr.trim()}`)); + }); + }); +} diff --git a/src/verify/runner.test.ts b/src/verify/runner.test.ts index ccbdd212..a77441d6 100644 --- a/src/verify/runner.test.ts +++ b/src/verify/runner.test.ts @@ -41,6 +41,19 @@ afterEach(async () => { }); describe('runVerify', () => { + it('logs verify sandbox bytes and the live sandbox count at creation and cleanup', async () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + try { + await runVerify({ projectPath: repo, commands: [verify('true')], baseRef: 'HEAD', sandboxScratchRoot: root }); + const lines = log.mock.calls.map(([line]) => String(line)).filter((line) => line.includes('[Verify] sandbox usage')); + expect(lines).toHaveLength(2); + expect(lines[0]).toMatch(/phase=create bytes=\d+ live=1/); + expect(lines[1]).toMatch(/phase=remove bytes=\d+ live=1/); + } finally { + log.mockRestore(); + } + }); + it('routes strict verification through the companion and keeps scratch checkouts under its allowed root', async () => { const canonicalScratchRoot = await realpath(root); const execute = vi.fn(async () => ({ diff --git a/src/verify/runner.ts b/src/verify/runner.ts index e78545b0..2d894074 100644 --- a/src/verify/runner.ts +++ b/src/verify/runner.ts @@ -20,25 +20,12 @@ import type { SandboxExecutorSession } from '../sandboxExecutor/protocol.js'; import type { VerifyCommand } from './manifest.js'; import { rebasePythonEnvironment } from './pythonEnvironment.js'; import { resourceAwareTestCommand, testResourceShellPrefix, withTestResourceBudget } from '../support/testResourceBudget.js'; +import { logPeriodicDiskUsage, logVerifySandboxUsage } from './diskUsage.js'; const OUTPUT_TAIL_BYTES = 8 * 1024; const FINGERPRINT_BYTES = 4 * 1024 * 1024; -const GIT_TIMEOUT_MS = 30_000; -/** - * The head sandbox is a `--no-hardlinks` clone: a full copy of the object store, - * because the sandbox runs the repository's own test commands and hardlinked - * packs would share inodes with the real checkout. That copy is bounded by disk - * throughput, not by git, and on a loaded host it outgrows the 30 s every other - * git call gets — cgf-portal's 135 MB took 33 s at load 55, and the runner fell - * back to the LLM tester 29 times in one day, which is how `ruff`-red code shipped - * as ready PRs (AGT-4416). The clone gets its own budget. - */ -export const CLONE_TIMEOUT_MS = 10 * 60_000; - -/** The budget a git invocation gets: the sandbox clone's own, everything else the short default. */ -export function gitTimeoutMsFor(args: readonly string[]): number { - return args[0] === 'clone' ? CLONE_TIMEOUT_MS : GIT_TIMEOUT_MS; -} +import { git, gitTimeoutMsFor, CLONE_TIMEOUT_MS, GIT_TIMEOUT_MS } from './git.js'; +export { gitTimeoutMsFor, CLONE_TIMEOUT_MS }; const execFileAsync = promisify(execFile); /** Dependency manifests the sandbox may observe when comparing base vs head. */ @@ -764,6 +751,7 @@ async function createVerifySandboxRoot(prefix: string, scratchRoot?: string): Pr return await mkdtemp(join(scratchRoot ?? tmpdir(), prefix)); } + async function createHeadSandbox( projectPath: string, commands: VerifyCommand[], @@ -806,59 +794,15 @@ async function createHeadSandbox( // Validate what was actually copied, closing the source validation/copy // race before any repository-controlled command can execute. await validateSandboxSymlinks(project, sharedPaths); + await logVerifySandboxUsage(root, 'create'); return { root, project }; } catch (error) { + await logVerifySandboxUsage(root, 'remove'); await rm(root, { recursive: true, force: true }); throw error; } } -async function git(projectPath: string, args: string[]): Promise { - const timeoutMs = gitTimeoutMsFor(args); - return await new Promise((resolveResult, reject) => { - const maxOutputBytes = 4 * 1024 * 1024; - // Checkout hooks belong to the live developer environment. Running them in - // a verification worktree can restore external data/secrets after filtering. - const child = spawn('git', ['-C', projectPath, '-c', 'core.hooksPath=/dev/null', ...args], { - stdio: ['ignore', 'pipe', 'pipe'], - detached: process.platform !== 'win32', - }); - let stdout = ''; - let stderr = ''; - let outputBytes = 0; - let settled = false; - let timer: ReturnType; - const fail = (error: Error) => { - if (settled) return; - settled = true; - clearTimeout(timer); - if (child.pid && process.platform !== 'win32') { - try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); } - } else child.kill('SIGKILL'); - reject(error); - }; - timer = setTimeout(() => fail(new Error(`git ${args[0] ?? ''} timed out after ${timeoutMs}ms`)), timeoutMs); - const append = (target: 'stdout' | 'stderr', chunk: Buffer) => { - outputBytes += chunk.length; - if (outputBytes > maxOutputBytes) { - fail(new Error(`git ${args[0] ?? ''} output exceeded ${maxOutputBytes} bytes`)); - return; - } - if (target === 'stdout') stdout += chunk.toString('utf8'); - else stderr += chunk.toString('utf8'); - }; - child.stdout.on('data', (chunk: Buffer) => append('stdout', chunk)); - child.stderr.on('data', (chunk: Buffer) => append('stderr', chunk)); - child.on('error', fail); - child.on('close', (code) => { - if (settled) return; - settled = true; - clearTimeout(timer); - if (code === 0) resolveResult(stdout.trim()); - else reject(new Error(`git exited with code ${code}: ${stderr.trim()}`)); - }); - }); -} async function runAtBase( projectPath: string, @@ -945,6 +889,14 @@ async function runVerifyInSlot(options: RunVerifyOptions): Promise { + void logPeriodicDiskUsage(options.projectPath, scratchRoot).catch((error) => { + console.warn(`[Verify] disk usage sample failed: ${error instanceof Error ? error.message : String(error)}`); + }); + }, 5 * 60_000); + usageTimer.unref(); + try { for (const command of options.commands) { const started = Date.now(); let sandbox: Awaited>; @@ -1031,8 +983,12 @@ async function runVerifyInSlot(options: RunVerifyOptions): Promise