feat(verify): log sandbox disk usage

src/verify/diskUsage.ts

@@ -0,0 +1,48 @@
+import { execFile } from 'node:child_process';
+import { lstat, readdir } from 'node:fs/promises';
+import { homedir, tmpdir } from 'node:os';
+import { basename, dirname, join } from 'node:path';
+import { promisify } from 'node:util';
+
+const execFileAsync = promisify(execFile);
+
+async function directoryBytes(path: string): Promise<number> {
+  let entries;
+  try { entries = await readdir(path); } catch { return 0; }
+  let bytes = 0;
+  for (const entry of entries) {
+    const child = join(path, entry);
+    try {
+      const info = await lstat(child);
+      if (info.isDirectory()) bytes += await directoryBytes(child);
+      else if (info.isFile()) bytes += info.size;
+    } catch { /* A concurrent cleanup is expected during a usage sample. */ }
+  }
+  return bytes;
+}
+
+export async function logVerifySandboxUsage(root: string, phase: 'create' | 'remove'): Promise<void> {
+  const [bytes, entries] = await Promise.all([
+    directoryBytes(root), readdir(dirname(root)).catch(() => []),
+  ]);
+  const live = entries.filter((entry) => entry.startsWith('.openswarm-verify-')).length;
+  console.log(`[Verify] sandbox usage phase=${phase} bytes=${bytes} live=${live} root=${root}`);
+}
+
+export async function logPeriodicDiskUsage(projectPath: string, scratchRoot?: string): Promise<void> {
+  const { stdout = '' } = await execFileAsync('git', ['-C', projectPath, 'worktree', 'list', '--porcelain'])
+    .catch(() => ({ stdout: '' }));
+  const worktrees = stdout.split('\n').filter((line) => line.startsWith('worktree ')).length;
+  const [sessionsBytes, verifyBytes] = await Promise.all([
+    directoryBytes(join(homedir(), '.openswarm', 'sessions')),
+    (async () => {
+      const root = scratchRoot ?? tmpdir();
+      let entries: string[];
+      try { entries = await readdir(root); } catch { return 0; }
+      const verifyRoots = entries.filter((entry) => entry.startsWith('.openswarm-verify-'));
+      return (await Promise.all(verifyRoots.map((entry) => directoryBytes(join(root, entry)))))
+        .reduce((total, bytes) => total + bytes, 0);
+    })(),
+  ]);
+  console.log(`[Verify] disk usage period=5m repo=${basename(projectPath)} worker_worktrees=${worktrees} sessions_bytes=${sessionsBytes} verify_tmp_bytes=${verifyBytes}`);
+}

src/verify/git.ts

@@ -0,0 +1,64 @@
+import { spawn } from 'node:child_process';
+
+export const GIT_TIMEOUT_MS = 30_000;
+/**
+ * The head sandbox is a `--no-hardlinks` clone: a full copy of the object store,
+ * because the sandbox runs the repository's own test commands and hardlinked
+ * packs would share inodes with the real checkout. That copy is bounded by disk
+ * throughput, not by git, and on a loaded host it outgrows the 30 s every other
+ * git call gets — cgf-portal's 135 MB took 33 s at load 55, and the runner fell
+ * back to the LLM tester 29 times in one day, which is how `ruff`-red code shipped
+ * as ready PRs (AGT-4416). The clone gets its own budget.
+ */
+export const CLONE_TIMEOUT_MS = 10 * 60_000;
+
+/** The budget a git invocation gets: the sandbox clone's own, everything else the short default. */
+export function gitTimeoutMsFor(args: readonly string[]): number {
+  return args[0] === 'clone' ? CLONE_TIMEOUT_MS : GIT_TIMEOUT_MS;
+}
+export async function git(projectPath: string, args: string[]): Promise<string> {
+  const timeoutMs = gitTimeoutMsFor(args);
+  return await new Promise((resolveResult, reject) => {
+    const maxOutputBytes = 4 * 1024 * 1024;
+    // Checkout hooks belong to the live developer environment. Running them in
+    // a verification worktree can restore external data/secrets after filtering.
+    const child = spawn('git', ['-C', projectPath, '-c', 'core.hooksPath=/dev/null', ...args], {
+      stdio: ['ignore', 'pipe', 'pipe'],
+      detached: process.platform !== 'win32',
+    });
+    let stdout = '';
+    let stderr = '';
+    let outputBytes = 0;
+    let settled = false;
+    let timer: ReturnType<typeof setTimeout>;
+    const fail = (error: Error) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (child.pid && process.platform !== 'win32') {
+        try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); }
+      } else child.kill('SIGKILL');
+      reject(error);
+    };
+    timer = setTimeout(() => fail(new Error(`git ${args[0] ?? ''} timed out after ${timeoutMs}ms`)), timeoutMs);
+    const append = (target: 'stdout' | 'stderr', chunk: Buffer) => {
+      outputBytes += chunk.length;
+      if (outputBytes > maxOutputBytes) {
+        fail(new Error(`git ${args[0] ?? ''} output exceeded ${maxOutputBytes} bytes`));
+        return;
+      }
+      if (target === 'stdout') stdout += chunk.toString('utf8');
+      else stderr += chunk.toString('utf8');
+    };
+    child.stdout.on('data', (chunk: Buffer) => append('stdout', chunk));
+    child.stderr.on('data', (chunk: Buffer) => append('stderr', chunk));
+    child.on('error', fail);
+    child.on('close', (code) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (code === 0) resolveResult(stdout.trim());
+      else reject(new Error(`git exited with code ${code}: ${stderr.trim()}`));
+    });
+  });
+}

src/verify/runner.test.ts

@@ -41,6 +41,19 @@ afterEach(async () => {
 });
 
 describe('runVerify', () => {
+  it('logs verify sandbox bytes and the live sandbox count at creation and cleanup', async () => {
+    const log = vi.spyOn(console, 'log').mockImplementation(() => undefined);
+    try {
+      await runVerify({ projectPath: repo, commands: [verify('true')], baseRef: 'HEAD', sandboxScratchRoot: root });
+      const lines = log.mock.calls.map(([line]) => String(line)).filter((line) => line.includes('[Verify] sandbox usage'));
+      expect(lines).toHaveLength(2);
+      expect(lines[0]).toMatch(/phase=create bytes=\d+ live=1/);
+      expect(lines[1]).toMatch(/phase=remove bytes=\d+ live=1/);
+    } finally {
+      log.mockRestore();
+    }
+  });
+
   it('routes strict verification through the companion and keeps scratch checkouts under its allowed root', async () => {
     const canonicalScratchRoot = await realpath(root);
     const execute = vi.fn(async () => ({

src/verify/runner.ts

@@ -1,3 +1,3 @@
 import { execFile, spawn, spawnSync } from 'node:child_process';
 import { createHash, randomUUID } from 'node:crypto';
 import { constants } from 'node:fs';
@@ -20,25 +20,12 @@
 import type { VerifyCommand } from './manifest.js';
 import { rebasePythonEnvironment } from './pythonEnvironment.js';
 import { resourceAwareTestCommand, testResourceShellPrefix, withTestResourceBudget } from '../support/testResourceBudget.js';
+import { logPeriodicDiskUsage, logVerifySandboxUsage } from './diskUsage.js';
 
 const OUTPUT_TAIL_BYTES = 8 * 1024;
 const FINGERPRINT_BYTES = 4 * 1024 * 1024;
-const GIT_TIMEOUT_MS = 30_000;
-/**
- * The head sandbox is a `--no-hardlinks` clone: a full copy of the object store,
- * because the sandbox runs the repository's own test commands and hardlinked
- * packs would share inodes with the real checkout. That copy is bounded by disk
- * throughput, not by git, and on a loaded host it outgrows the 30 s every other
- * git call gets — cgf-portal's 135 MB took 33 s at load 55, and the runner fell
- * back to the LLM tester 29 times in one day, which is how `ruff`-red code shipped
- * as ready PRs (AGT-4416). The clone gets its own budget.
- */
-export const CLONE_TIMEOUT_MS = 10 * 60_000;
-
-/** The budget a git invocation gets: the sandbox clone's own, everything else the short default. */
-export function gitTimeoutMsFor(args: readonly string[]): number {
-  return args[0] === 'clone' ? CLONE_TIMEOUT_MS : GIT_TIMEOUT_MS;
-}
+import { git, gitTimeoutMsFor, CLONE_TIMEOUT_MS, GIT_TIMEOUT_MS } from './git.js';
+export { gitTimeoutMsFor, CLONE_TIMEOUT_MS };
 const execFileAsync = promisify(execFile);
 
 /** Dependency manifests the sandbox may observe when comparing base vs head. */
@@ -764,6 +751,7 @@
   return await mkdtemp(join(scratchRoot ?? tmpdir(), prefix));
 }
 
+
 async function createHeadSandbox(
   projectPath: string,
   commands: VerifyCommand[],
@@ -806,59 +794,15 @@
     // Validate what was actually copied, closing the source validation/copy
     // race before any repository-controlled command can execute.
     await validateSandboxSymlinks(project, sharedPaths);
+    await logVerifySandboxUsage(root, 'create');
     return { root, project };
   } catch (error) {
+    await logVerifySandboxUsage(root, 'remove');
     await rm(root, { recursive: true, force: true });
     throw error;
   }
 }
 
-async function git(projectPath: string, args: string[]): Promise<string> {
-  const timeoutMs = gitTimeoutMsFor(args);
-  return await new Promise((resolveResult, reject) => {
-    const maxOutputBytes = 4 * 1024 * 1024;
-    // Checkout hooks belong to the live developer environment. Running them in
-    // a verification worktree can restore external data/secrets after filtering.
-    const child = spawn('git', ['-C', projectPath, '-c', 'core.hooksPath=/dev/null', ...args], {
-      stdio: ['ignore', 'pipe', 'pipe'],
-      detached: process.platform !== 'win32',
-    });
-    let stdout = '';
-    let stderr = '';
-    let outputBytes = 0;
-    let settled = false;
-    let timer: ReturnType<typeof setTimeout>;
-    const fail = (error: Error) => {
-      if (settled) return;
-      settled = true;
-      clearTimeout(timer);
-      if (child.pid && process.platform !== 'win32') {
-        try { process.kill(-child.pid, 'SIGKILL'); } catch { child.kill('SIGKILL'); }
-      } else child.kill('SIGKILL');
-      reject(error);
-    };
-    timer = setTimeout(() => fail(new Error(`git ${args[0] ?? ''} timed out after ${timeoutMs}ms`)), timeoutMs);
-    const append = (target: 'stdout' | 'stderr', chunk: Buffer) => {
-      outputBytes += chunk.length;
-      if (outputBytes > maxOutputBytes) {
-        fail(new Error(`git ${args[0] ?? ''} output exceeded ${maxOutputBytes} bytes`));
-        return;
-      }
-      if (target === 'stdout') stdout += chunk.toString('utf8');
-      else stderr += chunk.toString('utf8');
-    };
-    child.stdout.on('data', (chunk: Buffer) => append('stdout', chunk));
-    child.stderr.on('data', (chunk: Buffer) => append('stderr', chunk));
-    child.on('error', fail);
-    child.on('close', (code) => {
-      if (settled) return;
-      settled = true;
-      clearTimeout(timer);
-      if (code === 0) resolveResult(stdout.trim());
-      else reject(new Error(`git exited with code ${code}: ${stderr.trim()}`));
-    });
-  });
-}
 
 async function runAtBase(
   projectPath: string,
@@ -945,6 +889,14 @@
       throw new Error('[security] strict verification scratch root must contain, but not equal, the project checkout');
     }
   }
+  await logPeriodicDiskUsage(options.projectPath, scratchRoot);
+  const usageTimer = setInterval(() => {
+    void logPeriodicDiskUsage(options.projectPath, scratchRoot).catch((error) => {
+      console.warn(`[Verify] disk usage sample failed: ${error instanceof Error ? error.message : String(error)}`);
+    });
+  }, 5 * 60_000);
+  usageTimer.unref();
+  try {
   for (const command of options.commands) {
     const started = Date.now();
     let sandbox: Awaited<ReturnType<typeof createHeadSandbox>>;
@@ -1031,8 +983,12 @@
         durationMs: Date.now() - started,
       });
     } finally {
+      await logVerifySandboxUsage(sandbox.root, 'remove');
       await rm(sandbox.root, { recursive: true, force: true });
     }
   }
   return evidence;
+  } finally {
+    clearInterval(usageTimer);
+  }
 }