refactor: split cloud-hypervisor cleanup registry into focused modules

src/cloud-hypervisor/cleanup-dependencies.ts

@@ -0,0 +1,127 @@
+import { promises as fs } from 'fs';
+import * as path from 'path';
+import execa from 'execa';
+
+const CLEANUP_DIRECTORY_NAME = 'pending-cleanup';
+
+export interface CleanupRegistryDependencies {
+  readonly rootDirectory?: string;
+  readonly effectiveUid?: number;
+  readonly processId?: number;
+  readonly readFile?: typeof fs.readFile;
+  readonly readlink?: typeof fs.readlink;
+  readonly realpath?: typeof fs.realpath;
+  readonly lstat?: typeof fs.lstat;
+  readonly stat?: typeof fs.stat;
+  readonly mkdir?: typeof fs.mkdir;
+  readonly readdir?: typeof fs.readdir;
+  readonly rename?: typeof fs.rename;
+  readonly link?: typeof fs.link;
+  readonly unlink?: typeof fs.unlink;
+  readonly rm?: typeof fs.rm;
+  readonly rmdir?: typeof fs.rmdir;
+  readonly open?: typeof fs.open;
+  readonly kill?: typeof process.kill;
+  readonly run?: (
+    command: string,
+    args: readonly string[],
+  ) => Promise<{ exitCode: number; stdout: string; stderr: string }>;
+  readonly sleep?: (milliseconds: number) => Promise<void>;
+}
+
+export interface ResolvedCleanupDependencies {
+  readonly rootDirectory: string;
+  readonly effectiveUid: number;
+  readonly processId: number;
+  readonly readFile: typeof fs.readFile;
+  readonly readlink: typeof fs.readlink;
+  readonly realpath: typeof fs.realpath;
+  readonly lstat: typeof fs.lstat;
+  readonly stat: typeof fs.stat;
+  readonly mkdir: typeof fs.mkdir;
+  readonly readdir: typeof fs.readdir;
+  readonly rename: typeof fs.rename;
+  readonly link: typeof fs.link;
+  readonly unlink: typeof fs.unlink;
+  readonly rm: typeof fs.rm;
+  readonly rmdir: typeof fs.rmdir;
+  readonly open: typeof fs.open;
+  readonly kill: typeof process.kill;
+  readonly run: NonNullable<CleanupRegistryDependencies['run']>;
+  readonly sleep: NonNullable<CleanupRegistryDependencies['sleep']>;
+}
+
+export function resolveCleanupDependencies(
+  dependencies: CleanupRegistryDependencies = {},
+): ResolvedCleanupDependencies {
+  const runRoot = dependencies.rootDirectory ?? '/run/awf-cloud-hypervisor';
+  return {
+    rootDirectory: path.join(runRoot, CLEANUP_DIRECTORY_NAME),
+    effectiveUid: dependencies.effectiveUid ?? process.geteuid?.() ?? -1,
+    processId: dependencies.processId ?? process.pid,
+    readFile: dependencies.readFile ?? fs.readFile,
+    readlink: dependencies.readlink ?? fs.readlink,
+    realpath: dependencies.realpath ?? fs.realpath,
+    lstat: dependencies.lstat ?? fs.lstat,
+    stat: dependencies.stat ?? fs.stat,
+    mkdir: dependencies.mkdir ?? fs.mkdir,
+    readdir: dependencies.readdir ?? fs.readdir,
+    rename: dependencies.rename ?? fs.rename,
+    link: dependencies.link ?? fs.link,
+    unlink: dependencies.unlink ?? fs.unlink,
+    rm: dependencies.rm ?? fs.rm,
+    rmdir: dependencies.rmdir ?? fs.rmdir,
+    open: dependencies.open ?? fs.open,
+    kill: dependencies.kill ?? process.kill,
+    run: dependencies.run ?? runCommand,
+    sleep: dependencies.sleep ?? ((milliseconds) =>
+      new Promise((resolve) => setTimeout(resolve, milliseconds))),
+  };
+}
+
+async function runCommand(
+  command: string,
+  args: readonly string[],
+): Promise<{ exitCode: number; stdout: string; stderr: string }> {
+  // `command` is the absolute `ip` path returned by the root-only preflight.
+  // eslint-disable-next-line local/no-unsafe-execa
+  const result = await execa(command, [...args], {
+    reject: false,
+    stdio: ['ignore', 'pipe', 'pipe'],
+    env: { PATH: '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' },
+    extendEnv: false,
+    timeout: 10_000,
+  });
+  return { exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr };
+}
+
+export async function runChecked(
+  run: ResolvedCleanupDependencies['run'],
+  command: string,
+  args: readonly string[],
+): Promise<void> {
+  const result = await run(command, args);
+  if (result.exitCode !== 0) {
+    throw new Error(
+      `${command} ${args.join(' ')} failed with code ${result.exitCode}: ` +
+      `${result.stderr.trim() || result.stdout.trim()}`,
+    );
+  }
+}
+
+export async function pathExists(
+  filePath: string,
+  lstat: typeof fs.lstat,
+): Promise<boolean> {
+  try {
+    await lstat(filePath);
+    return true;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
+    throw error;
+  }
+}
+
+export function formatError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

src/cloud-hypervisor/cleanup-network.ts

@@ -0,0 +1,138 @@
+import type { CleanupRecord, FileIdentity, InterfaceIdentity } from './cleanup-identity';
+import {
+  bridgeForwardRule,
+  captureFileIdentity,
+  interfaceExists,
+  tryCaptureInterfaceIdentity,
+} from './cleanup-process';
+import { sameFileIdentity } from './cleanup-identity';
+import {
+  pathExists,
+  runChecked,
+  type ResolvedCleanupDependencies,
+} from './cleanup-dependencies';
+
+async function validateFileIfPresent(
+  dependencies: ResolvedCleanupDependencies,
+  filePath: string,
+  expected: FileIdentity | undefined,
+  label: string,
+): Promise<void> {
+  if (!(await pathExists(filePath, dependencies.lstat))) return;
+  if (!expected) throw new Error(`${label} exists but its immutable identity was never committed`);
+  const current = await captureFileIdentity(dependencies.lstat, filePath);
+  if (!sameFileIdentity(current, expected)) throw new Error(`${label} identity changed`);
+}
+
+async function validateInterfaceIfPresent(
+  dependencies: ResolvedCleanupDependencies,
+  ipPath: string,
+  name: string,
+  expected: InterfaceIdentity | undefined,
+  namespace: string | undefined,
+): Promise<void> {
+  const current = await tryCaptureInterfaceIdentity(dependencies.run, ipPath, name, namespace);
+  if (!current) return;
+  if (!expected) throw new Error(`interface "${name}" exists but its identity was never committed`);
+  if (current.ifindex !== expected.ifindex || current.namespace !== expected.namespace) {
+    throw new Error(`interface "${name}" identity changed`);
+  }
+}
+
+export async function validateRecordResources(
+  dependencies: ResolvedCleanupDependencies,
+  record: CleanupRecord,
+  ipPath: string,
+): Promise<void> {
+  const network = record.network;
+  const netnsExists = network ? await pathExists(network.netnsPath, dependencies.lstat) : false;
+  if (network) {
+    await validateFileIfPresent(dependencies, network.netnsPath, record.identities.netns, 'netns');
+  }
+  await validateFileIfPresent(
+    dependencies, record.paths.runDirectory, record.identities.runDirectory, 'run directory',
+  );
+  await validateFileIfPresent(
+    dependencies, record.paths.cgroupPath, record.identities.cgroup, 'cgroup',
+  );
+  await validateFileIfPresent(
+    dependencies,
+    record.paths.virtiofsdShareDirectory,
+    record.identities.virtiofsdShareDirectory,
+    'virtiofsd share directory',
+  );
+  if (record.paths.artifactSnapshotDirectory) {
+    await validateFileIfPresent(
+      dependencies,
+      record.paths.artifactSnapshotDirectory,
+      record.identities.artifactSnapshotDirectory,
+      'artifact snapshot directory',
+    );
+  }
+  if (network) {
+    await validateInterfaceIfPresent(
+      dependencies, ipPath, network.hostVethName, record.identities.hostVeth, undefined,
+    );
+  }
+  if (network && netnsExists) {
+    await validateInterfaceIfPresent(
+      dependencies,
+      ipPath,
+      network.namespaceVethName,
+      record.identities.namespaceVeth,
+      network.namespaceName,
+    );
+    await validateInterfaceIfPresent(
+      dependencies, ipPath, network.tapName, record.identities.tap, network.namespaceName,
+    );
+  }
+}
+
+export async function deleteNetwork(
+  dependencies: ResolvedCleanupDependencies,
+  record: CleanupRecord,
+  ipPath: string,
+): Promise<void> {
+  const network = requireNetwork(record);
+  if (await interfaceExists(dependencies.run, ipPath, network.hostVethName)) {
+    await validateInterfaceIfPresent(
+      dependencies,
+      ipPath,
+      network.hostVethName,
+      record.identities.hostVeth,
+      undefined,
+    );
+    await runChecked(dependencies.run, ipPath, ['link', 'delete', network.hostVethName]);
+  }
+  if (await pathExists(network.netnsPath, dependencies.lstat)) {
+    await validateFileIfPresent(
+      dependencies,
+      network.netnsPath,
+      record.identities.netns,
+      'netns',
+    );
+    await runChecked(dependencies.run, ipPath, ['netns', 'delete', network.namespaceName]);
+  }
+  const rule = bridgeForwardRule(
+    '-C',
+    network.infrastructureBridge,
+    network.hostForwardRuleComment,
+  );
+  const checked = await dependencies.run('iptables', rule);
+  if (checked.exitCode === 0) {
+    await runChecked(dependencies.run, 'iptables', bridgeForwardRule(
+      '-D',
+      network.infrastructureBridge,
+      network.hostForwardRuleComment,
+    ));
+  } else if (checked.exitCode !== 1) {
+    throw new Error(
+      `Could not revalidate per-run bridge rule: ${checked.stderr.trim() || checked.stdout.trim()}`,
+    );
+  }
+}
+
+function requireNetwork(record: CleanupRecord): NonNullable<CleanupRecord['network']> {
+  if (!record.network) throw new Error('Cleanup network plan is not committed');
+  return record.network;
+}

src/cloud-hypervisor/cleanup-process.ts

@@ -2,6 +2,9 @@ import { promises as fs } from 'fs';
 import type { FileIdentity, InterfaceIdentity, MountIdentity, ProcessIdentity, RecordedProcess } from './cleanup-identity';
 import { parseMountInfoLine, parseStatusIdentity, sameFileIdentity } from './cleanup-identity';
 
+const PROCESS_STOP_WAIT_MS = 2_000;
+const PROCESS_STOP_INTERVAL_MS = 50;
+
 export interface CleanupProcessDependencies {
   readonly readFile: typeof fs.readFile;
   readonly readlink: typeof fs.readlink;
@@ -96,6 +99,41 @@ export async function processMatches(
   }
 }
 
+export interface CleanupProcessStopDependencies extends CleanupProcessDependencies {
+  readonly kill: typeof process.kill;
+  readonly sleep: (milliseconds: number) => Promise<void>;
+}
+
+export async function stopProcess(
+  dependencies: CleanupProcessStopDependencies,
+  identity: ProcessIdentity,
+  recorded: RecordedProcess,
+): Promise<void> {
+  if (!tryKill(dependencies.kill, identity.pid, 'SIGTERM')) {
+    if (await processMatches(dependencies, identity, recorded)) {
+      throw new Error(`process ${identity.pid} still matches after kill reported ESRCH`);
+    }
+    return;
+  }
+  const deadline = Date.now() + PROCESS_STOP_WAIT_MS;
+  while (Date.now() < deadline) {
+    if (!(await processMatches(dependencies, identity, recorded))) return;
+    await dependencies.sleep(PROCESS_STOP_INTERVAL_MS);
+  }
+  if (!(await processMatches(dependencies, identity, recorded))) return;
+  if (!tryKill(dependencies.kill, identity.pid, 'SIGKILL')) {
+    if (await processMatches(dependencies, identity, recorded)) {
+      throw new Error(`process ${identity.pid} still matches after kill reported ESRCH`);
+    }
+    return;
+  }
+  for (let attempt = 0; attempt < PROCESS_STOP_WAIT_MS / PROCESS_STOP_INTERVAL_MS; attempt += 1) {
+    if (!(await processMatches(dependencies, identity, recorded))) return;
+    await dependencies.sleep(PROCESS_STOP_INTERVAL_MS);
+  }
+  throw new Error(`identity-validated process ${identity.pid} did not exit`);
+}
+
 export async function captureFileIdentity(
   lstat: typeof fs.lstat,
   filePath: string,