fix(daemon): a stop waits for health ticks and recoveries in flight

docs/internal/flows/startup.md

@@ -119,9 +119,9 @@ platform runs the deferred reclaim (step 21.2).
 2. Stop auxiliary fronts: agent-device edge (cancel its waiting allocates),
    HTTP (after its start settles), the uplink and its redial loop.
 3. Log, emit `daemon.stopping`.
-4. Unsubscribe pushes; dispose host load, dispatcher, reaper, then wait for every
-   cleanup run in flight (scheduled, dry-run or rule-scoped) to end; dispose health
-   monitor, physical watcher.
+4. Unsubscribe pushes; dispose host load, dispatcher, reaper, health monitor, then wait for every
+   cleanup run in flight (scheduled, dry-run or rule-scoped) and every health tick or
+   recovery reboot in flight to end; dispose physical watcher.
 5. Settle — later converge steps are skipped; leasing drains the physical
    watcher and background reclaims; core drains the warm pool.
 6. Dispose — core and leasing timers, `ComponentInstaller.close`, agent-device

src/daemon/main.test.ts

@@ -1641,6 +1641,124 @@ describe("startDaemon stop with a cleanup run in flight (#985)", () => {
   });
 });
 
+describe("startDaemon stop with a health recovery in flight (#1000)", () => {
+  const turnOfTheEventLoop = () => new Promise<void>((resolve) => setImmediate(resolve));
+  const turns = async (count: number) => {
+    for (let turn = 0; turn < count; turn += 1) await turnOfTheEventLoop();
+  };
+  const recovered = ["recover started", "recover finished", "dispose"];
+
+  it.each([
+    { name: "started before the stop", startsDuringStop: false, accepted: [recovered] },
+    // why: a stop may either wait for this recovery or keep it from starting; it may not leave it running.
+    {
+      name: "due while the stop waits for a cleanup run",
+      startsDuringStop: true,
+      accepted: [recovered, ["dispose"]],
+    },
+  ])(
+    "leaves no recovery reboot $name running once the drivers are disposed",
+    async ({ startsDuringStop, accepted }) => {
+      // why: records the order of the driver's recovery reboot and disposal, and holds the reboot mid-call.
+      class HeldRecoveryDriver extends FakeDriver {
+        readonly order: string[] = [];
+        readonly held: (() => void)[] = [];
+        override async makeReady(
+          device: Parameters<FakeDriver["makeReady"]>[0],
+          options: Parameters<FakeDriver["makeReady"]>[1],
+        ): ReturnType<FakeDriver["makeReady"]> {
+          if (options.purpose !== "recover") return super.makeReady(device, options);
+          this.order.push("recover started");
+          await new Promise<void>((resolve) => this.held.push(resolve));
+          const ready = await super.makeReady(device, options);
+          this.order.push("recover finished");
+          return ready;
+        }
+        async dispose(): Promise<void> {
+          this.order.push("dispose");
+        }
+      }
+      // why: holds a cleanup run on its disk read once asked to, so the stop waits on it.
+      class HeldDiskFree extends MemoryFilesystem {
+        holding = false;
+        readonly held: (() => void)[] = [];
+        override async diskFree(path: string): Promise<number> {
+          if (this.holding) await new Promise<void>((resolve) => this.held.push(resolve));
+          return super.diskFree(path);
+        }
+      }
+      const clock = new FakeClock(1_000);
+      const driver = new HeldRecoveryDriver({
+        availableOsVersions: ["26.5"],
+        clock,
+        platform: "ios",
+      });
+      const filesystem = new HeldDiskFree();
+      const { daemon } = await start({ clock, drivers: [driver], filesystem });
+      const agent = {
+        manageEventSubscription: () => undefined,
+        via: "socket",
+        principal: "agent",
+        role: "agent",
+      } as const;
+      await daemon.dispatch(
+        "lease.request",
+        { model: "iPhone 16", osVersion: "26.5", platform: "ios", requesterId: "agent-1" },
+        agent,
+      );
+      // why: the leased simulator dies outside simlock; the monitor recovers it on its second stopped reading, one 30 s tick apart.
+      const { devices } = await driver.listManaged();
+      expect(devices).toHaveLength(1);
+      driver.setManagedReality({
+        devices: devices.map((device) => ({ ...device, runState: "stopped" as const })),
+        processes: [],
+      });
+      clock.advance(30_000);
+      await turns(20);
+
+      if (startsDuringStop) {
+        filesystem.holding = true;
+        void daemon
+          .dispatch(
+            "cleanup.run",
+            { dryRun: true },
+            {
+              manageEventSubscription: () => undefined,
+              via: "socket",
+              principal: "admin",
+              role: "admin",
+            },
+          )
+          .catch(() => undefined);
+        await vi.waitFor(() => expect(filesystem.held).toHaveLength(1));
+      } else {
+        clock.advance(30_000);
+        await vi.waitFor(() => expect(driver.order).toEqual(["recover started"]));
+      }
+
+      const stopped = daemon.stop("test-stop");
+      let stopEnded = false;
+      void stopped.then(() => {
+        stopEnded = true;
+      });
+      if (startsDuringStop) {
+        await turns(20);
+        expect(stopEnded).toBe(false);
+        clock.advance(30_000);
+        await turns(50);
+        for (const release of filesystem.held.splice(0)) release();
+      }
+      for (let turn = 0; turn < 200 && !stopEnded; turn += 1) await turnOfTheEventLoop();
+      for (const release of driver.held.splice(0)) release();
+      await stopped;
+      // why: a reboot that outlived the stop records its end a few turns after it is released.
+      await turns(50);
+
+      expect(accepted).toContainEqual(driver.order);
+    },
+  );
+});
+
 describe("startDaemon stop with a warm target", () => {
   it("creates no further device for a target once a stop is asked for, though the creation in flight ends while the gateway stops", async () => {
     const directory = await mkdtemp(join(tmpdir(), "simlock-main-stop-target-"));

src/daemon/server.ts

@@ -637,10 +637,12 @@ export class DaemonServer {
     this.options.hostLoad?.dispose();
     this.#workerDispatcher?.dispose();
     this.#engine?.reaper.dispose();
+    this.#engine?.healthMonitor?.dispose();
     // why: every cleanup run in flight (scheduled, dry-run or rule-scoped) still reads the registry
     // and calls drivers; each ends before anything below closes them.
     await this.#engine?.reaper.settle();
-    this.#engine?.healthMonitor?.dispose();
+    // why: a health tick or recovery reboot already running still acts on a device; it ends before the drivers are disposed.
+    await this.#engine?.healthMonitor?.settle();
     this.#engine?.physicalDeviceWatcher?.dispose();
     // why: a stop releases nothing; leases persist with their deadline. Only work already in
     // flight is drained, so `daemon stop` finishes on a settled pool. Disposal follows rather than

src/leasing/lease-health-monitor.test.ts

@@ -713,4 +713,40 @@ describe("LeaseHealthMonitor", () => {
     await flush();
     expect(makeReadyCalls(harness.driver)).toBe(1);
   });
+
+  it("settle() waits for a tick running when the monitor is disposed, and that tick starts no recovery", async () => {
+    const harness = await createHarness();
+    const { device } = await seedLeased(harness);
+    harness.driver.setManagedReality({
+      devices: [observedDevice(device.driverDeviceId, "stopped")],
+      processes: [],
+    });
+    harness.monitor.start();
+    harness.clock.advance(30_000);
+    await flush();
+    // why: holds tick 2 on its observation so the monitor is disposed while that tick runs.
+    const listManaged = harness.driver.listManaged.bind(harness.driver);
+    let release: () => void = () => undefined;
+    harness.driver.listManaged = async () => {
+      await new Promise<void>((resolve) => {
+        release = resolve;
+      });
+      return listManaged();
+    };
+    harness.clock.advance(30_000);
+    await flush();
+
+    harness.monitor.dispose();
+    let settled = false;
+    const settling = harness.monitor.settle().then(() => {
+      settled = true;
+    });
+    await flush();
+    expect(settled).toBe(false);
+
+    release();
+    await settling;
+    expect(settled).toBe(true);
+    expect(makeReadyCalls(harness.driver)).toBe(0);
+  });
 });

src/leasing/lease-health-monitor.ts

@@ -54,6 +54,7 @@ export class LeaseHealthMonitor {
   readonly #inFlight = new Set<string>();
   readonly #logger: Logger;
   readonly #pendingBackoffs = new Map<TimerHandle, () => void>();
+  readonly #running = new Set<Promise<void>>();
   #disposed = false;
   #tickTimer: TimerHandle | undefined;
 
@@ -90,6 +91,21 @@ export class LeaseHealthMonitor {
     this.#pendingBackoffs.clear();
   }
 
+  /**
+   * why: Resolves once the running tick and every recovery have ended, including work a finishing
+   * one starts. Call after `dispose()`: a disposed monitor starts no recovery, so this ends.
+   */
+  async settle(): Promise<void> {
+    while (this.#running.size > 0) {
+      await Promise.allSettled(this.#running);
+    }
+  }
+
+  #track(work: Promise<void>): void {
+    this.#running.add(work);
+    void work.finally(() => this.#running.delete(work));
+  }
+
   #armTick(): void {
     if (this.#disposed) {
       return;
@@ -98,12 +114,14 @@ export class LeaseHealthMonitor {
       this.options.config.health.probeIntervalMs,
       () => {
         this.#tickTimer = undefined;
-        void this.#runTick().then(
-          () => this.#armTick(),
-          (error: unknown) => {
-            this.#logger.error("health monitor tick failed", { error: summarizeError(error) });
-            this.#armTick();
-          },
+        this.#track(
+          this.#runTick().then(
+            () => this.#armTick(),
+            (error: unknown) => {
+              this.#logger.error("health monitor tick failed", { error: summarizeError(error) });
+              this.#armTick();
+            },
+          ),
         );
       },
     );
@@ -249,19 +267,24 @@ export class LeaseHealthMonitor {
   }
 
   #dispatchRecovery(device: VirtualDevice, lease: LeaseRecord): void {
+    if (this.#disposed) {
+      return;
+    }
     if (this.#inFlight.size >= this.options.config.health.maxConcurrentRecoveries) {
       // Cap reached; this device's counters stay elevated and it is retried next tick.
       return;
     }
     this.#inFlight.add(device.id);
-    void this.#recover(device, lease)
-      .catch((error: unknown) => {
-        this.#logger.error("unexpected error recovering device", {
-          deviceId: device.id,
-          error: summarizeError(error),
-        });
-      })
-      .finally(() => this.#inFlight.delete(device.id));
+    this.#track(
+      this.#recover(device, lease)
+        .catch((error: unknown) => {
+          this.#logger.error("unexpected error recovering device", {
+            deviceId: device.id,
+            error: summarizeError(error),
+          });
+        })
+        .finally(() => this.#inFlight.delete(device.id)),
+    );
   }
 
   async #recover(device: VirtualDevice, lease: LeaseRecord): Promise<void> {