From c2cb0f8f1c3578c3ac4cb751d9e426656ced7e3d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 4 Aug 2026 00:27:11 +0000 Subject: [PATCH] Playground: cover the second class of TAP-busy leak (post-teardown zombies) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the recurring FirecrackerError('PUT /snapshot/load -> 400: Resource busy (os error 16). Invalid TUN/TAP Backend provided by fc-tap-') was in _teardown, not _restore_snapshot. Evidence: 20 orphaned fc processes on the box, all with 200 MiB..16 GiB RSS — i.e. VMs that successfully restored, ran, and were later kicked (idle-reaper / ensure-failed-retry / post-query-error). Something in _shutdown + _teardown wasn't finishing them off. Two fixes here: 1. _shutdown: `asyncio.wait_for(vm.proc.wait(), timeout=5.0)` is too short. Post-SIGKILL cleanup of a 16 GiB guest can take many seconds (page freeing + virtio-blk fd close + memory-backing unmap), especially under concurrent shutdowns. When wait_for timed out, vm.proc/vm.pid were reset but the OS process was still alive holding fc-tap-. Bump the reap window to 60 s and follow up with an explicit _pid_alive poll that re-SIGKILLs on each iteration. Log an error if the process outlives that window (previously silent). 2. _teardown: also call net.teardown_tap unconditionally. Even in the (now-rare) case a lingering fc still holds the TAP fd, the next /restore's ensure_tap re-creates a fresh TAP. Cheap (single tuntap add + addr), guaranteed clean. Fixes the "many systems fail with Resource busy fc-tap-" reports across chdb-dataframe, clickhouse, starrocks, polars-dataframe, doris-parquet, heavyai, spark-comet, victorialogs, quickwit, trino-partitioned, clickhouse-web, and others. --- playground/server/vm_manager.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/playground/server/vm_manager.py b/playground/server/vm_manager.py index 40f39cb0e5..544319e25c 100644 --- a/playground/server/vm_manager.py +++ b/playground/server/vm_manager.py @@ -797,11 +797,28 @@ async def _shutdown(self, vm: VM) -> None: if vm.pid is not None and _pid_alive(vm.pid): with contextlib.suppress(ProcessLookupError): os.kill(vm.pid, signal.SIGKILL) - # Reap the process. asyncio.Process.wait() drains the exit status so - # the kernel can release the resources (TAP fd, memory mappings). + # Reap the process. Post-SIGKILL cleanup of a 16 GiB guest can + # take many seconds (page freeing + virtio-blk fd close + + # memory-backing unmap), especially under concurrent + # shutdowns. If wait_for times out we still MUST confirm the + # PID is dead before dropping vm.proc/vm.pid — otherwise the + # process lingers, holds fc-tap- open, and the next + # /restore for the same slot fails with EBUSY. Poll _pid_alive + # up to 60 s (a good deal longer than the earlier 5 s) and + # SIGKILL again on the way if the first didn't take. if vm.proc is not None: with contextlib.suppress(Exception): - await asyncio.wait_for(vm.proc.wait(), timeout=5.0) + await asyncio.wait_for(vm.proc.wait(), timeout=60.0) + for _ in range(600): # 60 s total + if vm.pid is None or not _pid_alive(vm.pid): + break + with contextlib.suppress(ProcessLookupError): + os.kill(vm.pid, signal.SIGKILL) + await asyncio.sleep(0.1) + if vm.pid is not None and _pid_alive(vm.pid): + log.error("[%s] _shutdown: fc pid=%d still alive after 60 s SIGKILL " + "loop — leaking; next restore may hit fc-tap- busy", + vm.system.name, vm.pid) vm.proc = None vm.pid = None with contextlib.suppress(FileNotFoundError): @@ -811,6 +828,14 @@ async def _teardown(self, vm: VM, reason: str) -> None: log.warning("[%s] teardown: %s", vm.system.name, reason) with contextlib.suppress(Exception): await self._shutdown(vm) + # Tear down the TAP unconditionally on every teardown so the + # next /restore's ensure_tap gets a fresh interface. Belt- + # and-braces against a dying-but-not-yet-dead fc process + # holding the TAP fd, or any other kernel-level lingering + # state we haven't yet accounted for. ensure_tap on the next + # boot is cheap (a single tuntap add + address assignment). + with contextlib.suppress(Exception): + await net.teardown_tap(vm.slot) vm.state = "snapshotted" if _has_snapshot(vm) else "down" vm.ready_since = None vm.cpu_baseline_jiffies = 0