diff --git a/scripts/run-test-wave.py b/scripts/run-test-wave.py index ce7e3d344..390cb666d 100755 --- a/scripts/run-test-wave.py +++ b/scripts/run-test-wave.py @@ -40,6 +40,15 @@ # runner, survives only because it was already listed here. SLOW_SUITES = frozenset(("incremental", "store_arch", "daemon_runtime", "cli")) POLL_SECONDS = 0.05 +# Floor for how long the external Windows kill helper (taskkill.exe /T /F) may +# take to answer. This is deliberately NOT --kill-grace: that flag budgets how +# long a doomed process may take to die, while this budgets spawning the helper +# on a loaded runner, which routinely exceeds a second. Wiring the two together +# made a small kill grace flake the whole wave -- a timed-out taskkill is +# reported as "could not prove cleanup", which raises out of the wave loop and +# re-enters cleanup with the leader already dead. kill_grace still governs every +# actual death wait. The descendant probe has its own budget below. +WINDOWS_HELPER_TIMEOUT_SECONDS = 30 # WHY: the Windows descendant probe below is a cold `powershell.exe` + CIM # start. On a GitHub Windows runner that routinely costs seconds -- interpreter @@ -175,6 +184,10 @@ def start_suite( ) +def windows_helper_timeout(kill_grace: int) -> int: + return max(kill_grace, WINDOWS_HELPER_TIMEOUT_SECONDS) + + def windows_tree_cleanup_blocker(pid: int) -> str | None: """Why `pid`'s tree cannot be called clean, or None when it provably is. @@ -256,7 +269,7 @@ def terminate_process_tree(active: ActiveSuite, kill_grace: int) -> None: stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - timeout=kill_grace, + timeout=windows_helper_timeout(kill_grace), ) except (OSError, subprocess.TimeoutExpired): completed = None diff --git a/tests/test_parallel_harness_contract.sh b/tests/test_parallel_harness_contract.sh index 31854b995..3e0c6c731 100755 --- a/tests/test_parallel_harness_contract.sh +++ b/tests/test_parallel_harness_contract.sh @@ -366,6 +366,7 @@ python3 - "$scheduler" "$fixture" "$(command -v python3)" <<'PY' from __future__ import annotations import ctypes +import importlib.util import os import pathlib import signal @@ -385,6 +386,36 @@ release = barrier / "timeout_exit_race.release" descendant_path = fixture / "descendant.pid" +def scheduler_wait_budget() -> int: + """Seconds to allow the scheduler to finish refusing. + + Generous on purpose: the scheduler's refusal is the asserted state, and this + bound only has to exceed its worst case; it never decides the verdict. + + On POSIX the refusal is signal-driven and lands well inside --kill-grace. + On Windows it costs external helper spawns -- taskkill.exe, and powershell.exe + for the descendant probe -- which the scheduler deliberately budgets on their + own rather than with --kill-grace. Read those budgets from the scheduler + instead of restating them: hard-coding a number here silently turns a slow + runner into a harness failure the moment the two drift apart. The refusal + path can spend the budgets twice -- once in the wave loop, once in the + cleanup re-entry -- so allow both passes plus interpreter startup. + """ + if os.name != "nt": + return 8 + spec = importlib.util.spec_from_file_location("cbm_run_test_wave", scheduler) + module = importlib.util.module_from_spec(spec) + # Register before exec: @dataclass resolves annotations through + # sys.modules[cls.__module__], which is None for an unregistered module. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + per_pass = module.WINDOWS_HELPER_TIMEOUT_SECONDS + ( + module.WINDOWS_DESCENDANT_PROBE_SECONDS + * module.WINDOWS_DESCENDANT_PROBE_ATTEMPTS + ) + return per_pass * 2 + 10 + + def process_state(pid: int) -> str: if os.name == "nt": handle = ctypes.windll.kernel32.OpenProcess(0x101000, False, pid) @@ -480,11 +511,7 @@ try: raise SystemExit("FAIL: scheduler did not observe the forced leader exit") time.sleep(0.02) release.write_text("release\n", encoding="utf-8") - # Generous on purpose: the scheduler's refusal is the asserted state, and - # on Windows it now spends up to the descendant-probe budget (twice -- - # once in the wave loop, once in the cleanup pass) before refusing. This - # bound only has to exceed that worst case; it never decides the verdict. - stdout, stderr = process.communicate(timeout=120) + stdout, stderr = process.communicate(timeout=scheduler_wait_budget()) if os.name == "nt": # Assert the PROPERTY, not the wording. This used to require the phrase