From 82ac32f96db8c4360b6f70e9dfcbffd3082743d4 Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Thu, 20 Aug 2026 18:45:02 +0000 Subject: [PATCH 1/2] extend the unbounded lifetime to container sandboxes Containers support timeout_minutes=-1 too, but the platform requires a finite lifetime as a safety fallback when a container sets an idle timeout; give those a 30-day lifetime, effectively unbounded. The agent-timeout cap now skips prime entirely. run() polls start/get_background_job directly instead of passing a sentinel deadline to run_background_job, which cannot wait forever. Co-Authored-By: Claude Fable 5 --- verifiers/v1/runtimes/prime.py | 44 ++++++++++++++++------------------ verifiers/v1/utils/compile.py | 4 ++-- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/verifiers/v1/runtimes/prime.py b/verifiers/v1/runtimes/prime.py index c824d989a..0e913d6db 100644 --- a/verifiers/v1/runtimes/prime.py +++ b/verifiers/v1/runtimes/prime.py @@ -36,9 +36,10 @@ logger = logging.getLogger(__name__) -CONTAINER_LIFETIME = 24 * 60 * 60 -"""Fixed lifetime (seconds) of container sandboxes; only VM sandboxes support an -unbounded lifetime.""" +IDLE_FALLBACK_LIFETIME = 30 * 24 * 60 * 60 +"""Lifetime (seconds) given to container sandboxes that set an idle timeout: the +platform requires a finite lifetime there as a safety fallback should idle detection +fail. Far above any real run, so effectively unbounded.""" BASE_LABELS: list[str] = [] @@ -101,20 +102,6 @@ def _validate_egress(self) -> "PrimeConfig": ) return self - @model_validator(mode="after") - def _validate_idle_timeout(self) -> "PrimeConfig": - if ( - not self.vm - and self.idle_timeout is not None - and self.idle_timeout > CONTAINER_LIFETIME - ): - raise ValueError( - f"idle_timeout ({self.idle_timeout}s) must not exceed the " - f"{CONTAINER_LIFETIME}s ({CONTAINER_LIFETIME // 3600}h) container " - "sandbox lifetime" - ) - return self - class PrimeRuntimeInfo(PrimeConfig, BaseRuntimeInfo): image_cached: bool | None = None @@ -178,8 +165,13 @@ async def start(self) -> None: "memory_gb": self.config.memory, "disk_size_gb": self.config.disk, "gpu_count": gpu_count, - # -1 is prime's convention for no lifetime limit (VM-only) - "timeout_minutes": -1 if self.config.vm else CONTAINER_LIFETIME // 60, + # -1 is prime's convention for no lifetime limit; containers with an + # idle timeout must carry a finite lifetime as a safety fallback + "timeout_minutes": ( + -1 + if self.config.vm or idle_minutes is None + else IDLE_FALLBACK_LIFETIME // 60 + ), "idle_timeout_minutes": idle_minutes, "gpu_type": gpu_type, "region": self.config.region, @@ -281,17 +273,21 @@ async def prepare_execution(self, routes: list[str] | None) -> None: ) async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult: + # Poll the job by hand: the SDK's run_background_job needs a finite deadline, + # but the sandbox has no lifetime limit — the rollout's stage timeouts bound + # this via cancellation instead. try: - result = await self._client.run_background_job( + job = await self._client.start_background_job( self.info.id, shlex.join(argv), - # the SDK poll needs a finite deadline: the container lifetime, or - # effectively unbounded for VM sandboxes - timeout=365 * 24 * 60 * 60 if self.config.vm else CONTAINER_LIFETIME, working_dir=self.config.workdir, env=self.process_env(env), - poll_interval=1, ) + while True: + result = await self._client.get_background_job(self.info.id, job) + if result.completed: + break + await asyncio.sleep(1) except ( Exception ) as e: # a sandbox/API failure is one rollout's problem, not the eval's diff --git a/verifiers/v1/utils/compile.py b/verifiers/v1/utils/compile.py index b17127696..2ef933b0f 100644 --- a/verifiers/v1/utils/compile.py +++ b/verifiers/v1/utils/compile.py @@ -115,12 +115,12 @@ def validate_pairing( def cap_remote_agent_timeout( agent_timeout: float | None, runtime_config: RuntimeConfig, task: Task ) -> float | None: - """Remote sandboxes other than Prime VMs (which have no lifetime limit) live at + """Remote sandboxes other than Prime's (which have no lifetime limit) live at most 24 hours: cap the agent timeout there (with a warning) so a long run times out cleanly instead of the provider killing the box mid-run.""" if agent_timeout is None or runtime_is_local(runtime_config): return agent_timeout - if isinstance(runtime_config, PrimeConfig) and runtime_config.vm: + if isinstance(runtime_config, PrimeConfig): return agent_timeout if agent_timeout > 24 * 60 * 60: logger.warning( From 4c997393d2ab4f8dc0bbf95dd19841bef677fe4d Mon Sep 17 00:00:00 2001 From: Mika Senghaas Date: Thu, 20 Aug 2026 18:56:06 +0000 Subject: [PATCH 2/2] keep the container safety lifetime above the idle timeout An idle timeout past 30 days would otherwise exceed the fallback lifetime and fail the SDK's idle <= lifetime validation. Co-Authored-By: Claude Fable 5 --- verifiers/v1/runtimes/prime.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/verifiers/v1/runtimes/prime.py b/verifiers/v1/runtimes/prime.py index 0e913d6db..c45413666 100644 --- a/verifiers/v1/runtimes/prime.py +++ b/verifiers/v1/runtimes/prime.py @@ -166,11 +166,12 @@ async def start(self) -> None: "disk_size_gb": self.config.disk, "gpu_count": gpu_count, # -1 is prime's convention for no lifetime limit; containers with an - # idle timeout must carry a finite lifetime as a safety fallback + # idle timeout must carry a finite lifetime as a safety fallback (which + # must exceed the idle timeout) "timeout_minutes": ( -1 if self.config.vm or idle_minutes is None - else IDLE_FALLBACK_LIFETIME // 60 + else max(IDLE_FALLBACK_LIFETIME // 60, idle_minutes + 1) ), "idle_timeout_minutes": idle_minutes, "gpu_type": gpu_type,