From 9b155120f99f51776de6726797f1e2115266f2a8 Mon Sep 17 00:00:00 2001 From: Andrei Petraru Date: Thu, 10 Sep 2026 16:31:57 +0300 Subject: [PATCH] fix(sandbox): stop creating an empty venv that shadows the image's packages `SandboxConfig.python` defaults to a `PythonEnvConfig()` INSTANCE, and setup gated venv creation on that object alone. Every task therefore got a `uv venv` with nothing installed into it. An empty venv is not neutral. `uv venv` puts no pip in it, so inside a task image that provisions packages globally the two halves disagree: which pip -> /usr/local/bin/pip (image global) pip list -> langchain 1.4.0 (installed!) python -c "import langchain" -> ModuleNotFoundError and `venv/bin` was prepended to PATH for the agent AND for every `run_command` criterion, so criteria were graded under that same empty interpreter. Measured cost, run 2026-09-10_04-18-49, `skill-agent-guardrail-coded-escalation-smoke`: the agent finished the guardrail at turn 19, then spent turns 22-31 chasing the pip-vs-python contradiction, repaired it with `uv sync`, and hit `max_turns: 40` at turn 41 before writing the app resource into bindings.json. Scored 0.80 with correct code. The same task passes in 19-21 turns on the host driver, where there is no global environment for the empty venv to contradict. Nothing to install means nothing to create. Two existing tests asserted the old behavior (a venv from a config that requested no packages) and are updated; `_setup_template` already runs before venv creation, so the template test's "venv survives the copy" assertion was ordering-redundant. Co-Authored-By: Claude Opus 5 (1M context) --- docs/TASK_DEFINITION_GUIDE.md | 4 ++-- src/coder_eval/models/sandbox.py | 6 ++++- src/coder_eval/sandbox.py | 17 ++++++++----- tests/test_sandbox.py | 41 ++++++++++++++++++++++++++------ tests/test_sandbox_templates.py | 6 +++-- 5 files changed, 56 insertions(+), 18 deletions(-) diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 1548eb24..305720c0 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -521,7 +521,7 @@ The `sandbox` block is optional. When omitted, it defaults to `driver: "tempdir" ```yaml sandbox: driver: "tempdir" # Sandbox type ("tempdir" or "docker"); default: "tempdir" - python: # Python env config (null to skip venv) + python: # Python env config (a venv is created only if env_packages is non-empty) env_packages: # Packages to install in sandbox venv - pytest - pylint>=3.0 @@ -712,7 +712,7 @@ Every sandbox-relative path field accepts a glob — `path` on `file_exists`, `f Rules: - **A path that exists is never treated as a pattern.** A literal `path` behaves exactly as before, including one containing `*`, `?`, or `[` — a real file named `report[2024].json` is graded as itself, not as a character class that would match `report2.json`. Globbing only kicks in when the literal path does not exist. -- **Glob matches skip ignored directories.** Expansion runs over the live sandbox root, which also holds harness-created content the agent never wrote (`.venv` for any task with a `python:` block, `node_modules`, `dist`, `build`, `__pycache__`, …), so matches are filtered through the same [`ignore_patterns`](#sandbox-configuration) set used for template copying. A segment your pattern names *literally* is an opt-in and survives, so `dist/**/*.js` still grades `dist`; to un-ignore a directory a wildcard has to discover, use the negation escape hatch — `ignore_patterns: ["!dist"]`. +- **Glob matches skip ignored directories.** Expansion runs over the live sandbox root, which also holds harness-created content the agent never wrote (`.venv` for any task whose `python:` block lists `env_packages`, `node_modules`, `dist`, `build`, `__pycache__`, …), so matches are filtered through the same [`ignore_patterns`](#sandbox-configuration) set used for template copying. A segment your pattern names *literally* is an opt-in and survives, so `dist/**/*.js` still grades `dist`; to un-ignore a directory a wildcard has to discover, use the negation escape hatch — `ignore_patterns: ["!dist"]`. - Matches are sorted, and directories are skipped. - `file_exists` passes when the glob matches **at least one** file. - Content checks require the glob to match **exactly one** file. An ambiguous glob scores 0.0 and reports the matches (first 10, then `+N more`) rather than silently grading one of them — narrow the pattern. diff --git a/src/coder_eval/models/sandbox.py b/src/coder_eval/models/sandbox.py index 94108c47..746bad4e 100644 --- a/src/coder_eval/models/sandbox.py +++ b/src/coder_eval/models/sandbox.py @@ -428,7 +428,11 @@ class SandboxConfig(BaseModel): ) python: PythonEnvConfig | None = Field( default_factory=PythonEnvConfig, - description="Python environment config; set to null in YAML (or None in Python) to skip venv creation", + description=( + "Python environment config. A venv is created only when `env_packages` is non-empty -- an empty " + "venv would shadow the ambient interpreter without providing anything. Set to null in YAML (or " + "None in Python) to also opt out of adopting a venv the agent created itself." + ), ) node: NodeEnvConfig | None = Field( default=None, diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 73201419..043d4f5c 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -405,13 +405,18 @@ def _setup_tempdir(self, target_dir: Path | None = None) -> Path: # Mark mock binaries executable so the agent's PATH can shadow real CLIs self._prepare_mock_path_dirs() - # Set up Python virtual environment (only if python config is provided) - if self.config.python: + # Set up a Python virtual environment only when there are packages to + # put in it. `config.python` defaults to a `PythonEnvConfig()` INSTANCE, + # so gating creation on the object alone built a venv for every task and + # then installed nothing into it. An empty venv is not neutral: `uv venv` + # puts no pip in it, so inside a task image that provisions packages + # globally, `python` resolved to the venv (import fails) while `pip` + # resolved to the image's global pip (reports the package present), and + # the venv's bin/ was prepended to PATH for the agent and for every + # `run_command` criterion. Nothing to install means nothing to create. + if self.config.python and self.config.python.env_packages: self._setup_virtualenv() - - # Install required packages - if self.config.python.env_packages: - self._install_packages() + self._install_packages() # Install Node.js packages if self.config.node and self.config.node.env_packages: diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 1ae4e451..782bb7a8 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -61,12 +61,10 @@ def test_tempdir_sandbox_basic(): assert sandbox_dir.exists() assert sandbox_dir.is_dir() - # Check venv was created - venv_dir = sandbox_dir / ".venv" - assert venv_dir.exists() - scripts_dir = "Scripts" if os.name == "nt" else "bin" - python_name = "python.exe" if os.name == "nt" else "python" - assert (venv_dir / scripts_dir / python_name).exists() + # The default config asks for no packages, so it gets no venv; the + # sandbox runs under the ambient interpreter instead. + assert not (sandbox_dir / ".venv").exists() + assert sandbox.venv_dir is None finally: # Cleanup @@ -486,7 +484,12 @@ def test_sandbox_with_packages(): sandbox = Sandbox(config, task_id="test_packages") try: - sandbox.setup() + sandbox_dir = sandbox.setup() + + # Asking for packages is what earns a venv (see + # test_default_python_config_creates_no_venv_when_nothing_to_install). + assert (sandbox_dir / ".venv").exists() + assert sandbox.venv_dir == sandbox_dir / ".venv" # Test that requests is installed exit_code, stdout, stderr = sandbox.run_command('python -c "import requests; print(requests.__version__)"') @@ -1310,3 +1313,27 @@ def test_absent_when_the_task_declares_no_reference(self, tmp_path): assert "REFERENCE_DIR" not in sb._build_run_command_env() finally: sb.cleanup(preserve=False) + + +def test_default_python_config_creates_no_venv_when_nothing_to_install(): + """An empty `env_packages` must not leave an empty venv behind. + + `SandboxConfig.python` defaults to a `PythonEnvConfig()` instance, so every + task used to reach `_setup_virtualenv()` and get a `uv venv` with no packages + and no pip in it. That empty venv then shadowed a fully provisioned + environment: inside a task image with packages installed globally, `python` + resolved to the venv (import fails) while `pip` resolved to the image's + global pip (reports the package installed). Agents burned their turn budget + on that contradiction. Nothing to install means nothing to create. + """ + config = SandboxConfig(driver="tempdir") + assert config.python is not None, "default is an instance, not None -- the case this guards" + assert config.python.env_packages == [] + + sandbox = Sandbox(config, task_id="test_default_no_empty_venv") + try: + sandbox_dir = sandbox.setup() + assert not (sandbox_dir / ".venv").exists() + assert sandbox.venv_dir is None + finally: + sandbox.cleanup() diff --git a/tests/test_sandbox_templates.py b/tests/test_sandbox_templates.py index 12179300..43e4e9fb 100644 --- a/tests/test_sandbox_templates.py +++ b/tests/test_sandbox_templates.py @@ -34,8 +34,10 @@ def test_template_dir_basic(self, tmp_path): assert (sandbox_path / "main.py").read_text() == "print('hello')" assert (sandbox_path / "README.md").exists() - # Verify venv created (separate from template) - assert (sandbox_path / ".venv").exists() + # This config asks for no packages, so it gets no venv. Nothing here + # rides on one: `_setup_template` runs BEFORE venv creation, so a + # template copy can never clobber a venv in the first place. + assert not (sandbox_path / ".venv").exists() finally: sandbox.cleanup(preserve=False)