diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index 1548eb24..305720c0 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -521,7 +521,7 @@ The `sandbox` block is optional. When omitted, it defaults to `driver: "tempdir" ```yaml sandbox: driver: "tempdir" # Sandbox type ("tempdir" or "docker"); default: "tempdir" - python: # Python env config (null to skip venv) + python: # Python env config (a venv is created only if env_packages is non-empty) env_packages: # Packages to install in sandbox venv - pytest - pylint>=3.0 @@ -712,7 +712,7 @@ Every sandbox-relative path field accepts a glob — `path` on `file_exists`, `f Rules: - **A path that exists is never treated as a pattern.** A literal `path` behaves exactly as before, including one containing `*`, `?`, or `[` — a real file named `report[2024].json` is graded as itself, not as a character class that would match `report2.json`. Globbing only kicks in when the literal path does not exist. -- **Glob matches skip ignored directories.** Expansion runs over the live sandbox root, which also holds harness-created content the agent never wrote (`.venv` for any task with a `python:` block, `node_modules`, `dist`, `build`, `__pycache__`, …), so matches are filtered through the same [`ignore_patterns`](#sandbox-configuration) set used for template copying. A segment your pattern names *literally* is an opt-in and survives, so `dist/**/*.js` still grades `dist`; to un-ignore a directory a wildcard has to discover, use the negation escape hatch — `ignore_patterns: ["!dist"]`. +- **Glob matches skip ignored directories.** Expansion runs over the live sandbox root, which also holds harness-created content the agent never wrote (`.venv` for any task whose `python:` block lists `env_packages`, `node_modules`, `dist`, `build`, `__pycache__`, …), so matches are filtered through the same [`ignore_patterns`](#sandbox-configuration) set used for template copying. A segment your pattern names *literally* is an opt-in and survives, so `dist/**/*.js` still grades `dist`; to un-ignore a directory a wildcard has to discover, use the negation escape hatch — `ignore_patterns: ["!dist"]`. - Matches are sorted, and directories are skipped. - `file_exists` passes when the glob matches **at least one** file. - Content checks require the glob to match **exactly one** file. An ambiguous glob scores 0.0 and reports the matches (first 10, then `+N more`) rather than silently grading one of them — narrow the pattern. diff --git a/src/coder_eval/models/sandbox.py b/src/coder_eval/models/sandbox.py index 94108c47..746bad4e 100644 --- a/src/coder_eval/models/sandbox.py +++ b/src/coder_eval/models/sandbox.py @@ -428,7 +428,11 @@ class SandboxConfig(BaseModel): ) python: PythonEnvConfig | None = Field( default_factory=PythonEnvConfig, - description="Python environment config; set to null in YAML (or None in Python) to skip venv creation", + description=( + "Python environment config. A venv is created only when `env_packages` is non-empty -- an empty " + "venv would shadow the ambient interpreter without providing anything. Set to null in YAML (or " + "None in Python) to also opt out of adopting a venv the agent created itself." + ), ) node: NodeEnvConfig | None = Field( default=None, diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 73201419..043d4f5c 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -405,13 +405,18 @@ def _setup_tempdir(self, target_dir: Path | None = None) -> Path: # Mark mock binaries executable so the agent's PATH can shadow real CLIs self._prepare_mock_path_dirs() - # Set up Python virtual environment (only if python config is provided) - if self.config.python: + # Set up a Python virtual environment only when there are packages to + # put in it. `config.python` defaults to a `PythonEnvConfig()` INSTANCE, + # so gating creation on the object alone built a venv for every task and + # then installed nothing into it. An empty venv is not neutral: `uv venv` + # puts no pip in it, so inside a task image that provisions packages + # globally, `python` resolved to the venv (import fails) while `pip` + # resolved to the image's global pip (reports the package present), and + # the venv's bin/ was prepended to PATH for the agent and for every + # `run_command` criterion. Nothing to install means nothing to create. + if self.config.python and self.config.python.env_packages: self._setup_virtualenv() - - # Install required packages - if self.config.python.env_packages: - self._install_packages() + self._install_packages() # Install Node.js packages if self.config.node and self.config.node.env_packages: diff --git a/tests/test_sandbox.py b/tests/test_sandbox.py index 1ae4e451..782bb7a8 100644 --- a/tests/test_sandbox.py +++ b/tests/test_sandbox.py @@ -61,12 +61,10 @@ def test_tempdir_sandbox_basic(): assert sandbox_dir.exists() assert sandbox_dir.is_dir() - # Check venv was created - venv_dir = sandbox_dir / ".venv" - assert venv_dir.exists() - scripts_dir = "Scripts" if os.name == "nt" else "bin" - python_name = "python.exe" if os.name == "nt" else "python" - assert (venv_dir / scripts_dir / python_name).exists() + # The default config asks for no packages, so it gets no venv; the + # sandbox runs under the ambient interpreter instead. + assert not (sandbox_dir / ".venv").exists() + assert sandbox.venv_dir is None finally: # Cleanup @@ -486,7 +484,12 @@ def test_sandbox_with_packages(): sandbox = Sandbox(config, task_id="test_packages") try: - sandbox.setup() + sandbox_dir = sandbox.setup() + + # Asking for packages is what earns a venv (see + # test_default_python_config_creates_no_venv_when_nothing_to_install). + assert (sandbox_dir / ".venv").exists() + assert sandbox.venv_dir == sandbox_dir / ".venv" # Test that requests is installed exit_code, stdout, stderr = sandbox.run_command('python -c "import requests; print(requests.__version__)"') @@ -1310,3 +1313,27 @@ def test_absent_when_the_task_declares_no_reference(self, tmp_path): assert "REFERENCE_DIR" not in sb._build_run_command_env() finally: sb.cleanup(preserve=False) + + +def test_default_python_config_creates_no_venv_when_nothing_to_install(): + """An empty `env_packages` must not leave an empty venv behind. + + `SandboxConfig.python` defaults to a `PythonEnvConfig()` instance, so every + task used to reach `_setup_virtualenv()` and get a `uv venv` with no packages + and no pip in it. That empty venv then shadowed a fully provisioned + environment: inside a task image with packages installed globally, `python` + resolved to the venv (import fails) while `pip` resolved to the image's + global pip (reports the package installed). Agents burned their turn budget + on that contradiction. Nothing to install means nothing to create. + """ + config = SandboxConfig(driver="tempdir") + assert config.python is not None, "default is an instance, not None -- the case this guards" + assert config.python.env_packages == [] + + sandbox = Sandbox(config, task_id="test_default_no_empty_venv") + try: + sandbox_dir = sandbox.setup() + assert not (sandbox_dir / ".venv").exists() + assert sandbox.venv_dir is None + finally: + sandbox.cleanup() diff --git a/tests/test_sandbox_templates.py b/tests/test_sandbox_templates.py index 12179300..43e4e9fb 100644 --- a/tests/test_sandbox_templates.py +++ b/tests/test_sandbox_templates.py @@ -34,8 +34,10 @@ def test_template_dir_basic(self, tmp_path): assert (sandbox_path / "main.py").read_text() == "print('hello')" assert (sandbox_path / "README.md").exists() - # Verify venv created (separate from template) - assert (sandbox_path / ".venv").exists() + # This config asks for no packages, so it gets no venv. Nothing here + # rides on one: `_setup_template` runs BEFORE venv creation, so a + # template copy can never clobber a venv in the first place. + assert not (sandbox_path / ".venv").exists() finally: sandbox.cleanup(preserve=False)