feat(benchmark): pre-bake hermes agent into dataset images for closed-book runs

benchmark/agent-versions.env

@@ -5,3 +5,9 @@ CLAUDE_CODE_VERSION=2.1.211
 CODEX_VERSION=0.144.5
 OPENCODE_VERSION=1.18.3
 NODE_VERSION=20.11.1
+# Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time.
+# Must be an immutable ref — a tag or commit SHA. A moving ref such as "main" is
+# rejected at build time: the run manifest records this value as a pin, and two
+# builds recording the same string while installing different code is worse than
+# recording nothing. The installer script is fetched from this ref too.
+HERMES_VERSION=v2026.8.3

benchmark/patches/harbor-agent-patches.diff

@@ -426,3 +426,105 @@
 +++ b/harbor/switchyard_patch_id.txt
 @@ -0,0 +1 @@
 +switchyard-harbor-patches-2026-05-22-v2
+
+--- a/harbor/agents/installed/hermes.py
++++ b/harbor/agents/installed/hermes.py
+@@ -59,6 +59,16 @@
+         return 'export PATH="$HOME/.local/bin:$PATH"; hermes version'
+
+     async def install(self, environment: BaseEnvironment) -> None:
++        # Skip if hermes is already installed (e.g. from the dataset
++        # agent-bake layer). hermes installs to $HOME/.local/bin, so export
++        # that path before probing. Required for closed-book runs where the
++        # task-time curl install cannot reach github/pypi.
++        probe = await self.exec_as_agent(
++            environment,
++            command='export PATH="$HOME/.local/bin:$PATH"; command -v hermes >/dev/null 2>&1 && echo present || true',
++        )
++        if "present" in (getattr(probe, "stdout", "") or ""):
++            return
+         await self.exec_as_root(
+             environment,
+             command="apt-get update && apt-get install -y curl git ripgrep xz-utils",
+@@ -92,11 +92,31 @@
+     # ------------------------------------------------------------------
+
+     @staticmethod
+-    def _build_config_yaml(model: str) -> str:
+-        """Generate a hermes config.yaml with full capabilities enabled."""
++    def _build_config_yaml(
++        model: str, base_url: str | None = None, api_key: str | None = None
++    ) -> str:
++        """Generate a hermes config.yaml with full capabilities enabled.
++
++        When ``base_url`` is given, route the model through a custom
++        OpenAI-compatible endpoint (e.g. a Switchyard gateway): hermes only
++        honors a custom endpoint via its config.yaml ``model.provider=custom`` +
++        ``model.base_url`` (the ``OPENAI_BASE_URL`` env var is ignored for the
++        chat model), so we nest the provider/base_url/api_key under ``model``.
++        """
++        model_field: Any
++        if base_url:
++            model_field = {
++                "default": model,
++                "provider": "custom",
++                "base_url": base_url,
++            }
++            if api_key:
++                model_field["api_key"] = api_key
++        else:
++            model_field = model
+         config: dict[str, Any] = {
+-            "model": model,
+-            "provider": "auto",
++            "model": model_field,
++            **({} if base_url else {"provider": "auto"}),
+             "toolsets": ["hermes-cli"],
+             "agent": {"max_turns": 90},
+             "memory": {
+@@ -357,6 +377,9 @@
+         # Try native provider key first, fall back to OpenRouter.
+         hermes_provider_flag: str | None = None
+         use_native = False
++        # Custom OpenAI-compatible endpoint (e.g. Switchyard) for the chat model.
++        custom_base_url: str | None = None
++        custom_api_key: str | None = None
+
+         if provider in _NATIVE_PROVIDERS:
+             native_flag, key_names = _NATIVE_PROVIDERS[provider]
+@@ -367,11 +390,15 @@
+                     hermes_provider_flag = native_flag
+                     use_native = True
+                     break
+-            # Forward OPENAI_BASE_URL when using native OpenAI key
++            # Forward OPENAI_BASE_URL when using native OpenAI key. Hermes ignores
++            # this env var for the chat model, so also thread it into config.yaml
++            # as a custom provider (see _build_config_yaml).
+             if use_native and provider == "openai":
+                 base_url = os.environ.get("OPENAI_BASE_URL")
+                 if base_url:
+                     env["OPENAI_BASE_URL"] = base_url
++                    custom_base_url = base_url
++                    custom_api_key = os.environ.get("OPENAI_API_KEY")
+
+         if not use_native:
+             openrouter_key = os.environ.get("OPENROUTER_API_KEY")
+@@ -386,9 +413,15 @@
+             env["OPENROUTER_API_KEY"] = openrouter_key
+
+         # Native providers with --provider flag use just the model name;
+-        # everything else (OpenRouter, openai direct) uses provider/model.
+-        cli_model = model if hermes_provider_flag else self.model_name
+-        config_yaml = self._build_config_yaml(cli_model)
++        # a custom OpenAI-compatible endpoint also uses the bare model (the
++        # base_url already targets the gateway); everything else uses provider/model.
++        if custom_base_url:
++            cli_model = model
++        else:
++            cli_model = model if hermes_provider_flag else self.model_name
++        config_yaml = self._build_config_yaml(
++            cli_model, base_url=custom_base_url, api_key=custom_api_key
++        )
+
+         # Pass instruction via env var (safe from shell escaping issues)
+         env["HARBOR_INSTRUCTION"] = instruction

benchmark/prepare_harbor_dataset.py

@@ -187,10 +187,27 @@ def _install_layer(pins: dict[str, str]) -> str:
     claude_version = pins["CLAUDE_CODE_VERSION"]
     codex_version = pins["CODEX_VERSION"]
     opencode_version = pins["OPENCODE_VERSION"]
+    # Hermes (NousResearch hermes-agent) is a per-user uv app installed from
+    # GitHub, not an npm package. Baking it here (build-time, with host network)
+    # means the runtime install() skip-guard short-circuits, so tasks need no
+    # egress for it — enabling closed-book Hermes runs.
+    #
+    # HERMES_VERSION must name an immutable ref. A moving ref would let two builds
+    # record the same version string while installing different code, which is worse
+    # than not recording it: the manifest would assert a reproducibility it does not
+    # have. The installer script is fetched from the same ref for the same reason —
+    # pinning the agent but running whatever installer main has today reintroduces
+    # exactly the drift the pin exists to prevent.
+    hermes_version = pins["HERMES_VERSION"]
+    if hermes_version in ("main", "master", "HEAD"):
+        raise SystemExit(
+            f"HERMES_VERSION={hermes_version!r} is a moving ref and cannot be recorded "
+            "as a reproducible pin; use a tag or commit SHA"
+        )
     return f"""
 
 # Switchyard benchmark prebaked coding agents.
-ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version}"
+ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version},hermes={hermes_version}"
 RUN set -eux; \\
     if command -v apt-get >/dev/null 2>&1; then \\
         apt-get update; \\
@@ -231,6 +248,19 @@ def _install_layer(pins: dict[str, str]) -> str:
     claude --version; \\
     codex --version; \\
     opencode --version
+RUN set -eux; \\
+    export HOME=/root; \\
+    export PATH="/root/.local/bin:$PATH"; \\
+    if command -v apt-get >/dev/null 2>&1; then \\
+        apt-get update; \\
+        apt-get install -y --no-install-recommends git ripgrep xz-utils; \\
+        rm -rf /var/lib/apt/lists/*; \\
+    elif command -v apk >/dev/null 2>&1; then \\
+        apk add --no-cache git ripgrep xz; \\
+    fi; \\
+    curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/{hermes_version}/scripts/install.sh \\
+        | bash -s -- --skip-setup --branch {hermes_version}; \\
+    hermes version
 """
 
 

tests/test_prepare_harbor_dataset.py

@@ -8,6 +8,7 @@
 from pathlib import Path
 from types import ModuleType
 
+import pytest
 import yaml
 
 REPO = Path(__file__).resolve().parents[1]
@@ -284,6 +285,7 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Pat
     assert manifest["agent_versions"] == {
         "CLAUDE_CODE_VERSION": "2.1.211",
         "CODEX_VERSION": "0.144.5",
+        "HERMES_VERSION": "v2026.8.3",
         "NODE_VERSION": "20.11.1",
         "OPENCODE_VERSION": "1.18.3",
     }
@@ -306,3 +308,39 @@ def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None:
     proxy_env = "\n".join(compose["services"]["proxy"]["environment"])
     assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_env
     assert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_env
+
+def test_a_moving_hermes_ref_is_rejected() -> None:
+    """A moving ref cannot be recorded as a pin.
+
+    The dataset manifest presents HERMES_VERSION as a reproducibility guarantee. Two
+    builds recording the same string while installing different Hermes code is worse
+    than recording nothing, so the build fails rather than asserting a pin it does not
+    have.
+    """
+    base = {
+        "CLAUDE_CODE_VERSION": "1",
+        "CODEX_VERSION": "2",
+        "OPENCODE_VERSION": "3",
+        "NODE_VERSION": "4",
+    }
+    for moving in ("main", "master", "HEAD"):
+        with pytest.raises(SystemExit, match="moving ref"):
+            _load_generator_module()._install_layer({**base, "HERMES_VERSION": moving})
+
+
+def test_the_hermes_installer_is_fetched_at_the_pinned_ref() -> None:
+    """Pinning the agent but running main's installer reintroduces the same drift."""
+    pins = {
+        "CLAUDE_CODE_VERSION": "1",
+        "CODEX_VERSION": "2",
+        "OPENCODE_VERSION": "3",
+        "NODE_VERSION": "4",
+        "HERMES_VERSION": "v2026.8.3",
+    }
+
+    layer = _load_generator_module()._install_layer(pins)
+
+    assert "hermes-agent/v2026.8.3/scripts/install.sh" in layer
+    assert "hermes-agent/main/" not in layer
+    assert "--branch v2026.8.3" in layer
+