From 9a11d1b987530e4723b79c71ad94942956f72f7e Mon Sep 17 00:00:00 2001 From: Subhashree Radhakrishnan Date: Fri, 17 Jul 2026 05:31:59 +0000 Subject: [PATCH 1/3] feat(benchmark): pre-bake hermes agent into dataset images for closed-book runs Closed-book Harbor runs cannot curl-install agents at task time. Bake the NousResearch hermes-agent into the dataset image install layer (pinned via HERMES_VERSION in benchmark/agent-versions.env) and add an install-skip probe to the harbor hermes agent patch so the runtime install() short-circuits when hermes is already present. The agent patch also threads a custom OpenAI-compatible base_url (e.g. a Switchyard gateway) into hermes' config.yaml, which hermes requires for a custom chat-model endpoint (OPENAI_BASE_URL alone is ignored for the chat model). Co-Authored-By: Claude Fable 5 Signed-off-by: Subhashree Radhakrishnan --- benchmark/agent-versions.env | 3 + benchmark/patches/harbor-agent-patches.diff | 102 ++++++++++++++++++++ benchmark/prepare_harbor_dataset.py | 24 ++++- 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/benchmark/agent-versions.env b/benchmark/agent-versions.env index 843959969..f356b7858 100644 --- a/benchmark/agent-versions.env +++ b/benchmark/agent-versions.env @@ -5,3 +5,6 @@ CLAUDE_CODE_VERSION=2.1.211 CODEX_VERSION=0.144.5 OPENCODE_VERSION=1.18.3 NODE_VERSION=20.11.1 +# Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time. +# "main" tracks latest; set to a tag/commit to pin for reproducibility. +HERMES_VERSION=main diff --git a/benchmark/patches/harbor-agent-patches.diff b/benchmark/patches/harbor-agent-patches.diff index bc62838d1..54454cd68 100644 --- a/benchmark/patches/harbor-agent-patches.diff +++ b/benchmark/patches/harbor-agent-patches.diff @@ -426,3 +426,105 @@ +++ b/harbor/switchyard_patch_id.txt @@ -0,0 +1 @@ +switchyard-harbor-patches-2026-05-22-v2 + +--- a/harbor/agents/installed/hermes.py ++++ b/harbor/agents/installed/hermes.py +@@ -59,6 +59,16 @@ + return 'export PATH="$HOME/.local/bin:$PATH"; hermes version' + + async def install(self, environment: BaseEnvironment) -> None: ++ # Skip if hermes is already installed (e.g. from the dataset ++ # agent-bake layer). hermes installs to $HOME/.local/bin, so export ++ # that path before probing. Required for closed-book runs where the ++ # task-time curl install cannot reach github/pypi. ++ probe = await self.exec_as_agent( ++ environment, ++ command='export PATH="$HOME/.local/bin:$PATH"; command -v hermes >/dev/null 2>&1 && echo present || true', ++ ) ++ if "present" in (getattr(probe, "stdout", "") or ""): ++ return + await self.exec_as_root( + environment, + command="apt-get update && apt-get install -y curl git ripgrep xz-utils", +@@ -92,11 +92,31 @@ + # ------------------------------------------------------------------ + + @staticmethod +- def _build_config_yaml(model: str) -> str: +- """Generate a hermes config.yaml with full capabilities enabled.""" ++ def _build_config_yaml( ++ model: str, base_url: str | None = None, api_key: str | None = None ++ ) -> str: ++ """Generate a hermes config.yaml with full capabilities enabled. ++ ++ When ``base_url`` is given, route the model through a custom ++ OpenAI-compatible endpoint (e.g. a Switchyard gateway): hermes only ++ honors a custom endpoint via its config.yaml ``model.provider=custom`` + ++ ``model.base_url`` (the ``OPENAI_BASE_URL`` env var is ignored for the ++ chat model), so we nest the provider/base_url/api_key under ``model``. ++ """ ++ model_field: Any ++ if base_url: ++ model_field = { ++ "default": model, ++ "provider": "custom", ++ "base_url": base_url, ++ } ++ if api_key: ++ model_field["api_key"] = api_key ++ else: ++ model_field = model + config: dict[str, Any] = { +- "model": model, +- "provider": "auto", ++ "model": model_field, ++ **({} if base_url else {"provider": "auto"}), + "toolsets": ["hermes-cli"], + "agent": {"max_turns": 90}, + "memory": { +@@ -357,6 +377,9 @@ + # Try native provider key first, fall back to OpenRouter. + hermes_provider_flag: str | None = None + use_native = False ++ # Custom OpenAI-compatible endpoint (e.g. Switchyard) for the chat model. ++ custom_base_url: str | None = None ++ custom_api_key: str | None = None + + if provider in _NATIVE_PROVIDERS: + native_flag, key_names = _NATIVE_PROVIDERS[provider] +@@ -367,11 +390,15 @@ + hermes_provider_flag = native_flag + use_native = True + break +- # Forward OPENAI_BASE_URL when using native OpenAI key ++ # Forward OPENAI_BASE_URL when using native OpenAI key. Hermes ignores ++ # this env var for the chat model, so also thread it into config.yaml ++ # as a custom provider (see _build_config_yaml). + if use_native and provider == "openai": + base_url = os.environ.get("OPENAI_BASE_URL") + if base_url: + env["OPENAI_BASE_URL"] = base_url ++ custom_base_url = base_url ++ custom_api_key = os.environ.get("OPENAI_API_KEY") + + if not use_native: + openrouter_key = os.environ.get("OPENROUTER_API_KEY") +@@ -386,9 +413,15 @@ + env["OPENROUTER_API_KEY"] = openrouter_key + + # Native providers with --provider flag use just the model name; +- # everything else (OpenRouter, openai direct) uses provider/model. +- cli_model = model if hermes_provider_flag else self.model_name +- config_yaml = self._build_config_yaml(cli_model) ++ # a custom OpenAI-compatible endpoint also uses the bare model (the ++ # base_url already targets the gateway); everything else uses provider/model. ++ if custom_base_url: ++ cli_model = model ++ else: ++ cli_model = model if hermes_provider_flag else self.model_name ++ config_yaml = self._build_config_yaml( ++ cli_model, base_url=custom_base_url, api_key=custom_api_key ++ ) + + # Pass instruction via env var (safe from shell escaping issues) + env["HARBOR_INSTRUCTION"] = instruction diff --git a/benchmark/prepare_harbor_dataset.py b/benchmark/prepare_harbor_dataset.py index fb30a8ce3..14719feb6 100644 --- a/benchmark/prepare_harbor_dataset.py +++ b/benchmark/prepare_harbor_dataset.py @@ -187,10 +187,19 @@ def _install_layer(pins: dict[str, str]) -> str: claude_version = pins["CLAUDE_CODE_VERSION"] codex_version = pins["CODEX_VERSION"] opencode_version = pins["OPENCODE_VERSION"] + # Hermes (NousResearch hermes-agent) is a per-user uv app installed from + # GitHub, not an npm package. Default to main; pin a tag/commit via + # HERMES_VERSION for reproducibility. Baking it here (build-time, with host + # network) means the runtime install() skip-guard short-circuits, so tasks + # need no egress for it — enabling closed-book Hermes runs. + hermes_version = pins.get("HERMES_VERSION", "main") + hermes_branch_flag = ( + f" --branch {hermes_version}" if hermes_version and hermes_version != "main" else "" + ) return f""" # Switchyard benchmark prebaked coding agents. -ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version}" +ENV SWITCHYARD_PREBAKED_AGENT_VERSIONS="claude-code={claude_version},codex={codex_version},opencode={opencode_version},node={node_version},hermes={hermes_version}" RUN set -eux; \\ if command -v apt-get >/dev/null 2>&1; then \\ apt-get update; \\ @@ -231,6 +240,19 @@ def _install_layer(pins: dict[str, str]) -> str: claude --version; \\ codex --version; \\ opencode --version +RUN set -eux; \\ + export HOME=/root; \\ + export PATH="/root/.local/bin:$PATH"; \\ + if command -v apt-get >/dev/null 2>&1; then \\ + apt-get update; \\ + apt-get install -y --no-install-recommends git ripgrep xz-utils; \\ + rm -rf /var/lib/apt/lists/*; \\ + elif command -v apk >/dev/null 2>&1; then \\ + apk add --no-cache git ripgrep xz; \\ + fi; \\ + curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh \\ + | bash -s -- --skip-setup{hermes_branch_flag}; \\ + hermes version """ From 9ccc416d392a09f58c73bf0b36517d4b8a508360 Mon Sep 17 00:00:00 2001 From: Giedrius Burachas Date: Thu, 30 Jul 2026 15:33:32 +0000 Subject: [PATCH 2/3] test(benchmark): record hermes pin in dataset manifest expectation Signed-off-by: Giedrius Burachas --- tests/test_prepare_harbor_dataset.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.py index 1c69052ff..14493fc02 100644 --- a/tests/test_prepare_harbor_dataset.py +++ b/tests/test_prepare_harbor_dataset.py @@ -284,6 +284,7 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Pat assert manifest["agent_versions"] == { "CLAUDE_CODE_VERSION": "2.1.211", "CODEX_VERSION": "0.144.5", + "HERMES_VERSION": "main", "NODE_VERSION": "20.11.1", "OPENCODE_VERSION": "1.18.3", } From 4fcdde205713651afcd4bac435fe3e39af6dd5c0 Mon Sep 17 00:00:00 2001 From: Giedrius Burachas Date: Tue, 4 Aug 2026 18:18:43 +0000 Subject: [PATCH 3/3] fix(benchmark): pin hermes to an immutable ref and fetch its installer there Signed-off-by: Giedrius Burachas --- benchmark/agent-versions.env | 7 +++-- benchmark/prepare_harbor_dataset.py | 28 +++++++++++++------- tests/test_prepare_harbor_dataset.py | 39 +++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/benchmark/agent-versions.env b/benchmark/agent-versions.env index f356b7858..da2a84ea8 100644 --- a/benchmark/agent-versions.env +++ b/benchmark/agent-versions.env @@ -6,5 +6,8 @@ CODEX_VERSION=0.144.5 OPENCODE_VERSION=1.18.3 NODE_VERSION=20.11.1 # Hermes (NousResearch hermes-agent), installed from GitHub at dataset-bake time. -# "main" tracks latest; set to a tag/commit to pin for reproducibility. -HERMES_VERSION=main +# Must be an immutable ref — a tag or commit SHA. A moving ref such as "main" is +# rejected at build time: the run manifest records this value as a pin, and two +# builds recording the same string while installing different code is worse than +# recording nothing. The installer script is fetched from this ref too. +HERMES_VERSION=v2026.8.3 diff --git a/benchmark/prepare_harbor_dataset.py b/benchmark/prepare_harbor_dataset.py index 14719feb6..371d991f2 100644 --- a/benchmark/prepare_harbor_dataset.py +++ b/benchmark/prepare_harbor_dataset.py @@ -188,14 +188,22 @@ def _install_layer(pins: dict[str, str]) -> str: codex_version = pins["CODEX_VERSION"] opencode_version = pins["OPENCODE_VERSION"] # Hermes (NousResearch hermes-agent) is a per-user uv app installed from - # GitHub, not an npm package. Default to main; pin a tag/commit via - # HERMES_VERSION for reproducibility. Baking it here (build-time, with host - # network) means the runtime install() skip-guard short-circuits, so tasks - # need no egress for it — enabling closed-book Hermes runs. - hermes_version = pins.get("HERMES_VERSION", "main") - hermes_branch_flag = ( - f" --branch {hermes_version}" if hermes_version and hermes_version != "main" else "" - ) + # GitHub, not an npm package. Baking it here (build-time, with host network) + # means the runtime install() skip-guard short-circuits, so tasks need no + # egress for it — enabling closed-book Hermes runs. + # + # HERMES_VERSION must name an immutable ref. A moving ref would let two builds + # record the same version string while installing different code, which is worse + # than not recording it: the manifest would assert a reproducibility it does not + # have. The installer script is fetched from the same ref for the same reason — + # pinning the agent but running whatever installer main has today reintroduces + # exactly the drift the pin exists to prevent. + hermes_version = pins["HERMES_VERSION"] + if hermes_version in ("main", "master", "HEAD"): + raise SystemExit( + f"HERMES_VERSION={hermes_version!r} is a moving ref and cannot be recorded " + "as a reproducible pin; use a tag or commit SHA" + ) return f""" # Switchyard benchmark prebaked coding agents. @@ -250,8 +258,8 @@ def _install_layer(pins: dict[str, str]) -> str: elif command -v apk >/dev/null 2>&1; then \\ apk add --no-cache git ripgrep xz; \\ fi; \\ - curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh \\ - | bash -s -- --skip-setup{hermes_branch_flag}; \\ + curl -fsSL https://raw.githubusercontent.com/NousResearch/hermes-agent/{hermes_version}/scripts/install.sh \\ + | bash -s -- --skip-setup --branch {hermes_version}; \\ hermes version """ diff --git a/tests/test_prepare_harbor_dataset.py b/tests/test_prepare_harbor_dataset.py index 14493fc02..4b1e30e3f 100644 --- a/tests/test_prepare_harbor_dataset.py +++ b/tests/test_prepare_harbor_dataset.py @@ -8,6 +8,7 @@ from pathlib import Path from types import ModuleType +import pytest import yaml REPO = Path(__file__).resolve().parents[1] @@ -284,7 +285,7 @@ def test_generated_dataset_manifest_records_pins_tasks_and_digests(tmp_path: Pat assert manifest["agent_versions"] == { "CLAUDE_CODE_VERSION": "2.1.211", "CODEX_VERSION": "0.144.5", - "HERMES_VERSION": "main", + "HERMES_VERSION": "v2026.8.3", "NODE_VERSION": "20.11.1", "OPENCODE_VERSION": "1.18.3", } @@ -307,3 +308,39 @@ def test_generated_compose_bakes_task_id_into_proxy_env(tmp_path: Path) -> None: proxy_env = "\n".join(compose["services"]["proxy"]["environment"]) assert "SWITCHYARD_TASK_ID=task-id-check" in proxy_env assert "SWITCHYARD_TRIAL_DIR=${HOST_AGENT_LOGS_PATH:-}" in proxy_env + +def test_a_moving_hermes_ref_is_rejected() -> None: + """A moving ref cannot be recorded as a pin. + + The dataset manifest presents HERMES_VERSION as a reproducibility guarantee. Two + builds recording the same string while installing different Hermes code is worse + than recording nothing, so the build fails rather than asserting a pin it does not + have. + """ + base = { + "CLAUDE_CODE_VERSION": "1", + "CODEX_VERSION": "2", + "OPENCODE_VERSION": "3", + "NODE_VERSION": "4", + } + for moving in ("main", "master", "HEAD"): + with pytest.raises(SystemExit, match="moving ref"): + _load_generator_module()._install_layer({**base, "HERMES_VERSION": moving}) + + +def test_the_hermes_installer_is_fetched_at_the_pinned_ref() -> None: + """Pinning the agent but running main's installer reintroduces the same drift.""" + pins = { + "CLAUDE_CODE_VERSION": "1", + "CODEX_VERSION": "2", + "OPENCODE_VERSION": "3", + "NODE_VERSION": "4", + "HERMES_VERSION": "v2026.8.3", + } + + layer = _load_generator_module()._install_layer(pins) + + assert "hermes-agent/v2026.8.3/scripts/install.sh" in layer + assert "hermes-agent/main/" not in layer + assert "--branch v2026.8.3" in layer +