From 2de04cd9d6b98e2f9b9bd209c2803544da507926 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Wed, 14 Oct 2026 09:47:21 +0000 Subject: [PATCH 1/2] Fail Turns when the model host is unreachable --- apps/daemon/internal/agent/codex/options.go | 9 ++- .../internal/agent/codex/provider_config.go | 11 --- .../agent/codex/provider_config_test.go | 16 +++-- .../codex/unreachable_model_native_test.go | 70 +++++++++++++++++++ apps/web/src/i18n/locales/en/diagnostics.ts | 2 +- .../web/src/i18n/locales/zh-CN/diagnostics.ts | 2 +- contracts/agents-api/harness-onboarding.md | 2 + contracts/agents-api/zh/harness-onboarding.md | 4 +- docs/getting-started/nodes.md | 4 +- docs/zh/getting-started/nodes.md | 6 +- .../claude-sdk-adapter/src/native_failure.ts | 6 +- .../tests/native_failure.test.mjs | 8 +++ services/core/deploy/claude/README.md | 5 +- services/core/deploy/codex/README.md | 2 + services/core/deploy/mcode/README.md | 1 + 15 files changed, 122 insertions(+), 26 deletions(-) create mode 100644 apps/daemon/internal/agent/codex/unreachable_model_native_test.go diff --git a/apps/daemon/internal/agent/codex/options.go b/apps/daemon/internal/agent/codex/options.go index 02c1e194e..d3b87d438 100644 --- a/apps/daemon/internal/agent/codex/options.go +++ b/apps/daemon/internal/agent/codex/options.go @@ -72,9 +72,12 @@ type SessionPlan struct { // The codex binary itself is resolved via PATH only. func BuildSessionPlan(stateRoot, agentStateKey string, opts map[string]any, controls *proto.ExecutionControls) (SessionPlan, error) { plan := SessionPlan{ - // Harnesses run unattended: Codex never offers its ask-the-user tool. - ExtraConfig: [][2]string{{"tools.experimental_request_user_input.enabled", "false"}}, - Cleanup: func() {}, + // Harnesses run unattended: Codex never offers its ask-the-user tool, + // and an unreachable model host fails the Turn after the provider's + // bounded retries instead of waiting for the network forever. + ExtraConfig: [][2]string{{"tools.experimental_request_user_input.enabled", "false"}}, + DisableFeatures: []string{"unbounded_connection_retries"}, + Cleanup: func() {}, } nativeConfig, err := harnessconfiguration.Configuration().PrepareHarnessConfig(opts) diff --git a/apps/daemon/internal/agent/codex/provider_config.go b/apps/daemon/internal/agent/codex/provider_config.go index 9eca33486..bfdfe692f 100644 --- a/apps/daemon/internal/agent/codex/provider_config.go +++ b/apps/daemon/internal/agent/codex/provider_config.go @@ -42,10 +42,6 @@ type providerConfig struct { // WireAPI is "responses" (codex Responses API) or "chat" (rejected // upstream since codex-rs 0.140). Empty defaults to "responses". WireAPI string - // RequestMaxRetries / StreamMaxRetries map to codex's - // request_max_retries / stream_max_retries config keys. 0 omits. - RequestMaxRetries int - StreamMaxRetries int } // writeCodexProviderConfig writes (or appends to) /config.toml @@ -105,13 +101,6 @@ func writeCodexProviderConfig(codexHome string, cfg providerConfig) error { b.WriteString(tomlQuoteString(wireAPI)) b.WriteByte('\n') - if cfg.RequestMaxRetries > 0 { - fmt.Fprintf(&b, "request_max_retries = %d\n", cfg.RequestMaxRetries) - } - if cfg.StreamMaxRetries > 0 { - fmt.Fprintf(&b, "stream_max_retries = %d\n", cfg.StreamMaxRetries) - } - if len(cfg.HTTPHeaders) > 0 { keys := make([]string, 0, len(cfg.HTTPHeaders)) for k := range cfg.HTTPHeaders { diff --git a/apps/daemon/internal/agent/codex/provider_config_test.go b/apps/daemon/internal/agent/codex/provider_config_test.go index 9ca848612..7eef92ede 100644 --- a/apps/daemon/internal/agent/codex/provider_config_test.go +++ b/apps/daemon/internal/agent/codex/provider_config_test.go @@ -65,8 +65,6 @@ func TestWriteCodexProviderConfig_FullProvider(t *testing.T) { QueryParams: map[string]string{ "api-version": "2025-04-01-preview", }, - RequestMaxRetries: 4, - StreamMaxRetries: 3, } if err := writeCodexProviderConfig(dir, cfg); err != nil { t.Fatalf("write: %v", err) @@ -76,8 +74,6 @@ func TestWriteCodexProviderConfig_FullProvider(t *testing.T) { `name = "mygw"`, `base_url = "https://platform-api.example.com/v1"`, `env_key = "OAC_CODEX_PROVIDER_API_KEY"`, - `request_max_retries = 4`, - `stream_max_retries = 3`, `[model_providers.oac.http_headers]`, `"X-Sub-Module" = "codex-internal"`, `"X-Request-ID" = "abc"`, @@ -219,6 +215,18 @@ func mustReadFile(t *testing.T, path string) string { return string(b) } +// Codex's default unbounded connection retries keep a Turn whose model host +// is unreachable in progress forever. +func TestModelConnectionRetriesStayBounded(t *testing.T) { + plan, err := BuildSessionPlan(t.TempDir(), "bounded", map[string]any{}, nil) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(plan.DisableFeatures, "unbounded_connection_retries") { + t.Fatal("unbounded connection retries stay enabled") + } +} + func TestProviderCredentialOnlyInNativeLoopEnvironment(t *testing.T) { root := t.TempDir() const token = "synthetic-private-provider-token" diff --git a/apps/daemon/internal/agent/codex/unreachable_model_native_test.go b/apps/daemon/internal/agent/codex/unreachable_model_native_test.go new file mode 100644 index 000000000..c3bbc164f --- /dev/null +++ b/apps/daemon/internal/agent/codex/unreachable_model_native_test.go @@ -0,0 +1,70 @@ +package codex + +import ( + "context" + "log/slog" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" +) + +// A model host that does not resolve must fail the Turn with +// connection_failed instead of retrying forever. Opt-in: it needs the pinned +// native binary but no credentials or network. +func TestExecutorNativeUnreachableModelHost(t *testing.T) { + binary := os.Getenv("OAC_TEST_CODEX_BINARY") + if binary == "" { + t.Skip("requires the pinned native Codex binary") + } + if version, err := exec.Command(binary, "--version").Output(); err != nil || strings.TrimSpace(string(version)) != "codex-cli 0.153.4" { + t.Fatal("native artifact is not Codex 0.153.4") + } + t.Setenv("OAC_RUNTIME_HOME", t.TempDir()) + cfg := defaultSessionConfig(testStateRoot(t)) + cfg.codexBinary = binary + cfg.logger = slog.New(slog.DiscardHandler) + req := proto.PromptRequestPayload{ + AgentKind: "codex", AgentStateKey: "unreachable-model-host", + DisableExecutionEnvironment: true, DisableSubagents: true, + AgentOptions: map[string]any{"model": "gpt-5.5", "model_provider": map[string]any{"base_url": "https://model-host.invalid/v1", "protocol": "responses", "api_key": "unused"}}, + } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute) + defer cancel() + e, err := newExecutor(ctx, req, cfg) + if err != nil { + t.Fatalf("native prepare failed: %v", err) + } + t.Cleanup(func() { + c, stop := context.WithTimeout(context.Background(), 20*time.Second) + defer stop() + _ = e.Close(c) + }) + output := make(chan proto.Envelope, 64) + began := time.Now() + turn, err := e.StartTurn(ctx, "unreachable", proto.TextInput("Reply OK."), output) + if err != nil { + t.Fatalf("start failed: %v", err) + } + var failure proto.ErrorPayload + for done := false; !done; { + select { + case event := <-output: + t.Logf("+%s %s", time.Since(began).Round(time.Millisecond), event.Type) + if event.Type == proto.TypeError { + _ = event.DecodePayload(&failure) + } + done = event.Type == proto.TypeDone + case <-ctx.Done(): + t.Fatalf("Turn did not settle within %s", time.Since(began).Round(time.Second)) + } + } + _, _ = turn.AwaitSettlement(ctx) + t.Logf("failed after %s with %q", time.Since(began).Round(time.Millisecond), failure.Code) + if failure.Code != "connection_failed" { + t.Fatalf("failure code = %q, want connection_failed", failure.Code) + } +} diff --git a/apps/web/src/i18n/locales/en/diagnostics.ts b/apps/web/src/i18n/locales/en/diagnostics.ts index c907cd7c3..0919d55c1 100644 --- a/apps/web/src/i18n/locales/en/diagnostics.ts +++ b/apps/web/src/i18n/locales/en/diagnostics.ts @@ -47,7 +47,7 @@ export const diagnostics = { request_timeout: "Retry later.", context_length_exceeded: "Start a new Session or shorten the input.", cyber_policy: "Change the input before sending it again, or start a new Session.", - connection_failed: "Check the Base URL of the Session's model provider. Without one set on the Session or its saved Agent, a hosted sandbox Session uses the default in Settings → Models and credentials.", + connection_failed: "Check the Base URL of the Session's model provider. Without one set on the Session or its saved Agent, a hosted sandbox Session uses the default in Settings → Models and credentials. Then check that the URL's host resolves and is reachable from the Session's sandbox, or from the host of a self-hosted Session.", harness_error: "Core does not show the Harness's own message. Check the model and Harness on the Session page, then the Session's model provider. Without one set on the Session or its saved Agent, a hosted sandbox Session uses the default in Settings → Models and credentials.", model_provider_required: "Set a model provider on the Session or its saved Agent. A hosted sandbox Session can also use the default in Settings → Models and credentials.", runtime_unavailable: "Retry later. For a self-hosted Session, check Host connection on the Session page; for a hosted sandbox Session, check Settings → Sandbox and scheduling.", diff --git a/apps/web/src/i18n/locales/zh-CN/diagnostics.ts b/apps/web/src/i18n/locales/zh-CN/diagnostics.ts index 2b1112d4f..e7dfe2c14 100644 --- a/apps/web/src/i18n/locales/zh-CN/diagnostics.ts +++ b/apps/web/src/i18n/locales/zh-CN/diagnostics.ts @@ -47,7 +47,7 @@ export const diagnostics = { request_timeout: "请稍后重试。", context_length_exceeded: "请新建 Session 或缩短输入。", cyber_policy: "请修改输入后再发送,或新建 Session。", - connection_failed: "请检查该 Session 所用模型提供商的 Base URL。Session 及其保存的 Agent 都未设置模型提供商时,托管沙箱 Session 使用“设置 → 模型与凭据”中的默认配置。", + connection_failed: "请检查该 Session 所用模型提供商的 Base URL。Session 及其保存的 Agent 都未设置模型提供商时,托管沙箱 Session 使用“设置 → 模型与凭据”中的默认配置。再确认 Session 的沙箱(自托管 Session 则为其主机)能解析并访问该 URL 的主机。", harness_error: "Core 不显示 Harness 的原始消息。请检查 Session 页显示的模型和 Harness,再检查该 Session 所用的模型提供商。Session 及其保存的 Agent 都未设置模型提供商时,托管沙箱 Session 使用“设置 → 模型与凭据”中的默认配置。", model_provider_required: "请在 Session 或其保存的 Agent 上设置模型提供商。托管沙箱 Session 也可以使用“设置 → 模型与凭据”中的默认配置。", runtime_unavailable: "请稍后重试。自托管 Session 请检查 Session 页的“主机连接”;托管沙箱 Session 请检查“设置 → 沙箱与调度”。", diff --git a/contracts/agents-api/harness-onboarding.md b/contracts/agents-api/harness-onboarding.md index 2a16eceab..93acd1c66 100644 --- a/contracts/agents-api/harness-onboarding.md +++ b/contracts/agents-api/harness-onboarding.md @@ -248,6 +248,8 @@ Owned output pipes stay readable after the leader exits. Consumers drain stdout Adapters run native tools unattended with the launching user's permissions, and a Harness never asks a human. Codex runs with approval policy `never`, under which it settles MCP elicitation itself without reaching the client, and with full access; the adapter also disables its blocking `request_user_input` tool. Claude runs through the adapter's tool callback, which allows or denies without asking, in native `default` permission mode with the SDK sandbox disabled. MiniMax runs with bypassed permissions and its sandbox disabled; the adapter disables `askUser`, advertises no elicitation and answers `session/request_permission` with the ACP `cancelled` outcome, which MiniMax treats as a denial. No native permission or question request reaches Core: a human in the loop goes through a function tool, the Session reads [`requires_action`](./sessions-events.md#session-status) and the application submits the function result. Do not add permission profiles, bubblewrap wrappers or native sandbox settings; there is one execution path for every Environment origin. Resource paths are operator configuration, not a permission boundary. +An unreachable model host must not hold a Turn open. The adapter keeps the Harness's own connection retries bounded through native settings, and the Harness's final connection error becomes `connection_failed` when the Harness reports it as structured data ([native failure classification](../../docs/runtime-protocol.md#native-failure-classification)). Core adds no model timeout of its own. Each adapter guide states its Harness's bound. + Network admission follows [Restricted network](./environments.md#restricted-network). ## Native references diff --git a/contracts/agents-api/zh/harness-onboarding.md b/contracts/agents-api/zh/harness-onboarding.md index 804ff05c6..7dd671a16 100644 --- a/contracts/agents-api/zh/harness-onboarding.md +++ b/contracts/agents-api/zh/harness-onboarding.md @@ -1,7 +1,7 @@ --- title: "添加 Harness" source: contracts/agents-api/harness-onboarding.md -source_hash: f943975ae46fd7919700890d4f405c9a9e15385c3ab2b2e742098dcc2a2a8c20 +source_hash: a1db7d464a42cc82bbe285b12a3290d76d0535c5a6a8fb2563586631c6b9136c --- **Harness** 是一种运行模型和工具循环的原生代理引擎(Codex、Claude Code、MiniMax Code)。**Harness 适配器**将 Runtime 的 Executor 和 Turn 契约转换到该引擎的 SDK 或协议。本文档定义 Runtime–Harness 协议:适配器接口及其生命周期义务、注册、Core 资格认定和验收。[Harness capabilities](harness-capabilities.md) 记录了当前每个 Harness 支持的功能。 @@ -250,6 +250,8 @@ daemon 的 `clirunner` 让每个原生子进程在自己的 Unix 进程组中启 适配器以启动用户的权限无人值守运行原生工具,Harness 从不询问人类。Codex 使用批准策略 `never` 和完全访问权限,在该策略下 Codex 自行处理 MCP elicitation,不会发给客户端;适配器还禁用其阻塞式 `request_user_input` 工具。Claude 通过适配器的工具回调运行,回调直接允许或拒绝、不会询问,使用原生 `default` 权限模式并禁用 SDK sandbox。MiniMax 绕过权限并禁用 sandbox;适配器禁用 `askUser`、不声明 elicitation,并以 ACP `cancelled` 结果答复 `session/request_permission`,MiniMax 将其视为拒绝。原生权限或问题请求都不会到达 Core:需要人工介入时通过 function 工具完成,Session 读取 [`requires_action`](./sessions-events.md#session-status),由应用提交 function 结果。不要添加权限 profile、bubblewrap 包装器或原生 sandbox 设置;每种 Environment 来源都只有一条执行路径。资源路径属于操作员配置,而不是权限边界。 +模型主机不可达时不能让 Turn 一直挂起。适配器通过原生设置让 Harness 自身的连接重试保持有界;Harness 以结构化数据报告最终连接错误时,该错误归为 `connection_failed`([原生失败分类](../../../docs/zh/runtime-protocol.md#native-failure-classification))。Core 不另设模型超时。各适配器指南写明其 Harness 的上限。 + 网络准入遵循 [Restricted network](environments.md#restricted-network)。 ## 原生参考 {#native-references} diff --git a/docs/getting-started/nodes.md b/docs/getting-started/nodes.md index cd2154da9..a3f396779 100644 --- a/docs/getting-started/nodes.md +++ b/docs/getting-started/nodes.md @@ -68,12 +68,14 @@ The proxy applies only to installation downloads. It is not saved in the node's | Node state | `/var/lib/oac-node/.oac/nodes//`: identity, configuration, node files. microsandbox keeps its images and sandboxes under `/var/lib/oac-node/.oac/m/` | | Records | `/etc/oac-node/`: what the installer created or changed, used by reruns and uninstall | | Docker | The Runtime image, imported once, and a network `oac-node-` | -| microsandbox network policy | Sandboxes may reach Core, DNS on the host and public addresses, and nothing else: no inbound connections and no private networks. A private model or MCP endpoint needs a policy change on the node | +| microsandbox network policy | Sandboxes may reach Core, DNS on the host and public addresses, and nothing else: no inbound connections and no private networks | Root only prepares the account, the group and the unit; everything else, the Docker network included, runs as `oac-node`, in its own session without a terminal. The installer never installs Docker, KVM or packages, never starts Docker, never changes device permissions, sudoers, firewall or SELinux settings, and never touches other accounts. **Docker mode is root-equivalent.** Membership in the `docker` group lets `oac-node`, and so anything that controls the node, act as root on the host. Containers also share the host kernel, so a container escape reaches the host. Add Docker nodes only on hosts dedicated to sandboxes. microsandbox nodes need only the `kvm` group and run each sandbox in its own microVM. +**microsandbox blocks private endpoints.** A sandbox can't reach a model provider or MCP server at a private address. Sandboxes resolve names through the node host's resolver, and DNS rebind protection discards any answer that contains a private address, so a name that the host resolves to a private address doesn't resolve in the sandbox at all, even for a public service such as a model API behind a cloud VPC resolver. The Turn then fails once the Harness stops retrying. Installer-added nodes have no setting to change this. Make the host's resolver return the service's public addresses, for example with entries in the host's `/etc/hosts`; use a Docker node, which doesn't filter sandbox traffic; or [register the node manually](#register-a-node-manually) with a `network` policy that also allows the private range in a rule without a port, because rebind protection ignores port-scoped rules. + **One Core per host.** Nodes share the `oac-node` account, so a host serves one Core; a command from a second Core is refused. **One release.** A node runs the program of the console that added it and is never upgraded in place; see the [installation version policy](./operations.md#installation-version-policy). diff --git a/docs/zh/getting-started/nodes.md b/docs/zh/getting-started/nodes.md index fc24c564b..aa380ec94 100644 --- a/docs/zh/getting-started/nodes.md +++ b/docs/zh/getting-started/nodes.md @@ -1,7 +1,7 @@ --- title: "添加和管理节点" source: docs/getting-started/nodes.md -source_hash: 56719d358e65d838d05bfb60b902a8d8f0803f833048be4a7c56c4731776ed0a +source_hash: 00b2655a0a6af8e210696b894a9dbce17590a91ff964665be0ecc4787c0accf6 --- 节点是一台 Linux 主机,在沙箱提供商为 microsandbox(推荐)或 Docker 时,为 Core 托管 Session 运行沙箱。Core 将新 Session 分配给有空余容量的节点;节点创建沙箱,沙箱回连 Core。E2B 不需要节点。应用为自己的 Session 连接的机器是[自托管执行器](self-hosted.md),而不是节点。 @@ -70,12 +70,14 @@ Microsandbox 的检查点执行兼容性包含主机内核和 CPU profile。因 | 节点状态 | `/var/lib/oac-node/.oac/nodes//`:身份、配置和节点文件。microsandbox 在 `/var/lib/oac-node/.oac/m/` 保存镜像和沙箱 | | 记录 | `/etc/oac-node/`:安装程序创建或修改的内容,用于重新运行和卸载 | | Docker | 一次性导入 Runtime 镜像,以及网络 `oac-node-` | -| microsandbox 网络策略 | 沙箱仅能访问 Core、主机上的 DNS 和公开地址:无入站连接、不可访问私有网络。私有模型或 MCP 端点需要在节点上修改策略 | +| microsandbox 网络策略 | 沙箱仅能访问 Core、主机上的 DNS 和公开地址:无入站连接、不可访问私有网络 | root 只准备账号、组和服务单元;其他操作(包括 Docker 网络)都以 `oac-node` 在没有终端的独立会话中执行。安装程序不安装 Docker、KVM 或软件包,不启动 Docker,不修改设备权限、sudoers、防火墙或 SELinux 设置,也不修改其他账号。 **Docker 模式等同于 root 权限。** 加入 `docker` 组后,`oac-node` 以及任何能控制节点的主体,都可以在主机上执行 root 操作。容器共享主机内核,因此容器逃逸能访问主机。只在沙箱专用主机上添加 Docker 节点。microsandbox 节点只需要 `kvm` 组,每个沙箱运行在独立 microVM 中。 +**microsandbox 拦截私有端点。** 沙箱无法访问位于私有地址的模型服务或 MCP 服务器。沙箱通过节点主机的解析器解析域名,DNS 重绑定保护会丢弃任何包含私有地址的应答;因此,主机解析为私有地址的域名在沙箱中完全无法解析,即使它是公开服务,例如云 VPC 解析器后的模型 API。Harness 停止重试后,Turn 随即失败。通过安装程序添加的节点没有更改此行为的设置。可以让主机解析器返回该服务的公开地址,例如在主机的 `/etc/hosts` 中添加条目;或使用不过滤沙箱流量的 Docker 节点;或[手动注册节点](#register-a-node-manually),在 `network` 策略中为该私有网段添加不带端口的允许规则,因为重绑定保护忽略带端口的规则。 + **每台主机只服务一个 Core。** 节点共享 `oac-node` 账号,因此主机只服务一个 Core;第二个 Core 的命令会被拒绝。 **一个发行版本。** 节点运行添加它的控制台所提供的程序,不进行原地升级;参阅[安装版本策略](operations.md#installation-version-policy)。 diff --git a/packages/claude-sdk-adapter/src/native_failure.ts b/packages/claude-sdk-adapter/src/native_failure.ts index cd7631f77..273aeb121 100644 --- a/packages/claude-sdk-adapter/src/native_failure.ts +++ b/packages/claude-sdk-adapter/src/native_failure.ts @@ -27,7 +27,11 @@ export class NativeFailure { const candidate = this.candidates.get(pending[0]!); const same = pending.every(id => this.candidates.get(id) === candidate); for (const id of pending) this.candidates.delete(id); - if (same && (message.subtype === "error_during_execution" || message.subtype === "success" && message.is_error)) return candidate; + if (same && message.subtype === "error_during_execution") return candidate; + // Claude Code also reports a request that got no HTTP response (DNS, refused or + // reset connection, timeout) as server_error; only a provider 5xx has a status. + if (same && message.subtype === "success" && message.is_error) + return candidate === "server_error" && typeof message.api_error_status !== "number" ? "connection_failed" : candidate; } } } diff --git a/packages/claude-sdk-adapter/tests/native_failure.test.mjs b/packages/claude-sdk-adapter/tests/native_failure.test.mjs index cf87b7d99..2f663090e 100644 --- a/packages/claude-sdk-adapter/tests/native_failure.test.mjs +++ b/packages/claude-sdk-adapter/tests/native_failure.test.mjs @@ -18,6 +18,14 @@ test("pinned SDK finite native mapping", () => { } }); +test("a server error without an HTTP status is a connection failure", () => { + for (const [status, code] of [[null, "connection_failed"], [502, "server_error"]]) { + const f = new NativeFailure(); + f.observe(assistant("server_error"), "native", ["input"]); + assert.equal(f.observe(result({ subtype: "success", api_error_status: status }), "native", ["input"]), code); + } +}); + test("classification requires same root, input and unsuccessful terminal result", () => { for (const transform of [m=>({...m,parent_tool_use_id:"child"}),m=>({...m,session_id:"other"}),m=>({...m,isReplay:true}),m=>({...m,isSynthetic:true})]) { const f = new NativeFailure(); f.observe(transform(assistant("rate_limit")),"native",["input"]); diff --git a/services/core/deploy/claude/README.md b/services/core/deploy/claude/README.md index f790d50b2..f5fcd19ea 100644 --- a/services/core/deploy/claude/README.md +++ b/services/core/deploy/claude/README.md @@ -30,7 +30,10 @@ The bridge ([`native_failure.ts`](../../../../packages/claude-sdk-adapter/src/na | `overloaded` | `server_overloaded` | | `invalid_request` | `invalid_request` | | `model_not_found` | `resource_not_found` | -| `server_error` | `server_error` | +| `server_error` with an HTTP status in the result's `api_error_status` | `server_error` | +| `server_error` whose result has no `api_error_status`: no HTTP response, such as a failed DNS lookup, a refused or reset connection or a timeout | `connection_failed` | + +Claude Code `2.1.269` retries a failed model request up to 10 times, with backoff that grows to about 35 seconds, so a model host that does not resolve or refuses connections fails the Turn after about three minutes. The adapter leaves that bound as it is. `unknown`, `max_output_tokens` and other results stay unclassified. The bridge drops the classification when any other error ends the Turn, when cleanup fails or when no native process ran. The Go adapter keeps it only for the reported failed result of the same native Session, and only after it received the Turn settlement ([`executor_turn.go`](../../../../apps/daemon/internal/agent/claudesdk/executor_turn.go)). diff --git a/services/core/deploy/codex/README.md b/services/core/deploy/codex/README.md index aeaddef93..d48b0a082 100644 --- a/services/core/deploy/codex/README.md +++ b/services/core/deploy/codex/README.md @@ -14,6 +14,8 @@ Codex accepts only the `responses` model protocol ([`harnessconfig/codex`](../.. Each Session has its own `CODEX_HOME` at `/daemon/agent-sessions//`, where the Runtime supplies `` from the [state directory setting](../../../../docs/configuration.md#runtime-resource-directories). Native history stays there; Runtime device credentials stay in the Runtime home. The state layout alone does not qualify cross-node native continuation. The adapter regenerates that directory's `config.toml` on every prompt: the Session's frozen provider bundle becomes the `[model_providers.oac]` block, with `wire_api` `responses`, and the thread is pinned to that provider, so Codex never falls back to its built-in `openai` provider. Provider bearer credentials use the native `env_key` reference and enter the model-loop child process environment; they are not written into retained `config.toml`. The adapter excludes that variable from native shell environments with `shell_environment_policy.exclude` and disables `shell_snapshot`, whose capture in the pinned native version can otherwise persist the model-loop environment. This retains native thread history without retaining native shell snapshots; it introduces no user setting. +By default Codex retries a failed model connection forever, waiting up to 60 seconds between attempts, so a Turn whose model host is unreachable stays in progress with no events. The adapter disables that `unbounded_connection_retries` feature, so Codex falls back to its bounded defaults: up to 4 retries per request and 5 per response stream. A model host that does not resolve or refuses connections then fails the Turn with `connection_failed` in about 25 seconds. Retries start only after a request fails, so slow responses are unaffected, and a stream still ends after Codex's 300 seconds without data. Codex sets no connect timeout: against a host that silently drops connections, each of the 30 attempts waits for the operating system's connect timeout. + ## Execution controls The adapter applies Core's typed controls and opt-ins, described in the [Runtime protocol](../../../../docs/runtime-protocol.md#capability-declarations), through native Codex settings on both new and resumed Turns. diff --git a/services/core/deploy/mcode/README.md b/services/core/deploy/mcode/README.md index f993eb85d..499a6cc18 100644 --- a/services/core/deploy/mcode/README.md +++ b/services/core/deploy/mcode/README.md @@ -37,6 +37,7 @@ In the workspace profile, the frozen installation's Skills are linked into the S - MiniMax Code reports no usage: ACP context occupancy and cumulative cost are not per-Turn usage. - Workspace `workspace_bash` calls become command observations with the command text, output text and status; exit code and duration stay unknown. Native task and Skill utilities produce no public Items. - The adapter attaches no [native failure classification](../../../../docs/runtime-protocol.md#native-failure-classification) to its errors: MiniMax Code `0.4.12` reports a model provider rejection over ACP as JSON-RPC internal error `-32603` whose only detail is prose, so these failures stay unclassified. +- MiniMax Code `0.4.12` retries a failed model request up to 5 times and stops retrying after 120 seconds, so a model host that does not resolve fails the Turn, unclassified, in about two minutes. Its retries stay inside MiniMax Code and never reach ACP. ## Runtime image From 0902c941e2cf16f1c4fe5f9f9745fee07880fc5d Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Wed, 14 Oct 2026 10:26:28 +0000 Subject: [PATCH 2/2] Classify connection failures from retries without a response --- .../sessions/session-diagnostics.test.tsx | 3 +++ apps/web/src/i18n/locales/en/diagnostics.ts | 3 ++- .../web/src/i18n/locales/zh-CN/diagnostics.ts | 3 ++- apps/web/src/lib/diagnostic-failure.ts | 14 ++++++++++---- docs/getting-started/nodes.md | 2 +- docs/zh/getting-started/nodes.md | 4 ++-- .../claude-sdk-adapter/src/native_failure.ts | 19 ++++++++++++------- .../tests/native_failure.test.mjs | 10 +++++++--- services/core/deploy/claude/README.md | 8 ++++---- services/core/deploy/codex/README.md | 2 +- 10 files changed, 44 insertions(+), 24 deletions(-) diff --git a/apps/web/src/features/sessions/session-diagnostics.test.tsx b/apps/web/src/features/sessions/session-diagnostics.test.tsx index f30a93f41..1f179df34 100644 --- a/apps/web/src/features/sessions/session-diagnostics.test.tsx +++ b/apps/web/src/features/sessions/session-diagnostics.test.tsx @@ -70,6 +70,9 @@ describe("Core diagnostics in the console", () => { const providerDefault = "Without a model provider set on the Session or its saved Agent, a hosted sandbox Session uses the default in Settings → Models and credentials."; expect(diagnosticFailure({ code: "invalid_request", params: {}, failed_at: null }, t)).toBe(`The model provider rejected the request. Check the model and parameters, starting with the model on the Session page. ${providerDefault}`); expect(diagnosticFailure({ code: "connection_failed", params: { http_status: 502 }, failed_at: null }, t)).toContain("HTTP 502. Check the Base URL of the Session's model provider."); + // Only a failure without any HTTP response points at name resolution and reachability. + expect(diagnosticFailure({ code: "connection_failed", params: { http_status: 502 }, failed_at: null }, t)).not.toContain("resolves"); + expect(diagnosticFailure({ code: "connection_failed", params: { http_status: null }, failed_at: null }, t)).toContain("Then check that the URL's host resolves and is reachable"); expect(diagnosticFailure({ code: "runtime_unavailable", params: {}, failed_at: null }, t)).toBe("The Runtime is unavailable. Retry later. For a self-hosted Session, check Host connection on the Session page; for a hosted sandbox Session, check Settings → Sandbox and scheduling."); expect(diagnosticFailure({ code: "harness_error", params: {}, failed_at: null }, t)).toContain("Core does not show the Harness's own message"); expect(diagnosticFailure({ code: "invalid_request", params: {}, failed_at: null }, i18n.getFixedT("zh-CN", "diagnostics"))).toBe("模型提供商拒绝了请求。请检查模型和参数,先看 Session 页显示的模型。Session 及其保存的 Agent 都未设置模型提供商时,托管沙箱 Session 使用“设置 → 模型与凭据”中的默认配置。"); diff --git a/apps/web/src/i18n/locales/en/diagnostics.ts b/apps/web/src/i18n/locales/en/diagnostics.ts index 0919d55c1..6ce80d5a7 100644 --- a/apps/web/src/i18n/locales/en/diagnostics.ts +++ b/apps/web/src/i18n/locales/en/diagnostics.ts @@ -47,7 +47,7 @@ export const diagnostics = { request_timeout: "Retry later.", context_length_exceeded: "Start a new Session or shorten the input.", cyber_policy: "Change the input before sending it again, or start a new Session.", - connection_failed: "Check the Base URL of the Session's model provider. Without one set on the Session or its saved Agent, a hosted sandbox Session uses the default in Settings → Models and credentials. Then check that the URL's host resolves and is reachable from the Session's sandbox, or from the host of a self-hosted Session.", + connection_failed: "Check the Base URL of the Session's model provider. Without one set on the Session or its saved Agent, a hosted sandbox Session uses the default in Settings → Models and credentials.", harness_error: "Core does not show the Harness's own message. Check the model and Harness on the Session page, then the Session's model provider. Without one set on the Session or its saved Agent, a hosted sandbox Session uses the default in Settings → Models and credentials.", model_provider_required: "Set a model provider on the Session or its saved Agent. A hosted sandbox Session can also use the default in Settings → Models and credentials.", runtime_unavailable: "Retry later. For a self-hosted Session, check Host connection on the Session page; for a hosted sandbox Session, check Settings → Sandbox and scheduling.", @@ -60,6 +60,7 @@ export const diagnostics = { setupIndex: "Setup command {{number}}.", exitCode: "Exit code {{code}}.", httpStatus: "HTTP {{status}}.", + unreachableHost: "Then check that the URL's host resolves and is reachable from the Session's sandbox, or from the host of a self-hosted Session.", step: { setup: "setup", python: "Python dependencies", npm: "npm dependencies", system: "system packages", file: "file preparation", skill: "Skill preparation" }, unavailable: "Failure details unavailable", unavailableHelp: "The diagnostics read failed or no longer matches this status. Refresh to check again.", diff --git a/apps/web/src/i18n/locales/zh-CN/diagnostics.ts b/apps/web/src/i18n/locales/zh-CN/diagnostics.ts index e7dfe2c14..130ac4caf 100644 --- a/apps/web/src/i18n/locales/zh-CN/diagnostics.ts +++ b/apps/web/src/i18n/locales/zh-CN/diagnostics.ts @@ -47,7 +47,7 @@ export const diagnostics = { request_timeout: "请稍后重试。", context_length_exceeded: "请新建 Session 或缩短输入。", cyber_policy: "请修改输入后再发送,或新建 Session。", - connection_failed: "请检查该 Session 所用模型提供商的 Base URL。Session 及其保存的 Agent 都未设置模型提供商时,托管沙箱 Session 使用“设置 → 模型与凭据”中的默认配置。再确认 Session 的沙箱(自托管 Session 则为其主机)能解析并访问该 URL 的主机。", + connection_failed: "请检查该 Session 所用模型提供商的 Base URL。Session 及其保存的 Agent 都未设置模型提供商时,托管沙箱 Session 使用“设置 → 模型与凭据”中的默认配置。", harness_error: "Core 不显示 Harness 的原始消息。请检查 Session 页显示的模型和 Harness,再检查该 Session 所用的模型提供商。Session 及其保存的 Agent 都未设置模型提供商时,托管沙箱 Session 使用“设置 → 模型与凭据”中的默认配置。", model_provider_required: "请在 Session 或其保存的 Agent 上设置模型提供商。托管沙箱 Session 也可以使用“设置 → 模型与凭据”中的默认配置。", runtime_unavailable: "请稍后重试。自托管 Session 请检查 Session 页的“主机连接”;托管沙箱 Session 请检查“设置 → 沙箱与调度”。", @@ -60,6 +60,7 @@ export const diagnostics = { setupIndex: "第 {{number}} 条设置命令。", exitCode: "退出码 {{code}}。", httpStatus: "HTTP {{status}}。", + unreachableHost: "再确认 Session 的沙箱(自托管 Session 则为其主机)能解析并访问该 URL 的主机。", step: { setup: "设置", python: "Python 依赖", npm: "npm 依赖", system: "系统软件包", file: "文件准备", skill: "Skill 准备" }, unavailable: "失败详情不可用", unavailableHelp: "诊断读取失败或与当前状态不一致,请刷新后重试。", diff --git a/apps/web/src/lib/diagnostic-failure.ts b/apps/web/src/lib/diagnostic-failure.ts index 14336b107..3df058ab7 100644 --- a/apps/web/src/lib/diagnostic-failure.ts +++ b/apps/web/src/lib/diagnostic-failure.ts @@ -1,6 +1,11 @@ import type { DiagnosticFailure } from "@oac/agents-client"; import type { TFunction } from "i18next"; +/** The provider's HTTP status on a connection failure; null when no response arrived. */ +function connectionStatus(failure: DiagnosticFailure): number | null { + return failure.code === "connection_failed" && "http_status" in failure.params ? failure.params.http_status : null; +} + /** Translate the catalog, never native error text or guessed message categories. */ export function diagnosticReason(failure: DiagnosticFailure, t: TFunction<"diagnostics">): string { const gap = t("sentenceGap"); @@ -10,15 +15,16 @@ export function diagnosticReason(failure: DiagnosticFailure, t: TFunction<"diagn if (failure.params.index !== null) message += gap + t("setupIndex", { number: failure.params.index + 1 }); if (failure.params.exit_code !== null) message += gap + t("exitCode", { code: failure.params.exit_code }); } - if (failure.code === "connection_failed" && "http_status" in failure.params && failure.params.http_status !== null) { - message += gap + t("httpStatus", { status: failure.params.http_status }); - } + const status = connectionStatus(failure); + if (status !== null) message += gap + t("httpStatus", { status }); return message; } /** The catalog's next step for the reason; empty when it has none. */ export function diagnosticNext(failure: DiagnosticFailure, t: TFunction<"diagnostics">): string { - return t(`next.${failure.code}`, { defaultValue: "" }); + const next: string = t(`next.${failure.code}`, { defaultValue: "" }); + // Without an HTTP status no provider answered, so its host may not resolve or be reachable. + return failure.code === "connection_failed" && connectionStatus(failure) === null ? next + t("sentenceGap") + t("unreachableHost") : next; } /** The classified reason, then its next step when the catalog has one. */ diff --git a/docs/getting-started/nodes.md b/docs/getting-started/nodes.md index a3f396779..daa6510f2 100644 --- a/docs/getting-started/nodes.md +++ b/docs/getting-started/nodes.md @@ -74,7 +74,7 @@ Root only prepares the account, the group and the unit; everything else, the Doc **Docker mode is root-equivalent.** Membership in the `docker` group lets `oac-node`, and so anything that controls the node, act as root on the host. Containers also share the host kernel, so a container escape reaches the host. Add Docker nodes only on hosts dedicated to sandboxes. microsandbox nodes need only the `kvm` group and run each sandbox in its own microVM. -**microsandbox blocks private endpoints.** A sandbox can't reach a model provider or MCP server at a private address. Sandboxes resolve names through the node host's resolver, and DNS rebind protection discards any answer that contains a private address, so a name that the host resolves to a private address doesn't resolve in the sandbox at all, even for a public service such as a model API behind a cloud VPC resolver. The Turn then fails once the Harness stops retrying. Installer-added nodes have no setting to change this. Make the host's resolver return the service's public addresses, for example with entries in the host's `/etc/hosts`; use a Docker node, which doesn't filter sandbox traffic; or [register the node manually](#register-a-node-manually) with a `network` policy that also allows the private range in a rule without a port, because rebind protection ignores port-scoped rules. +**microsandbox blocks private endpoints.** A sandbox can't reach a model provider or MCP server at a private address. Sandbox DNS queries go to the nameservers listed in the host's `/etc/resolv.conf`, and DNS rebind protection discards any answer that contains a private address, so a name those nameservers resolve to a private address doesn't resolve in the sandbox at all, even for a public service such as a model API behind a cloud VPC resolver. The Turn then fails once the Harness stops retrying. Installer-added nodes have no setting to change this. Make those nameservers return the service's public addresses; microsandbox doesn't read the host's `/etc/hosts`, so an entry there helps only when `/etc/resolv.conf` points at a local stub resolver that serves it, such as systemd-resolved at `127.0.0.53`. Otherwise use a Docker node, which doesn't filter sandbox traffic, or [register the node manually](#register-a-node-manually) with a `network` policy that also allows the private range in a rule without a port, because rebind protection ignores port-scoped rules. **One Core per host.** Nodes share the `oac-node` account, so a host serves one Core; a command from a second Core is refused. diff --git a/docs/zh/getting-started/nodes.md b/docs/zh/getting-started/nodes.md index aa380ec94..988eca509 100644 --- a/docs/zh/getting-started/nodes.md +++ b/docs/zh/getting-started/nodes.md @@ -1,7 +1,7 @@ --- title: "添加和管理节点" source: docs/getting-started/nodes.md -source_hash: 00b2655a0a6af8e210696b894a9dbce17590a91ff964665be0ecc4787c0accf6 +source_hash: 197acaddceb84ff46a833b883a8751c24d3f4a2ae49e077280e0c677a083424e --- 节点是一台 Linux 主机,在沙箱提供商为 microsandbox(推荐)或 Docker 时,为 Core 托管 Session 运行沙箱。Core 将新 Session 分配给有空余容量的节点;节点创建沙箱,沙箱回连 Core。E2B 不需要节点。应用为自己的 Session 连接的机器是[自托管执行器](self-hosted.md),而不是节点。 @@ -76,7 +76,7 @@ root 只准备账号、组和服务单元;其他操作(包括 Docker 网络 **Docker 模式等同于 root 权限。** 加入 `docker` 组后,`oac-node` 以及任何能控制节点的主体,都可以在主机上执行 root 操作。容器共享主机内核,因此容器逃逸能访问主机。只在沙箱专用主机上添加 Docker 节点。microsandbox 节点只需要 `kvm` 组,每个沙箱运行在独立 microVM 中。 -**microsandbox 拦截私有端点。** 沙箱无法访问位于私有地址的模型服务或 MCP 服务器。沙箱通过节点主机的解析器解析域名,DNS 重绑定保护会丢弃任何包含私有地址的应答;因此,主机解析为私有地址的域名在沙箱中完全无法解析,即使它是公开服务,例如云 VPC 解析器后的模型 API。Harness 停止重试后,Turn 随即失败。通过安装程序添加的节点没有更改此行为的设置。可以让主机解析器返回该服务的公开地址,例如在主机的 `/etc/hosts` 中添加条目;或使用不过滤沙箱流量的 Docker 节点;或[手动注册节点](#register-a-node-manually),在 `network` 策略中为该私有网段添加不带端口的允许规则,因为重绑定保护忽略带端口的规则。 +**microsandbox 拦截私有端点。** 沙箱无法访问位于私有地址的模型提供商或 MCP 服务器。沙箱的 DNS 查询发往主机 `/etc/resolv.conf` 中列出的域名服务器,DNS 重绑定保护会丢弃任何包含私有地址的应答;因此,这些域名服务器解析为私有地址的域名在沙箱中完全无法解析,即使它是公开服务,例如云 VPC 解析器后的模型 API。Harness 停止重试后,Turn 随即失败。通过安装程序添加的节点没有更改此行为的设置。请让这些域名服务器返回该服务的公开地址;microsandbox 不读取主机的 `/etc/hosts`,因此只有当 `/etc/resolv.conf` 指向会读取该文件的本地 stub 解析器(例如位于 `127.0.0.53` 的 systemd-resolved)时,在其中添加条目才有效。否则,可以使用不过滤沙箱流量的 Docker 节点,或[手动注册节点](#register-a-node-manually),在 `network` 策略中为该私有网段添加不带端口的允许规则,因为重绑定保护忽略带端口的规则。 **每台主机只服务一个 Core。** 节点共享 `oac-node` 账号,因此主机只服务一个 Core;第二个 Core 的命令会被拒绝。 diff --git a/packages/claude-sdk-adapter/src/native_failure.ts b/packages/claude-sdk-adapter/src/native_failure.ts index 273aeb121..3a7f54697 100644 --- a/packages/claude-sdk-adapter/src/native_failure.ts +++ b/packages/claude-sdk-adapter/src/native_failure.ts @@ -13,12 +13,21 @@ const codes: Record = { // last error text. A matching native error result alone commits a classification. export class NativeFailure { private readonly candidates = new Map(); + // Claude Code reports both retries that got no HTTP response and a server error in + // an already streaming response as server_error without a status. Only the first + // ends with a retry whose error_status is null and no model output after it. + private connectionRetry = false; observe(message: SDKMessage, sessionID: string, pending: readonly string[]): string | undefined { - if (!sessionID || message.session_id !== sessionID || !pending.length || + if (!sessionID || message.session_id !== sessionID || ("isReplay" in message && message.isReplay) || ("isSynthetic" in message && message.isSynthetic)) return; + if (message.type === "system" && message.subtype === "api_retry") this.connectionRetry = message.error_status === null; + const connectionRetry = this.connectionRetry; + if (message.type === "stream_event" || message.type === "assistant" || message.type === "result") this.connectionRetry = false; + if (!pending.length) return; if (message.type === "assistant" && message.parent_tool_use_id === null) { - const code = typeof message.error !== "string" || !Object.hasOwn(codes, message.error) ? undefined : codes[message.error]; + const native = typeof message.error !== "string" || !Object.hasOwn(codes, message.error) ? undefined : codes[message.error]; + const code = native === "server_error" && connectionRetry ? "connection_failed" : native; for (const id of pending) { this.candidates.delete(id); if (code) this.candidates.set(id, code); @@ -27,11 +36,7 @@ export class NativeFailure { const candidate = this.candidates.get(pending[0]!); const same = pending.every(id => this.candidates.get(id) === candidate); for (const id of pending) this.candidates.delete(id); - if (same && message.subtype === "error_during_execution") return candidate; - // Claude Code also reports a request that got no HTTP response (DNS, refused or - // reset connection, timeout) as server_error; only a provider 5xx has a status. - if (same && message.subtype === "success" && message.is_error) - return candidate === "server_error" && typeof message.api_error_status !== "number" ? "connection_failed" : candidate; + if (same && (message.subtype === "error_during_execution" || message.subtype === "success" && message.is_error)) return candidate; } } } diff --git a/packages/claude-sdk-adapter/tests/native_failure.test.mjs b/packages/claude-sdk-adapter/tests/native_failure.test.mjs index 2f663090e..f7246d739 100644 --- a/packages/claude-sdk-adapter/tests/native_failure.test.mjs +++ b/packages/claude-sdk-adapter/tests/native_failure.test.mjs @@ -18,11 +18,15 @@ test("pinned SDK finite native mapping", () => { } }); -test("a server error without an HTTP status is a connection failure", () => { - for (const [status, code] of [[null, "connection_failed"], [502, "server_error"]]) { +test("a server error is a connection failure only after retries without an HTTP response", () => { + const retry = error_status => ({ type: "system", subtype: "api_retry", session_id: "native", error_status, error: "unknown" }); + const text = { type: "stream_event", session_id: "native", parent_tool_use_id: null, event: { type: "content_block_start" } }; + for (const [before, code] of [[[retry(null)], "connection_failed"], [[retry(503)], "server_error"], + [[retry(null), text, assistant(undefined)], "server_error"], [[], "server_error"]]) { const f = new NativeFailure(); + for (const message of before) f.observe(message, "native", message.type === "assistant" ? ["input"] : []); f.observe(assistant("server_error"), "native", ["input"]); - assert.equal(f.observe(result({ subtype: "success", api_error_status: status }), "native", ["input"]), code); + assert.equal(f.observe(result({ subtype: "success", api_error_status: null }), "native", ["input"]), code); } }); diff --git a/services/core/deploy/claude/README.md b/services/core/deploy/claude/README.md index f5fcd19ea..227af93f8 100644 --- a/services/core/deploy/claude/README.md +++ b/services/core/deploy/claude/README.md @@ -20,7 +20,7 @@ Recovery uses the SDK's history APIs ([`recovery.ts`](../../../../packages/claud ## Native failure classification -The bridge ([`native_failure.ts`](../../../../packages/claude-sdk-adapter/src/native_failure.ts)) classifies a failure only from root assistant messages (no parent tool use) of the same native Session that answer Core-submitted inputs not yet completed; replayed and synthetic messages are ignored. A classification is committed only when the matching native result is an error; a successful result clears it. The [Runtime protocol](../../../../docs/runtime-protocol.md#native-failure-classification) defines the codes. +The bridge ([`native_failure.ts`](../../../../packages/claude-sdk-adapter/src/native_failure.ts)) classifies a failure only from root assistant messages (no parent tool use) of the same native Session that answer Core-submitted inputs not yet completed, and from the `api_retry` message right before one; replayed and synthetic messages are ignored. A classification is committed only when the matching native result is an error; a successful result clears it. The [Runtime protocol](../../../../docs/runtime-protocol.md#native-failure-classification) defines the codes. | SDK assistant error | Code | | --- | --- | @@ -30,10 +30,10 @@ The bridge ([`native_failure.ts`](../../../../packages/claude-sdk-adapter/src/na | `overloaded` | `server_overloaded` | | `invalid_request` | `invalid_request` | | `model_not_found` | `resource_not_found` | -| `server_error` with an HTTP status in the result's `api_error_status` | `server_error` | -| `server_error` whose result has no `api_error_status`: no HTTP response, such as a failed DNS lookup, a refused or reset connection or a timeout | `connection_failed` | +| `server_error` right after an `api_retry` whose `error_status` is null, with no model output between them | `connection_failed` | +| any other `server_error` | `server_error` | -Claude Code `2.1.269` retries a failed model request up to 10 times, with backoff that grows to about 35 seconds, so a model host that does not resolve or refuses connections fails the Turn after about three minutes. The adapter leaves that bound as it is. +Claude Code `2.1.269` reports both a request that got no HTTP response, such as a failed DNS lookup, a refused connection or a timeout, and a server error in a response that already started streaming as `server_error` without a status. Only the first comes right after an `api_retry` whose `error_status` is null, with no model output in between. Claude Code retries such a request up to 10 times, with backoff that grows to about 35 seconds, so a model host that does not resolve or refuses connections fails the Turn after about three minutes. The adapter leaves that bound as it is. `unknown`, `max_output_tokens` and other results stay unclassified. The bridge drops the classification when any other error ends the Turn, when cleanup fails or when no native process ran. The Go adapter keeps it only for the reported failed result of the same native Session, and only after it received the Turn settlement ([`executor_turn.go`](../../../../apps/daemon/internal/agent/claudesdk/executor_turn.go)). diff --git a/services/core/deploy/codex/README.md b/services/core/deploy/codex/README.md index d48b0a082..f5ecb72ab 100644 --- a/services/core/deploy/codex/README.md +++ b/services/core/deploy/codex/README.md @@ -14,7 +14,7 @@ Codex accepts only the `responses` model protocol ([`harnessconfig/codex`](../.. Each Session has its own `CODEX_HOME` at `/daemon/agent-sessions//`, where the Runtime supplies `` from the [state directory setting](../../../../docs/configuration.md#runtime-resource-directories). Native history stays there; Runtime device credentials stay in the Runtime home. The state layout alone does not qualify cross-node native continuation. The adapter regenerates that directory's `config.toml` on every prompt: the Session's frozen provider bundle becomes the `[model_providers.oac]` block, with `wire_api` `responses`, and the thread is pinned to that provider, so Codex never falls back to its built-in `openai` provider. Provider bearer credentials use the native `env_key` reference and enter the model-loop child process environment; they are not written into retained `config.toml`. The adapter excludes that variable from native shell environments with `shell_environment_policy.exclude` and disables `shell_snapshot`, whose capture in the pinned native version can otherwise persist the model-loop environment. This retains native thread history without retaining native shell snapshots; it introduces no user setting. -By default Codex retries a failed model connection forever, waiting up to 60 seconds between attempts, so a Turn whose model host is unreachable stays in progress with no events. The adapter disables that `unbounded_connection_retries` feature, so Codex falls back to its bounded defaults: up to 4 retries per request and 5 per response stream. A model host that does not resolve or refuses connections then fails the Turn with `connection_failed` in about 25 seconds. Retries start only after a request fails, so slow responses are unaffected, and a stream still ends after Codex's 300 seconds without data. Codex sets no connect timeout: against a host that silently drops connections, each of the 30 attempts waits for the operating system's connect timeout. +By default Codex retries a failed model connection forever, waiting up to 60 seconds between attempts, so a Turn whose model host is unreachable stays in progress with no events. The adapter disables that `unbounded_connection_retries` feature, so Codex falls back to its bounded defaults: up to 4 retries per request and 5 per response stream. A model host that does not resolve or refuses connections then fails the Turn with `connection_failed` within about a minute, depending on how long each lookup or connection attempt takes. Retries start only after a request fails, so slow responses are unaffected, and a stream still ends after Codex's 300 seconds without data. Codex sets no connect timeout: against a host that silently drops connections, each of the 30 attempts waits for the operating system's connect timeout. ## Execution controls