Skip to content

Commit 0902c94

Browse files
committed
Classify connection failures from retries without a response
1 parent 2de04cd commit 0902c94

10 files changed

Lines changed: 44 additions & 24 deletions

File tree

‎apps/web/src/features/sessions/session-diagnostics.test.tsx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ describe("Core diagnostics in the console", () => {
7070
const providerDefault = "Without a model provider set on the Session or its saved Agent, a hosted sandbox Session uses the default in Settings → Models and credentials.";
7171
expect(diagnosticFailure({ code: "invalid_request", params: {}, failed_at: null }, t)).toBe(`The model provider rejected the request. Check the model and parameters, starting with the model on the Session page. ${providerDefault}`);
7272
expect(diagnosticFailure({ code: "connection_failed", params: { http_status: 502 }, failed_at: null }, t)).toContain("HTTP 502. Check the Base URL of the Session's model provider.");
73+
// Only a failure without any HTTP response points at name resolution and reachability.
74+
expect(diagnosticFailure({ code: "connection_failed", params: { http_status: 502 }, failed_at: null }, t)).not.toContain("resolves");
75+
expect(diagnosticFailure({ code: "connection_failed", params: { http_status: null }, failed_at: null }, t)).toContain("Then check that the URL's host resolves and is reachable");
7376
expect(diagnosticFailure({ code: "runtime_unavailable", params: {}, failed_at: null }, t)).toBe("The Runtime is unavailable. Retry later. For a self-hosted Session, check Host connection on the Session page; for a hosted sandbox Session, check Settings → Sandbox and scheduling.");
7477
expect(diagnosticFailure({ code: "harness_error", params: {}, failed_at: null }, t)).toContain("Core does not show the Harness's own message");
7578
expect(diagnosticFailure({ code: "invalid_request", params: {}, failed_at: null }, i18n.getFixedT("zh-CN", "diagnostics"))).toBe("模型提供商拒绝了请求。请检查模型和参数,先看 Session 页显示的模型。Session 及其保存的 Agent 都未设置模型提供商时,托管沙箱 Session 使用“设置 → 模型与凭据”中的默认配置。");

‎apps/web/src/i18n/locales/en/diagnostics.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ export const diagnostics = {
4747
request_timeout: "Retry later.",
4848
context_length_exceeded: "Start a new Session or shorten the input.",
4949
cyber_policy: "Change the input before sending it again, or start a new Session.",
50-
connection_failed: "Check the Base URL of the Session's model provider. Without one set on the Session or its saved Agent, a hosted sandbox Session uses the default in Settings → Models and credentials. Then check that the URL's host resolves and is reachable from the Session's sandbox, or from the host of a self-hosted Session.",
50+
connection_failed: "Check the Base URL of the Session's model provider. Without one set on the Session or its saved Agent, a hosted sandbox Session uses the default in Settings → Models and credentials.",
5151
harness_error: "Core does not show the Harness's own message. Check the model and Harness on the Session page, then the Session's model provider. Without one set on the Session or its saved Agent, a hosted sandbox Session uses the default in Settings → Models and credentials.",
5252
model_provider_required: "Set a model provider on the Session or its saved Agent. A hosted sandbox Session can also use the default in Settings → Models and credentials.",
5353
runtime_unavailable: "Retry later. For a self-hosted Session, check Host connection on the Session page; for a hosted sandbox Session, check Settings → Sandbox and scheduling.",
@@ -60,6 +60,7 @@ export const diagnostics = {
6060
setupIndex: "Setup command {{number}}.",
6161
exitCode: "Exit code {{code}}.",
6262
httpStatus: "HTTP {{status}}.",
63+
unreachableHost: "Then check that the URL's host resolves and is reachable from the Session's sandbox, or from the host of a self-hosted Session.",
6364
step: { setup: "setup", python: "Python dependencies", npm: "npm dependencies", system: "system packages", file: "file preparation", skill: "Skill preparation" },
6465
unavailable: "Failure details unavailable",
6566
unavailableHelp: "The diagnostics read failed or no longer matches this status. Refresh to check again.",

‎apps/web/src/i18n/locales/zh-CN/diagnostics.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ export const diagnostics = {
4747
request_timeout: "请稍后重试。",
4848
context_length_exceeded: "请新建 Session 或缩短输入。",
4949
cyber_policy: "请修改输入后再发送,或新建 Session。",
50-
connection_failed: "请检查该 Session 所用模型提供商的 Base URL。Session 及其保存的 Agent 都未设置模型提供商时,托管沙箱 Session 使用“设置 → 模型与凭据”中的默认配置。再确认 Session 的沙箱(自托管 Session 则为其主机)能解析并访问该 URL 的主机。",
50+
connection_failed: "请检查该 Session 所用模型提供商的 Base URL。Session 及其保存的 Agent 都未设置模型提供商时,托管沙箱 Session 使用“设置 → 模型与凭据”中的默认配置。",
5151
harness_error: "Core 不显示 Harness 的原始消息。请检查 Session 页显示的模型和 Harness,再检查该 Session 所用的模型提供商。Session 及其保存的 Agent 都未设置模型提供商时,托管沙箱 Session 使用“设置 → 模型与凭据”中的默认配置。",
5252
model_provider_required: "请在 Session 或其保存的 Agent 上设置模型提供商。托管沙箱 Session 也可以使用“设置 → 模型与凭据”中的默认配置。",
5353
runtime_unavailable: "请稍后重试。自托管 Session 请检查 Session 页的“主机连接”;托管沙箱 Session 请检查“设置 → 沙箱与调度”。",
@@ -60,6 +60,7 @@ export const diagnostics = {
6060
setupIndex: "第 {{number}} 条设置命令。",
6161
exitCode: "退出码 {{code}}。",
6262
httpStatus: "HTTP {{status}}。",
63+
unreachableHost: "再确认 Session 的沙箱(自托管 Session 则为其主机)能解析并访问该 URL 的主机。",
6364
step: { setup: "设置", python: "Python 依赖", npm: "npm 依赖", system: "系统软件包", file: "文件准备", skill: "Skill 准备" },
6465
unavailable: "失败详情不可用",
6566
unavailableHelp: "诊断读取失败或与当前状态不一致,请刷新后重试。",

‎apps/web/src/lib/diagnostic-failure.ts‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
import type { DiagnosticFailure } from "@oac/agents-client";
22
import type { TFunction } from "i18next";
33

4+
/** The provider's HTTP status on a connection failure; null when no response arrived. */
5+
function connectionStatus(failure: DiagnosticFailure): number | null {
6+
return failure.code === "connection_failed" && "http_status" in failure.params ? failure.params.http_status : null;
7+
}
8+
49
/** Translate the catalog, never native error text or guessed message categories. */
510
export function diagnosticReason(failure: DiagnosticFailure, t: TFunction<"diagnostics">): string {
611
const gap = t("sentenceGap");
@@ -10,15 +15,16 @@ export function diagnosticReason(failure: DiagnosticFailure, t: TFunction<"diagn
1015
if (failure.params.index !== null) message += gap + t("setupIndex", { number: failure.params.index + 1 });
1116
if (failure.params.exit_code !== null) message += gap + t("exitCode", { code: failure.params.exit_code });
1217
}
13-
if (failure.code === "connection_failed" && "http_status" in failure.params && failure.params.http_status !== null) {
14-
message += gap + t("httpStatus", { status: failure.params.http_status });
15-
}
18+
const status = connectionStatus(failure);
19+
if (status !== null) message += gap + t("httpStatus", { status });
1620
return message;
1721
}
1822

1923
/** The catalog's next step for the reason; empty when it has none. */
2024
export function diagnosticNext(failure: DiagnosticFailure, t: TFunction<"diagnostics">): string {
21-
return t(`next.${failure.code}`, { defaultValue: "" });
25+
const next: string = t(`next.${failure.code}`, { defaultValue: "" });
26+
// Without an HTTP status no provider answered, so its host may not resolve or be reachable.
27+
return failure.code === "connection_failed" && connectionStatus(failure) === null ? next + t("sentenceGap") + t("unreachableHost") : next;
2228
}
2329

2430
/** The classified reason, then its next step when the catalog has one. */

‎docs/getting-started/nodes.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ Root only prepares the account, the group and the unit; everything else, the Doc
7474

7575
**Docker mode is root-equivalent.** Membership in the `docker` group lets `oac-node`, and so anything that controls the node, act as root on the host. Containers also share the host kernel, so a container escape reaches the host. Add Docker nodes only on hosts dedicated to sandboxes. microsandbox nodes need only the `kvm` group and run each sandbox in its own microVM.
7676

77-
**microsandbox blocks private endpoints.** A sandbox can't reach a model provider or MCP server at a private address. Sandboxes resolve names through the node host's resolver, and DNS rebind protection discards any answer that contains a private address, so a name that the host resolves to a private address doesn't resolve in the sandbox at all, even for a public service such as a model API behind a cloud VPC resolver. The Turn then fails once the Harness stops retrying. Installer-added nodes have no setting to change this. Make the host's resolver return the service's public addresses, for example with entries in the host's `/etc/hosts`; use a Docker node, which doesn't filter sandbox traffic; or [register the node manually](#register-a-node-manually) with a `network` policy that also allows the private range in a rule without a port, because rebind protection ignores port-scoped rules.
77+
**microsandbox blocks private endpoints.** A sandbox can't reach a model provider or MCP server at a private address. Sandbox DNS queries go to the nameservers listed in the host's `/etc/resolv.conf`, and DNS rebind protection discards any answer that contains a private address, so a name those nameservers resolve to a private address doesn't resolve in the sandbox at all, even for a public service such as a model API behind a cloud VPC resolver. The Turn then fails once the Harness stops retrying. Installer-added nodes have no setting to change this. Make those nameservers return the service's public addresses; microsandbox doesn't read the host's `/etc/hosts`, so an entry there helps only when `/etc/resolv.conf` points at a local stub resolver that serves it, such as systemd-resolved at `127.0.0.53`. Otherwise use a Docker node, which doesn't filter sandbox traffic, or [register the node manually](#register-a-node-manually) with a `network` policy that also allows the private range in a rule without a port, because rebind protection ignores port-scoped rules.
7878

7979
**One Core per host.** Nodes share the `oac-node` account, so a host serves one Core; a command from a second Core is refused.
8080

‎docs/zh/getting-started/nodes.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
---
22
title: "添加和管理节点"
33
source: docs/getting-started/nodes.md
4-
source_hash: 00b2655a0a6af8e210696b894a9dbce17590a91ff964665be0ecc4787c0accf6
4+
source_hash: 197acaddceb84ff46a833b883a8751c24d3f4a2ae49e077280e0c677a083424e
55
---
66

77
节点是一台 Linux 主机,在沙箱提供商为 microsandbox(推荐)或 Docker 时,为 Core 托管 Session 运行沙箱。Core 将新 Session 分配给有空余容量的节点;节点创建沙箱,沙箱回连 Core。E2B 不需要节点。应用为自己的 Session 连接的机器是[自托管执行器](self-hosted.md),而不是节点。
@@ -76,7 +76,7 @@ root 只准备账号、组和服务单元;其他操作(包括 Docker 网络
7676

7777
**Docker 模式等同于 root 权限。** 加入 `docker` 组后,`oac-node` 以及任何能控制节点的主体,都可以在主机上执行 root 操作。容器共享主机内核,因此容器逃逸能访问主机。只在沙箱专用主机上添加 Docker 节点。microsandbox 节点只需要 `kvm` 组,每个沙箱运行在独立 microVM 中。
7878

79-
**microsandbox 拦截私有端点。** 沙箱无法访问位于私有地址的模型服务或 MCP 服务器。沙箱通过节点主机的解析器解析域名,DNS 重绑定保护会丢弃任何包含私有地址的应答;因此,主机解析为私有地址的域名在沙箱中完全无法解析,即使它是公开服务,例如云 VPC 解析器后的模型 API。Harness 停止重试后,Turn 随即失败。通过安装程序添加的节点没有更改此行为的设置。可以让主机解析器返回该服务的公开地址,例如在主机的 `/etc/hosts` 中添加条目;或使用不过滤沙箱流量的 Docker 节点;或[手动注册节点](#register-a-node-manually),在 `network` 策略中为该私有网段添加不带端口的允许规则,因为重绑定保护忽略带端口的规则。
79+
**microsandbox 拦截私有端点。** 沙箱无法访问位于私有地址的模型提供商或 MCP 服务器。沙箱的 DNS 查询发往主机 `/etc/resolv.conf` 中列出的域名服务器,DNS 重绑定保护会丢弃任何包含私有地址的应答;因此,这些域名服务器解析为私有地址的域名在沙箱中完全无法解析,即使它是公开服务,例如云 VPC 解析器后的模型 API。Harness 停止重试后,Turn 随即失败。通过安装程序添加的节点没有更改此行为的设置。请让这些域名服务器返回该服务的公开地址;microsandbox 不读取主机的 `/etc/hosts`,因此只有当 `/etc/resolv.conf` 指向会读取该文件的本地 stub 解析器(例如位于 `127.0.0.53` 的 systemd-resolved)时,在其中添加条目才有效。否则,可以使用不过滤沙箱流量的 Docker 节点,或[手动注册节点](#register-a-node-manually),在 `network` 策略中为该私有网段添加不带端口的允许规则,因为重绑定保护忽略带端口的规则。
8080

8181
**每台主机只服务一个 Core。** 节点共享 `oac-node` 账号,因此主机只服务一个 Core;第二个 Core 的命令会被拒绝。
8282

‎packages/claude-sdk-adapter/src/native_failure.ts‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,21 @@ const codes: Record<SDKAssistantMessageError, string | undefined> = {
1313
// last error text. A matching native error result alone commits a classification.
1414
export class NativeFailure {
1515
private readonly candidates = new Map<string, string>();
16+
// Claude Code reports both retries that got no HTTP response and a server error in
17+
// an already streaming response as server_error without a status. Only the first
18+
// ends with a retry whose error_status is null and no model output after it.
19+
private connectionRetry = false;
1620

1721
observe(message: SDKMessage, sessionID: string, pending: readonly string[]): string | undefined {
18-
if (!sessionID || message.session_id !== sessionID || !pending.length ||
22+
if (!sessionID || message.session_id !== sessionID ||
1923
("isReplay" in message && message.isReplay) || ("isSynthetic" in message && message.isSynthetic)) return;
24+
if (message.type === "system" && message.subtype === "api_retry") this.connectionRetry = message.error_status === null;
25+
const connectionRetry = this.connectionRetry;
26+
if (message.type === "stream_event" || message.type === "assistant" || message.type === "result") this.connectionRetry = false;
27+
if (!pending.length) return;
2028
if (message.type === "assistant" && message.parent_tool_use_id === null) {
21-
const code = typeof message.error !== "string" || !Object.hasOwn(codes, message.error) ? undefined : codes[message.error];
29+
const native = typeof message.error !== "string" || !Object.hasOwn(codes, message.error) ? undefined : codes[message.error];
30+
const code = native === "server_error" && connectionRetry ? "connection_failed" : native;
2231
for (const id of pending) {
2332
this.candidates.delete(id);
2433
if (code) this.candidates.set(id, code);
@@ -27,11 +36,7 @@ export class NativeFailure {
2736
const candidate = this.candidates.get(pending[0]!);
2837
const same = pending.every(id => this.candidates.get(id) === candidate);
2938
for (const id of pending) this.candidates.delete(id);
30-
if (same && message.subtype === "error_during_execution") return candidate;
31-
// Claude Code also reports a request that got no HTTP response (DNS, refused or
32-
// reset connection, timeout) as server_error; only a provider 5xx has a status.
33-
if (same && message.subtype === "success" && message.is_error)
34-
return candidate === "server_error" && typeof message.api_error_status !== "number" ? "connection_failed" : candidate;
39+
if (same && (message.subtype === "error_during_execution" || message.subtype === "success" && message.is_error)) return candidate;
3540
}
3641
}
3742
}

‎packages/claude-sdk-adapter/tests/native_failure.test.mjs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,15 @@ test("pinned SDK finite native mapping", () => {
1818
}
1919
});
2020

21-
test("a server error without an HTTP status is a connection failure", () => {
22-
for (const [status, code] of [[null, "connection_failed"], [502, "server_error"]]) {
21+
test("a server error is a connection failure only after retries without an HTTP response", () => {
22+
const retry = error_status => ({ type: "system", subtype: "api_retry", session_id: "native", error_status, error: "unknown" });
23+
const text = { type: "stream_event", session_id: "native", parent_tool_use_id: null, event: { type: "content_block_start" } };
24+
for (const [before, code] of [[[retry(null)], "connection_failed"], [[retry(503)], "server_error"],
25+
[[retry(null), text, assistant(undefined)], "server_error"], [[], "server_error"]]) {
2326
const f = new NativeFailure();
27+
for (const message of before) f.observe(message, "native", message.type === "assistant" ? ["input"] : []);
2428
f.observe(assistant("server_error"), "native", ["input"]);
25-
assert.equal(f.observe(result({ subtype: "success", api_error_status: status }), "native", ["input"]), code);
29+
assert.equal(f.observe(result({ subtype: "success", api_error_status: null }), "native", ["input"]), code);
2630
}
2731
});
2832

0 commit comments

Comments
 (0)