Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions docs/tool-approval-replay.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,28 @@ records fail closed. The replay module has no HTTP, database or host UI dependen

## Recovery guarantees and host obligations

Hosts should set `TOOL_APPROVAL_EXECUTION_SCOPE_CONFIG_KEY` in `configurable`
to a generation identifier when the SDK advertises
`TOOL_APPROVAL_EXECUTION_SCOPE_CAPABLE`. The scope key predates that capability;
older SDKs can derive scoped owners but still attempt to replay completed batches
on later graph tasks. Hosts must check the capability before enabling the scope.
Keep it stable through every approval resume,
including rebuilt `Run` instances, and change it for each new generation. A
conversation id alone is insufficient; edits that reuse a response id also need
a generation epoch. Explicit scoping removes LangGraph's changing task namespace
from the composite owner while retaining the executing agent id.

Carried evidence is active only for its resumed interrupt and consuming task.
The interrupt id is checked against LangGraph's resume map, and the task's
scratchpad distinguishes a resumed node from later steps of the same invocation.
This applies to every ToolNode interrupt, including `ask_user_question` pauses
that have replay records but no approval-review evidence.
A validated parent replaying a child approval can forward that child's evidence
without a local resume value. Stale review and settled-batch evidence are ignored
together. An active approval with a different execution owner still fails closed;
starting a fresh generation may select a different agent without inheriting the
prior approval.

This protocol guarantees replay of **checkpointed completed work** when resuming
an approval pause. Rebuilding a Run/ToolNode with the same checkpointer is a
supported operation; retaining the old hooks or instances is not required.
Expand Down
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@librechat/agents",
"version": "3.8.2",
"version": "3.8.4",
"reova": {
"enabled": true,
"endpoint": "https://telemetry.reo.dev/data"
Expand Down
3 changes: 3 additions & 0 deletions src/hooks/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,3 +116,6 @@ export type {
PostCompactHookOutput,
} from './types';
export type { ExecuteHooksOptions } from './executeHooks';

/** Hosts may opt into stable generation scopes across every ToolNode interrupt type. */
export const TOOL_APPROVAL_EXECUTION_SCOPE_CAPABLE = true;
6 changes: 6 additions & 0 deletions src/hooks/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@ export const HOOK_EVENTS = [
'PostCompact',
] as const;

/**
* Host-owned generation identity, stable across every resume and rebuilt Run.
* A new generation must use a new scope, even when it reuses a thread or response
* id. Explicit scoping keeps LangGraph task namespaces out of approval owners;
* the executing agent id remains part of the owner and cannot change on resume.
*/
export const TOOL_APPROVAL_EXECUTION_SCOPE_CONFIG_KEY =
'__librechat_tool_approval_execution_scope';

Expand Down
4 changes: 2 additions & 2 deletions src/tools/BashExecutor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ function createBashExecutionTool(
};

const proxyAgent = resolveFetchProxyAgent(execEndpoint);
if (proxyAgent) {
if (proxyAgent != null) {
fetchOptions.agent = proxyAgent;
}
const response = await fetch(execEndpoint, fetchOptions);
Expand Down Expand Up @@ -343,7 +343,7 @@ function createBashExecutionTool(
normalizeCodeApiRequestError(error).message,
command
);
throw new Error(`Execution error:\n\n${messageWithReminder}`);
throw new CodeApiRequestError(`Execution error:\n\n${messageWithReminder}`);
}
},
{
Expand Down
8 changes: 5 additions & 3 deletions src/tools/BashProgrammaticToolCalling.ts
Original file line number Diff line number Diff line change
Expand Up @@ -542,9 +542,11 @@ export function createBashProgrammaticToolCallingTool(
(error as Error).message,
code
);
throw new Error(
`Bash programmatic execution failed: ${messageWithReminder}`
);
const message = `Bash programmatic execution failed: ${messageWithReminder}`;
if (error instanceof CodeApiRequestError) {
throw new CodeApiRequestError(message);
}
throw new Error(message, { cause: error });
}
},
{
Expand Down
128 changes: 111 additions & 17 deletions src/tools/CodeExecutor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ export function appendFailedExecutionFileReminder(
code: string
): string {
if (
output.includes(CODE_API_CAPABILITY_ERROR_MESSAGE) ||
!MNT_DATA_PATH_PATTERN.test(code) ||
output.includes(FAILED_EXECUTION_FILE_REMINDER)
) {
Expand Down Expand Up @@ -157,10 +158,67 @@ const SAFE_CODE_API_EXECUTION_ERROR_DETAILS: Readonly<
'stdout length exceeded': 'Execution output exceeded the size limit.',
};

export const CODE_API_CAPABILITY_ERROR_MESSAGE =
'Code execution is not supported by the selected environment.';
const CODE_API_CAPABILITY_ERROR_GUIDANCE =
'This is a permanent capability mismatch. Do not retry this tool in this environment; select a compatible environment or use an available workspace tool.';

const CODE_API_CAPABILITY_LIMITATIONS: Readonly<
Partial<Record<string, string>>
> = {
bridge_worker_mismatch:
'The selected worker does not support the requested execution capabilities (such as the stateful workspace required by programmatic tool calling).',
execution_profile_mismatch:
'The selected environment does not provide the requested execution profile.',
capability_mismatch:
'The selected environment lacks a required execution capability.',
unsupported_capability:
'The selected environment lacks a required execution capability.',
stateful_workspace_unsupported:
'The selected worker does not provide a stateful workspace.',
};

function getCodeApiCapabilityErrorMessage(
responseBody: string
): string | undefined {
try {
const parsed = JSON.parse(responseBody) as {
error?: string;
code?: string;
message?: string;
} | null;
const code = parsed?.code ?? parsed?.error;
if (typeof code !== 'string') return undefined;
const normalizedCode = code.toLowerCase();
if (!Object.hasOwn(CODE_API_CAPABILITY_LIMITATIONS, normalizedCode))
return undefined;
let limitation = CODE_API_CAPABILITY_LIMITATIONS[normalizedCode];
if (limitation == null) return undefined;
if (
normalizedCode === 'bridge_worker_mismatch' &&
typeof parsed?.message === 'string' &&
/^Bridge worker [A-Za-z0-9._:-]+ does not provide a stateful workspace$/.test(
parsed.message
)
) {
limitation =
CODE_API_CAPABILITY_LIMITATIONS.stateful_workspace_unsupported;
}
return `${CODE_API_CAPABILITY_ERROR_MESSAGE} ${limitation} ${CODE_API_CAPABILITY_ERROR_GUIDANCE}`;
} catch {
return undefined;
}
}

export class CodeApiRequestError extends Error {
readonly retryable: boolean;

constructor(message = CODE_API_UNAVAILABLE_ERROR_MESSAGE) {
super(message);
this.name = 'CodeApiRequestError';
this.retryable =
message.includes(CODE_API_UNAVAILABLE_ERROR_MESSAGE) ||
message.includes('Code execution is temporarily rate-limited.');
}
}

Expand Down Expand Up @@ -277,11 +335,7 @@ type CodeApiErrorResponse = {
body?: NodeJS.ReadableStream | null;
};

/**
* Only the 429 branch reads the body, and node-fetch keeps the stream and its
* socket alive until something does. Repeated backend failures would otherwise
* accumulate connections holding payloads this module deliberately discards.
*/
/** Releases unread or incomplete error responses after bounded classification. */
function discardResponseBody(response: CodeApiErrorResponse): void {
const body = response.body;
if (body == null || !('destroy' in body)) {
Expand All @@ -298,15 +352,57 @@ function discardResponseBody(response: CodeApiErrorResponse): void {
}
}

const MAX_CODE_API_ERROR_BODY_BYTES = 64 * 1024;
const CODE_API_ERROR_BODY_TIMEOUT_MS = 1_000;

/** Reads only small error envelopes, with a deadline for stalled upstreams. */
async function readCodeApiErrorBody(
response: CodeApiErrorResponse
): Promise<string> {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
const read = async (): Promise<string> => {
if (response.body != null && Symbol.asyncIterator in response.body) {
let bytes = 0;
const chunks: Buffer[] = [];
for await (const chunk of response.body as AsyncIterable<
Buffer | string
>) {
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
bytes += buffer.length;
if (bytes > MAX_CODE_API_ERROR_BODY_BYTES) return '';
chunks.push(buffer);
}
return Buffer.concat(chunks).toString('utf8');
}
const body = await response.text();
return Buffer.byteLength(body) <= MAX_CODE_API_ERROR_BODY_BYTES
? body
: '';
};
return await Promise.race([
read(),
new Promise<string>((resolve) => {
timer = setTimeout(() => resolve(''), CODE_API_ERROR_BODY_TIMEOUT_MS);
}),
]);
} catch {
return '';
} finally {
clearTimeout(timer);
discardResponseBody(response);
}
}

export async function buildCodeApiHttpErrorMessage(
method: CodeApiMethod,
_endpoint: string,
response: CodeApiErrorResponse,
options?: { recoverable?: boolean; profile?: t.CodeApiExecutionProfile }
): Promise<string> {
/* Logged before the body is touched. A non-OK response can leave a chunked
body open, and there is no read timeout here, so draining first would
withhold the diagnostic indefinitely for exactly the backend it identifies.
body open, so logging first preserves the diagnostic even if the bounded
classification read times out.
The body itself is never logged — it is upstream free text that can echo
the header that was sent — and the endpoint is host-configured, so the
backend is named by the profile this module chose rather than by its
Expand All @@ -325,16 +421,12 @@ export async function buildCodeApiHttpErrorMessage(
}
);
}
if (response.status !== 429) {
discardResponseBody(response);
const responseBody = await readCodeApiErrorBody(response);
const capabilityError = getCodeApiCapabilityErrorMessage(responseBody);
if (capabilityError != null) {
return capabilityError;
}
if (response.status === 429) {
let responseBody = '';
try {
responseBody = await response.text();
} catch {
responseBody = '';
}
const retryAfterSeconds = getRetryAfterSeconds(responseBody);
return retryAfterSeconds != null
? `Code execution is temporarily rate-limited. Retry after ${retryAfterSeconds} seconds.`
Expand Down Expand Up @@ -543,7 +635,7 @@ function createCodeExecutionTool(
};

const proxyAgent = resolveFetchProxyAgent(execEndpoint);
if (proxyAgent) {
if (proxyAgent != null) {
fetchOptions.agent = proxyAgent;
}
const response = await fetch(execEndpoint, fetchOptions);
Expand Down Expand Up @@ -610,7 +702,9 @@ function createCodeExecutionTool(
normalizeCodeApiRequestError(error).message,
code
);
throw new Error(`Execution error:\n\n${messageWithReminder}`);
throw new CodeApiRequestError(
`Execution error:\n\n${messageWithReminder}`
);
}
},
{
Expand Down
18 changes: 10 additions & 8 deletions src/tools/ProgrammaticToolCalling.ts
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,7 @@ export async function fetchSessionFiles(
};

const proxyAgent = resolveFetchProxyAgent(filesEndpoint, proxy);
if (proxyAgent) {
if (proxyAgent != null) {
fetchOptions.agent = proxyAgent;
}

Expand Down Expand Up @@ -531,7 +531,7 @@ export async function makeRequest(
};

const proxyAgent = resolveFetchProxyAgent(endpoint, proxy);
if (proxyAgent) {
if (proxyAgent != null) {
fetchOptions.agent = proxyAgent;
}

Expand Down Expand Up @@ -689,7 +689,7 @@ type ToolInputSchemaKind = {
function detectSchemaKind(schema: unknown): ToolInputSchemaKind {
const kind: ToolInputSchemaKind = { object: false, string: false };

if (!schema || typeof schema !== 'object') {
if (schema == null || typeof schema !== 'object') {
return kind;
}

Expand All @@ -704,7 +704,7 @@ function detectSchemaKind(schema: unknown): ToolInputSchemaKind {
}

const zodDef = (schema as { _def?: unknown })._def;
if (!zodDef || typeof zodDef !== 'object') {
if (zodDef == null || typeof zodDef !== 'object') {
return kind;
}

Expand All @@ -726,7 +726,7 @@ function detectSchemaKind(schema: unknown): ToolInputSchemaKind {
type?: unknown;
}
).innerType ?? (zodDef as { schema?: unknown }).schema;
if (innerSchema) {
if (innerSchema != null) {
const innerKind = detectSchemaKind(innerSchema);
kind.object ||= innerKind.object;
kind.string ||= innerKind.string;
Expand Down Expand Up @@ -1221,9 +1221,11 @@ export function createProgrammaticToolCallingTool(
(error as Error).message,
code
);
throw new Error(
`Programmatic execution failed: ${messageWithReminder}`
);
const message = `Programmatic execution failed: ${messageWithReminder}`;
if (error instanceof CodeApiRequestError) {
throw new CodeApiRequestError(message);
}
throw new Error(message, { cause: error });
}
},
{
Expand Down
Loading
Loading