From 69713fe6ed4f4ebce23000c82de592083045f077 Mon Sep 17 00:00:00 2001 From: madrynweb Date: Fri, 22 May 2026 20:18:24 -0300 Subject: [PATCH] fix: sanitize unescaped control characters in JSON tool call responses The provider sometimes returns JSON with literal \n, \r, and \t inside string values instead of escaped \\n, \\r, \\t. This caused JSON.parse to fail silently, so tools (function calling) were shown as text in the chat instead of being executed. Added sanitizeJsonString() to escape control characters only when inside JSON strings. Also improved brace counting to respect string boundaries and added graceful recovery when a tool call fails to parse. --- tools.js | 189 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 136 insertions(+), 53 deletions(-) diff --git a/tools.js b/tools.js index 1c7a265..4a01f35 100644 --- a/tools.js +++ b/tools.js @@ -59,13 +59,13 @@ When you receive a tool result, analyze it and provide a helpful response to the return result; } - + function normalizeMessages(messages) { return messages.map(msg => { const normalized = { role: msg.role }; - + if (msg.role === 'assistant' && msg.tool_calls) { - normalized.content = msg.tool_calls.map(tc => + normalized.content = msg.tool_calls.map(tc => `TOOL_CALL: ${tc.function.name}\nARGUMENTS: ${tc.function.arguments}` ).join('\n\n'); } else if (typeof msg.content === 'string') { @@ -76,7 +76,7 @@ function normalizeMessages(messages) { } else if (msg.content) { normalized.content = msg.content; } - + return normalized; }); } @@ -120,72 +120,72 @@ function parseMinimaxXML(responseText) { const toolCalls = []; const invokeMatches = minimaxMatch[1].matchAll(/]+)>(.*?)<\/invoke>/gs); - + for (const invokeMatch of invokeMatches) { const attrs = invokeMatch[1]; const innerContent = invokeMatch[2]; - + let toolName = ''; const args = {}; - + const attrPairs = attrs.matchAll(/(\w+):\s*"([^"]*)"/g); for (const attrPair of attrPairs) { const [, attrName, attrValue] = attrPair; if (!toolName) { toolName = extractToolName(attrName); } - + const mappedName = mapToolAttribute(attrName); if (mappedName) { args[mappedName] = attrValue; } } - + const paramMatches = innerContent.matchAll(/([^<]*)<\/parameter>/g); for (const paramMatch of paramMatches) { args[paramMatch[1]] = paramMatch[2].trim(); } - + if (toolName && Object.keys(args).length > 0) { toolCalls.push(createToolCallObject(toolName, args, toolCalls.length)); } } - + return toolCalls; } function parseClaudeXML(responseText) { const toolCalls = []; const functionCallMatches = responseText.matchAll(/\s*\s*(.*?)\s*<\/parameter_list>\s*<\/invoke>/gs); - + for (const match of functionCallMatches) { const [, toolName, parametersXml] = match; const args = {}; - + const paramMatches = parametersXml.matchAll(/(.*?)<\/parameter>/gs); for (const paramMatch of paramMatches) { const [, paramName, paramValue] = paramMatch; args[paramName] = paramValue.trim(); } - + if (toolName && Object.keys(args).length > 0) { toolCalls.push(createToolCallObject(toolName, args, toolCalls.length)); } } - + return toolCalls; } function parseOpenAIToolCalls(responseText) { const toolCalls = []; - + try { const toolCallMatch = responseText.match(/"tool_calls"\s*:\s*\[(.*?)\]/s); if (!toolCallMatch) return []; - + const toolCallArray = `[${toolCallMatch[1]}]`; const parsedToolCalls = JSON.parse(toolCallArray); - + for (const toolCall of parsedToolCalls) { if (toolCall.function && toolCall.function.name) { toolCalls.push({ @@ -201,70 +201,153 @@ function parseOpenAIToolCalls(responseText) { } catch (error) { return []; } - + return toolCalls; } +/** + * Sanitizes a JSON string by escaping unescaped control characters + * (newlines, tabs, etc.) that appear inside JSON string values. + * This handles patch texts and other multi-line content. + */ +function sanitizeJsonString(jsonStr) { + let result = ''; + let inString = false; + let escapeNext = false; + + for (let i = 0; i < jsonStr.length; i++) { + const char = jsonStr[i]; + + if (escapeNext) { + result += char; + escapeNext = false; + continue; + } + + if (char === '\\') { + result += char; + escapeNext = true; + continue; + } + + if (char === '"' && !inString) { + inString = true; + result += char; + continue; + } else if (char === '"' && inString) { + inString = false; + result += char; + continue; + } + + // Inside a string: escape control characters + if (inString) { + if (char === '\n') { + result += '\\n'; + } else if (char === '\r') { + result += '\\r'; + } else if (char === '\t') { + result += '\\t'; + } else if (char.charCodeAt(0) < 0x20) { + result += '\\u' + ('000' + char.charCodeAt(0).toString(16)).slice(-4); + } else { + result += char; + } + } else { + result += char; + } + } + + return result; +} + function parseTextFormat(responseText) { const toolCalls = []; let currentIndex = 0; - - const patterns = [ - /TOOL_CALL:\s*(\w+).*?ARGUMENTS:\s*\{/is, - /TOOL_CALL:\s*(\w+)ARGUMENTS:\s*\{/is, - ]; - + + const toolCallPattern = /TOOL_CALL:\s*(\w+)[\s\S]*?ARGUMENTS:\s*\{/i; + while (currentIndex < responseText.length) { - let toolCallMatch = null; - - for (const pattern of patterns) { - toolCallMatch = responseText.substring(currentIndex).match(pattern); - if (toolCallMatch) break; - } - + const remainingText = responseText.substring(currentIndex); + const toolCallMatch = remainingText.match(toolCallPattern); + if (!toolCallMatch) break; - + const toolName = toolCallMatch[1]; - const fullMatchText = toolCallMatch[0]; const matchStartIndex = currentIndex + toolCallMatch.index; - const argsStartIndex = matchStartIndex + fullMatchText.lastIndexOf('{'); - + const argsStartIndex = matchStartIndex + toolCallMatch[0].lastIndexOf('{'); + + // Brace counting that respects JSON strings (doesn't count braces inside strings) let braceCount = 0; + let inString = false; + let escapeNext = false; let argsEndIndex = argsStartIndex; let foundClosingBrace = false; - + for (let i = argsStartIndex; i < responseText.length; i++) { const char = responseText[i]; - if (char === '{') { - braceCount++; - } else if (char === '}') { - braceCount--; - if (braceCount === 0) { - argsEndIndex = i + 1; - foundClosingBrace = true; - break; + + if (escapeNext) { + escapeNext = false; + continue; + } + + if (char === '\\') { + escapeNext = true; + continue; + } + + if (char === '"' && !inString) { + inString = true; + continue; + } else if (char === '"' && inString) { + inString = false; + continue; + } + + if (!inString) { + if (char === '{') { + braceCount++; + } else if (char === '}') { + braceCount--; + if (braceCount === 0) { + argsEndIndex = i + 1; + foundClosingBrace = true; + break; + } } } } - + if (!foundClosingBrace) { console.warn('[Tool Parsing] Incomplete JSON for tool call:', toolName); break; } - - const argsString = responseText.substring(argsStartIndex, argsEndIndex); - + + let argsString = responseText.substring(argsStartIndex, argsEndIndex); + + // Sanitize: escape unescaped control characters inside JSON strings + argsString = sanitizeJsonString(argsString); + try { const args = JSON.parse(argsString); toolCalls.push(createToolCallObject(toolName, args, toolCalls.length)); currentIndex = argsEndIndex; } catch (error) { - console.error('Failed to parse tool call arguments:', error); - console.debug('[Tool Parsing] Failed to parse:', argsString.substring(0, 200)); - break; + console.error('Failed to parse tool call arguments:', error.message); + console.debug('[Tool Parsing] Failed substring (first 300 chars):', argsString.substring(0, 300)); + + // Try to recover: find the next TOOL_CALL and skip this broken one + const nextToolCall = responseText.substring(argsEndIndex).match(/TOOL_CALL:\s*(\w+)/i); + if (nextToolCall) { + currentIndex = argsEndIndex + nextToolCall.index; + console.warn('[Tool Parsing] Skipping broken JSON, trying next tool call at index', currentIndex); + } else { + break; + } } } - + return toolCalls; }