diff --git a/crates/tinytools-agent/src/parse/mod.rs b/crates/tinytools-agent/src/parse/mod.rs index 05196b0..44f59d9 100644 --- a/crates/tinytools-agent/src/parse/mod.rs +++ b/crates/tinytools-agent/src/parse/mod.rs @@ -76,16 +76,49 @@ pub fn parse_text(text: &str, options: &ParseOptions<'_>) -> ParseOutcome { let diagnostics = scan.diagnostics; if calls.is_empty() { - let joined = parts.join("\n"); - let (cleaned, glm_calls) = grammar::glm::parse_and_strip(&joined); + let (cleaned, glm_calls) = parse_unprotected_glm(text, &scan.kept); if !glm_calls.is_empty() { calls = glm_calls; - return finalize(cleaned.trim().to_string(), calls, diagnostics, options); + return finalize(cleaned, calls, diagnostics, options); } } finalize(parts.join("\n"), calls, diagnostics, options) } +/// Parses GLM calls outside fences and keeps fenced examples in the narrative. +fn parse_unprotected_glm(text: &str, kept: &[Range]) -> (String, Vec) { + let protected = protected::fence_ranges(text); + let mut parts = Vec::new(); + let mut calls = Vec::new(); + for range in kept { + let mut cursor = range.start; + let mut cleaned = String::new(); + for fence in &protected { + if fence.end <= cursor || fence.start >= range.end { + continue; + } + if cursor < fence.start { + let (part, parsed) = grammar::glm::parse_and_strip(&text[cursor..fence.start]); + cleaned.push_str(&part); + calls.extend(parsed); + } + let end = fence.end.min(range.end); + cleaned.push_str(&text[cursor.max(fence.start)..end]); + cursor = end; + } + if cursor < range.end { + let (part, parsed) = grammar::glm::parse_and_strip(&text[cursor..range.end]); + cleaned.push_str(&part); + calls.extend(parsed); + } + let cleaned = cleaned.trim(); + if !cleaned.is_empty() { + parts.push(cleaned.to_string()); + } + } + (parts.join("\n"), calls) +} + /// Name resolution and the diagnostics it produces. fn finalize( text: String, diff --git a/crates/tinytools-agent/src/parse/test/glm.rs b/crates/tinytools-agent/src/parse/test/glm.rs index 1eb6cd4..42ee463 100644 --- a/crates/tinytools-agent/src/parse/test/glm.rs +++ b/crates/tinytools-agent/src/parse/test/glm.rs @@ -11,6 +11,19 @@ fn glm_lines_parse_when_nothing_else_matched() { assert_eq!(calls[0].name, "shell"); } +#[test] +fn fenced_example_survives_a_real_glm_call() { + let input = "Before\n```text\nshell/command>example\n```\nAfter\nshell/command>real"; + let (text, calls) = parse(input); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "shell"); + assert_eq!(calls[0].arguments, serde_json::json!({"command": "real"})); + assert!(text.contains("```text\nshell/command>example\n```")); + assert!(text.contains("Before")); + assert!(text.contains("After")); + assert!(!text.contains("shell/command>real")); +} + #[test] fn glm_parser_covers_json_payloads_invalid_urls_and_plain_commands() { let calls = parse_glm_style_tool_calls(concat!( diff --git a/crates/tinytools-agent/src/parse/test/markup.rs b/crates/tinytools-agent/src/parse/test/markup.rs index dca5d3e..9c572d4 100644 --- a/crates/tinytools-agent/src/parse/test/markup.rs +++ b/crates/tinytools-agent/src/parse/test/markup.rs @@ -3,6 +3,19 @@ use crate::parse::contains_call_markup; +#[test] +fn glm_fallback_keeps_fenced_text_within_a_kept_range() { + let input = "discard\nintro\n```text\nshell/command>example\n```\nshell/command>real"; + let start = input.find("intro").unwrap(); + let kept = start..input.len(); + let (text, calls) = super::super::parse_unprotected_glm(input, std::slice::from_ref(&kept)); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments, serde_json::json!({"command": "real"})); + assert!(text.starts_with("intro\n```text\nshell/command>example\n```")); + assert!(!text.contains("discard")); + assert!(!text.contains("shell/command>real")); +} + /// What `DeepSeek` V4 returned in place of a summary when its request declared /// no tools (captured from the `OpenHuman` harness benchmark). const DSML: &str = "<||DSML|| calls>\n<||DSML|| invoke name=\"shell\">\n\ @@ -46,5 +59,6 @@ fn prose_a_bare_json_answer_and_a_quoted_example_are_not_markup() { assert!(!contains_call_markup( "The format is:\n```xml\nls\n```" )); + assert!(!contains_call_markup("```text\nshell/command>ls\n```")); assert!(!contains_call_markup("")); }