From 8e5008c375f9e717e6cbb81fa9532599b98ff0a8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 3 Oct 2026 10:59:01 +0300 Subject: [PATCH 1/5] Ignore fenced examples in GLM fallback Co-authored-by: Medulla --- crates/tinytools-agent/src/parse/mod.rs | 28 ++++++++++++++++++- .../tinytools-agent/src/parse/test/markup.rs | 1 + 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/tinytools-agent/src/parse/mod.rs b/crates/tinytools-agent/src/parse/mod.rs index 05196b0..de892ee 100644 --- a/crates/tinytools-agent/src/parse/mod.rs +++ b/crates/tinytools-agent/src/parse/mod.rs @@ -76,7 +76,7 @@ pub fn parse_text(text: &str, options: &ParseOptions<'_>) -> ParseOutcome { let diagnostics = scan.diagnostics; if calls.is_empty() { - let joined = parts.join("\n"); + let joined = unprotected_narrative(text, &scan.kept); let (cleaned, glm_calls) = grammar::glm::parse_and_strip(&joined); if !glm_calls.is_empty() { calls = glm_calls; @@ -86,6 +86,32 @@ pub fn parse_text(text: &str, options: &ParseOptions<'_>) -> ParseOutcome { finalize(parts.join("\n"), calls, diagnostics, options) } +/// Keeps narrative for the GLM fallback while excluding fenced examples. +fn unprotected_narrative(text: &str, kept: &[Range]) -> String { + let protected = protected::fence_ranges(text); + let mut parts = Vec::new(); + for range in kept { + let mut cursor = range.start; + for fence in &protected { + if fence.end <= cursor || fence.start >= range.end { + continue; + } + if cursor < fence.start { + parts.push(text[cursor..fence.start.min(range.end)].trim()); + } + cursor = fence.end.min(range.end); + } + if cursor < range.end { + parts.push(text[cursor..range.end].trim()); + } + } + parts + .into_iter() + .filter(|part| !part.is_empty()) + .collect::>() + .join("\n") +} + /// Name resolution and the diagnostics it produces. fn finalize( text: String, diff --git a/crates/tinytools-agent/src/parse/test/markup.rs b/crates/tinytools-agent/src/parse/test/markup.rs index dca5d3e..28ad09e 100644 --- a/crates/tinytools-agent/src/parse/test/markup.rs +++ b/crates/tinytools-agent/src/parse/test/markup.rs @@ -46,5 +46,6 @@ fn prose_a_bare_json_answer_and_a_quoted_example_are_not_markup() { assert!(!contains_call_markup( "The format is:\n```xml\nls\n```" )); + assert!(!contains_call_markup("```text\nshell/command>ls\n```")); assert!(!contains_call_markup("")); } From a6573ae1eaba4cc1fe7547e1a9196b9e35de3c54 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 3 Oct 2026 19:48:07 +0300 Subject: [PATCH 2/5] test(parse): add regression test for fenced examples in glm parsing Adds a test covering a real glm call where a fenced code block contains a tool-call-like line that should not be parsed as a call. The test verifies that only the actual call outside the fence is extracted and that the fenced content remains intact in the text output. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-agent/src/parse/test/glm.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/tinytools-agent/src/parse/test/glm.rs b/crates/tinytools-agent/src/parse/test/glm.rs index 1eb6cd4..42ee463 100644 --- a/crates/tinytools-agent/src/parse/test/glm.rs +++ b/crates/tinytools-agent/src/parse/test/glm.rs @@ -11,6 +11,19 @@ fn glm_lines_parse_when_nothing_else_matched() { assert_eq!(calls[0].name, "shell"); } +#[test] +fn fenced_example_survives_a_real_glm_call() { + let input = "Before\n```text\nshell/command>example\n```\nAfter\nshell/command>real"; + let (text, calls) = parse(input); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, "shell"); + assert_eq!(calls[0].arguments, serde_json::json!({"command": "real"})); + assert!(text.contains("```text\nshell/command>example\n```")); + assert!(text.contains("Before")); + assert!(text.contains("After")); + assert!(!text.contains("shell/command>real")); +} + #[test] fn glm_parser_covers_json_payloads_invalid_urls_and_plain_commands() { let calls = parse_glm_style_tool_calls(concat!( From 6b3f8b8520288a5db694d77ef34c4927e79081f7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 3 Oct 2026 19:48:44 +0300 Subject: [PATCH 3/5] refactor(parse): extract GLM parsing into unprotected narrative helper The unprotected narrative function now parses GLM calls directly while preserving fenced examples, instead of returning raw text for a separate parsing pass. This consolidates the GLM fallback logic into a single function that returns both the cleaned narrative and any parsed tool calls, eliminating the need for a second parse step and reducing redundant processing. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-agent/src/parse/mod.rs | 33 +++++++++++++++---------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/crates/tinytools-agent/src/parse/mod.rs b/crates/tinytools-agent/src/parse/mod.rs index de892ee..44f59d9 100644 --- a/crates/tinytools-agent/src/parse/mod.rs +++ b/crates/tinytools-agent/src/parse/mod.rs @@ -76,40 +76,47 @@ pub fn parse_text(text: &str, options: &ParseOptions<'_>) -> ParseOutcome { let diagnostics = scan.diagnostics; if calls.is_empty() { - let joined = unprotected_narrative(text, &scan.kept); - let (cleaned, glm_calls) = grammar::glm::parse_and_strip(&joined); + let (cleaned, glm_calls) = parse_unprotected_glm(text, &scan.kept); if !glm_calls.is_empty() { calls = glm_calls; - return finalize(cleaned.trim().to_string(), calls, diagnostics, options); + return finalize(cleaned, calls, diagnostics, options); } } finalize(parts.join("\n"), calls, diagnostics, options) } -/// Keeps narrative for the GLM fallback while excluding fenced examples. -fn unprotected_narrative(text: &str, kept: &[Range]) -> String { +/// Parses GLM calls outside fences and keeps fenced examples in the narrative. +fn parse_unprotected_glm(text: &str, kept: &[Range]) -> (String, Vec) { let protected = protected::fence_ranges(text); let mut parts = Vec::new(); + let mut calls = Vec::new(); for range in kept { let mut cursor = range.start; + let mut cleaned = String::new(); for fence in &protected { if fence.end <= cursor || fence.start >= range.end { continue; } if cursor < fence.start { - parts.push(text[cursor..fence.start.min(range.end)].trim()); + let (part, parsed) = grammar::glm::parse_and_strip(&text[cursor..fence.start]); + cleaned.push_str(&part); + calls.extend(parsed); } - cursor = fence.end.min(range.end); + let end = fence.end.min(range.end); + cleaned.push_str(&text[cursor.max(fence.start)..end]); + cursor = end; } if cursor < range.end { - parts.push(text[cursor..range.end].trim()); + let (part, parsed) = grammar::glm::parse_and_strip(&text[cursor..range.end]); + cleaned.push_str(&part); + calls.extend(parsed); + } + let cleaned = cleaned.trim(); + if !cleaned.is_empty() { + parts.push(cleaned.to_string()); } } - parts - .into_iter() - .filter(|part| !part.is_empty()) - .collect::>() - .join("\n") + (parts.join("\n"), calls) } /// Name resolution and the diagnostics it produces. From cd5a9fbcd1cdb1a57b92c3db9eb8391416990d8f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 3 Oct 2026 19:49:07 +0300 Subject: [PATCH 4/5] test(parse): add regression test for glm fallback range handling Adds a test covering the case where a fenced text block falls entirely within a kept range during GLM fallback parsing, ensuring the fence content is preserved while the discarded prefix and the real command after the fence are handled correctly. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-agent/src/parse/test/markup.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/tinytools-agent/src/parse/test/markup.rs b/crates/tinytools-agent/src/parse/test/markup.rs index 28ad09e..fdd2d8b 100644 --- a/crates/tinytools-agent/src/parse/test/markup.rs +++ b/crates/tinytools-agent/src/parse/test/markup.rs @@ -3,6 +3,18 @@ use crate::parse::contains_call_markup; +#[test] +fn glm_fallback_keeps_fenced_text_within_a_kept_range() { + let input = "discard\nintro\n```text\nshell/command>example\n```\nshell/command>real"; + let start = input.find("intro").unwrap(); + let (text, calls) = super::super::parse_unprotected_glm(input, &[start..input.len()]); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].arguments, serde_json::json!({"command": "real"})); + assert!(text.starts_with("intro\n```text\nshell/command>example\n```")); + assert!(!text.contains("discard")); + assert!(!text.contains("shell/command>real")); +} + /// What `DeepSeek` V4 returned in place of a summary when its request declared /// no tools (captured from the `OpenHuman` harness benchmark). const DSML: &str = "<｜｜DSML｜｜ calls>\n<｜｜DSML｜｜ invoke name=\"shell\">\n\ From c95677f31a7cb4c7669b3f86a39b2b3f60aeeaea Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 3 Oct 2026 19:49:45 +0300 Subject: [PATCH 5/5] test(parse): update GLM fallback test for kept-range slice The test now passes the kept range as a slice reference rather than a direct range, matching the updated signature of `parse_unprotected_glm` which expects a slice of ranges. This ensures the test exercises the function with the correct argument type while preserving the original assertion of fenced text retention. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinytools-agent/src/parse/test/markup.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tinytools-agent/src/parse/test/markup.rs b/crates/tinytools-agent/src/parse/test/markup.rs index fdd2d8b..9c572d4 100644 --- a/crates/tinytools-agent/src/parse/test/markup.rs +++ b/crates/tinytools-agent/src/parse/test/markup.rs @@ -7,7 +7,8 @@ use crate::parse::contains_call_markup; fn glm_fallback_keeps_fenced_text_within_a_kept_range() { let input = "discard\nintro\n```text\nshell/command>example\n```\nshell/command>real"; let start = input.find("intro").unwrap(); - let (text, calls) = super::super::parse_unprotected_glm(input, &[start..input.len()]); + let kept = start..input.len(); + let (text, calls) = super::super::parse_unprotected_glm(input, std::slice::from_ref(&kept)); assert_eq!(calls.len(), 1); assert_eq!(calls[0].arguments, serde_json::json!({"command": "real"})); assert!(text.starts_with("intro\n```text\nshell/command>example\n```"));