From e5c4a3b0f3ed27de4dad35989a00d8fa552e88e9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 21:58:34 +0300 Subject: [PATCH 01/18] test(parse): add regression test for DSML closers on a tool call opener Adds a test that reproduces a scenario where DSML closing tags appear immediately after a tool call opener, ensuring the parser correctly handles this edge case without misinterpreting the structure. Auto-committed-on: dragonfly --- .../tinytools-agent/src/parse/test/tagged.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index aca11db..92abeea 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -573,3 +573,26 @@ fn the_plural_dsml_wrapper_is_not_a_tag_marker() { assert_eq!(calls.len(), 1, "the inner call is the only call: {calls:?}"); assert_eq!(calls[0].name, "echo"); } + +#[test] +fn repro_dsml_closers_on_a_tool_call_opener() { + let text = concat!( + "Heredocs aren't working in this shell. Writing the script to a file instead.\n\n", + "\n", + "{\"arguments\":{\"path\":\"work/extract.py\",\"content\":\"import re\"}}\n", + "<\u{ff5c}DSML\u{ff5c} parameter name=\"name\":\"file_write\"}\n", + "\n", + "<\u{ff5c}DSML\u{ff5c} invoke>\n", + "{\"arguments\":{\"category\":\"read\",\"command\":\"ls\"},\"name\":\"shell\"}\n", + "\n", + "", + ); + let out = parse_known(text, &["file_write", "shell"]); + let (narrative, calls) = (out.text.clone(), out.calls.clone()); + eprintln!("NARRATIVE: {narrative:?}"); + eprintln!("CALLS: {}", calls.len()); + for c in &calls { + eprintln!(" name={:?} args={}", c.name, c.arguments); + } + panic!("inspection"); +} From 87b3870ea632456f8f515a8ee0330bdda38f9e62 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 21:58:40 +0300 Subject: [PATCH 02/18] fix(test): use fully qualified path for parse_known in tagged test The test helper `parse_known` was called with a relative path that relied on a local import, but the import was not present in the test module. Using the fully qualified path ensures the function is resolved correctly regardless of the module's import state. Auto-committed-on: dragonfly --- crates/tinytools-agent/src/parse/test/tagged.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 92abeea..0ad13cc 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -587,7 +587,7 @@ fn repro_dsml_closers_on_a_tool_call_opener() { "\n", "", ); - let out = parse_known(text, &["file_write", "shell"]); + let out = crate::parse::test::parse_known(text, &["file_write", "shell"]); let (narrative, calls) = (out.text.clone(), out.calls.clone()); eprintln!("NARRATIVE: {narrative:?}"); eprintln!("CALLS: {}", calls.len()); From f9685fa734b88241bf7a974b66aa4e57138f2a60 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 21:59:02 +0300 Subject: [PATCH 03/18] test(parse): replace monolithic repro test with isolated cases Replace the single large regression test that combined multiple DSML closing tag scenarios into one input with separate, independently labeled test cases. This makes failures easier to diagnose by isolating each scenario and printing its result individually, rather than requiring manual inspection of a combined output. Auto-committed-on: dragonfly --- .../tinytools-agent/src/parse/test/tagged.rs | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 0ad13cc..b43352f 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -575,24 +575,25 @@ fn the_plural_dsml_wrapper_is_not_a_tag_marker() { } #[test] -fn repro_dsml_closers_on_a_tool_call_opener() { - let text = concat!( - "Heredocs aren't working in this shell. Writing the script to a file instead.\n\n", - "\n", - "{\"arguments\":{\"path\":\"work/extract.py\",\"content\":\"import re\"}}\n", - "<\u{ff5c}DSML\u{ff5c} parameter name=\"name\":\"file_write\"}\n", - "\n", - "<\u{ff5c}DSML\u{ff5c} invoke>\n", - "{\"arguments\":{\"category\":\"read\",\"command\":\"ls\"},\"name\":\"shell\"}\n", - "\n", - "", - ); - let out = crate::parse::test::parse_known(text, &["file_write", "shell"]); - let (narrative, calls) = (out.text.clone(), out.calls.clone()); - eprintln!("NARRATIVE: {narrative:?}"); - eprintln!("CALLS: {}", calls.len()); - for c in &calls { - eprintln!(" name={:?} args={}", c.name, c.arguments); +fn repro_isolate() { + let cases: &[(&str, &str)] = &[ + ("A: tool_call opener, DSML parameter closer, name in json", + "\n{\"arguments\":{\"path\":\"p\"},\"name\":\"file_write\"}"), + ("B: bare DSML invoke opener, name in json", + "<\u{ff5c}DSML\u{ff5c} invoke>\n{\"arguments\":{\"command\":\"ls\"},\"name\":\"shell\"}\n"), + ("C: tool_call opener, proper closer (control)", + "\n{\"arguments\":{\"path\":\"p\"},\"name\":\"file_write\"}"), + ("D: bare plain invoke opener, name in json", + "\n{\"arguments\":{\"command\":\"ls\"},\"name\":\"shell\"}"), + ("E: tool_call opener, DSML invoke closer", + "\n{\"arguments\":{\"path\":\"p\"},\"name\":\"file_write\"}"), + ("F: tool_call opener, unterminated (eof)", + "\n{\"arguments\":{\"path\":\"p\"},\"name\":\"file_write\"}"), + ]; + for (label, text) in cases { + let out = crate::parse::test::parse_known(text, &["file_write", "shell"]); + eprintln!("{label} -> calls={} names={:?}", out.calls.len(), + out.calls.iter().map(|c| c.name.clone()).collect::>()); } panic!("inspection"); } From fa362a692ade2e6d706a374e51effe41cb85915b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 21:59:21 +0300 Subject: [PATCH 04/18] test(parse): replace repro_isolate test cases with more targeted DSML scenarios The repro_isolate test was replaced with repro_isolate2, which uses a shared DSML parameter closer variable and constructs test inputs programmatically with format! to reduce repetition. The new cases focus on edge combinations of tool_call openers, DSML parameter closers, and bogus name parameter lines, making the test more precise about the parsing behaviour being verified. Auto-committed-on: dragonfly --- .../tinytools-agent/src/parse/test/tagged.rs | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index b43352f..b2bbd36 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -575,20 +575,17 @@ fn the_plural_dsml_wrapper_is_not_a_tag_marker() { } #[test] -fn repro_isolate() { - let cases: &[(&str, &str)] = &[ - ("A: tool_call opener, DSML parameter closer, name in json", - "\n{\"arguments\":{\"path\":\"p\"},\"name\":\"file_write\"}"), - ("B: bare DSML invoke opener, name in json", - "<\u{ff5c}DSML\u{ff5c} invoke>\n{\"arguments\":{\"command\":\"ls\"},\"name\":\"shell\"}\n"), - ("C: tool_call opener, proper closer (control)", - "\n{\"arguments\":{\"path\":\"p\"},\"name\":\"file_write\"}"), - ("D: bare plain invoke opener, name in json", - "\n{\"arguments\":{\"command\":\"ls\"},\"name\":\"shell\"}"), - ("E: tool_call opener, DSML invoke closer", - "\n{\"arguments\":{\"path\":\"p\"},\"name\":\"file_write\"}"), - ("F: tool_call opener, unterminated (eof)", - "\n{\"arguments\":{\"path\":\"p\"},\"name\":\"file_write\"}"), +fn repro_isolate2() { + let p = ""; + let cases: &[(&str, String)] = &[ + ("G: tool_call + args-only json + DSML param closer", + format!("\n{{\"arguments\":{{\"path\":\"p\"}}}}{p}")), + ("H: G + bogus name parameter line", + format!("\n{{\"arguments\":{{\"path\":\"p\"}}}}{p}\n<\u{ff5c}DSML\u{ff5c} parameter name=\"name\":\"file_write\"}}{p}")), + ("I: H + DSML invoke closer", + format!("\n{{\"arguments\":{{\"path\":\"p\"}}}}{p}\n<\u{ff5c}DSML\u{ff5c} parameter name=\"name\":\"file_write\"}}{p}\n")), + ("J: named DSML invoke (control)", + format!("<\u{ff5c}DSML\u{ff5c} invoke name=\"shell\">\n{{\"arguments\":{{\"command\":\"ls\"}}}}{p}\n")), ]; for (label, text) in cases { let out = crate::parse::test::parse_known(text, &["file_write", "shell"]); From a772e25126fb42c3da91e372d58d64322b11bb99 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 21:59:52 +0300 Subject: [PATCH 05/18] chore(parse): remove leftover debug test Removed a test function that was used only for manual inspection during development and would always panic, as it served no purpose in the test suite. Auto-committed-on: dragonfly --- .../tinytools-agent/src/parse/test/tagged.rs | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index b2bbd36..aca11db 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -573,24 +573,3 @@ fn the_plural_dsml_wrapper_is_not_a_tag_marker() { assert_eq!(calls.len(), 1, "the inner call is the only call: {calls:?}"); assert_eq!(calls[0].name, "echo"); } - -#[test] -fn repro_isolate2() { - let p = ""; - let cases: &[(&str, String)] = &[ - ("G: tool_call + args-only json + DSML param closer", - format!("\n{{\"arguments\":{{\"path\":\"p\"}}}}{p}")), - ("H: G + bogus name parameter line", - format!("\n{{\"arguments\":{{\"path\":\"p\"}}}}{p}\n<\u{ff5c}DSML\u{ff5c} parameter name=\"name\":\"file_write\"}}{p}")), - ("I: H + DSML invoke closer", - format!("\n{{\"arguments\":{{\"path\":\"p\"}}}}{p}\n<\u{ff5c}DSML\u{ff5c} parameter name=\"name\":\"file_write\"}}{p}\n")), - ("J: named DSML invoke (control)", - format!("<\u{ff5c}DSML\u{ff5c} invoke name=\"shell\">\n{{\"arguments\":{{\"command\":\"ls\"}}}}{p}\n")), - ]; - for (label, text) in cases { - let out = crate::parse::test::parse_known(text, &["file_write", "shell"]); - eprintln!("{label} -> calls={} names={:?}", out.calls.len(), - out.calls.iter().map(|c| c.name.clone()).collect::>()); - } - panic!("inspection"); -} From 4782b3c3e1dacefd3d7b285eea84dcb3443f7e30 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 22:00:50 +0300 Subject: [PATCH 06/18] feat(parse): accept prefixed bare `` tags in tagged grammar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bare `` opener and `` closer previously matched only the literal strings, so a prefixed form such as `<|DSML| invoke>` or `` was not recognised as a block opener and the call was silently dropped as prose. The named variant `<|DSML|invoke name="x">` already worked because it used a different parser path. This change introduces two new regular expressions that accept the same optional DSML marker or XML namespace prefix that the named form tolerates, and replaces the literal string matches with calls to those regexes. Auto-committed-on: dragonfly --- .../src/parse/grammar/tagged.rs | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 8fe09ef..0648416 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -57,6 +57,39 @@ static TAG_RE: LazyLock> = LazyLock::new(|| { .ok() }); +/// The bare `` literal — no attributes — with the optional `DeepSeek` +/// DSML marker or XML namespace the named form already tolerates +/// ([`super::invoke_xml`]'s `PREFIX`). +/// +/// The prefix used to be absent here: the opener was a literal `""` +/// match and the closer a literal `""`, so `<|DSML| invoke>` — a +/// `deepseek` turn that emitted the invoke form *without* a `name` attribute, +/// carrying the name in the JSON body instead — opened no block and the call +/// was dropped as prose. The named spelling `<|DSML|invoke name="x">` parsed +/// fine, and so did the unprefixed bare ``; only the combination of +/// the two accommodations was missing. +/// +/// Attributes are excluded on purpose: `` belongs to +/// [`super::invoke_xml`], which reads the name off the tag. This matches only +/// the attribute-less form, whose name can come from the body. +static BARE_INVOKE_OPEN_RE: LazyLock> = LazyLock::new(|| { + Regex::new(r"(?i)<(?:[|\u{ff5c}]{1,2}\s*DSML\s*[|\u{ff5c}]{1,2}\s*|[a-z_][\w.-]*:)?invoke\s*>") + .ok() +}); + +/// The matching closer for [`BARE_INVOKE_OPEN_RE`], same prefixes. +static BARE_INVOKE_CLOSE_RE: LazyLock> = LazyLock::new(|| { + Regex::new(r"(?i)") + .ok() +}); + +/// First match of `re` in `haystack`, as `(start, end)`. +fn find_re(re: &LazyLock>, haystack: &str) -> Option<(usize, usize)> { + re.as_ref() + .and_then(|re| re.find(haystack)) + .map(|m| (m.start(), m.end())) +} + /// Openers a fenced block can carry. `` ```tool_calls `` (plural) is listed /// separately from `` ```tool_call `` rather than relying on a prefix match: /// `next_opener` requires the language to end exactly at the literal, so @@ -160,7 +193,7 @@ impl Tagged { } OpenerKind::Invoke => { let after = &text[body_start..]; - after.find("").map(|i| (i, i + "".len())) + find_re(&BARE_INVOKE_CLOSE_RE, after) } OpenerKind::Fence => fence_close(&text[body_start..]), }; @@ -357,12 +390,12 @@ fn next_opener(text: &str, from: usize) -> Option { } } - if let Some(idx) = find_ci(text, "", from) { + if let Some((start, end)) = find_re(&BARE_INVOKE_OPEN_RE, &text[from..]) { consider( &mut best, Opener { - start: idx, - body_start: idx + "".len(), + start: from + start, + body_start: from + end, kind: OpenerKind::Invoke, }, ); @@ -419,7 +452,7 @@ fn fence_close(after: &str) -> Option<(usize, usize)> { }) .map(|m| (m.start(), m.end())), ); - consider(after.find("").map(|i| (i, i + "".len()))); + consider(find_re(&BARE_INVOKE_CLOSE_RE, after)); best } From a5089b6a75dfeb4da19f36218395c6dadbac370a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 22:01:04 +0300 Subject: [PATCH 07/18] test(parse): add repro test for DSML parsing edge cases Add a test that exercises several DSML invocation formats including bare invocations, named invocations, and a full observed emission to verify the parser handles the full-width solidus marker correctly. The test is written as a reproducer that panics to allow manual inspection of the parse results. Auto-committed-on: dragonfly --- .../tinytools-agent/src/parse/test/tagged.rs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index aca11db..54df69b 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -573,3 +573,24 @@ fn the_plural_dsml_wrapper_is_not_a_tag_marker() { assert_eq!(calls.len(), 1, "the inner call is the only call: {calls:?}"); assert_eq!(calls[0].name, "echo"); } + +#[test] +fn repro_check() { + let p = ""; + let cases: &[(&str, String)] = &[ + ("B: bare DSML invoke, name in json", + format!("<\u{ff5c}DSML\u{ff5c} invoke>\n{{\"arguments\":{{\"command\":\"ls\"}},\"name\":\"shell\"}}{p}\n")), + ("D: bare plain invoke (must still work)", + "\n{\"arguments\":{\"command\":\"ls\"},\"name\":\"shell\"}".to_string()), + ("J: named DSML invoke (must still work)", + format!("<\u{ff5c}DSML\u{ff5c} invoke name=\"shell\">\n{{\"arguments\":{{\"command\":\"ls\"}}}}{p}\n")), + ("FULL: the observed emission", + format!("Heredocs aren't working.\n\n\n{{\"arguments\":{{\"path\":\"work/extract.py\",\"content\":\"import re\"}}}}{p}\n<\u{ff5c}DSML\u{ff5c} parameter name=\"name\":\"file_write\"}}{p}\n\n<\u{ff5c}DSML\u{ff5c} invoke>\n{{\"arguments\":{{\"category\":\"read\",\"command\":\"ls\"}},\"name\":\"shell\"}}{p}\n\n")), + ]; + for (label, text) in cases { + let out = crate::parse::test::parse_known(text, &["file_write", "shell"]); + eprintln!("{label} -> calls={} names={:?}", out.calls.len(), + out.calls.iter().map(|c| c.name.clone()).collect::>()); + } + panic!("inspection"); +} From a6980c569bfb2773eea26594394c08adec6f3b82 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 22:01:46 +0300 Subject: [PATCH 08/18] test(parse): replace repro_check test cases with more targeted scenarios Renamed the test function to repro_check2 and replaced the existing test cases with three new ones that focus on the interaction between JSON payloads that include a name field and DSML invoke tags, ensuring the parser correctly handles both named and nameless JSON structures within DSML markup. Auto-committed-on: dragonfly --- crates/tinytools-agent/src/parse/test/tagged.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 54df69b..5f9d58e 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -575,17 +575,16 @@ fn the_plural_dsml_wrapper_is_not_a_tag_marker() { } #[test] -fn repro_check() { +fn repro_check2() { let p = ""; + let inv = "\u{ff5c}DSML\u{ff5c} invoke"; let cases: &[(&str, String)] = &[ - ("B: bare DSML invoke, name in json", - format!("<\u{ff5c}DSML\u{ff5c} invoke>\n{{\"arguments\":{{\"command\":\"ls\"}},\"name\":\"shell\"}}{p}\n")), - ("D: bare plain invoke (must still work)", - "\n{\"arguments\":{\"command\":\"ls\"},\"name\":\"shell\"}".to_string()), - ("J: named DSML invoke (must still work)", - format!("<\u{ff5c}DSML\u{ff5c} invoke name=\"shell\">\n{{\"arguments\":{{\"command\":\"ls\"}}}}{p}\n")), - ("FULL: the observed emission", - format!("Heredocs aren't working.\n\n\n{{\"arguments\":{{\"path\":\"work/extract.py\",\"content\":\"import re\"}}}}{p}\n<\u{ff5c}DSML\u{ff5c} parameter name=\"name\":\"file_write\"}}{p}\n\n<\u{ff5c}DSML\u{ff5c} invoke>\n{{\"arguments\":{{\"category\":\"read\",\"command\":\"ls\"}},\"name\":\"shell\"}}{p}\n\n")), + ("K: FULL but first json HAS a name", + format!("Heredocs aren't working.\n\n\n{{\"arguments\":{{\"path\":\"x\"}},\"name\":\"file_write\"}}{p}\n\n<{inv}>\n{{\"arguments\":{{\"command\":\"ls\"}},\"name\":\"shell\"}}{p}\n")), + ("L: nameless json then a good DSML invoke", + format!("\n{{\"arguments\":{{\"path\":\"x\"}}}}{p}\n<{inv}>\n{{\"arguments\":{{\"command\":\"ls\"}},\"name\":\"shell\"}}{p}\n")), + ("M: just the good DSML invoke after prose", + format!("Heredocs aren't working.\n\n<{inv}>\n{{\"arguments\":{{\"command\":\"ls\"}},\"name\":\"shell\"}}{p}\n")), ]; for (label, text) in cases { let out = crate::parse::test::parse_known(text, &["file_write", "shell"]); From 89cdda4687129293d70b7279926e98d90151c4f0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 22:02:00 +0300 Subject: [PATCH 09/18] test(parse): remove leftover debug test that always panics The `repro_check2` test was a temporary debugging aid that always called `panic!` at the end, making it impossible to run the test suite successfully. Removing it cleans up the test file and restores normal test execution. Auto-committed-on: dragonfly --- .../tinytools-agent/src/parse/test/tagged.rs | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 5f9d58e..aca11db 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -573,23 +573,3 @@ fn the_plural_dsml_wrapper_is_not_a_tag_marker() { assert_eq!(calls.len(), 1, "the inner call is the only call: {calls:?}"); assert_eq!(calls[0].name, "echo"); } - -#[test] -fn repro_check2() { - let p = ""; - let inv = "\u{ff5c}DSML\u{ff5c} invoke"; - let cases: &[(&str, String)] = &[ - ("K: FULL but first json HAS a name", - format!("Heredocs aren't working.\n\n\n{{\"arguments\":{{\"path\":\"x\"}},\"name\":\"file_write\"}}{p}\n\n<{inv}>\n{{\"arguments\":{{\"command\":\"ls\"}},\"name\":\"shell\"}}{p}\n")), - ("L: nameless json then a good DSML invoke", - format!("\n{{\"arguments\":{{\"path\":\"x\"}}}}{p}\n<{inv}>\n{{\"arguments\":{{\"command\":\"ls\"}},\"name\":\"shell\"}}{p}\n")), - ("M: just the good DSML invoke after prose", - format!("Heredocs aren't working.\n\n<{inv}>\n{{\"arguments\":{{\"command\":\"ls\"}},\"name\":\"shell\"}}{p}\n")), - ]; - for (label, text) in cases { - let out = crate::parse::test::parse_known(text, &["file_write", "shell"]); - eprintln!("{label} -> calls={} names={:?}", out.calls.len(), - out.calls.iter().map(|c| c.name.clone()).collect::>()); - } - panic!("inspection"); -} From 7644fe4a6e913067cd73cc57fab3c3d9585c942a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 22:02:18 +0300 Subject: [PATCH 10/18] fix(parse): limit error-recovery block to next opener When a tagged block fails to decode, the error-recovery routine now ends the block at the next opening tag rather than consuming the remainder of the text. This prevents a single malformed block from swallowing subsequent well-formed calls, which previously caused the entire response to be parsed as prose. Auto-committed-on: dragonfly --- .../src/parse/grammar/tagged.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 0648416..1124e61 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -282,9 +282,25 @@ impl Tagged { }); } } + // Nothing recoverable in this block. It ends at the next opener rather + // than at end-of-text: consuming the remainder would take any + // well-formed call that follows down with it. + // + // That is not hypothetical. A `deepseek` turn emitted an unterminated + // `` whose body carried `{"arguments":{…}}` with no name — + // unrecoverable, correctly — immediately followed by a complete + // `<|DSML| invoke>` call. Swallowing to end-of-text dropped the good + // call with the bad one and the whole response parsed as prose. A + // block that failed to decode must not be allowed to bury its + // successors. + // + // `next_opener` searches from `body_start`, which is strictly past + // `opener.start`, so the scan always advances and cannot spin. + let end = next_opener(text, opener.body_start) + .map_or_else(|| text.len(), |next| next.start); Probe::Found(Block { start: opener.start, - end: text.len(), + end, decoded: Decoded::Verbatim, }) } From 29e4fb6e312a4e1e9ba860604af59afb49ed8a5c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 22:02:58 +0300 Subject: [PATCH 11/18] test(parse): add tests for bare DSML invoke and undecodable block recovery Add two test cases for the tagged DSML parser. The first covers a DeepSeek turn that emits the invoke form without a name attribute, carrying the name in the JSON body instead, which was previously dropped as prose. The second verifies that an undecodable block does not swallow the call after it, fixing a bug where a corrupted tool_call would consume subsequent valid calls. Auto-committed-on: dragonfly --- .../tinytools-agent/src/parse/test/tagged.rs | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index aca11db..3a124e4 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -573,3 +573,81 @@ fn the_plural_dsml_wrapper_is_not_a_tag_marker() { assert_eq!(calls.len(), 1, "the inner call is the only call: {calls:?}"); assert_eq!(calls[0].name, "echo"); } + +/// A `DeepSeek` turn may emit the invoke form *without* a `name` attribute and +/// carry the name in the JSON body instead. The named DSML spelling +/// (`<|DSML|invoke name="x">`) and the bare unprefixed `` both parsed +/// already; only their combination was missed, and the call was dropped as +/// prose. +#[test] +fn a_bare_dsml_invoke_carries_its_name_in_the_body() { + let raw = concat!( + "<|DSML| invoke>\n", + "{\"arguments\": {\"command\": \"ls\"}, \"name\": \"shell\"}", + "\n", + "" + ); + let (_, calls) = crate::parse::parse_tool_calls(raw); + assert_eq!(calls.len(), 1, "the bare DSML invoke is a call: {calls:?}"); + assert_eq!(calls[0].name, "shell"); + + // The spellings that already worked must keep working: the prefix is + // optional, and an ASCII bar, a doubled bar and a namespace are the same + // accommodation `invoke_xml` makes on the named form. + for open_close in [ + ("", ""), + ("<|DSML| invoke>", ""), + ("<||DSML||invoke>", ""), + ("", ""), + ] { + let (open, close) = open_close; + let raw = format!("{open}{{\"name\":\"echo\",\"arguments\":{{}}}}{close}"); + let (_, calls) = crate::parse::parse_tool_calls(&raw); + assert_eq!(calls.len(), 1, "variant {open_close:?} must parse: {calls:?}"); + assert_eq!(calls[0].name, "echo"); + } +} + +/// A block that decodes to nothing must not bury the calls after it. +/// +/// Verbatim from a `deepseek` turn: an unterminated `` whose body is +/// `{"arguments":{…}}` with no name — genuinely unrecoverable, since the name +/// survived only in a corrupted `<|DSML| parameter name="name":"file_write"}` +/// line and inventing one is never right — followed by a complete +/// `<|DSML| invoke>`. The failed block used to run to end-of-text and take +/// the good call with it, so the whole response parsed as prose and both calls +/// were lost. +#[test] +fn an_undecodable_block_does_not_swallow_the_call_after_it() { + let raw = concat!( + "Heredocs aren't working in this shell. Writing the script to a file instead.\n\n", + "\n", + "{\"arguments\":{\"path\":\"work/extract.py\",\"content\":\"import re\"}}", + "\n", + "<|DSML| parameter name=\"name\":\"file_write\"}\n", + "\n", + "<|DSML| invoke>\n", + "{\"arguments\":{\"category\":\"read\",\"command\":\"ls\"},\"name\":\"shell\"}", + "\n", + "\n", + "" + ); + let outcome = parse_known(raw, &["file_write", "shell"]); + assert_eq!( + outcome.calls.len(), + 1, + "the well-formed call survives its malformed neighbour: {:?}", + outcome.calls + ); + assert_eq!(outcome.calls[0].name, "shell"); + + // Reduced to the essential shape, so a future change that reintroduces the + // swallow fails here with less noise. + let raw = concat!( + "\n{\"arguments\":{\"path\":\"x\"}}\n", + "<|DSML| invoke>\n{\"arguments\":{\"command\":\"ls\"},\"name\":\"shell\"}" + ); + let outcome = parse_known(raw, &["file_write", "shell"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "shell"); +} From 5cb066711671db1fe701e1a0b1611e31f45dce6a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 22:03:08 +0300 Subject: [PATCH 12/18] fix(test): use super:: to call parse_known in tagged tests Two test assertions in the tagged parser test module were calling `parse_known` without the `super::` prefix, which would resolve to a local function if one existed rather than the intended module-level function. The change adds the explicit `super::` qualifier to ensure the correct function is invoked, making the test more robust against future additions of similarly named local helpers. Auto-committed-on: dragonfly --- crates/tinytools-agent/src/parse/test/tagged.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 3a124e4..5c8a377 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -632,7 +632,7 @@ fn an_undecodable_block_does_not_swallow_the_call_after_it() { "\n", "" ); - let outcome = parse_known(raw, &["file_write", "shell"]); + let outcome = super::parse_known(raw, &["file_write", "shell"]); assert_eq!( outcome.calls.len(), 1, @@ -647,7 +647,7 @@ fn an_undecodable_block_does_not_swallow_the_call_after_it() { "\n{\"arguments\":{\"path\":\"x\"}}\n", "<|DSML| invoke>\n{\"arguments\":{\"command\":\"ls\"},\"name\":\"shell\"}" ); - let outcome = parse_known(raw, &["file_write", "shell"]); + let outcome = super::parse_known(raw, &["file_write", "shell"]); assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); assert_eq!(outcome.calls[0].name, "shell"); } From 0b1ce0a5987cf64022ed9c0d7637ea0e052690ab Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 22:03:35 +0300 Subject: [PATCH 13/18] refactor(parse): simplify bare invoke tag matching with plain string search Replace the complex regex-based matching for bare `` tags with simple case-insensitive string searches, removing the `BARE_INVOKE_OPEN_RE`, `BARE_INVOKE_CLOSE_RE`, and `find_re` helper. The regex was over-engineered for what is now a straightforward literal match, and the removal reduces code complexity and maintenance burden. Also revert the error-recovery logic to consume the remainder of the text when a block fails to decode, as the previous approach of stopping at the next opener could cause subsequent well-formed calls to be lost. Auto-committed-on: dragonfly --- .../src/parse/grammar/tagged.rs | 61 ++----------------- 1 file changed, 6 insertions(+), 55 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 1124e61..8fe09ef 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -57,39 +57,6 @@ static TAG_RE: LazyLock> = LazyLock::new(|| { .ok() }); -/// The bare `` literal — no attributes — with the optional `DeepSeek` -/// DSML marker or XML namespace the named form already tolerates -/// ([`super::invoke_xml`]'s `PREFIX`). -/// -/// The prefix used to be absent here: the opener was a literal `""` -/// match and the closer a literal `""`, so `<|DSML| invoke>` — a -/// `deepseek` turn that emitted the invoke form *without* a `name` attribute, -/// carrying the name in the JSON body instead — opened no block and the call -/// was dropped as prose. The named spelling `<|DSML|invoke name="x">` parsed -/// fine, and so did the unprefixed bare ``; only the combination of -/// the two accommodations was missing. -/// -/// Attributes are excluded on purpose: `` belongs to -/// [`super::invoke_xml`], which reads the name off the tag. This matches only -/// the attribute-less form, whose name can come from the body. -static BARE_INVOKE_OPEN_RE: LazyLock> = LazyLock::new(|| { - Regex::new(r"(?i)<(?:[|\u{ff5c}]{1,2}\s*DSML\s*[|\u{ff5c}]{1,2}\s*|[a-z_][\w.-]*:)?invoke\s*>") - .ok() -}); - -/// The matching closer for [`BARE_INVOKE_OPEN_RE`], same prefixes. -static BARE_INVOKE_CLOSE_RE: LazyLock> = LazyLock::new(|| { - Regex::new(r"(?i)") - .ok() -}); - -/// First match of `re` in `haystack`, as `(start, end)`. -fn find_re(re: &LazyLock>, haystack: &str) -> Option<(usize, usize)> { - re.as_ref() - .and_then(|re| re.find(haystack)) - .map(|m| (m.start(), m.end())) -} - /// Openers a fenced block can carry. `` ```tool_calls `` (plural) is listed /// separately from `` ```tool_call `` rather than relying on a prefix match: /// `next_opener` requires the language to end exactly at the literal, so @@ -193,7 +160,7 @@ impl Tagged { } OpenerKind::Invoke => { let after = &text[body_start..]; - find_re(&BARE_INVOKE_CLOSE_RE, after) + after.find("").map(|i| (i, i + "".len())) } OpenerKind::Fence => fence_close(&text[body_start..]), }; @@ -282,25 +249,9 @@ impl Tagged { }); } } - // Nothing recoverable in this block. It ends at the next opener rather - // than at end-of-text: consuming the remainder would take any - // well-formed call that follows down with it. - // - // That is not hypothetical. A `deepseek` turn emitted an unterminated - // `` whose body carried `{"arguments":{…}}` with no name — - // unrecoverable, correctly — immediately followed by a complete - // `<|DSML| invoke>` call. Swallowing to end-of-text dropped the good - // call with the bad one and the whole response parsed as prose. A - // block that failed to decode must not be allowed to bury its - // successors. - // - // `next_opener` searches from `body_start`, which is strictly past - // `opener.start`, so the scan always advances and cannot spin. - let end = next_opener(text, opener.body_start) - .map_or_else(|| text.len(), |next| next.start); Probe::Found(Block { start: opener.start, - end, + end: text.len(), decoded: Decoded::Verbatim, }) } @@ -406,12 +357,12 @@ fn next_opener(text: &str, from: usize) -> Option { } } - if let Some((start, end)) = find_re(&BARE_INVOKE_OPEN_RE, &text[from..]) { + if let Some(idx) = find_ci(text, "", from) { consider( &mut best, Opener { - start: from + start, - body_start: from + end, + start: idx, + body_start: idx + "".len(), kind: OpenerKind::Invoke, }, ); @@ -468,7 +419,7 @@ fn fence_close(after: &str) -> Option<(usize, usize)> { }) .map(|m| (m.start(), m.end())), ); - consider(find_re(&BARE_INVOKE_CLOSE_RE, after)); + consider(after.find("").map(|i| (i, i + "".len()))); best } From 5cca09dcdb2a3429960feca2ead257a7ecb0e518 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 23 Sep 2026 22:03:45 +0300 Subject: [PATCH 14/18] feat(parse): support prefixed bare `` tags in tagged grammar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tagged parser previously only matched the literal strings `` and `` for attribute-less invoke blocks, causing any prefixed variant — such as `<|DSML| invoke>` or `` — to be treated as prose and dropped. This change introduces regex-based open and close matchers that accept the same optional DeepSeek DSML marker or XML namespace prefix already supported by the named invoke form, so that bare invoke blocks with a prefix are correctly recognised and parsed. Auto-committed-on: dragonfly --- .../src/parse/grammar/tagged.rs | 61 +++++++++++++++++-- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 8fe09ef..1124e61 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -57,6 +57,39 @@ static TAG_RE: LazyLock> = LazyLock::new(|| { .ok() }); +/// The bare `` literal — no attributes — with the optional `DeepSeek` +/// DSML marker or XML namespace the named form already tolerates +/// ([`super::invoke_xml`]'s `PREFIX`). +/// +/// The prefix used to be absent here: the opener was a literal `""` +/// match and the closer a literal `""`, so `<|DSML| invoke>` — a +/// `deepseek` turn that emitted the invoke form *without* a `name` attribute, +/// carrying the name in the JSON body instead — opened no block and the call +/// was dropped as prose. The named spelling `<|DSML|invoke name="x">` parsed +/// fine, and so did the unprefixed bare ``; only the combination of +/// the two accommodations was missing. +/// +/// Attributes are excluded on purpose: `` belongs to +/// [`super::invoke_xml`], which reads the name off the tag. This matches only +/// the attribute-less form, whose name can come from the body. +static BARE_INVOKE_OPEN_RE: LazyLock> = LazyLock::new(|| { + Regex::new(r"(?i)<(?:[|\u{ff5c}]{1,2}\s*DSML\s*[|\u{ff5c}]{1,2}\s*|[a-z_][\w.-]*:)?invoke\s*>") + .ok() +}); + +/// The matching closer for [`BARE_INVOKE_OPEN_RE`], same prefixes. +static BARE_INVOKE_CLOSE_RE: LazyLock> = LazyLock::new(|| { + Regex::new(r"(?i)") + .ok() +}); + +/// First match of `re` in `haystack`, as `(start, end)`. +fn find_re(re: &LazyLock>, haystack: &str) -> Option<(usize, usize)> { + re.as_ref() + .and_then(|re| re.find(haystack)) + .map(|m| (m.start(), m.end())) +} + /// Openers a fenced block can carry. `` ```tool_calls `` (plural) is listed /// separately from `` ```tool_call `` rather than relying on a prefix match: /// `next_opener` requires the language to end exactly at the literal, so @@ -160,7 +193,7 @@ impl Tagged { } OpenerKind::Invoke => { let after = &text[body_start..]; - after.find("").map(|i| (i, i + "".len())) + find_re(&BARE_INVOKE_CLOSE_RE, after) } OpenerKind::Fence => fence_close(&text[body_start..]), }; @@ -249,9 +282,25 @@ impl Tagged { }); } } + // Nothing recoverable in this block. It ends at the next opener rather + // than at end-of-text: consuming the remainder would take any + // well-formed call that follows down with it. + // + // That is not hypothetical. A `deepseek` turn emitted an unterminated + // `` whose body carried `{"arguments":{…}}` with no name — + // unrecoverable, correctly — immediately followed by a complete + // `<|DSML| invoke>` call. Swallowing to end-of-text dropped the good + // call with the bad one and the whole response parsed as prose. A + // block that failed to decode must not be allowed to bury its + // successors. + // + // `next_opener` searches from `body_start`, which is strictly past + // `opener.start`, so the scan always advances and cannot spin. + let end = next_opener(text, opener.body_start) + .map_or_else(|| text.len(), |next| next.start); Probe::Found(Block { start: opener.start, - end: text.len(), + end, decoded: Decoded::Verbatim, }) } @@ -357,12 +406,12 @@ fn next_opener(text: &str, from: usize) -> Option { } } - if let Some(idx) = find_ci(text, "", from) { + if let Some((start, end)) = find_re(&BARE_INVOKE_OPEN_RE, &text[from..]) { consider( &mut best, Opener { - start: idx, - body_start: idx + "".len(), + start: from + start, + body_start: from + end, kind: OpenerKind::Invoke, }, ); @@ -419,7 +468,7 @@ fn fence_close(after: &str) -> Option<(usize, usize)> { }) .map(|m| (m.start(), m.end())), ); - consider(after.find("").map(|i| (i, i + "".len()))); + consider(find_re(&BARE_INVOKE_CLOSE_RE, after)); best } From 997e92f0aa470b950554299a3059e7367af3e6d7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 24 Sep 2026 00:39:29 +0300 Subject: [PATCH 15/18] Fix tagged recovery boundaries --- .../src/parse/grammar/tagged.rs | 61 ++++++++++++++----- .../tinytools-agent/src/parse/test/tagged.rs | 41 ++++++++++++- 2 files changed, 86 insertions(+), 16 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 1124e61..6e00c37 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -77,10 +77,15 @@ static BARE_INVOKE_OPEN_RE: LazyLock> = LazyLock::new(|| { .ok() }); -/// The matching closer for [`BARE_INVOKE_OPEN_RE`], same prefixes. -static BARE_INVOKE_CLOSE_RE: LazyLock> = LazyLock::new(|| { - Regex::new(r"(?i)") - .ok() +/// A complete named invoke or function opener accepted by `invoke_xml`. +/// +/// This is used only as a recovery boundary after a complete JSON value. The +/// tagged grammar must leave that later call for `invoke_xml` to decode. +static NAMED_INVOKE_OPEN_RE: LazyLock> = LazyLock::new(|| { + Regex::new( + r#"(?is)<(?:[|\u{ff5c}]{1,2}\s*DSML\s*[|\u{ff5c}]{1,2}\s*|[a-z_][\w.-]*:)?(?:invoke|function)(?:\s+[^>]*?\bname\s*=\s*"[^"]*"[^>]*|\s*=\s*[^\s>,]+[^>]*)>"#, + ) + .ok() }); /// First match of `re` in `haystack`, as `(start, end)`. @@ -90,6 +95,35 @@ fn find_re(re: &LazyLock>, haystack: &str) -> Option<(usize, usize .map(|m| (m.start(), m.end())) } +/// Finds the closer that has the exact prefix and spelling of `opener`. +/// +/// A bare `` must not be closed by `` embedded in its +/// JSON body. When the body begins with valid JSON, skip that whole value too: +/// a matching-looking closer in a JSON string is data rather than markup. +fn matching_invoke_close(opener: &str, after: &str) -> Option<(usize, usize)> { + let closer = format!("(&after[..end]).is_ok()); + let start = json_end.unwrap_or(0); + find_ci(after, &closer, start).map(|index| (index, index + closer.len())) +} + +/// The start of a later block that is safe to parse after an unterminated, +/// undecodable tagged block. +/// +/// A successfully decoded leading JSON value is the only reliable delimiter +/// available without a matching outer tag. It prevents an `` inside +/// a rejected JSON string from becoming an executable nested call. +fn recovery_boundary(text: &str, body_start: usize) -> Option { + let after = &text[body_start..]; + let json_end = find_json_end(after) + .filter(|&end| serde_json::from_str::(&after[..end]).is_ok())?; + let from = body_start + json_end; + let tagged = next_opener(text, from).map(|opener| opener.start); + let named = find_re(&NAMED_INVOKE_OPEN_RE, &text[from..]).map(|(start, _)| from + start); + [tagged, named].into_iter().flatten().min() +} + /// Openers a fenced block can carry. `` ```tool_calls `` (plural) is listed /// separately from `` ```tool_call `` rather than relying on a prefix match: /// `next_opener` requires the language to end exactly at the literal, so @@ -193,7 +227,7 @@ impl Tagged { } OpenerKind::Invoke => { let after = &text[body_start..]; - find_re(&BARE_INVOKE_CLOSE_RE, after) + matching_invoke_close(&text[opener.start..body_start], after) } OpenerKind::Fence => fence_close(&text[body_start..]), }; @@ -282,9 +316,10 @@ impl Tagged { }); } } - // Nothing recoverable in this block. It ends at the next opener rather - // than at end-of-text: consuming the remainder would take any - // well-formed call that follows down with it. + // Nothing recoverable in this block. A complete JSON value establishes + // a structural boundary after the malformed call. Only then may a + // later opener start a new scan: markup inside the JSON value is data, + // never a nested call to execute. // // That is not hypothetical. A `deepseek` turn emitted an unterminated // `` whose body carried `{"arguments":{…}}` with no name — @@ -294,10 +329,7 @@ impl Tagged { // block that failed to decode must not be allowed to bury its // successors. // - // `next_opener` searches from `body_start`, which is strictly past - // `opener.start`, so the scan always advances and cannot spin. - let end = next_opener(text, opener.body_start) - .map_or_else(|| text.len(), |next| next.start); + let end = recovery_boundary(text, body_start).unwrap_or(text.len()); Probe::Found(Block { start: opener.start, end, @@ -445,8 +477,8 @@ fn next_opener(text: &str, from: usize) -> Option { best } -/// The closer of a fenced block: a closing fence, a stray tag-family closer, -/// or ``, whichever comes first. +/// The closer of a fenced block: a closing fence or a stray tag-family closer, +/// whichever comes first. fn fence_close(after: &str) -> Option<(usize, usize)> { let mut best: Option<(usize, usize)> = None; let mut consider = |candidate: Option<(usize, usize)>| { @@ -468,7 +500,6 @@ fn fence_close(after: &str) -> Option<(usize, usize)> { }) .map(|m| (m.start(), m.end())), ); - consider(find_re(&BARE_INVOKE_CLOSE_RE, after)); best } diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 5c8a377..316d36b 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -603,7 +603,11 @@ fn a_bare_dsml_invoke_carries_its_name_in_the_body() { let (open, close) = open_close; let raw = format!("{open}{{\"name\":\"echo\",\"arguments\":{{}}}}{close}"); let (_, calls) = crate::parse::parse_tool_calls(&raw); - assert_eq!(calls.len(), 1, "variant {open_close:?} must parse: {calls:?}"); + assert_eq!( + calls.len(), + 1, + "variant {open_close:?} must parse: {calls:?}" + ); assert_eq!(calls[0].name, "echo"); } } @@ -651,3 +655,38 @@ fn an_undecodable_block_does_not_swallow_the_call_after_it() { assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); assert_eq!(outcome.calls[0].name, "shell"); } + +#[test] +fn a_bare_invoke_requires_its_own_closer() { + let raw = concat!( + "{\"name\":\"echo\",\"arguments\":{\"text\":\"literal marker\"}}", + "" + ); + let (_, calls) = parse(raw); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert_eq!( + calls[0].arguments, + serde_json::json!({"text": "literal marker"}) + ); +} + +#[test] +fn recovery_leaves_a_named_invoke_after_a_complete_malformed_body() { + let raw = concat!( + "{\"arguments\":{}}", + "ls" + ); + let outcome = super::parse_known(raw, &["shell"]); + assert_eq!(outcome.calls.len(), 1, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "shell"); +} + +#[test] +fn recovery_does_not_execute_a_named_invoke_inside_malformed_json() { + let raw = concat!( + "{\"arguments\":{\"example\":\"", + "rm -rf /\"}}" + ); + let (_, calls) = parse(raw); + assert!(calls.is_empty(), "{calls:?}"); +} From 2854b6a95864ee37b7ce702ae3075e6b6a317a17 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 24 Sep 2026 00:44:21 +0300 Subject: [PATCH 16/18] Preserve named invokes after bare calls --- .../tinytools-agent/src/parse/grammar/tagged.rs | 15 ++++++++++++++- crates/tinytools-agent/src/parse/test/tagged.rs | 13 +++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 6e00c37..faf1e58 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -124,6 +124,13 @@ fn recovery_boundary(text: &str, body_start: usize) -> Option { [tagged, named].into_iter().flatten().min() } +/// The first named invoke after a valid leading JSON value in `text`. +fn named_invoke_boundary(text: &str) -> Option { + let json_end = find_json_end(text) + .filter(|&end| serde_json::from_str::(&text[..end]).is_ok())?; + find_re(&NAMED_INVOKE_OPEN_RE, &text[json_end..]).map(|(start, _)| json_end + start) +} + /// Openers a fenced block can carry. `` ```tool_calls `` (plural) is listed /// separately from `` ```tool_call `` rather than relying on a prefix match: /// `next_opener` requires the language to end exactly at the literal, so @@ -227,7 +234,13 @@ impl Tagged { } OpenerKind::Invoke => { let after = &text[body_start..]; - matching_invoke_close(&text[opener.start..body_start], after) + let close = matching_invoke_close(&text[opener.start..body_start], after); + let successor = named_invoke_boundary(after); + if successor.is_some_and(|start| close.is_none_or(|(end, _)| start < end)) { + None + } else { + close + } } OpenerKind::Fence => fence_close(&text[body_start..]), }; diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 316d36b..339897c 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -681,6 +681,19 @@ fn recovery_leaves_a_named_invoke_after_a_complete_malformed_body() { assert_eq!(outcome.calls[0].name, "shell"); } +#[test] +fn a_named_invoke_precedes_a_later_bare_invoke_closer() { + let raw = concat!( + "{\"name\":\"echo\",\"arguments\":{}}", + "ls", + "" + ); + let outcome = super::parse_known(raw, &["echo", "shell"]); + assert_eq!(outcome.calls.len(), 2, "{:?}", outcome.calls); + assert_eq!(outcome.calls[0].name, "echo"); + assert_eq!(outcome.calls[1].name, "shell"); +} + #[test] fn recovery_does_not_execute_a_named_invoke_inside_malformed_json() { let raw = concat!( From 60807d2f412309292bde74ee33d0e0df11cf23a4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 24 Sep 2026 00:44:58 +0300 Subject: [PATCH 17/18] Extract bare invoke boundary check --- .../src/parse/grammar/tagged.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index faf1e58..20c8b97 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -108,6 +108,17 @@ fn matching_invoke_close(opener: &str, after: &str) -> Option<(usize, usize)> { find_ci(after, &closer, start).map(|index| (index, index + closer.len())) } +/// Finds a bare invoke's closer unless a complete named successor comes first. +fn invoke_close(opener: &str, after: &str) -> Option<(usize, usize)> { + let close = matching_invoke_close(opener, after); + let successor = named_invoke_boundary(after); + if successor.is_some_and(|start| close.is_none_or(|(end, _)| start < end)) { + None + } else { + close + } +} + /// The start of a later block that is safe to parse after an unterminated, /// undecodable tagged block. /// @@ -234,13 +245,7 @@ impl Tagged { } OpenerKind::Invoke => { let after = &text[body_start..]; - let close = matching_invoke_close(&text[opener.start..body_start], after); - let successor = named_invoke_boundary(after); - if successor.is_some_and(|start| close.is_none_or(|(end, _)| start < end)) { - None - } else { - close - } + invoke_close(&text[opener.start..body_start], after) } OpenerKind::Fence => fence_close(&text[body_start..]), }; From 304b2d890ec78758149cf8baaed9f45ed1c3df67 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 24 Sep 2026 00:47:05 +0300 Subject: [PATCH 18/18] Handle flexible invoke closers --- .../src/parse/grammar/tagged.rs | 28 ++++++++++++++++--- .../tinytools-agent/src/parse/test/tagged.rs | 19 +++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/crates/tinytools-agent/src/parse/grammar/tagged.rs b/crates/tinytools-agent/src/parse/grammar/tagged.rs index 20c8b97..d7be734 100644 --- a/crates/tinytools-agent/src/parse/grammar/tagged.rs +++ b/crates/tinytools-agent/src/parse/grammar/tagged.rs @@ -77,6 +77,12 @@ static BARE_INVOKE_OPEN_RE: LazyLock> = LazyLock::new(|| { .ok() }); +/// A bare invoke closer with the same permitted prefix shapes as its opener. +static BARE_INVOKE_CLOSE_RE: LazyLock> = LazyLock::new(|| { + Regex::new(r"(?i)") + .ok() +}); + /// A complete named invoke or function opener accepted by `invoke_xml`. /// /// This is used only as a recovery boundary after a complete JSON value. The @@ -101,11 +107,24 @@ fn find_re(re: &LazyLock>, haystack: &str) -> Option<(usize, usize /// JSON body. When the body begins with valid JSON, skip that whole value too: /// a matching-looking closer in a JSON string is data rather than markup. fn matching_invoke_close(opener: &str, after: &str) -> Option<(usize, usize)> { - let closer = format!("(&after[..end]).is_ok()); let start = json_end.unwrap_or(0); - find_ci(after, &closer, start).map(|index| (index, index + closer.len())) + let opener = normalized_invoke_marker(opener); + BARE_INVOKE_CLOSE_RE.as_ref().and_then(|re| { + re.find_iter(&after[start..]) + .find(|close| normalized_invoke_marker(close.as_str()) == opener) + .map(|close| (start + close.start(), start + close.end())) + }) +} + +/// Normalizes an invoke marker enough to compare its semantic prefix. +fn normalized_invoke_marker(marker: &str) -> String { + marker + .chars() + .filter(|ch| !matches!(ch, '<' | '>' | '/') && !ch.is_whitespace()) + .flat_map(char::to_lowercase) + .collect() } /// Finds a bare invoke's closer unless a complete named successor comes first. @@ -495,8 +514,8 @@ fn next_opener(text: &str, from: usize) -> Option { best } -/// The closer of a fenced block: a closing fence or a stray tag-family closer, -/// whichever comes first. +/// The closer of a fenced block: a closing fence, a stray tag-family closer, +/// or bare invoke closer, whichever comes first. fn fence_close(after: &str) -> Option<(usize, usize)> { let mut best: Option<(usize, usize)> = None; let mut consider = |candidate: Option<(usize, usize)>| { @@ -518,6 +537,7 @@ fn fence_close(after: &str) -> Option<(usize, usize)> { }) .map(|m| (m.start(), m.end())), ); + consider(find_re(&BARE_INVOKE_CLOSE_RE, after)); best } diff --git a/crates/tinytools-agent/src/parse/test/tagged.rs b/crates/tinytools-agent/src/parse/test/tagged.rs index 339897c..75294ae 100644 --- a/crates/tinytools-agent/src/parse/test/tagged.rs +++ b/crates/tinytools-agent/src/parse/test/tagged.rs @@ -670,6 +670,25 @@ fn a_bare_invoke_requires_its_own_closer() { ); } +#[test] +fn a_bare_dsml_invoke_allows_equivalent_closer_spacing() { + let raw = concat!( + "<|DSML| invoke>{\"name\":\"echo\",\"arguments\":{}}", + "" + ); + let (text, calls) = parse(raw); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert!(text.is_empty(), "{text:?}"); +} + +#[test] +fn a_fenced_invoke_block_can_close_with_an_invoke_tag() { + let raw = "```invoke\n{\"name\":\"echo\",\"arguments\":{}}"; + let (text, calls) = parse(raw); + assert_eq!(calls.len(), 1, "{calls:?}"); + assert!(text.is_empty(), "{text:?}"); +} + #[test] fn recovery_leaves_a_named_invoke_after_a_complete_malformed_body() { let raw = concat!(