fix(harness): keep tool-call markup out of summaries and tool-less answers

crates/tinyagents-harness/src/agent_loop/README.md

@@ -110,6 +110,30 @@ deterministic) and enabled per policy via
 `retry::RetryPolicy::with_backoff_sleep`. A real provider integration retries
 after a genuine, growing delay while unit tests stay sleep-free.
 
+## Calls written on a turn with no callable tool
+
+A request can withdraw its tools — a final wrap-up call, a spent research
+budget — or set `ToolChoice::None`, while its transcript is still full of tool
+calls. A native model then has no tool channel and writes its next call as
+plain text in its own markup (DeepSeek V4: `<｜DSML｜invoke name="shell">…`).
+Declaring the tools with `tool_choice: "none"` does not prevent it. Taken as
+the final answer, that markup ends the turn with a stray command where the
+result should be.
+
+On such a turn `TextRecovery::withholding` is in force: the streaming
+scrubber and, for unary replies, `withhold_text_calls` remove any complete
+call the tool-call grammars recognise from the visible text, keep reasoning,
+and count it in `DroppedBlocks::withheld`. Nothing is dispatched — the
+I-2 rule (prose never becomes a call when none could be accepted) holds. If a
+call was withheld and a model call remains, the loop drops that assistant row
+and re-prompts once with `WITHHELD_TOOL_CALL_NUDGE`, which says plainly that
+tools are unavailable, emitting `ControlApplied { control:
+"withheld_tool_call" }` and `RetryScheduled`. Re-prompts are bounded by
+`RunPolicy::dropped_tool_call_nudges` and run before the empty-reply retries,
+since a bare re-send of the same transcript leaks the same way. With no call
+left, the scrubbed reply stands. Language-tagged fenced examples and bare JSON
+answers are left untouched.
+
 ## Truncated-empty recovery
 
 Local reasoning models (for example `qwen3` via Ollama) intermittently spend

crates/tinyagents-harness/src/agent_loop/dialect.rs

@@ -344,6 +344,11 @@ pub(super) struct TextRecovery {
     /// Call blocks the model attempted that produced no call, on the
     /// attempt whose response the loop is about to read.
     pub(super) dropped: Arc<DroppedBlocks>,
+    /// No tool could be called this turn (none offered, or an effective
+    /// `ToolChoice::None`), yet call markup is still scrubbed from the
+    /// visible text and counted in [`DroppedBlocks::withheld`] instead of
+    /// becoming a call. See [`TextRecovery::withholding`].
+    pub(super) withhold: bool,
 }
 
 /// Tool-call blocks a grammar recognised on one model attempt that did not
@@ -357,6 +362,9 @@ pub(super) struct DroppedBlocks {
     /// Openers with no closer at end of text (`UnterminatedBlock`); the raw
     /// markup is released as visible text.
     unterminated: AtomicUsize,
+    /// Complete calls the model wrote on a turn that could not take one;
+    /// scrubbed from the visible text and never dispatched.
+    withheld: AtomicUsize,
 }
 
 impl DroppedBlocks {
@@ -370,11 +378,32 @@ impl DroppedBlocks {
         self.unterminated.load(Ordering::Relaxed)
     }
 
+    /// Calls written on a turn that offered no callable tool.
+    pub(super) fn withheld(&self) -> usize {
+        self.withheld.load(Ordering::Relaxed)
+    }
+
     /// Forgets the counts: the next response comes from a new attempt, or
     /// the attempt that produced them failed and its response is discarded.
     pub(super) fn reset(&self) {
         self.malformed.store(0, Ordering::Relaxed);
         self.unterminated.store(0, Ordering::Relaxed);
+        self.withheld.store(0, Ordering::Relaxed);
+    }
+
+    /// Counts `calls` complete calls withheld on a turn that could not take
+    /// them.
+    fn record_withheld(&self, calls: usize, model_call_id: &CallId) {
+        if calls == 0 {
+            return;
+        }
+        self.withheld.fetch_add(calls, Ordering::Relaxed);
+        tracing::warn!(
+            target: "tinyagents::agent_loop",
+            call_id = %model_call_id,
+            withheld = calls,
+            "[agent_loop] tool call written on a turn with no callable tools; scrubbed, not run"
+        );
     }
 
     /// Adds the dropped blocks among `diagnostics`, logging any it finds.
@@ -403,10 +432,28 @@ impl DroppedBlocks {
 }
 
 impl TextRecovery {
+    /// Recovery for a turn on which no tool can be called: nothing is ever
+    /// recovered as a call, but call markup is still kept out of the answer.
+    ///
+    /// A request that withdraws its tools (a final wrap-up call, a spent
+    /// research budget, a summarizer) still carries a transcript full of tool
+    /// calls, and a native model with no tool channel open writes its next
+    /// call in its own markup as plain text: DeepSeek V4's
+    /// `<｜DSML｜invoke name="shell">…`. In replayed bench captures that
+    /// happened on most such requests, and declaring the tools with
+    /// `tool_choice: "none"` did not stop it. Read as a final answer the
+    /// markup ended the turn with a stray command in place of a result.
+    pub(super) fn withholding() -> Self {
+        Self {
+            withhold: true,
+            ..Self::default()
+        }
+    }
+
     /// A scrubber for one streamed model call, or `None` when no tools were
-    /// offered and there is nothing to recover.
+    /// offered and no markup has to be withheld either.
     pub(super) fn scrubber(&self, model_call_id: &CallId) -> Option<DeltaScrubber> {
-        (!self.offered.is_empty()).then(|| {
+        (!self.offered.is_empty() || self.withhold).then(|| {
             // A fresh scrubber is one provider attempt; a retried attempt
             // must not inherit the previous one's count.
             self.dropped.reset();
@@ -416,10 +463,34 @@ impl TextRecovery {
                 self.registry.clone(),
                 Arc::clone(&self.dropped),
             )
+            .withholding(self.withhold)
         })
     }
 }
 
+/// Scrubs tool-call markup out of a completed response on a turn that could
+/// not take a call, counting what it removed in `dropped` and dispatching
+/// nothing. Reasoning and other non-text blocks survive.
+///
+/// The streamed path does the same incrementally through a withholding
+/// [`DeltaScrubber`]; this covers a unary reply, and a streamed one whose
+/// terminal text was not reconciled from the deltas.
+pub(super) fn withhold_text_calls(
+    response: &mut ModelResponse,
+    model_call_id: &CallId,
+    dropped: &DroppedBlocks,
+) {
+    let text = response.text();
+    // Bare JSON stays: an answer may legitimately be a JSON object.
+    let outcome = tinytools_agent::parse_text(&text, &ParseOptions::new().without_bare_json());
+    if outcome.calls.is_empty() {
+        return;
+    }
+    dropped.record_withheld(outcome.calls.len(), model_call_id);
+    response.message.content =
+        replace_text_blocks(std::mem::take(&mut response.message.content), outcome.text);
+}
+
 /// How one model call is made: streamed or unary, and what it needs to
 /// recover text-dialect calls from the answer.
 #[derive(Debug, Clone, Default)]
@@ -533,6 +604,11 @@ pub(super) struct DeltaScrubber {
     model_call_id: CallId,
     calls: Vec<ToolCall>,
     dropped: Arc<DroppedBlocks>,
+    /// Scrub complete calls without collecting them (a turn with no callable
+    /// tool); they are counted in [`DroppedBlocks::withheld`] instead.
+    withhold: bool,
+    /// Complete calls withheld on this attempt.
+    withheld: usize,
 }
 
 impl DeltaScrubber {
@@ -553,9 +629,17 @@ impl DeltaScrubber {
             model_call_id,
             calls: Vec::new(),
             dropped,
+            withhold: false,
+            withheld: 0,
         }
     }
 
+    /// Scrub calls without dispatching them (see [`TextRecovery::withholding`]).
+    pub(super) fn withholding(mut self, withhold: bool) -> Self {
+        self.withhold = withhold;
+        self
+    }
+
     /// Feeds one text delta; returns the text safe to forward.
     pub(super) fn feed(&mut self, text: &str) -> String {
         let step = self.inner.feed(text);
@@ -572,6 +656,12 @@ impl DeltaScrubber {
 
     fn collect(&mut self, calls: Vec<ParsedToolCall>, diagnostics: &[ParseDiagnostic]) {
         self.dropped.record(diagnostics, &self.model_call_id);
+        if self.withhold {
+            self.withheld += calls.len();
+            self.dropped
+                .record_withheld(calls.len(), &self.model_call_id);
+            return;
+        }
         for call in calls {
             let slot = self.calls.len() + 1;
             self.calls
@@ -588,4 +678,9 @@ impl DeltaScrubber {
     pub(super) fn has_calls(&self) -> bool {
         !self.calls.is_empty()
     }
+
+    /// Whether any complete call was withheld during the stream.
+    pub(super) fn has_withheld(&self) -> bool {
+        self.withheld > 0
+    }
 }

crates/tinyagents-harness/src/agent_loop/model_call.rs

@@ -1178,9 +1178,12 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
             // alongside the structured calls the scrubber recovered below,
             // and gets persisted into the transcript to be replayed back to
             // the model next turn.
+            // A withheld call (a turn with no callable tool) was scrubbed the
+            // same way, so its raw markup must not come back from the
+            // terminal text either.
             let scrubber_recovered_calls = text_scrubber
                 .as_ref()
-                .is_some_and(super::dialect::DeltaScrubber::has_calls);
+                .is_some_and(|scrubber| scrubber.has_calls() || scrubber.has_withheld());
             if let ModelStreamItem::Completed(response) = &mut item
                 && (saw_streamed_content || scrubber_recovered_calls)
             {

crates/tinyagents-harness/src/agent_loop/run_loop.rs

@@ -405,6 +405,9 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
         // Consecutive "you said tool_calls but sent none" re-prompts
         // (see `RunPolicy::dropped_tool_call_nudges`).
         let mut dropped_tool_call_nudges_used: u32 = 0;
+        // Consecutive re-prompts after a call written on a turn with no
+        // callable tool (bounded by the same `dropped_tool_call_nudges`).
+        let mut withheld_call_nudges_used: u32 = 0;
         let mut boosted_max_tokens: Option<u32> = None;
         let mut truncation_base: Option<u32> = None;
 
@@ -1005,9 +1008,13 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
                     offered: Arc::new(request.tools.clone()),
                     registry: dialect.registry_for(&request.tools),
                     dropped: Arc::default(),
+                    withhold: false,
                 }
             } else {
-                super::dialect::TextRecovery::default()
+                // Nothing can be recovered as a call, but a call the model
+                // writes anyway is kept out of the answer (see
+                // `TextRecovery::withholding`).
+                super::dialect::TextRecovery::withholding()
             };
             // Applied before budget preflight below: for a text dialect this
             // rewrite folds the protocol block and full tool catalogue into
@@ -1174,6 +1181,15 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
             if forced_text_dialect || text_dialect_recovery_enabled {
                 recover_text_dialect_calls(ctx, &mut response, &call_id, &recovery);
             }
+            // A turn that could not take a call: whatever call markup the
+            // model wrote is scrubbed from the answer and never run. Applied
+            // whatever `text_dialect_recovery` says — that policy decides
+            // whether prose can *become* a call, and here none can. A
+            // streamed reply was already scrubbed delta by delta; this
+            // catches a unary one.
+            if recovery.withhold {
+                super::dialect::withhold_text_calls(&mut response, &call_id, &recovery.dropped);
+            }
 
             // Account for the completed provider response before fallible
             // response middleware. A middleware rejection must not erase
@@ -1429,6 +1445,7 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
                 // otherwise receive fewer than the policy's configured number
                 // of consecutive re-prompts.
                 dropped_tool_call_nudges_used = 0;
+                withheld_call_nudges_used = 0;
                 empty_response_retries_used = 0;
                 reset_truncated_empty_recovery(
                     &mut truncated_empty_retries_used,
@@ -1470,6 +1487,48 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
             }
 
             if real_tool_calls.is_empty() {
+                // A call written on a turn that could not take one (tools
+                // withdrawn for a concluding answer, or `ToolChoice::None`).
+                // It was scrubbed and not run; what is left is either nothing
+                // or a lead-in to a step that never happened, so it is not the
+                // answer the request asked for. Drop that row and ask once
+                // more, telling the model plainly that tools are gone.
+                // Replaying the bench request that leaked (DeepSeek V4, tools
+                // withdrawn), the unchanged request leaked 6 times in 8; with
+                // the row dropped and this re-prompt added it leaked 0 times
+                // in 12. Runs before the empty-reply retries: a bare re-send
+                // of the same transcript leaks the same way.
+                let withheld_calls = recovery.dropped.withheld();
+                if withheld_calls > 0
+                    && withheld_call_nudges_used < self.policy.dropped_tool_call_nudges
+                    && ctx.limits.remaining_model_calls() > 0
+                {
+                    withheld_call_nudges_used += 1;
+                    messages.pop();
+                    tracing::info!(
+                        target: "tinyagents::agent_loop",
+                        run_id = %ctx.run_id(),
+                        call_id = %call_id,
+                        withheld_calls,
+                        attempt = withheld_call_nudges_used,
+                        "[agent_loop] re-prompting after a tool call on a turn with no callable tools"
+                    );
+                    ctx.emit(AgentEvent::ControlApplied {
+                        control: "withheld_tool_call".to_string(),
+                        detail: format!(
+                            "{withheld_calls} tool call(s) written while no tool was callable \
+                             in model call `{call_id}`; scrubbed, not run, re-prompted"
+                        ),
+                    });
+                    messages.push(Message::user(WITHHELD_TOOL_CALL_NUDGE));
+                    let record = ctx.emit(AgentEvent::RetryScheduled {
+                        call_id: call_id.clone(),
+                        attempt: withheld_call_nudges_used as usize,
+                    });
+                    status.set_last_event(record.id);
+                    continue;
+                }
+
                 // Truncated-empty recovery (runs before structured extraction,
                 // which would otherwise fail on the empty completion). A local
                 // reasoning model can burn the whole token budget on its hidden
@@ -1596,6 +1655,7 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
                     continue;
                 }
                 dropped_tool_call_nudges_used = 0;
+                withheld_call_nudges_used = 0;
                 empty_response_retries_used = 0;
 
                 // This turn resolved without scheduling a truncated-empty
@@ -1710,6 +1770,7 @@ impl<State: Send + Sync, Ctx: Send + Sync> AgentHarness<State, Ctx> {
             // recovery state before the tools run so the next turn starts from
             // the caller's configured cap and a full retry budget.
             dropped_tool_call_nudges_used = 0;
+            withheld_call_nudges_used = 0;
             empty_response_retries_used = 0;
             reset_truncated_empty_recovery(
                 &mut truncated_empty_retries_used,
@@ -2437,6 +2498,13 @@ const DROPPED_TOOL_CALL_NUDGE: &str = "Your previous turn indicated a tool call
      included. If you meant to call a tool, issue the actual tool call now; otherwise answer \
      directly.";
 
+/// The re-prompt sent when the model wrote a tool call on a turn that offered
+/// no callable tool. The call was scrubbed and not run; the wording names that
+/// plainly, because a model told only to "answer" keeps trying to act.
+const WITHHELD_TOOL_CALL_NUDGE: &str = "Your previous reply was a tool call, but tools are not \
+     available for this reply, so it did not run. Do not write tool calls. Answer now in plain \
+     text from the results already gathered, and state any remaining uncertainty.";
+
 /// The re-prompt sent when a text-dialect tool-call block could not be
 /// decoded: no tool ran, and the model should know why rather than assume
 /// its call went through.
@@ -2513,6 +2581,10 @@ fn reset_truncated_empty_recovery(
 #[path = "run_loop_recovery_tests.rs"]
 mod recovery_tests;
 
+#[cfg(test)]
+#[path = "run_loop_withheld_tests.rs"]
+mod withheld_tests;
+
 /// Attaches `default` to `request` when the request carries no reasoning
 /// config of its own. A request-level config always wins.
 pub(crate) fn apply_default_reasoning(

crates/tinyagents-harness/src/agent_loop/run_loop_recovery_tests.rs

@@ -17,6 +17,7 @@ fn offered(names: &[&str]) -> TextRecovery {
         ),
         registry: None,
         dropped: Arc::default(),
+        withhold: false,
     }
 }