Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,15 @@ use crate::agent::tinyagents::host::OpenHumanRunContext;
/// a batched `web_contents_tool` read, each of which counts once.
pub(super) const DIRECT_WEB_READ_LIMIT: usize = 8;

const RESEARCH_CLOSE_INSTRUCTION: &str = "The direct web research budget for this turn is exhausted. Answer the user's latest request now using the results already available. State any remaining uncertainty. Do not search again, repeat a page fetch, or merely describe what you plan to read.";
/// The instruction appended when the budget is spent and the tools are
/// withdrawn. It says outright that tools are gone and a call will not run:
/// told only to "answer", a native model with a transcript full of tool calls
/// and no tool channel writes its next call as plain-text markup (DeepSeek V4's
/// `<|DSML|invoke …>`), which then stood as the turn's answer. Replaying a
/// bench request that hit this budget, the old wording leaked a call 6 times in
/// 8 and this wording 0 times in 8. The harness also withholds and re-prompts
/// any call that still arrives (tinyagents `TextRecovery::withholding`).
pub(crate) const RESEARCH_CLOSE_INSTRUCTION: &str = "The direct web research budget for this turn is exhausted, and tools are no longer available for this reply: any tool call you write now will not run. Answer the user's latest request now in plain text, using only the results already available. State any remaining uncertainty. Do not search again, repeat a page fetch, or merely describe what you plan to read.";

#[derive(Default)]
pub(crate) struct ResearchBudgetMiddleware {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,32 @@ async fn web_research_concludes_after_eight_reads() {
assert!(request.messages.last().unwrap().text().contains("Answer"));
}

/// With the tools withdrawn, the instruction must say that a tool call will
/// not run: without it DeepSeek V4 wrote its next call as plain-text markup
/// that stood as the answer.
#[tokio::test]
async fn the_concluding_instruction_says_tools_are_gone() {
let mw = ResearchBudgetMiddleware::new();
let mut run = ctx();
for i in 0..research_budget::DIRECT_WEB_READ_LIMIT {
mw.after_tool(
&mut run,
&(),
&invocation(format!("web-{i}"), "web_fetch"),
&mut tool_result("web_fetch", "page"),
)
.await
.unwrap();
}
let mut request = research_request();
mw.before_model(&mut run, &(), &mut request).await.unwrap();
let instruction = request.messages.last().unwrap().text();
assert_eq!(instruction, research_budget::RESEARCH_CLOSE_INSTRUCTION);
assert!(instruction.contains("tools are no longer available"));
assert!(instruction.contains("will not run"));
assert!(instruction.contains("plain text"));
}

#[tokio::test]
async fn unrelated_tools_do_not_spend_the_web_research_budget() {
let mw = ResearchBudgetMiddleware::new();
Expand Down
Loading