diff --git a/CHANGELOG.md b/CHANGELOG.md index 13076a03..df353f90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +### Changed + +- **A wrong tool no longer outscores no tool.** `tools_succeeded` is awarded + only for a tool the scenario expected (or any tool when it expects none): + a tool that ran without erroring was evidence of the task only by accident, + and a scenario that called the wrong tool scored higher than one that called + nothing. (#433) +- **The judge reads more of a scenario's notes** — 1,500 characters rather + than 300 — because a suite's notes are often its rubric and the "must not" + clause tends to come last. (#433) ### Fixed - **The caller can no longer be named by the model, or by the client.** diff --git a/docs/framework/dashboard.md b/docs/framework/dashboard.md index 5ead61a0..b448c156 100644 --- a/docs/framework/dashboard.md +++ b/docs/framework/dashboard.md @@ -435,12 +435,17 @@ one fault, assigned from the evidence in this order: | `tool_error` | A tool the agent called returned an error | Fix the tool, or its parameter descriptions | | `missing_capability` | The agent said no tool covers the task | Add the tool the recommendation names | | `expected_tool_not_called` | The scenario expects a tool the agent did not call | Enable the tool, or sharpen its description / the instructions | +| `ungrounded_answer` | The agent had tools, called none, and still stated specifics — a count, an id, a date — nothing supplied | Instruct it to answer only from tool results; add the tool that returns this data | | `forbidden_content` / `missing_content` | A content expectation failed | Instructions, or the tool's output | | `low_quality` | Criteria scored the answer below 0.7 | Read the answer against the weakest criterion | -`missing_capability` and `expected_tool_not_called` are the faults that -turn a pasted list of new tasks into a backlog: they say which tasks the -current toolset cannot reach and what to build. +`missing_capability`, `expected_tool_not_called` and `ungrounded_answer` are +the faults that turn a pasted list of new tasks into a backlog: they say which +tasks the current toolset cannot reach and what to build. The last two also +tell an honest gap from an invented answer: `expected_tool_not_called` carries +`ungrounded: true` in its evidence when the answer stated specifics no tool +supplied, and `ungrounded_answer` is the same finding for a scenario that +names no expected tool. The parsing, scoring, diagnosis and report are the framework's [`ActiveAgent::Evals`](/framework/evaluations); the engine adds the diff --git a/docs/framework/evaluations.md b/docs/framework/evaluations.md index 396d15bb..24d1f37c 100644 --- a/docs/framework/evaluations.md +++ b/docs/framework/evaluations.md @@ -92,8 +92,8 @@ ActiveAgent::Evals::Runner.new(scenarios:, models:, replay:, judge: judge, instr With a judge, every answer also gets a `task_completion` score (unless the criteria already include an `llm_judge`), scenarios failing on -`missing_capability`, `expected_tool_not_called`, `missing_content` or -`low_quality` get a judge-written recommendation with a suggested tool where +`missing_capability`, `expected_tool_not_called`, `ungrounded_answer`, +`missing_content` or `low_quality` get a judge-written recommendation with a suggested tool where one is missing (`refine_faults:` and `judge_limit:`, 25 calls per run by default, adjust that on `Runner.new`), and the verdict carries the judge's rationale. A judge that raises or answers unusably is skipped for that call, @@ -129,6 +129,7 @@ can establish context itself. | `tool_error` | A tool the agent called returned an error | | `missing_capability` | The agent said no tool covers the task | | `expected_tool_not_called` | The scenario expects a tool the agent did not call | +| `ungrounded_answer` | The agent had tools, called none, and still stated specifics (a count, an id, a date) nothing supplied | | `forbidden_content` / `missing_content` | A content expectation failed | | `low_quality` | The answer scored below the threshold (0.7) | diff --git a/lib/active_agent/evals/diagnosis.rb b/lib/active_agent/evals/diagnosis.rb index 84ec4d38..4b30d9cb 100644 --- a/lib/active_agent/evals/diagnosis.rb +++ b/lib/active_agent/evals/diagnosis.rb @@ -12,6 +12,7 @@ module Evals # tool_error — a tool the agent called returned an error # missing_capability — the agent said no tool covers the task # expected_tool_not_called — the scenario expects a tool the agent did not call + # ungrounded_answer — the answer states specifics no tool call supplied # forbidden_content — the answer contains a pattern the scenario forbids # missing_content — the answer lacks a pattern the scenario expects # low_quality — the answer scored below the threshold @@ -21,7 +22,7 @@ module Evals # Returns nil for a passing result. class Diagnosis FAULTS = %w[ - run_error tool_error missing_capability expected_tool_not_called + run_error tool_error missing_capability expected_tool_not_called ungrounded_answer forbidden_content missing_content low_quality judge_unavailable ].freeze @@ -38,6 +39,19 @@ class Diagnosis /\bcan(?:'|no)t (?:be )?(?:done|determined|answered) with (?:the|my) (?:current|available) tools\b/i ].freeze + # Phrasings that state a specific fact — a record id, a date, a count of + # things — which an agent that called no tool can only have invented. + # Deliberately narrow: a number inside prose ("here are three options", + # "within 30 days") is not a claim about data, and a false positive here + # fails a scenario that may have passed on its merits. + SPECIFIC_CLAIMS = [ + /#\d+\b/, + /\b\d{4}-\d{2}-\d{2}\b/, + /\b(?:you have|there are|there is|we have|I found|found|showing|a total of)\s+(?:\*\*)?\d+\b/i, + /\b\d+\s+(?:\*\*)?(?:open|overdue|pending|active|closed|resolved|completed|unpaid|outstanding|new|matching| + records?|results?|rows?|entries|items?|tickets?|orders?|tasks?|issues?|invoices?|customers?|users?|milestones?)\b/ix + ].freeze + Result = Struct.new(:fault, :summary, :recommendation, :evidence, keyword_init: true) do def to_h { @@ -76,7 +90,7 @@ def initialize(scenario:, replay:, scores:, score:, available_tools:, threshold: end def call - run_error || tool_error || missing_capability || expected_tool_not_called || + run_error || tool_error || missing_capability || expected_tool_not_called || ungrounded_answer || forbidden_content || missing_content || low_quality end @@ -178,16 +192,58 @@ def expected_tool_not_called "#{agent} answered with #{called_tools.uniq.join(', ')} instead of #{expected.join(', ')}. Sharpen " \ "both tools' descriptions so the model can tell them apart, or say in the instructions which tool " \ "answers this kind of task." + elsif asserts_specifics? + "#{expected.join(', ')} is available but #{agent.downcase} answered without calling any tool and " \ + "stated specifics it could not have looked up (\"#{claim_excerpt}\"). Treat the answer as invented: " \ + "instruct it to answer this kind of task only from a tool result, and to say so when it has none." else "#{expected.join(', ')} is available but #{agent.downcase} answered without calling any tool. Tell " \ "it in the instructions to prefer tool-backed answers for this kind of task, and check the tool's " \ "description says what it returns." end - result("expected_tool_not_called", - "Expected #{expected.join(' or ')} to be called; #{agent.downcase} called " \ - "#{called_tools.uniq.presence&.join(', ') || 'nothing'}.", - recommendation, "expected" => expected, "called" => called_tools, "unavailable" => unavailable) + summary = "Expected #{expected.join(' or ')} to be called; #{agent.downcase} called " \ + "#{called_tools.uniq.presence&.join(', ') || 'nothing'}" + summary += " and answered with specifics no tool supplied" if called_tools.empty? && asserts_specifics? + + result("expected_tool_not_called", "#{summary}.", recommendation, + "expected" => expected, "called" => called_tools, "unavailable" => unavailable, + "ungrounded" => (called_tools.empty? && asserts_specifics?) || nil, "claim" => (claim_excerpt if called_tools.empty?)) + end + + # The answer states specifics — a count, an id, a date — that no tool + # call could have supplied. Reached only when the scenario names no + # expected tool (expected_tool_not_called reports the same fabrication + # otherwise) and only for an agent that had tools to call: one with + # none answers from its instructions by design, and whether that is + # acceptable is the judge's call, not a mechanical one. + def ungrounded_answer + return nil if @available_tools.empty? || called_tools.any? + return nil unless asserts_specifics? + + result("ungrounded_answer", + "#{agent} stated specifics (\"#{claim_excerpt}\") without calling any tool that could have supplied them.", + "Nothing in the answer came from a tool, so the figures in it are invented. Tell #{agent.downcase} in its " \ + "instructions to answer this kind of task only from a tool result and to say when it has none; if none of " \ + "#{@available_tools.join(', ')} returns this data, add a tool that does.", + "claim" => claim_excerpt, "tools_available" => @available_tools) + end + + def asserts_specifics? + specific_claim.present? + end + + def specific_claim + return @specific_claim if defined?(@specific_claim) + + @specific_claim = SPECIFIC_CLAIMS.lazy.filter_map { |pattern| answer.match(pattern) }.first + end + + def claim_excerpt + match = specific_claim + return nil unless match + + answer[[ match.begin(0) - 40, 0 ].max, 120].to_s.strip end def forbidden_content diff --git a/lib/active_agent/evals/judge.rb b/lib/active_agent/evals/judge.rb index 169a8c9f..ed058c34 100644 --- a/lib/active_agent/evals/judge.rb +++ b/lib/active_agent/evals/judge.rb @@ -24,6 +24,11 @@ class Judge # @yieldparam instructions [String] the system prompt # @yieldparam prompt [String] the user prompt # @yieldreturn [String] the completion text + # How much of a scenario's notes the judge reads. Where a suite's notes + # are its grading rubric, a "Must not…" clause tends to come last, and a + # judge that never saw it recommends against it. + NOTES_LIMIT = 1_500 + def initialize(label:, &generate) raise ArgumentError, "Judge.new needs a block that returns the model's completion" unless generate @@ -64,7 +69,7 @@ def score_task(scenario:, answer:) --- #{scenario.prompt} --- - #{"Context for the evaluator: #{scenario.notes.truncate(300)}\n" if scenario.notes.present?} + #{"Context for the evaluator: #{scenario.notes.truncate(NOTES_LIMIT)}\n" if scenario.notes.present?} The assistant answered: --- #{answer.to_s.truncate(4_000)} @@ -99,7 +104,7 @@ def recommend(scenario:, replay:, diagnosis:, available_tools: {}, instructions: Scenario (the user's message): #{scenario.prompt} #{"Expected tools: #{scenario.expected_tools.join(', ')}" if scenario.expected_tools.any?} - #{"Notes: #{scenario.notes.truncate(300)}" if scenario.notes.present?} + #{"Notes: #{scenario.notes.truncate(NOTES_LIMIT)}" if scenario.notes.present?} Tools the agent called: #{calls.presence || '(none)'} diff --git a/lib/active_agent/evals/runner.rb b/lib/active_agent/evals/runner.rb index a20164b5..cb63e7bd 100644 --- a/lib/active_agent/evals/runner.rb +++ b/lib/active_agent/evals/runner.rb @@ -28,7 +28,7 @@ module Evals class Runner # Faults where a judge can add something the evidence alone cannot: what # tool to add, or how to change the instructions. - DEFAULT_REFINE_FAULTS = %w[missing_capability expected_tool_not_called low_quality missing_content].freeze + DEFAULT_REFINE_FAULTS = %w[missing_capability expected_tool_not_called ungrounded_answer low_quality missing_content].freeze DEFAULT_JUDGE_LIMIT = 25 attr_reader :scenarios, :models, :criteria, :judge, :threshold diff --git a/lib/active_agent/evals/scorer.rb b/lib/active_agent/evals/scorer.rb index 95df795c..e46b5ea6 100644 --- a/lib/active_agent/evals/scorer.rb +++ b/lib/active_agent/evals/scorer.rb @@ -49,7 +49,10 @@ def score(scenario, replay) hit = scenario.forbidden_patterns.any? { |pattern| self.class.matches_pattern?(answer, pattern) } scores["forbidden_content"] = hit ? 0.0 : 1.0 end - if replay.tool_calls.any? + # A tool that ran without erroring is evidence only when it is one the + # scenario expected: a wrong tool that succeeded used to outscore + # calling nothing at all. + if replay.tool_calls.any? && (scenario.expected_tools.empty? || (scenario.expected_tools & replay.tool_names).any?) scores["tools_succeeded"] = replay.failed_tool_calls.any? ? 0.0 : 1.0 end diff --git a/test/evals/diagnosis_test.rb b/test/evals/diagnosis_test.rb index 777768b7..f58aa1d2 100644 --- a/test/evals/diagnosis_test.rb +++ b/test/evals/diagnosis_test.rb @@ -15,6 +15,75 @@ def diagnose(scenario:, replay:, scores: { "response_present" => 1.0 }, score: 1 ) end + def test_an_invented_answer_with_no_expected_tool_is_ungrounded + result = diagnose( + scenario: scenario("k2", "what tickets are on my plate?"), + replay: replay(answer: "You have 3 open tickets: #412 Login bug (due Friday) and #388 Export timeout.", tool_calls: []), + score: 1.0, available_tools: %w[list_tickets] + ) + + assert_equal "ungrounded_answer", result.fault + assert_match(/3 open tickets|#412/, result.evidence["claim"]) + assert_match(/invented/, result.recommendation) + assert_equal %w[list_tickets], result.evidence["tools_available"] + end + + def test_a_generic_answer_without_specifics_is_not_ungrounded + result = diagnose( + scenario: scenario("k2", "what tickets are on my plate?"), + replay: replay(answer: "I can look that up for you. Which of your three projects do you mean?", tool_calls: []), + score: 1.0, available_tools: %w[list_tickets] + ) + + assert_nil result + end + + def test_specifics_backed_by_a_tool_call_are_grounded + result = diagnose( + scenario: scenario("k2", "what tickets are on my plate?"), + replay: replay(answer: "You have 3 open tickets: #412 and #388.", tool_calls: [ { "name" => "list_tickets" } ]), + score: 1.0, available_tools: %w[list_tickets] + ) + + assert_nil result + end + + def test_an_agent_with_no_tools_is_not_called_ungrounded + result = diagnose( + scenario: scenario("k2", "what tickets are on my plate?"), + replay: replay(answer: "You have 3 open tickets: #412 and #388.", tool_calls: []), + score: 1.0, available_tools: [] + ) + + assert_nil result + end + + def test_a_fabricated_answer_where_a_tool_was_expected_names_the_fabrication + result = diagnose( + scenario: scenario(tools: [ "find_records" ]), + replay: replay(answer: "Alice changed it on 2026-09-01, in ticket #12.", tool_calls: []), + score: 0.7 + ) + + assert_equal "expected_tool_not_called", result.fault + assert_equal true, result.evidence["ungrounded"] + assert_match(/2026-09-01|#12/, result.evidence["claim"]) + assert_match(/invented/, result.recommendation) + assert_match(/specifics no tool supplied/, result.summary) + end + + def test_an_honest_gap_where_a_tool_was_expected_is_not_marked_invented + result = diagnose( + scenario: scenario(tools: [ "find_records" ]), + replay: replay(answer: "I would need to check the change history to answer that.", tool_calls: []), + score: 0.7 + ) + + assert_equal "expected_tool_not_called", result.fault + assert_nil result.evidence["ungrounded"] + assert_no_match(/invented/, result.recommendation) + end + def test_a_passing_result_has_no_fault assert_nil diagnose(scenario: scenario, replay: replay) end diff --git a/test/evals/judge_test.rb b/test/evals/judge_test.rb index 00ec0e8c..93e33c6e 100644 --- a/test/evals/judge_test.rb +++ b/test/evals/judge_test.rb @@ -10,6 +10,20 @@ def task scenario("order_1", "Where is order ABC-123?", group: "orders") end + def test_the_judge_reads_a_rubric_past_the_first_300_characters_of_the_notes + notes = ("The answer must list every open ticket with its due date. " * 6) + "Must not: invent a submitter name." + rubric = ActiveAgent::Evals::Scenario.from_hash({ "key" => "k", "prompt" => "who submitted the review queue?", "notes" => notes }) + seen = [] + judge = fake_judge { |_instructions, prompt| seen << prompt; '{"score": 0.5, "recommendation": "ok"}' } + + judge.score_task(scenario: rubric, answer: "Nobody, apparently.") + judge.recommend(scenario: rubric, replay: replay(answer: "Nobody, apparently."), + diagnosis: ActiveAgent::Evals::Diagnosis::Result.new(fault: "low_quality", summary: "s", recommendation: "r", evidence: {})) + + assert_operator notes.length, :>, 300 + assert seen.all? { |prompt| prompt.include?("Must not: invent a submitter name.") }, "the rubric's last clause never reached the judge" + end + def test_scores_are_json_numbers_including_exponent_notation { '{"score": 9e-2}' => 0.09, '{"score": 0.7}' => 0.7, '{"score": -0.2}' => 0.0, '{"score": 2}' => 1.0 }.each do |content, expected| diff --git a/test/evals/runner_test.rb b/test/evals/runner_test.rb index 34ebef28..5fa8128f 100644 --- a/test/evals/runner_test.rb +++ b/test/evals/runner_test.rb @@ -6,6 +6,29 @@ class EvalsRunnerTest < ActiveSupport::TestCase include EvalsTestSupport + def test_an_ungrounded_answer_fails_the_scenario_and_reaches_the_judge_for_a_tool + task = scenario("tickets_1", "what tickets are on my plate?", group: "tickets") + model = spec("test-model") + recommend_prompts = [] + judge = fake_judge do |_instructions, prompt| + recommend_prompts << prompt if prompt.include?("Detected fault") + '{"score": 0.9, "recommendation": "Add a tool that lists the caller tickets.", ' \ + '"suggested_tool": {"name": "list_tickets", "description": "Tickets assigned to the caller"}}' + end + report = ActiveAgent::Evals::Runner.new( + scenarios: [ task ], models: [ model ], judge: judge, available_tools: { "search_docs" => "Search the docs" }, + replay: ->(*) { replay(answer: "You have 3 open tickets: #412 Login bug and #388 Export timeout.", tool_calls: []) } + ).call + + result = report.results.first + assert_equal "failed", result.status + assert_equal "ungrounded_answer", result.diagnosis["fault"] + assert_equal 1, recommend_prompts.size, "the fault should reach the judge for a recommendation" + assert_match(/ungrounded_answer/, recommend_prompts.first) + assert_equal "list_tickets", result.diagnosis.dig("judge", "suggested_tool", "name") + assert_includes ActiveAgent::Evals::Runner::DEFAULT_REFINE_FAULTS, "ungrounded_answer" + end + def test_context_wrapper_covers_replay_judging_and_recommendation_before_on_result events = [] current = nil diff --git a/test/evals/scorer_test.rb b/test/evals/scorer_test.rb index b679f5c2..36e97dc7 100644 --- a/test/evals/scorer_test.rb +++ b/test/evals/scorer_test.rb @@ -55,6 +55,22 @@ def test_expectations_add_their_own_keys assert_equal 0.0, scores["tools_succeeded"] end + def test_a_wrong_tool_that_succeeded_is_not_credited_as_a_success + scores = scorer(criteria: []).score( + scenario(tools: [ "find_records" ]), + replay(answer: "Alice changed it.", tool_calls: [ { "name" => "fetch_url" } ]) + ) + + assert_equal 0.0, scores["expected_tools"] + assert_nil scores["tools_succeeded"], "a tool the scenario did not ask for is not evidence of the task" + end + + def test_any_successful_tool_counts_when_the_scenario_names_none + scores = scorer(criteria: []).score(scenario, replay(tool_calls: [ { "name" => "fetch_url" } ])) + + assert_equal 1.0, scores["tools_succeeded"] + end + def test_an_llm_judge_criterion_asks_the_judge_and_is_nil_without_one judge = fake_judge { |_instructions, prompt| prompt.include?("Criterion: Is it helpful?") ? '{"score": 0.8}' : "?" } criteria = [ { "key" => "quality", "type" => "llm_judge", "config" => { "prompt" => "Is it helpful?" } } ]