From ff46f1f70b2ce6d42777f783422dae408d76f06c Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 25 Aug 2026 14:39:12 -0700 Subject: [PATCH 1/2] feat: head-to-head judging and assert_pairwise client.evaluations.compare_pairwise(a, b) asks a judge which of two runs answered each question better, instead of comparing two absolute averages that drift with the rubric and cluster in the 7-8 band. get_pairwise / list_pairwise read past comparisons back. agentx.testing.assert_pairwise is the pytest side. Its checks are the ones that actually catch regressions rather than flattering them: must_win treats a tie as a failure (a change that cannot win its own comparison has not earned a green test), max_losses catches a change that lifts the average by improving easy cases while breaking hard ones, and max_flip_rate fails a comparison whose verdicts reversed with the presentation order, because that measured position bias and an inconclusive comparison must not read as a pass. With no both_orders pass there is no flip rate, and the check is skipped rather than passing on a fabricated zero. Verified live against a local engine: the stronger run passed must_win + max_losses=0 + max_flip_rate=0.25, and the same pair with the runs swapped failed with "run A did not win (0 vs 3)". Co-Authored-By: Claude Fable 5 --- agentx/evaluations/client.py | 58 +++++++++++++++++ agentx/evaluations/models.py | 61 ++++++++++++++++++ agentx/evaluations/runner.py | 14 +++++ agentx/testing.py | 75 ++++++++++++++++++++++ tests/test_pairwise.py | 119 +++++++++++++++++++++++++++++++++++ 5 files changed, 327 insertions(+) create mode 100644 tests/test_pairwise.py diff --git a/agentx/evaluations/client.py b/agentx/evaluations/client.py index 4814192..004a945 100644 --- a/agentx/evaluations/client.py +++ b/agentx/evaluations/client.py @@ -17,6 +17,7 @@ EvaluationSettings, EvaluationSubject, ModelInfo, + PairwiseComparison, Prompt, Report, ) @@ -541,6 +542,63 @@ def publish_prompt_version( "POST", f"/evaluate/prompts/{prompt_id}/versions", base=self._api_root, json=payload ) + # ------------------------------------------------------------------ + # Head-to-head (pairwise) judging. Rides the /evaluate dialect via _api_root(), same + # precedent as get_report and the prompt loop above. + # ------------------------------------------------------------------ + + def compare_pairwise( + self, + run_a_id: str, + run_b_id: str, + *, + criteria: Optional[str] = None, + judge_model: Optional[str] = None, + both_orders: bool = False, + ) -> PairwiseComparison: + """Ask a judge which of two runs answered each question better. + + Both runs must be of the same dataset. Absolute ratings answer "is this above the bar"; + this answers "did the change help", which is the question a diff between two runs is + actually asking. The judge sees the two answers as "Answer 1"/"Answer 2" with the order + alternating case by case, so it never learns which run is the candidate. + + ``both_orders=True`` judges every pair twice with the sides swapped. It doubles the judge + cost and is the only real defense against position bias: a pair whose winner reverses is + recorded as a tie, and the batch reports its ``flip_rate``. + + ``criteria`` and ``judge_model`` default to the dataset's own evaluation criteria and + judge model, so a head-to-head grades on the same terms a normal run of it does. + """ + payload: Dict[str, Any] = {"runAId": run_a_id, "runBId": run_b_id} + if criteria: + payload["criteria"] = criteria + if judge_model: + payload["judgeModel"] = judge_model + if both_orders: + payload["bothOrders"] = True + response = self._request("POST", "/evaluate/runs/pairwise", json=payload, base=self._api_root) + return PairwiseComparison(**response["comparison"]) + + def get_pairwise(self, batch_id: str) -> PairwiseComparison: + """Read back a stored head-to-head by its batch id.""" + response = self._request("GET", f"/evaluate/runs/pairwise/{batch_id}", base=self._api_root) + return PairwiseComparison(**response["comparison"]) + + def list_pairwise( + self, *, run_a_id: Optional[str] = None, run_b_id: Optional[str] = None + ) -> List[Dict[str, Any]]: + """Summaries of past head-to-heads, newest first, optionally narrowed to one run.""" + params: Dict[str, Any] = {} + if run_a_id: + params["runAId"] = run_a_id + if run_b_id: + params["runBId"] = run_b_id + response = self._request( + "GET", "/evaluate/runs/pairwise", params=params or None, base=self._api_root + ) + return response.get("comparisons", []) + # ------------------------------------------------------------------ # Tool schema registry (same version-scoped propose/publish loop as prompts) # ------------------------------------------------------------------ diff --git a/agentx/evaluations/models.py b/agentx/evaluations/models.py index 807d4a2..db5aaff 100644 --- a/agentx/evaluations/models.py +++ b/agentx/evaluations/models.py @@ -742,3 +742,64 @@ class Config: @property def is_terminal(self) -> bool: return self.status in ("completed", "partially_failed", "failed") + + +class PairwiseSummary(BaseModel): + """Batch-level verdict of a head-to-head comparison (``compare_pairwise``). + + ``winner`` is "a", "b", or "tie" - a dead heat is reported as a tie rather than broken + arbitrarily. ``flip_rate`` is only populated for a ``both_orders=True`` comparison: it is the + share of cases whose winner reversed when the two answers were swapped, which is position + bias rather than quality. A high flip rate means the batch is inconclusive, so it is reported + instead of being folded away.""" + + total: int = 0 + a_wins: int = Field(default=0, alias="aWins") + b_wins: int = Field(default=0, alias="bWins") + ties: int = 0 + winner: str = "tie" + flip_rate: Optional[float] = Field(default=None, alias="flipRate") + + class Config: + populate_by_name = True + extra = "ignore" + + +class PairwiseCase(BaseModel): + """One question's verdict. ``presented_first`` records which run's answer the judge read + first, because that is the confound pairwise judging exists to control for.""" + + id: Optional[str] = Field(default=None, alias="_id") + question_index: Optional[int] = Field(default=None, alias="questionIndex") + query: Optional[str] = None + winner: str = "tie" + presented_first: str = Field(default="a", alias="presentedFirst") + flipped: bool = False + justification: Optional[str] = None + judge_model: Optional[str] = Field(default=None, alias="judgeModel") + + class Config: + populate_by_name = True + extra = "ignore" + + +class PairwiseComparison(BaseModel): + """A full head-to-head between two runs of the same dataset. + + ``skipped`` names the cases that could not be judged (one side produced no answer, or the + batch hit the server's per-comparison cap) with the reason - a comparison that quietly + dropped half the dataset would read as a clean sweep.""" + + batch_id: str = Field(alias="batchId") + run_a_id: str = Field(alias="runAId") + run_b_id: str = Field(alias="runBId") + both_orders: bool = Field(default=False, alias="bothOrders") + judge_model: Optional[str] = Field(default=None, alias="judgeModel") + summary: PairwiseSummary = Field(default_factory=PairwiseSummary) + cases: List[PairwiseCase] = Field(default_factory=list) + skipped: List[Dict[str, Any]] = Field(default_factory=list) + created_at: Optional[str] = Field(default=None, alias="createdAt") + + class Config: + populate_by_name = True + extra = "ignore" diff --git a/agentx/evaluations/runner.py b/agentx/evaluations/runner.py index 9791740..ef33aa0 100644 --- a/agentx/evaluations/runner.py +++ b/agentx/evaluations/runner.py @@ -446,6 +446,20 @@ def simulate_conversation(self, **kwargs) -> dict: EvaluationsClient.simulate_conversation for parameters.""" return self._client.simulate_conversation(**kwargs) + def compare_pairwise(self, run_a_id: str, run_b_id: str, **kwargs): + """Head-to-head judging between two runs of the same dataset - "which answer is better" + rather than two absolute scores. See EvaluationsClient.compare_pairwise for the options, + and agentx.testing.assert_pairwise for the pytest-side check.""" + return self._client.compare_pairwise(run_a_id, run_b_id, **kwargs) + + def get_pairwise(self, batch_id: str): + """Read back a stored head-to-head by its batch id.""" + return self._client.get_pairwise(batch_id) + + def list_pairwise(self, **kwargs) -> list: + """Summaries of past head-to-heads, newest first, optionally narrowed to one run.""" + return self._client.list_pairwise(**kwargs) + def get_run(self, run_id: str) -> dict: """Run summary + per-result rows by id, without needing the EvaluationRunContext that created it (e.g. from a separate process).""" diff --git a/agentx/testing.py b/agentx/testing.py index 9913dfe..6ccec09 100644 --- a/agentx/testing.py +++ b/agentx/testing.py @@ -88,3 +88,78 @@ def assert_evaluation( f"Evaluation run {run_id} failed its quality gate:\n{_format_failures(gate)}", gate=gate, ) + + +def _format_pairwise(comparison: Any) -> str: + summary = getattr(comparison, "summary", None) + lines: List[str] = [] + if summary is not None: + flip = getattr(summary, "flip_rate", None) + lines.append( + f" A won {getattr(summary, 'a_wins', 0)}, B won {getattr(summary, 'b_wins', 0)}, " + f"{getattr(summary, 'ties', 0)} tied, out of {getattr(summary, 'total', 0)}" + + (f" (flip rate {flip})" if flip is not None else "") + ) + for case in getattr(comparison, "cases", None) or []: + if getattr(case, "winner", None) == "b": + query = (getattr(case, "query", None) or "").strip() + lines.append(f" lost: {query[:80]} - {(getattr(case, 'justification', None) or '')[:120]}") + return "\n".join(lines) if lines else f" comparison: {comparison!r}" + + +def assert_pairwise( + comparison: Any, + *, + must_win: bool = False, + max_losses: Optional[int] = None, + max_flip_rate: Optional[float] = None, +) -> Any: + """Assert a head-to-head comparison went the candidate's way. + + ``comparison`` is what ``client.evaluations.compare_pairwise(a, b)`` returns; run A is the + candidate and run B is the baseline it has to beat. + + - ``must_win`` - fail unless A won more cases than B. A tie fails: "no worse than before" is + not the same claim as "better", and a change that cannot win its own comparison has not + earned a green test. + - ``max_losses`` - fail when A lost more than this many individual cases, even if it won + overall. This is the check that catches a change that lifts the average by improving easy + cases while breaking hard ones. + - ``max_flip_rate`` - fail when too many verdicts reversed with the presentation order. That + is position bias rather than quality, and it means the comparison itself is inconclusive, + so treating it as a pass would be worse than a red test. Only meaningful for a comparison + run with ``both_orders=True``; a comparison without it has no flip rate and this check is + skipped rather than quietly passing. + + At least one check is required. Returns the comparison on success; raises + :class:`EvaluationAssertionError` naming the cases that lost. + """ + if not must_win and max_losses is None and max_flip_rate is None: + raise ValueError( + "assert_pairwise needs at least one check: must_win, max_losses, and/or max_flip_rate" + ) + summary = getattr(comparison, "summary", None) + if summary is None: + raise ValueError("assert_pairwise expects the result of compare_pairwise()") + + failures: List[str] = [] + a_wins = getattr(summary, "a_wins", 0) + b_wins = getattr(summary, "b_wins", 0) + if must_win and a_wins <= b_wins: + failures.append(f"run A did not win ({a_wins} vs {b_wins})") + if max_losses is not None and b_wins > max_losses: + failures.append(f"run A lost {b_wins} cases, more than the {max_losses} allowed") + flip_rate = getattr(summary, "flip_rate", None) + if max_flip_rate is not None and flip_rate is not None and flip_rate > max_flip_rate: + failures.append( + f"verdicts flipped on {flip_rate:.0%} of cases with the presentation order, above the " + f"{max_flip_rate:.0%} allowed - this comparison is inconclusive, not a pass" + ) + + if not failures: + return comparison + batch_id = getattr(comparison, "batch_id", "?") + raise EvaluationAssertionError( + f"Head-to-head {batch_id} failed: {'; '.join(failures)}\n{_format_pairwise(comparison)}", + gate=comparison, + ) diff --git a/tests/test_pairwise.py b/tests/test_pairwise.py new file mode 100644 index 0000000..e490281 --- /dev/null +++ b/tests/test_pairwise.py @@ -0,0 +1,119 @@ +"""Head-to-head judging: the client's request shape and the pytest assertion over the result.""" + +import pytest + +from agentx.evaluations.models import PairwiseComparison +from agentx.testing import EvaluationAssertionError, assert_pairwise + + +def comparison(**over) -> PairwiseComparison: + payload = { + "batchId": "batch-1", + "runAId": "run-candidate", + "runBId": "run-baseline", + "bothOrders": False, + "judgeModel": "gpt-5.6-luna", + "summary": {"total": 3, "aWins": 2, "bWins": 1, "ties": 0, "winner": "a", "flipRate": None}, + "cases": [ + { + "questionIndex": 2, + "query": "Who pays return shipping?", + "winner": "b", + "presentedFirst": "a", + "justification": "Answer 2 names both cases explicitly.", + } + ], + "skipped": [], + } + payload.update(over) + return PairwiseComparison(**payload) + + +class FakeClient: + """Captures the request instead of sending it - the wire shape is the contract with the + engine, and camelCase is the convention it has to keep.""" + + # The pairwise routes live on the /evaluate dialect, reached through this property. + _api_root = "https://engine.example/api/v1" + + def __init__(self, response=None): + self.calls = [] + self._response = response or {"comparison": comparison().model_dump(by_alias=True)} + + def _request(self, method, path, **kwargs): + self.calls.append((method, path, kwargs)) + return self._response + + +def test_compare_pairwise_sends_camelcase_and_omits_unset_options(): + from agentx.evaluations.client import EvaluationsClient + + client = FakeClient() + result = EvaluationsClient.compare_pairwise(client, "run-candidate", "run-baseline") + + method, path, kwargs = client.calls[0] + assert (method, path) == ("POST", "/evaluate/runs/pairwise") + # Defaults are the server's to choose; the SDK does not invent a criteria string or a + # judge model, and does not send bothOrders unless the caller asked for it. + assert kwargs["json"] == {"runAId": "run-candidate", "runBId": "run-baseline"} + assert result.summary.a_wins == 2 + assert result.cases[0].presented_first == "a" + + +def test_compare_pairwise_forwards_the_options_it_is_given(): + from agentx.evaluations.client import EvaluationsClient + + client = FakeClient() + EvaluationsClient.compare_pairwise( + client, "a", "b", criteria="Which is more concise?", judge_model="gpt-5.6-luna", both_orders=True + ) + assert client.calls[0][2]["json"] == { + "runAId": "a", + "runBId": "b", + "criteria": "Which is more concise?", + "judgeModel": "gpt-5.6-luna", + "bothOrders": True, + } + + +def test_assert_pairwise_passes_a_clear_win(): + result = assert_pairwise(comparison(), must_win=True, max_losses=1) + assert result.summary.winner == "a" + + +def test_a_tie_is_not_a_win(): + tied = comparison(summary={"total": 2, "aWins": 1, "bWins": 1, "ties": 0, "winner": "tie", "flipRate": None}) + with pytest.raises(EvaluationAssertionError) as excinfo: + assert_pairwise(tied, must_win=True) + assert "did not win" in str(excinfo.value) + + +def test_max_losses_catches_a_win_that_broke_hard_cases(): + # Wins overall, but lost more individual cases than the caller tolerates. + lossy = comparison(summary={"total": 10, "aWins": 5, "bWins": 4, "ties": 1, "winner": "a", "flipRate": None}) + with pytest.raises(EvaluationAssertionError) as excinfo: + assert_pairwise(lossy, max_losses=2) + message = str(excinfo.value) + assert "lost 4 cases" in message + # The failure names the case that lost, so the test output is actionable on its own. + assert "Who pays return shipping?" in message + + +def test_a_high_flip_rate_fails_instead_of_passing_on_position_bias(): + biased = comparison( + bothOrders=True, + summary={"total": 4, "aWins": 3, "bWins": 1, "ties": 0, "winner": "a", "flipRate": 0.5}, + ) + with pytest.raises(EvaluationAssertionError) as excinfo: + assert_pairwise(biased, max_flip_rate=0.2) + assert "inconclusive" in str(excinfo.value) + + +def test_flip_rate_check_is_skipped_when_both_orders_was_not_run(): + # No flip rate exists to check, so this must not silently fail or silently pass a made-up 0. + assert_pairwise(comparison(), max_flip_rate=0.0) + + +def test_requires_at_least_one_check(): + with pytest.raises(ValueError): + assert_pairwise(comparison()) From bef5d01b270cd6344758801c22ffc8a827b8297e Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 25 Aug 2026 14:46:11 -0700 Subject: [PATCH 2/2] docs: cover the pytest assertions in CICD_EVAL.md assert_evaluation shipped without a mention in any SDK markdown, and assert_pairwise would have followed it. Both are now documented where a CI reader looks, with the distinction that matters between them: one asks whether a run clears the bar, the other whether it beats what it replaces. Co-Authored-By: Claude Fable 5 --- CICD_EVAL.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/CICD_EVAL.md b/CICD_EVAL.md index e0cd650..61d72c7 100644 --- a/CICD_EVAL.md +++ b/CICD_EVAL.md @@ -445,6 +445,68 @@ sys.exit(0 if result.gate == "pass" else 1) --- +## pytest assertions + +`agentx.testing` turns a run into a plain pytest failure, so quality checks live in the same +suite as everything else. Both helpers raise `AssertionError` subclasses - no plugin, no +registration, works in any runner. + +### `assert_evaluation` - does it clear the bar + +```python +from agentx import AgentX +from agentx.testing import assert_evaluation + +def test_support_agent_quality(): + client = AgentX.from_env() + report = ( + client.evaluations + .run(dataset_id=DATASET_ID, scorer_id=SCORER_ID, subject=SUBJECT) + .execute(my_agent) + .finalize() + ) + assert_evaluation(report, min_rating=7.0, no_regression=True) +``` + +`min_rating` is an absolute floor; `no_regression` compares against the dataset's previous +completed run (`tolerance` defaults to 0.5, since judge scores are noisy). The check rides the +engine's CI gate, so every pytest verdict also appears in the dashboard's gate history with +`caller="pytest"` - the red test and the dashboard row are one event, not two systems drifting. + +### `assert_pairwise` - is it better than what it replaces + +An average clearing a floor does not mean a change helped. For that, compare the two runs +head to head and assert the comparison went the candidate's way: + +```python +from agentx.testing import assert_pairwise + +def test_new_prompt_beats_the_old_one(): + comparison = client.evaluations.compare_pairwise( + candidate_run_id, + baseline_run_id, + both_orders=True, + ) + assert_pairwise( + comparison, + must_win=True, + max_losses=2, + max_flip_rate=0.2, + ) +``` + +| Check | Fails when | +|---|---| +| `must_win` | Run A did not win more cases than run B. A tie fails - "no worse than before" is not the claim being made. | +| `max_losses` | Run A lost more individual cases than allowed, even if it won overall. Catches a change that lifts the average by improving easy cases while breaking hard ones. | +| `max_flip_rate` | Too many verdicts reversed when the answers were swapped. That is position bias, and an inconclusive comparison must not read as a pass. | + +`max_flip_rate` needs `both_orders=True` to mean anything: without it there is no flip rate, and +the check is skipped rather than passing on a fabricated zero. The failure message names the +cases that lost, so the test output is actionable without opening the dashboard. + +--- + ## Exceptions | Exception | When raised |