Feat/pairwise judging

CICD_EVAL.md

@@ -445,6 +445,68 @@ sys.exit(0 if result.gate == "pass" else 1)
 
 ---
 
+## pytest assertions
+
+`agentx.testing` turns a run into a plain pytest failure, so quality checks live in the same
+suite as everything else. Both helpers raise `AssertionError` subclasses - no plugin, no
+registration, works in any runner.
+
+### `assert_evaluation` - does it clear the bar
+
+```python
+from agentx import AgentX
+from agentx.testing import assert_evaluation
+
+def test_support_agent_quality():
+    client = AgentX.from_env()
+    report = (
+        client.evaluations
+        .run(dataset_id=DATASET_ID, scorer_id=SCORER_ID, subject=SUBJECT)
+        .execute(my_agent)
+        .finalize()
+    )
+    assert_evaluation(report, min_rating=7.0, no_regression=True)
+```
+
+`min_rating` is an absolute floor; `no_regression` compares against the dataset's previous
+completed run (`tolerance` defaults to 0.5, since judge scores are noisy). The check rides the
+engine's CI gate, so every pytest verdict also appears in the dashboard's gate history with
+`caller="pytest"` - the red test and the dashboard row are one event, not two systems drifting.
+
+### `assert_pairwise` - is it better than what it replaces
+
+An average clearing a floor does not mean a change helped. For that, compare the two runs
+head to head and assert the comparison went the candidate's way:
+
+```python
+from agentx.testing import assert_pairwise
+
+def test_new_prompt_beats_the_old_one():
+    comparison = client.evaluations.compare_pairwise(
+        candidate_run_id,
+        baseline_run_id,
+        both_orders=True,
+    )
+    assert_pairwise(
+        comparison,
+        must_win=True,
+        max_losses=2,
+        max_flip_rate=0.2,
+    )
+```
+
+| Check | Fails when |
+|---|---|
+| `must_win` | Run A did not win more cases than run B. A tie fails - "no worse than before" is not the claim being made. |
+| `max_losses` | Run A lost more individual cases than allowed, even if it won overall. Catches a change that lifts the average by improving easy cases while breaking hard ones. |
+| `max_flip_rate` | Too many verdicts reversed when the answers were swapped. That is position bias, and an inconclusive comparison must not read as a pass. |
+
+`max_flip_rate` needs `both_orders=True` to mean anything: without it there is no flip rate, and
+the check is skipped rather than passing on a fabricated zero. The failure message names the
+cases that lost, so the test output is actionable without opening the dashboard.
+
+---
+
 ## Exceptions
 
 | Exception | When raised |

agentx/evaluations/client.py

@@ -17,6 +17,7 @@
     EvaluationSettings,
     EvaluationSubject,
     ModelInfo,
+    PairwiseComparison,
     Prompt,
     Report,
 )
@@ -541,6 +542,63 @@ def publish_prompt_version(
             "POST", f"/evaluate/prompts/{prompt_id}/versions", base=self._api_root, json=payload
         )
 
+    # ------------------------------------------------------------------
+    # Head-to-head (pairwise) judging. Rides the /evaluate dialect via _api_root(), same
+    # precedent as get_report and the prompt loop above.
+    # ------------------------------------------------------------------
+
+    def compare_pairwise(
+        self,
+        run_a_id: str,
+        run_b_id: str,
+        *,
+        criteria: Optional[str] = None,
+        judge_model: Optional[str] = None,
+        both_orders: bool = False,
+    ) -> PairwiseComparison:
+        """Ask a judge which of two runs answered each question better.
+
+        Both runs must be of the same dataset. Absolute ratings answer "is this above the bar";
+        this answers "did the change help", which is the question a diff between two runs is
+        actually asking. The judge sees the two answers as "Answer 1"/"Answer 2" with the order
+        alternating case by case, so it never learns which run is the candidate.
+
+        ``both_orders=True`` judges every pair twice with the sides swapped. It doubles the judge
+        cost and is the only real defense against position bias: a pair whose winner reverses is
+        recorded as a tie, and the batch reports its ``flip_rate``.
+
+        ``criteria`` and ``judge_model`` default to the dataset's own evaluation criteria and
+        judge model, so a head-to-head grades on the same terms a normal run of it does.
+        """
+        payload: Dict[str, Any] = {"runAId": run_a_id, "runBId": run_b_id}
+        if criteria:
+            payload["criteria"] = criteria
+        if judge_model:
+            payload["judgeModel"] = judge_model
+        if both_orders:
+            payload["bothOrders"] = True
+        response = self._request("POST", "/evaluate/runs/pairwise", json=payload, base=self._api_root)
+        return PairwiseComparison(**response["comparison"])
+
+    def get_pairwise(self, batch_id: str) -> PairwiseComparison:
+        """Read back a stored head-to-head by its batch id."""
+        response = self._request("GET", f"/evaluate/runs/pairwise/{batch_id}", base=self._api_root)
+        return PairwiseComparison(**response["comparison"])
+
+    def list_pairwise(
+        self, *, run_a_id: Optional[str] = None, run_b_id: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        """Summaries of past head-to-heads, newest first, optionally narrowed to one run."""
+        params: Dict[str, Any] = {}
+        if run_a_id:
+            params["runAId"] = run_a_id
+        if run_b_id:
+            params["runBId"] = run_b_id
+        response = self._request(
+            "GET", "/evaluate/runs/pairwise", params=params or None, base=self._api_root
+        )
+        return response.get("comparisons", [])
+
     # ------------------------------------------------------------------
     # Tool schema registry (same version-scoped propose/publish loop as prompts)
     # ------------------------------------------------------------------

agentx/evaluations/models.py

@@ -742,3 +742,64 @@ class Config:
     @property
     def is_terminal(self) -> bool:
         return self.status in ("completed", "partially_failed", "failed")
+
+
+class PairwiseSummary(BaseModel):
+    """Batch-level verdict of a head-to-head comparison (``compare_pairwise``).
+
+    ``winner`` is "a", "b", or "tie" - a dead heat is reported as a tie rather than broken
+    arbitrarily. ``flip_rate`` is only populated for a ``both_orders=True`` comparison: it is the
+    share of cases whose winner reversed when the two answers were swapped, which is position
+    bias rather than quality. A high flip rate means the batch is inconclusive, so it is reported
+    instead of being folded away."""
+
+    total: int = 0
+    a_wins: int = Field(default=0, alias="aWins")
+    b_wins: int = Field(default=0, alias="bWins")
+    ties: int = 0
+    winner: str = "tie"
+    flip_rate: Optional[float] = Field(default=None, alias="flipRate")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class PairwiseCase(BaseModel):
+    """One question's verdict. ``presented_first`` records which run's answer the judge read
+    first, because that is the confound pairwise judging exists to control for."""
+
+    id: Optional[str] = Field(default=None, alias="_id")
+    question_index: Optional[int] = Field(default=None, alias="questionIndex")
+    query: Optional[str] = None
+    winner: str = "tie"
+    presented_first: str = Field(default="a", alias="presentedFirst")
+    flipped: bool = False
+    justification: Optional[str] = None
+    judge_model: Optional[str] = Field(default=None, alias="judgeModel")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"
+
+
+class PairwiseComparison(BaseModel):
+    """A full head-to-head between two runs of the same dataset.
+
+    ``skipped`` names the cases that could not be judged (one side produced no answer, or the
+    batch hit the server's per-comparison cap) with the reason - a comparison that quietly
+    dropped half the dataset would read as a clean sweep."""
+
+    batch_id: str = Field(alias="batchId")
+    run_a_id: str = Field(alias="runAId")
+    run_b_id: str = Field(alias="runBId")
+    both_orders: bool = Field(default=False, alias="bothOrders")
+    judge_model: Optional[str] = Field(default=None, alias="judgeModel")
+    summary: PairwiseSummary = Field(default_factory=PairwiseSummary)
+    cases: List[PairwiseCase] = Field(default_factory=list)
+    skipped: List[Dict[str, Any]] = Field(default_factory=list)
+    created_at: Optional[str] = Field(default=None, alias="createdAt")
+
+    class Config:
+        populate_by_name = True
+        extra = "ignore"

agentx/evaluations/runner.py

@@ -446,6 +446,20 @@ def simulate_conversation(self, **kwargs) -> dict:
         EvaluationsClient.simulate_conversation for parameters."""
         return self._client.simulate_conversation(**kwargs)
 
+    def compare_pairwise(self, run_a_id: str, run_b_id: str, **kwargs):
+        """Head-to-head judging between two runs of the same dataset - "which answer is better"
+        rather than two absolute scores. See EvaluationsClient.compare_pairwise for the options,
+        and agentx.testing.assert_pairwise for the pytest-side check."""
+        return self._client.compare_pairwise(run_a_id, run_b_id, **kwargs)
+
+    def get_pairwise(self, batch_id: str):
+        """Read back a stored head-to-head by its batch id."""
+        return self._client.get_pairwise(batch_id)
+
+    def list_pairwise(self, **kwargs) -> list:
+        """Summaries of past head-to-heads, newest first, optionally narrowed to one run."""
+        return self._client.list_pairwise(**kwargs)
+
     def get_run(self, run_id: str) -> dict:
         """Run summary + per-result rows by id, without needing the EvaluationRunContext that
         created it (e.g. from a separate process)."""

agentx/testing.py

@@ -88,3 +88,78 @@ def assert_evaluation(
         f"Evaluation run {run_id} failed its quality gate:\n{_format_failures(gate)}",
         gate=gate,
     )
+
+
+def _format_pairwise(comparison: Any) -> str:
+    summary = getattr(comparison, "summary", None)
+    lines: List[str] = []
+    if summary is not None:
+        flip = getattr(summary, "flip_rate", None)
+        lines.append(
+            f"  A won {getattr(summary, 'a_wins', 0)}, B won {getattr(summary, 'b_wins', 0)}, "
+            f"{getattr(summary, 'ties', 0)} tied, out of {getattr(summary, 'total', 0)}"
+            + (f" (flip rate {flip})" if flip is not None else "")
+        )
+    for case in getattr(comparison, "cases", None) or []:
+        if getattr(case, "winner", None) == "b":
+            query = (getattr(case, "query", None) or "").strip()
+            lines.append(f"  lost: {query[:80]} - {(getattr(case, 'justification', None) or '')[:120]}")
+    return "\n".join(lines) if lines else f"  comparison: {comparison!r}"
+
+
+def assert_pairwise(
+    comparison: Any,
+    *,
+    must_win: bool = False,
+    max_losses: Optional[int] = None,
+    max_flip_rate: Optional[float] = None,
+) -> Any:
+    """Assert a head-to-head comparison went the candidate's way.
+
+    ``comparison`` is what ``client.evaluations.compare_pairwise(a, b)`` returns; run A is the
+    candidate and run B is the baseline it has to beat.
+
+    - ``must_win`` - fail unless A won more cases than B. A tie fails: "no worse than before" is
+      not the same claim as "better", and a change that cannot win its own comparison has not
+      earned a green test.
+    - ``max_losses`` - fail when A lost more than this many individual cases, even if it won
+      overall. This is the check that catches a change that lifts the average by improving easy
+      cases while breaking hard ones.
+    - ``max_flip_rate`` - fail when too many verdicts reversed with the presentation order. That
+      is position bias rather than quality, and it means the comparison itself is inconclusive,
+      so treating it as a pass would be worse than a red test. Only meaningful for a comparison
+      run with ``both_orders=True``; a comparison without it has no flip rate and this check is
+      skipped rather than quietly passing.
+
+    At least one check is required. Returns the comparison on success; raises
+    :class:`EvaluationAssertionError` naming the cases that lost.
+    """
+    if not must_win and max_losses is None and max_flip_rate is None:
+        raise ValueError(
+            "assert_pairwise needs at least one check: must_win, max_losses, and/or max_flip_rate"
+        )
+    summary = getattr(comparison, "summary", None)
+    if summary is None:
+        raise ValueError("assert_pairwise expects the result of compare_pairwise()")
+
+    failures: List[str] = []
+    a_wins = getattr(summary, "a_wins", 0)
+    b_wins = getattr(summary, "b_wins", 0)
+    if must_win and a_wins <= b_wins:
+        failures.append(f"run A did not win ({a_wins} vs {b_wins})")
+    if max_losses is not None and b_wins > max_losses:
+        failures.append(f"run A lost {b_wins} cases, more than the {max_losses} allowed")
+    flip_rate = getattr(summary, "flip_rate", None)
+    if max_flip_rate is not None and flip_rate is not None and flip_rate > max_flip_rate:
+        failures.append(
+            f"verdicts flipped on {flip_rate:.0%} of cases with the presentation order, above the "
+            f"{max_flip_rate:.0%} allowed - this comparison is inconclusive, not a pass"
+        )
+
+    if not failures:
+        return comparison
+    batch_id = getattr(comparison, "batch_id", "?")
+    raise EvaluationAssertionError(
+        f"Head-to-head {batch_id} failed: {'; '.join(failures)}\n{_format_pairwise(comparison)}",
+        gate=comparison,
+    )