From 8d76a4aea33619f4c1200fdbb22c24fced657ad6 Mon Sep 17 00:00:00 2001 From: ernestprovo23 Date: Tue, 9 Jun 2026 17:57:43 -0400 Subject: [PATCH] feat(debate): optional early-stop on convergence (closes #4) Add an opt-in debate early-stop when answers converge, preserving the historic fixed-rounds behavior exactly when disabled (default). Signal: round-over-round answer stability -- per-member difflib.SequenceMatcher ratio averaged across members usable in both the previous and current round. Deterministic, stdlib-only, offline-testable. A degenerate score (no comparable answers) or any scoring failure degrades to running the remaining fixed rounds; it never crashes the debate. Opt-in is consistent with the cache feature: - config: ConclaveConfig.converge_threshold (float | None, off by default) - library: Council.debate/debate_sync + run_debate converge_threshold param (None defers to config, mirroring cache's None-defers-to-config) - CLI: --converge-threshold FLOAT plus --converge/--no-converge Recorded on CouncilResult.converged + convergence_score (minimal, mirrors how the cache feature added .cached); actual rounds run is len(rounds). converge_threshold is added to the debate cache key (make_key / _cache_key / _cached_run plumbing) so a converged run and a fixed run over identical inputs never collide. Convergence logic lives in modes.py (not council.py). 9 new offline, transport-mocked tests cover: off=full rounds, on+converge=early stop with recorded score, on+never-converge=full rounds, degenerate input fallback, config-defer, the scorer unit, and a cache-key no-collision check (142 -> 151 tests). ruff check + format clean. --- README.md | 24 +++- config.example.yml | 9 ++ docs/PRODUCT_DESIGN_DOCUMENT.md | 11 +- src/conclave/cache.py | 11 +- src/conclave/cli.py | 67 ++++++++++- src/conclave/config.py | 36 ++++++ src/conclave/council.py | 45 ++++++- src/conclave/models.py | 11 ++ src/conclave/modes.py | 112 +++++++++++++++++- tests/test_cache.py | 60 ++++++++++ tests/test_modes.py | 204 ++++++++++++++++++++++++++++++++ 11 files changed, 566 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index d6c8579..4c910b3 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,10 @@ conclave ask "Name three sorting algorithms." -c grok,perplexity --mode raw conclave ask "Is a service mesh worth it for 8 services?" \ -c grok,gemini,claude --mode debate --rounds 3 +# Debate with early-stop: stop before --rounds once answers stop changing +conclave ask "Is a service mesh worth it for 8 services?" \ + -c grok,gemini,claude --mode debate --rounds 5 --converge-threshold 0.95 + # Adversarial: one model proposes, the rest refute, the synthesizer judges conclave ask "Defend event sourcing for this ledger." \ -c grok,gemini,perplexity --mode adversarial --proposer grok @@ -89,9 +93,11 @@ conclave ask "..." -c grok,perplexity --mode debate --json ``` Mode flags at a glance: `--mode synthesize|raw|debate|adversarial`. `--rounds N` -(default 2) applies to `debate`; `--proposer NAME` (default: first member) applies -to `adversarial`. `--synthesizer/-s` overrides the synthesizer *and* the adversarial -judge. +(default 2) is the *maximum* round count for `debate`; `--converge-threshold FLOAT` +(or `--converge`/`--no-converge`) optionally stops a debate early once answers +stabilize round-over-round (off by default — `--rounds` runs in full). `--proposer +NAME` (default: first member) applies to `adversarial`. `--synthesizer/-s` overrides +the synthesizer *and* the adversarial judge. `--council` accepts either a comma-separated list of friendly names or the name of a council defined in your config (see below). The built-in `default` council @@ -127,6 +133,10 @@ for rnd in debate.rounds: print("round", rnd.round_number, [a.name for a in rnd.successful_answers]) print("FINAL:\n", debate.synthesis) +# debate with optional early-stop: stop before `rounds` once answers converge +quick = council.debate_sync("Is P=NP likely false?", rounds=5, converge_threshold=0.95) +print("ran", len(quick.rounds), "rounds; converged:", quick.converged, quick.convergence_score) + # adversarial: propose -> refute -> verdict adv = council.adversarial_sync("Defend CRDTs for offline-first apps.", proposer="grok") print("PROPOSAL by", adv.adversarial.proposer, "->", adv.adversarial.proposal.answer) @@ -138,7 +148,8 @@ print("VERDICT:\n", adv.adversarial.verdict) # also mirrored to adv.synthesis `CouncilResult` exposes `mode`, `answers` (per-model `ModelAnswer` with `model_id`, `latency_s`, `usage`, `error`), `synthesis`, `synthesizer`, `skipped`, plus `successful_answers` / `failed_answers` helpers. For `debate` it also carries -`rounds` (a list of `DebateRound`, each with per-member `answers`); for +`rounds` (a list of `DebateRound`, each with per-member `answers`) plus +`converged`/`convergence_score` (set when an early-stop fired); for `adversarial` it carries `adversarial` (an `AdversarialResult` with `proposer`, `proposal`, `critiques`, `verdict`). For debate the final round is mirrored into `answers` and the synthesis into `synthesis`; for adversarial the proposal + @@ -168,8 +179,9 @@ Repeated or eval runs can be served from an on-disk cache instead of re-calling the providers. It is **off by default** and **never persists API keys** — the cache key is a SHA-256 over the normalized prompt, the ordered council members (friendly name + resolved model id), the mode, the synthesizer/judge identity, -and the mode params (temperature, debate `rounds`, adversarial `proposer`). No -key value or env-var name ever reaches the key or the stored payload. +and the mode params (temperature, debate `rounds` + `converge_threshold`, +adversarial `proposer`). No key value or env-var name ever reaches the key or the +stored payload. Enable it per run with `--cache` (or disable a config default with `--no-cache`): diff --git a/config.example.yml b/config.example.yml index f9340a1..10874fb 100644 --- a/config.example.yml +++ b/config.example.yml @@ -25,3 +25,12 @@ synthesizer: claude # The cache is keyed on (prompt, council, mode, model ids) and NEVER stores keys. # Stored under $XDG_CACHE_HOME/conclave (else ~/.cache/conclave). cache: false + +# Optional debate early-stop on convergence (OFF by default, unset). When set to +# a value in [0.0, 1.0], a `--mode debate` run stops before `--rounds` is +# exhausted once answers stabilize round-over-round (a difflib similarity ratio +# averaged across members reaches this threshold). Leave unset for the historic +# fixed-rounds behavior. Override per invocation with `--converge-threshold` or +# `--converge` / `--no-converge`. A high value (e.g. 0.95) only stops on +# near-identical successive answers. +# converge_threshold: 0.95 diff --git a/docs/PRODUCT_DESIGN_DOCUMENT.md b/docs/PRODUCT_DESIGN_DOCUMENT.md index 0a7018f..3178fe7 100644 --- a/docs/PRODUCT_DESIGN_DOCUMENT.md +++ b/docs/PRODUCT_DESIGN_DOCUMENT.md @@ -345,8 +345,15 @@ Ordered roughly by strategic value to the origin use case and to mcp-warden. 1. **`vote` mode** — structured majority with reported split. Needs a constrained answer schema so votes are comparable. -2. **Debate convergence/stop criteria** — today debate runs a fixed `--rounds`; add optional - early-stop when answers converge (and a configurable convergence signal). +2. ~~**Debate convergence/stop criteria** — today debate runs a fixed `--rounds`; add optional + early-stop when answers converge (and a configurable convergence signal).~~ **LANDED** + (issue #4): opt-in early-stop via `converge_threshold` (config field, `Council.debate` + param, and `--converge-threshold` / `--converge`/`--no-converge` CLI flags). The signal is + round-over-round answer stability — per-member `difflib.SequenceMatcher` ratio averaged + across members — deterministic, stdlib-only, offline-testable. Off by default (`None`), + preserving fixed-rounds behavior exactly. Recorded on `CouncilResult.converged` + + `convergence_score`; `converge_threshold` is part of the debate cache key so a converged + run and a fixed run never collide. Kept in the list, struck through, for traceability. 3. **More first-class providers** — additional friendly-name defaults (e.g. more OpenAI, Anthropic, Google, and open-weights endpoints). New OpenAI-compatible vendors are already config-only via `endpoints:`; this item is about promoting common ones to typed defaults diff --git a/src/conclave/cache.py b/src/conclave/cache.py index 74edb2f..455e303 100644 --- a/src/conclave/cache.py +++ b/src/conclave/cache.py @@ -84,6 +84,7 @@ def make_key( temperature: float, rounds: int | None = None, proposer: str | None = None, + converge_threshold: float | None = None, ) -> str: """Build the stable cache key (sha256 hex) for a council run. @@ -95,8 +96,8 @@ def make_key( * ordered ``(friendly_name, resolved_model_id)`` member pairs (order matters -- see module docstring), * synthesizer/judge friendly name + resolved model id, - * temperature, and mode params (``rounds`` for debate, ``proposer`` for - adversarial) when they apply. + * temperature, and mode params (``rounds`` + ``converge_threshold`` for + debate, ``proposer`` for adversarial) when they apply. Args: prompt: The raw user prompt (normalized internally). @@ -108,6 +109,9 @@ def make_key( rounds: Debate round count (included only for ``debate``). proposer: Adversarial proposer friendly name (included only for ``adversarial``). + converge_threshold: Debate early-stop threshold (included only for + ``debate``). A converged run and a fixed-rounds run over otherwise + identical inputs must not collide, so this is part of the key. Returns: A 64-char lowercase hex SHA-256 digest. Contains zero key material. @@ -124,6 +128,9 @@ def make_key( } if mode == "debate": payload["rounds"] = rounds + # Early-stop changes how many rounds actually run and thus the output; + # it must distinguish a converged run from a fixed-rounds run. + payload["converge_threshold"] = converge_threshold if mode == "adversarial": payload["proposer"] = proposer diff --git a/src/conclave/cli.py b/src/conclave/cli.py index b5f4e3d..dbddce1 100644 --- a/src/conclave/cli.py +++ b/src/conclave/cli.py @@ -92,6 +92,12 @@ def _render_debate(result: CouncilResult) -> None: console.rule(f"[bold]Round {rnd.round_number}[/bold]") for ans in rnd.answers: console.print(_answer_panel(ans, border="magenta")) + if result.converged: + score = result.convergence_score if result.convergence_score is not None else 0.0 + err_console.print( + f"[green]Converged after {len(result.rounds)} round(s) " + f"(score {score:.3f}); stopped early.[/green]" + ) _print_synthesis(result, title="FINAL SYNTHESIS") @@ -134,6 +140,39 @@ def _render_adversarial(result: CouncilResult) -> None: _VALID_MODES = {"synthesize", "raw", "debate", "adversarial"} +# Threshold used when --converge is passed without an explicit --converge-threshold. +# High by design: an early stop should require answers that are nearly stable +# round-over-round, so the default is conservative and rarely fires spuriously. +_DEFAULT_CONVERGE_THRESHOLD = 0.95 + + +def _resolve_converge_threshold( + converge: bool | None, + converge_threshold: float | None, + config_threshold: float | None, +) -> float | None: + """Resolve the CLI convergence flags + config into a single threshold. + + Returns the threshold to run with, or ``None`` for early-stop off. Precedence, + mirroring how ``--cache/--no-cache`` overrides config per invocation: + + * an explicit ``--converge-threshold`` value wins (and implies on); + * else ``--no-converge`` forces off regardless of config; + * else ``--converge`` turns on at the default threshold; + * else (both flags unset) defer to the config value. + + Resolution happens here (not in the council) so ``--no-converge`` can force + off even when config enables it; the resulting concrete threshold is then + passed to :meth:`Council.debate_sync`. + """ + if converge_threshold is not None: + return converge_threshold + if converge is False: + return None + if converge is True: + return _DEFAULT_CONVERGE_THRESHOLD + return config_threshold + @app.command() def ask( @@ -154,7 +193,28 @@ def ask( None, "--synthesizer", "-s", help="Override the synthesizer/judge model name." ), rounds: int = typer.Option( - 2, "--rounds", "-r", help="Number of debate rounds (debate mode only).", min=1 + 2, "--rounds", "-r", help="Maximum number of debate rounds (debate mode only).", min=1 + ), + converge: bool | None = typer.Option( + None, + "--converge/--no-converge", + help=( + "Enable/disable debate early-stop on answer convergence (debate mode " + "only; off by default, defers to config when unset). --converge with " + "no --converge-threshold uses a default threshold of " + f"{_DEFAULT_CONVERGE_THRESHOLD}; --no-converge forces it off." + ), + ), + converge_threshold: float | None = typer.Option( + None, + "--converge-threshold", + help=( + "Debate early-stop threshold in [0.0, 1.0] (debate mode only). When " + "set, the debate stops once round-over-round answer stability reaches " + "this value. Implies --converge. Overrides config; unset defers to it." + ), + min=0.0, + max=1.0, ), proposer: str | None = typer.Option( None, @@ -201,7 +261,10 @@ def ask( c = Council(models=members, synthesizer=synthesizer, config=cfg, cache=cache) if mode_lower == "debate": - result = c.debate_sync(prompt, rounds=rounds) + threshold = _resolve_converge_threshold( + converge, converge_threshold, cfg.converge_threshold + ) + result = c.debate_sync(prompt, rounds=rounds, converge_threshold=threshold) elif mode_lower == "adversarial": result = c.adversarial_sync(prompt, proposer=proposer) else: diff --git a/src/conclave/config.py b/src/conclave/config.py index 0815221..eef2716 100644 --- a/src/conclave/config.py +++ b/src/conclave/config.py @@ -66,6 +66,13 @@ class ConclaveConfig(BaseModel): repeat run is served from the on-disk cache (see :mod:`conclave.cache`) instead of re-calling the providers. The cache never persists keys. A ``--cache/--no-cache`` CLI flag overrides this per invocation. + converge_threshold: opt-in debate early-stop threshold (off by default, + ``None``). When set to a value in ``[0.0, 1.0]``, a ``debate`` run + stops before ``rounds`` is exhausted once the round-over-round answer + stability (a :mod:`difflib` similarity ratio averaged across members) + reaches the threshold. ``None`` keeps the historic fixed-rounds + behavior exactly. A ``--converge-threshold`` / ``--converge/--no-converge`` + CLI flag overrides this per invocation. See :func:`conclave.modes.run_debate`. """ models: dict[str, str] = Field(default_factory=dict) @@ -73,6 +80,7 @@ class ConclaveConfig(BaseModel): synthesizer: str = DEFAULT_SYNTHESIZER endpoints: dict[str, CustomEndpoint] = Field(default_factory=dict) cache: bool = False + converge_threshold: float | None = None def resolve_model_id(self, name: str) -> str: """Map a friendly name to a provider-prefixed model id. @@ -191,10 +199,38 @@ def _load_config_uncached(path: Path) -> ConclaveConfig: # Off by default; only an explicit truthy ``cache: true`` in the file enables it. cache = bool(raw.get("cache", False)) + # Off by default (None). A malformed value degrades to off rather than raising, + # keeping config loading resilient like the rest of this module. + converge_threshold = _coerce_threshold(raw.get("converge_threshold")) + return ConclaveConfig( models=merged_models, councils=councils, synthesizer=synthesizer, endpoints=endpoints, cache=cache, + converge_threshold=converge_threshold, ) + + +def _coerce_threshold(value: Any) -> float | None: + """Coerce a config ``converge_threshold`` value to a float in ``[0.0, 1.0]``. + + Returns ``None`` (early-stop off) for an absent, non-numeric, or out-of-range + value, logging a warning in the latter cases. This mirrors the module's + resilient loading: a bad config field never crashes a run, it just disables + the optional feature. + """ + if value is None: + return None + try: + threshold = float(value) + except (TypeError, ValueError): + logger.warning("converge_threshold %r is not a number; disabling early-stop", value) + return None + if not 0.0 <= threshold <= 1.0: + logger.warning( + "converge_threshold %s is outside [0.0, 1.0]; disabling early-stop", threshold + ) + return None + return threshold diff --git a/src/conclave/council.py b/src/conclave/council.py index d5b2700..3ee28f9 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -105,6 +105,7 @@ def _cache_key( *, rounds: int | None = None, proposer: str | None = None, + converge_threshold: float | None = None, ) -> str: """Build the cache key for a run from the resolved, secret-free identity. @@ -125,6 +126,7 @@ def _cache_key( temperature=self.temperature, rounds=rounds, proposer=proposer, + converge_threshold=converge_threshold, ) async def _cached_run( @@ -135,6 +137,7 @@ async def _cached_run( *, rounds: int | None = None, proposer: str | None = None, + converge_threshold: float | None = None, ) -> CouncilResult: """Serve ``run`` from the result cache when caching is enabled. @@ -146,7 +149,13 @@ async def _cached_run( if not self.cache_enabled: return await run() - key = self._cache_key(prompt, mode, rounds=rounds, proposer=proposer) + key = self._cache_key( + prompt, + mode, + rounds=rounds, + proposer=proposer, + converge_threshold=converge_threshold, + ) hit = cache_mod.load(key) if hit is not None: logger.info("cache hit for %s run (%s)", mode, key[:12]) @@ -307,20 +316,39 @@ async def synthesize_blocks(self, system_prompt: str, user_content: str) -> Mode timeout=self.timeout, ) - async def debate(self, prompt: str, rounds: int = 2) -> CouncilResult: + async def debate( + self, prompt: str, rounds: int = 2, converge_threshold: float | None = None + ) -> CouncilResult: """Run a multi-round debate. See :func:`conclave.modes.run_debate`. Round 1 is an independent fan-out; rounds 2..N show each member its peers' anonymized prior answers; the synthesizer consolidates survivors. - Cache-served when caching is enabled (``rounds`` is part of the key). + Cache-served when caching is enabled (``rounds`` and the resolved + ``converge_threshold`` are part of the key). + + Args: + prompt: The user prompt. + rounds: Maximum number of rounds (the historic fixed count). + converge_threshold: Opt-in early-stop threshold. ``None`` (default) + defers to ``config.converge_threshold`` (off unless set in + ``~/.conclave/config.yml``); an explicit value overrides it for + this call. With early-stop off the debate runs exactly ``rounds``, + identical to the historic behavior. Mirrors the ``cache`` + None-defers-to-config convention. """ from .modes import run_debate + # Resolve the opt-in here (mirrors ``cache``: explicit arg wins, else + # config) so the cache key reflects what will actually run. + threshold = ( + self.config.converge_threshold if converge_threshold is None else converge_threshold + ) return await self._cached_run( prompt, "debate", - lambda: run_debate(self, prompt, rounds=rounds), + lambda: run_debate(self, prompt, rounds=rounds, converge_threshold=threshold), rounds=rounds, + converge_threshold=threshold, ) async def adversarial(self, prompt: str, proposer: str | None = None) -> CouncilResult: @@ -398,9 +426,14 @@ def ask_sync(self, prompt: str, synthesize: bool = True) -> CouncilResult: """ return self._run_sync(lambda: self.ask(prompt, synthesize=synthesize), "ask_sync") - def debate_sync(self, prompt: str, rounds: int = 2) -> CouncilResult: + def debate_sync( + self, prompt: str, rounds: int = 2, converge_threshold: float | None = None + ) -> CouncilResult: """Synchronous wrapper around :meth:`debate`.""" - return self._run_sync(lambda: self.debate(prompt, rounds=rounds), "debate_sync") + return self._run_sync( + lambda: self.debate(prompt, rounds=rounds, converge_threshold=converge_threshold), + "debate_sync", + ) def adversarial_sync(self, prompt: str, proposer: str | None = None) -> CouncilResult: """Synchronous wrapper around :meth:`adversarial`.""" diff --git a/src/conclave/models.py b/src/conclave/models.py index ef57e4e..2fac00e 100644 --- a/src/conclave/models.py +++ b/src/conclave/models.py @@ -114,6 +114,15 @@ class CouncilResult(BaseModel): cache rather than produced by a live run. ``False`` for every live run and for freshly stored entries. Lets a consumer detect a cache hit without re-running. See :mod:`conclave.cache`. + converged: ``True`` when a ``debate`` run stopped early because answers + converged (the convergence score crossed the configured threshold) + before ``rounds`` was exhausted. ``False`` for every other run, + including a debate that ran its full round count. The actual number + of rounds run is always ``len(rounds)``. See + :func:`conclave.modes.run_debate`. + convergence_score: The convergence score (0.0--1.0) of the round that + triggered an early stop, or ``None`` when no early stop occurred. + Higher means more stable round-over-round (more converged). """ prompt: str @@ -127,6 +136,8 @@ class CouncilResult(BaseModel): rounds: list[DebateRound] = Field(default_factory=list) adversarial: AdversarialResult | None = None cached: bool = False + converged: bool = False + convergence_score: float | None = None @property def successful_answers(self) -> list[ModelAnswer]: diff --git a/src/conclave/modes.py b/src/conclave/modes.py index f8b0d75..4ca5e90 100644 --- a/src/conclave/modes.py +++ b/src/conclave/modes.py @@ -22,6 +22,7 @@ from __future__ import annotations +from difflib import SequenceMatcher from typing import TYPE_CHECKING from . import prompts @@ -35,20 +36,36 @@ logger = get_logger("modes") -async def run_debate(council: Council, prompt: str, rounds: int = 2) -> CouncilResult: +async def run_debate( + council: Council, + prompt: str, + rounds: int = 2, + converge_threshold: float | None = None, +) -> CouncilResult: """Run a multi-round debate and return a structured :class:`CouncilResult`. Args: council: The :class:`Council` providing fan-out, config, and synthesizer. prompt: The user prompt. - rounds: Number of rounds (clamped to ``>= 1``). Round 1 is independent; - each later round shows members their peers' anonymized prior answers. + rounds: Maximum number of rounds (clamped to ``>= 1``). Round 1 is + independent; each later round shows members their peers' anonymized + prior answers. + converge_threshold: Opt-in early-stop threshold in ``[0.0, 1.0]``. When + ``None`` (default), the debate runs exactly ``rounds`` -- identical + to the historic fixed-rounds behavior. When set, after each round + ``>= 2`` the round-over-round answer stability is scored (see + :func:`_round_convergence`); if it reaches the threshold the debate + stops early. A degenerate score (e.g. no comparable answers) never + triggers a stop and never crashes -- it falls back to running the + remaining fixed rounds. Returns: A :class:`CouncilResult` with ``rounds`` (per-round answers), ``answers`` mirroring the final round, and ``synthesis`` from the final consolidation. - Survivors are tracked per round: a member that errors drops out of the - next round. Zero available members yields an empty result, not an error. + ``converged``/``convergence_score`` record whether (and at what score) an + early stop fired; the actual rounds run is ``len(rounds)``. Survivors are + tracked per round: a member that errors drops out of the next round. Zero + available members yields an empty result, not an error. """ rounds = max(1, rounds) members, skipped = council._available_members() @@ -75,8 +92,20 @@ async def run_debate(council: Council, prompt: str, rounds: int = 2) -> CouncilR answers = await council.fan_out(survivors, messages_for) result.rounds.append(DebateRound(round_number=round_no, answers=answers)) - # Survivors for the next round = members that succeeded this round. by_name = {a.name: a for a in answers} + + # Early-stop check: only from round 2 on (round 1 has no prior to compare + # against), only when opted in, and only if more rounds would otherwise + # run. A detector failure degrades to continuing the fixed rounds. + if ( + converge_threshold is not None + and round_no >= 2 + and round_no < rounds + and _should_stop(prior, by_name, converge_threshold, result, round_no) + ): + break + + # Survivors for the next round = members that succeeded this round. prior = by_name next_survivors = [(n, m) for (n, m) in survivors if by_name[n].ok] dropped = [n for (n, _m) in survivors if not by_name[n].ok] @@ -96,6 +125,77 @@ async def run_debate(council: Council, prompt: str, rounds: int = 2) -> CouncilR return result +def _should_stop( + prev: dict[str, ModelAnswer], + curr: dict[str, ModelAnswer], + threshold: float, + result: CouncilResult, + round_no: int, +) -> bool: + """Decide whether the debate has converged enough to stop after this round. + + Scores the current round against the previous round via + :func:`_round_convergence` and, when the score reaches ``threshold``, records + the early stop on ``result`` (``converged`` + ``convergence_score``) and + returns ``True``. Any unexpected failure in scoring is swallowed and treated + as "not converged" so a detector bug can never crash the debate -- it simply + degrades to running the remaining fixed rounds. + """ + try: + score = _round_convergence(prev, curr) + except Exception as exc: # noqa: BLE001 -- detector must never crash the run + logger.warning("convergence scoring failed at round %d (%s); continuing", round_no, exc) + return False + + if score is None: + # No comparable answers (degenerate input): cannot conclude convergence. + logger.info("round %d: no comparable answers for convergence; continuing", round_no) + return False + + logger.info("round %d convergence score: %.3f (threshold %.3f)", round_no, score, threshold) + if score >= threshold: + result.converged = True + result.convergence_score = score + logger.info("debate converged at round %d (score %.3f); stopping early", round_no, score) + return True + return False + + +def _round_convergence( + prev: dict[str, ModelAnswer], + curr: dict[str, ModelAnswer], +) -> float | None: + """Score round-over-round answer stability in ``[0.0, 1.0]``, or ``None``. + + The signal is **round-over-round stability**: for each member that produced a + usable answer in *both* the previous and current round, compute the + :class:`difflib.SequenceMatcher` ratio between the two answer texts (1.0 = + identical, 0.0 = nothing in common), then average across those members. A high + mean means members stopped revising -- the debate has stabilized. + + This signal is deliberately simple, deterministic, and dependency-free + (stdlib ``difflib`` only), so it is fully offline-testable and adds no heavy + dependency. It is preferred over cross-member agreement because a debate + converging means members *stop changing their answers*, which is faithfully + captured by self-stability and does not penalize a legitimate, stable + disagreement between members. + + Returns: + The mean stability ratio, or ``None`` when no member has a usable answer + in both rounds (degenerate input -- the caller treats this as "not + converged" rather than crashing or falsely stopping). + """ + ratios: list[float] = [] + for name, curr_ans in curr.items(): + prev_ans = prev.get(name) + if prev_ans is None or not prev_ans.ok or not curr_ans.ok: + continue + ratios.append(SequenceMatcher(None, prev_ans.answer or "", curr_ans.answer or "").ratio()) + if not ratios: + return None + return sum(ratios) / len(ratios) + + def _debate_messages_for( prompt: str, round_no: int, diff --git a/tests/test_cache.py b/tests/test_cache.py index 0af860e..d6b7688 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -28,6 +28,7 @@ from conclave import cache as cache_mod from conclave.config import ConclaveConfig from conclave.models import ModelAnswer +from tests.conftest import make_response @pytest.fixture @@ -316,3 +317,62 @@ def test_make_key_normalizes_whitespace(): assert cache_mod.make_key(prompt="a b\n c", **common) == cache_mod.make_key( prompt=" a b c ", **common ) + + +def test_make_key_debate_converge_threshold_differs(monkeypatch): + """A converged-config debate and a no-converge debate must NOT collide (issue #4). + + Otherwise identical inputs: only ``converge_threshold`` differs. The cache key + must differ so a converged run (which may stop early) is never served for a + fixed-rounds request, and vice versa. + """ + base = dict( + prompt="hello world", + mode="debate", + members=[("a", "x/1"), ("b", "y/2")], + synthesizer="claude", + synthesizer_model_id="anthropic/claude-sonnet-4-6", + temperature=0.7, + rounds=5, + ) + k_off = cache_mod.make_key(converge_threshold=None, **base) + k_on = cache_mod.make_key(converge_threshold=0.9, **base) + k_on2 = cache_mod.make_key(converge_threshold=0.95, **base) + assert k_off != k_on # converge on vs off -> different keys + assert k_on != k_on2 # different thresholds -> different keys + # Determinism preserved. + assert k_off == cache_mod.make_key(converge_threshold=None, **base) + + +async def test_cache_converge_vs_fixed_no_collision(cache_home, monkeypatch, patch_call_model): + """End-to-end: a converged debate and a fixed debate get distinct cache entries. + + With caching enabled, running the same prompt as a converged debate then as a + fixed (no-converge) debate must produce two separate cache files -- the second + run must not be served the first run's result. + """ + _set_keys(monkeypatch) + + def handler(model, messages, **kwargs): + system = next((m["content"] for m in messages if m.get("role") == "system"), "") + if "synthesizer concluding a multi-round" in system: + return make_response("SYNTH") + return make_response(f"identical stable answer from {model}") + + patch_call_model(handler) + + cfg = _config(cache=True) + council = Council(models=["grok", "gemini"], synthesizer="claude", config=cfg, cache=True) + + converged = await council.debate("q", rounds=5, converge_threshold=0.9) + fixed = await council.debate("q", rounds=5) # no convergence + + # The converged run stopped early; the fixed run ran all 5 rounds. If they had + # collided, the fixed run would have been served the converged (2-round) entry. + assert converged.converged is True + assert len(converged.rounds) == 2 + assert fixed.converged is False + assert len(fixed.rounds) == 5 + # Two distinct cache files exist. + files = list(cache_home.glob("*.json")) + assert len(files) == 2 diff --git a/tests/test_modes.py b/tests/test_modes.py index ab88dae..2305bb5 100644 --- a/tests/test_modes.py +++ b/tests/test_modes.py @@ -255,6 +255,210 @@ def handler(model, messages, **kwargs): assert result.synthesis == "SYNC DEBATE" +# --------------------------------------------------------------------------- # +# Debate convergence / early-stop (issue #4) +# --------------------------------------------------------------------------- # + + +def _round_no(messages) -> int: + """Infer the debate round from a member message list. + + Round 1 is a bare user prompt (no system message); later rounds carry the + debate system prompt and a 'Round N of M' marker in the user content. + """ + system = _system_text(messages) + if "debating a prompt over several rounds" not in system: + return 1 + user = next((m["content"] for m in messages if m["role"] == "user"), "") + # The user content embeds 'Round {n} of {m}.' + import re + + match = re.search(r"Round (\d+) of", user) + return int(match.group(1)) if match else 1 + + +async def test_debate_convergence_off_runs_full_rounds(monkeypatch, patch_call_model): + """Convergence OFF (default) -> debate runs exactly --rounds, unchanged. + + Even with answers that would converge, with no threshold the historic + fixed-rounds behavior must be byte-for-byte preserved: all rounds run and + the result reports no early stop. + """ + _all_keys(monkeypatch) + + def handler(model, messages, **kwargs): + system = _system_text(messages) + if "synthesizer concluding a multi-round" in system: + return make_response("SYNTH") + # Identical answer every round (would converge if checked). + return make_response("the answer is stable and unchanging across rounds") + + patch_call_model(handler) + + council = Council(models=["grok", "gemini"], synthesizer="claude", config=_config()) + result = await council.debate("q", rounds=3) # no converge_threshold + + assert len(result.rounds) == 3 + assert result.converged is False + assert result.convergence_score is None + + +async def test_debate_convergence_on_stops_early(monkeypatch, patch_call_model): + """Convergence ON + converging answers -> stops early, records score + rounds. + + Each member returns the same text in rounds 1 and 2, so round-2 stability is + 1.0 and the debate stops after round 2 instead of running the full 5 rounds. + """ + _all_keys(monkeypatch) + + rounds_seen: list[int] = [] + + def handler(model, messages, **kwargs): + system = _system_text(messages) + if "synthesizer concluding a multi-round" in system: + return make_response("SYNTH") + rounds_seen.append(_round_no(messages)) + # Same body each round -> round-over-round stability == 1.0. + return make_response(f"final stable answer from {model}") + + patch_call_model(handler) + + council = Council(models=["grok", "gemini"], synthesizer="claude", config=_config()) + result = await council.debate("q", rounds=5, converge_threshold=0.9) + + # Stopped after round 2 (the first round eligible for a convergence check). + assert len(result.rounds) == 2 + assert result.converged is True + assert result.convergence_score is not None + assert result.convergence_score >= 0.9 + # No round beyond 2 ever ran. + assert max(rounds_seen) == 2 + # Final synthesis still runs over the converged round. + assert result.synthesis == "SYNTH" + + +async def test_debate_convergence_on_never_converges_runs_full(monkeypatch, patch_call_model): + """Convergence ON + answers that never stabilize -> full --rounds, no early stop.""" + _all_keys(monkeypatch) + + def handler(model, messages, **kwargs): + system = _system_text(messages) + if "synthesizer concluding a multi-round" in system: + return make_response("SYNTH") + # A wholly different, long answer each round -> low stability ratio. + rnd = _round_no(messages) + unique = "alpha beta gamma delta " * rnd + str(rnd) * (rnd * 50) + return make_response(f"round {rnd} {model} {unique}") + + patch_call_model(handler) + + council = Council(models=["grok", "gemini"], synthesizer="claude", config=_config()) + result = await council.debate("q", rounds=3, converge_threshold=0.95) + + assert len(result.rounds) == 3 # no false early stop + assert result.converged is False + assert result.convergence_score is None + + +async def test_debate_convergence_degenerate_input_falls_back(monkeypatch, patch_call_model): + """A degenerate convergence input (no comparable answers) -> fixed rounds, no crash. + + Every member fails round 1, so it has no usable prior answer to compare round + 2 against. The detector must not crash or falsely stop; with all members + dropped the debate ends on its own, but the run completes cleanly with the + convergence fields untouched. + """ + _all_keys(monkeypatch) + + def handler(model, messages, **kwargs): + # Everyone always fails -> no usable answers in any round. + raise RuntimeError("provider down") + + patch_call_model(handler) + + council = Council(models=["grok", "gemini"], synthesizer="claude", config=_config()) + result = await council.debate("q", rounds=3, converge_threshold=0.5) + + # All failed round 1 -> debate ends after round 1 (no survivors); the + # convergence check never falsely fires and nothing crashes. + assert len(result.rounds) == 1 + assert result.converged is False + assert result.convergence_score is None + assert result.synthesis is None + + +async def test_debate_convergence_partial_degenerate_does_not_crash(monkeypatch, patch_call_model): + """Mixed usable/failed answers in a round still score without crashing. + + One member converges (stable answer) while another fails in round 2. The + detector scores only the member usable in both rounds; the run does not crash + and behaves deterministically. + """ + _all_keys(monkeypatch) + + def handler(model, messages, **kwargs): + system = _system_text(messages) + if "synthesizer concluding a multi-round" in system: + return make_response("SYNTH") + rnd = _round_no(messages) + if model == "gemini/gemini-2.5-pro" and rnd >= 2: + raise RuntimeError("gemini failed round 2") + return make_response(f"stable body from {model}") + + patch_call_model(handler) + + council = Council(models=["grok", "gemini"], synthesizer="claude", config=_config()) + result = await council.debate("q", rounds=5, converge_threshold=0.9) + + # grok stayed stable across rounds 1->2 (ratio 1.0); mean over the one + # comparable member is >= threshold, so the debate stops after round 2. + assert len(result.rounds) == 2 + assert result.converged is True + assert result.convergence_score is not None + assert result.convergence_score >= 0.9 + + +async def test_debate_convergence_defers_to_config(monkeypatch, patch_call_model): + """converge_threshold=None on the call defers to config.converge_threshold.""" + _all_keys(monkeypatch) + + def handler(model, messages, **kwargs): + if "synthesizer concluding a multi-round" in _system_text(messages): + return make_response("SYNTH") + return make_response(f"identical stable answer from {model}") + + patch_call_model(handler) + + cfg = _config() + cfg.converge_threshold = 0.9 # enable via config, not the call arg + council = Council(models=["grok", "gemini"], synthesizer="claude", config=cfg) + result = await council.debate("q", rounds=5) # no explicit threshold + + assert len(result.rounds) == 2 + assert result.converged is True + + +def test_round_convergence_helper_direct(): + """The convergence scorer: identical pairs -> 1.0, no comparable pairs -> None.""" + from conclave.models import ModelAnswer + from conclave.modes import _round_convergence + + prev = { + "a": ModelAnswer(name="a", model_id="x/1", answer="hello world"), + "b": ModelAnswer(name="b", model_id="y/2", answer="foo bar"), + } + curr_same = { + "a": ModelAnswer(name="a", model_id="x/1", answer="hello world"), + "b": ModelAnswer(name="b", model_id="y/2", answer="foo bar"), + } + assert _round_convergence(prev, curr_same) == 1.0 + + # No member has a usable answer in both rounds -> None (degenerate). + failed = {"a": ModelAnswer(name="a", model_id="x/1", error="boom")} + assert _round_convergence(prev, failed) is None + assert _round_convergence({}, {}) is None + + # --------------------------------------------------------------------------- # # Adversarial # --------------------------------------------------------------------------- #