diff --git a/README.md b/README.md index ac404af..d6c8579 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,32 @@ synthesizer: claude Then: `conclave ask "..." --council fast`. +## Result cache (optional, off by default) + +Repeated or eval runs can be served from an on-disk cache instead of re-calling +the providers. It is **off by default** and **never persists API keys** — the +cache key is a SHA-256 over the normalized prompt, the ordered council members +(friendly name + resolved model id), the mode, the synthesizer/judge identity, +and the mode params (temperature, debate `rounds`, adversarial `proposer`). No +key value or env-var name ever reaches the key or the stored payload. + +Enable it per run with `--cache` (or disable a config default with `--no-cache`): + +```bash +conclave ask "..." --council fast --cache +``` + +or set a default in `~/.conclave/config.yml`: + +```yaml +cache: true +``` + +A cache hit returns the prior `CouncilResult` with `cached: true` set and does +not touch the network. Entries live under `$XDG_CACHE_HOME/conclave` (else +`~/.cache/conclave`); a corrupt or unreadable entry is treated as a miss and +never crashes a run. + ## Test ```bash diff --git a/config.example.yml b/config.example.yml index 91f1386..f9340a1 100644 --- a/config.example.yml +++ b/config.example.yml @@ -18,3 +18,10 @@ councils: # Default synthesizer (friendly name). Overridable with `--synthesizer`. synthesizer: claude + +# Optional result cache (OFF by default). When true, an identical repeat run is +# served from an on-disk cache instead of re-calling the providers -- handy for +# repeated/eval runs. Override per invocation with `--cache` / `--no-cache`. +# The cache is keyed on (prompt, council, mode, model ids) and NEVER stores keys. +# Stored under $XDG_CACHE_HOME/conclave (else ~/.cache/conclave). +cache: false diff --git a/src/conclave/cache.py b/src/conclave/cache.py new file mode 100644 index 0000000..74edb2f --- /dev/null +++ b/src/conclave/cache.py @@ -0,0 +1,201 @@ +"""Optional on-disk result cache for council runs (off by default). + +This is the §9 #4 roadmap item: an opt-in cache keyed on +``(prompt, council, mode, model ids)`` so repeated or eval runs are cheap. It is +**off by default** and **never persists key material** -- the cache key and the +stored payload are derived solely from the normalized prompt, the ordered council +member friendly-names + resolved model ids, the run mode, the synthesizer/judge +identity, and the mode parameters that affect output. No environment variable is +read here; no key value reaches the key string or the on-disk artifact. + +Storage +======= +Entries live one-per-file under ``$XDG_CACHE_HOME/conclave`` (falling back to +``~/.cache/conclave``). Each file is named ``.json`` and holds the +JSON serialization of a :class:`conclave.models.CouncilResult` (via +``model_dump(mode="json")``), which by construction carries no secrets. + +Graceful degradation +==================== +A corrupt, unreadable, or schema-incompatible cache entry is treated as a **miss** +(logged at warning level), never an error: a bad cache file can never crash a run. +Writes that fail (e.g. a read-only cache dir) are likewise logged and swallowed -- +caching is a best-effort optimization, never a correctness dependency. + +Key-ordering choice +=================== +Member order is **preserved** (not sorted) in the cache key. For ``synthesize`` / +``raw`` the member order does not change the output, but for ``debate`` and +``adversarial`` it does: the adversarial proposer defaults to the first member and +debate assigns stable letter labels by member position. Preserving order is +therefore the conservative, always-correct choice -- two runs collide only when +they would genuinely produce equivalent results. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from pathlib import Path + +from pydantic import ValidationError + +from .logging import get_logger +from .models import CouncilResult + +logger = get_logger("cache") + +# Bumped if the cache-key composition or stored schema changes incompatibly, so +# old entries simply miss instead of being mis-served against new code. +_CACHE_VERSION = 1 + +_WHITESPACE = re.compile(r"\s+") + + +def cache_dir() -> Path: + """Return the conclave cache directory, honoring ``XDG_CACHE_HOME``. + + Falls back to ``~/.cache/conclave`` when ``XDG_CACHE_HOME`` is unset or empty. + The directory is not created here; :func:`store` creates it lazily on write. + """ + xdg = os.environ.get("XDG_CACHE_HOME", "").strip() + base = Path(xdg) if xdg else Path.home() / ".cache" + return base / "conclave" + + +def _normalize_prompt(prompt: str) -> str: + """Collapse runs of whitespace and strip ends for a stable prompt key. + + Two prompts that differ only in incidental whitespace should hit the same + cache entry; semantic content is otherwise preserved verbatim. + """ + return _WHITESPACE.sub(" ", prompt).strip() + + +def make_key( + *, + prompt: str, + mode: str, + members: list[tuple[str, str]], + synthesizer: str | None, + synthesizer_model_id: str | None, + temperature: float, + rounds: int | None = None, + proposer: str | None = None, +) -> str: + """Build the stable cache key (sha256 hex) for a council run. + + The key is a SHA-256 over a canonical JSON document of only output-affecting, + secret-free identity: + + * normalized prompt, + * run mode, + * ordered ``(friendly_name, resolved_model_id)`` member pairs (order matters -- + see module docstring), + * synthesizer/judge friendly name + resolved model id, + * temperature, and mode params (``rounds`` for debate, ``proposer`` for + adversarial) when they apply. + + Args: + prompt: The raw user prompt (normalized internally). + mode: ``"synthesize" | "raw" | "debate" | "adversarial"``. + members: Ordered ``(friendly_name, resolved_model_id)`` pairs actually run. + synthesizer: Synthesizer/judge friendly name (``None`` when not applicable). + synthesizer_model_id: Resolved synthesizer/judge model id. + temperature: Sampling temperature (affects output). + rounds: Debate round count (included only for ``debate``). + proposer: Adversarial proposer friendly name (included only for + ``adversarial``). + + Returns: + A 64-char lowercase hex SHA-256 digest. Contains zero key material. + """ + payload: dict[str, object] = { + "v": _CACHE_VERSION, + "prompt": _normalize_prompt(prompt), + "mode": mode, + # Pairs as lists so JSON round-trips; order preserved deliberately. + "members": [[name, model_id] for name, model_id in members], + "synthesizer": synthesizer, + "synthesizer_model_id": synthesizer_model_id, + "temperature": temperature, + } + if mode == "debate": + payload["rounds"] = rounds + if mode == "adversarial": + payload["proposer"] = proposer + + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _entry_path(key: str) -> Path: + """Map a cache key to its on-disk entry path.""" + return cache_dir() / f"{key}.json" + + +def load(key: str) -> CouncilResult | None: + """Return the cached :class:`CouncilResult` for ``key``, or ``None`` on miss. + + A missing file is a normal miss (silent). A present-but-unreadable or + schema-incompatible file is a degraded miss: it is logged at warning level and + treated as absent so a corrupt entry can never crash a run. + + Args: + key: The cache key from :func:`make_key`. + + Returns: + The deserialized result with ``cached=True`` set, or ``None``. + """ + try: + path = _entry_path(key) + raw = path.read_text(encoding="utf-8") + except FileNotFoundError: + return None + except OSError as exc: + logger.warning("cache read failed for key %s: %s; treating as miss", key[:12], exc) + return None + + try: + data = json.loads(raw) + result = CouncilResult.model_validate(data) + except (json.JSONDecodeError, ValidationError, TypeError) as exc: + logger.warning("corrupt cache entry %s: %s; treating as miss", path, exc) + return None + + # Mark as cache-served so consumers can distinguish a hit from a live run. + result.cached = True + return result + + +def store(key: str, result: CouncilResult) -> None: + """Persist ``result`` under ``key``, best-effort (failures are swallowed). + + The cache directory is created lazily. Any write failure (read-only dir, disk + full, serialization error) is logged at warning level and ignored -- caching + must never turn a successful run into a failure. The stored payload is + ``result.model_dump(mode="json")``, which carries no secrets. + + The ``cached`` flag is normalized to ``False`` before writing so a stored + entry reflects how it was produced (live), not how it will later be served. + + Args: + key: The cache key from :func:`make_key`. + result: The live :class:`CouncilResult` to persist. + """ + try: + path = _entry_path(key) + path.parent.mkdir(parents=True, exist_ok=True) + payload = result.model_dump(mode="json") + payload["cached"] = False + # Atomic-ish write: write to a temp sibling then replace, so a crash mid + # write never leaves a half-written (corrupt) entry behind. + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + tmp.replace(path) + except (OSError, TypeError, ValueError) as exc: + logger.warning( + "cache write failed for key %s: %s; continuing without caching", key[:12], exc + ) diff --git a/src/conclave/cli.py b/src/conclave/cli.py index 379b740..b5f4e3d 100644 --- a/src/conclave/cli.py +++ b/src/conclave/cli.py @@ -165,6 +165,15 @@ def ask( as_json: bool = typer.Option( False, "--json", help="Emit the full result as JSON instead of panels." ), + cache: bool | None = typer.Option( + None, + "--cache/--no-cache", + help=( + "Use the on-disk result cache (off by default; defers to config when " + "unset). On a hit an identical prior run is returned without calling " + "the providers. The cache never stores API keys." + ), + ), ) -> None: """Fan PROMPT out to a council and synthesize, debate, or adversarially review. @@ -190,7 +199,7 @@ def ask( err_console.print(f"[red]No council members resolved from '{council}'.[/red]") raise typer.Exit(code=2) - c = Council(models=members, synthesizer=synthesizer, config=cfg) + c = Council(models=members, synthesizer=synthesizer, config=cfg, cache=cache) if mode_lower == "debate": result = c.debate_sync(prompt, rounds=rounds) elif mode_lower == "adversarial": diff --git a/src/conclave/config.py b/src/conclave/config.py index e70690e..0815221 100644 --- a/src/conclave/config.py +++ b/src/conclave/config.py @@ -62,12 +62,17 @@ class ConclaveConfig(BaseModel): councils: named lists of friendly names. synthesizer: friendly name of the default synthesizer model. endpoints: prefix -> custom OpenAI-compatible endpoint declaration. + cache: opt-in result cache (off by default). When ``True`` an identical + repeat run is served from the on-disk cache (see :mod:`conclave.cache`) + instead of re-calling the providers. The cache never persists keys. + A ``--cache/--no-cache`` CLI flag overrides this per invocation. """ models: dict[str, str] = Field(default_factory=dict) councils: dict[str, list[str]] = Field(default_factory=dict) synthesizer: str = DEFAULT_SYNTHESIZER endpoints: dict[str, CustomEndpoint] = Field(default_factory=dict) + cache: bool = False def resolve_model_id(self, name: str) -> str: """Map a friendly name to a provider-prefixed model id. @@ -183,9 +188,13 @@ def _load_config_uncached(path: Path) -> ConclaveConfig: if isinstance(spec, dict) } + # Off by default; only an explicit truthy ``cache: true`` in the file enables it. + cache = bool(raw.get("cache", False)) + return ConclaveConfig( models=merged_models, councils=councils, synthesizer=synthesizer, endpoints=endpoints, + cache=cache, ) diff --git a/src/conclave/council.py b/src/conclave/council.py index da6beab..d5b2700 100644 --- a/src/conclave/council.py +++ b/src/conclave/council.py @@ -13,8 +13,9 @@ from __future__ import annotations import asyncio -from collections.abc import Callable +from collections.abc import Awaitable, Callable +from . import cache as cache_mod from . import transport from .config import ConclaveConfig, load_config from .logging import get_logger @@ -48,6 +49,12 @@ class Council: config: Pre-loaded config; if ``None``, loaded from disk + defaults. temperature: Sampling temperature for member calls. timeout: Per-call timeout in seconds. + cache: Opt-in result cache. ``None`` (default) defers to + ``config.cache`` (off unless enabled in ``~/.conclave/config.yml``); + ``True``/``False`` overrides it for this council. When enabled, an + identical repeat run is served from the on-disk cache instead of + re-calling the providers. The cache never persists API keys -- + see :mod:`conclave.cache`. Example: >>> council = Council(models=["grok", "perplexity"], synthesizer="claude") @@ -62,12 +69,15 @@ def __init__( config: ConclaveConfig | None = None, temperature: float = 0.7, timeout: float = 120.0, + cache: bool | None = None, ) -> None: self.config = config or load_config() self.requested_models = list(models) self.synthesizer = synthesizer or self.config.synthesizer self.temperature = temperature self.timeout = timeout + # Explicit override wins; otherwise defer to config (off by default). + self.cache_enabled = self.config.cache if cache is None else cache def _available_members(self) -> tuple[list[tuple[str, str]], list[str]]: """Partition requested members into (available, skipped-for-no-key). @@ -88,6 +98,64 @@ def _available_members(self) -> tuple[list[tuple[str, str]], list[str]]: skipped.append(name) return members, skipped + def _cache_key( + self, + prompt: str, + mode: str, + *, + rounds: int | None = None, + proposer: str | None = None, + ) -> str: + """Build the cache key for a run from the resolved, secret-free identity. + + Uses the *resolved* member ids and the synthesizer/judge identity so two + runs collide only when they would genuinely produce equivalent output. + Members that would be skipped for a missing key are excluded -- a cache + entry reflects the council that actually ran, so a key reappearing later + produces the same membership. No environment value is read here. + """ + members, _skipped = self._available_members() + synth_id = self.config.resolve_model_id(self.synthesizer) + return cache_mod.make_key( + prompt=prompt, + mode=mode, + members=members, + synthesizer=self.synthesizer, + synthesizer_model_id=synth_id, + temperature=self.temperature, + rounds=rounds, + proposer=proposer, + ) + + async def _cached_run( + self, + prompt: str, + mode: str, + run: Callable[[], Awaitable[CouncilResult]], + *, + rounds: int | None = None, + proposer: str | None = None, + ) -> CouncilResult: + """Serve ``run`` from the result cache when caching is enabled. + + On a hit the cached :class:`CouncilResult` is returned with ``cached=True`` + and the providers are not called. On a miss (or when caching is off) the + live ``run`` executes; a successful live run is stored best-effort. Cache + read/write failures never propagate -- they degrade to a normal live run. + """ + if not self.cache_enabled: + return await run() + + key = self._cache_key(prompt, mode, rounds=rounds, proposer=proposer) + hit = cache_mod.load(key) + if hit is not None: + logger.info("cache hit for %s run (%s)", mode, key[:12]) + return hit + + result = await run() + cache_mod.store(key, result) + return result + async def fan_out( self, members: list[tuple[str, str]], @@ -141,6 +209,9 @@ async def fan_out( async def ask(self, prompt: str, synthesize: bool = True) -> CouncilResult: """Run the council asynchronously. + When the result cache is enabled, an identical prior run is returned from + cache (``CouncilResult.cached is True``) without calling the providers. + Args: prompt: The user prompt to fan out. synthesize: When True (default), merge answers via the synthesizer. @@ -150,6 +221,13 @@ async def ask(self, prompt: str, synthesize: bool = True) -> CouncilResult: synthesis. A run with zero available members returns an empty-answer result rather than raising. """ + mode = "synthesize" if synthesize else "raw" + return await self._cached_run( + prompt, mode, lambda: self._ask_uncached(prompt, synthesize=synthesize) + ) + + async def _ask_uncached(self, prompt: str, synthesize: bool = True) -> CouncilResult: + """The live ask path (no cache consultation). See :meth:`ask`.""" members, skipped = self._available_members() result = CouncilResult( prompt=prompt, @@ -234,19 +312,31 @@ async def debate(self, prompt: str, rounds: int = 2) -> CouncilResult: Round 1 is an independent fan-out; rounds 2..N show each member its peers' anonymized prior answers; the synthesizer consolidates survivors. + Cache-served when caching is enabled (``rounds`` is part of the key). """ from .modes import run_debate - return await run_debate(self, prompt, rounds=rounds) + return await self._cached_run( + prompt, + "debate", + lambda: run_debate(self, prompt, rounds=rounds), + rounds=rounds, + ) async def adversarial(self, prompt: str, proposer: str | None = None) -> CouncilResult: """Run propose -> refute -> verdict. See :func:`conclave.modes.run_adversarial`. ``proposer`` (friendly name) defaults to the first requested member. + Cache-served when caching is enabled (``proposer`` is part of the key). """ from .modes import run_adversarial - return await run_adversarial(self, prompt, proposer=proposer) + return await self._cached_run( + prompt, + "adversarial", + lambda: run_adversarial(self, prompt, proposer=proposer), + proposer=proposer, + ) async def aclose(self) -> None: """Close the shared pooled HTTP client. diff --git a/src/conclave/models.py b/src/conclave/models.py index 4abbd66..ef57e4e 100644 --- a/src/conclave/models.py +++ b/src/conclave/models.py @@ -110,6 +110,10 @@ class CouncilResult(BaseModel): rounds: Per-round answers for ``debate`` mode (empty otherwise). adversarial: The proposal/critique/verdict structure for ``adversarial`` mode (``None`` otherwise). + cached: ``True`` when this result was served from the optional result + cache rather than produced by a live run. ``False`` for every live + run and for freshly stored entries. Lets a consumer detect a cache + hit without re-running. See :mod:`conclave.cache`. """ prompt: str @@ -122,6 +126,7 @@ class CouncilResult(BaseModel): skipped: list[str] = Field(default_factory=list) rounds: list[DebateRound] = Field(default_factory=list) adversarial: AdversarialResult | None = None + cached: bool = False @property def successful_answers(self) -> list[ModelAnswer]: diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..0af860e --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,318 @@ +"""Tests for the optional result cache (issue #6 / PDD §9 #4). + +All tests run offline. The cache is redirected to a per-test ``tmp_path`` via the +``XDG_CACHE_HOME`` env var so the real ``~/.cache`` is never touched and each test +starts empty. + +Pinned behaviors: + +* **Off by default** -- two identical runs both execute and nothing is written. +* **On -> miss then hit** -- the second identical run does NOT call the providers + (asserted by a call counter on the patched call path) and is flagged ``cached``. +* **Key sensitivity** -- changing prompt / council / mode / model id misses. +* **Security** -- no API key VALUE appears in the cache key or the persisted + on-disk payload, even with a fake key env var set. +* **Graceful degradation** -- a corrupt entry is a miss, the run completes, no + crash, and the corrupt entry is overwritten with a valid one. +""" + +from __future__ import annotations + +import asyncio +import json + +import pytest + +import conclave.council as council_mod +from conclave import Council +from conclave import cache as cache_mod +from conclave.config import ConclaveConfig +from conclave.models import ModelAnswer + + +@pytest.fixture +def cache_home(tmp_path, monkeypatch): + """Redirect the cache dir into tmp_path and return it.""" + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + return tmp_path / "conclave" + + +def _config(cache: bool = False) -> ConclaveConfig: + """A deterministic config independent of any on-disk ~/.conclave file.""" + return ConclaveConfig( + models={ + "grok": "xai/grok-4.3", + "gemini": "gemini/gemini-2.5-pro", + "claude": "anthropic/claude-sonnet-4-6", + "perplexity": "perplexity/sonar-pro", + }, + councils={"default": ["grok", "gemini", "claude", "perplexity"]}, + synthesizer="claude", + cache=cache, + ) + + +def _set_keys(monkeypatch) -> None: + """Set every provider key to a dummy non-empty value.""" + for var in ("XAI_API_KEY", "GEMINI_API_KEY", "ANTHROPIC_API_KEY", "PERPLEXITY_API_KEY"): + monkeypatch.setenv(var, "dummy-key") + + +@pytest.fixture +def counting_call_model(monkeypatch): + """Patch ``conclave.council.call_model`` with a call-counting fake. + + Returns the mutable counter dict ``{"n": int}``. Every member + synthesizer + call increments it, so a cache HIT is provable by the counter not advancing. + """ + counter = {"n": 0} + + async def fake_call_model(name, model_id, messages, *, temperature=0.7, timeout=120.0): + counter["n"] += 1 + await asyncio.sleep(0) + # Synthesizer call is the 2-message (system+user) one. + text = "MERGED" if len(messages) == 2 else f"answer from {model_id}" + return ModelAnswer(name=name, model_id=model_id, answer=text) + + monkeypatch.setattr(council_mod, "call_model", fake_call_model) + return counter + + +# --------------------------------------------------------------------------- # +# Off by default +# --------------------------------------------------------------------------- # + + +async def test_cache_off_by_default_no_file_no_hit(monkeypatch, counting_call_model, cache_home): + """Cache OFF (default): two identical runs both execute; nothing is written.""" + _set_keys(monkeypatch) + council = Council(models=["grok", "perplexity"], synthesizer="claude", config=_config()) + assert council.cache_enabled is False + + r1 = await council.ask("what is 2+2?") + after_first = counting_call_model["n"] + r2 = await council.ask("what is 2+2?") + + assert r1.cached is False + assert r2.cached is False + # Second run executed again -> counter advanced. + assert counting_call_model["n"] == after_first * 2 + # No cache artifacts written at all. + assert not cache_home.exists() or not list(cache_home.glob("*.json")) + + +# --------------------------------------------------------------------------- # +# On: miss then hit +# --------------------------------------------------------------------------- # + + +async def test_cache_on_first_miss_then_hit(monkeypatch, counting_call_model, cache_home): + """Cache ON: first run populates; second identical run is a hit, no provider calls.""" + _set_keys(monkeypatch) + council = Council( + models=["grok", "perplexity"], synthesizer="claude", config=_config(), cache=True + ) + + r1 = await council.ask("what is 2+2?") + calls_after_first = counting_call_model["n"] + assert calls_after_first > 0 + assert r1.cached is False + # Exactly one entry written. + entries = list(cache_home.glob("*.json")) + assert len(entries) == 1 + + r2 = await council.ask("what is 2+2?") + # The hit must NOT call the providers again. + assert counting_call_model["n"] == calls_after_first + assert r2.cached is True + # Same content served. + assert r2.synthesis == r1.synthesis + assert [a.answer for a in r2.answers] == [a.answer for a in r1.answers] + + +async def test_cache_on_via_config_flag(monkeypatch, counting_call_model, cache_home): + """Cache enabled through config.cache (no explicit Council arg) also hits.""" + _set_keys(monkeypatch) + council = Council(models=["grok"], synthesizer="claude", config=_config(cache=True)) + assert council.cache_enabled is True + + await council.ask("hello") + n = counting_call_model["n"] + r2 = await council.ask("hello") + assert counting_call_model["n"] == n + assert r2.cached is True + + +async def test_explicit_no_cache_overrides_config(monkeypatch, counting_call_model, cache_home): + """An explicit cache=False overrides config.cache=True (the --no-cache path).""" + _set_keys(monkeypatch) + council = Council(models=["grok"], config=_config(cache=True), cache=False) + assert council.cache_enabled is False + await council.ask("hello") + n = counting_call_model["n"] + await council.ask("hello") + assert counting_call_model["n"] == n * 2 # ran again, no hit + + +# --------------------------------------------------------------------------- # +# Key sensitivity +# --------------------------------------------------------------------------- # + + +async def test_changing_prompt_misses(monkeypatch, counting_call_model, cache_home): + _set_keys(monkeypatch) + council = Council(models=["grok"], synthesizer="claude", config=_config(), cache=True) + await council.ask("prompt one") + n = counting_call_model["n"] + r = await council.ask("prompt two") + assert counting_call_model["n"] > n + assert r.cached is False + + +async def test_changing_council_membership_misses(monkeypatch, counting_call_model, cache_home): + _set_keys(monkeypatch) + c1 = Council(models=["grok"], synthesizer="claude", config=_config(), cache=True) + await c1.ask("same prompt") + n = counting_call_model["n"] + c2 = Council(models=["grok", "perplexity"], synthesizer="claude", config=_config(), cache=True) + r = await c2.ask("same prompt") + assert counting_call_model["n"] > n + assert r.cached is False + + +async def test_changing_mode_misses(monkeypatch, counting_call_model, cache_home): + _set_keys(monkeypatch) + council = Council(models=["grok"], synthesizer="claude", config=_config(), cache=True) + await council.ask("same prompt", synthesize=True) + n = counting_call_model["n"] + r = await council.ask("same prompt", synthesize=False) # raw mode -> different key + assert counting_call_model["n"] > n + assert r.cached is False + + +async def test_changing_model_id_misses(monkeypatch, counting_call_model, cache_home): + """Same friendly name but a different resolved model id -> different key.""" + _set_keys(monkeypatch) + cfg_a = _config() + await Council(models=["grok"], synthesizer="claude", config=cfg_a, cache=True).ask("p") + n = counting_call_model["n"] + cfg_b = _config() + cfg_b.models["grok"] = "xai/grok-4.3-mini" # different resolved id + r = await Council(models=["grok"], synthesizer="claude", config=cfg_b, cache=True).ask("p") + assert counting_call_model["n"] > n + assert r.cached is False + + +# --------------------------------------------------------------------------- # +# Security: no key material on disk or in the key +# --------------------------------------------------------------------------- # + + +async def test_no_key_value_in_cache_key_or_payload(monkeypatch, counting_call_model, cache_home): + """A fake key VALUE must not appear in the cache key string or persisted file.""" + secret = "sk-CONCLAVE-SUPER-SECRET-KEY-VALUE-9f8e7d6c" + for var in ("XAI_API_KEY", "ANTHROPIC_API_KEY"): + monkeypatch.setenv(var, secret) + + council = Council(models=["grok"], synthesizer="claude", config=_config(), cache=True) + + # The cache key itself must contain zero key material. + key = council._cache_key("audit prompt", "synthesize") + assert secret not in key + + await council.ask("audit prompt") + + entries = list(cache_home.glob("*.json")) + assert len(entries) == 1 + blob = entries[0].read_text(encoding="utf-8") + assert secret not in blob + # Also sanity-check the env var NAMES are absent from the stored payload. + assert "XAI_API_KEY" not in blob + assert "ANTHROPIC_API_KEY" not in blob + # And the filename (the key) carries no secret. + assert secret not in entries[0].name + + +# --------------------------------------------------------------------------- # +# Graceful degradation +# --------------------------------------------------------------------------- # + + +async def test_corrupt_entry_is_miss_no_crash(monkeypatch, counting_call_model, cache_home): + """A corrupt cache file is treated as a miss; the run completes and rewrites it.""" + _set_keys(monkeypatch) + council = Council(models=["grok"], synthesizer="claude", config=_config(), cache=True) + + # Pre-write a corrupt entry at the exact key the run will use. + key = council._cache_key("q", "synthesize") + cache_home.mkdir(parents=True, exist_ok=True) + (cache_home / f"{key}.json").write_text("{ this is not valid json", encoding="utf-8") + + r = await council.ask("q") # must not raise + assert r.cached is False # corrupt entry was a miss, ran live + assert counting_call_model["n"] > 0 + # The corrupt entry was overwritten with a valid one -> next run hits. + n = counting_call_model["n"] + r2 = await council.ask("q") + assert r2.cached is True + assert counting_call_model["n"] == n + + +async def test_unreadable_payload_schema_is_miss(monkeypatch, cache_home): + """A JSON file that is not a valid CouncilResult is a miss, not a crash.""" + cache_home.mkdir(parents=True, exist_ok=True) + key = "deadbeef" + (cache_home / f"{key}.json").write_text(json.dumps({"not": "a result"}), encoding="utf-8") + assert cache_mod.load(key) is None + + +async def test_write_failure_does_not_crash_run(monkeypatch, counting_call_model, cache_home): + """A failing cache write degrades to a normal live run (no exception).""" + _set_keys(monkeypatch) + + # Simulate a low-level failure in path resolution: both load() and store() + # must swallow it and degrade to a normal live run with no caching. + def raise_oserror(key): + raise OSError("simulated cache path failure") + + monkeypatch.setattr(cache_mod, "_entry_path", raise_oserror) + + council = Council(models=["grok"], synthesizer="claude", config=_config(), cache=True) + # Even though path resolution fails inside store(), the run completes. + r = await council.ask("q") + assert r.cached is False + assert counting_call_model["n"] > 0 + + +# --------------------------------------------------------------------------- # +# Cache key direct unit checks +# --------------------------------------------------------------------------- # + + +def test_make_key_is_deterministic_and_order_sensitive(): + base = dict( + prompt="hello world", + mode="synthesize", + synthesizer="claude", + synthesizer_model_id="anthropic/claude-sonnet-4-6", + temperature=0.7, + ) + k1 = cache_mod.make_key(members=[("a", "x/1"), ("b", "y/2")], **base) + k2 = cache_mod.make_key(members=[("a", "x/1"), ("b", "y/2")], **base) + k3 = cache_mod.make_key(members=[("b", "y/2"), ("a", "x/1")], **base) + assert k1 == k2 # deterministic + assert k1 != k3 # member order matters (debate/adversarial ordering) + assert len(k1) == 64 # sha256 hex + + +def test_make_key_normalizes_whitespace(): + common = dict( + mode="raw", + members=[("a", "x/1")], + synthesizer=None, + synthesizer_model_id=None, + temperature=0.7, + ) + assert cache_mod.make_key(prompt="a b\n c", **common) == cache_mod.make_key( + prompt=" a b c ", **common + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9a89356..ccf745c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -267,6 +267,58 @@ def handler(model, messages, **kwargs): assert "gemini" in result.output +def test_cache_flag_serves_second_run_from_cache(monkeypatch, patch_cli_config, tmp_path): + """`--cache` makes a second identical CLI run a hit (providers not re-called).""" + import conclave.council as council_mod + from conclave.models import ModelAnswer + + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + _all_keys(monkeypatch) + + counter = {"n": 0} + + async def fake_call_model(name, model_id, messages, *, temperature=0.7, timeout=120.0): + counter["n"] += 1 + return ModelAnswer(name=name, model_id=model_id, answer=f"ans-{model_id}") + + monkeypatch.setattr(council_mod, "call_model", fake_call_model) + + args = ["ask", "what is 2+2?", "--council", "grok", "--mode", "raw", "--cache", "--json"] + first = runner.invoke(cli.app, args) + assert first.exit_code == 0 + n_after_first = counter["n"] + assert n_after_first > 0 + assert json.loads(first.stdout)["cached"] is False + + second = runner.invoke(cli.app, args) + assert second.exit_code == 0 + assert counter["n"] == n_after_first # no new provider calls -> served from cache + assert json.loads(second.stdout)["cached"] is True + + +def test_no_cache_flag_runs_live_each_time(monkeypatch, patch_cli_config, tmp_path): + """`--no-cache` forces a live run even if config enabled caching.""" + import conclave.council as council_mod + from conclave.models import ModelAnswer + + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path)) + _all_keys(monkeypatch) + + counter = {"n": 0} + + async def fake_call_model(name, model_id, messages, *, temperature=0.7, timeout=120.0): + counter["n"] += 1 + return ModelAnswer(name=name, model_id=model_id, answer="ans") + + monkeypatch.setattr(council_mod, "call_model", fake_call_model) + + args = ["ask", "p", "--council", "grok", "--mode", "raw", "--no-cache", "--json"] + runner.invoke(cli.app, args) + n = counter["n"] + runner.invoke(cli.app, args) + assert counter["n"] == n * 2 # ran live both times + + def test_providers_command_lists_keys_without_values(monkeypatch, patch_cli_config): """`conclave providers` prints a table marking present/absent keys, no values.""" monkeypatch.setenv("XAI_API_KEY", "super-secret-value")