diff --git a/.env.example b/.env.example index 1641cf6f..bebea1ed 100644 --- a/.env.example +++ b/.env.example @@ -24,3 +24,7 @@ OPENAI_API_KEY="sk-..." # LOG_LEVEL="INFO" # DEBUG dumps effective upstream sampling params + extra_body per request # LITELLM_DEBUG_HTTP="false" # verbose: dump full litellm kwargs/response, enable litellm.set_verbose + httpx/httpcore DEBUG. Use with LOG_LEVEL=DEBUG. + +# PROXY_DEBUG_CACHE_DUMP="false" # debug: write outgoing payloads to $cwd/.claude-code-proxy/prompts/ when an outgoing isn't a prefix extension of a prior one (cache-busting hot spot finder). Default: false. + +# PROXY_DEBUG_INBOUND_DUMP="false" # debug: write raw Anthropic request (pre-conversion) to $cwd/.claude-code-proxy/anthropic-prompts/ on every request. Default: false. diff --git a/CLAUDE.md b/CLAUDE.md index 47aae4c4..e4ecefb4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -112,6 +112,17 @@ Logs go to stderr with a single timestamped format. `log_request` prints one lin Follow the convention: add a unit test by writing a `test_foo` function; it'll be picked up automatically. Use `assert` statements — failures are caught and printed with the function name. +## Code review + +After every `/code-review` invocation on a feature branch, run a **findings triage** before acting on fixes: + +- **Scope check** — for each finding, determine whether it belongs to the current branch's commits (`fix/...`, `feat/...`) or was already present on `main` before the branch diverged. Use `git log main..HEAD --stat` and `git merge-base HEAD main` to draw the boundary. +- **Relevance check** — if a finding predates the branch, it is out of scope for *this* review. Note it in the triage summary so it isn't forgotten, but do not bundle the fix into the branch's commits (keep diffs reviewable; queue the finding separately). +- **Verification** — re-read the cited file:line against the current `HEAD`, since file:line anchors drift across commits. Findings citing a line that no longer contains the alleged code are stale and should be discarded. +- **Outcome** — report each finding as one of: `fix in branch`, `pre-existing → file separate ticket`, `stale → discard`. Only act on the first category. + +The triage happens once per `/code-review` invocation, before any code changes — not on every finding individually. + ## Files to know - `server.py` — everything (proxy, models, translation, streaming, config loader) diff --git a/README.md b/README.md index c59ccbe6..ab556f00 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,22 @@ Merge chain is `[global] → [bucket] → [tier]` (later wins per leaf). Samplin Pick the knobs your backend actually understands — don't mix `reasoning_effort` (OpenAI o-series), `chat_template_kwargs` (llama.cpp), or Anthropic-native `thinking` in one section. They belong to different backends. +### Anthropic-shaped field translations + +Claude Code sends a few Anthropic-native fields that don't map 1:1 to OpenAI Chat Completions. The proxy translates them so you don't have to round-trip via config: + +| Anthropic field | Where it lands on the wire | Notes | +|---------------------------|-------------------------------------|-------| +| `thinking` | `extra_body.thinking` (body-only) | Anthropic-compatible backends like MiniMax-M3 read this verbatim. Body-only because the openai SDK rejects unknown top-level kwargs; the key still reaches the JSON body via litellm's `extra_body` lift (see `openai_like/chat/handler.py:241,258`). | +| `output_config.effort` | top-level `reasoning_effort` | OpenAI's reasoning-effort axis (o-series, etc.). Only translated when `effort` is set; other `output_config` keys are ignored. | +| `metadata`, `context_management` | dropped | Anthropic-only; not forwarded. | + +Three more rules apply when these fields overlap with `[tier].extra_body`: + +- `thinking`: deep-merged with `[tier].extra_body.thinking` — config wins per leaf (e.g. config can flip `type` from `adaptive` to `disabled` while the client's `budget_tokens` survives). +- `output_config.effort`: translated first, then `[tier].extra_body.reasoning_effort` (if any) overrides it. +- The merged key list is published as `allowed_openai_params` at both top-level and inside `extra_body` so the litellm hop and any cascade proxy forward these vendor keys verbatim instead of filtering them. + ### Lookup order For each setting, the proxy uses the first non-empty value from this list: @@ -131,7 +147,7 @@ Env wins so `docker run -e KEY=VAL` and `docker-compose.yml: environment:` overr ### Per-tier merge semantics - **Model selection**: per tier, the resolver walks `{TIER}_MODEL` env → `{BIG|SMALL}_MODEL` env → `[tier].model` → `[bucket].model` → `[global].model` → built-in default. First non-empty wins. `[global].model` is the catch-all for any model — including unmapped ones (tier=None). -- **extra_body merge chain**: `[global] → [bucket] → [tier]` (haiku → `small`, others → `big`). Each layer deep-merges; later wins per leaf. Keys are lifted to top-level kwargs on the upstream call. The merged key list is published as `allowed_openai_params` (top-level + inside `extra_body`) so the litellm hop and any cascade proxy forward vendor keys (`chat_template_kwargs`, `cache_prompt`, `n_predict`, `reasoning_effort`, …) instead of dropping them. +- **extra_body merge chain**: `[global] → [bucket] → [tier]` (haiku → `small`, others → `big`). Each layer deep-merges; later wins per leaf. Keys are lifted to top-level kwargs on the upstream call, except `thinking` and any other Anthropic-only fields that the openai SDK rejects at signature time — those stay in `extra_body` and reach the JSON body via litellm's cascade lift. The merged key list is published as `allowed_openai_params` (top-level + inside `extra_body`) so the litellm hop and any cascade proxy forward vendor keys (`chat_template_kwargs`, `cache_prompt`, `n_predict`, `reasoning_effort`, …) instead of dropping them. - **Sampling / reasoning / vendor fields** all live inside `[tier].extra_body` (and `[global].extra_body` / `[bucket].extra_body`). There is no per-key whitelist — pass any top-level key the upstream OpenAI Chat Completions API (or your compatible backend) accepts: `temperature`, `top_p`, `top_k`, `stop`, `seed`, `max_completion_tokens`, `reasoning_effort`, `chat_template_kwargs`, `cache_prompt`, `n_predict`, … - **Conflict resolution**: when both a config layer (`[global]` / `[bucket]` / `[tier]`) and the client request set the same key (whether via Pydantic sampling fields or a request-level `extra_body`), **config wins** per leaf. - **No defaults applied**: when neither config nor the request sets a key, it is **omitted from the upstream call** (we don't auto-apply Anthropic defaults like `temperature=1.0`). @@ -158,6 +174,46 @@ extra_body = { temperature = 0.3, cache_prompt = true, n_predict = 4096, chat_te Inspect upstream logs (or use `mitmproxy`) to confirm `cache_prompt`, `chat_template_kwargs`, etc. land in the body. For offline checks, set `LOG_LEVEL=DEBUG` — the proxy logs the effective `extra_body` per request (sourced from request or `[tier]` config). +### Cache-busting diagnostics (`PROXY_DEBUG_CACHE_DUMP=true`) + +Off by default. When enabled, every outgoing payload is matched against a rolling window of prior ones — a request whose outgoing starts with a prior one is a `prefix_hit` (the desired case); anything that crosses the 0.6 fuzzy threshold but isn't a structural prefix is `fuzzy_match` (suspect a cache-busting edit). + +Artifacts land in `$cwd/.claude-code-proxy/prompts/`: + +- `----new.json` — the current outgoing payload +- `----old.json` — the prior payload it matched against + +Compare them with your diff tool of choice (the system prompt is one long +string, so `diff -u` won't be useful — a side-by-side viewer or a script +that splits on `\n\n` works better). Adjust `[[prompt_remap]]` to canonicalise +whichever side drifted. + +```bash +PROXY_DEBUG_CACHE_DUMP=true uv run uvicorn server:app +``` + +### Inbound request dumps (`PROXY_DEBUG_INBOUND_DUMP=true`) + +Off by default. When enabled, every Anthropic request Claude Code sends is +written verbatim (before conversion to OpenAI shape) to +`$cwd/.claude-code-proxy/anthropic-prompts/--inbound.json`. + +Use this when you need to see the raw Anthropic payload — in particular: + +- Where Claude Code places its `` injections across turns + (top-level `system` field vs in-band `role="system"` messages vs user content) +- The exact shape of the `system` field (`str` vs `list[ContentBlock]`) +- Whether the conversation history is growing or staying stable + +This complements `PROXY_DEBUG_CACHE_DUMP` (which captures post-conversion +outgoing payloads to `.claude-code-proxy/prompts/`). Inbound dumps let you +inspect what the *client* sends; cache dumps let you inspect what +*upstream* receives. + +```bash +PROXY_DEBUG_INBOUND_DUMP=true uv run uvicorn server:app +``` + ### System-prompt rewrites (`[[prompt_remap]]`) Two things at once: @@ -167,7 +223,7 @@ Two things at once: ```toml [[prompt_remap]] -match = "The TodoWrite tool hasn't been used recently.*?ignore if not applicable\\.\\n+" +match = "The (?:TodoWrite tool hasn't|task tools haven't) been used recently.*?ignore if not applicable\\.\\n+" replacement = "" ``` diff --git a/bugtracker.md b/bugtracker.md index d54511c7..8c30b9a8 100644 --- a/bugtracker.md +++ b/bugtracker.md @@ -9,23 +9,45 @@ surrounding code changes. ## Active -### Streaming resilience +### Security / proxy invariants + +#### `extra_body` spread doesn't filter protected keys — `server.py:1478` + +- **Severity**: medium — operator or client can override the proxy's own + `model`, `messages`, `stream`, `tools` keys via `request.extra_body` or + `[tier].extra_body`. The WARNING log at server.py:1464 reads "ignoring + protected key" but the value still reaches the JSON body upstream via + litellm's `openai_like/chat/handler.py:241,258` spread path. +- **Where**: `_apply_merged_extra_body` does + `litellm_request["extra_body"] = {"allowed_openai_params": keys, **merged_extra}`. + The loop at server.py:1462-1470 correctly skips protected keys for the + top-level kwarg lift, but the `**merged_extra` spread below re-includes + them in `extra_body`, which is forwarded verbatim into the wire body. +- **Repro**: `[sonnet].extra_body.model = "evil-model"` → + `WARNING ignoring protected key in extra_body: model` is logged, but + `out["extra_body"]["model"] == "evil-model"` and upstream receives a + second `model` field alongside the legitimate `out["model"]`. Same + hazard for `stream=true` (silently toggles streaming against the + route's expectation) and `messages=...` (replaces the proxy-built + messages list). +- **Why caught now**: existing test `test_extra_body_protected_keys_blocked` + only asserts on top-level kwargs (`out["model"]`, `out["messages"]`, + `out["stream"]`); it doesn't check `out["extra_body"]` so the bypass + slipped through. Reviewer-verified via runtime execution of + `_prepare_litellm_request` against a malicious config. +- **Suggested fix**: filter `merged_extra` before the spread — + `safe_extra = {k: v for k, v in merged_extra.items() if k not in _PROTECTED_KEYS}` + — and use `safe_extra` (not `merged_extra`) in the `extra_body` dict. + Extend `test_extra_body_protected_keys_blocked` to assert + `"model" not in out["extra_body"]`, etc. +- **Source**: introduced on `fix/bugs` by commit `d8f953e feat(server): + translate output_config.effort + pass thinking through` (the spread + was added so body-only keys like `thinking` could reach the JSON body + — but the spread didn't filter `_PROTECTED_KEYS`). Pre-merge-base, on + `main`, `extra_body` only held `{"allowed_openai_params": keys}` — no + spread, no bypass. -#### `end_turn` hardcoded when upstream omits finish_reason — `server.py:1635` - -- **Severity**: high — silently drops tool_use when upstream closes the - stream before sending `finish_reason`. -- **Where**: `_stream_epilogue` always calls `_SseFormatter.finish("end_turn", …)`, - even when a `tool_use` block is mid-emission. The Anthropic SDK treats - `end_turn` as "no pending work" and never asks the user for tool results. -- **Repro**: tool_use stream where upstream emits two valid tool_use blocks, - then closes without `finish_reason`. -- **Suggested fix**: track whether any `tool_use` block was emitted in - `_StreamState`; pick `_to_anthropic_stop_reason("tool_use")` when true, - `end_turn` otherwise. Requires the in-flight tool_use block to remain - open through the finish event (Anthropic SSE expects - `content_block_stop` before `message_delta`). -- **Source**: pre-existing in `main`. +### Streaming resilience #### `_log_request` emits STATUS_OK before upstream call — `server.py:1907` diff --git a/config.toml.example b/config.toml.example index 6414ceea..947d8b9d 100644 --- a/config.toml.example +++ b/config.toml.example @@ -100,9 +100,10 @@ # rewritten text. [[prompt_remap]] -# Claude Code periodically injects a TodoWrite reminder into the system -# prompt; without stripping it, every flip state is a cache miss and the +# Claude Code periodically injects a reminder into the system prompt (one +# variant mentions TodoWrite, another mentions task tools / TaskCreate / +# TaskUpdate). Without stripping, every flip state is a cache miss and the # whole prompt is re-processed. -match = "The TodoWrite tool hasn't been used recently.*?ignore if not applicable\\.\\n+" +match = "The (?:TodoWrite tool hasn't|task tools haven't) been used recently.*?ignore if not applicable\\.\\n+" replacement = "" diff --git a/pyproject.toml b/pyproject.toml index ecc01b8d..97e1eabe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,6 +121,7 @@ known-first-party = ["server", "tests"] "magic-value-comparison", # tests assert against literal sampling temps/byte sizes/HTTP codes — naming each one adds ceremony without clarity "D", # tests.py has its own module docstring already; per-test docstrings are noise "T20", # integration runner prints scenarios + pass/fail lines + "S108", # /tmp/... in test fixture strings isn't a filesystem op ] [tool.ty.environment] diff --git a/server.py b/server.py index 830ee8ca..e60548ed 100644 --- a/server.py +++ b/server.py @@ -4,6 +4,8 @@ LiteLLM and converts the response back. Single FastAPI app, single code path. """ +import difflib +import itertools import json import logging import os @@ -13,7 +15,7 @@ import time import tomllib import uuid -from collections.abc import AsyncGenerator, AsyncIterator, Iterator +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Iterator from contextlib import asynccontextmanager from copy import deepcopy from dataclasses import dataclass @@ -21,7 +23,7 @@ from typing import Any, Literal, cast from dotenv import load_dotenv -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Request, Response from fastapi.responses import StreamingResponse from pydantic import BaseModel, field_validator, model_validator @@ -68,6 +70,15 @@ def _litellm_debug_http_enabled() -> bool: return _str_to_bool(os.environ.get("LITELLM_DEBUG_HTTP"), default=False) +def _debug_cache_dump_enabled() -> bool: + """PROXY_DEBUG_CACHE_DUMP=true records outgoing payloads to $cwd/.claude-code-proxy/prompts/. + + Matches prefix equivalence to surface cache-busting hot spots. + Off by default; each request writes 0-2 files when enabled. + """ + return _str_to_bool(os.environ.get("PROXY_DEBUG_CACHE_DUMP"), default=False) + + def _debug_json_dump(label: str, obj: object) -> None: """Best-effort debug-level JSON dump. @@ -79,6 +90,61 @@ def _debug_json_dump(label: str, obj: object) -> None: logger.debug("%s dump failed: %s", label, e) +_INBOUND_DUMP_DIR = pathlib.Path(".claude-code-proxy/anthropic-prompts") + + +def _inbound_dump_enabled() -> bool: + """PROXY_DEBUG_INBOUND_DUMP=true writes the raw Anthropic request to a file before conversion.""" + return _str_to_bool(os.environ.get("PROXY_DEBUG_INBOUND_DUMP"), default=False) + + +_INBOUND_DUMP_SEQ = itertools.count(1) + + +def _inbound_dump_path() -> pathlib.Path: + """Build a timestamped dump path; creates the directory if missing.""" + _INBOUND_DUMP_DIR.mkdir(parents=True, exist_ok=True) + ts = time.strftime("%Y%m%d-%H%M%S") + seq = next(_INBOUND_DUMP_SEQ) + return _INBOUND_DUMP_DIR / f"{ts}-{os.getpid()}-inbound-{seq:04d}.json" + + +def _format_inbound_body(body: bytes) -> str: + """Pretty-print JSON; on parse failure return raw text with replacement chars.""" + try: + return json.dumps(json.loads(body), indent=2, ensure_ascii=False) + except (ValueError, UnicodeDecodeError): + return body.decode("utf-8", errors="replace") + + +def _dump_inbound_anthropic_body(body: bytes) -> None: + """Write the raw Anthropic request body before Pydantic validation. + + Diagnostic aid for verifying how Claude Code shapes its requests — + in particular where it places ``system`` reminders across turns and + which fields it omits (the validated object would silently fill in + defaults and hide that). Body is pretty-printed for readability; + content matches the wire bytes modulo whitespace. Best-effort: a + failed write or parse must never break the proxy. + """ + if not _inbound_dump_enabled(): + return + try: + path = _inbound_dump_path() + except Exception as e: + logger.debug("inbound dump failed: %s", e) + return + try: + text = _format_inbound_body(body) + except Exception as e: + logger.debug("inbound dump failed: %s", e) + return + try: + path.write_text(text, encoding="utf-8") + except Exception as e: + logger.debug("inbound dump failed: %s", e) + + def _resolve_tiktoken_offline() -> bool: """[proxy].tiktoken_offline from CONFIG_PATH, TIKTOKEN_OFFLINE env, then True.""" path = os.environ.get("CONFIG_PATH", "./config.toml") @@ -152,6 +218,12 @@ def _encoding_for_model(_model_name: str) -> _OfflineEncoding: _VALID_SECTIONS = {"proxy", "global", "big", "small"} | _VALID_TIERS _PROXY_KEYS = {"openai_api_key", "openai_base_url", "openai_tls_verify"} _PROTECTED_KEYS = {"model", "messages", "stream", "tools"} # OpenAI Chat Completions keys the proxy owns +# Keys that must NOT be lifted to top-level kwargs — the openai SDK's +# signature rejects them as unknown kwargs (TypeError), but the upstream +# HTTP body still accepts them via ``extra_body``. ``thinking`` is the +# canonical case: Claude Code / MiniMax-M3 exchange it via JSON body, not +# as a typed SDK param. +_BODY_ONLY_KEYS = {"thinking"} _BUCKET_FOR_TIER = {t: ("small" if t == "haiku" else "big") for t in TIER_KEYS} _PROVIDER_PREFIXES = ("anthropic/", "openai/", "gemini/") _BOOL_TLS_VERIFY = {"openai_tls_verify"} @@ -323,12 +395,7 @@ class Message(BaseModel): """A single turn in the conversation — user, assistant, or system reminder.""" role: Literal["user", "assistant", "system"] - content: ( - str - | list[ - ContentBlockText | ContentBlockThinking | ContentBlockImage | ContentBlockToolUse | ContentBlockToolResult - ] - ) + content: str | list[ContentBlockText | ContentBlockThinking | ContentBlockImage | ContentBlockToolUse | ContentBlockToolResult] class Tool(BaseModel): @@ -359,6 +426,11 @@ class MessagesRequest(BaseModel): top_k: int | None = None tools: list[Tool] | None = None tool_choice: dict[str, Any] | None = None + # Anthropic-shaped knobs. ``output_config.effort`` translates to OpenAI's + # ``reasoning_effort``; ``thinking`` passes through to ``extra_body.thinking`` + # for Anthropic-compatible backends (e.g. MiniMax-M3) — see convert_anthropic_to_litellm. + output_config: dict[str, Any] | None = None + thinking: dict[str, Any] | None = None # Pass-through bag for arbitrary OpenAI Chat Completions keys. Merged # with [tier].extra_body at convert time; per-leaf config-wins. extra_body: dict[str, Any] | None = None @@ -564,6 +636,18 @@ def _parse_prompt_remap_section(body: object) -> list[dict[str, str]]: return out +def _apply_known_section(out: dict[str, Any], section: str, body: dict[str, Any]) -> None: + """Parse a recognised [proxy] / [bucket] / [global] / [tier] body into ``out``.""" + if section in {"big", "small"}: + out[section] = _parse_bucket_section(body, section) + elif section == "proxy": + out["proxy"] = _parse_proxy_section(body) + elif section == "global": + out["global"] = _parse_tier_section(body, section) + else: # per-tier section (haiku/sonnet/opus/fable/mythos) + out["tiers"][section] = _parse_bucket_section(body, section) + + def _load_config(path: str) -> dict[str, Any]: """Parse TOML at path; fail-open on every parse failure (logged, not raised).""" out: dict[str, Any] = {"proxy": {}, "global": {}, "big": {}, "small": {}, "tiers": {}, "prompt_remap": []} @@ -593,18 +677,12 @@ def _load_config(path: str) -> dict[str, Any]: if not isinstance(body, dict): logger.warning("[%s] must be a table, got %s; ignoring", section, type(body).__name__) continue - if section == "proxy": - out["proxy"] = _parse_proxy_section(body) - elif section in {"big", "small"}: - out[section] = _parse_bucket_section(body, section) - elif section == "global": - out["global"] = _parse_tier_section(body, section) - else: # per-tier section - out["tiers"][section] = _parse_bucket_section(body, section) + _apply_known_section(out, section, body) return out CONFIG_PATH = os.environ.get("CONFIG_PATH", "./config.toml") +CONFIG: dict[str, Any] try: CONFIG = _load_config(CONFIG_PATH) except Exception: @@ -847,6 +925,24 @@ def _log_request(ctx: _LogContext) -> None: app = FastAPI(lifespan=_configure_logging) +@app.middleware("http") +async def _inbound_body_dump_middleware(request: Request, call_next: Callable[[Request], Awaitable[Response]]) -> Response: + """Capture the raw request body for PROXY_DEBUG_INBOUND_DUMP before Pydantic validation. + + Reading the body here also lets the downstream handler re-parse it + (FastAPI caches the body on ``request._body`` so the route handler + still sees the same bytes). Best-effort: a failed read or write + must never break the request. + """ + if _inbound_dump_enabled() and request.url.path == "/v1/messages" and request.method == "POST": + try: + body = await request.body() + _dump_inbound_anthropic_body(body) + except Exception: # debug-only — never break the request + logger.debug("inbound body dump failed", exc_info=True) + return await call_next(request) + + OPENAI_API_KEY = _proxy_value("openai_api_key", "OPENAI_API_KEY") OPENAI_BASE_URL = _proxy_value("openai_base_url", "OPENAI_BASE_URL") @@ -864,12 +960,57 @@ def _log_request(ctx: _LogContext) -> None: # --------------------------------------------------------------------------- +def _convert_tool_result_to_parts(content: object) -> str | list[dict[str, Any]]: + """Convert Anthropic tool_result content to OpenAI Chat Completions shape. + + Returns a string when content is text-only (preserves the existing wire + format for backwards compatibility). Returns a list of content parts + when content contains any image block — text parts stay as text, image + parts become image_url blocks via convert_image_block. + """ + if content is None: + return "No content provided" + if isinstance(content, str): + return content + if isinstance(content, dict): + # Route single-dict shapes through the list branch for one source of truth + return _convert_tool_result_to_parts([content]) + if isinstance(content, list): + return _tool_result_parts_from_list(content) + return _str_or_unparseable(content) + + +def _tool_result_parts_from_list(items: list[object]) -> str | list[dict[str, Any]]: + parts = [_build_tool_result_part(item) for item in items] + # Text-only result: flatten back to a string for wire-format stability + if all(p.get("type") == "text" for p in parts): + return "\n".join(p.get("text", "") for p in parts).strip() + return parts + + +def _build_tool_result_part(item: object) -> dict[str, Any]: + if isinstance(item, dict): + item_type = item.get("type") + if item_type == "text": + text = item.get("text") + return {"type": "text", "text": text if isinstance(text, str) else ""} + if item_type == "image": + return convert_image_block(item.get("source")) + # Unknown block type — render via existing prose path, wrapped as text part + return {"type": "text", "text": _parse_tool_result_content(item)} + if isinstance(item, str): + return {"type": "text", "text": item} + return {"type": "text", "text": _str_or_unparseable(item)} + + def _parse_tool_result_content(content: object) -> str: """Normalise a tool_result ``content`` field into a plain string. Anthropic allows None, str, list of blocks, or a single dict (sometimes ``{"type": "text", ...}``). We stringify whatever shape we get so the - model sees prose rather than a raw JSON blob. + model sees prose rather than a raw JSON blob. Image-bearing tool results + are routed through _convert_tool_result_to_parts at the call site instead + so the image can travel as an image_url block. """ match content: case None: @@ -971,17 +1112,16 @@ def _build_system_message( Anthropic's spec only allows system at the top level, but Claude Code 2.1.154+ has started embedding system reminders inline. We hoist them all - to the start so OpenAI sees one system message at the top. Order is - preserved: in-band messages come first, then the top-level field — which - is the order Claude Code most likely intended when it injected the - reminders inline. + to the start so OpenAI sees one system message at the top. Order + mirrors the natural Anthropic chronology: the top-level ``system`` + field is the agent's identity and goes first, in-band reminders follow + in the order they appear in the ``messages`` array. Configured ``[[prompt_remap]]`` regexes are applied last. """ - parts = [text for text in (_extract_text(m.content) for m in messages if m.role == "system") if text] top = _extract_text(system_field) - if top: - parts.append(top) + inband = [text for text in (_extract_text(m.content) for m in messages if m.role == "system") if text] + parts = ([top] if top else []) + inband if not parts: return None text = _apply_prompt_remaps("\n\n".join(parts)).strip() @@ -992,8 +1132,24 @@ def _build_system_message( def _apply_prompt_remaps(text: str) -> str: """Apply configured prompt remappings in order.""" + before_len = len(text) + fired = 0 + total_matches = 0 for pattern, replacement in _PROMPT_REMAPS: - text = pattern.sub(replacement, text) + new_text, n = pattern.subn(replacement, text) + if n: + fired += 1 + total_matches += n + text = new_text + if fired: + logger.warning( + "prompt_remap: stripped %d chars in %d match%s via %d %s", + before_len - len(text), + total_matches, + "es" if total_matches != 1 else "", + fired, + "entry" if fired == 1 else "entries", + ) return text @@ -1014,24 +1170,41 @@ def _collect_tool_ids(messages: list[Message]) -> tuple[set[str], set[str]]: return call_ids, result_ids # ty: ignore[unsound-return-statement] — elements come from cast / _get_field; ty can't trace set element types -def _convert_assistant_message(msg: Message, result_ids: set[str]) -> dict[str, Any]: +def _convert_assistant_message(msg: Message, result_ids: set[str], id_map: dict[str, str]) -> dict[str, Any]: text_parts = [] tool_calls = [] + reasoning_parts = [] for block in msg.content: block_type = _get_field(block, "type") if block_type == "text": text_parts.append(cast("ContentBlockText", block).text) + elif block_type == "thinking": + # Round-trip upstream reasoning_content back into the assistant turn so the + # model sees its own prior chain. Forwarded as the canonical ``reasoning_content`` + # field on the OpenAI message dict; litellm's openai_like/chat handler spreads + # messages straight into the JSON body, and the openai SDK serialises them + # with plain json.dumps (TypedDict = no runtime validation), so the extra key + # reaches the wire untouched. See vertex_ai/gemini/transformation.py:826 for + # the canonical round-trip contract. + reasoning_parts.append(cast("ContentBlockThinking", block).thinking) elif block_type == "tool_use": tool_block = cast("ContentBlockToolUse", block) if tool_block.id in result_ids: tool_calls.append( { - "id": tool_block.id, + "id": id_map[tool_block.id], "type": "function", "function": { "name": tool_block.name, - "arguments": json.dumps(tool_block.input), + # Compact JSON: cache-stable rendering when llama-server's + # chat template interpolates the arguments string. + "arguments": json.dumps( + tool_block.input, + separators=(",", ":"), + sort_keys=True, + ensure_ascii=False, + ), }, }, ) @@ -1046,10 +1219,14 @@ def _convert_assistant_message(msg: Message, result_ids: set[str]) -> dict[str, out["content"] = text or (None if tool_calls else "") if tool_calls: out["tool_calls"] = tool_calls + if reasoning_parts: + # Anthropic may emit multiple thinking blocks per turn (rare). Concatenate so the + # model sees one continuous chain matching what it produced originally. + out["reasoning_content"] = "\n\n".join(reasoning_parts) return out -def _convert_user_message(msg: Message, call_ids: set[str]) -> list[dict[str, Any]]: +def _convert_user_message(msg: Message, call_ids: set[str], id_map: dict[str, str]) -> list[dict[str, Any]]: tool_messages = [] user_parts = [] @@ -1061,13 +1238,13 @@ def _convert_user_message(msg: Message, call_ids: set[str]) -> list[dict[str, An user_parts.append(convert_image_block(cast("ContentBlockImage", block).source)) elif block_type == "tool_result": tool_use_id = _get_field(block, "tool_use_id", "") or "" - result_text = _parse_tool_result_content(_get_field(block, "content")) + raw_content = _get_field(block, "content") if tool_use_id in call_ids: tool_messages.append( { "role": "tool", - "tool_call_id": tool_use_id, - "content": result_text, + "tool_call_id": id_map[tool_use_id], + "content": _convert_tool_result_to_parts(raw_content), }, ) else: @@ -1075,7 +1252,7 @@ def _convert_user_message(msg: Message, call_ids: set[str]) -> list[dict[str, An user_parts.append( { "type": "text", - "text": f"(Result from an earlier tool call:)\n{result_text}", + "text": f"(Result from an earlier tool call:)\n{_parse_tool_result_content(raw_content)}", }, ) @@ -1094,12 +1271,18 @@ def _convert_message( msg: Message, result_ids: set[str], call_ids: set[str], + id_map: dict[str, str], ) -> list[dict[str, Any]]: if isinstance(msg.content, str): return [{"role": msg.role, "content": msg.content}] if msg.role == "assistant": - return [_convert_assistant_message(msg, result_ids)] - return _convert_user_message(msg, call_ids) + return [_convert_assistant_message(msg, result_ids, id_map)] + return _convert_user_message(msg, call_ids, id_map) + + +# Anthropic-only keys that may sit inside an input_schema; OpenAI clients +# never emit them, so a ref forwarder would render the prompt without them. +_TOOL_PARAM_ANTHROPIC_KEYS = frozenset({"cache_control", "strict"}) def _convert_tool_definitions(tools: list[Tool]) -> list[dict[str, Any]]: @@ -1109,7 +1292,7 @@ def _convert_tool_definitions(tools: list[Tool]) -> list[dict[str, Any]]: "function": { "name": tool.name, "description": tool.description or "", - "parameters": tool.input_schema, + "parameters": {k: v for k, v in tool.input_schema.items() if k not in _TOOL_PARAM_ANTHROPIC_KEYS}, }, } for tool in tools @@ -1133,17 +1316,25 @@ def sanitize_messages_for_openai(messages: list[dict[str, Any]]) -> None: """Strip message keys OpenAI doesn't accept and coerce empty content. Mutates in place. Keeps ``role``, ``content``, ``name``, ``tool_call_id``, - and ``tool_calls``; everything else is dropped. Empty/None ``content`` is - replaced with ``"..."`` when no tool_calls are present — OpenAI rejects - empty content outright. + ``tool_calls``, and ``reasoning_content``; everything else is dropped. + ``reasoning_content`` is the canonical round-trip field for prior-turn + reasoning — OpenAI's TypedDict doesn't declare it but the SDK's wire + serializer is plain json.dumps so it reaches the wire untouched (verified + via openapi_dumps), and litellm's vertex_ai/gemini handler reads it back + (vertex_ai/gemini/transformation.py:826). Empty/None ``content`` is replaced + with ``"..."`` when no tool_calls are present — OpenAI rejects empty + content outright. Tool result bodies are left as-is even when empty: the + placeholder would change the prompt's tokens. """ - allowed_keys = {"role", "content", "name", "tool_call_id", "tool_calls"} + allowed_keys = {"role", "content", "name", "tool_call_id", "tool_calls", "reasoning_content"} for msg in messages: for key in list(msg.keys()): if key not in allowed_keys: logger.debug("Removing unsupported message field: %s", key) del msg[key] - if msg.get("content") in {None, ""} and not msg.get("tool_calls"): + if msg.get("role") == "tool": + continue + if not msg.get("content") and not msg.get("tool_calls"): msg["content"] = "..." @@ -1185,6 +1376,10 @@ def convert_anthropic_to_litellm(anthropic_request: MessagesRequest) -> dict[str """ call_ids, result_ids = _collect_tool_ids(anthropic_request.messages) + # Rewrite Anthropic's ``toolu_*`` ids to OpenAI's ``call_*`` so the + # chat template renders the same prefix any OpenAI client would. + id_map = {old: f"call_{uuid.uuid4().hex[:24]}" for old in call_ids} + messages = [] if system := _build_system_message(anthropic_request.system, anthropic_request.messages): messages.append(system) @@ -1193,7 +1388,7 @@ def convert_anthropic_to_litellm(anthropic_request: MessagesRequest) -> dict[str # models to emit tool calls as literal text and broke tool use. for msg in anthropic_request.messages: if msg.role != "system": - messages.extend(_convert_message(msg, result_ids, call_ids)) + messages.extend(_convert_message(msg, result_ids, call_ids, id_map)) litellm_request: dict[str, Any] = { "model": anthropic_request.model, @@ -1244,8 +1439,21 @@ def _apply_merged_extra_body( whitelist twice — top-level so litellm extends supported_params (utils.py:3877), and inside extra_body so cascade proxies forward vendor keys instead of filtering them (openai_like/chat/handler.py:241,254-259). + + Three client-side sources are folded in (later wins per leaf): + 1. ``request.thinking`` (Anthropic-shaped) → passes through to + Anthropic-compatible backends like MiniMax-M3 verbatim. + 2. ``request.output_config.effort`` → translated to ``reasoning_effort`` + (OpenAI's reasoning-effort axis used by o-series). + 3. ``request.extra_body`` → arbitrary keys the client wants to forward. + ``[tier].extra_body`` overrides any of the above per leaf — apply last. """ merged_extra: dict[str, Any] = {} + if anthropic_request.thinking: + merged_extra = _deep_merge(merged_extra, {"thinking": anthropic_request.thinking}) + effort = anthropic_request.output_config.get("effort") if anthropic_request.output_config else None + if effort: + merged_extra = _deep_merge(merged_extra, {"reasoning_effort": effort}) if anthropic_request.extra_body: merged_extra = _deep_merge(merged_extra, anthropic_request.extra_body) tier_extra = tier_cfg.get("extra_body") @@ -1256,12 +1464,19 @@ def _apply_merged_extra_body( if k in _PROTECTED_KEYS: logger.warning("ignoring protected key in extra_body: %s", k) continue + if k in _BODY_ONLY_KEYS: + # Don't lift to top-level — the openai SDK rejects unknown kwargs. + # The key still lands in the JSON body via extra_body below. + continue litellm_request[k] = v if merged_extra: keys = list(merged_extra.keys()) + # Body-only keys still need to be in allowed_openai_params so litellm + # forwards them; they reach the JSON body through extra_body in + # openai_like/chat/handler.py:241,258 which bypasses the SDK signature. litellm_request["allowed_openai_params"] = keys - litellm_request["extra_body"] = {"allowed_openai_params": keys} + litellm_request["extra_body"] = {"allowed_openai_params": keys, **merged_extra} # --------------------------------------------------------------------------- @@ -1671,10 +1886,13 @@ class _StreamState: Lives only as long as ``handle_streaming``'s iteration. ``should_stop`` is set by chunk processors when they emit a finish_reason so the outer - loop can break cleanly. + loop can break cleanly. ``tool_use_emitted`` lets the epilogue pick + ``tool_use`` over ``end_turn`` when the upstream closes without a + finish_reason mid-tool-call. """ tool_index: int | None = None + tool_use_emitted: bool = False input_tokens: int = 0 output_tokens: int = 0 has_sent_stop_reason: bool = False @@ -1711,10 +1929,11 @@ def _stream_prologue(original_request: MessagesRequest, tracker: _BlockTracker) yield _SseFormatter.ping() -def _stream_epilogue(tracker: _BlockTracker, think_parser: _ThinkStreamParser, output_tokens: int) -> Iterator[str]: +def _stream_epilogue(state: _StreamState, tracker: _BlockTracker, think_parser: _ThinkStreamParser) -> Iterator[str]: yield from _translate_parser_events(think_parser.flush(), tracker) yield from tracker.close() - yield from _SseFormatter.finish("end_turn", output_tokens) + stop_reason = "tool_use" if state.tool_use_emitted else "end_turn" + yield from _SseFormatter.finish(stop_reason, state.output_tokens) def _log_stream_finished(state: _StreamState) -> None: @@ -1835,6 +2054,7 @@ def _process_single_tool_call(tool_call: object, tracker: _BlockTracker, state: if state.tool_index is not None: yield from tracker.close() state.tool_index = current_index + state.tool_use_emitted = True function = _get_field(tool_call, "function", {}) or {} name = _get_field(function, "name", "") tool_id = _get_field(tool_call, "id") or _new_tool_id() @@ -1930,7 +2150,7 @@ async def handle_streaming( # Skip epilogue if chunk loop already terminated the stream via _emit_failure # — calling _translate_parser_events again here would re-emit the error frame. if not state.has_sent_stop_reason and not state.should_stop: - for event in _stream_epilogue(tracker, think_parser, state.output_tokens): + for event in _stream_epilogue(state, tracker, think_parser): yield event _log_stream_finished(state) except Exception as e: @@ -1959,18 +2179,167 @@ def _prepare_litellm_request(request: MessagesRequest) -> dict[str, Any]: return litellm_request +_CACHE_DEBUG_DIR = pathlib.Path(".claude-code-proxy/prompts") +_CACHE_MATCHER_HISTORY = 50 +_CACHE_MATCHER_THRESHOLD = 0.6 + + +class _CacheMatcher: + """Rolling-window prefix-equivalence matcher for llama-server cache hits. + + The rendered prompt is keyed by the messages list (and tools list) — + metadata fields like ``model``/``max_tokens`` don't affect the prefix + cache. We canonicalise just the messages+tools slices, so the + ``startswith`` check returns true when the new outgoing is a prefix + extension of a prior one (the cache-hit case). On mismatch we run a + fuzzy match (SequenceMatcher ratio) and, when similarity crosses the + threshold, persist both payloads as artifacts for offline analysis. + """ + + def __init__(self, *, max_history: int, fuzzy_threshold: float, out_dir: pathlib.Path) -> None: + self._recent: list[tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]] = [] + self._max_history = max_history + self._fuzzy_threshold = fuzzy_threshold + self._out_dir = out_dir + + @staticmethod + def _strip_ids(payload: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Return copies of (messages, tools) with tool_call ids scrubbed (random per request). + + Does not mutate ``payload`` — callers can rely on its tool_call ids + after observe() returns. + """ + messages = [] + for msg in payload.get("messages", []): + scrubbed = {**msg} + if "tool_calls" in scrubbed: + scrubbed["tool_calls"] = [{**tc, "id": ""} for tc in scrubbed["tool_calls"]] + if "tool_call_id" in scrubbed: + scrubbed["tool_call_id"] = "" + messages.append(scrubbed) + return messages, list(payload.get("tools", [])) + + def observe(self, payload: dict[str, Any]) -> None: + new_messages, new_tools = self._strip_ids(payload) + new_canonical = json.dumps(new_messages, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + # Newest-first so the matched prior is the immediately-prior request: + # logs show incremental growth (msgs N-1 vs N), not always the oldest entry. + for old_messages, old_tools, old_payload in reversed(self._recent): + prefix_match = ( + new_tools == old_tools + and old_messages + and len(new_messages) >= len(old_messages) + and new_messages[: len(old_messages)] == old_messages + ) + if prefix_match: + logger.debug( + "cache debug: prefix hit (msgs %d vs %d) — saving to %s", + len(old_messages), + len(new_messages), + self._out_dir, + ) + self._record(payload, old_payload, 1.0, "prefix_hit") + self._append(new_messages, new_tools, payload) + return + # Fuzzy fallback over the canonical wire form. + best_score = 0.0 + best_old_payload: dict[str, Any] | None = None + for old_messages, _old_tools, old_payload in self._recent: + old_canonical = json.dumps(old_messages, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + score = difflib.SequenceMatcher(None, new_canonical, old_canonical).ratio() + if score > best_score: + best_score = score + best_old_payload = old_payload + if best_score >= self._fuzzy_threshold: + logger.info( + "cache debug: fuzzy match %.2f — saving artifacts to %s", + best_score, + self._out_dir, + ) + self._record(payload, best_old_payload, best_score, "fuzzy_match") + else: + logger.debug("cache debug: no match (best=%.2f)", best_score) + self._append(new_messages, new_tools, payload) + + def _append(self, messages: list[dict[str, Any]], tools: list[dict[str, Any]], payload: dict[str, Any]) -> None: + self._recent.append((messages, tools, payload)) + if len(self._recent) > self._max_history: + self._recent.pop(0) + + def _record( + self, + new_payload: dict[str, Any], + old_payload: dict[str, Any] | None, + score: float, + kind: str, + ) -> None: + # Write scrubbed forms so random per-request tool_call ids don't drown the + # real structural change. + new_messages, new_tools = self._strip_ids(new_payload) + new_for_disk = {**new_payload, "messages": new_messages, "tools": new_tools} + stamp = _cache_debug_stamp(score, kind) + self._out_dir.mkdir(parents=True, exist_ok=True) + _safe_write_json(self._out_dir / f"{stamp}-new.json", new_for_disk) + if old_payload is not None: + old_messages, old_tools = self._strip_ids(old_payload) + old_for_disk = {**old_payload, "messages": old_messages, "tools": old_tools} + _safe_write_json(self._out_dir / f"{stamp}-old.json", old_for_disk) + + +_cache_debug_seq = itertools.count(1) + + +def _cache_debug_stamp(score: float, kind: str) -> str: + # Trailing seq disambiguates same-second same-pid same-score writes (every prefix_hit has score=1.00). + seq = next(_cache_debug_seq) + return f"{time.strftime('%Y%m%d-%H%M%S')}-{os.getpid()}-{kind}-{score:.2f}-{seq:04d}" + + +def _safe_write_json(path: pathlib.Path, payload: dict[str, Any]) -> None: + try: + path.write_text(json.dumps(payload, default=str, ensure_ascii=False), encoding="utf-8") + except Exception as e: + logger.debug("cache debug save failed: %s", e) + + +@cache +def _get_cache_matcher() -> _CacheMatcher: + return _CacheMatcher( + max_history=_CACHE_MATCHER_HISTORY, + fuzzy_threshold=_CACHE_MATCHER_THRESHOLD, + out_dir=pathlib.Path.cwd() / _CACHE_DEBUG_DIR, + ) + + +def _debug_dump_outgoing_payload(litellm_request: dict[str, Any]) -> None: + """Record outgoing payloads and match prefix equivalence when ``PROXY_DEBUG_CACHE_DUMP=true``. + + Each request adds to a rolling window of canonical wire forms. A new + outgoing that starts with any prior one is a cache hit (the desired + case). Otherwise we fuzzy-match against the window and, on similarity + ≥ 0.6, save both payloads to ``$cwd/.claude-code-proxy/prompts/`` for analysis. + + Artifacts: ``----new.json`` (current) and + ``----old.json`` (prior that matched). Compare them + manually — unified diff of pretty-printed JSON collapses multi-KB system + prompts onto a single line and isn't readable. + """ + if not _debug_cache_dump_enabled(): + return + _get_cache_matcher().observe(litellm_request) + + def _log_upstream_params_debug(litellm_request: dict[str, Any]) -> None: - # Skip the bulky fields (messages, tools) — they dominate the dump and - # are visible in litellm.set_verbose anyway. + # Skip bulky fields (dumped by litellm.set_verbose) and the api_key secret. if not logger.isEnabledFor(logging.DEBUG): return - debug = {k: v for k, v in litellm_request.items() if k not in {"messages", "tools"}} + debug = {k: v for k, v in litellm_request.items() if k not in {"messages", "tools", "api_key"}} logger.debug("upstream params: %s", debug) if _litellm_debug_http_enabled(): - # Verbose: dump the entire kwargs dict going into litellm - # (messages, tools, tool_choice, …) so we can confirm the - # exact payload upstream sees, not just the sampling subset. - _debug_json_dump("litellm.completion kwargs (full)", litellm_request) + # Dump the full kwargs (messages, tools, tool_choice, …) to confirm the + # exact payload upstream sees — api_key is masked first. + redacted = {**litellm_request, "api_key": "***"} + _debug_json_dump("litellm.completion kwargs (full)", redacted) def _log_response_debug(litellm_response: object, model: str, start_time: float) -> None: @@ -1990,6 +2359,7 @@ def _log_response_debug(litellm_response: object, model: str, start_time: float) async def _handle_request(request: MessagesRequest) -> MessagesResponse | StreamingResponse: litellm_request = _prepare_litellm_request(request) + _debug_dump_outgoing_payload(litellm_request) _log_upstream_params_debug(litellm_request) _log_request( _LogContext( diff --git a/tests.py b/tests.py index 88c41b68..0ad5988f 100644 --- a/tests.py +++ b/tests.py @@ -16,15 +16,18 @@ import asyncio import contextlib import inspect +import itertools import json import os import pathlib +import shutil import sys import tempfile import time from collections.abc import AsyncGenerator, Callable, Iterator from dataclasses import dataclass, field from typing import Any, NoReturn +from unittest.mock import patch import httpx from dotenv import load_dotenv @@ -149,9 +152,9 @@ def _parse_sse_block(block: str) -> dict[str, Any] | None: data_lines: list[str] = [] for line in block.splitlines(): if line.startswith("event: "): - event_type = line[len("event: "):] + event_type = line[len("event: ") :] elif line.startswith("data: "): - data_lines.append(line[len("data: "):]) + data_lines.append(line[len("data: ") :]) if not data_lines: return None payload = "".join(data_lines) @@ -174,10 +177,12 @@ async def _run_stream(chunks: list[Any], req: srv.MessagesRequest) -> list[dict[ def _text_chunk(text: str, **extra: Any) -> dict[str, Any]: chunk = { - "choices": [{ - "delta": {"content": text}, - "finish_reason": None, - }], + "choices": [ + { + "delta": {"content": text}, + "finish_reason": None, + }, + ], } chunk["choices"][0].update(extra.get("choice_extra", {})) chunk.update({k: v for k, v in extra.items() if k != "choice_extra"}) @@ -203,10 +208,12 @@ def _tool_delta_chunk( if function: tool_call["function"] = function return { - "choices": [{ - "delta": {"tool_calls": [tool_call]}, - "finish_reason": finish_reason, - }], + "choices": [ + { + "delta": {"tool_calls": [tool_call]}, + "finish_reason": finish_reason, + }, + ], } @@ -224,77 +231,94 @@ def _finish_chunk(reason: str, *, output_tokens: int = 5) -> dict[str, Any]: # --- Model mapping --- + def test_capture_original_model_copies_model_field() -> None: - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.original_model == "claude-3-5-sonnet-20241022" assert req.model == f"openai/{srv._default_model_for_tier('sonnet')}" def test_capture_original_model_preserves_explicit_override() -> None: - req = _make_request({ - "model": "openai/gpt-4.1", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "openai/gpt-4.1", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.original_model == "openai/gpt-4.1" assert req.model == "openai/gpt-4.1" def test_validate_model_field_haiku_mapping() -> None: - req = _make_request({ - "model": "claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-haiku-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == f"openai/{srv._default_model_for_tier('haiku')}" def test_validate_model_field_sonnet_mapping() -> None: - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == f"openai/{srv._default_model_for_tier('sonnet')}" def test_validate_model_field_opus_maps_to_big_model() -> None: - req = _make_request({ - "model": "claude-opus-5", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-opus-5", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == f"openai/{srv._default_model_for_tier('opus')}" def test_validate_model_field_opus_with_dated_id() -> None: - req = _make_request({ - "model": "claude-opus-5-20251215", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-opus-5-20251215", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == f"openai/{srv._default_model_for_tier('opus')}" def test_validate_model_field_fable_maps_to_big_model() -> None: - req = _make_request({ - "model": "claude-fable-5", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-fable-5", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == f"openai/{srv._default_model_for_tier('fable')}" def test_validate_model_field_mythos_maps_to_big_model() -> None: - req = _make_request({ - "model": "claude-mythos-5", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-mythos-5", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == f"openai/{srv._default_model_for_tier('mythos')}" @@ -302,11 +326,13 @@ def test_validate_model_field_sonnet_override_takes_precedence() -> None: saved = _scrub_model_envs() try: with _patched_config('[sonnet]\nmodel = "custom-sonnet-model"'): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == "openai/custom-sonnet-model" finally: for k, v in saved.items(): @@ -318,17 +344,21 @@ def test_validate_model_field_opus_override_is_independent() -> None: saved = _scrub_model_envs() try: with _patched_config('[opus]\nmodel = "custom-opus-model"'): - opus_req = _make_request({ - "model": "claude-opus-5", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + opus_req = _make_request( + { + "model": "claude-opus-5", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert opus_req.model == "openai/custom-opus-model" - sonnet_req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + sonnet_req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert sonnet_req.model == f"openai/{srv._default_model_for_tier('sonnet')}" finally: for k, v in saved.items(): @@ -340,11 +370,13 @@ def test_validate_model_field_haiku_override() -> None: saved = _scrub_model_envs() try: with _patched_config('[haiku]\nmodel = "custom-haiku-model"'): - req = _make_request({ - "model": "claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-haiku-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == "openai/custom-haiku-model" finally: for k, v in saved.items(): @@ -353,48 +385,58 @@ def test_validate_model_field_haiku_override() -> None: def test_validate_model_field_known_openai_model_gets_prefix() -> None: - req = _make_request({ - "model": "gpt-4.1", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "gpt-4.1", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == "openai/gpt-4.1" def test_validate_model_field_existing_openai_prefix_passthrough() -> None: - req = _make_request({ - "model": "openai/custom-model", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "openai/custom-model", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == "openai/custom-model" def test_validate_model_field_unknown_name_gets_prefix() -> None: - req = _make_request({ - "model": "my-local-llama", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "my-local-llama", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == "openai/my-local-llama" def test_validate_model_field_strips_anthropic_prefix() -> None: - req = _make_request({ - "model": "anthropic/claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "anthropic/claude-3-5-haiku-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == f"openai/{srv._default_model_for_tier('haiku')}" assert req.original_model == "anthropic/claude-3-5-haiku-20241022" def test_validate_model_field_strips_gemini_prefix() -> None: - req = _make_request({ - "model": "gemini/claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "gemini/claude-3-5-haiku-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == f"openai/{srv._default_model_for_tier('haiku')}" assert req.original_model == "gemini/claude-3-5-haiku-20241022" @@ -406,6 +448,7 @@ def test_tls_verify_wiring_matches_module_setting() -> None: # --- Message sanitization --- + def test_sanitize_messages_for_openai_removes_foreign_keys() -> None: messages = [ {"role": "user", "content": "hi", "stop_reason": "end_turn", "type": "message"}, @@ -437,15 +480,56 @@ def test_sanitize_messages_for_openai_keeps_allowed_keys() -> None: assert messages[0]["content"] == "42" +def test_sanitize_messages_for_openai_handles_list_content() -> None: + """Image messages arrive with content as a list of content blocks — not a set member. + + Post-conversion (what sanitize actually sees) uses the OpenAI ``image_url`` + shape; the Anthropic shape is also exercised so any regression in + conversion order would still fail this test. + """ + messages = [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}}, + {"type": "text", "text": "what do you see?"}, + ], + }, + { + "role": "user", + "content": [ + {"type": "image", "source": {"type": "base64", "data": "..."}}, + {"type": "text", "text": "raw anthropic shape"}, + ], + }, + {"role": "user", "content": []}, + {"role": "assistant", "content": [], "tool_calls": [{"id": "x"}]}, + ] + srv.sanitize_messages_for_openai(messages) + assert messages[0]["content"] == [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}}, + {"type": "text", "text": "what do you see?"}, + ] + assert messages[1]["content"] == [ + {"type": "image", "source": {"type": "base64", "data": "..."}}, + {"type": "text", "text": "raw anthropic shape"}, + ] + assert messages[2]["content"] == "..." + assert messages[3]["content"] == [] + + # --- Request conversion --- + def test_convert_anthropic_to_litellm_minimal_request() -> None: with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 200, - "messages": [{"role": "user", "content": "Hello"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [{"role": "user", "content": "Hello"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["model"] == f"openai/{srv._default_model_for_tier('sonnet')}" assert out["max_completion_tokens"] == 200 @@ -464,38 +548,44 @@ def test_no_max_tokens_clamp() -> None: Users explicitly want 24000 when they ask for 24000. """ with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": srv.MAX_OUTPUT_TOKENS + 1000, - "messages": [{"role": "user", "content": "Hello"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": srv.MAX_OUTPUT_TOKENS + 1000, + "messages": [{"role": "user", "content": "Hello"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["max_completion_tokens"] == srv.MAX_OUTPUT_TOKENS + 1000 def test_convert_anthropic_to_litellm_with_string_system() -> None: with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": "You are helpful.", - "messages": [{"role": "user", "content": "Hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "You are helpful.", + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["messages"][0] == {"role": "system", "content": "You are helpful."} def test_convert_anthropic_to_litellm_with_list_system_joins_text_blocks() -> None: with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": [ - {"type": "text", "text": "Be concise."}, - {"type": "text", "text": "Answer in English."}, - ], - "messages": [{"role": "user", "content": "Hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": [ + {"type": "text", "text": "Be concise."}, + {"type": "text", "text": "Answer in English."}, + ], + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["messages"][0]["role"] == "system" assert "Be concise." in out["messages"][0]["content"] @@ -504,39 +594,47 @@ def test_convert_anthropic_to_litellm_with_list_system_joins_text_blocks() -> No def test_convert_anthropic_to_litellm_with_tools_and_choice() -> None: with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 200, - "messages": [{"role": "user", "content": "What is 2+2?"}], - "tools": [{ - "name": "calc", - "description": "calculator", - "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}, "required": ["q"]}, - }], - "tool_choice": {"type": "tool", "name": "calc"}, - }) - out = srv.convert_anthropic_to_litellm(req) - assert out["tools"] == [{ - "type": "function", - "function": { - "name": "calc", - "description": "calculator", - "parameters": {"type": "object", "properties": {"q": {"type": "string"}}, "required": ["q"]}, + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [{"role": "user", "content": "What is 2+2?"}], + "tools": [ + { + "name": "calc", + "description": "calculator", + "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}, "required": ["q"]}, + }, + ], + "tool_choice": {"type": "tool", "name": "calc"}, }, - }] + ) + out = srv.convert_anthropic_to_litellm(req) + assert out["tools"] == [ + { + "type": "function", + "function": { + "name": "calc", + "description": "calculator", + "parameters": {"type": "object", "properties": {"q": {"type": "string"}}, "required": ["q"]}, + }, + }, + ] assert out["tool_choice"] == {"type": "function", "function": {"name": "calc"}} def test_convert_anthropic_to_litellm_passes_optional_sampling() -> None: with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hi"}], - "stop_sequences": ["END"], - "top_p": 0.9, - "top_k": 40, - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + "stop_sequences": ["END"], + "top_p": 0.9, + "top_k": 40, + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["stop"] == ["END"] assert out["top_p"] == 0.9 @@ -549,14 +647,16 @@ def test_explicit_null_sampling_is_dropped() -> None: must not include the field. """ with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hi"}], - "temperature": None, - "top_p": None, - "stop_sequences": None, - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + "temperature": None, + "top_p": None, + "stop_sequences": None, + }, + ) # Pydantic v2 includes explicit null in fields_set — confirm semantics. assert "temperature" in req.model_fields_set out = srv.convert_anthropic_to_litellm(req) @@ -565,57 +665,1109 @@ def test_explicit_null_sampling_is_dropped() -> None: assert "stop" not in out +def test_output_config_effort_translates_to_reasoning_effort() -> None: + """Anthropic's ``output_config.effort`` lifts to OpenAI's ``reasoning_effort``. + + Claude Code sends ``output_config`` to namespace reasoning controls; OpenAI + Chat Completions exposes the same axis at the top level for o-series and + other reasoning backends. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + "output_config": {"effort": "low"}, + }, + ) + out = srv.convert_anthropic_to_litellm(req) + assert out["reasoning_effort"] == "low" + + +def test_output_config_without_effort_does_not_set_reasoning_effort() -> None: + """``output_config`` present but no ``effort`` key → no ``reasoning_effort``.""" + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + "output_config": {"some_future_key": "x"}, + }, + ) + out = srv.convert_anthropic_to_litellm(req) + assert "reasoning_effort" not in out + + +def test_output_config_empty_string_effort_is_dropped() -> None: + """``output_config.effort=""`` is treated as absent — the empty string is not + a valid enum value for any reasoning backend (Anthropic, OpenAI o-series, + Moonshot kimi), and forwarding it produces a 422. Truthy guard matches the + proxy's general rule of not auto-applying defaults when the client sent + nothing meaningful. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + "output_config": {"effort": ""}, + }, + ) + out = srv.convert_anthropic_to_litellm(req) + assert "reasoning_effort" not in out + + +def test_thinking_passes_through_to_extra_body_for_anthropic_compatible_backends() -> None: + """``request.thinking`` lands in ``extra_body.thinking`` (not top-level) — the + openai SDK rejects unknown top-level kwargs, so body-only keys reach the JSON + body via extra_body instead. Whitelist still published so litellm forwards it. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + "thinking": {"type": "adaptive"}, + }, + ) + out = srv.convert_anthropic_to_litellm(req) + # Body-only: NOT a top-level kwarg, but in extra_body for the JSON wire. + assert "thinking" not in out + assert out["extra_body"] == { + "allowed_openai_params": ["thinking"], + "thinking": {"type": "adaptive"}, + } + # Whitelist published so cascade proxies forward verbatim. + assert "thinking" in out["allowed_openai_params"] + + +def test_tier_extra_body_thinking_overrides_request_thinking_per_leaf() -> None: + """Config wins per leaf — ``[tier].extra_body.thinking.type`` overrides + the client's type, but unrelated leaves the client sent (budget_tokens) + survive the deep-merge. ``thinking`` stays body-only. + """ + with _patched_config('[global]\nextra_body = { thinking = { type = "disabled" } }\n'): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + "thinking": {"type": "enabled", "budget_tokens": 1024}, + }, + ) + out = srv.convert_anthropic_to_litellm(req) + # Config wins on the type leaf; budget_tokens only present in request survives. + assert "thinking" not in out + assert out["extra_body"]["thinking"] == {"type": "disabled", "budget_tokens": 1024} + + +def test_thinking_absent_means_no_thinking_field_in_outbound() -> None: + """No ``thinking`` in request, none in config → no ``thinking`` on the wire.""" + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + assert "thinking" not in out + assert "allowed_openai_params" not in out + + +def test_thinking_block_round_trips_as_reasoning_content() -> None: + """Anthropic thinking blocks on prior assistant turns reach upstream as + ``reasoning_content`` on the assistant message — without this, models + trained on reasoning chains (e.g. qwen3.8 with preserve_thinking=true) lose + continuity and re-derive reasoning every turn. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "What is 2+2?"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "The user asks a simple math question."}, + {"type": "text", "text": "The answer is 4."}, + ], + }, + {"role": "user", "content": "And 3+3?"}, + ], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + asst = out["messages"][1] + assert asst["role"] == "assistant" + assert asst["content"] == "The answer is 4." + assert asst["reasoning_content"] == "The user asks a simple math question." + + +def test_multiple_thinking_blocks_concatenate() -> None: + """Rare: multiple thinking blocks in one turn → joined with blank line.""" + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "Hi"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "First thought."}, + {"type": "thinking", "thinking": "Second thought."}, + {"type": "text", "text": "Hi back."}, + ], + }, + ], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + assert out["messages"][1]["reasoning_content"] == "First thought.\n\nSecond thought." + + +def test_assistant_without_thinking_has_no_reasoning_content_field() -> None: + """Pure text or tool-only assistant turns → no spurious reasoning_content.""" + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello."}, + ], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + assert "reasoning_content" not in out["messages"][1] + + +def test_sanitize_preserves_reasoning_content() -> None: + """The wire-format sanitizer must not strip reasoning_content — it's the + round-trip field for prior-turn reasoning. Without this the round-trip is + silent: the message leaves _convert_assistant_message with reasoning_content + attached but loses it before reaching the wire. + """ + msgs = [{"role": "assistant", "content": "ok", "reasoning_content": "thoughtful"}] + srv.sanitize_messages_for_openai(msgs) + assert msgs[0]["reasoning_content"] == "thoughtful" + + def test_convert_anthropic_to_litellm_pairs_tool_call_with_tool_result() -> None: with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 200, - "messages": [ - {"role": "user", "content": "What is 2+2?"}, - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "t1", "name": "calc", "input": {"q": "2+2"}}, - ]}, - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": "4"}, - ]}, - ], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "What is 2+2?"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "t1", "name": "calc", "input": {"q": "2+2"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "4"}, + ], + }, + ], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["messages"][1]["role"] == "assistant" - assert out["messages"][1]["tool_calls"] == [{ - "id": "t1", "type": "function", - "function": {"name": "calc", "arguments": '{"q": "2+2"}'}, - }] - assert out["messages"][2] == {"role": "tool", "tool_call_id": "t1", "content": "4"} + # Compact JSON, sort_keys, ensure_ascii=False; id rewritten to call_* prefix. + tc = out["messages"][1]["tool_calls"][0] + assert tc["type"] == "function" + assert tc["function"]["name"] == "calc" + assert tc["function"]["arguments"] == '{"q":"2+2"}', f"Compact JSON expected; got {tc['function']['arguments']!r}" + assert tc["id"].startswith("call_"), f"Tool call id must be rewritten to call_* prefix; got {tc['id']!r}" + tool_msg = out["messages"][2] + assert tool_msg["role"] == "tool" + assert tool_msg["content"] == "4" + assert tool_msg["tool_call_id"] == tc["id"], "tool_call_id must match tool_calls[].id after rewrite" def test_user_content_list_with_single_text_block() -> None: """A user message as a list of content blocks must be flattened to string content.""" with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": [ - {"type": "text", "text": "Hello there"}, - ]}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello there"}, + ], + }, + ], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["messages"] == [{"role": "user", "content": "Hello there"}] +def test_tool_call_arguments_use_compact_json_for_cache_stability() -> None: + """Regression test for llama-server prefix-cache misses. + + ``tool_call.function.arguments`` is rendered verbatim by llama-server's + chat template. Default ``json.dumps`` inserts ``', '`` and ``': '`` + separators; reference OpenAI clients (and Anthropic's own tool output) + use compact JSON. The whitespace difference produces different tokens + after BPE, busting the prefix cache when the same conversation is + routed via the proxy vs. a reference client. + + This test pins the proxy to compact JSON — fixing the bug means + tightening ``json.dumps`` in ``_convert_assistant_message`` to + ``separators=(",", ":")``. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "lookup"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "t1", "name": "lookup", "input": {"key": "value", "count": 42, "tag": "a b"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "ok"}, + ], + }, + ], + "tools": [ + { + "name": "lookup", + "description": "x", + "input_schema": {"type": "object"}, + }, + ], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + args_str = out["messages"][1]["tool_calls"][0]["function"]["arguments"] + # Compact JSON: no whitespace after the comma or colon separators. + assert ", " not in args_str, f"Compact JSON required for cache stability; got {args_str!r}" + assert ": " not in args_str, f"Compact JSON required for cache stability; got {args_str!r}" + # And it must round-trip back to the original dict. + assert json.loads(args_str) == {"key": "value", "count": 42, "tag": "a b"} + + +# --------------------------------------------------------------------------- +# Prefix-equivalence tests for llama-server prompt-cache stability. +# +# llama-server's prefix cache is keyed by the rendered prompt's token sequence. +# Two requests share cache if and only if the proxy's outgoing OpenAI payload +# is byte-identical up to the length of the shorter one. The tests below pin +# the canonical wire form so we can detect any drift — every failure is a +# candidate cache-busting hotspot. +# --------------------------------------------------------------------------- + + +def _canonical_wire(payload: dict[str, Any]) -> str: + """Canonical, byte-stable JSON serialisation of the outgoing payload. + + Mirrors what a well-behaved OpenAI client would put on the wire: + sorted keys, no whitespace, UTF-8 passthrough. The proxy's job is to + emit messages that hash to this canonical form regardless of client + quirks. + """ + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def test_tool_call_id_normalised_to_call_prefix() -> None: + """Anthropic emits ``toolu_*`` ids; OpenAI clients use ``call_*``. + + The chat template renders the id in the prompt for many models, so the + prefix differs from a reference client's request. The proxy must rewrite + both ``tool_calls[].id`` and the matching ``tool_call_id``. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "lookup"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_01abc123", "name": "lookup", "input": {"q": "x"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01abc123", "content": "ok"}, + ], + }, + ], + "tools": [{"name": "lookup", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + tc_id = out["messages"][1]["tool_calls"][0]["id"] + tool_msg_id = out["messages"][2]["tool_call_id"] + assert tc_id.startswith("call_"), f"Expected call_* prefix, got {tc_id!r}" + assert tool_msg_id.startswith("call_"), f"Expected call_* prefix, got {tool_msg_id!r}" + assert tc_id == tool_msg_id, f"tool_calls[].id ({tc_id!r}) must match tool_call_id ({tool_msg_id!r})" + + +def test_tool_call_arguments_preserve_unicode() -> None: + """Non-ASCII characters in tool arguments must serialise as UTF-8. + + Default ``json.dumps`` uses ``ensure_ascii=True`` and escapes unicode to + ``\\uXXXX``. The reference wire form is UTF-8 — the rendered prompt + diverges whenever tool inputs contain non-ASCII. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "搜索"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_01abc", "name": "lookup", "input": {"city": "北京", "emoji": "🔍"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01abc", "content": "ok"}, + ], + }, + ], + "tools": [{"name": "lookup", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + args_str = out["messages"][1]["tool_calls"][0]["function"]["arguments"] + assert "\\u" not in args_str, f"Unicode must be passed through; got {args_str!r}" + assert "北京" in args_str + assert "🔍" in args_str + + +def test_tool_call_arguments_stable_key_order() -> None: + """The arguments JSON must have a deterministic key order. + + Reference wire form sorts keys (``sort_keys=True``). Python dicts preserve + insertion order, so the proxy must canonicalise via ``json.dumps`` rather + than embedding the input dict directly to disk. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "lookup"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_01abc", "name": "lookup", "input": {"z": 1, "a": 2, "m": 3}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01abc", "content": "ok"}, + ], + }, + ], + "tools": [{"name": "lookup", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + args_str = out["messages"][1]["tool_calls"][0]["function"]["arguments"] + parsed = json.loads(args_str) + keys = list(parsed.keys()) + assert keys == sorted(keys), f"Arguments keys must be sorted; got {keys!r}" + + +def test_outgoing_payload_is_canonical_byte_stable() -> None: + """The outgoing payload must serialise to a single canonical form. + + Two requests with the same content must produce the same bytes on the + wire, regardless of input dict ordering. We assert the canonical form + round-trips cleanly and that the proxy's output preserves key order. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hello"}], + "tools": [{"name": "f", "description": "d", "input_schema": {"type": "object"}}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + # Round-trip must be idempotent (no whitespace, sorted keys). + wire = _canonical_wire(out) + reparsed = json.loads(wire) + again = _canonical_wire(reparsed) + assert wire == again, "Canonical wire form must be idempotent" + # And no leading whitespace, no indentation. + assert "\n" not in wire, "Wire form must not contain newlines" + assert ": " not in wire, "Wire form must not contain ': ' separators" + assert ", " not in wire, "Wire form must not contain ', ' separators" + + +def test_prefix_equivalence_across_turns() -> None: + """Turn N's outgoing messages must START with turn N-1's outgoing messages. + + llama-server's prefix cache only reuses if the rendered prompt of turn + N equals the rendered prompt of turn N-1 plus the new turn appended. + Structural prefix equivalence on the message list is the proxy-side + precondition for that. + """ + with _patched_empty_config(): + # Turn 1: just a user message. + req1 = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hello"}], + }, + ) + out1 = srv.convert_anthropic_to_litellm(req1) + # Turn 2: append an assistant reply. + req2 = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "hi"}, + ], + }, + ) + out2 = srv.convert_anthropic_to_litellm(req2) + # The first len(out1.messages) messages of out2 must equal out1's + # messages byte-for-byte. Use canonical wire form for comparison. + n = len(out1["messages"]) + prefix_new = [_canonical_wire(m) for m in out2["messages"][:n]] + prefix_old = [_canonical_wire(m) for m in out1["messages"]] + assert prefix_new == prefix_old, f"Turn N's prefix drifted:\n new: {prefix_new}\n old: {prefix_old}" + + +def test_prefix_equivalence_with_tool_turns() -> None: + """Tool-call turn's prefix must match the previous turn's full messages. + + Most common cache-busting scenario: a turn that adds an assistant + tool_call plus the user tool_result must have the prefix (everything + before the new tool turns) identical to the previous turn. + """ + with _patched_empty_config(): + # Turn 1: system + user. + req1 = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "system": "You are helpful.", + "messages": [ + {"role": "user", "content": "What is the weather?"}, + ], + "tools": [{"name": "weather", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out1 = srv.convert_anthropic_to_litellm(req1) + # Turn 2: same prefix + assistant tool_call + user tool_result. + req2 = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "system": "You are helpful.", + "messages": [ + {"role": "user", "content": "What is the weather?"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_01abc", "name": "weather", "input": {"city": "Paris"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01abc", "content": "sunny"}, + ], + }, + ], + "tools": [{"name": "weather", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out2 = srv.convert_anthropic_to_litellm(req2) + n = len(out1["messages"]) + prefix_new = [_canonical_wire(m) for m in out2["messages"][:n]] + prefix_old = [_canonical_wire(m) for m in out1["messages"]] + assert prefix_new == prefix_old, "Turn N's prefix (system + user) must match turn N-1 exactly." + + +def test_system_message_idempotent_across_turns() -> None: + """Same system field ⇒ byte-identical system message in outgoing payload.""" + with _patched_empty_config(): + req1 = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "You are a helpful assistant.", + "messages": [{"role": "user", "content": "hi"}], + }, + ) + req2 = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "You are a helpful assistant.", + "messages": [{"role": "user", "content": "bye"}], + }, + ) + out1 = srv.convert_anthropic_to_litellm(req1) + out2 = srv.convert_anthropic_to_litellm(req2) + assert _canonical_wire(out1["messages"][0]) == _canonical_wire(out2["messages"][0]), ( + "System message must be byte-stable when the input system field is unchanged." + ) + + +def test_tool_definitions_order_preserved() -> None: + """Tool list ordering must equal input ordering — clients append-only.""" + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "x"}], + "tools": [ + {"name": "z_last", "description": "", "input_schema": {"type": "object"}}, + {"name": "a_first", "description": "", "input_schema": {"type": "object"}}, + {"name": "m_mid", "description": "", "input_schema": {"type": "object"}}, + ], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + names = [t["function"]["name"] for t in out["tools"]] + assert names == ["z_last", "a_first", "m_mid"], f"Tool order must match input; got {names!r}" + + +def test_determinism_same_input_same_output() -> None: + """Converting the same request twice must produce the same wire form. + + Tool call ids are randomised (``call_``) so the byte-level hash + differs across runs. Assert determinism on the canonicalised message + payload after stripping the random-generated ids — that's what + llama-server's prefix cache is keyed on. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "system": "help", + "messages": [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_01abc", "name": "f", "input": {"a": 1, "b": 2, "c": "x y"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01abc", "content": "ok"}, + ], + }, + ], + "tools": [{"name": "f", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out1 = srv.convert_anthropic_to_litellm(req) + out2 = srv.convert_anthropic_to_litellm(req) + + # Strip random ids so the deterministic content can be compared. + def _strip_ids(payload: dict[str, Any]) -> dict[str, Any]: + payload = json.loads(_canonical_wire(payload)) + for msg in payload.get("messages", []): + for tc in msg.get("tool_calls", []): + tc["id"] = "" + if "tool_call_id" in msg: + msg["tool_call_id"] = "" + return payload + + assert _strip_ids(out1) == _strip_ids(out2), "Same input must produce deterministic output (modulo random ids)" + + +def test_no_anthropic_specific_fields_in_outgoing_messages() -> None: + """Only OpenAI fields must appear in outgoing message dicts. + + Anthropic uses ``tool_use_id``, ``cache_control``, ``signature``; + OpenAI uses ``tool_call_id``, ``tool_calls``. Any leakage busts the + prefix cache because the chat template renders the extra fields. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "x"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_01abc", "name": "f", "input": {"a": 1}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01abc", "content": "ok"}, + ], + }, + ], + "tools": [{"name": "f", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + allowed = {"role", "content", "name", "tool_call_id", "tool_calls"} + for msg in out["messages"]: + extras = set(msg.keys()) - allowed + assert not extras, f"Anthropic-specific fields leaked: {extras!r} in {msg!r}" + + +def test_tool_definitions_parameters_not_none() -> None: + """Tool ``parameters`` must always be a dict — never ``null``. + + OpenAI-compatible upstreams reject ``parameters: null``; reference + clients emit ``{}`` for empty schemas. The proxy must normalise. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "x"}], + "tools": [{"name": "f", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + for tool in out["tools"]: + params = tool["function"]["parameters"] + assert isinstance(params, dict), f"Tool parameters must be dict; got {type(params).__name__}" + assert "type" in params, "Tool parameters must declare 'type' so upstream validates schema" + + +def test_tool_definition_cache_control_stripped_from_parameters() -> None: + """Anthropic allows ``cache_control`` inside ``input_schema``; OpenAI doesn't. + + Reference wire form has no cache_control. Any leftover field renders + in the prompt and busts the prefix. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "x"}], + "tools": [ + { + "name": "f", + "description": "x", + "input_schema": { + "type": "object", + "cache_control": {"type": "ephemeral"}, + }, + }, + ], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + params = out["tools"][0]["function"]["parameters"] + assert "cache_control" not in params, f"cache_control must be stripped from parameters; got {params!r}" + + +def test_tool_result_string_content_passes_through_unchanged() -> None: + """String ``tool_result.content`` must hit the wire as the same string. + + The proxy currently appends trailing newlines / strips — both change + the prompt's tokens. Reference form is verbatim. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "x"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_01abc", "name": "f", "input": {}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01abc", "content": "exact result"}, + ], + }, + ], + "tools": [{"name": "f", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + tool_msg = next(m for m in out["messages"] if m.get("role") == "tool") + assert tool_msg["content"] == "exact result", f"Tool result content must be verbatim; got {tool_msg['content']!r}" + + +def test_empty_string_tool_result_content_not_replaced() -> None: + """Empty string ``tool_result.content`` must stay empty — not ``"..."``. + + The current code normalises empty content to the placeholder ``"..."`` + at the message level. That substitution is for empty ``user`` content, + not for tool result bodies. Exercise the full pipeline so the test + catches the post-sanitisation substitution. + """ + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "x"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_01abc", "name": "f", "input": {}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01abc", "content": ""}, + ], + }, + ], + "tools": [{"name": "f", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + srv.sanitize_messages_for_openai(out["messages"]) + tool_msg = next(m for m in out["messages"] if m.get("role") == "tool") + assert not tool_msg["content"], f"Empty tool result must stay empty; got {tool_msg['content']!r}" + + +# --- Image-bearing tool_result --- + + +def _make_tool_result_request(tool_result_content: object) -> srv.MessagesRequest: + """Build an Anthropic Messages request whose last user turn is a single + tool_result block with the given content. Wraps the boilerplate so each + image-tool-result test stays focused on the content shape it exercises. + """ + return _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "x"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_01abc", "name": "Read", "input": {"path": "/tmp/s.png"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01abc", "content": tool_result_content}, + ], + }, + ], + "tools": [{"name": "Read", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + + +def test_tool_result_with_image_block_emits_image_url() -> None: + """Image-only tool_result content list must emit an image_url part — + not a 100+KB stringified-JSON blob that the model can't see. + """ + img_source = {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="} + with _patched_empty_config(): + req = _make_tool_result_request([{"type": "image", "source": img_source}]) + out = srv.convert_anthropic_to_litellm(req) + tool_msg = next(m for m in out["messages"] if m.get("role") == "tool") + assert isinstance(tool_msg["content"], list), f"Image-bearing tool result must be a list, got {type(tool_msg['content']).__name__}" + assert tool_msg["content"] == [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}, + }, + ] + + +def test_tool_result_with_text_and_image_mixed() -> None: + """Mixed text + image list preserves source order: text part, then image_url part.""" + img_source = {"type": "base64", "media_type": "image/png", "data": "abc"} + with _patched_empty_config(): + req = _make_tool_result_request( + [ + {"type": "text", "text": "here is the file"}, + {"type": "image", "source": img_source}, + ], + ) + out = srv.convert_anthropic_to_litellm(req) + tool_msg = next(m for m in out["messages"] if m.get("role") == "tool") + assert tool_msg["content"] == [ + {"type": "text", "text": "here is the file"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}}, + ] + + +def test_tool_result_image_only_dict_single() -> None: + """Single image dict (not wrapped in a list) still produces an image_url list.""" + img_source = {"type": "base64", "media_type": "image/jpeg", "data": "xyz"} + with _patched_empty_config(): + req = _make_tool_result_request({"type": "image", "source": img_source}) + out = srv.convert_anthropic_to_litellm(req) + tool_msg = next(m for m in out["messages"] if m.get("role") == "tool") + assert tool_msg["content"] == [ + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,xyz"}, + }, + ] + + +def test_tool_result_text_only_unchanged_string() -> None: + """Regression: a list of text-only blocks still flattens to a string. + + Three pre-existing tests assert string tool content; this is the + list-of-text-blocks equivalent of test_tool_result_string_content_passes_through_unchanged. + """ + with _patched_empty_config(): + req = _make_tool_result_request( + [ + {"type": "text", "text": "first"}, + {"type": "text", "text": "second"}, + ], + ) + out = srv.convert_anthropic_to_litellm(req) + tool_msg = next(m for m in out["messages"] if m.get("role") == "tool") + assert tool_msg["content"] == "first\nsecond", f"Text-only list must flatten to newline-joined string; got {tool_msg['content']!r}" + + +def test_tool_result_text_block_with_null_text_does_not_crash() -> None: + """``{"type": "text", "text": null}`` must not raise ``TypeError`` later when + ``_tool_result_parts_from_list`` joins text parts — null is not a str. + """ + with _patched_empty_config(): + req = _make_tool_result_request( + [ + {"type": "text", "text": None}, + {"type": "text", "text": "after"}, + ], + ) + out = srv.convert_anthropic_to_litellm(req) + tool_msg = next(m for m in out["messages"] if m.get("role") == "tool") + assert tool_msg["content"] == "after", f"Null text must collapse to empty; got {tool_msg['content']!r}" + + +def test_tool_result_with_multiple_images() -> None: + """Multiple images in one tool_result emit multiple image_url parts.""" + src1 = {"type": "base64", "media_type": "image/png", "data": "aaa"} + src2 = {"type": "base64", "media_type": "image/png", "data": "bbb"} + with _patched_empty_config(): + req = _make_tool_result_request( + [ + {"type": "image", "source": src1}, + {"type": "image", "source": src2}, + ], + ) + out = srv.convert_anthropic_to_litellm(req) + tool_msg = next(m for m in out["messages"] if m.get("role") == "tool") + assert tool_msg["content"] == [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,aaa"}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,bbb"}}, + ] + + +def test_tool_result_image_with_url_source() -> None: + """``source.type=url`` exercises the bare-URL branch of convert_image_block.""" + with _patched_empty_config(): + req = _make_tool_result_request( + [ + {"type": "image", "source": {"type": "url", "url": "https://example.com/cat.jpg"}}, + ], + ) + out = srv.convert_anthropic_to_litellm(req) + tool_msg = next(m for m in out["messages"] if m.get("role") == "tool") + assert tool_msg["content"] == [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/cat.jpg"}, + }, + ] + + +def test_tool_result_empty_list_content_stays_empty() -> None: + """``content=[]`` must round-trip as empty string (parity with empty-string test).""" + with _patched_empty_config(): + req = _make_tool_result_request([]) + out = srv.convert_anthropic_to_litellm(req) + tool_msg = next(m for m in out["messages"] if m.get("role") == "tool") + assert not tool_msg["content"], f"Empty list tool result must stay empty; got {tool_msg['content']!r}" + + +def test_orphaned_tool_result_with_image_folded_as_prose() -> None: + """Orphan tool_result (no matching tool_use) folds into user text — image lost to prose. + + The ghost id is meaningless, so we never emit a ``role=tool`` for it; the + image content gets stringified via the existing prose path. + """ + img_source = {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="} + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "x"}, + # tool_result references an id that no assistant turn emitted + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_ghost", "content": [{"type": "image", "source": img_source}]}, + ], + }, + ], + "tools": [{"name": "Read", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + tool_msgs = [m for m in out["messages"] if m.get("role") == "tool"] + assert not tool_msgs, f"Orphan tool_result must not produce role=tool; got {tool_msgs}" + folded = [ + m + for m in out["messages"] + if m.get("role") == "user" and isinstance(m.get("content"), str) and "Result from an earlier tool call" in m["content"] + ] + assert len(folded) == 1, f"Expected one folded user message; got {len(folded)}" + assert "image" in folded[0]["content"] + assert "iVBORw0KGgo" in folded[0]["content"] + + +def test_tool_result_with_image_after_sanitize_passes_list_through() -> None: + """Regression guard: sanitize_messages_for_openai must preserve list + content for ``role=tool`` (it has an early-continue for tool role). + """ + img_source = {"type": "base64", "media_type": "image/png", "data": "abc"} + with _patched_empty_config(): + req = _make_tool_result_request([{"type": "image", "source": img_source}]) + out = srv.convert_anthropic_to_litellm(req) + srv.sanitize_messages_for_openai(out["messages"]) + tool_msg = next(m for m in out["messages"] if m.get("role") == "tool") + assert isinstance(tool_msg["content"], list) + assert tool_msg["content"] == [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,abc"}, + }, + ] + + +def test_tool_result_image_survives_full_pipeline() -> None: + """End-to-end: tool_use → tool_result with image → next assistant turn. + + Confirms the image_url block survives the full convert → sanitize pipeline + and lives next to a fresh assistant message. + """ + img_source = {"type": "base64", "media_type": "image/png", "data": "img"} + with _patched_empty_config(): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "describe this"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_read", "name": "Read", "input": {"path": "/tmp/x.png"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_read", "content": [{"type": "image", "source": img_source}]}, + ], + }, + {"role": "assistant", "content": "looks like a chart"}, + ], + "tools": [{"name": "Read", "description": "x", "input_schema": {"type": "object"}}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + srv.sanitize_messages_for_openai(out["messages"]) + + roles = [m["role"] for m in out["messages"]] + assert roles == ["user", "assistant", "tool", "assistant"], f"Expected [user, assistant, tool, assistant]; got {roles}" + tool_msg = out["messages"][2] + assert tool_msg["content"] == [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,img"}, + }, + ] + + def test_user_content_list_with_text_and_image() -> None: """Text + image must produce a structured OpenAI content array (not flattened).""" with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": [ - {"type": "text", "text": "What is this?"}, - {"type": "image", "source": { - "type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo=", - }}, - ]}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgo=", + }, + }, + ], + }, + ], + }, + ) out = srv.convert_anthropic_to_litellm(req) msg = out["messages"][0] assert msg["role"] == "user" @@ -640,21 +1792,27 @@ def test_convert_image_block_unknown_source_falls_back_gracefully() -> None: # --- Tool edge cases --- + def test_dangling_tool_use_folded_into_text() -> None: """A tool_use with no matching tool_result (truncated history) must be turned into prose, not emitted as a tool_call — otherwise the model would have to answer for an unanswerable call. """ with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 200, - "messages": [ - {"role": "user", "content": "What's the weather?"}, - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "t1", "name": "get_weather", "input": {"city": "SF"}}, - ]}, - ], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "t1", "name": "get_weather", "input": {"city": "SF"}}, + ], + }, + ], + }, + ) out = srv.convert_anthropic_to_litellm(req) assistant = out["messages"][1] assert assistant["role"] == "assistant" @@ -668,15 +1826,20 @@ def test_orphaned_tool_result_folded_into_user_text() -> None: than emitted as a role='tool' message (which would dangle). """ with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 200, - "messages": [ - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "ghost", "content": "old data"}, - ]}, - ], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "ghost", "content": "old data"}, + ], + }, + ], + }, + ) out = srv.convert_anthropic_to_litellm(req) user_msgs = [m for m in out["messages"] if m["role"] == "user"] tool_msgs = [m for m in out["messages"] if m["role"] == "tool"] @@ -691,20 +1854,28 @@ def test_tool_use_and_tool_result_ordering() -> None: the assistant tool_call turn. """ with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 200, - "messages": [ - {"role": "user", "content": "Q"}, - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "t1", "name": "calc", "input": {"x": 1}}, - ]}, - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": "42"}, - {"type": "text", "text": "Thanks. Now also do Y."}, - ]}, - ], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 200, + "messages": [ + {"role": "user", "content": "Q"}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "t1", "name": "calc", "input": {"x": 1}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "42"}, + {"type": "text", "text": "Thanks. Now also do Y."}, + ], + }, + ], + }, + ) out = srv.convert_anthropic_to_litellm(req) roles = [m["role"] for m in out["messages"]] assert roles == ["user", "assistant", "tool", "user"], f"got {roles}" @@ -712,57 +1883,68 @@ def test_tool_use_and_tool_result_ordering() -> None: def test_tool_choice_any_passes_through() -> None: with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "x"}], - "tools": [{"name": "t", "description": "t", "input_schema": {"type": "object"}}], - "tool_choice": {"type": "any"}, - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "x"}], + "tools": [{"name": "t", "description": "t", "input_schema": {"type": "object"}}], + "tool_choice": {"type": "any"}, + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["tool_choice"] == "any" def test_tool_choice_unknown_type_falls_back_to_auto() -> None: with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "x"}], - "tools": [{"name": "t", "description": "t", "input_schema": {"type": "object"}}], - "tool_choice": {"type": "bogus_type"}, - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "x"}], + "tools": [{"name": "t", "description": "t", "input_schema": {"type": "object"}}], + "tool_choice": {"type": "bogus_type"}, + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["tool_choice"] == "auto" def test_tool_choice_tool_with_missing_name_falls_back_to_auto() -> None: with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "x"}], - "tools": [{"name": "t", "description": "t", "input_schema": {"type": "object"}}], - "tool_choice": {"type": "tool"}, - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "x"}], + "tools": [{"name": "t", "description": "t", "input_schema": {"type": "object"}}], + "tool_choice": {"type": "tool"}, + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["tool_choice"] == "auto" # --- Response conversion --- + def test_convert_litellm_to_anthropic_text_response() -> None: - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + }, + ) response = { "id": "resp-1", - "choices": [{ - "message": {"role": "assistant", "content": "Hello there"}, - "finish_reason": "stop", - }], + "choices": [ + { + "message": {"role": "assistant", "content": "Hello there"}, + "finish_reason": "stop", + }, + ], "usage": {"prompt_tokens": 11, "completion_tokens": 5}, } out = srv.convert_litellm_to_anthropic(response, req) @@ -776,24 +1958,30 @@ def test_convert_litellm_to_anthropic_text_response() -> None: def test_convert_litellm_to_anthropic_tool_use_response() -> None: - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + }, + ) response = { - "choices": [{ - "message": { - "role": "assistant", - "content": None, - "tool_calls": [{ - "id": "call_1", - "type": "function", - "function": {"name": "calc", "arguments": '{"q": "2+2"}'}, - }], - }, - "finish_reason": "tool_calls", - }], + "choices": [ + { + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "calc", "arguments": '{"q": "2+2"}'}, + }, + ], + }, + "finish_reason": "tool_calls", + }, + ], "usage": {"prompt_tokens": 7, "completion_tokens": 3}, } out = srv.convert_litellm_to_anthropic(response, req) @@ -801,16 +1989,21 @@ def test_convert_litellm_to_anthropic_tool_use_response() -> None: assert len(out.content) == 1 block = out.content[0] assert block.model_dump() == { - "type": "tool_use", "id": "call_1", "name": "calc", "input": {"q": "2+2"}, + "type": "tool_use", + "id": "call_1", + "name": "calc", + "input": {"q": "2+2"}, } def test_convert_litellm_to_anthropic_generates_id_when_missing() -> None: - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + }, + ) response = { "choices": [{"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], } @@ -821,11 +2014,13 @@ def test_convert_litellm_to_anthropic_generates_id_when_missing() -> None: def test_convert_litellm_to_anthropic_maps_length_stop_reason() -> None: - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 5, - "messages": [{"role": "user", "content": "Tell me a long story"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 5, + "messages": [{"role": "user", "content": "Tell me a long story"}], + }, + ) response = { "choices": [{"message": {"role": "assistant", "content": "Once upon..."}, "finish_reason": "length"}], } @@ -834,11 +2029,13 @@ def test_convert_litellm_to_anthropic_maps_length_stop_reason() -> None: def test_convert_litellm_to_anthropic_handles_empty_choices() -> None: - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + }, + ) response = {"choices": []} out = srv.convert_litellm_to_anthropic(response, req) assert [b.model_dump() for b in out.content] == [{"type": "text", "text": ""}] @@ -847,11 +2044,13 @@ def test_convert_litellm_to_anthropic_handles_empty_choices() -> None: def test_convert_litellm_to_anthropic_uses_keyword_usage_args() -> None: """Regression: Usage(...) must be built with keyword args, not positional.""" - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + }, + ) response = { "choices": [{"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 4, "completion_tokens": 2}, @@ -863,11 +2062,13 @@ def test_convert_litellm_to_anthropic_uses_keyword_usage_args() -> None: def test_convert_litellm_to_anthropic_recovers_from_broken_usage() -> None: """Regression: even if usage is malformed, we still return a usable response.""" - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "Hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + }, + ) response = { "choices": [{"message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], "usage": "not-a-dict", @@ -882,16 +2083,20 @@ def test_convert_litellm_to_anthropic_handles_string_arguments_gracefully() -> N """Tool arguments are typically strings (JSON-encoded); a non-string must not crash.""" req = _base_request() response = { - "choices": [{ - "message": { - "content": None, - "tool_calls": [{ - "id": "c1", - "function": {"name": "calc", "arguments": {"already": "a dict"}}, - }], - }, - "finish_reason": "tool_calls", - }], + "choices": [ + { + "message": { + "content": None, + "tool_calls": [ + { + "id": "c1", + "function": {"name": "calc", "arguments": {"already": "a dict"}}, + }, + ], + }, + "finish_reason": "tool_calls", + }, + ], } out = srv.convert_litellm_to_anthropic(response, req) block = out.content[0] @@ -904,16 +2109,20 @@ def test_convert_litellm_to_anthropic_recovers_from_invalid_json_arguments() -> """If tool arguments are not valid JSON, we must not crash — fall back to a raw wrapper.""" req = _base_request() response = { - "choices": [{ - "message": { - "content": None, - "tool_calls": [{ - "id": "c1", - "function": {"name": "calc", "arguments": "{this is not json"}, - }], - }, - "finish_reason": "tool_calls", - }], + "choices": [ + { + "message": { + "content": None, + "tool_calls": [ + { + "id": "c1", + "function": {"name": "calc", "arguments": "{this is not json"}, + }, + ], + }, + "finish_reason": "tool_calls", + }, + ], } out = srv.convert_litellm_to_anthropic(response, req) block = out.content[0] @@ -923,17 +2132,20 @@ def test_convert_litellm_to_anthropic_recovers_from_invalid_json_arguments() -> # --- System messages --- + def test_system_message_list_with_only_text_blocks() -> None: with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": [ - {"type": "text", "text": "You are concise."}, - {"type": "text", "text": "Answer in English."}, - ], - "messages": [{"role": "user", "content": "Hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": [ + {"type": "text", "text": "You are concise."}, + {"type": "text", "text": "Answer in English."}, + ], + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) sys_msg = out["messages"][0] assert sys_msg["role"] == "system" @@ -948,18 +2160,20 @@ def test_system_role_message_in_messages_array_is_hoisted() -> None: request — never inline as a 'system' message in the middle. """ with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": "You are concise.", - "messages": [ - {"role": "system", "content": "[skill: foo] description"}, - {"role": "user", "content": "Hi"}, - {"role": "assistant", "content": "Hello!"}, - {"role": "system", "content": [{"type": "text", "text": "[skill: baz] more"}]}, - {"role": "user", "content": "and now?"}, - ], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "You are concise.", + "messages": [ + {"role": "system", "content": "[skill: foo] description"}, + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "system", "content": [{"type": "text", "text": "[skill: baz] more"}]}, + {"role": "user", "content": "and now?"}, + ], + }, + ) out = srv.convert_anthropic_to_litellm(req) roles = [m["role"] for m in out["messages"]] assert roles == ["system", "user", "assistant", "user"], f"got {roles}" @@ -967,20 +2181,22 @@ def test_system_role_message_in_messages_array_is_hoisted() -> None: foo = sys_content.index("[skill: foo]") baz = sys_content.index("[skill: baz]") sys = sys_content.index("You are concise.") - assert foo < baz < sys + assert sys < foo < baz def test_system_role_message_with_string_content_is_hoisted() -> None: """A system message with string content is treated identically to a list.""" with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [ - {"role": "system", "content": "top-of-stream reminder"}, - {"role": "user", "content": "Hi"}, - ], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [ + {"role": "system", "content": "top-of-stream reminder"}, + {"role": "user", "content": "Hi"}, + ], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["messages"][0] == {"role": "system", "content": "top-of-stream reminder"} assert out["messages"][1]["role"] == "user" @@ -1010,21 +2226,109 @@ def test_prompt_remap_strip_todo_reminder() -> None: "on. Use it if it's relevant to the current work. This is just a " "gentle reminder - ignore if not applicable.\n\n" ) - with _patched_empty_config(), _patched_prompt_remaps([ - {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, - ]): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": "You are concise." + "\n\n" + reminder, - "messages": [{"role": "user", "content": "Hi"}], - }) + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, + ], + ), + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "You are concise." + "\n\n" + reminder, + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) sys = out["messages"][0]["content"] assert "TodoWrite" not in sys, f"Reminder must be stripped; got {sys!r}" assert "You are concise." in sys +def test_prompt_remap_strip_task_tools_reminder() -> None: + """TaskCreate/TaskUpdate reminder (alt variant) is also stripped.""" + reminder = ( + "The task tools haven't been used recently. If you're working on " + "tasks that would benefit from tracking progress, consider using " + "TaskCreate to add new tasks and TaskUpdate to update task status " + "(set to in_progress when starting, completed when done). Also " + "consider cleaning up the task list if it has become stale. Only " + "use these if relevant to the current work. This is just a gentle " + "reminder - ignore if not applicable.\n\n\n" + ) + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"The task tools haven't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, + ], + ), + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "You are concise." + "\n\n" + reminder, + "messages": [{"role": "user", "content": "Hi"}], + }, + ) + out = srv.convert_anthropic_to_litellm(req) + sys = out["messages"][0]["content"] + assert "TaskCreate" not in sys, f"Reminder must be stripped; got {sys!r}" + assert "TaskUpdate" not in sys, f"Reminder must be stripped; got {sys!r}" + assert "You are concise." in sys + + +def test_prompt_remap_canonical_across_reminder_variants() -> None: + """Both reminder variants yield the same outgoing system prompt as the no-reminder case.""" + request_body = { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hi"}], + } + todo_reminder = ( + "\n\nThe TodoWrite tool hasn't been used recently. If you're working " + "on tasks that would benefit from tracking progress, consider using " + "the TodoWrite tool to track progress. Also consider cleaning up the " + "todo list if has become stale and no longer matches what you are " + "working on. Use it if it's relevant to the current work. This is " + "just a gentle reminder - ignore if not applicable.\n\n" + ) + task_tools_reminder = ( + "\n\nThe task tools haven't been used recently. If you're working on " + "tasks that would benefit from tracking progress, consider using " + "TaskCreate to add new tasks and TaskUpdate to update task status " + "(set to in_progress when starting, completed when done). Also " + "consider cleaning up the task list if it has become stale. Only " + "use these if relevant to the current work. This is just a gentle " + "reminder - ignore if not applicable.\n\n\n" + ) + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + { + "match": r"The (?:TodoWrite tool hasn't|task tools haven't) been used recently" + r".*?ignore if not applicable\.?\n+", + "replacement": "", + }, + ], + ), + ): + out_none = srv.convert_anthropic_to_litellm(_make_request({**request_body, "system": "You are concise."})) + out_todo = srv.convert_anthropic_to_litellm(_make_request({**request_body, "system": "You are concise." + todo_reminder})) + out_task = srv.convert_anthropic_to_litellm(_make_request({**request_body, "system": "You are concise." + task_tools_reminder})) + sys_none = out_none["messages"][0]["content"] + sys_todo = out_todo["messages"][0]["content"] + sys_task = out_task["messages"][0]["content"] + assert sys_none == sys_todo == sys_task, ( + f"All three outgoing prompts must be byte-identical;\nnone: {sys_none!r}\ntodo: {sys_todo!r}\ntask: {sys_task!r}" + ) + + def test_prompt_remap_canonical_across_reminder_states() -> None: """Reminder on vs off → byte-identical outgoing system prompt.""" request_body = { @@ -1040,31 +2344,40 @@ def test_prompt_remap_canonical_across_reminder_states() -> None: "working on. Use it if it's relevant to the current work. This is " "just a gentle reminder - ignore if not applicable.\n\n" ) - with _patched_empty_config(), _patched_prompt_remaps([ - {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, - ]): + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, + ], + ), + ): req_on = _make_request({**request_body, "system": "You are concise." + reminder}) req_off = _make_request({**request_body, "system": "You are concise."}) out_on = srv.convert_anthropic_to_litellm(req_on) out_off = srv.convert_anthropic_to_litellm(req_off) sys_on = out_on["messages"][0]["content"] sys_off = out_off["messages"][0]["content"] - assert sys_on == sys_off, ( - f"Outgoing system prompt must match across reminder states;\n" - f"on: {sys_on!r}\noff: {sys_off!r}" - ) + assert sys_on == sys_off, f"Outgoing system prompt must match across reminder states;\non: {sys_on!r}\noff: {sys_off!r}" def test_prompt_remap_strips_trailing_newlines() -> None: - with _patched_empty_config(), _patched_prompt_remaps([ - {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, - ]): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": "You are concise.\n\nThe TodoWrite tool hasn't been used recently. ignore if not applicable.\n\n", - "messages": [{"role": "user", "content": "Hi"}], - }) + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, + ], + ), + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "You are concise.\n\nThe TodoWrite tool hasn't been used recently. ignore if not applicable.\n\n", + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) # No stray newlines, no leading/trailing whitespace. assert out["messages"][0]["content"] == "You are concise.", ( @@ -1074,49 +2387,149 @@ def test_prompt_remap_strips_trailing_newlines() -> None: def test_prompt_remap_no_match_passes_through() -> None: """Patterns with no match leave the system prompt untouched.""" - with _patched_empty_config(), _patched_prompt_remaps([ - {"match": r"never-present-pattern-\d+", "replacement": ""}, - ]): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": "You are concise.", - "messages": [{"role": "user", "content": "Hi"}], - }) + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"never-present-pattern-\d+", "replacement": ""}, + ], + ), + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "You are concise.", + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["messages"][0]["content"] == "You are concise." +def test_prompt_remap_logs_when_stripping() -> None: + """WARNING log fires when any entry actually replaces text.""" + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"The task tools haven't been used recently.*?ignore if not applicable\.\n+", "replacement": ""}, + ], + ), + patch.object(srv.logger, "warning") as warning, + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "You are concise.\n\nThe task tools haven't been used recently. ignore if not applicable.\n", + "messages": [{"role": "user", "content": "Hi"}], + }, + ) + srv.convert_anthropic_to_litellm(req) + strip_logs = [c for c in warning.call_args_list if c.args and c.args[0].startswith("prompt_remap: stripped")] + assert len(strip_logs) == 1, f"expected one strip log, got {len(strip_logs)}: {strip_logs}" + _msg, stripped, matches, _es, fired, _word = strip_logs[0].args + assert stripped > 0 + assert matches == 1 + assert fired == 1 + + +def test_prompt_remap_silent_when_no_strip() -> None: + """No WARNING log fires when no entry replaces anything.""" + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"never-present-pattern-\d+", "replacement": ""}, + ], + ), + patch.object(srv.logger, "warning") as warning, + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "You are concise.", + "messages": [{"role": "user", "content": "Hi"}], + }, + ) + srv.convert_anthropic_to_litellm(req) + strip_logs = [c for c in warning.call_args_list if c.args and c.args[0].startswith("prompt_remap: stripped")] + assert strip_logs == [], f"expected no strip log, got {strip_logs}" + + +def test_prompt_remap_logs_match_count_when_pattern_fires_multiple_times() -> None: + """When the same reminder appears N times, log says 'N matches' so accumulation is visible.""" + reminder = "The task tools haven't been used recently. ignore if not applicable.\n" + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"The task tools haven't been used recently.*?ignore if not applicable\.\n+", "replacement": ""}, + ], + ), + patch.object(srv.logger, "warning") as warning, + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "You are concise.\n\n" + reminder + reminder + reminder, + "messages": [{"role": "user", "content": "Hi"}], + }, + ) + srv.convert_anthropic_to_litellm(req) + strip_logs = [c for c in warning.call_args_list if c.args and c.args[0].startswith("prompt_remap: stripped")] + assert len(strip_logs) == 1 + _msg, _stripped, matches, _es, _fired, _word = strip_logs[0].args + assert matches == 3, f"expected 3 matches for 3 reminder copies; got {matches}" + + def test_prompt_remap_multiple_entries_applied_in_order() -> None: """Entries are applied sequentially; later matches see the already-rewritten text.""" - with _patched_empty_config(), _patched_prompt_remaps([ - {"match": r"REMINDER_INNER", "replacement": ""}, - {"match": r"REMINDER_OUTER", "replacement": ""}, - ]): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": "Keep REMINDER_OUTER marker [REMINDER_INNER content] visible.", - "messages": [{"role": "user", "content": "Hi"}], - }) + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"REMINDER_INNER", "replacement": ""}, + {"match": r"REMINDER_OUTER", "replacement": ""}, + ], + ), + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "Keep REMINDER_OUTER marker [REMINDER_INNER content] visible.", + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["messages"][0]["content"] == "Keep marker [ content] visible." def test_prompt_remap_handles_list_system_field() -> None: """Anthropic's `system` can be a list of content blocks, not just a string.""" - with _patched_empty_config(), _patched_prompt_remaps([ - {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, - ]): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": [ - {"type": "text", "text": "You are concise."}, - {"type": "text", "text": "The TodoWrite tool hasn't been used recently. ignore if not applicable.\n\n"}, + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, ], - "messages": [{"role": "user", "content": "Hi"}], - }) + ), + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": [ + {"type": "text", "text": "You are concise."}, + {"type": "text", "text": "The TodoWrite tool hasn't been used recently. ignore if not applicable.\n\n"}, + ], + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) sys = out["messages"][0]["content"] assert "TodoWrite" not in sys @@ -1125,17 +2538,24 @@ def test_prompt_remap_handles_list_system_field() -> None: def test_prompt_remap_handles_inband_system_messages() -> None: """In-band role='system' messages (Claude Code 2.1.154+) are also remapped.""" - with _patched_empty_config(), _patched_prompt_remaps([ - {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, - ]): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [ - {"role": "system", "content": "The TodoWrite tool hasn't been used recently. ignore if not applicable.\n\n"}, - {"role": "user", "content": "Hi"}, + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, ], - }) + ), + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [ + {"role": "system", "content": "The TodoWrite tool hasn't been used recently. ignore if not applicable.\n\n"}, + {"role": "user", "content": "Hi"}, + ], + }, + ) out = srv.convert_anthropic_to_litellm(req) # The system message was the entire reminder — stripped to "" → dropped. assert out["messages"][0]["role"] == "user" @@ -1144,15 +2564,22 @@ def test_prompt_remap_handles_inband_system_messages() -> None: def test_prompt_remap_empty_after_strip_returns_no_system_message() -> None: """If the entire system prompt is the reminder, the system message is dropped.""" - with _patched_empty_config(), _patched_prompt_remaps([ - {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, - ]): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": "The TodoWrite tool hasn't been used recently. ignore if not applicable.\n\n", - "messages": [{"role": "user", "content": "Hi"}], - }) + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"The TodoWrite tool hasn't been used recently.*?ignore if not applicable\.?\n+", "replacement": ""}, + ], + ), + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "The TodoWrite tool hasn't been used recently. ignore if not applicable.\n\n", + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["messages"][0]["role"] == "user" assert len(out["messages"]) == 1 @@ -1161,66 +2588,222 @@ def test_prompt_remap_empty_after_strip_returns_no_system_message() -> None: def test_prompt_remap_no_remaps_no_change() -> None: """Empty config leaves the system prompt verbatim — default behaviour unchanged.""" with _patched_empty_config(), _patched_prompt_remaps([]): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": "The TodoWrite tool hasn't been used recently.", - "messages": [{"role": "user", "content": "Hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "The TodoWrite tool hasn't been used recently.", + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert "TodoWrite" in out["messages"][0]["content"] def test_prompt_remap_compile_failure_skipped() -> None: """Bad regexes are warned and skipped — boot must not fail.""" - with _patched_empty_config(), _patched_prompt_remaps([ - {"match": r"[unclosed", "replacement": ""}, # re.error on compile - {"match": r"valid-pattern", "replacement": "X"}, - ]): - # The valid one still applies; the broken one is dropped. - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": "Then valid-pattern here.", - "messages": [{"role": "user", "content": "Hi"}], - }) + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"[unclosed", "replacement": ""}, # re.error on compile + {"match": r"valid-pattern", "replacement": "X"}, + ], + ), + ): + # The valid one still applies; the broken one is dropped. + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "Then valid-pattern here.", + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["messages"][0]["content"] == "Then X here." def test_prompt_remap_bad_match_string_skipped() -> None: """Non-string `match` entries are warned and skipped.""" - with _patched_empty_config(), _patched_prompt_remaps([ - {"match": 123, "replacement": ""}, # ty: ignore[list-item] - {"match": r"real-match", "replacement": "Y"}, - ]): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": "real-match applies", - "messages": [{"role": "user", "content": "Hi"}], - }) + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ # ty: ignore[invalid-argument-type] — non-string match is the regression input + {"match": 123, "replacement": ""}, + {"match": r"real-match", "replacement": "Y"}, + ], + ), + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "real-match applies", + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["messages"][0]["content"] == "Y applies" def test_prompt_remap_replacement_supports_newlines() -> None: """`\\n` in the replacement string becomes a real newline.""" - with _patched_empty_config(), _patched_prompt_remaps([ - {"match": r"BLOCK", "replacement": "\n"}, - ]): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "system": "before BLOCK after", - "messages": [{"role": "user", "content": "Hi"}], - }) + with ( + _patched_empty_config(), + _patched_prompt_remaps( + [ + {"match": r"BLOCK", "replacement": "\n"}, + ], + ), + ): + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "system": "before BLOCK after", + "messages": [{"role": "user", "content": "Hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["messages"][0]["content"] == "before \n after" +@contextlib.contextmanager +def _patched_env(name: str, value: str) -> Iterator[None]: + """Temporarily set an env var; restore on exit.""" + original = os.environ.get(name) + os.environ[name] = value + try: + yield + finally: + if original is None: + os.environ.pop(name, None) + else: + os.environ[name] = original + + +@contextlib.contextmanager +def _patched_cwd(path: pathlib.Path) -> Iterator[None]: + """Temporarily chdir; restore on exit.""" + original = pathlib.Path.cwd() + os.chdir(path) + try: + yield + finally: + os.chdir(original) + + +def _reset_cache_matcher() -> None: + """Drop the @cache singleton so each test gets a fresh matcher.""" + srv._get_cache_matcher.cache_clear() + + +def test_debug_cache_dump_disabled_by_default() -> None: + """Without PROXY_DEBUG_CACHE_DUMP, no debug directory is created and no files written.""" + tmp = tempfile.mkdtemp(prefix="ccp-debug-") + try: + with _patched_cwd(pathlib.Path(tmp)), _patched_env("PROXY_DEBUG_CACHE_DUMP", ""): + _reset_cache_matcher() + srv._debug_dump_outgoing_payload({"messages": [{"role": "user", "content": "hi"}]}) + srv._debug_dump_outgoing_payload({"messages": [{"role": "user", "content": "hi again"}]}) + assert not (pathlib.Path(tmp) / ".claude-code-proxy").exists(), "Disabled flag must not create the debug directory" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def test_debug_cache_dump_does_not_mutate_payload() -> None: + """The matcher scrubs ids in copies; the caller's payload keeps its original tool_call ids.""" + payload = { + "messages": [ + { + "role": "assistant", + "tool_calls": [{"id": "call_abc123", "type": "function", "function": {"name": "f"}}], + }, + {"role": "tool", "tool_call_id": "call_abc123", "content": "result"}, + ], + "tools": [{"type": "function", "function": {"name": "f"}}], + } + snapshot = json.loads(json.dumps(payload)) # deep copy for comparison + tmp = tempfile.mkdtemp(prefix="ccp-debug-") + try: + with _patched_cwd(pathlib.Path(tmp)), _patched_env("PROXY_DEBUG_CACHE_DUMP", "true"): + _reset_cache_matcher() + srv._debug_dump_outgoing_payload(payload) + srv._debug_dump_outgoing_payload(payload) # second call hits history + assert payload == snapshot, f"observe() must not mutate the caller's payload; got diff:\nbefore: {snapshot}\nafter: {payload}" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def test_debug_cache_dump_enabled_prefix_hit_creates_artifacts() -> None: + """PROXY_DEBUG_CACHE_DUMP=true creates $cwd/.claude-code-proxy/prompts/ when an outgoing + is a prefix extension of a prior one (cache-hit evidence).""" + tmp = tempfile.mkdtemp(prefix="ccp-debug-") + try: + prompts_dir = pathlib.Path(tmp) / ".claude-code-proxy" / "prompts" + payload_v1 = {"messages": [{"role": "user", "content": "hi"}]} + payload_v2 = { + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ], + } + with _patched_cwd(pathlib.Path(tmp)), _patched_env("PROXY_DEBUG_CACHE_DUMP", "true"): + _reset_cache_matcher() + srv._debug_dump_outgoing_payload(payload_v1) + # First call: no history → no directory, no files. + assert not prompts_dir.exists(), "No history → no debug directory should be created" + srv._debug_dump_outgoing_payload(payload_v2) + assert prompts_dir.is_dir(), f"Expected {prompts_dir} to exist after a prefix hit" + json_files = list(prompts_dir.glob("*-prefix_hit-*.json")) + assert len(json_files) == 2, f"Expected -new.json + -old.json; got {json_files}" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def test_debug_cache_dump_matches_immediately_prior_request() -> None: + """Monotonic growth → each new request matches the immediately-prior one, not the oldest.""" + tmp = tempfile.mkdtemp(prefix="ccp-debug-") + try: + prompts_dir = pathlib.Path(tmp) / ".claude-code-proxy" / "prompts" + + def make(n: int) -> dict[str, object]: + return {"messages": [{"role": "user", "content": f"msg-{i}"} for i in range(n)]} + + with _patched_cwd(pathlib.Path(tmp)), _patched_env("PROXY_DEBUG_CACHE_DUMP", "true"): + _reset_cache_matcher() + for n in (3, 5, 8): + srv._debug_dump_outgoing_payload(make(n)) + hits = sorted(prompts_dir.glob("*-prefix_hit-*.json")) + assert len(hits) == 4, f"Expected 2 prefix_hits x 2 files; got {hits}" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def test_debug_cache_dump_enabled_fuzzy_match_writes_artifacts() -> None: + """Two payloads that share ≥ 0.6 SequenceMatcher ratio but aren't a structural prefix + trigger ``fuzzy_match`` artifacts (cache-busting suspect).""" + tmp = tempfile.mkdtemp(prefix="ccp-debug-") + try: + prompts_dir = pathlib.Path(tmp) / ".claude-code-proxy" / "prompts" + # Same messages except one trailing punctuation flip — high SequenceMatcher ratio, + # not a structural prefix (different final char). + payload_v1 = {"messages": [{"role": "user", "content": "Hello"}]} + payload_v2 = {"messages": [{"role": "user", "content": "Hello!"}]} + with _patched_cwd(pathlib.Path(tmp)), _patched_env("PROXY_DEBUG_CACHE_DUMP", "true"): + _reset_cache_matcher() + srv._debug_dump_outgoing_payload(payload_v1) + srv._debug_dump_outgoing_payload(payload_v2) + fuzzy_files = list(prompts_dir.glob("*-fuzzy_match-*.json")) + assert fuzzy_files, f"Expected fuzzy_match artifacts; got {list(prompts_dir.glob('*'))}" + finally: + shutil.rmtree(tmp, ignore_errors=True) + + # --- Content block assembly --- + def test_build_content_blocks_text_only() -> None: """No reasoning -> just a text block.""" blocks = srv._build_content_blocks("hi", "", []) @@ -1253,13 +2836,15 @@ def test_convert_litellm_to_anthropic_uses_reasoning_content() -> None: """ req = _base_request() response = { - "choices": [{ - "message": { - "content": "final answer", - "reasoning_content": "step by step", + "choices": [ + { + "message": { + "content": "final answer", + "reasoning_content": "step by step", + }, + "finish_reason": "stop", }, - "finish_reason": "stop", - }], + ], } out = srv.convert_litellm_to_anthropic(response, req) dumped = [b.model_dump(exclude_none=True) for b in out.content] @@ -1276,18 +2861,23 @@ def test_request_accepts_thinking_block_in_history() -> None: OpenAI call (OpenAI has no equivalent concept). """ with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [ - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": [ - {"type": "text", "text": "hello"}, - {"type": "thinking", "thinking": "I should say hi back.", "signature": ""}, - ]}, - {"role": "user", "content": "and now?"}, - ], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "hello"}, + {"type": "thinking", "thinking": "I should say hi back.", "signature": ""}, + ], + }, + {"role": "user", "content": "and now?"}, + ], + }, + ) out = srv.convert_anthropic_to_litellm(req) assistant = out["messages"][1] assert assistant["role"] == "assistant" @@ -1297,6 +2887,7 @@ def test_request_accepts_thinking_block_in_history() -> None: # --- Think stream parser --- + def test_think_stream_parser_text_only() -> None: """Plain text with no tags passes through verbatim.""" p = srv._ThinkStreamParser() @@ -1405,6 +2996,7 @@ def test_think_stream_parser_unclosed_at_flush() -> None: # --- Streaming --- + async def test_streaming_text_only_emits_required_events() -> None: """A pure-text stream must emit message_start, ping, deltas, message_delta, message_stop, [DONE].""" req = _base_request(stream=True) @@ -1416,8 +3008,7 @@ async def test_streaming_text_only_emits_required_events() -> None: events = await _run_stream(chunks, req) types = [e["type"] for e in events] - for required in ("message_start", "content_block_start", "content_block_delta", - "content_block_stop", "message_delta", "message_stop"): + for required in ("message_start", "content_block_start", "content_block_delta", "content_block_stop", "message_delta", "message_stop"): assert required in types, f"missing {required}; got {types}" assert events[-1] == {"type": "[DONE]"}, "stream must end with [DONE]" @@ -1428,10 +3019,7 @@ async def test_streaming_text_only_accumulates_text() -> None: chunks = [_text_chunk("foo"), _text_chunk("bar"), _finish_chunk("stop")] events = await _run_stream(chunks, req) text = "".join( - e["delta"]["text"] - for e in events - if e["type"] == "content_block_delta" - and e.get("delta", {}).get("type") == "text_delta" + e["delta"]["text"] for e in events if e["type"] == "content_block_delta" and e.get("delta", {}).get("type") == "text_delta" ) assert text == "foobar" @@ -1440,9 +3028,16 @@ async def test_streaming_text_then_tool_call_closes_text_block_first() -> None: """When the model emits text and then a tool call, the text block must be closed before the tool_use block starts — Anthropic's SSE protocol requires this ordering. """ - req = _base_request(stream=True, tools=[{ - "name": "calc", "description": "calc", "input_schema": {"type": "object"}, - }]) + req = _base_request( + stream=True, + tools=[ + { + "name": "calc", + "description": "calc", + "input_schema": {"type": "object"}, + }, + ], + ) chunks = [ _text_chunk("Let me calculate"), _tool_delta_chunk(0, tool_id="call_1", name="calc", arguments='{"x":1}'), @@ -1455,30 +3050,30 @@ def find(predicate: Callable[[dict[str, Any]], bool]) -> int: text_close_idx = find(lambda e: e["type"] == "content_block_stop" and e.get("index") == 0) tool_start_idx = find( - lambda e: e["type"] == "content_block_start" - and (e.get("content_block") or {}).get("type") == "tool_use", - ) - assert text_close_idx < tool_start_idx, ( - f"text block (idx {text_close_idx}) must close before tool block opens (idx {tool_start_idx})" + lambda e: e["type"] == "content_block_start" and (e.get("content_block") or {}).get("type") == "tool_use", ) + assert text_close_idx < tool_start_idx, f"text block (idx {text_close_idx}) must close before tool block opens (idx {tool_start_idx})" async def test_streaming_tool_call_then_text_opens_new_block() -> None: """Text emitted after a tool_use must open a fresh text block (close-before-open).""" - req = _base_request(stream=True, tools=[{ - "name": "calc", "description": "calc", "input_schema": {"type": "object"}, - }]) + req = _base_request( + stream=True, + tools=[ + { + "name": "calc", + "description": "calc", + "input_schema": {"type": "object"}, + }, + ], + ) chunks = [ _tool_delta_chunk(0, tool_id="call_1", name="calc", arguments='{"x":1}'), _text_chunk("after tool"), _finish_chunk("tool_calls"), ] events = await _run_stream(chunks, req) - text_starts = [ - e for e in events - if e["type"] == "content_block_start" - and (e.get("content_block") or {}).get("type") == "text" - ] + text_starts = [e for e in events if e["type"] == "content_block_start" and (e.get("content_block") or {}).get("type") == "text"] assert len(text_starts) == 2, f"expected two text blocks (initial + post-tool), got {len(text_starts)}" # Canonical sequence: tool_use stop(0) → text start(1) → text stop(1). types = [e["type"] for e in events] @@ -1487,19 +3082,22 @@ async def test_streaming_tool_call_then_text_opens_new_block() -> None: async def test_streaming_tool_only_no_text() -> None: """Tool-only response: no text deltas, just tool_use block and finish.""" - req = _base_request(stream=True, tools=[{ - "name": "calc", "description": "calc", "input_schema": {"type": "object"}, - }]) + req = _base_request( + stream=True, + tools=[ + { + "name": "calc", + "description": "calc", + "input_schema": {"type": "object"}, + }, + ], + ) chunks = [ _tool_delta_chunk(0, tool_id="call_1", name="calc", arguments='{"x":1}'), _finish_chunk("tool_calls"), ] events = await _run_stream(chunks, req) - text_deltas = [ - e for e in events - if e["type"] == "content_block_delta" - and e.get("delta", {}).get("type") == "text_delta" - ] + text_deltas = [e for e in events if e["type"] == "content_block_delta" and e.get("delta", {}).get("type") == "text_delta"] assert text_deltas == [] stop = next(e for e in events if e["type"] == "message_delta") assert stop["delta"]["stop_reason"] == "tool_use" @@ -1517,6 +3115,38 @@ async def test_streaming_no_finish_reason_falls_back_to_end_turn() -> None: assert stop["delta"]["stop_reason"] == "end_turn" +async def test_streaming_no_finish_reason_with_tool_call_uses_tool_use_stop() -> None: + """Upstream closes the stream mid-tool-call without finish_reason. + + Previously the epilogue hardcoded end_turn, which made the Anthropic + SDK treat the response as "no pending work" and skip the tool result + request. Pick tool_use instead. + """ + req = _base_request( + stream=True, + tools=[ + { + "name": "calc", + "description": "x", + "input_schema": {"type": "object"}, + }, + ], + ) + chunks = [ + _tool_delta_chunk(index=0, tool_id="call_1", name="calc", arguments='{"q":'), + _tool_delta_chunk(index=0, arguments='"2"}'), + # No finish_reason chunk — upstream just terminated. + ] + events = await _run_stream(chunks, req) + types = [e["type"] for e in events] + assert "message_stop" in types + assert "content_block_stop" in types, "in-flight tool_use block must close" + stop = next(e for e in events if e["type"] == "message_delta") + assert stop["delta"]["stop_reason"] == "tool_use", ( + f"expected tool_use when upstream omits finish_reason mid-tool-call; got {stop['delta']['stop_reason']!r}" + ) + + async def test_streaming_emits_error_frame_on_chunk_failure() -> None: """A chunk that crashes the inner pipeline must yield `event: error` so the SDK raises APIStatusError instead of silently terminating the stream. @@ -1596,11 +3226,7 @@ def _boom_once(*args: Any, **kwargs: Any) -> Any: # Stream terminated cleanly via message_stop (not error). assert "message_stop" in types # good2/good3 must have flowed through — counter reset on next success. - text_deltas = [ - e for e in events - if e["type"] == "content_block_delta" - and e.get("delta", {}).get("type") == "text_delta" - ] + text_deltas = [e for e in events if e["type"] == "content_block_delta" and e.get("delta", {}).get("type") == "text_delta"] text_blob = "".join(d["delta"]["text"] for d in text_deltas) assert "good1" in text_blob assert "good2" in text_blob @@ -1636,9 +3262,7 @@ async def _interleaved() -> AsyncGenerator[dict[str, Any], None]: events = _parse_sse(raw) types = [e["type"] for e in events] # No error frame — interleaved bad chunks reset on the good one. - assert "error" not in types, ( - f"counter did not reset on success; got error frame. types={types}" - ) + assert "error" not in types, f"counter did not reset on success; got error frame. types={types}" assert "message_stop" in types @@ -1682,11 +3306,7 @@ def test_emit_failure_flushes_buffered_think_content() -> None: raw = list(srv._emit_failure(parser, tracker, 0, exc, "test failed")) events = _parse_sse(raw) - thinking_deltas = [ - e for e in events - if e["type"] == "content_block_delta" - and e.get("delta", {}).get("type") == "thinking_delta" - ] + thinking_deltas = [e for e in events if e["type"] == "content_block_delta" and e.get("delta", {}).get("type") == "thinking_delta"] assert thinking_deltas, "buffered think content must be flushed before error frame" assert any("partial reasoning" in d["delta"]["thinking"] for d in thinking_deltas) @@ -1715,20 +3335,23 @@ async def test_streaming_multiple_tool_calls_use_distinct_indices() -> None: """Parallel tool calls must each get their own SSE block index, with content_block_stop(N) emitted before content_block_start(N+1). """ - req = _base_request(stream=True, tools=[{ - "name": "calc", "description": "calc", "input_schema": {"type": "object"}, - }]) + req = _base_request( + stream=True, + tools=[ + { + "name": "calc", + "description": "calc", + "input_schema": {"type": "object"}, + }, + ], + ) chunks = [ _tool_delta_chunk(0, tool_id="call_1", name="calc", arguments='{"a":1}'), _tool_delta_chunk(1, tool_id="call_2", name="calc", arguments='{"b":2}'), _finish_chunk("tool_calls"), ] events = await _run_stream(chunks, req) - tool_starts = [ - e for e in events - if e["type"] == "content_block_start" - and (e.get("content_block") or {}).get("type") == "tool_use" - ] + tool_starts = [e for e in events if e["type"] == "content_block_start" and (e.get("content_block") or {}).get("type") == "tool_use"] assert len(tool_starts) == 2 indices = {t["index"] for t in tool_starts} assert len(indices) == 2, f"tool blocks must have distinct indices, got {indices}" @@ -1738,23 +3361,82 @@ async def test_streaming_multiple_tool_calls_use_distinct_indices() -> None: assert len(stops) >= len(starts), "each tool_use must be closed before the next opens" # In event order, the first tool stop precedes the second tool start. second_tool_start_idx = next( - i for i, e in enumerate(events) - if e["type"] == "content_block_start" - and (e.get("content_block") or {}).get("type") == "tool_use" - and e["index"] == 1 - ) - first_tool_stop_idx = next( - i for i, e in enumerate(events) - if e["type"] == "content_block_stop" and e["index"] == 0 + i + for i, e in enumerate(events) + if e["type"] == "content_block_start" and (e.get("content_block") or {}).get("type") == "tool_use" and e["index"] == 1 ) + first_tool_stop_idx = next(i for i, e in enumerate(events) if e["type"] == "content_block_stop" and e["index"] == 0) assert first_tool_stop_idx < second_tool_start_idx +async def test_streaming_preserves_partial_tool_arguments_when_index_changes() -> None: + """Repro for the malformed-Bash-before-Read pattern we saw against MiniMax. + + When two parallel tool calls arrive and the first one's argument stream + ends mid-JSON (literally ``"{"``), the proxy must forward it faithfully — + no synthetic completion, no drop. Claude Code surfaces the malformed + input via ``__unparsedToolInput`` and the model retries with a clean + call. + """ + req = _base_request( + stream=True, + tools=[ + { + "name": "Bash", + "description": "x", + "input_schema": {"type": "object", "properties": {"command": {"type": "string"}}}, + }, + { + "name": "Read", + "description": "x", + "input_schema": {"type": "object", "properties": {"file_path": {"type": "string"}}}, + }, + ], + ) + chunks = [ + _tool_delta_chunk(0, tool_id="call_bash", name="Bash", arguments="{"), + _tool_delta_chunk(1, tool_id="call_read", name="Read", arguments='{"file_path":"/tmp/x.png"}'), + _finish_chunk("tool_calls"), + ] + events = await _run_stream(chunks, req) + + # Find each tool_use block by name, then assert what deltas fed it. + blocks: dict[str, dict[str, object]] = {} + current_name: str | None = None + for e in events: + if e["type"] == "content_block_start": + cb = e.get("content_block") or {} + if cb.get("type") == "tool_use": + current_name = cb.get("name") + blocks[current_name] = {"index": e["index"], "deltas": []} # ty: ignore[invalid-assignment] — current_name is str|None, narrowed by tool_use branch above + elif e["type"] == "content_block_stop": + current_name = None + elif e["type"] == "content_block_delta" and current_name: + d = e.get("delta") or {} + if d.get("type") == "input_json_delta": + blocks[current_name]["deltas"].append(d.get("partial_json", "")) # ty: ignore[unresolved-attribute] — narrowed by `and current_name` guard above + + assert set(blocks) == {"Bash", "Read"}, f"expected Bash and Read blocks; got {set(blocks)}" + assert blocks["Bash"]["deltas"] == ["{"], ( + f"Bash must carry the literal partial_json from upstream — no synthetic closing; got {blocks['Bash']['deltas']!r}" + ) + assert blocks["Read"]["deltas"] == ['{"file_path":"/tmp/x.png"}'], ( + f"Read must carry its full arguments; got {blocks['Read']['deltas']!r}" + ) + + async def test_streaming_tool_arguments_streamed_as_partial_json() -> None: """Tool argument fragments must be wrapped in input_json_delta deltas.""" - req = _base_request(stream=True, tools=[{ - "name": "calc", "description": "calc", "input_schema": {"type": "object"}, - }]) + req = _base_request( + stream=True, + tools=[ + { + "name": "calc", + "description": "calc", + "input_schema": {"type": "object"}, + }, + ], + ) chunks = [ _tool_delta_chunk(0, tool_id="call_1", name="calc", arguments='{"x":'), _tool_delta_chunk(0, arguments="1}"), @@ -1764,8 +3446,7 @@ async def test_streaming_tool_arguments_streamed_as_partial_json() -> None: arg_deltas = [ e["delta"]["partial_json"] for e in events - if e["type"] == "content_block_delta" - and e.get("delta", {}).get("type") == "input_json_delta" + if e["type"] == "content_block_delta" and e.get("delta", {}).get("type") == "input_json_delta" ] assert "".join(arg_deltas) == '{"x":1}' @@ -1788,7 +3469,7 @@ async def test_streaming_message_id_format() -> None: msg_id = start["message"]["id"] assert msg_id.startswith("msg_") assert len(msg_id) == len("msg_") + 24 - assert all(c in "0123456789abcdef" for c in msg_id[len("msg_"):]) + assert all(c in "0123456789abcdef" for c in msg_id[len("msg_") :]) async def test_streaming_emits_thinking_block_for_think_tags() -> None: @@ -1802,24 +3483,14 @@ async def test_streaming_emits_thinking_block_for_think_tags() -> None: _finish_chunk("stop"), ] events = await _run_stream(chunks, req) - thinking_starts = [ - e for e in events - if e["type"] == "content_block_start" - and (e.get("content_block") or {}).get("type") == "thinking" - ] + thinking_starts = [e for e in events if e["type"] == "content_block_start" and (e.get("content_block") or {}).get("type") == "thinking"] assert len(thinking_starts) == 1, "exactly one thinking block expected" thinking_deltas = [ - e["delta"]["thinking"] - for e in events - if e["type"] == "content_block_delta" - and e.get("delta", {}).get("type") == "thinking_delta" + e["delta"]["thinking"] for e in events if e["type"] == "content_block_delta" and e.get("delta", {}).get("type") == "thinking_delta" ] assert "".join(thinking_deltas) == "step 1; step 2;" text_deltas = [ - e["delta"]["text"] - for e in events - if e["type"] == "content_block_delta" - and e.get("delta", {}).get("type") == "text_delta" + e["delta"]["text"] for e in events if e["type"] == "content_block_delta" and e.get("delta", {}).get("type") == "text_delta" ] assert "".join(text_deltas) == "final answer" @@ -1830,17 +3501,15 @@ async def test_streaming_emits_thinking_for_native_reasoning_content() -> None: """ req = _base_request(stream=True) chunk = { - "choices": [{ - "delta": {"content": "answer", "reasoning_content": "thinking"}, - "finish_reason": "stop", - }], + "choices": [ + { + "delta": {"content": "answer", "reasoning_content": "thinking"}, + "finish_reason": "stop", + }, + ], } events = await _run_stream([chunk], req) - thinking_starts = [ - e for e in events - if e["type"] == "content_block_start" - and (e.get("content_block") or {}).get("type") == "thinking" - ] + thinking_starts = [e for e in events if e["type"] == "content_block_start" and (e.get("content_block") or {}).get("type") == "thinking"] assert len(thinking_starts) == 1 @@ -1894,6 +3563,7 @@ def _patched_empty_config() -> Iterator[None]: # --- Loader --- + def test_load_config_happy_path() -> None: with _patched_config(""" [proxy] @@ -2041,15 +3711,16 @@ def test_convert_request_explicit_none_top_p_is_dropped() -> None: to upstream — OpenAI-compatible backends reject null with 400. """ with _patched_empty_config(): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - "top_p": None, - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "top_p": None, + }, + ) assert "top_p" in req.model_fields_set, ( - "test setup: top_p=None must land in model_fields_set so the " - "request-time drop branch actually executes" + "test setup: top_p=None must land in model_fields_set so the request-time drop branch actually executes" ) out = srv.convert_anthropic_to_litellm(req) assert "top_p" not in out @@ -2064,11 +3735,13 @@ def test_convert_config_empty_stop_in_extra_body_passes_through() -> None: [sonnet] extra_body = { stop = [] } """): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["stop"] == [] @@ -2079,19 +3752,18 @@ def test_resolve_tier_config_does_not_share_nested_dicts() -> None: [global] extra_body = { chat_template_kwargs = { enable_thinking = false } } """): - req = _make_request({ - "model": "claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-haiku-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) resolved = srv._resolve_tier_config(req) resolved["extra_body"]["chat_template_kwargs"]["enable_thinking"] = True resolved["extra_body"]["new_key"] = 1 # CONFIG is unchanged - assert ( - srv.CONFIG["global"]["extra_body"]["chat_template_kwargs"]["enable_thinking"] - is False - ) + assert srv.CONFIG["global"]["extra_body"]["chat_template_kwargs"]["enable_thinking"] is False assert "new_key" not in srv.CONFIG["global"]["extra_body"] @@ -2101,11 +3773,13 @@ def test_convert_config_zero_sampling_in_extra_body_is_preserved() -> None: [sonnet] extra_body = { temperature = 0, top_p = 0, top_k = 0 } """): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["temperature"] == 0 assert out["top_p"] == 0 @@ -2125,6 +3799,7 @@ def test_proxy_value_empty_string_in_config_falls_through_to_env() -> None: # --- Deep merge --- + def test_deep_merge_config_wins_per_leaf() -> None: merged = srv._deep_merge({"a": 1, "b": 2}, {"b": 99, "c": 3}) assert merged == {"a": 1, "b": 99, "c": 3} @@ -2176,6 +3851,7 @@ def test_proxy_bool_garbage_string_falls_back_to_caller_default() -> None: # --- Tier capture --- + def test_derive_tier_sets_tier_for_each_known_substring() -> None: expected = { "claude-3-5-haiku-20241022": "haiku", @@ -2185,33 +3861,30 @@ def test_derive_tier_sets_tier_for_each_known_substring() -> None: "claude-mythos-5": "mythos", } for model, tier in expected.items(): - req = _make_request({"model": model, "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": model, "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) assert req.tier == tier, f"model={model!r} expected tier={tier!r}, got {req.tier!r}" def test_derive_tier_is_none_for_unknown_model() -> None: - req = _make_request({"model": "my-local-llama", "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "my-local-llama", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) assert req.tier is None def test_derive_tier_strips_anthropic_prefix() -> None: - req = _make_request({"model": "anthropic/claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request( + {"model": "anthropic/claude-3-5-haiku-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}, + ) assert req.tier == "haiku" def test_derive_tier_strips_gemini_prefix() -> None: - req = _make_request({"model": "gemini/claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "gemini/claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) assert req.tier == "sonnet" # --- Per-tier lookup --- + def test_resolve_tier_config_prefers_tier_over_global() -> None: with _patched_config(""" [global] @@ -2220,9 +3893,7 @@ def test_resolve_tier_config_prefers_tier_over_global() -> None: [sonnet] extra_body = { temperature = 0.9 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg["extra_body"]["temperature"] == 0.9 @@ -2232,9 +3903,7 @@ def test_resolve_tier_config_falls_back_to_global_when_tier_missing() -> None: [global] extra_body = { temperature = 0.5 } """): - req = _make_request({"model": "claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-haiku-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg["extra_body"]["temperature"] == 0.5 @@ -2244,9 +3913,7 @@ def test_resolve_tier_config_falls_back_to_global_when_tier_none() -> None: [global] extra_body = { temperature = 0.4 } """): - req = _make_request({"model": "my-local-llama", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "my-local-llama", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg["extra_body"]["temperature"] == 0.4 @@ -2259,9 +3926,7 @@ def test_resolve_tier_config_deep_merges_extra_body_over_global() -> None: [sonnet] extra_body = { chat_template_kwargs = { enable_thinking = false }, foo = 2 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg["extra_body"]["cache_prompt"] is True # from global assert cfg["extra_body"]["foo"] == 2 # tier overrides global @@ -2270,9 +3935,7 @@ def test_resolve_tier_config_deep_merges_extra_body_over_global() -> None: def test_resolve_tier_config_empty_when_nothing_loaded() -> None: with _patched_empty_config(): - req = _make_request({"model": "claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-haiku-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg == {} @@ -2282,10 +3945,8 @@ def test_resolve_tier_config_handles_none_global() -> None: downstream .get access (regression — None guard added at the resolver). """ with _patched_empty_config(): - srv.CONFIG["global"] = None # ty: ignore[invalid-assignment] — None-handling regression test - req = _make_request({"model": "claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + srv.CONFIG["global"] = None # None-handling regression test + req = _make_request({"model": "claude-3-5-haiku-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg == {} @@ -2296,9 +3957,7 @@ def test_resolve_tier_config_handles_none_tier_value() -> None: """ with _patched_empty_config(): srv.CONFIG["tiers"] = {"haiku": None} - req = _make_request({"model": "claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-haiku-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg == {} @@ -2309,9 +3968,7 @@ def test_convert_extra_body_non_dict_is_skipped() -> None: """ with _patched_empty_config(): srv.CONFIG["tiers"] = {"haiku": {"extra_body": "not-a-dict"}} - req = _make_request({"model": "claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-haiku-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) assert "extra_body" not in out @@ -2320,21 +3977,25 @@ def test_validate_model_field_preserves_bare_name_case() -> None: """Custom (non-OpenAI) model names must keep their original case in the rewritten upstream model. """ - req = _make_request({ - "model": "MyModel-V1", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "MyModel-V1", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == "openai/MyModel-V1" def test_validate_model_field_openai_prefix_is_case_insensitive() -> None: """``OpenAI/MyModel-V1`` should pass through unchanged (any-case prefix).""" - req = _make_request({ - "model": "OpenAI/MyModel-V1", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "OpenAI/MyModel-V1", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.model == "OpenAI/MyModel-V1" @@ -2343,11 +4004,13 @@ def test_validate_model_field_anthropic_prefix_preserves_case() -> None: rewritten upstream model must use the resolved sonnet default (lowercased because it's a known OpenAI model). """ - req = _make_request({ - "model": "anthropic/Claude-3-5-Sonnet", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - }) + req = _make_request( + { + "model": "anthropic/Claude-3-5-Sonnet", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + }, + ) assert req.tier == "sonnet" assert req.model == f"openai/{srv._default_model_for_tier('sonnet')}" @@ -2383,16 +4046,16 @@ def test_extra_body_is_deep_copied_from_raw_toml() -> None: # --- Injection: sampling --- + def test_convert_extra_body_overrides_request_sampling_via_config() -> None: """[tier].extra_body {temperature=0.2} wins over client temperature=0.9.""" with _patched_config(""" [sonnet] extra_body = { temperature = 0.2 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - "temperature": 0.9}) + req = _make_request( + {"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}], "temperature": 0.9}, + ) out = srv.convert_anthropic_to_litellm(req) assert out["temperature"] == 0.2 @@ -2400,10 +4063,14 @@ def test_convert_extra_body_overrides_request_sampling_via_config() -> None: def test_convert_request_sampling_preserved_when_config_omits_key() -> None: """Request value flows through unchanged when config doesn't touch the key.""" with _patched_config(""): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - "temperature": 0.42}) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "temperature": 0.42, + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["temperature"] == 0.42 @@ -2413,9 +4080,7 @@ def test_convert_sampling_field_omitted_when_neither_set() -> None: the upstream call doesn't include it (was previously always set to 1.0). """ with _patched_empty_config(): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) assert "temperature" not in out assert "top_p" not in out @@ -2428,9 +4093,7 @@ def test_convert_max_completion_tokens_via_extra_body_overrides_request() -> Non [sonnet] extra_body = { max_completion_tokens = 500 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 1000, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 1000, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) assert out["max_completion_tokens"] == 500 @@ -2441,9 +4104,7 @@ def test_convert_config_only_seed_field() -> None: [sonnet] extra_body = { seed = 42 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) assert out["seed"] == 42 @@ -2454,23 +4115,20 @@ def test_convert_global_extra_body_applies_to_unmapped_tier() -> None: [global] extra_body = { cache_prompt = true } """): - req = _make_request({"model": "my-local-llama", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "my-local-llama", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) assert out["cache_prompt"] is True # --- Injection: extra_body --- + def test_convert_merges_extra_body_from_config() -> None: with _patched_config(""" [sonnet] extra_body = { cache_prompt = true, n_predict = 1024 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) # extra_body keys are lifted to top-level kwargs in the new pipeline. assert out["cache_prompt"] is True @@ -2482,9 +4140,7 @@ def test_convert_extra_body_deep_merges_nested_dicts() -> None: [sonnet] extra_body = { chat_template_kwargs = { enable_thinking = false }, n_predict = 256 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) assert out["chat_template_kwargs"] == {"enable_thinking": False} assert out["n_predict"] == 256 @@ -2498,9 +4154,7 @@ def test_convert_global_extra_body_preserved_when_tier_section_has_no_extra_body [sonnet] extra_body = { temperature = 0.7 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) # Global extra_body survives; tier extra_body sits alongside. assert out["cache_prompt"] is True @@ -2516,15 +4170,14 @@ def test_convert_tier_extra_body_overrides_global() -> None: [sonnet] extra_body = { cache_prompt = false } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) assert out["cache_prompt"] is False # --- End-to-end: realistic llama-server config --- + def test_convert_with_full_llama_server_config() -> None: with _patched_config(""" [global] @@ -2537,9 +4190,7 @@ def test_convert_with_full_llama_server_config() -> None: [sonnet] extra_body = { chat_template_kwargs = { enable_thinking = false } } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 256, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 256, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) # No temperature in request → not set (no defaults applied). assert "temperature" not in out @@ -2553,15 +4204,14 @@ def test_convert_with_full_llama_server_config() -> None: # --- extra_body pipeline (post-simplification) --- + def test_extra_body_simple() -> None: """Vendor key from config extra_body reaches the upstream call as a top-level kwarg.""" with _patched_config(""" [sonnet] extra_body = { reasoning_effort = "low" } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) assert out["reasoning_effort"] == "low" assert out["allowed_openai_params"] == ["reasoning_effort"] @@ -2574,10 +4224,9 @@ def test_extra_body_overrides_pydantic_sampling() -> None: [sonnet] extra_body = { temperature = 0.3 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - "temperature": 0.5}) + req = _make_request( + {"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}], "temperature": 0.5}, + ) out = srv.convert_anthropic_to_litellm(req) assert out["temperature"] == 0.3 @@ -2585,11 +4234,15 @@ def test_extra_body_overrides_pydantic_sampling() -> None: def test_extra_body_pydantic_sampling_passes_through() -> None: """When config doesn't touch a sampling key, the client's Pydantic value reaches upstream.""" with _patched_empty_config(): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - "temperature": 0.5, - "top_p": 0.95}) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "temperature": 0.5, + "top_p": 0.95, + }, + ) out = srv.convert_anthropic_to_litellm(req) assert out["temperature"] == 0.5 assert out["top_p"] == 0.95 @@ -2601,16 +4254,20 @@ def test_extra_body_deep_merge_with_client() -> None: [sonnet] extra_body = { thinking = { type = "disabled" }, temperature = 0.3 } """): - req = _make_request({ - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}], - "extra_body": {"thinking": {"effort": "high"}, "top_p": 0.9}, - }) + req = _make_request( + { + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "hi"}], + "extra_body": {"thinking": {"effort": "high"}, "top_p": 0.9}, + }, + ) out = srv.convert_anthropic_to_litellm(req) + # thinking is body-only (openai SDK rejects unknown kwargs); lives in extra_body. + assert "thinking" not in out # thinking: both leaves present; client effort + config type (config wins per leaf) - assert out["thinking"]["type"] == "disabled" # from config - assert out["thinking"]["effort"] == "high" # from client (config didn't set) + assert out["extra_body"]["thinking"]["type"] == "disabled" # from config + assert out["extra_body"]["thinking"]["effort"] == "high" # from client (config didn't set) # temperature: config wins outright assert out["temperature"] == 0.3 # top_p: client-only → reaches upstream @@ -2623,9 +4280,7 @@ def test_extra_body_protected_keys_blocked() -> None: [sonnet] extra_body = { model = "evil-model", messages = "stolen", stream = true } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) # The proxy's own values are untouched. assert out["model"] != "evil-model" @@ -2639,9 +4294,7 @@ def test_extra_body_allowed_openai_params_set() -> None: [sonnet] extra_body = { reasoning_effort = "low", top_k = 5 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) assert set(out["allowed_openai_params"]) == {"reasoning_effort", "top_k"} assert set(out["extra_body"]["allowed_openai_params"]) == {"reasoning_effort", "top_k"} @@ -2650,9 +4303,7 @@ def test_extra_body_allowed_openai_params_set() -> None: def test_no_max_tokens_clamp_arbitrary_large_value() -> None: """Any large max_tokens value (including well above MAX_OUTPUT_TOKENS) flows through unchanged.""" with _patched_empty_config(): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 99999, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 99999, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) assert out["max_completion_tokens"] == 99999 @@ -2666,9 +4317,7 @@ def test_global_extra_body_precedes_tier() -> None: [sonnet] extra_body = { x = 2, y = 3 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) out = srv.convert_anthropic_to_litellm(req) # 'x' is overridden by the tier (wins over global) assert out["x"] == 2 @@ -2760,10 +4409,18 @@ def _scrub_model_envs() -> dict[str, str | None]: """Pop all model-related env vars so a test starts from a clean slate. Returns a dict suitable for restoring in `finally`. """ - return {k: os.environ.pop(k, None) for k in ( - "BIG_MODEL", "SMALL_MODEL", - "HAIKU_MODEL", "SONNET_MODEL", "OPUS_MODEL", "FABLE_MODEL", "MYTHOS_MODEL", - )} + return { + k: os.environ.pop(k, None) + for k in ( + "BIG_MODEL", + "SMALL_MODEL", + "HAIKU_MODEL", + "SONNET_MODEL", + "OPUS_MODEL", + "FABLE_MODEL", + "MYTHOS_MODEL", + ) + } def test_default_model_for_tier_haiku_uses_small_default() -> None: @@ -2912,7 +4569,7 @@ def test_default_model_for_tier_handles_none_bucket() -> None: try: with _patched_empty_config(): srv._default_model_for_tier.cache_clear() - srv.CONFIG["big"] = None # ty: ignore[invalid-assignment] — None-handling regression test + srv.CONFIG["big"] = None # None-handling regression test assert srv._default_model_for_tier("sonnet") == "gpt-4.1" finally: for k, v in saved.items(): @@ -3038,9 +4695,7 @@ def test_resolve_tier_config_merges_global_bucket_tier_for_sonnet() -> None: [sonnet] extra_body = { from_tier = 3, shared = "t" } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) eb = cfg["extra_body"] assert eb["from_global"] == 1 @@ -3061,9 +4716,7 @@ def test_resolve_tier_config_merges_global_bucket_tier_for_haiku() -> None: [haiku] extra_body = { from_haiku = 3 } """): - req = _make_request({"model": "claude-3-5-haiku-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-haiku-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg["extra_body"]["from_global"] == 1 assert cfg["extra_body"]["from_small"] == 2 @@ -3077,9 +4730,7 @@ def test_resolve_tier_config_strips_model_from_result() -> None: model = "bucket-big" extra_body = { x = 1 } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert "model" not in cfg assert cfg["extra_body"] == {"x": 1} @@ -3094,9 +4745,7 @@ def test_resolve_tier_config_unknown_tier_only_global() -> None: [big] extra_body = { y = 2 } """): - req = _make_request({"model": "my-local-llama", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "my-local-llama", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg["extra_body"] == {"x": 1} @@ -3107,9 +4756,7 @@ def test_resolve_tier_config_omitted_tier_section_still_inherits_bucket() -> Non [big] extra_body = { cache_prompt = true } """): - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg["extra_body"]["cache_prompt"] is True @@ -3117,9 +4764,7 @@ def test_resolve_tier_config_omitted_tier_section_still_inherits_bucket() -> Non def test_resolve_tier_config_empty_and_unknown_tier() -> None: """No CONFIG, no tier — empty dict.""" with _patched_empty_config(): - req = _make_request({"model": "my-local-llama", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "my-local-llama", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg == {} @@ -3127,11 +4772,9 @@ def test_resolve_tier_config_empty_and_unknown_tier() -> None: def test_resolve_tier_config_handles_none_global_cfg() -> None: """CONFIG['global'] patched to None — must not crash on `.is not None` guard.""" with _patched_empty_config(): - srv.CONFIG["global"] = None # ty: ignore[invalid-assignment] — None-handling regression test + srv.CONFIG["global"] = None # None-handling regression test srv.CONFIG["big"] = {"extra_body": {"from_big": 1}} - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) assert cfg == {"extra_body": {"from_big": 1}} @@ -3142,14 +4785,103 @@ def test_resolve_tier_config_handles_none_tier_cfg() -> None: srv.CONFIG["global"] = {"extra_body": {"from_global": 1}} srv.CONFIG["big"] = {"extra_body": {"from_big": 2}} srv.CONFIG["tiers"] = {"sonnet": None} - req = _make_request({"model": "claude-3-5-sonnet-20241022", - "max_tokens": 100, - "messages": [{"role": "user", "content": "hi"}]}) + req = _make_request({"model": "claude-3-5-sonnet-20241022", "max_tokens": 100, "messages": [{"role": "user", "content": "hi"}]}) cfg = srv._resolve_tier_config(req) # global + big layers merge; the None [sonnet] layer is skipped. assert cfg["extra_body"] == {"from_global": 1, "from_big": 2} +def test_dump_inbound_anthropic_body_disabled_by_default() -> None: + dump_dir = pathlib.Path(tempfile.mkdtemp()) / "anthropic-prompts" + original_dir = srv._INBOUND_DUMP_DIR + srv._INBOUND_DUMP_DIR = dump_dir + try: + with _patched_env("PROXY_DEBUG_INBOUND_DUMP", "false"): + srv._dump_inbound_anthropic_body(b'{"model":"x","max_tokens":1,"messages":[{"role":"user","content":"hi"}]}') + assert list(dump_dir.glob("*")) == [] + finally: + srv._INBOUND_DUMP_DIR = original_dir + + +def test_dump_inbound_anthropic_body_writes_raw_bytes() -> None: + dump_dir = pathlib.Path(tempfile.mkdtemp()) / "anthropic-prompts" + original_dir = srv._INBOUND_DUMP_DIR + srv._INBOUND_DUMP_DIR = dump_dir + try: + raw = ( + b'{"model":"claude-sonnet","max_tokens":100,' + b'"system":"top",' + b'"messages":[{"role":"system","content":"inband"},{"role":"user","content":"hello"}]}' + ) + with _patched_env("PROXY_DEBUG_INBOUND_DUMP", "true"): + srv._dump_inbound_anthropic_body(raw) + files = list(dump_dir.glob("*-inbound-*.json")) + assert len(files) == 1 + # Pretty-printed JSON whose parsed payload matches the wire bytes + # verbatim — clients add non-Pydantic fields (thinking, metadata, + # output_config, context_management, …) that must survive. + payload = json.loads(files[0].read_text()) + assert payload == json.loads(raw) + assert payload["system"] == "top" + assert payload["messages"][0]["role"] == "system" + # Whitespace was rewritten, not copied verbatim. + assert "\n " in files[0].read_text() + finally: + srv._INBOUND_DUMP_DIR = original_dir + + +def test_inbound_dump_path_disambiguates_same_second_writes() -> None: + """Two dumps within the same second must produce distinct filenames — a + seq counter stops the second from clobbering the first. + """ + dump_dir = pathlib.Path(tempfile.mkdtemp()) + original_dir = srv._INBOUND_DUMP_DIR + srv._INBOUND_DUMP_DIR = dump_dir + try: + srv._INBOUND_DUMP_SEQ = itertools.count(1) + path_a = srv._inbound_dump_path() + path_b = srv._inbound_dump_path() + assert path_a != path_b, f"Same-second dumps collided: {path_a.name} == {path_b.name}" + assert path_a.name.endswith("-inbound-0001.json") + assert path_b.name.endswith("-inbound-0002.json") + finally: + srv._INBOUND_DUMP_DIR = original_dir + + +def test_dump_inbound_anthropic_body_handles_write_errors() -> None: + dump_dir = pathlib.Path(tempfile.mkdtemp()) / "anthropic-prompts" + original_dir = srv._INBOUND_DUMP_DIR + srv._INBOUND_DUMP_DIR = dump_dir + + def boom(_self: pathlib.Path, *_args: Any, **_kwargs: Any) -> NoReturn: + raise OSError("disk full") + + original_write_bytes = pathlib.Path.write_bytes + try: + with _patched_env("PROXY_DEBUG_INBOUND_DUMP", "true"): + pathlib.Path.write_bytes = boom # type: ignore[method-assign] + # Must not raise + srv._dump_inbound_anthropic_body(b"{}") + finally: + pathlib.Path.write_bytes = original_write_bytes # type: ignore[method-assign] + srv._INBOUND_DUMP_DIR = original_dir + + +def test_dump_inbound_anthropic_body_falls_back_to_raw_on_invalid_json() -> None: + """Non-JSON body (e.g. truncated upload) still lands on disk verbatim — never crash.""" + dump_dir = pathlib.Path(tempfile.mkdtemp()) / "anthropic-prompts" + original_dir = srv._INBOUND_DUMP_DIR + srv._INBOUND_DUMP_DIR = dump_dir + try: + with _patched_env("PROXY_DEBUG_INBOUND_DUMP", "true"): + srv._dump_inbound_anthropic_body(b'{"truncated"') + files = list(dump_dir.glob("*-inbound-*.json")) + assert len(files) == 1 + assert files[0].read_text(encoding="utf-8") == '{"truncated"' + finally: + srv._INBOUND_DUMP_DIR = original_dir + + # --------------------------------------------------------------------------- # Integration smoke tests # --------------------------------------------------------------------------- @@ -3195,6 +4927,7 @@ class _StreamAgg: """Aggregated state from consuming an SSE stream: set of seen event types, accumulated text, whether a tool_use block opened, whether [DONE] arrived. Lives only as long as one integration scenario.""" + event_types: set[str] = field(default_factory=set) text_content: str = "" saw_tool_use: bool = False @@ -3260,7 +4993,7 @@ def _record_stream_event(data: dict[str, Any], agg: _StreamAgg) -> None: def _extract_sse_data(event_block: str) -> str | None: """Concatenate ``data: `` lines from a single SSE event block; return None for blocks with no data line.""" - data_lines = [line[len("data: "):] for line in event_block.splitlines() if line.startswith("data: ")] + data_lines = [line[len("data: ") :] for line in event_block.splitlines() if line.startswith("data: ")] if not data_lines: return None return "".join(data_lines) @@ -3309,8 +5042,7 @@ def filter_scenarios(scenarios: dict[str, dict[str, Any]], args: argparse.Namesp def discover_unit_tests() -> list[str]: """Collect every top-level test_* function defined in this module.""" - return [name for name, _ in inspect.getmembers(sys.modules[__name__], inspect.isfunction) - if name.startswith("test_")] + return [name for name, _ in inspect.getmembers(sys.modules[__name__], inspect.isfunction) if name.startswith("test_")] def run_unit_tests(names: list[str]) -> list[bool]: