Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,11 @@ call → response with metrics. See `claude_explain.md` for detailed architectur
`redacted_thinking` blocks (encrypted reasoning when safety filters trip); the same filter excludes them
correctly, but be aware "no text block" can mean either max_tokens starvation *or* a redacted-thinking-only
response — the error message is the same.
- **Safety refusals are handled before the empty-response path.** Claude 5-family classifiers can decline a
request (HTTP 200 with `stop_reason: "refusal"`, empty or partial content) — plausible here since CE users
compile arbitrary, sometimes exploit-adjacent code. `app/explain.py` returns a distinct user-facing message,
discards any partial output, and emits `ClaudeExplainRefusal` so it's separable from token starvation on
dashboards.
- **Empty responses are not 500s.** When the model returns no text block, `app/explain.py` returns
`ExplainResponse(status="error")` with `usage` populated and emits `ClaudeExplainEmptyResponse`. The cache
layer skips storing error responses so retries hit the API. Don't change this to raise — the structured error
Expand Down
32 changes: 32 additions & 0 deletions app/explain.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,38 @@ async def _call_anthropic_api(
output_tokens = message.usage.output_tokens
total_tokens = input_tokens + output_tokens

if message.stop_reason == "refusal":
# Claude 5-family safety classifiers can decline a request: a normal
# HTTP 200 whose stop_reason is "refusal", with empty (or discarded
# partial) content. CE users compile arbitrary code, so exploit-adjacent
# input can occasionally trip this. Handle it before the generic
# empty-response path so it gets a clear user-facing message and its
# own metric rather than looking like token starvation on dashboards.
message_text = (
"Claude declined to explain this code (safety filters). This can occasionally "
"trigger on benign security-related code; trimming the input to the relevant "
"part may help."
)
LOGGER.warning("Refusal from model (in=%d, out=%d)", input_tokens, output_tokens)
metrics_provider.set_property("language", body.language)
metrics_provider.set_property("compiler", body.compiler)
metrics_provider.set_property("instructionSet", body.instructionSet or "unknown")
metrics_provider.set_property("cached", "false")
metrics_provider.put_metric("ClaudeExplainRequest", 1)
metrics_provider.put_metric("ClaudeExplainRefusal", 1)
metrics_provider.put_metric("ClaudeExplainInputTokens", input_tokens)
metrics_provider.put_metric("ClaudeExplainOutputTokens", output_tokens)
return ExplainResponse(
status="error",
message=message_text,
model=prompt_data["model"],
usage=TokenUsage(
inputTokens=input_tokens,
outputTokens=output_tokens,
totalTokens=total_tokens,
),
)

# Pick the last text block — when thinking is enabled the response
# contains thinking blocks before the final text block.
text_blocks = [c for c in message.content if getattr(c, "type", None) == "text"]
Expand Down
53 changes: 53 additions & 0 deletions app/test_explain.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,59 @@ async def test_returns_error_when_no_text_block(self, sample_request, noop_metri
assert response.usage.inputTokens == 100
assert response.usage.outputTokens == 50

@pytest.mark.asyncio
async def test_refusal_returns_clear_error(self, sample_request, noop_metrics):
"""A safety-classifier refusal (HTTP 200, stop_reason='refusal', empty
content) must return a user-facing message distinct from the generic
empty-response error, with usage populated."""
mock_message = MagicMock()
mock_message.content = []
mock_message.usage = MagicMock(input_tokens=80, output_tokens=0)
mock_message.stop_reason = "refusal"

mock_client = MagicMock()
mock_client.messages.create = AsyncMock(return_value=mock_message)

test_prompt = Prompt(Path("app/prompt.yaml"))
response = await process_request(sample_request, mock_client, test_prompt, noop_metrics)

assert response.status == "error"
assert response.explanation is None
assert "declined" in response.message
assert "no text content" not in response.message
assert response.usage is not None
assert response.usage.inputTokens == 80

@pytest.mark.asyncio
async def test_refusal_discards_partial_output(self, sample_request, noop_metrics):
"""A mid-stream refusal can carry partial text; it must be discarded
rather than served as if it were a complete explanation."""
partial = MagicMock()
partial.type = "text"
partial.text = "This function starts by..."

mock_message = MagicMock()
mock_message.content = [partial]
mock_message.usage = MagicMock(input_tokens=80, output_tokens=40)
mock_message.stop_reason = "refusal"

mock_client = MagicMock()
mock_client.messages.create = AsyncMock(return_value=mock_message)

test_prompt = Prompt(Path("app/prompt.yaml"))
response = await process_request(sample_request, mock_client, test_prompt, noop_metrics)

assert response.status == "error"
assert response.explanation is None
# The refusal contract: the distinct declined message, not the generic
# empty-response error, and no trace of the partial output anywhere.
assert "declined" in response.message
assert "no text content" not in response.message
assert partial.text not in response.message
assert response.usage is not None
assert response.usage.inputTokens == 80
assert response.usage.outputTokens == 40

@pytest.mark.asyncio
async def test_returns_error_when_call_exceeds_deadline(self, sample_request, noop_metrics):
"""A Claude call that overruns the wall-clock budget must return a
Expand Down