diff --git a/changelog/unreleased/responses-cache-usage.md b/changelog/unreleased/responses-cache-usage.md new file mode 100644 index 0000000..623c041 --- /dev/null +++ b/changelog/unreleased/responses-cache-usage.md @@ -0,0 +1,7 @@ +### English + +- Preserve cached input token usage in Responses API output for both streaming and non-streaming requests without double-counting total tokens. + +### 中文 + +- 在 Responses API 的流式与非流式输出中保留缓存输入 Token 用量，同时避免在总 Token 数中重复计数。 diff --git a/internal/gateway/compat_stream.go b/internal/gateway/compat_stream.go index b4703d5..0688050 100644 --- a/internal/gateway/compat_stream.go +++ b/internal/gateway/compat_stream.go @@ -610,7 +610,11 @@ func RelayResponsesStream(writer io.Writer, body io.Reader, requestID, model str } } terminal := responsesTerminalForFinishReason(output.finishReason) - response := responsesResponse(requestID, model, content, output.reasoning.String(), calls, derefInt(stats.PromptTokens), derefInt(stats.CompletionTokens), output.finishReason) + response := responsesResponse( + requestID, model, content, output.reasoning.String(), calls, + derefInt(stats.PromptTokens), derefInt(stats.CompletionTokens), output.finishReason, + stats.CacheReadTokens, stats.CachedTokens, + ) if err := eventWriter.write(terminal.event, map[string]any{"type": terminal.event, "response": response}); err != nil { return stats, err } diff --git a/internal/gateway/openai_stream.go b/internal/gateway/openai_stream.go index 26d27de..ed704c2 100644 --- a/internal/gateway/openai_stream.go +++ b/internal/gateway/openai_stream.go @@ -429,10 +429,14 @@ func ParseStreamUsageLine(line string) (StreamRelayStats, bool) { if credits == nil { credits = parsed.Usage.Credit } + cacheRead := parsed.Usage.CacheReadTokens + if cacheRead == nil { + cacheRead = parsed.Usage.PromptDetails.CachedTokens + } return StreamRelayStats{ PromptTokens: parsed.Usage.PromptTokens, CompletionTokens: parsed.Usage.CompletionTokens, - CacheReadTokens: parsed.Usage.CacheReadTokens, + CacheReadTokens: cacheRead, CacheWriteTokens: parsed.Usage.CacheWriteTokens, CachedTokens: parsed.Usage.PromptDetails.CachedTokens, UsageSource: firstNonEmpty(parsed.Usage.Source, "estimate"), diff --git a/internal/gateway/responses.go b/internal/gateway/responses.go index 601a88b..688862b 100644 --- a/internal/gateway/responses.go +++ b/internal/gateway/responses.go @@ -50,7 +50,11 @@ func (h *Handler) HandleResponses(w http.ResponseWriter, r *http.Request) { CachedTokens: result.CachedTokens, UsageSource: result.UsageSource, Credits: result.Credits, ConsumedCredits: result.ConsumedCredits, Model: result.Model, FinishReason: result.FinishReason, }, nil, result.AttemptCount, result.ReasoningLevel) - response := responsesResponse(execution.RequestID, firstNonEmpty(result.Model, execution.PublicModel), result.Content, result.Reasoning, decodeOpenAIToolCalls(result.ToolCalls), result.PromptTokens, result.CompletionTokens, result.FinishReason) + response := responsesResponse( + execution.RequestID, firstNonEmpty(result.Model, execution.PublicModel), result.Content, result.Reasoning, + decodeOpenAIToolCalls(result.ToolCalls), result.PromptTokens, result.CompletionTokens, result.FinishReason, + result.CacheReadTokens, result.CachedTokens, + ) translate.RestoreResponseToolNames(response, execution.Request.ResponseToolNames) writeJSON(w, http.StatusOK, response) } @@ -130,12 +134,18 @@ func responsesRequestStatus(finishReason string) string { return accounts.RequestStatusOK } -func responsesResponse(requestID, model, content, reasoning string, toolCalls []proxyToolCall, promptTokens, completionTokens int, finishReason string) map[string]any { +func responsesResponse( + requestID, model, content, reasoning string, + toolCalls []proxyToolCall, + promptTokens, completionTokens int, + finishReason string, + cacheReadTokens, cachedTokens *int, +) map[string]any { terminal := responsesTerminalForFinishReason(finishReason) response := map[string]any{ "id": "resp_" + requestID, "object": "response", "created_at": time.Now().Unix(), "status": terminal.status, "model": model, "output": responsesOutputItems(requestID, content, reasoning, toolCalls), - "usage": responsesUsage(promptTokens, completionTokens), + "usage": responsesUsage(promptTokens, completionTokens, cacheReadTokens, cachedTokens), } if terminal.incompleteDetails != nil { response["incomplete_details"] = terminal.incompleteDetails @@ -167,6 +177,13 @@ func responseFunctionCallItem(requestID string, callIndex int, call proxyToolCal } } -func responsesUsage(promptTokens, completionTokens int) map[string]any { - return map[string]any{"input_tokens": promptTokens, "output_tokens": completionTokens, "total_tokens": promptTokens + completionTokens} +func responsesUsage(promptTokens, completionTokens int, cacheReadTokens, cachedTokens *int) map[string]any { + usage := map[string]any{"input_tokens": promptTokens, "output_tokens": completionTokens, "total_tokens": promptTokens + completionTokens} + if cacheReadTokens == nil { + cacheReadTokens = cachedTokens + } + if cacheReadTokens != nil { + usage["input_tokens_details"] = map[string]any{"cached_tokens": *cacheReadTokens} + } + return usage } diff --git a/internal/gateway/responses_test.go b/internal/gateway/responses_test.go index 414b2c7..f3c60dc 100644 --- a/internal/gateway/responses_test.go +++ b/internal/gateway/responses_test.go @@ -7,7 +7,7 @@ import ( ) func TestResponsesResponseMapsLengthToIncomplete(t *testing.T) { - response := responsesResponse("req", "model", "", "reasoning", nil, 10, 32, "length") + response := responsesResponse("req", "model", "", "reasoning", nil, 10, 32, "length", nil, nil) if response["status"] != "incomplete" { t.Fatalf("status=%v", response["status"]) } @@ -54,3 +54,63 @@ func TestRelayResponsesStreamEmitsIncomplete(t *testing.T) { t.Fatalf("response=%#v", response) } } +func TestResponsesUsagePreservesCachedInputTokens(t *testing.T) { + zero := 0 + read := 64 + cached := 48 + + withoutCache := responsesUsage(100, 20, nil, nil) + if _, ok := withoutCache["input_tokens_details"]; ok { + t.Fatalf("missing cache usage must not be fabricated: %#v", withoutCache) + } + + withZero := responsesUsage(100, 20, &zero, nil) + zeroDetails := withZero["input_tokens_details"].(map[string]any) + if zeroDetails["cached_tokens"] != 0 { + t.Fatalf("explicit zero cache usage was lost: %#v", withZero) + } + + withFallback := responsesUsage(100, 20, nil, &cached) + fallbackDetails := withFallback["input_tokens_details"].(map[string]any) + if fallbackDetails["cached_tokens"] != 48 { + t.Fatalf("cached token fallback mismatch: %#v", withFallback) + } + + withTopLevel := responsesUsage(100, 20, &read, &cached) + topLevelDetails := withTopLevel["input_tokens_details"].(map[string]any) + if topLevelDetails["cached_tokens"] != 64 { + t.Fatalf("cache_read_tokens must win: %#v", withTopLevel) + } + if withTopLevel["total_tokens"] != 120 { + t.Fatalf("cached input must not be added twice: %#v", withTopLevel) + } +} + +func TestParseStreamUsageLineCacheReadFallback(t *testing.T) { + for _, test := range []struct { + name string + usage string + want *int + }{ + {name: "detail only", usage: `"prompt_tokens_details":{"cached_tokens":2176}`, want: ptrInt(2176)}, + {name: "explicit zero", usage: `"prompt_tokens_details":{"cached_tokens":0}`, want: ptrInt(0)}, + {name: "top-level wins", usage: `"cache_read_tokens":12,"prompt_tokens_details":{"cached_tokens":2176}`, want: ptrInt(12)}, + {name: "unknown stays absent", usage: `"prompt_tokens":16`, want: nil}, + } { + t.Run(test.name, func(t *testing.T) { + stats, ok := ParseStreamUsageLine(`data: {"usage":{` + test.usage + `}}`) + if !ok { + t.Fatal("usage not parsed") + } + if test.want == nil { + if stats.CacheReadTokens != nil { + t.Fatalf("fabricated cache read: %v", *stats.CacheReadTokens) + } + return + } + if stats.CacheReadTokens == nil || *stats.CacheReadTokens != *test.want { + t.Fatalf("cache read = %v, want %d", stats.CacheReadTokens, *test.want) + } + }) + } +}