From 582f7b30561964a6bab081be86e9636dcc368c1e Mon Sep 17 00:00:00 2001 From: young1lin <2550110827@qq.com> Date: Mon, 25 May 2026 11:37:30 +0800 Subject: [PATCH] feat(chat): surface cached prompt tokens from upstream usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI-compatible providers (Qwen, DeepSeek, OpenAI, Azure, etc.) report prompt-cache hits in `usage.prompt_tokens_details.cached_tokens`. This value was being read by go-openai but dropped at the WeKnora boundary, so there was no way to tell whether prompt caching was actually working. This change plumbs the field end-to-end: - `types.TokenUsage.CachedTokens` (json:"cached_tokens,omitempty") — zero-values are omitted so payloads stay quiet for providers that never report cache hits. - `cachedTokens` helper in remote_api.go guards against nil `PromptTokensDetails` (Ollama and older OpenAI-compat backends omit the details block entirely). - All three response-parsing paths populate it: * `parseCompletionResponse` (non-streaming) * `processStream` (SDK streaming) * `processRawHTTPStream` (raw-HTTP streaming, used when callers need to inject custom fields like `cache_control`) - The five `[LLM Usage]` log lines now print `cached_tokens=%d` so cache hit rate is visible in `journalctl` / log tail without going through metrics. Together with the deterministic tool ordering from the previous commit, this makes Qwen explicit caching observable: a warmed prefix should show `cached_tokens` ≈ system + tools token count (typically several thousand) on subsequent requests within the 5-minute TTL. Tests: - `TestCachedTokensHelper` — nil safety + round-trip - `TestParseCompletionResponse_CachedTokens` — populated + missing details paths through `parseCompletionResponse` - `TestTokenUsage_CachedTokensJSONOmitempty` — zero is omitted, non-zero is emitted --- internal/models/chat/remote_api.go | 33 ++++++--- internal/models/chat/remote_api_test.go | 91 +++++++++++++++++++++++++ internal/types/chat.go | 6 ++ 3 files changed, 120 insertions(+), 10 deletions(-) diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go index a72c16fe..10e0f319 100644 --- a/internal/models/chat/remote_api.go +++ b/internal/models/chat/remote_api.go @@ -424,8 +424,8 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat if err != nil { return nil, err } - logger.Infof(timeoutCtx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d", - c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens) + logger.Infof(timeoutCtx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d", + c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens, result.Usage.CachedTokens) return result, nil } @@ -488,8 +488,8 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu if err != nil { return nil, err } - logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d", - c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens) + logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d", + c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens, result.Usage.CachedTokens) return result, nil } @@ -512,6 +512,7 @@ func (c *RemoteAPIChat) parseCompletionResponse(resp *openai.ChatCompletionRespo PromptTokens: resp.Usage.PromptTokens, CompletionTokens: resp.Usage.CompletionTokens, TotalTokens: resp.Usage.TotalTokens, + CachedTokens: cachedTokens(resp.Usage.PromptTokensDetails), }, } @@ -696,8 +697,8 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo if err != nil { if err == io.EOF { if state.usage != nil { - logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d", - c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens) + logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d", + c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens) } toolCalls := state.buildOrderedToolCalls() streamChan <- types.StreamResponse{ @@ -723,6 +724,7 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo PromptTokens: response.Usage.PromptTokens, CompletionTokens: response.Usage.CompletionTokens, TotalTokens: response.Usage.TotalTokens, + CachedTokens: cachedTokens(response.Usage.PromptTokensDetails), } } @@ -745,8 +747,8 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res if err != nil { if err == io.EOF { if state.usage != nil { - logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d", - c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens) + logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d", + c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens) } toolCalls := state.buildOrderedToolCalls() streamChan <- types.StreamResponse{ @@ -773,8 +775,8 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res if event.Done { if state.usage != nil { - logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d", - c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens) + logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d", + c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens) } toolCalls := state.buildOrderedToolCalls() streamChan <- types.StreamResponse{ @@ -814,6 +816,7 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res PromptTokens: streamResp.Usage.PromptTokens, CompletionTokens: streamResp.Usage.CompletionTokens, TotalTokens: streamResp.Usage.TotalTokens, + CachedTokens: cachedTokens(streamResp.Usage.PromptTokensDetails), } } @@ -1169,3 +1172,13 @@ func (c *RemoteAPIChat) GetBaseURL() string { func (c *RemoteAPIChat) GetAPIKey() string { return c.apiKey } + +// cachedTokens returns the cached prompt-token count from an OpenAI-compatible +// usage detail block, or zero when the provider did not report one. Some +// providers omit PromptTokensDetails entirely, so the nil guard is required. +func cachedTokens(d *openai.PromptTokensDetails) int { + if d == nil { + return 0 + } + return d.CachedTokens +} diff --git a/internal/models/chat/remote_api_test.go b/internal/models/chat/remote_api_test.go index 1b1d605f..d0ce0048 100644 --- a/internal/models/chat/remote_api_test.go +++ b/internal/models/chat/remote_api_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/Tencent/WeKnora/internal/types" + "github.com/sashabaranov/go-openai" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -383,3 +384,93 @@ func TestRemoteAPIChat(t *testing.T) { }) } } + +// TestCachedTokensHelper covers the nil-safety contract of the cachedTokens +// helper. Some providers omit PromptTokensDetails entirely; the helper must +// return zero rather than panic. +func TestCachedTokensHelper(t *testing.T) { + assert.Equal(t, 0, cachedTokens(nil), "nil details must return zero") + assert.Equal(t, 0, cachedTokens(&openai.PromptTokensDetails{}), + "empty details must return zero") + assert.Equal(t, 1234, cachedTokens(&openai.PromptTokensDetails{CachedTokens: 1234}), + "populated cached_tokens must round-trip") +} + +// TestParseCompletionResponse_CachedTokens verifies that +// prompt_tokens_details.cached_tokens from an OpenAI-compatible response is +// propagated into TokenUsage.CachedTokens. This is the field Qwen explicit +// caching populates on a cache hit. +func TestParseCompletionResponse_CachedTokens(t *testing.T) { + c := newTestRemoteChat(t) + + t.Run("cached_tokens populated from prompt_tokens_details", func(t *testing.T) { + resp := &openai.ChatCompletionResponse{ + Choices: []openai.ChatCompletionChoice{ + { + Message: openai.ChatCompletionMessage{Role: "assistant", Content: "hi"}, + FinishReason: openai.FinishReasonStop, + }, + }, + Usage: openai.Usage{ + PromptTokens: 6929, + CompletionTokens: 42, + TotalTokens: 6971, + PromptTokensDetails: &openai.PromptTokensDetails{ + CachedTokens: 6900, + }, + }, + } + + got, err := c.parseCompletionResponse(resp) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, 6929, got.Usage.PromptTokens) + assert.Equal(t, 42, got.Usage.CompletionTokens) + assert.Equal(t, 6971, got.Usage.TotalTokens) + assert.Equal(t, 6900, got.Usage.CachedTokens, + "cached_tokens must mirror prompt_tokens_details.cached_tokens") + }) + + t.Run("missing prompt_tokens_details yields zero cached_tokens", func(t *testing.T) { + resp := &openai.ChatCompletionResponse{ + Choices: []openai.ChatCompletionChoice{ + { + Message: openai.ChatCompletionMessage{Role: "assistant", Content: "hi"}, + FinishReason: openai.FinishReasonStop, + }, + }, + Usage: openai.Usage{ + PromptTokens: 100, + CompletionTokens: 10, + TotalTokens: 110, + // PromptTokensDetails intentionally nil — providers like Ollama + // and older OpenAI-compat backends omit this block entirely. + }, + } + + got, err := c.parseCompletionResponse(resp) + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, 0, got.Usage.CachedTokens, + "missing details must surface as zero, not panic") + }) +} + +// TestTokenUsage_CachedTokensJSONOmitempty ensures the new CachedTokens field +// stays out of serialized payloads when it is zero. This keeps logs and API +// responses unchanged for providers that never report cache hits. +func TestTokenUsage_CachedTokensJSONOmitempty(t *testing.T) { + t.Run("zero is omitted", func(t *testing.T) { + u := types.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15} + b, err := json.Marshal(u) + require.NoError(t, err) + assert.NotContains(t, string(b), "cached_tokens") + }) + + t.Run("non-zero is emitted", func(t *testing.T) { + u := types.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15, CachedTokens: 7} + b, err := json.Marshal(u) + require.NoError(t, err) + assert.Contains(t, string(b), `"cached_tokens":7`) + }) +} diff --git a/internal/types/chat.go b/internal/types/chat.go index d6e7071b..412f5bb1 100644 --- a/internal/types/chat.go +++ b/internal/types/chat.go @@ -10,6 +10,12 @@ type TokenUsage struct { PromptTokens int `json:"prompt_tokens"` CompletionTokens int `json:"completion_tokens"` TotalTokens int `json:"total_tokens"` + // CachedTokens is the subset of PromptTokens that hit a provider-side + // prompt cache (OpenAI prompt_tokens_details.cached_tokens, Qwen explicit + // caching, etc.). Zero when the provider does not report cache hits or + // when no cache was hit. Omitted from JSON when zero to keep payloads + // quiet for providers that never populate it. + CachedTokens int `json:"cached_tokens,omitempty"` } // LLMToolCall represents a function/tool call from the LLM