feat(chat): surface cached prompt tokens from upstream usage

OpenAI-compatible providers (Qwen, DeepSeek, OpenAI, Azure, etc.) report
prompt-cache hits in `usage.prompt_tokens_details.cached_tokens`. This
value was being read by go-openai but dropped at the WeKnora boundary,
so there was no way to tell whether prompt caching was actually working.

This change plumbs the field end-to-end:

- `types.TokenUsage.CachedTokens` (json:"cached_tokens,omitempty") —
  zero-values are omitted so payloads stay quiet for providers that
  never report cache hits.
- `cachedTokens` helper in remote_api.go guards against nil
  `PromptTokensDetails` (Ollama and older OpenAI-compat backends omit
  the details block entirely).
- All three response-parsing paths populate it:
    * `parseCompletionResponse` (non-streaming)
    * `processStream` (SDK streaming)
    * `processRawHTTPStream` (raw-HTTP streaming, used when callers
      need to inject custom fields like `cache_control`)
- The five `[LLM Usage]` log lines now print `cached_tokens=%d` so
  cache hit rate is visible in `journalctl` / log tail without going
  through metrics.

Together with the deterministic tool ordering from the previous commit,
this makes Qwen explicit caching observable: a warmed prefix should
show `cached_tokens` ≈ system + tools token count (typically several
thousand) on subsequent requests within the 5-minute TTL.

Tests:
- `TestCachedTokensHelper` — nil safety + round-trip
- `TestParseCompletionResponse_CachedTokens` — populated + missing
  details paths through `parseCompletionResponse`
- `TestTokenUsage_CachedTokensJSONOmitempty` — zero is omitted,
  non-zero is emitted
This commit is contained in:
young1lin
2026-05-25 11:37:30 +08:00
committed by lyingbug
parent 29bec4204a
commit 582f7b3056
3 changed files with 120 additions and 10 deletions
+23 -10
View File
@@ -424,8 +424,8 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
if err != nil {
return nil, err
}
logger.Infof(timeoutCtx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens)
logger.Infof(timeoutCtx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d",
c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens, result.Usage.CachedTokens)
return result, nil
}
@@ -488,8 +488,8 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu
if err != nil {
return nil, err
}
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens)
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d",
c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens, result.Usage.CachedTokens)
return result, nil
}
@@ -512,6 +512,7 @@ func (c *RemoteAPIChat) parseCompletionResponse(resp *openai.ChatCompletionRespo
PromptTokens: resp.Usage.PromptTokens,
CompletionTokens: resp.Usage.CompletionTokens,
TotalTokens: resp.Usage.TotalTokens,
CachedTokens: cachedTokens(resp.Usage.PromptTokensDetails),
},
}
@@ -696,8 +697,8 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo
if err != nil {
if err == io.EOF {
if state.usage != nil {
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens)
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d",
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens)
}
toolCalls := state.buildOrderedToolCalls()
streamChan <- types.StreamResponse{
@@ -723,6 +724,7 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo
PromptTokens: response.Usage.PromptTokens,
CompletionTokens: response.Usage.CompletionTokens,
TotalTokens: response.Usage.TotalTokens,
CachedTokens: cachedTokens(response.Usage.PromptTokensDetails),
}
}
@@ -745,8 +747,8 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res
if err != nil {
if err == io.EOF {
if state.usage != nil {
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens)
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d",
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens)
}
toolCalls := state.buildOrderedToolCalls()
streamChan <- types.StreamResponse{
@@ -773,8 +775,8 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res
if event.Done {
if state.usage != nil {
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens)
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d",
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens)
}
toolCalls := state.buildOrderedToolCalls()
streamChan <- types.StreamResponse{
@@ -814,6 +816,7 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res
PromptTokens: streamResp.Usage.PromptTokens,
CompletionTokens: streamResp.Usage.CompletionTokens,
TotalTokens: streamResp.Usage.TotalTokens,
CachedTokens: cachedTokens(streamResp.Usage.PromptTokensDetails),
}
}
@@ -1169,3 +1172,13 @@ func (c *RemoteAPIChat) GetBaseURL() string {
func (c *RemoteAPIChat) GetAPIKey() string {
return c.apiKey
}
// cachedTokens returns the cached prompt-token count from an OpenAI-compatible
// usage detail block, or zero when the provider did not report one. Some
// providers omit PromptTokensDetails entirely, so the nil guard is required.
func cachedTokens(d *openai.PromptTokensDetails) int {
if d == nil {
return 0
}
return d.CachedTokens
}
+91
View File
@@ -8,6 +8,7 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/types"
"github.com/sashabaranov/go-openai"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -383,3 +384,93 @@ func TestRemoteAPIChat(t *testing.T) {
})
}
}
// TestCachedTokensHelper covers the nil-safety contract of the cachedTokens
// helper. Some providers omit PromptTokensDetails entirely; the helper must
// return zero rather than panic.
func TestCachedTokensHelper(t *testing.T) {
assert.Equal(t, 0, cachedTokens(nil), "nil details must return zero")
assert.Equal(t, 0, cachedTokens(&openai.PromptTokensDetails{}),
"empty details must return zero")
assert.Equal(t, 1234, cachedTokens(&openai.PromptTokensDetails{CachedTokens: 1234}),
"populated cached_tokens must round-trip")
}
// TestParseCompletionResponse_CachedTokens verifies that
// prompt_tokens_details.cached_tokens from an OpenAI-compatible response is
// propagated into TokenUsage.CachedTokens. This is the field Qwen explicit
// caching populates on a cache hit.
func TestParseCompletionResponse_CachedTokens(t *testing.T) {
c := newTestRemoteChat(t)
t.Run("cached_tokens populated from prompt_tokens_details", func(t *testing.T) {
resp := &openai.ChatCompletionResponse{
Choices: []openai.ChatCompletionChoice{
{
Message: openai.ChatCompletionMessage{Role: "assistant", Content: "hi"},
FinishReason: openai.FinishReasonStop,
},
},
Usage: openai.Usage{
PromptTokens: 6929,
CompletionTokens: 42,
TotalTokens: 6971,
PromptTokensDetails: &openai.PromptTokensDetails{
CachedTokens: 6900,
},
},
}
got, err := c.parseCompletionResponse(resp)
require.NoError(t, err)
require.NotNil(t, got)
assert.Equal(t, 6929, got.Usage.PromptTokens)
assert.Equal(t, 42, got.Usage.CompletionTokens)
assert.Equal(t, 6971, got.Usage.TotalTokens)
assert.Equal(t, 6900, got.Usage.CachedTokens,
"cached_tokens must mirror prompt_tokens_details.cached_tokens")
})
t.Run("missing prompt_tokens_details yields zero cached_tokens", func(t *testing.T) {
resp := &openai.ChatCompletionResponse{
Choices: []openai.ChatCompletionChoice{
{
Message: openai.ChatCompletionMessage{Role: "assistant", Content: "hi"},
FinishReason: openai.FinishReasonStop,
},
},
Usage: openai.Usage{
PromptTokens: 100,
CompletionTokens: 10,
TotalTokens: 110,
// PromptTokensDetails intentionally nil — providers like Ollama
// and older OpenAI-compat backends omit this block entirely.
},
}
got, err := c.parseCompletionResponse(resp)
require.NoError(t, err)
require.NotNil(t, got)
assert.Equal(t, 0, got.Usage.CachedTokens,
"missing details must surface as zero, not panic")
})
}
// TestTokenUsage_CachedTokensJSONOmitempty ensures the new CachedTokens field
// stays out of serialized payloads when it is zero. This keeps logs and API
// responses unchanged for providers that never report cache hits.
func TestTokenUsage_CachedTokensJSONOmitempty(t *testing.T) {
t.Run("zero is omitted", func(t *testing.T) {
u := types.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15}
b, err := json.Marshal(u)
require.NoError(t, err)
assert.NotContains(t, string(b), "cached_tokens")
})
t.Run("non-zero is emitted", func(t *testing.T) {
u := types.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15, CachedTokens: 7}
b, err := json.Marshal(u)
require.NoError(t, err)
assert.Contains(t, string(b), `"cached_tokens":7`)
})
}
+6
View File
@@ -10,6 +10,12 @@ type TokenUsage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
// CachedTokens is the subset of PromptTokens that hit a provider-side
// prompt cache (OpenAI prompt_tokens_details.cached_tokens, Qwen explicit
// caching, etc.). Zero when the provider does not report cache hits or
// when no cache was hit. Omitted from JSON when zero to keep payloads
// quiet for providers that never populate it.
CachedTokens int `json:"cached_tokens,omitempty"`
}
// LLMToolCall represents a function/tool call from the LLM