feat(chat): surface cached prompt tokens from upstream usage
OpenAI-compatible providers (Qwen, DeepSeek, OpenAI, Azure, etc.) report
prompt-cache hits in `usage.prompt_tokens_details.cached_tokens`. This
value was being read by go-openai but dropped at the WeKnora boundary,
so there was no way to tell whether prompt caching was actually working.
This change plumbs the field end-to-end:
- `types.TokenUsage.CachedTokens` (json:"cached_tokens,omitempty") —
zero-values are omitted so payloads stay quiet for providers that
never report cache hits.
- `cachedTokens` helper in remote_api.go guards against nil
`PromptTokensDetails` (Ollama and older OpenAI-compat backends omit
the details block entirely).
- All three response-parsing paths populate it:
* `parseCompletionResponse` (non-streaming)
* `processStream` (SDK streaming)
* `processRawHTTPStream` (raw-HTTP streaming, used when callers
need to inject custom fields like `cache_control`)
- The five `[LLM Usage]` log lines now print `cached_tokens=%d` so
cache hit rate is visible in `journalctl` / log tail without going
through metrics.
Together with the deterministic tool ordering from the previous commit,
this makes Qwen explicit caching observable: a warmed prefix should
show `cached_tokens` ≈ system + tools token count (typically several
thousand) on subsequent requests within the 5-minute TTL.
Tests:
- `TestCachedTokensHelper` — nil safety + round-trip
- `TestParseCompletionResponse_CachedTokens` — populated + missing
details paths through `parseCompletionResponse`
- `TestTokenUsage_CachedTokensJSONOmitempty` — zero is omitted,
non-zero is emitted
This commit is contained in:
@@ -424,8 +424,8 @@ func (c *RemoteAPIChat) Chat(ctx context.Context, messages []Message, opts *Chat
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logger.Infof(timeoutCtx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
|
||||
c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens)
|
||||
logger.Infof(timeoutCtx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d",
|
||||
c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens, result.Usage.CachedTokens)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -488,8 +488,8 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
|
||||
c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens)
|
||||
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d",
|
||||
c.modelName, result.Usage.PromptTokens, result.Usage.CompletionTokens, result.Usage.TotalTokens, result.Usage.CachedTokens)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -512,6 +512,7 @@ func (c *RemoteAPIChat) parseCompletionResponse(resp *openai.ChatCompletionRespo
|
||||
PromptTokens: resp.Usage.PromptTokens,
|
||||
CompletionTokens: resp.Usage.CompletionTokens,
|
||||
TotalTokens: resp.Usage.TotalTokens,
|
||||
CachedTokens: cachedTokens(resp.Usage.PromptTokensDetails),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -696,8 +697,8 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
if state.usage != nil {
|
||||
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
|
||||
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens)
|
||||
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d",
|
||||
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens)
|
||||
}
|
||||
toolCalls := state.buildOrderedToolCalls()
|
||||
streamChan <- types.StreamResponse{
|
||||
@@ -723,6 +724,7 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo
|
||||
PromptTokens: response.Usage.PromptTokens,
|
||||
CompletionTokens: response.Usage.CompletionTokens,
|
||||
TotalTokens: response.Usage.TotalTokens,
|
||||
CachedTokens: cachedTokens(response.Usage.PromptTokensDetails),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -745,8 +747,8 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
if state.usage != nil {
|
||||
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
|
||||
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens)
|
||||
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d",
|
||||
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens)
|
||||
}
|
||||
toolCalls := state.buildOrderedToolCalls()
|
||||
streamChan <- types.StreamResponse{
|
||||
@@ -773,8 +775,8 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res
|
||||
|
||||
if event.Done {
|
||||
if state.usage != nil {
|
||||
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d",
|
||||
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens)
|
||||
logger.Infof(ctx, "[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d",
|
||||
c.modelName, state.usage.PromptTokens, state.usage.CompletionTokens, state.usage.TotalTokens, state.usage.CachedTokens)
|
||||
}
|
||||
toolCalls := state.buildOrderedToolCalls()
|
||||
streamChan <- types.StreamResponse{
|
||||
@@ -814,6 +816,7 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res
|
||||
PromptTokens: streamResp.Usage.PromptTokens,
|
||||
CompletionTokens: streamResp.Usage.CompletionTokens,
|
||||
TotalTokens: streamResp.Usage.TotalTokens,
|
||||
CachedTokens: cachedTokens(streamResp.Usage.PromptTokensDetails),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1169,3 +1172,13 @@ func (c *RemoteAPIChat) GetBaseURL() string {
|
||||
func (c *RemoteAPIChat) GetAPIKey() string {
|
||||
return c.apiKey
|
||||
}
|
||||
|
||||
// cachedTokens returns the cached prompt-token count from an OpenAI-compatible
|
||||
// usage detail block, or zero when the provider did not report one. Some
|
||||
// providers omit PromptTokensDetails entirely, so the nil guard is required.
|
||||
func cachedTokens(d *openai.PromptTokensDetails) int {
|
||||
if d == nil {
|
||||
return 0
|
||||
}
|
||||
return d.CachedTokens
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/sashabaranov/go-openai"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -383,3 +384,93 @@ func TestRemoteAPIChat(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCachedTokensHelper covers the nil-safety contract of the cachedTokens
|
||||
// helper. Some providers omit PromptTokensDetails entirely; the helper must
|
||||
// return zero rather than panic.
|
||||
func TestCachedTokensHelper(t *testing.T) {
|
||||
assert.Equal(t, 0, cachedTokens(nil), "nil details must return zero")
|
||||
assert.Equal(t, 0, cachedTokens(&openai.PromptTokensDetails{}),
|
||||
"empty details must return zero")
|
||||
assert.Equal(t, 1234, cachedTokens(&openai.PromptTokensDetails{CachedTokens: 1234}),
|
||||
"populated cached_tokens must round-trip")
|
||||
}
|
||||
|
||||
// TestParseCompletionResponse_CachedTokens verifies that
|
||||
// prompt_tokens_details.cached_tokens from an OpenAI-compatible response is
|
||||
// propagated into TokenUsage.CachedTokens. This is the field Qwen explicit
|
||||
// caching populates on a cache hit.
|
||||
func TestParseCompletionResponse_CachedTokens(t *testing.T) {
|
||||
c := newTestRemoteChat(t)
|
||||
|
||||
t.Run("cached_tokens populated from prompt_tokens_details", func(t *testing.T) {
|
||||
resp := &openai.ChatCompletionResponse{
|
||||
Choices: []openai.ChatCompletionChoice{
|
||||
{
|
||||
Message: openai.ChatCompletionMessage{Role: "assistant", Content: "hi"},
|
||||
FinishReason: openai.FinishReasonStop,
|
||||
},
|
||||
},
|
||||
Usage: openai.Usage{
|
||||
PromptTokens: 6929,
|
||||
CompletionTokens: 42,
|
||||
TotalTokens: 6971,
|
||||
PromptTokensDetails: &openai.PromptTokensDetails{
|
||||
CachedTokens: 6900,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
got, err := c.parseCompletionResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, 6929, got.Usage.PromptTokens)
|
||||
assert.Equal(t, 42, got.Usage.CompletionTokens)
|
||||
assert.Equal(t, 6971, got.Usage.TotalTokens)
|
||||
assert.Equal(t, 6900, got.Usage.CachedTokens,
|
||||
"cached_tokens must mirror prompt_tokens_details.cached_tokens")
|
||||
})
|
||||
|
||||
t.Run("missing prompt_tokens_details yields zero cached_tokens", func(t *testing.T) {
|
||||
resp := &openai.ChatCompletionResponse{
|
||||
Choices: []openai.ChatCompletionChoice{
|
||||
{
|
||||
Message: openai.ChatCompletionMessage{Role: "assistant", Content: "hi"},
|
||||
FinishReason: openai.FinishReasonStop,
|
||||
},
|
||||
},
|
||||
Usage: openai.Usage{
|
||||
PromptTokens: 100,
|
||||
CompletionTokens: 10,
|
||||
TotalTokens: 110,
|
||||
// PromptTokensDetails intentionally nil — providers like Ollama
|
||||
// and older OpenAI-compat backends omit this block entirely.
|
||||
},
|
||||
}
|
||||
|
||||
got, err := c.parseCompletionResponse(resp)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, got)
|
||||
assert.Equal(t, 0, got.Usage.CachedTokens,
|
||||
"missing details must surface as zero, not panic")
|
||||
})
|
||||
}
|
||||
|
||||
// TestTokenUsage_CachedTokensJSONOmitempty ensures the new CachedTokens field
|
||||
// stays out of serialized payloads when it is zero. This keeps logs and API
|
||||
// responses unchanged for providers that never report cache hits.
|
||||
func TestTokenUsage_CachedTokensJSONOmitempty(t *testing.T) {
|
||||
t.Run("zero is omitted", func(t *testing.T) {
|
||||
u := types.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15}
|
||||
b, err := json.Marshal(u)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(b), "cached_tokens")
|
||||
})
|
||||
|
||||
t.Run("non-zero is emitted", func(t *testing.T) {
|
||||
u := types.TokenUsage{PromptTokens: 10, CompletionTokens: 5, TotalTokens: 15, CachedTokens: 7}
|
||||
b, err := json.Marshal(u)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(b), `"cached_tokens":7`)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -10,6 +10,12 @@ type TokenUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
// CachedTokens is the subset of PromptTokens that hit a provider-side
|
||||
// prompt cache (OpenAI prompt_tokens_details.cached_tokens, Qwen explicit
|
||||
// caching, etc.). Zero when the provider does not report cache hits or
|
||||
// when no cache was hit. Omitted from JSON when zero to keep payloads
|
||||
// quiet for providers that never populate it.
|
||||
CachedTokens int `json:"cached_tokens,omitempty"`
|
||||
}
|
||||
|
||||
// LLMToolCall represents a function/tool call from the LLM
|
||||
|
||||
Reference in New Issue
Block a user