feat(llm): add prompt-cache observability and wiki cache-friendly prompts

Normalize provider-specific cache counters into TokenUsage, propagate call
purpose and prefix fingerprints through context for logging and Langfuse,
and restructure wiki page-modify prompts with warmup/singleflight to improve
prefix cache hit rates during reduce batches.
This commit is contained in:
wizardchen
2026-07-23 18:00:35 +08:00
committed by lyingbug
parent 6c794f64cf
commit 6fb858105a
36 changed files with 877 additions and 175 deletions
+4
View File
@@ -233,6 +233,10 @@ Dev 相关容器都带 `-dev` 后缀、用独立网络 `WeKnora-network-dev`,
| Input Tokens | `TokenUsage.PromptTokens` | 来自模型返回的 usage 字段。 |
| Output Tokens | `TokenUsage.CompletionTokens` | 来自模型返回的 usage 字段。 |
| Total Tokens | `TokenUsage.TotalTokens` | 大多数厂商返回;未返回时自动求和。 |
| Cache Read Tokens | `TokenUsage.CacheReadTokens` | 从 OpenAI-compatible `cached_tokens`、DeepSeek 原生 hit tokens 或 Anthropic cache read tokens 归一化。 |
| Cache Write Tokens | `TokenUsage.CacheWriteTokens` | Anthropic cache creation 或供应商等价字段;不与 Input Tokens 重复相加。 |
| Cache Miss Tokens | `TokenUsage.CacheMissTokens` | 已上报缓存口径下未从缓存读取的输入 token。 |
| Generation Metadata | `call_purpose` / `prompt_prefix_fingerprint` | 用调用用途分组缓存指标;前缀只上报不可逆短哈希,不上报原始 prompt。 |
| `userId` | `X-User-ID` / 空间 ID | 未登录时退化为 `tenant:<id>`,方便按空间汇总消耗;enqueue 时会写入 payload,worker 在无上游 trace 的场景也能保留归属。 |
| `sessionId` | URL 中的 `:session_id`(或 `RequestID` 兜底) | 可以在 Langfuse 的 Sessions 视图聚合一整场对话,或按单次异步批次聚合。 |
| Time-To-First-Token | 流式调用首条有效 chunk 的时间 | 通过 `generation-update.completionStartTime` 上报。 |
+2
View File
@@ -12,6 +12,7 @@ import (
"github.com/Tencent/WeKnora/internal/agent/token"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
)
const (
@@ -220,6 +221,7 @@ func (c *Consolidator) summarizeWithRetry(
for attempt := 1; attempt <= maxConsolidationAttempts; attempt++ {
summarizeCtx, cancel := context.WithTimeout(ctx, consolidationTimeout)
summarizeCtx = types.WithLLMCallMetadata(summarizeCtx, "agent_memory_consolidation", "")
resp, err := c.chatModel.Chat(summarizeCtx, []chat.Message{
{Role: "system", Content: consolidationSystemPrompt},
+28 -10
View File
@@ -315,13 +315,36 @@ If nothing in this batch is cite-worthy, return: {"citations": {}, "new_slugs":
Now apply the instructions above to the chunks and output ONLY the JSON.`
// WikiPageModifyPrompt updates an existing wiki page with new additions and removes stale/deleted information in a single pass.
const WikiPageModifyPrompt = `You are a wiki editor tasked with updating an existing wiki page. You must process a set of NEW information to add, AND/OR a set of deleted documents whose exclusive contributions must be REMOVED.
// WikiPageModifySystemPrompt contains only rules shared by every page update.
// Keeping page identity and source data out of this message gives providers a
// long byte-stable prefix to cache across a reduce batch.
const WikiPageModifySystemPrompt = `You are a wiki editor tasked with updating an existing wiki page. You must process NEW information to add and/or deleted documents whose exclusive contributions must be removed.
### SOURCE GROUNDING & MERGE RULES (CRITICAL):
1. **No Inline Chunk IDs:** Chunk aliases such as [c003] are internal processing metadata. NEVER output them in the page body or summary, and remove any legacy inline chunk aliases from existing content while editing. Source associations are stored separately by the system.
2. **Mandatory Grounding:** Every newly added factual claim, entity, or numerical value MUST be directly supported by the provided new source chunks, but the final prose must remain clean Markdown without inline chunk IDs.
3. **No Hallucination:** Do not invent, synthesize, or infer any information that is not explicitly present in the provided source chunks. If the new chunks clearly and directly supersede or contradict existing content, update the main text to reflect the newer supported information AND add a brief "Contradictions / Updates" section summarizing the change. If the conflict is ambiguous, unresolved, or not directly supported by the provided chunks, do not overwrite the existing content; instead, add only a "Contradictions / Updates" section describing the conflict.
4. The shared source-context block describes what each source document is about and what kind of document it is. Use it only to calibrate scope, attribution, and tone. Never copy source-context wording into the page as factual evidence.
5. Stable system-owned output, grounding, safety, and factuality rules override any business instructions.
### EDITING AND OUTPUT RULES:
1. You are a COMPILER, not a creative writer. Stay close to the verbatim source wording. You may lightly reorder, deduplicate, and join related sentences, but must not rephrase for style, expand short statements, or invent transitions.
2. Do not over-structure. Introduce a section heading only if the source or existing page uses it. Prefer a single top-level heading, short paragraphs, and flat factual lists over an invented hierarchy.
3. Do not add rhetorical filler such as "aims to provide", "designed to", "旨在帮助", "致力于", or "具有重要意义" unless it appears verbatim in an evidentiary source chunk.
4. Keep self-reported claims scoped and attributed. Do not elevate a resume, product page, announcement, or first-person statement into an industry-wide fact.
5. Preserve existing information that remains valid and on-topic. Maintain the existing page's structure and formatting style where possible.
6. Keep a [[slug|name]] link only when its slug is present in the supplied valid-link list. Never invent a slug and never link a page to itself.
7. Images may be included only from supplied new information. Treat each Markdown image URL as an opaque token and reproduce it exactly without altering, shortening, or normalizing it.
8. The first output line must be "SUMMARY: {one sentence, 15-40 words}", followed immediately by clean Markdown page content.
Output the SUMMARY line first, followed by the updated Markdown content, with no other preamble.`
// WikiPageModifyUserPrompt contains the per-batch and per-page data. The document-
// level source context deliberately comes first: all pages generated from one
// source then share the longest possible prefix before page metadata diverges.
const WikiPageModifyUserPrompt = `{{if .HasAdditions}}<shared_source_contexts>
{{.SharedSourceContexts}}</shared_source_contexts>
{{end}}
<page_metadata>
<slug>{{.PageSlug}}</slug>
@@ -341,7 +364,7 @@ This wiki page is specifically about **{{.PageTitle}}** (a {{.PageType}}). Every
{{.NewContent}}
</new_information>
The <new_information> block above is assembled from VERBATIM source chunks that were already cited as directly supporting this page. An optional <source_context> block inside each document is a document-level summary that tells you BOTH what the document is about AND what KIND of document it is (e.g. a resume, an announcement, a product page, a schedule) — use it to calibrate tone, stay on-topic, and avoid over-promotion. Do NOT quote the source_context text into the page; it is framing only.
The <new_information> block above is assembled from VERBATIM source chunks already cited as directly supporting this page. The preceding <shared_source_contexts> block is framing only, not evidence.
{{end}}
{{if .HasRetractions}}
@@ -367,19 +390,14 @@ The <new_information> block above is assembled from VERBATIM source chunks that
3. ADD and MERGE the facts from <new_information> into the page. You are a COMPILER, not a writer:
- **CRITICAL CONFLICT CHECK**: First verify that the <new_information> is actually about **{{.PageTitle}}** (as declared in <page_metadata>). If a piece of new info clearly belongs to a DIFFERENT but related thing (e.g., this page is about "Hunyuan Model" but the new info is about "Qwen3"; or this page is about "居民身份证" but the new info is about "工作居住证"), you MUST REJECT that part of the new information and DO NOT add it.
- If it is genuinely about {{.PageTitle}} and contradicts old content, prefer the newer information.
- **Stay close to source wording.** The chunks are verbatim. Reuse the source's own sentences; you MAY lightly reorder, deduplicate, and join related sentences, but do NOT rephrase for style, do NOT expand short statements into longer ones, and do NOT invent transitional sentences.
- **Do NOT over-structure.** Only introduce a section heading (##, ###) if the source itself uses that heading OR the page already has one from existing content. For a new page with flat source text, a single "# {{.PageTitle}}" heading plus 1-2 short paragraphs and a flat bullet list of facts is PREFERRED over inventing a hierarchy of subsections.
- **Do NOT add rhetorical filler.** Phrases like "旨在帮助…", "该平台致力于…", "具有重要意义", "designed to…", "aims to provide…" MUST NOT appear unless they are literally present in the source chunks.
- **Scope discipline.** The source_context tells you whether the document is self-reported (e.g. a resume) or third-party authoritative. If the source is self-reported, do NOT elevate claims into industry-wide statements — stay descriptive and attribute when useful ("根据简历所述…" / "as described by…" is acceptable when the source is first-person).
{{end}}
4. Preserve existing information that is still valid and still about {{.PageTitle}}.
5. Keep [[slug|name]] wiki-link references ONLY if the slug appears in the <valid_wiki_links> list above. Remove any [[slug|name]] whose slug is NOT in that list. Do NOT invent new wiki-link slugs. The page's own slug ({{.PageSlug}}) MUST NOT appear as a [[...]] link inside its own content.
6. Maintain the existing page structure and formatting style. Use "# {{.PageTitle}}" as the top-level heading if the page does not already have one. Do NOT introduce new heading levels beyond what the source or existing page justifies.
7. **Image rule**: Include relevant images using Markdown syntax: ![caption](url) from new information if applicable. The URL inside ![caption](url) is an opaque token; reproduce it EXACTLY and VERBATIM, do not alter, shorten, or normalize it.
{{if .HasRetractions}}
8. If after removing deleted content the page becomes nearly empty and there is no new information to add, output just: "SUMMARY: (empty page)\n# {{.PageTitle}}\n\n*This page's primary source document was removed.*"
7. If after removing deleted content the page becomes nearly empty and there is no new information to add, output just: "SUMMARY: (empty page)\n# {{.PageTitle}}\n\n*This page's primary source document was removed.*"
{{end}}
9. Write in {{.Language}}.
8. Write in {{.Language}}.
</instructions>
Output the SUMMARY line first, then the updated Markdown content. Do not include any other preamble.`
+45 -5
View File
@@ -124,20 +124,60 @@ func TestWikiChunkCitationPrompt_PreservesPlaceholders(t *testing.T) {
}
}
func TestWikiPageModifyPrompt_HidesInternalChunkAliases(t *testing.T) {
func TestWikiPageModifyUserPrompt_HidesInternalChunkAliases(t *testing.T) {
combined := WikiPageModifySystemPrompt + "\n" + WikiPageModifyUserPrompt
for _, guidance := range []string{
"NEVER output them in the page body or summary",
"Source associations are stored separately by the system",
"clean Markdown without inline chunk IDs",
} {
if !strings.Contains(WikiPageModifyPrompt, guidance) {
t.Errorf("WikiPageModifyPrompt missing chunk-alias guidance %q", guidance)
if !strings.Contains(combined, guidance) {
t.Errorf("WikiPageModifyUserPrompt missing chunk-alias guidance %q", guidance)
}
}
for _, obsolete := range []string{"Preserve Citations", "followed by an inline citation"} {
if strings.Contains(WikiPageModifyPrompt, obsolete) {
t.Errorf("WikiPageModifyPrompt still contains obsolete inline-citation rule %q", obsolete)
if strings.Contains(combined, obsolete) {
t.Errorf("WikiPageModifyUserPrompt still contains obsolete inline-citation rule %q", obsolete)
}
}
}
func renderWikiPageModify(t *testing.T, sourceContexts, title string) string {
t.Helper()
tmpl, err := template.New("modify").Parse(WikiPageModifyUserPrompt)
if err != nil {
t.Fatalf("parse template: %v", err)
}
var b strings.Builder
if err := tmpl.Execute(&b, map[string]string{
"HasAdditions": "1",
"SharedSourceContexts": sourceContexts,
"PageSlug": "concept/" + strings.ToLower(title),
"PageTitle": title,
"PageType": "concept",
"ExistingContent": "(New page)",
"NewContent": "page-specific chunks for " + title,
"Language": "English",
}); err != nil {
t.Fatalf("execute template: %v", err)
}
return b.String()
}
func TestWikiPageModifyUserPrompt_SharedSourceContextPrecedesPageVariables(t *testing.T) {
const shared = "<document><title>Same Source</title><context>long shared summary</context></document>"
a := renderWikiPageModify(t, shared, "Alpha")
b := renderWikiPageModify(t, shared, "Beta")
marker := "\n<page_metadata>\n"
ia, ib := strings.Index(a, marker), strings.Index(b, marker)
if ia < 0 || ib < 0 {
t.Fatalf("rendered prompt missing page metadata marker")
}
if a[:ia] != b[:ib] {
t.Fatalf("shared source prefix differs across pages\nA: %s\nB: %s", a[:ia], b[:ib])
}
if !strings.Contains(a[:ia], shared) {
t.Fatalf("shared source context is not part of the cacheable prefix: %s", a[:ia])
}
}
+2
View File
@@ -41,6 +41,8 @@ func (e *AgentEngine) streamLLMToEventBus(
messages = e.sourceRefs.EncodeMessages(messages)
messages = e.resourceRefs.EncodeMessages(messages)
prefixFingerprint := chat.PromptPrefixFingerprint(messages, opts)
llmCtx = types.WithLLMCallMetadata(llmCtx, "agent_round", prefixFingerprint)
stream, err := e.chatModel.ChatStream(llmCtx, messages, opts)
if err != nil {
logger.Errorf(ctx, "[Agent][Stream] Failed to start LLM stream: %v", err)
+2 -1
View File
@@ -785,7 +785,8 @@ Output only the scores, no explanations or additional text.`,
// Each score line is ~15 tokens, add buffer for safety
maxTokens := len(batch)*20 + 100
response, err := t.chatModel.Chat(ctx, messages, &chat.ChatOptions{
modelCtx := types.WithLLMCallMetadata(ctx, "knowledge_search_rerank", "")
response, err := t.chatModel.Chat(modelCtx, messages, &chat.ChatOptions{
Temperature: 0.1, // Low temperature for consistent scoring
MaxTokens: maxTokens,
})
+2 -1
View File
@@ -384,7 +384,8 @@ Web page content:
},
}
response, err := t.chatModel.Chat(ctx, messages, &chat.ChatOptions{
modelCtx := types.WithLLMCallMetadata(ctx, "web_fetch_summary", "")
response, err := t.chatModel.Chat(modelCtx, messages, &chat.ChatOptions{
Temperature: 0.3,
MaxTokens: 1024,
})
@@ -55,6 +55,7 @@ func (p *PluginChatCompletion) OnEvent(
resourceRefs := llmresource.NewRegistry()
chatMessages = sourceRefs.EncodeMessages(chatMessages)
chatMessages = resourceRefs.EncodeMessages(chatMessages)
ctx = withPromptCacheMetadata(ctx, chatModel, chatMessages, opt, "knowledge_qa")
// Call the chat model to generate response
pipelineInfo(ctx, "Completion", "model_call", map[string]interface{}{
@@ -60,6 +60,7 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
resourceRefs := llmresource.NewRegistry()
chatMessages = sourceRefs.EncodeMessages(chatMessages)
chatMessages = resourceRefs.EncodeMessages(chatMessages)
ctx = withPromptCacheMetadata(ctx, chatModel, chatMessages, opt, "knowledge_qa")
pipelineInfo(ctx, "Stream", "messages_ready", map[string]interface{}{
"message_count": len(chatMessages),
"system_prompt": chatMessages[0].Content,
@@ -113,6 +113,18 @@ func prepareMessagesWithHistory(chatManage *types.ChatManage) []chat.Message {
return chatMessages
}
func withPromptCacheMetadata(
ctx context.Context,
chatModel chat.Chat,
messages []chat.Message,
opts *chat.ChatOptions,
purpose string,
) context.Context {
prefixFingerprint := chat.PromptPrefixFingerprint(messages, opts)
_ = chatModel // model identity is already captured by the usage sink
return types.WithLLMCallMetadata(ctx, purpose, prefixFingerprint)
}
// AppendHistoryMessages appends prior Q&A rounds in chronological order.
// History is already filtered and truncated upstream by the load_history plugin.
func AppendHistoryMessages(messages []chat.Message, history []*types.History) []chat.Message {
@@ -119,7 +119,8 @@ If NO, leave the sql field empty.
Return your response in the specified JSON format.`, chatManage.Query, knowledge.ID, schema.Description())
response, err := chatModel.Chat(ctx, []chat.Message{
modelCtx := types.WithLLMCallMetadata(ctx, "data_analysis_plan", "")
response, err := chatModel.Chat(modelCtx, []chat.Message{
{Role: "user", Content: analysisPrompt},
}, &chat.ChatOptions{
Temperature: 0.1,
@@ -184,7 +184,8 @@ func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphDa
// logger.Debugf(ctx, "chat system: %s", generator.System(ctx))
// logger.Debugf(ctx, "chat user: %s", generator.User(ctx, content))
chatResponse, err := e.chat.Chat(ctx, generator.Render(ctx, content), e.chatOpt)
modelCtx := types.WithLLMCallMetadata(ctx, "entity_extraction", "")
chatResponse, err := e.chat.Chat(modelCtx, generator.Render(ctx, content), e.chatOpt)
if err != nil {
logger.Errorf(ctx, "failed to chat: %v", err)
return nil, err
@@ -114,7 +114,8 @@ func (p *PluginQueryUnderstand) OnEvent(ctx context.Context,
// --- Call model ---
thinking := false
response, err := rewriteModel.Chat(ctx, []chat.Message{
modelCtx := types.WithLLMCallMetadata(ctx, "query_rewrite", "")
response, err := rewriteModel.Chat(modelCtx, []chat.Message{
{Role: "system", Content: systemContent},
userMsg,
}, &chat.ChatOptions{
@@ -395,7 +395,7 @@ func (s *knowledgeService) scrubWikiPendingIngest(ctx context.Context, kbID, kno
// actual swap happens asynchronously inside mapOneDocument (see its
// oldPageSlugs handling) — that's where we have both the old page set and
// the freshly extracted candidate slugs, which is exactly the information
// the WikiPageModifyPrompt needs to do a correct replace-not-append.
// the WikiPageModifyUserPrompt needs to do a correct replace-not-append.
//
// So the only thing worth doing synchronously at reparse time is keeping
// the Redis pending list clean so the re-ingest enqueued by
@@ -829,7 +829,8 @@ func (s *knowledgeService) getSummary(ctx context.Context,
"language": types.LanguageNameFromContext(ctx),
})
thinking := false
summary, err := summaryModel.Chat(ctx, []chat.Message{
modelCtx := types.WithLLMCallMetadata(ctx, "document_summary", "")
summary, err := summaryModel.Chat(modelCtx, []chat.Message{
{
Role: "system",
Content: summaryPrompt,
@@ -1928,7 +1929,8 @@ func (s *knowledgeService) generateQuestionsWithContext(ctx context.Context,
prompt = types.AppendCustomPromptInstructions(prompt, customInstructions, "question_generation")
thinking := false
response, err := chatModel.Chat(ctx, []chat.Message{
modelCtx := types.WithLLMCallMetadata(ctx, "question_generation", "")
response, err := chatModel.Chat(modelCtx, []chat.Message{
{
Role: "user",
Content: prompt,
+166 -33
View File
@@ -23,6 +23,7 @@ import (
"github.com/google/uuid"
"github.com/hibiken/asynq"
"github.com/redis/go-redis/v9"
"golang.org/x/sync/singleflight"
)
// ErrWikiIngestConcurrent is returned by the wiki ingest handler in Lite mode
@@ -344,6 +345,17 @@ type wikiIngestService struct {
// wiki:finalize:active:<kbID> lock, keeping two finalize runs for the
// same KB from overlapping when there is no Redis.
liteFinalizeLocks sync.Map
// llmRequests coalesces byte-identical concurrent prompts within this
// process. Keys include tenant and model to preserve isolation.
llmRequests singleflight.Group
// promptWarmups serializes only the first request for a reusable Wiki page
// prefix. Other prefixes and already-warmed cohorts stay parallel.
promptWarmups sync.Map
}
type wikiPromptWarmup struct {
done chan struct{}
once sync.Once
}
// NewWikiIngestService creates a new wiki ingest service
@@ -1399,7 +1411,7 @@ var wikiLinkRE = regexp.MustCompile(`\[\[([^\[\]\|\s]+)(?:\|([^\]]+))?\]\]`)
// Background: WikiSummaryPrompt instructs the LLM to embed wiki links
// for every extracted slug it knows about, but slug extraction happens
// during map (parallel with summary generation) and the actual page
// creation happens later in reduce. When reduce's WikiPageModifyPrompt
// creation happens later in reduce. When reduce's WikiPageModifyUserPrompt
// fails on an entity/concept slug the page never gets written — and
// the already-persisted summary is left holding a `[[entity/foo|name]]`
// link that 404s.
@@ -2379,42 +2391,163 @@ func (s *wikiIngestService) generateWithTemplate(ctx context.Context, chatModel
}
prompt := buf.String()
prompt = types.AppendCustomPromptInstructions(prompt, data["CustomInstructions"], data["InstructionScope"])
thinking := false
var lastErr error
for attempt := 1; attempt <= wikiLLMMaxAttempts; attempt++ {
response, err := chatModel.Chat(ctx, []chat.Message{
purpose := wikiPromptPurpose(promptTpl)
messages := []chat.Message{{Role: "user", Content: prompt}}
if promptTpl == agent.WikiPageModifyUserPrompt {
systemPrompt := types.AppendCustomPromptInstructions(
agent.WikiPageModifySystemPrompt,
maskedData["CustomInstructions"],
maskedData["InstructionScope"],
)
messages = []chat.Message{
{Role: "system", Content: systemPrompt},
{Role: "user", Content: prompt},
}, &chat.ChatOptions{
Temperature: 0.3,
Thinking: &thinking,
})
if err == nil {
return unmaskImageURLs(response.Content, urlMap), nil
}
lastErr = err
// Abort immediately on non-retryable errors (4xx except 408/429,
// parse/marshal failures, tool-side bugs, etc.). Retrying a
// hard "invalid arguments" error just wastes the model's budget.
if !isTransientLLMError(ctx, err) {
return "", fmt.Errorf("LLM call failed: %w", err)
}
if attempt == wikiLLMMaxAttempts {
break
}
backoff := wikiLLMBackoffBase << (attempt - 1)
logger.Warnf(ctx, "wiki ingest: LLM call failed (attempt %d/%d), retrying in %s: %v",
attempt, wikiLLMMaxAttempts, backoff, err)
select {
case <-ctx.Done():
return "", fmt.Errorf("LLM call aborted during backoff: %w", ctx.Err())
case <-time.After(backoff):
} else {
messages[0].Content = types.AppendCustomPromptInstructions(
prompt, maskedData["CustomInstructions"], maskedData["InstructionScope"],
)
}
thinking := false
opts := &chat.ChatOptions{Temperature: 0.3, Thinking: &thinking}
prefixFingerprint := chat.PromptPrefixFingerprint(messages, opts)
warmupKey := ""
if promptTpl == agent.WikiPageModifyUserPrompt {
prefixFingerprint = chat.FingerprintPromptPrefix(
messages[0].Content, maskedData["SharedSourceContexts"],
)
if tenantID, ok := types.TenantIDFromContext(ctx); ok {
warmupKey = chat.BuildPromptCacheKey(
tenantID, chatModel.GetModelID(), purpose, prefixFingerprint,
)
}
}
return "", fmt.Errorf("LLM call failed after %d attempts: %w", wikiLLMMaxAttempts, lastErr)
ctx = types.WithLLMCallMetadata(ctx, purpose, prefixFingerprint)
tenantID, tenantScoped := types.TenantIDFromContext(ctx)
requestJSON, _ := json.Marshal(struct {
Messages []chat.Message `json:"messages"`
Options *chat.ChatOptions `json:"options"`
}{Messages: messages, Options: opts})
requestKey := chat.BuildPromptCacheKey(
tenantID, chatModel.GetModelID(), "wiki_exact_request",
chat.FingerprintPromptPrefix(string(requestJSON)),
)
execute := func() (interface{}, error) {
releaseWarmup := func() {}
if tenantScoped && promptTpl == agent.WikiPageModifyUserPrompt && strings.TrimSpace(maskedData["SharedSourceContexts"]) != "" {
var warmupErr error
releaseWarmup, warmupErr = s.awaitWikiPromptWarmup(ctx, warmupKey)
if warmupErr != nil {
return "", warmupErr
}
}
defer releaseWarmup()
var lastErr error
for attempt := 1; attempt <= wikiLLMMaxAttempts; attempt++ {
response, callErr := chatModel.Chat(ctx, messages, opts)
if callErr == nil && response != nil {
return response.Content, nil
}
if callErr == nil {
callErr = errors.New("LLM returned nil response")
}
lastErr = callErr
if !isTransientLLMError(ctx, callErr) {
return "", fmt.Errorf("LLM call failed: %w", callErr)
}
if attempt == wikiLLMMaxAttempts {
break
}
backoff := wikiLLMBackoffBase << (attempt - 1)
logger.Warnf(ctx, "wiki ingest: LLM call failed (attempt %d/%d), retrying in %s: %v",
attempt, wikiLLMMaxAttempts, backoff, callErr)
timer := time.NewTimer(backoff)
select {
case <-ctx.Done():
timer.Stop()
return "", fmt.Errorf("LLM call aborted during backoff: %w", ctx.Err())
case <-timer.C:
}
}
return "", fmt.Errorf("LLM call failed after %d attempts: %w", wikiLLMMaxAttempts, lastErr)
}
// Missing tenant context is unexpected for production Wiki work. Fail safe
// by skipping cross-call coalescing instead of putting unrelated requests
// into a synthetic tenant-0 bucket.
if !tenantScoped {
value, executeErr := execute()
if executeErr != nil {
return "", executeErr
}
content, _ := value.(string)
return unmaskImageURLs(content, urlMap), nil
}
resultCh := s.llmRequests.DoChan(requestKey, execute)
select {
case <-ctx.Done():
return "", ctx.Err()
case result := <-resultCh:
if result.Err != nil {
return "", result.Err
}
content, _ := result.Val.(string)
return unmaskImageURLs(content, urlMap), nil
}
}
func wikiPromptPurpose(promptTpl string) string {
switch promptTpl {
case agent.WikiPageModifyUserPrompt:
return "wiki_page_modify"
case agent.WikiChunkCitationPrompt:
return "wiki_chunk_citation"
case agent.WikiCandidateSlugPrompt:
return "wiki_candidate_slug"
case agent.WikiSummaryPrompt:
return "wiki_summary"
case agent.WikiKnowledgeExtractPrompt:
return "wiki_knowledge_extract"
case agent.WikiTaxonomyPlanPrompt:
return "wiki_taxonomy_plan"
case agent.WikiDeduplicationPrompt:
return "wiki_deduplication"
case agent.WikiIndexIntroPrompt, agent.WikiIndexIntroUpdatePrompt:
return "wiki_index_intro"
default:
return "wiki_generation"
}
}
func (s *wikiIngestService) awaitWikiPromptWarmup(ctx context.Context, key string) (func(), error) {
if key == "" {
return func() {}, nil
}
candidate := &wikiPromptWarmup{done: make(chan struct{})}
actual, loaded := s.promptWarmups.LoadOrStore(key, candidate)
entry := actual.(*wikiPromptWarmup)
if !loaded {
return func() {
entry.once.Do(func() { close(entry.done) })
// Keep the local warmed marker long enough to cover the parallel
// reduce burst without turning it into a persistent application cache.
time.AfterFunc(4*time.Minute, func() {
s.promptWarmups.CompareAndDelete(key, entry)
})
}, nil
}
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-entry.done:
return func() {}, nil
}
}
// isTransientLLMError reports whether an error from the chat provider
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
@@ -1538,7 +1539,7 @@ func (s *wikiIngestService) mapOneDocument(
// swap: emit BOTH a "retract" (carrying the doc's PRIOR summary
// body as the old-version signal) AND the normal addition. The
// reduce stage sees HasAdditions=1 + HasRetractions=1 and the
// WikiPageModifyPrompt correctly tells the editor model to
// WikiPageModifyUserPrompt correctly tells the editor model to
// remove the old K section and add the new K section in one
// pass — giving us replace-not-append semantics that "append
// new K on top of old K" would otherwise violate.
@@ -1706,7 +1707,7 @@ func (s *wikiIngestService) extractEntitiesAndConceptsNoUpsert(
// - changed: whether the wiki page was created or updated
// - affectedType: "ingest" or "retract" — drives downstream bookkeeping
// - additionFailed: true iff the slug had entity/concept additions queued
// AND the WikiPageModifyPrompt LLM call failed, so no page exists/was
// AND the WikiPageModifyUserPrompt LLM call failed, so no page exists/was
// refreshed for it. Callers use this to sanitize dead [[slug]] links
// elsewhere (e.g. in the doc's summary page) and to drop the slug from
// the wiki log feed so users don't see a clickable entry that 404s.
@@ -1875,6 +1876,7 @@ func (s *wikiIngestService) reduceSlugUpdates(
var deletedContent strings.Builder
var relatedSlugs strings.Builder
var newContentBuilder strings.Builder
var sharedSourceContexts strings.Builder
var docTitles []string
var language string
@@ -1936,6 +1938,11 @@ func (s *wikiIngestService) reduceSlugUpdates(
// knowledge ID, so the <new_information> block can quote the chunks
// verbatim instead of relying on the short Details paraphrase.
chunkContentByID := s.resolveCitedChunks(ctx, tenantID, additions)
// A document summary is shared by every page derived from that document.
// Render a deterministic, de-duplicated block before any page-specific
// metadata so provider prefix caches can reuse it across parallel reduce
// calls.
sourceContextByRef := make(map[string]string)
for _, add := range additions {
cited := collectCitedChunkContent(add.SourceChunks, chunkContentByID)
@@ -1947,21 +1954,27 @@ func (s *wikiIngestService) reduceSlugUpdates(
// source documents, and calibrating tone (self-reported vs
// third-party authoritative) benefits from the richer context.
sourceCtx := strings.TrimSpace(add.DocSummary)
sourceCtxBlock := ""
if sourceCtx != "" {
sourceCtxBlock = fmt.Sprintf("<source_context>\n%s\n</source_context>\n", sourceCtx)
contextKey := add.SourceRef
if contextKey == "" {
contextKey = add.KnowledgeID + "\x00" + add.DocTitle
}
sourceContextByRef[contextKey] = fmt.Sprintf(
"<document>\n<title>%s</title>\n<context>\n%s\n</context>\n</document>\n",
add.DocTitle, sourceCtx,
)
}
if cited != "" {
fmt.Fprintf(&newContentBuilder,
"<document>\n<title>%s</title>\n%s<content>\n**%s**: %s\n\n%s\n</content>\n</document>\n\n",
add.DocTitle, sourceCtxBlock, add.Item.Name, add.Item.Description, cited)
"<document>\n<title>%s</title>\n<content>\n**%s**: %s\n\n%s\n</content>\n</document>\n\n",
add.DocTitle, add.Item.Name, add.Item.Description, cited)
} else {
// Fallback: no citations available (legacy path, citation pass
// failed, or bad chunk IDs were filtered out) — stick with
// the short Details summary so the page still gets real text.
fmt.Fprintf(&newContentBuilder,
"<document>\n<title>%s</title>\n%s<content>\n**%s**: %s\n\n%s\n</content>\n</document>\n\n",
add.DocTitle, sourceCtxBlock, add.Item.Name, add.Item.Description, add.Item.Details)
"<document>\n<title>%s</title>\n<content>\n**%s**: %s\n\n%s\n</content>\n</document>\n\n",
add.DocTitle, add.Item.Name, add.Item.Description, add.Item.Details)
}
docTitles = appendUnique(docTitles, add.DocTitle)
@@ -1977,6 +1990,15 @@ func (s *wikiIngestService) reduceSlugUpdates(
page.PageType = add.Type
}
}
contextKeys := make([]string, 0, len(sourceContextByRef))
for key := range sourceContextByRef {
contextKeys = append(contextKeys, key)
}
sort.Strings(contextKeys)
for _, key := range contextKeys {
sharedSourceContexts.WriteString(sourceContextByRef[key])
}
}
if len(additions) > 0 || len(retracts) > 0 {
@@ -2019,7 +2041,7 @@ func (s *wikiIngestService) reduceSlugUpdates(
pageAliases := strings.Join(page.Aliases, ", ")
var updatedContent string
updatedContent, err = s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyPrompt, map[string]string{
updatedContent, err = s.generateWithTemplate(ctx, chatModel, agent.WikiPageModifyUserPrompt, map[string]string{
"HasAdditions": hasAdditionsStr,
"HasRetractions": hasRetractionsStr,
"PageSlug": slug,
@@ -2027,6 +2049,7 @@ func (s *wikiIngestService) reduceSlugUpdates(
"PageType": pageType,
"PageAliases": pageAliases,
"ExistingContent": existingContent,
"SharedSourceContexts": sharedSourceContexts.String(),
"NewContent": newContentBuilder.String(),
"DeletedContent": deletedContent.String(),
"RemainingSourcesContent": remainingSourcesContent.String(),
@@ -4,9 +4,12 @@ import (
"context"
"errors"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/agent"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -334,16 +337,25 @@ func TestGenerateWithTemplateMasksImageURLsBeforeLLM(t *testing.T) {
type templateCaptureChatModel struct {
prompt string
response string
messages []chat.Message
options chat.ChatOptions
purpose string
prefix string
}
func (m *templateCaptureChatModel) Chat(
_ context.Context,
ctx context.Context,
messages []chat.Message,
_ *chat.ChatOptions,
opts *chat.ChatOptions,
) (*types.ChatResponse, error) {
if len(messages) > 0 {
m.prompt = messages[0].Content
}
m.messages = append([]chat.Message(nil), messages...)
if opts != nil {
m.options = *opts
}
m.purpose, m.prefix = types.LLMCallMetadataFromContext(ctx)
return &types.ChatResponse{Content: m.response}, nil
}
@@ -358,6 +370,118 @@ func (m *templateCaptureChatModel) ChatStream(
func (m *templateCaptureChatModel) GetModelName() string { return "capture" }
func (m *templateCaptureChatModel) GetModelID() string { return "capture" }
func TestGenerateWikiPageModifyUsesCacheableMessageLayout(t *testing.T) {
model := &templateCaptureChatModel{response: "SUMMARY: page\n# Alpha"}
service := &wikiIngestService{}
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
_, err := service.generateWithTemplate(ctx, model, agent.WikiPageModifyUserPrompt, map[string]string{
"HasAdditions": "1",
"SharedSourceContexts": "<document><context>shared source summary</context></document>\n",
"PageSlug": "concept/alpha",
"PageTitle": "Alpha",
"PageType": "concept",
"ExistingContent": "(New page)",
"NewContent": "page-specific evidence",
"AvailableSlugs": "concept/beta (Beta)",
"Language": "English",
"InstructionScope": "wiki_content",
})
if err != nil {
t.Fatalf("generateWithTemplate() error = %v", err)
}
if len(model.messages) != 2 || model.messages[0].Role != "system" || model.messages[1].Role != "user" {
t.Fatalf("unexpected message layout: %#v", model.messages)
}
if !strings.Contains(model.messages[0].Content, "SOURCE GROUNDING & MERGE RULES") {
t.Fatalf("system prompt missing stable rules: %q", model.messages[0].Content)
}
if !strings.HasPrefix(model.messages[1].Content, "<shared_source_contexts>") {
t.Fatalf("shared source context must lead user message: %q", model.messages[1].Content)
}
if model.purpose != "wiki_page_modify" || model.prefix == "" {
t.Fatalf("missing cache metadata: purpose=%q prefix=%q", model.purpose, model.prefix)
}
}
func TestAwaitWikiPromptWarmupBlocksFollowersUntilLeaderCompletes(t *testing.T) {
service := &wikiIngestService{}
release, err := service.awaitWikiPromptWarmup(context.Background(), "same-prefix")
if err != nil {
t.Fatal(err)
}
followerDone := make(chan error, 1)
go func() {
_, followErr := service.awaitWikiPromptWarmup(context.Background(), "same-prefix")
followerDone <- followErr
}()
select {
case <-followerDone:
t.Fatal("follower passed warmup gate before leader completed")
case <-time.After(20 * time.Millisecond):
}
release()
select {
case followErr := <-followerDone:
if followErr != nil {
t.Fatal(followErr)
}
case <-time.After(time.Second):
t.Fatal("follower did not resume after leader completed")
}
}
type blockingTemplateChatModel struct {
calls atomic.Int32
started chan struct{}
release chan struct{}
once sync.Once
}
func (m *blockingTemplateChatModel) Chat(
context.Context,
[]chat.Message,
*chat.ChatOptions,
) (*types.ChatResponse, error) {
m.calls.Add(1)
m.once.Do(func() { close(m.started) })
<-m.release
return &types.ChatResponse{Content: "shared result"}, nil
}
func (m *blockingTemplateChatModel) ChatStream(context.Context, []chat.Message, *chat.ChatOptions) (<-chan types.StreamResponse, error) {
return nil, nil
}
func (m *blockingTemplateChatModel) GetModelName() string { return "blocking" }
func (m *blockingTemplateChatModel) GetModelID() string { return "blocking" }
func TestGenerateWithTemplateCoalescesIdenticalConcurrentRequests(t *testing.T) {
model := &blockingTemplateChatModel{started: make(chan struct{}), release: make(chan struct{})}
service := &wikiIngestService{}
result := make(chan error, 2)
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(7))
call := func() {
_, err := service.generateWithTemplate(ctx, model, "same {{.Value}}", map[string]string{"Value": "input"})
result <- err
}
go call()
<-model.started
go call()
time.Sleep(20 * time.Millisecond)
if got := model.calls.Load(); got != 1 {
t.Fatalf("identical requests reached provider %d times before release", got)
}
close(model.release)
for range 2 {
if err := <-result; err != nil {
t.Fatal(err)
}
}
if got := model.calls.Load(); got != 1 {
t.Fatalf("identical requests reached provider %d times, want 1", got)
}
}
func TestWikiIngestCleanupContextDetachedFromCancelledParent(t *testing.T) {
parent := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(42))
ctx, cancel := context.WithCancel(parent)
+83 -24
View File
@@ -50,8 +50,10 @@ type anthropicResponse struct {
} `json:"content"`
StopReason string `json:"stop_reason"`
Usage struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheCreationInputTokens *int `json:"cache_creation_input_tokens"`
CacheReadInputTokens *int `json:"cache_read_input_tokens"`
} `json:"usage"`
Error *struct {
Type string `json:"type"`
@@ -63,10 +65,10 @@ type anthropicStreamEvent struct {
Type string `json:"type"`
Message *struct {
Usage struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
CacheReadInputTokens int `json:"cache_read_input_tokens"`
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheCreationInputTokens *int `json:"cache_creation_input_tokens"`
CacheReadInputTokens *int `json:"cache_read_input_tokens"`
} `json:"usage"`
} `json:"message,omitempty"`
Delta *struct {
@@ -75,10 +77,10 @@ type anthropicStreamEvent struct {
StopReason string `json:"stop_reason"`
} `json:"delta,omitempty"`
Usage *struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
CacheReadInputTokens int `json:"cache_read_input_tokens"`
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheCreationInputTokens *int `json:"cache_creation_input_tokens"`
CacheReadInputTokens *int `json:"cache_read_input_tokens"`
} `json:"usage,omitempty"`
Error *struct {
Type string `json:"type"`
@@ -316,14 +318,20 @@ func (c *AnthropicChat) parseResponse(resp *anthropicResponse) *types.ChatRespon
}
inputTokens := resp.Usage.InputTokens
outputTokens := resp.Usage.OutputTokens
cacheRead := valueOrZero(resp.Usage.CacheReadInputTokens)
cacheWrite := valueOrZero(resp.Usage.CacheCreationInputTokens)
promptTokens := inputTokens + cacheRead + cacheWrite
usage := types.TokenUsage{
PromptTokens: promptTokens,
CompletionTokens: outputTokens,
TotalTokens: promptTokens + outputTokens,
}
usage.SetPromptCacheUsage(cacheRead, cacheWrite, max(0, promptTokens-cacheRead),
resp.Usage.CacheReadInputTokens != nil || resp.Usage.CacheCreationInputTokens != nil)
return &types.ChatResponse{
Content: strings.Join(parts, ""),
FinishReason: resp.StopReason,
Usage: types.TokenUsage{
PromptTokens: inputTokens,
CompletionTokens: outputTokens,
TotalTokens: inputTokens + outputTokens,
},
Usage: usage,
}
}
@@ -333,6 +341,9 @@ func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
var finishReason string
var inputTokens int
var outputTokens int
var cacheReadTokens int
var cacheWriteTokens int
var cacheReported bool
for {
event, err := sseReader.ReadEvent()
@@ -359,6 +370,11 @@ func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
if streamEvent.Message != nil {
inputTokens = max(inputTokens, streamEvent.Message.Usage.InputTokens)
outputTokens = max(outputTokens, streamEvent.Message.Usage.OutputTokens)
cacheReadTokens, cacheWriteTokens, cacheReported = mergeAnthropicCacheCounters(
cacheReadTokens, cacheWriteTokens, cacheReported,
streamEvent.Message.Usage.CacheReadInputTokens,
streamEvent.Message.Usage.CacheCreationInputTokens,
)
}
if streamEvent.Delta != nil {
if streamEvent.Delta.Type == "text_delta" && streamEvent.Delta.Text != "" {
@@ -371,17 +387,26 @@ func parseAnthropicSSE(reader io.Reader) (*types.ChatResponse, error) {
if streamEvent.Usage != nil {
inputTokens = max(inputTokens, streamEvent.Usage.InputTokens)
outputTokens = max(outputTokens, streamEvent.Usage.OutputTokens)
cacheReadTokens, cacheWriteTokens, cacheReported = mergeAnthropicCacheCounters(
cacheReadTokens, cacheWriteTokens, cacheReported,
streamEvent.Usage.CacheReadInputTokens,
streamEvent.Usage.CacheCreationInputTokens,
)
}
}
promptTokens := inputTokens + cacheReadTokens + cacheWriteTokens
usage := types.TokenUsage{
PromptTokens: promptTokens,
CompletionTokens: outputTokens,
TotalTokens: promptTokens + outputTokens,
}
usage.SetPromptCacheUsage(cacheReadTokens, cacheWriteTokens,
max(0, promptTokens-cacheReadTokens), cacheReported)
return &types.ChatResponse{
Content: strings.Join(contentParts, ""),
FinishReason: finishReason,
Usage: types.TokenUsage{
PromptTokens: inputTokens,
CompletionTokens: outputTokens,
TotalTokens: inputTokens + outputTokens,
},
Usage: usage,
}, nil
}
@@ -447,7 +472,10 @@ func processAnthropicStream(ctx context.Context, model string, resp *http.Respon
return
}
if streamEvent.Message != nil {
usage = mergeAnthropicUsage(usage, streamEvent.Message.Usage.InputTokens, streamEvent.Message.Usage.OutputTokens)
usage = mergeAnthropicUsage(usage, streamEvent.Message.Usage.InputTokens,
streamEvent.Message.Usage.OutputTokens,
streamEvent.Message.Usage.CacheReadInputTokens,
streamEvent.Message.Usage.CacheCreationInputTokens)
}
if streamEvent.Delta != nil {
if streamEvent.Delta.StopReason != "" {
@@ -462,17 +490,48 @@ func processAnthropicStream(ctx context.Context, model string, resp *http.Respon
}
}
if streamEvent.Usage != nil {
usage = mergeAnthropicUsage(usage, streamEvent.Usage.InputTokens, streamEvent.Usage.OutputTokens)
usage = mergeAnthropicUsage(usage, streamEvent.Usage.InputTokens,
streamEvent.Usage.OutputTokens,
streamEvent.Usage.CacheReadInputTokens,
streamEvent.Usage.CacheCreationInputTokens)
}
}
}
func mergeAnthropicUsage(current *types.TokenUsage, inputTokens, outputTokens int) *types.TokenUsage {
func mergeAnthropicUsage(
current *types.TokenUsage,
inputTokens, outputTokens int,
cacheRead, cacheWrite *int,
) *types.TokenUsage {
if current == nil {
current = &types.TokenUsage{}
}
current.PromptTokens = max(current.PromptTokens, inputTokens)
read, write, reported := mergeAnthropicCacheCounters(
current.CacheReadTokens, current.CacheWriteTokens, current.CacheReported,
cacheRead, cacheWrite,
)
uncachedInput := max(0, current.PromptTokens-current.CacheReadTokens-current.CacheWriteTokens)
uncachedInput = max(uncachedInput, inputTokens)
current.PromptTokens = uncachedInput + read + write
current.CompletionTokens = max(current.CompletionTokens, outputTokens)
current.TotalTokens = current.PromptTokens + current.CompletionTokens
current.SetPromptCacheUsage(read, write, max(0, current.PromptTokens-read), reported)
return current
}
func mergeAnthropicCacheCounters(
currentRead, currentWrite int,
currentReported bool,
cacheRead, cacheWrite *int,
) (read, write int, reported bool) {
read = currentRead
write = currentWrite
reported = currentReported || cacheRead != nil || cacheWrite != nil
if cacheRead != nil {
read = max(read, *cacheRead)
}
if cacheWrite != nil {
write = max(write, *cacheWrite)
}
return read, write, reported
}
+35 -4
View File
@@ -70,6 +70,33 @@ func TestAnthropicChat(t *testing.T) {
assert.Equal(t, 5, resp.Usage.TotalTokens)
}
func TestAnthropicChat_CacheUsage(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"id":"msg_cached","type":"message","role":"assistant",
"content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn",
"usage":{"input_tokens":24,"output_tokens":2,"cache_creation_input_tokens":100,"cache_read_input_tokens":900}
}`))
}))
defer server.Close()
chat, err := NewAnthropicChat(&ChatConfig{
Source: types.ModelSourceRemote, BaseURL: server.URL, ModelName: "claude-sonnet-4-5",
APIKey: "test-key", Provider: string(provider.ProviderAnthropic),
})
require.NoError(t, err)
resp, err := chat.Chat(context.Background(), []Message{{Role: "user", Content: "Hi"}}, nil)
require.NoError(t, err)
assert.Equal(t, 1024, resp.Usage.PromptTokens)
assert.Equal(t, 900, resp.Usage.CacheReadTokens)
assert.Equal(t, 100, resp.Usage.CacheWriteTokens)
assert.Equal(t, 124, resp.Usage.CacheMissTokens)
assert.True(t, resp.Usage.CacheReported)
assert.Equal(t, types.PromptCacheStatusHit, resp.Usage.CacheStatus)
}
func TestAnthropicChat_FullEndpoint(t *testing.T) {
t.Setenv("SSRF_WHITELIST", "127.0.0.1")
@@ -145,13 +172,13 @@ func TestAnthropicChat_SSEResponse(t *testing.T) {
assert.Equal(t, "/v1/messages", r.URL.Path)
w.Header().Set("Content-Type", "text/event-stream")
_, _ = w.Write([]byte(`event: message_start
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
data: {"type":"message_start","message":{"usage":{"input_tokens":14,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":100}}}
event: content_block_delta
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
event: message_delta
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}
event: message_stop
data: {"type":"message_stop"}
@@ -177,6 +204,8 @@ data: {"type":"message_stop"}
assert.Equal(t, 114, resp.Usage.PromptTokens)
assert.Equal(t, 5, resp.Usage.CompletionTokens)
assert.Equal(t, 119, resp.Usage.TotalTokens)
assert.Equal(t, 100, resp.Usage.CacheReadTokens)
assert.Equal(t, 14, resp.Usage.CacheMissTokens)
}
func TestAnthropicChat_ChatStream(t *testing.T) {
@@ -189,13 +218,13 @@ func TestAnthropicChat_ChatStream(t *testing.T) {
require.NoError(t, json.NewDecoder(r.Body).Decode(&capturedRequest))
w.Header().Set("Content-Type", "text/event-stream")
_, _ = w.Write([]byte(`event: message_start
data: {"type":"message_start","message":{"usage":{"input_tokens":0,"output_tokens":0}}}
data: {"type":"message_start","message":{"usage":{"input_tokens":14,"output_tokens":0,"cache_creation_input_tokens":0,"cache_read_input_tokens":100}}}
event: content_block_delta
data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"pong"}}
event: message_delta
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":114,"output_tokens":5}}
data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":5}}
event: message_stop
data: {"type":"message_stop"}
@@ -231,6 +260,8 @@ data: {"type":"message_stop"}
assert.Equal(t, 114, chunks[1].Usage.PromptTokens)
assert.Equal(t, 5, chunks[1].Usage.CompletionTokens)
assert.Equal(t, 119, chunks[1].Usage.TotalTokens)
assert.Equal(t, 100, chunks[1].Usage.CacheReadTokens)
assert.Equal(t, 14, chunks[1].Usage.CacheMissTokens)
}
func TestNewRemoteChat_AnthropicProvider(t *testing.T) {
+19 -10
View File
@@ -25,15 +25,18 @@ func (l *langfuseChat) Chat(ctx context.Context, messages []Message, opts *ChatO
return l.inner.Chat(ctx, messages, opts)
}
purpose, prefixFingerprint := types.LLMCallMetadataFromContext(ctx)
genCtx, gen := mgr.StartGeneration(ctx, langfuse.GenerationOptions{
Name: "chat.completion",
Model: l.inner.GetModelName(),
Input: buildLangfuseMessages(messages),
ModelParameters: buildLangfuseModelParams(opts),
Metadata: map[string]interface{}{
"model_id": l.inner.GetModelID(),
"streaming": false,
"has_tools": opts != nil && len(opts.Tools) > 0,
"model_id": l.inner.GetModelID(),
"streaming": false,
"has_tools": opts != nil && len(opts.Tools) > 0,
"call_purpose": purpose,
"prompt_prefix_fingerprint": prefixFingerprint,
},
})
@@ -57,15 +60,18 @@ func (l *langfuseChat) ChatStream(ctx context.Context, messages []Message, opts
return l.inner.ChatStream(ctx, messages, opts)
}
purpose, prefixFingerprint := types.LLMCallMetadataFromContext(ctx)
genCtx, gen := mgr.StartGeneration(ctx, langfuse.GenerationOptions{
Name: "chat.completion.stream",
Model: l.inner.GetModelName(),
Input: buildLangfuseMessages(messages),
ModelParameters: buildLangfuseModelParams(opts),
Metadata: map[string]interface{}{
"model_id": l.inner.GetModelID(),
"streaming": true,
"has_tools": opts != nil && len(opts.Tools) > 0,
"model_id": l.inner.GetModelID(),
"streaming": true,
"has_tools": opts != nil && len(opts.Tools) > 0,
"call_purpose": purpose,
"prompt_prefix_fingerprint": prefixFingerprint,
},
})
@@ -211,10 +217,13 @@ func convertUsage(u *types.TokenUsage) *langfuse.TokenUsage {
return nil
}
return &langfuse.TokenUsage{
Input: u.PromptTokens,
Output: u.CompletionTokens,
Total: u.TotalTokens,
Unit: "TOKENS",
Input: u.PromptTokens,
Output: u.CompletionTokens,
Total: u.TotalTokens,
CacheRead: u.CacheReadTokens,
CacheWrite: u.CacheWriteTokens,
CacheMiss: u.CacheMissTokens,
Unit: "TOKENS",
}
}
@@ -43,3 +43,16 @@ func TestBuildLangfuseMessagesReasoningContent(t *testing.T) {
t.Fatalf("reasoning_content = %v; want chain of thought", msgs[0]["reasoning_content"])
}
}
func TestConvertUsageIncludesPromptCacheCounters(t *testing.T) {
got := convertUsage(&types.TokenUsage{
PromptTokens: 1000, CompletionTokens: 50, TotalTokens: 1050,
CacheReadTokens: 800, CacheWriteTokens: 100, CacheMissTokens: 200,
})
if got == nil {
t.Fatal("convertUsage returned nil")
}
if got.CacheRead != 800 || got.CacheWrite != 100 || got.CacheMiss != 200 {
t.Fatalf("cache usage = read:%d write:%d miss:%d", got.CacheRead, got.CacheWrite, got.CacheMiss)
}
}
+3 -2
View File
@@ -108,8 +108,9 @@ func buildResponseToolCalls(tcs []types.LLMToolCall) []logger.LLMToolCallInfo {
}
func usageString(u types.TokenUsage) string {
return fmt.Sprintf("Prompt: %d, Completion: %d, Total: %d",
u.PromptTokens, u.CompletionTokens, u.TotalTokens)
return fmt.Sprintf("Prompt: %d, Completion: %d, Total: %d, CacheRead: %d, CacheWrite: %d, CacheMiss: %d, CacheStatus: %s",
u.PromptTokens, u.CompletionTokens, u.TotalTokens, u.CacheReadTokens,
u.CacheWriteTokens, u.CacheMissTokens, u.CacheStatus)
}
// logLLMDebugCall logs a complete non-stream LLM chat call.
+2
View File
@@ -165,6 +165,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
CompletionTokens: completionTokens,
TotalTokens: promptTokens + completionTokens,
}
usage.MarkPromptCacheUnsupported()
logUsage(ctx, c.modelName, &usage)
return &types.ChatResponse{
@@ -260,6 +261,7 @@ func (c *OllamaChat) ChatStream(
CompletionTokens: resp.EvalCount,
TotalTokens: resp.PromptEvalCount + resp.EvalCount,
}
usage.MarkPromptCacheUnsupported()
}
logUsage(ctx, c.modelName, usage)
streamChan <- types.StreamResponse{
+7 -38
View File
@@ -26,15 +26,11 @@ func (c *RemoteAPIChat) parseCompletionResponse(resp *openai.ChatCompletionRespo
// 为设置了 Thinking=false 但模型仍返回思考内容的情况和部分不支持Thinking=false的思考模型(例如Miniax-M2.1)提供兜底策略
content := removeThinkingContent(choice.Message.Content)
usage := tokenUsageFromOpenAI(resp.Usage, c.provider)
response := &types.ChatResponse{
Content: content,
FinishReason: string(choice.FinishReason),
Usage: types.TokenUsage{
PromptTokens: resp.Usage.PromptTokens,
CompletionTokens: resp.Usage.CompletionTokens,
TotalTokens: resp.Usage.TotalTokens,
CachedTokens: cachedTokens(resp.Usage.PromptTokensDetails),
},
Usage: usage,
}
if len(choice.Message.ToolCalls) > 0 {
@@ -148,12 +144,8 @@ func (c *RemoteAPIChat) processStream(
}
if response.Usage != nil {
state.usage = &types.TokenUsage{
PromptTokens: response.Usage.PromptTokens,
CompletionTokens: response.Usage.CompletionTokens,
TotalTokens: response.Usage.TotalTokens,
CachedTokens: cachedTokens(response.Usage.PromptTokensDetails),
}
usage := tokenUsageFromOpenAI(*response.Usage, c.provider)
state.usage = &usage
}
if len(response.Choices) > 0 {
@@ -246,12 +238,9 @@ func (c *RemoteAPIChat) processRawHTTPStream(
}
if streamResp.Usage != nil {
state.usage = &types.TokenUsage{
PromptTokens: streamResp.Usage.PromptTokens,
CompletionTokens: streamResp.Usage.CompletionTokens,
TotalTokens: streamResp.Usage.TotalTokens,
CachedTokens: cachedTokens(streamResp.Usage.PromptTokensDetails),
}
usage := tokenUsageFromOpenAI(*streamResp.Usage, c.provider)
applyRawPromptCacheUsage(event.Data, &usage)
state.usage = &usage
}
if len(streamResp.Choices) > 0 {
@@ -600,23 +589,3 @@ func (c *RemoteAPIChat) processToolCallsDelta(
}
}
}
// cachedTokens returns the cached prompt-token count from an OpenAI-compatible
// usage detail block, or zero when the provider did not report one. Some
// providers omit PromptTokensDetails entirely, so the nil guard is required.
//
// Note on provider semantics:
// - Implicit-cache providers (OpenAI, Azure OpenAI, DeepSeek, …) populate
// `cached_tokens` automatically whenever the prompt prefix matches a
// previous request — no caller opt-in is required.
// - Explicit-cache providers (Qwen on Aliyun, Anthropic Claude, …) only
// populate `cached_tokens` after the caller attaches `cache_control:
// {"type": "ephemeral"}` to the relevant message / content block. This
// helper still returns zero for those providers until that opt-in is
// applied upstream of the request.
func cachedTokens(d *openai.PromptTokensDetails) int {
if d == nil {
return 0
}
return d.CachedTokens
}
+144
View File
@@ -0,0 +1,144 @@
package chat
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"github.com/Tencent/WeKnora/internal/models/provider"
"github.com/Tencent/WeKnora/internal/types"
"github.com/sashabaranov/go-openai"
)
// FingerprintPromptPrefix returns a short, non-reversible identifier suitable
// for logs and cache routing. Raw prompts must never be used as metric labels.
func FingerprintPromptPrefix(parts ...string) string {
h := sha256.New()
for _, part := range parts {
_, _ = h.Write([]byte(part))
_, _ = h.Write([]byte{0})
}
return hex.EncodeToString(h.Sum(nil))[:16]
}
// PromptPrefixFingerprint hashes the stable portion common to normal chat and
// agent requests: leading system messages plus the deterministic tool schema.
// Dynamic conversation/user messages intentionally do not participate.
func PromptPrefixFingerprint(messages []Message, opts *ChatOptions) string {
type stablePrefix struct {
System []Message `json:"system,omitempty"`
Tools []Tool `json:"tools,omitempty"`
}
prefix := stablePrefix{}
for _, message := range messages {
if message.Role != "system" {
break
}
prefix.System = append(prefix.System, message)
}
if opts != nil {
prefix.Tools = opts.Tools
}
data, _ := json.Marshal(prefix)
return FingerprintPromptPrefix(string(data))
}
// BuildPromptCacheKey derives an opaque process-local coordination key.
// Tenant and model identifiers are hashed rather than retained in memory.
func BuildPromptCacheKey(tenantID uint64, modelID, purpose, prefixFingerprint string) string {
return "wk-" + FingerprintPromptPrefix(
fmt.Sprintf("%d", tenantID), modelID, purpose, prefixFingerprint,
)
}
func providerCacheAccountingStatus(name provider.ProviderName) types.PromptCacheStatus {
switch name {
case provider.ProviderOpenAI,
provider.ProviderAzureOpenAI,
provider.ProviderDeepSeek,
provider.ProviderAliyun,
provider.ProviderAnthropic:
return types.PromptCacheStatusUnreported
default:
return types.PromptCacheStatusUnsupported
}
}
func tokenUsageFromOpenAI(usage openai.Usage, providerName provider.ProviderName) types.TokenUsage {
u := types.TokenUsage{
PromptTokens: usage.PromptTokens,
CompletionTokens: usage.CompletionTokens,
TotalTokens: usage.TotalTokens,
}
if usage.PromptTokensDetails != nil {
read := usage.PromptTokensDetails.CachedTokens
u.SetPromptCacheUsage(read, 0, max(0, usage.PromptTokens-read), true)
return u
}
if providerCacheAccountingStatus(providerName) == types.PromptCacheStatusUnsupported {
u.MarkPromptCacheUnsupported()
} else {
u.SetPromptCacheUsage(0, 0, 0, false)
}
return u
}
// cachedTokens is retained as the nil-safe primitive used by older callers
// and focused tests; normalization happens in tokenUsageFromOpenAI.
func cachedTokens(details *openai.PromptTokensDetails) int {
if details == nil {
return 0
}
return details.CachedTokens
}
type rawPromptCacheUsage struct {
Usage struct {
PromptTokens int `json:"prompt_tokens"`
PromptCacheHit *int `json:"prompt_cache_hit_tokens"`
PromptCacheMiss *int `json:"prompt_cache_miss_tokens"`
CacheReadInput *int `json:"cache_read_input_tokens"`
CacheCreationInput *int `json:"cache_creation_input_tokens"`
PromptTokensDetails *struct {
CachedTokens *int `json:"cached_tokens"`
CacheWriteTokens *int `json:"cache_write_tokens"`
} `json:"prompt_tokens_details"`
} `json:"usage"`
}
// applyRawPromptCacheUsage captures native fields discarded by the pinned
// OpenAI-compatible SDK (notably DeepSeek hit/miss counters).
func applyRawPromptCacheUsage(data []byte, usage *types.TokenUsage) {
if usage == nil || len(data) == 0 {
return
}
var raw rawPromptCacheUsage
if json.Unmarshal(data, &raw) != nil {
return
}
if raw.Usage.PromptCacheHit != nil || raw.Usage.PromptCacheMiss != nil {
read := valueOrZero(raw.Usage.PromptCacheHit)
miss := valueOrZero(raw.Usage.PromptCacheMiss)
usage.SetPromptCacheUsage(read, 0, miss, true)
return
}
if raw.Usage.CacheReadInput != nil || raw.Usage.CacheCreationInput != nil {
read := valueOrZero(raw.Usage.CacheReadInput)
write := valueOrZero(raw.Usage.CacheCreationInput)
usage.SetPromptCacheUsage(read, write, max(0, usage.PromptTokens-read), true)
return
}
if details := raw.Usage.PromptTokensDetails; details != nil {
read := valueOrZero(details.CachedTokens)
write := valueOrZero(details.CacheWriteTokens)
usage.SetPromptCacheUsage(read, write, max(0, usage.PromptTokens-read), true)
}
}
func valueOrZero(value *int) int {
if value == nil {
return 0
}
return *value
}
+4
View File
@@ -146,6 +146,10 @@ func (lkeapProvider) Thinking() ThinkingStrategy { return thinkingTypeField{} }
type deepseekProvider struct{ baseProvider }
func (deepseekProvider) Name() provider.ProviderName { return provider.ProviderDeepSeek }
// Native DeepSeek cache counters are not represented by go-openai v1.41.2;
// use the raw path so prompt_cache_hit_tokens/miss_tokens remain observable.
func (deepseekProvider) ForceRawHTTP() bool { return true }
func (deepseekProvider) ShapeRequest(req *openai.ChatCompletionRequest, opts *ChatOptions, _ bool) {
if opts != nil && opts.ToolChoice != "" {
req.ToolChoice = nil
+4 -3
View File
@@ -147,9 +147,10 @@ func TestBuildOutbound_ShapeRequest(t *testing.T) {
c := newOutboundChat(t, string(provider.ProviderDeepSeek), "deepseek-chat", nil)
body, _, useRaw, err := c.buildOutbound(msgs, &ChatOptions{ToolChoice: "auto"}, false)
require.NoError(t, err)
assert.False(t, useRaw)
req := body.(*openai.ChatCompletionRequest)
assert.Nil(t, req.ToolChoice)
assert.True(t, useRaw)
request, ok := body.(map[string]any)
require.True(t, ok)
assert.NotContains(t, request, "tool_choice")
})
t.Run("moonshot pins temperature to 1", func(t *testing.T) {
+4 -1
View File
@@ -67,6 +67,8 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) {
config = openai.DefaultConfig(apiKey)
if baseURL := chatConfig.BaseURL; baseURL != "" {
config.BaseURL = baseURL
} else if providerName == provider.ProviderDeepSeek {
config.BaseURL = provider.DeepSeekBaseURL
}
}
@@ -100,7 +102,7 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) {
modelName: modelName,
client: openai.NewClientWithConfig(config),
modelID: chatConfig.ModelID,
baseURL: chatConfig.BaseURL,
baseURL: strings.TrimRight(config.BaseURL, "/"),
apiKey: apiKey,
provider: providerName,
appID: chatConfig.AppID,
@@ -255,6 +257,7 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu
return nil, err
}
c.applyCompletionToolCallMetadata(body, result)
applyRawPromptCacheUsage(body, &result.Usage)
logUsage(ctx, c.modelName, &result.Usage)
return result, nil
}
+15
View File
@@ -488,6 +488,10 @@ func TestParseCompletionResponse_CachedTokens(t *testing.T) {
assert.Equal(t, 6971, got.Usage.TotalTokens)
assert.Equal(t, 6900, got.Usage.CachedTokens,
"cached_tokens must mirror prompt_tokens_details.cached_tokens")
assert.Equal(t, 6900, got.Usage.CacheReadTokens)
assert.Equal(t, 29, got.Usage.CacheMissTokens)
assert.True(t, got.Usage.CacheReported)
assert.Equal(t, types.PromptCacheStatusHit, got.Usage.CacheStatus)
})
t.Run("missing prompt_tokens_details yields zero cached_tokens", func(t *testing.T) {
@@ -512,9 +516,20 @@ func TestParseCompletionResponse_CachedTokens(t *testing.T) {
require.NotNil(t, got)
assert.Equal(t, 0, got.Usage.CachedTokens,
"missing details must surface as zero, not panic")
assert.False(t, got.Usage.CacheReported)
assert.Equal(t, types.PromptCacheStatusUnsupported, got.Usage.CacheStatus)
})
}
func TestApplyRawPromptCacheUsage_DeepSeekNativeFields(t *testing.T) {
usage := types.TokenUsage{PromptTokens: 4096, CompletionTokens: 10, TotalTokens: 4106}
applyRawPromptCacheUsage([]byte(`{"usage":{"prompt_tokens":4096,"prompt_cache_hit_tokens":3072,"prompt_cache_miss_tokens":1024}}`), &usage)
assert.Equal(t, 3072, usage.CacheReadTokens)
assert.Equal(t, 1024, usage.CacheMissTokens)
assert.True(t, usage.CacheReported)
assert.Equal(t, types.PromptCacheStatusHit, usage.CacheStatus)
}
// TestTokenUsage_CachedTokensJSONOmitempty ensures the new CachedTokens field
// stays out of serialized payloads when it is zero. This keeps logs and API
// responses unchanged for providers that never report cache hits.
+5 -2
View File
@@ -14,7 +14,10 @@ func logUsage(ctx context.Context, model string, u *types.TokenUsage) {
if u == nil {
return
}
purpose, prefixFingerprint := types.LLMCallMetadataFromContext(ctx)
logger.Infof(ctx,
"[LLM Usage] model=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d",
model, u.PromptTokens, u.CompletionTokens, u.TotalTokens, u.CachedTokens)
"[LLM Usage] model=%s, purpose=%s, prompt_prefix=%s, prompt_tokens=%d, completion_tokens=%d, total_tokens=%d, cached_tokens=%d, cache_read_tokens=%d, cache_write_tokens=%d, cache_miss_tokens=%d, cache_reported=%t, cache_status=%s",
model, purpose, prefixFingerprint, u.PromptTokens, u.CompletionTokens, u.TotalTokens,
u.CachedTokens, u.CacheReadTokens, u.CacheWriteTokens, u.CacheMissTokens,
u.CacheReported, u.CacheStatus)
}
+7 -4
View File
@@ -8,10 +8,13 @@ import (
// underlying model, in Langfuse's canonical schema. Set as the value of the
// langfuse.observation.usage_details span attribute (JSON-serialized).
type TokenUsage struct {
Input int `json:"input,omitempty"`
Output int `json:"output,omitempty"`
Total int `json:"total,omitempty"`
Unit string `json:"unit,omitempty"`
Input int `json:"input,omitempty"`
Output int `json:"output,omitempty"`
Total int `json:"total,omitempty"`
CacheRead int `json:"cache_read_input_tokens,omitempty"`
CacheWrite int `json:"cache_creation_input_tokens,omitempty"`
CacheMiss int `json:"cache_miss_input_tokens,omitempty"`
Unit string `json:"unit,omitempty"`
}
// Langfuse / OpenTelemetry semantic-convention attribute keys mirrored from
+61 -20
View File
@@ -5,31 +5,72 @@ import (
"encoding/json"
)
// PromptCacheStatus distinguishes a real cache miss from providers that do not
// expose prompt-cache accounting. Treating both as zero makes fleet-level hit
// rate dashboards misleading.
type PromptCacheStatus string
const (
PromptCacheStatusUnsupported PromptCacheStatus = "unsupported"
PromptCacheStatusUnreported PromptCacheStatus = "unreported"
PromptCacheStatusMiss PromptCacheStatus = "miss"
PromptCacheStatusHit PromptCacheStatus = "hit"
)
// TokenUsage holds token consumption statistics returned by the model API.
type TokenUsage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
// CachedTokens is the subset of PromptTokens that hit a provider-side
// prompt cache. Populated from `usage.prompt_tokens_details.cached_tokens`
// in OpenAI-compatible responses.
//
// Whether this field is non-zero depends on the provider's caching mode:
//
// - Implicit caching (OpenAI, Azure OpenAI, DeepSeek, …) — automatic.
// The field populates whenever the prompt prefix matches a previous
// request within the provider's cache TTL. No client-side opt-in.
//
// - Explicit caching (Qwen on Aliyun, Anthropic Claude, …) — opt-in
// required. The caller must attach `cache_control: {"type":
// "ephemeral"}` to the relevant message or content block to make
// the provider create and read the cache. Until that opt-in is
// applied, CachedTokens stays zero even when the prompt prefix is
// otherwise byte-stable.
//
// Omitted from JSON when zero so payloads stay quiet for providers
// that never populate it.
CachedTokens int `json:"cached_tokens,omitempty"`
// CachedTokens is the legacy alias for CacheReadTokens. It remains on the
// wire for compatibility with existing API consumers.
CachedTokens int `json:"cached_tokens,omitempty"`
CacheReadTokens int `json:"cache_read_tokens,omitempty"`
CacheWriteTokens int `json:"cache_write_tokens,omitempty"`
CacheMissTokens int `json:"cache_miss_tokens,omitempty"`
CacheReported bool `json:"cache_reported"`
CacheStatus PromptCacheStatus `json:"cache_status,omitempty"`
}
// SetPromptCacheUsage normalizes provider-specific cache counters into the
// shared usage model. promptTokens remains the provider's total input-token
// count; read/write/miss are descriptive subsets and must not be added to it.
func (u *TokenUsage) SetPromptCacheUsage(read, write, miss int, reported bool) {
if u == nil {
return
}
if read < 0 {
read = 0
}
if write < 0 {
write = 0
}
if miss < 0 {
miss = 0
}
u.CachedTokens = read
u.CacheReadTokens = read
u.CacheWriteTokens = write
u.CacheMissTokens = miss
u.CacheReported = reported
switch {
case !reported:
u.CacheStatus = PromptCacheStatusUnreported
case read > 0:
u.CacheStatus = PromptCacheStatusHit
default:
u.CacheStatus = PromptCacheStatusMiss
}
}
// MarkPromptCacheUnsupported marks a provider/model path that cannot report
// provider-side prompt-cache usage.
func (u *TokenUsage) MarkPromptCacheUnsupported() {
if u == nil {
return
}
u.SetPromptCacheUsage(0, 0, 0, false)
u.CacheStatus = PromptCacheStatusUnsupported
}
// LLMToolCall represents a function/tool call from the LLM
+6
View File
@@ -45,6 +45,12 @@ const (
// concurrency governor uses this to throttle only background LLM traffic,
// leaving interactive chat latency untouched. See WithBackgroundTask.
BackgroundTaskContextKey ContextKey = "BackgroundTask"
// LLMCallPurposeContextKey labels the product-level reason for a model
// request (for example agent_round or wiki_page_modify).
LLMCallPurposeContextKey ContextKey = "LLMCallPurpose"
// LLMPromptPrefixFingerprintContextKey carries a non-sensitive hash of the
// intended reusable prompt prefix for cache diagnostics.
LLMPromptPrefixFingerprintContextKey ContextKey = "LLMPromptPrefixFingerprint"
// MCPOAuthNonInteractiveContextKey marks a request whose channel cannot
// resolve an in-conversation MCP OAuth prompt (e.g. an IM bot: there is no
// live client to click "Authorize" and call the resolve endpoint). When set,
+23
View File
@@ -183,6 +183,29 @@ func IsBackgroundTask(ctx context.Context) bool {
return v
}
// WithLLMCallMetadata annotates a provider call for cache observability. The
// fingerprint must be a hash, never raw prompt content.
func WithLLMCallMetadata(ctx context.Context, purpose, prefixFingerprint string) context.Context {
if strings.TrimSpace(purpose) != "" {
ctx = context.WithValue(ctx, LLMCallPurposeContextKey, strings.TrimSpace(purpose))
}
if strings.TrimSpace(prefixFingerprint) != "" {
ctx = context.WithValue(ctx, LLMPromptPrefixFingerprintContextKey, strings.TrimSpace(prefixFingerprint))
}
return ctx
}
// LLMCallMetadataFromContext returns the cache-observability labels attached
// by the orchestration layer.
func LLMCallMetadataFromContext(ctx context.Context) (purpose, prefixFingerprint string) {
if ctx == nil {
return "", ""
}
purpose, _ = ctx.Value(LLMCallPurposeContextKey).(string)
prefixFingerprint, _ = ctx.Value(LLMPromptPrefixFingerprintContextKey).(string)
return purpose, prefixFingerprint
}
// LanguageFromContext extracts the language locale string from ctx (e.g. "zh-CN", "en-US").
// Returns ("zh-CN", false) when the key is absent.
func LanguageFromContext(ctx context.Context) (string, bool) {
+8
View File
@@ -145,6 +145,14 @@ func TestLanguageFromContext(t *testing.T) {
}
}
func TestLLMCallMetadataContext(t *testing.T) {
ctx := WithLLMCallMetadata(context.Background(), "wiki_page_modify", "abc123")
purpose, prefix := LLMCallMetadataFromContext(ctx)
if purpose != "wiki_page_modify" || prefix != "abc123" {
t.Fatalf("metadata = (%q, %q)", purpose, prefix)
}
}
// BenchmarkLanguageLocaleName benchmarks the language name lookup
func BenchmarkLanguageLocaleName(b *testing.B) {
testCases := []string{"zh", "en", "zh-CN", "ko", "unknown"}