diff --git a/go.mod b/go.mod index a9a9f988..69425a67 100644 --- a/go.mod +++ b/go.mod @@ -114,6 +114,7 @@ require ( github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/duckdb/duckdb-go-bindings v0.1.24 // indirect github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.24 // indirect @@ -240,6 +241,7 @@ require ( github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect + github.com/tiktoken-go/tokenizer v0.7.0 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect diff --git a/go.sum b/go.sum index 810ada80..d54d80ef 100644 --- a/go.sum +++ b/go.sum @@ -1562,6 +1562,8 @@ github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= @@ -2434,6 +2436,8 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tiktoken-go/tokenizer v0.7.0 h1:VMu6MPT0bXFDHr7UPh9uii7CNItVt3X9K90omxL54vw= +github.com/tiktoken-go/tokenizer v0.7.0/go.mod h1:6UCYI/DtOallbmL7sSy30p6YQv60qNyU/4aVigPOx6w= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= diff --git a/internal/agent/engine.go b/internal/agent/engine.go index f41da21f..549b2211 100644 --- a/internal/agent/engine.go +++ b/internal/agent/engine.go @@ -36,6 +36,8 @@ type AgentEngine struct { imageDescriber ImageDescriberFunc // VLM function for describing images in tool results (optional) tokenEstimator *agenttoken.Estimator // Token estimator for context window management memoryConsolidator *agentmemory.Consolidator // Memory consolidator for LLM-powered summarization (optional) + lastUsage types.TokenUsage // Token usage from the most recent LLM call + lastSentMsgCount int // Number of messages sent in the most recent LLM call } // ImageDescriberFunc generates a text description of an image. @@ -57,7 +59,10 @@ func NewAgentEngine( if eventBus == nil { eventBus = event.NewEventBus() } - tokenEst := agenttoken.NewEstimator(0) + tokenEst, err := agenttoken.NewEstimator() + if err != nil { + return nil + } engine := &AgentEngine{ config: config, toolRegistry: toolRegistry, @@ -133,6 +138,18 @@ func (e *AgentEngine) GetSkillsManager() *skills.Manager { return e.skillsManager } +// estimateCurrentTokens returns the best estimate of the current context token count. +// When API-reported usage from a previous round is available, it uses that as a +// baseline and only BPE-estimates the delta (newly appended messages). Otherwise it +// falls back to a full BPE estimation of all messages. +func (e *AgentEngine) estimateCurrentTokens(messages []chat.Message) int { + if e.lastUsage.TotalTokens > 0 && e.lastSentMsgCount > 0 && e.lastSentMsgCount < len(messages) { + delta := e.tokenEstimator.EstimateMessages(messages[e.lastSentMsgCount:]) + return e.lastUsage.TotalTokens + delta + } + return e.tokenEstimator.EstimateMessages(messages) +} + // Execute executes the agent with conversation history and streaming output // All events are emitted to EventBus and handled by subscribers (like Handler layer) func (e *AgentEngine) Execute( @@ -272,11 +289,14 @@ func (e *AgentEngine) executeLoop( roundStart := time.Now() - // Context window management - messages = e.manageContextWindow(ctx, messages, state.CurrentRound+1) + // Context window management: estimate current token count using + // the API-reported usage from the previous round plus a BPE delta + // for newly appended messages (assistant reply + tool results). + currentTokens := e.estimateCurrentTokens(messages) + messages = e.manageContextWindow(ctx, messages, state.CurrentRound+1, currentTokens) - logger.Infof(ctx, "[Agent][Round-%d/%d] Starting: %d messages, %d tools", - state.CurrentRound+1, e.config.MaxIterations, len(messages), len(tools)) + logger.Infof(ctx, "[Agent][Round-%d/%d] Starting: %d messages, %d tools, est_tokens=%d", + state.CurrentRound+1, e.config.MaxIterations, len(messages), len(tools), currentTokens) common.PipelineInfo(ctx, "Agent", "round_start", map[string]interface{}{ "iteration": state.CurrentRound, "round": state.CurrentRound + 1, @@ -286,14 +306,20 @@ func (e *AgentEngine) executeLoop( }) // 1. Think: Call LLM with function calling (includes retry + graceful degradation) + e.lastSentMsgCount = len(messages) response, err := e.callLLMWithRetry(ctx, messages, tools, state, query, state.CurrentRound, sessionID) if err != nil { return state, err } if response == nil { - // Graceful degradation succeeded — state.IsComplete already set break } + if response.Usage.TotalTokens > 0 { + e.lastUsage = response.Usage + logger.Debugf(ctx, "[Agent][Round-%d] Usage: prompt=%d, completion=%d, total=%d", + state.CurrentRound+1, response.Usage.PromptTokens, + response.Usage.CompletionTokens, response.Usage.TotalTokens) + } // Create agent step step := types.AgentStep{ diff --git a/internal/agent/finalize.go b/internal/agent/finalize.go index 754ef0ae..28ef967b 100644 --- a/internal/agent/finalize.go +++ b/internal/agent/finalize.go @@ -87,7 +87,7 @@ Now generate the final answer:`, query) answerID := generateEventID("answer") logger.Debugf(ctx, "[Agent][FinalAnswer] AnswerID: %s", answerID) - fullAnswer, _, err := e.streamLLMToEventBus( + llmResult, err := e.streamLLMToEventBus( ctx, messages, &chat.ChatOptions{Temperature: e.config.Temperature, Thinking: e.config.Thinking}, @@ -95,7 +95,7 @@ Now generate the final answer:`, query) if chunk.Content != "" { logger.Debugf(ctx, "[Agent][FinalAnswer] Emitting answer chunk: %d chars", len(chunk.Content)) e.eventBus.Emit(ctx, event.Event{ - ID: answerID, // Same ID for all chunks in this stream + ID: answerID, Type: event.EventAgentFinalAnswer, SessionID: sessionID, Data: event.AgentFinalAnswerData{ @@ -115,6 +115,7 @@ Now generate the final answer:`, query) return err } + fullAnswer := llmResult.Content logger.Infof(ctx, "[Agent][FinalAnswer] Final answer generated: %d characters", len(fullAnswer)) common.PipelineInfo(ctx, "Agent", "final_answer_done", map[string]interface{}{ "session_id": sessionID, diff --git a/internal/agent/memory/consolidator.go b/internal/agent/memory/consolidator.go index 470ad0f3..e2c68275 100644 --- a/internal/agent/memory/consolidator.go +++ b/internal/agent/memory/consolidator.go @@ -58,14 +58,14 @@ func NewConsolidator( } } -// ShouldConsolidate checks if the messages need consolidation based on token count. -func (c *Consolidator) ShouldConsolidate(messages []chat.Message) bool { - if c.maxTokens <= 0 || len(messages) <= 3 { +// ShouldConsolidate checks if consolidation is needed based on the caller-provided +// token estimate. The estimate should come from the model API's Usage when available. +func (c *Consolidator) ShouldConsolidate(currentTokens int) bool { + if c.maxTokens <= 0 { return false } - tokenCount := c.estimator.EstimateMessages(messages) triggerAt := int(float64(c.maxTokens) * c.threshold) - return tokenCount > triggerAt + return currentTokens > triggerAt } // Consolidate summarizes older messages and returns a compressed message array. diff --git a/internal/agent/memory/consolidator_test.go b/internal/agent/memory/consolidator_test.go index ac55ca4f..c9bad31b 100644 --- a/internal/agent/memory/consolidator_test.go +++ b/internal/agent/memory/consolidator_test.go @@ -9,7 +9,8 @@ import ( ) func TestConsolidator_ShouldConsolidate(t *testing.T) { - est := token.NewEstimator(0) + est, err := token.NewEstimator() + assert.NoError(t, err) t.Run("below threshold returns false", func(t *testing.T) { c := NewConsolidator(nil, est, 100000, 0) @@ -19,27 +20,18 @@ func TestConsolidator_ShouldConsolidate(t *testing.T) { {Role: "assistant", Content: "hi"}, {Role: "user", Content: "current"}, } - assert.False(t, c.ShouldConsolidate(messages)) + tokens := est.EstimateMessages(messages) + assert.False(t, c.ShouldConsolidate(tokens)) }) - t.Run("too few messages returns false", func(t *testing.T) { - c := NewConsolidator(nil, est, 10, 0) - messages := []chat.Message{ - {Role: "system", Content: "system"}, - {Role: "user", Content: "hello"}, - } - assert.False(t, c.ShouldConsolidate(messages)) + t.Run("over threshold returns true", func(t *testing.T) { + c := NewConsolidator(nil, est, 10, 0) // very low threshold + assert.True(t, c.ShouldConsolidate(100)) }) t.Run("disabled returns false", func(t *testing.T) { - c := NewConsolidator(nil, est, 0, 0) // disabled - messages := []chat.Message{ - {Role: "system", Content: "system"}, - {Role: "user", Content: "hello"}, - {Role: "assistant", Content: "hi"}, - {Role: "user", Content: "current"}, - } - assert.False(t, c.ShouldConsolidate(messages)) + c := NewConsolidator(nil, est, 0, 0) + assert.False(t, c.ShouldConsolidate(99999)) }) } @@ -61,7 +53,8 @@ func TestTruncateForPrompt(t *testing.T) { } func TestRawArchive(t *testing.T) { - est := token.NewEstimator(0) + est, err := token.NewEstimator() + assert.NoError(t, err) c := NewConsolidator(nil, est, 100000, 0) messages := []chat.Message{ @@ -83,7 +76,8 @@ func TestRawArchive(t *testing.T) { } func TestBuildConsolidationPrompt(t *testing.T) { - est := token.NewEstimator(0) + est, err := token.NewEstimator() + assert.NoError(t, err) c := NewConsolidator(nil, est, 100000, 0) messages := []chat.Message{ diff --git a/internal/agent/observe.go b/internal/agent/observe.go index 80dc783a..c355f7f4 100644 --- a/internal/agent/observe.go +++ b/internal/agent/observe.go @@ -16,29 +16,29 @@ import ( ) // manageContextWindow consolidates or compresses messages if approaching the token limit. -// It returns the (possibly shortened) message slice. -func (e *AgentEngine) manageContextWindow(ctx context.Context, messages []chat.Message, round int) []chat.Message { +// currentTokens is the caller's best estimate of the current context size (using +// API-reported Usage when available, falling back to BPE estimation). +func (e *AgentEngine) manageContextWindow(ctx context.Context, messages []chat.Message, round, currentTokens int) []chat.Message { if e.config.MaxContextTokens <= 0 { return messages } beforeLen := len(messages) - // Try LLM-powered consolidation first (preserves more semantic content) - if e.memoryConsolidator != nil && e.memoryConsolidator.ShouldConsolidate(messages) { - logger.Infof(ctx, "[Agent][Round-%d] Token threshold exceeded, consolidating memory", - round) + if e.memoryConsolidator != nil && e.memoryConsolidator.ShouldConsolidate(currentTokens) { + logger.Infof(ctx, "[Agent][Round-%d] Token threshold exceeded (est=%d), consolidating memory", + round, currentTokens) consolidated, consolidateErr := e.memoryConsolidator.Consolidate(ctx, messages) if consolidateErr != nil { logger.Warnf(ctx, "[Agent][Round-%d] Memory consolidation failed: %v, "+ "falling back to simple compression", round, consolidateErr) } else { messages = consolidated + currentTokens = e.tokenEstimator.EstimateMessages(messages) } } - // Fallback: simple context compression if still over the hard limit (80%) - messages = agenttoken.CompressContext(messages, e.tokenEstimator, e.config.MaxContextTokens) + messages = agenttoken.CompressContext(messages, e.tokenEstimator, e.config.MaxContextTokens, currentTokens) if len(messages) < beforeLen { logger.Infof(ctx, "[Agent][Round-%d] Context managed: %d → %d messages (max_tokens=%d)", diff --git a/internal/agent/think.go b/internal/agent/think.go index 2e96d69b..87562e70 100644 --- a/internal/agent/think.go +++ b/internal/agent/think.go @@ -13,58 +13,59 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) +// streamLLMResult holds accumulated output from a streaming LLM call. +type streamLLMResult struct { + Content string + ToolCalls []types.LLMToolCall + Usage *types.TokenUsage +} + // streamLLMToEventBus streams LLM response through EventBus (generic method) // emitFunc: callback to emit each chunk event -// Returns: full accumulated content, tool calls (if any), error func (e *AgentEngine) streamLLMToEventBus( ctx context.Context, messages []chat.Message, opts *chat.ChatOptions, emitFunc func(chunk *types.StreamResponse, fullContent string), -) (string, []types.LLMToolCall, error) { +) (*streamLLMResult, error) { logger.Debugf(ctx, "[Agent][Stream] Starting LLM stream with %d messages", len(messages)) - // Create a per-call timeout context so that a single LLM call gets a - // guaranteed time window even when the parent context's deadline is almost - // exhausted after previous iterations. The shorter of the two deadlines wins, - // so the parent pipeline timeout is still respected. llmCtx, llmCancel := context.WithTimeout(ctx, e.getLLMCallTimeout()) defer llmCancel() stream, err := e.chatModel.ChatStream(llmCtx, messages, opts) if err != nil { logger.Errorf(ctx, "[Agent][Stream] Failed to start LLM stream: %v", err) - return "", nil, err + return nil, err } - fullContent := "" - var toolCalls []types.LLMToolCall + result := &streamLLMResult{} chunkCount := 0 for chunk := range stream { chunkCount++ if chunk.Content != "" { - // Only accumulate LLM's own text output, not content extracted from tool arguments - // (e.g. final_answer's answer or thinking tool's thought are streamed separately) isExtracted := chunk.Data != nil && chunk.Data["source"] != nil if !isExtracted { - fullContent += chunk.Content + result.Content += chunk.Content } } - // Collect tool calls if present if len(chunk.ToolCalls) > 0 { - toolCalls = chunk.ToolCalls + result.ToolCalls = chunk.ToolCalls + } + + if chunk.Usage != nil { + result.Usage = chunk.Usage } - // Emit event through callback if emitFunc != nil { - emitFunc(&chunk, fullContent) + emitFunc(&chunk, result.Content) } } - return fullContent, toolCalls, nil + return result, nil } // streamReflectionToEventBus streams reflection process through EventBus @@ -93,14 +94,14 @@ Think: // Generate a single ID for this entire reflection stream reflectionID := generateEventID("reflection") - fullReflection, _, err := e.streamLLMToEventBus( + llmResult, err := e.streamLLMToEventBus( ctx, messages, &chat.ChatOptions{Temperature: 0.5}, func(chunk *types.StreamResponse, fullContent string) { if chunk.Content != "" { e.eventBus.Emit(ctx, event.Event{ - ID: reflectionID, // Same ID for all chunks in this stream + ID: reflectionID, Type: event.EventAgentReflection, SessionID: sessionID, Data: event.AgentReflectionData{ @@ -118,7 +119,7 @@ Think: return "", err } - return fullReflection, nil + return llmResult.Content, nil } // streamThinkingToEventBus streams the thinking process through EventBus @@ -145,7 +146,7 @@ func (e *AgentEngine) streamThinkingToEventBus( thinkingID := generateEventID("thinking") answerID := generateEventID("answer") - fullContent, toolCalls, err := e.streamLLMToEventBus( + llmResult, err := e.streamLLMToEventBus( ctx, messages, opts, @@ -230,17 +231,19 @@ func (e *AgentEngine) streamThinkingToEventBus( } logger.Debugf(ctx, "[Agent][Thinking] Iteration-%d completed: content=%d chars, tool_calls=%d", - iteration+1, len(fullContent), len(toolCalls)) + iteration+1, len(llmResult.Content), len(llmResult.ToolCalls)) - // Strip … blocks that some models (DeepSeek, Qwen, etc.) embed in content. - fullContent = agenttools.StripThinkBlocks(fullContent) + fullContent := agenttools.StripThinkBlocks(llmResult.Content) - // Build response - return &types.ChatResponse{ + resp := &types.ChatResponse{ Content: fullContent, - ToolCalls: toolCalls, + ToolCalls: llmResult.ToolCalls, FinishReason: "stop", - }, nil + } + if llmResult.Usage != nil { + resp.Usage = *llmResult.Usage + } + return resp, nil } // callLLMWithRetry logs messages, sanitizes them, calls the LLM with retry on transient errors, diff --git a/internal/agent/token/compress.go b/internal/agent/token/compress.go index 88b116b4..692da5f3 100644 --- a/internal/agent/token/compress.go +++ b/internal/agent/token/compress.go @@ -9,39 +9,35 @@ import ( const DefaultContextThresholdRatio = 0.8 // CompressContext trims older messages to bring total token count below the threshold. +// currentTokens is the caller's best estimate of the current context size (from API +// Usage when available, falling back to BPE estimation). The estimator is still used +// internally for per-message-group cost calculation during trimming. +// // It preserves: // - The first message (system prompt) // - The last message (current user query) // - tool_call / tool_result message pairs (never splits them) -// -// Returns the compressed message slice. If no compression is needed, returns the original. func CompressContext( messages []chat.Message, estimator *Estimator, maxTokens int, + currentTokens int, ) []chat.Message { if maxTokens <= 0 || len(messages) <= 2 { return messages } threshold := int(float64(maxTokens) * DefaultContextThresholdRatio) - currentTokens := estimator.EstimateMessages(messages) if currentTokens <= threshold { return messages } - // Strategy: keep system prompt (first) + current query (last), - // trim from the oldest messages inward, respecting tool_call/tool_result boundaries. - systemMsg := messages[0] lastMsg := messages[len(messages)-1] middle := messages[1 : len(messages)-1] - // Find groups of messages that form tool_call/tool_result pairs. - // A group = one assistant message with tool_calls + all following tool result messages. groups := groupToolMessages(middle) - // Remove groups from the front until we're under the threshold. tokensToFree := currentTokens - threshold freed := 0 removeUpTo := 0 @@ -58,7 +54,6 @@ func CompressContext( } } - // Rebuild messages: system + remaining groups + current query remaining := make([]chat.Message, 0, len(messages)) remaining = append(remaining, systemMsg) for i := removeUpTo; i < len(groups); i++ { diff --git a/internal/agent/token/estimator.go b/internal/agent/token/estimator.go index 3ebcf5e7..076bab5b 100644 --- a/internal/agent/token/estimator.go +++ b/internal/agent/token/estimator.go @@ -1,116 +1,87 @@ // Package token provides token estimation for LLM context window management. +// +// The primary source of truth for token counts is the model API's Usage field +// (types.TokenUsage), returned with every LLM response. This package serves as +// a supplementary estimator used in two scenarios: +// +// 1. Delta estimation — after an LLM call, new messages (assistant reply + +// tool results) are appended before the next call. The Estimator computes +// the token cost of these new messages so the engine can decide whether +// context compression is needed without making an extra API round-trip. +// +// 2. First-round fallback — on the very first round of a session, no prior +// API Usage is available, so the Estimator provides a full estimate. +// +// The encoding used (cl100k_base) is an approximation. Different model families +// use different tokenizers, so the numbers will not be exact for non-OpenAI +// models. This is acceptable because the estimate only needs to be close enough +// to trigger compression at roughly the right time; over- or under-estimating +// by a small margin is corrected on the next API call. package token import ( - "unicode" + "fmt" "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/tiktoken-go/tokenizer" ) const ( - // DefaultCharsPerToken is a blended ratio for mixed English/CJK text. - // English ≈ 4 chars/token, CJK ≈ 1.5 chars/token, blended ≈ 3.5. - DefaultCharsPerToken = 3.5 - - // perMessageOverhead is the estimated token overhead per message - // (role label, formatting, separator tokens). - perMessageOverhead = 4 + perMessageOverhead = 3 + perConversationTail = 3 ) -// Estimator estimates token counts for messages and strings. -// Uses a character-based heuristic that accounts for CJK characters. +// Estimator counts tokens for messages and strings using BPE tokenization. +// It is intended for incremental (delta) estimation between LLM calls; the +// authoritative token count comes from the API's Usage response. type Estimator struct { - charsPerToken float64 + codec tokenizer.Codec } -// NewEstimator creates a token estimator. -// If charsPerToken <= 0, DefaultCharsPerToken is used. -func NewEstimator(charsPerToken float64) *Estimator { - if charsPerToken <= 0 { - charsPerToken = DefaultCharsPerToken +// NewEstimator creates a token estimator using the cl100k_base encoding. +func NewEstimator() (*Estimator, error) { + codec, err := tokenizer.Get(tokenizer.Cl100kBase) + if err != nil { + return nil, fmt.Errorf("token: failed to initialize tokenizer: %w", err) } - return &Estimator{charsPerToken: charsPerToken} + return &Estimator{codec: codec}, nil } -// EstimateString estimates the token count for a string. -// It uses a CJK-aware heuristic: CJK characters are counted at ~1.5 chars/token, -// while ASCII/Latin characters use the configured charsPerToken ratio. -func (e *Estimator) EstimateString(s string) int { - if len(s) == 0 { - return 0 - } - - var cjkRunes, otherRunes int - for _, r := range s { - if isCJK(r) { - cjkRunes++ - } else { - otherRunes++ - } - } - - // CJK characters are roughly 1.5 chars per token - cjkTokens := float64(cjkRunes) / 1.5 - otherTokens := float64(otherRunes) / e.charsPerToken - - return int(cjkTokens+otherTokens) + 1 // +1 to avoid underestimation -} - -// EstimateMessages estimates the total token count for a slice of chat messages. +// EstimateMessages returns the estimated token count for a slice of messages. +// Prefer using API Usage for the full context and this method only for deltas. func (e *Estimator) EstimateMessages(messages []chat.Message) int { total := 0 for i := range messages { total += e.EstimateMessage(&messages[i]) } + total += perConversationTail return total } -// EstimateMessage estimates the token count for a single message. +// EstimateString returns the token count for a string using BPE tokenization. +func (e *Estimator) EstimateString(s string) int { + if len(s) == 0 { + return 0 + } + ids, _, err := e.codec.Encode(s) + if err != nil { + return (len(s) + 3) / 4 + } + return len(ids) +} + +// EstimateMessage returns the token count for a single message. func (e *Estimator) EstimateMessage(msg *chat.Message) int { tokens := perMessageOverhead tokens += e.EstimateString(msg.Role) tokens += e.EstimateString(msg.Content) tokens += e.EstimateString(msg.Name) - // Tool calls in assistant messages for _, tc := range msg.ToolCalls { tokens += e.EstimateString(tc.Function.Name) tokens += e.EstimateString(tc.Function.Arguments) - tokens += 4 // overhead for tool call structure + tokens += 4 } return tokens } - -// isCJK checks if a rune is a CJK (Chinese, Japanese, Korean) character. -func isCJK(r rune) bool { - // CJK Unified Ideographs - if r >= 0x4E00 && r <= 0x9FFF { - return true - } - // CJK Unified Ideographs Extension A - if r >= 0x3400 && r <= 0x4DBF { - return true - } - // CJK Unified Ideographs Extension B - if r >= 0x20000 && r <= 0x2A6DF { - return true - } - // CJK Compatibility Ideographs - if r >= 0xF900 && r <= 0xFAFF { - return true - } - // Hiragana + Katakana - if r >= 0x3040 && r <= 0x30FF { - return true - } - // Hangul Syllables - if r >= 0xAC00 && r <= 0xD7AF { - return true - } - // Full-width characters - if unicode.Is(unicode.Han, r) { - return true - } - return false -} diff --git a/internal/agent/token/estimator_test.go b/internal/agent/token/estimator_test.go index f1083f74..2c3a269e 100644 --- a/internal/agent/token/estimator_test.go +++ b/internal/agent/token/estimator_test.go @@ -12,34 +12,34 @@ import ( ) func TestEstimator(t *testing.T) { - e := NewEstimator(0) // use default + e, err := NewEstimator() + if err != nil { + t.Fatalf("Failed to create estimator: %v", err) + } t.Run("empty string", func(t *testing.T) { assert.Equal(t, 0, e.EstimateString("")) }) t.Run("english text", func(t *testing.T) { - // "hello world" = 11 chars ≈ 11/3.5 + 1 ≈ 4 tokens tokens := e.EstimateString("hello world") - assert.Greater(t, tokens, 0) - assert.Less(t, tokens, 20) + assert.Equal(t, 2, tokens) }) t.Run("chinese text", func(t *testing.T) { - // 10 Chinese chars at ~1.5 chars/token ≈ 7 tokens tokens := e.EstimateString("你好世界测试数据中文") - assert.Greater(t, tokens, 3) - assert.Less(t, tokens, 20) + fmt.Println(tokens) + assert.Greater(t, tokens, 0) }) - t.Run("CJK is more tokens per char than latin", func(t *testing.T) { - // Same number of characters, but CJK should estimate more tokens + t.Run("CJK produces more tokens per char than latin", func(t *testing.T) { latin := strings.Repeat("a", 100) cjk := strings.Repeat("中", 100) latinTokens := e.EstimateString(latin) cjkTokens := e.EstimateString(cjk) + fmt.Println(latinTokens, cjkTokens) assert.Greater(t, cjkTokens, latinTokens, - "CJK text should estimate more tokens per character") + "CJK text should produce more tokens per character") }) t.Run("message estimation includes overhead", func(t *testing.T) { @@ -49,6 +49,7 @@ func TestEstimator(t *testing.T) { } tokens := e.EstimateMessage(&msg) contentTokens := e.EstimateString("hello") + fmt.Println(tokens, contentTokens) assert.Greater(t, tokens, contentTokens, "message tokens should include overhead beyond just content") }) @@ -82,26 +83,28 @@ func TestEstimator(t *testing.T) { } func TestCompressContext(t *testing.T) { - e := NewEstimator(0) + e, err := NewEstimator() + require.NoError(t, err) t.Run("no compression needed", func(t *testing.T) { messages := []chat.Message{ {Role: "system", Content: "system prompt"}, {Role: "user", Content: "hello"}, } - result := CompressContext(messages, e, 100000) + tokens := e.EstimateMessages(messages) + result := CompressContext(messages, e, 100000, tokens) assert.Equal(t, messages, result) }) t.Run("preserves system and last message", func(t *testing.T) { - // Create messages that exceed the token limit messages := []chat.Message{ {Role: "system", Content: "system prompt"}, {Role: "user", Content: strings.Repeat("old message ", 1000)}, {Role: "assistant", Content: strings.Repeat("old response ", 1000)}, {Role: "user", Content: "current query"}, } - result := CompressContext(messages, e, 100) // very low limit to force compression + tokens := e.EstimateMessages(messages) + result := CompressContext(messages, e, 100, tokens) require.GreaterOrEqual(t, len(result), 2) assert.Equal(t, "system", result[0].Role) assert.Equal(t, "current query", result[len(result)-1].Content) @@ -110,27 +113,23 @@ func TestCompressContext(t *testing.T) { t.Run("keeps tool_call and tool_result paired", func(t *testing.T) { messages := []chat.Message{ {Role: "system", Content: "system prompt"}, - // Old conversation {Role: "user", Content: strings.Repeat("x", 500)}, {Role: "assistant", Content: strings.Repeat("y", 500)}, - // Tool interaction that should stay together {Role: "assistant", Content: "thinking", ToolCalls: []chat.ToolCall{ {ID: "call_1", Function: chat.FunctionCall{Name: "search", Arguments: `{"q":"test"}`}}, }}, {Role: "tool", Content: strings.Repeat("result ", 100), ToolCallID: "call_1", Name: "search"}, - // Current query {Role: "user", Content: "what did you find?"}, } - result := CompressContext(messages, e, 200) // force some compression + tokens := e.EstimateMessages(messages) + result := CompressContext(messages, e, 200, tokens) b, _ := json.Marshal(result) fmt.Println(string(b)) - // Check that tool_call and tool_result are still paired for i, msg := range result { if msg.Role == "tool" { - // The previous message should be an assistant with tool_calls require.Greater(t, i, 0) assert.Equal(t, "assistant", result[i-1].Role) assert.Greater(t, len(result[i-1].ToolCalls), 0) @@ -143,7 +142,7 @@ func TestCompressContext(t *testing.T) { {Role: "system", Content: "system"}, {Role: "user", Content: "hello"}, } - result := CompressContext(messages, e, 0) + result := CompressContext(messages, e, 0, 0) assert.Equal(t, messages, result) }) @@ -152,7 +151,7 @@ func TestCompressContext(t *testing.T) { {Role: "system", Content: "system"}, {Role: "user", Content: "hello"}, } - result := CompressContext(messages, e, 1) // even with tiny limit + result := CompressContext(messages, e, 1, 999) assert.Equal(t, messages, result) }) } @@ -179,8 +178,8 @@ func TestGroupToolMessages(t *testing.T) { {Role: "user", Content: "thanks"}, } groups := groupToolMessages(messages) - assert.Len(t, groups, 3) // user, (assistant+2 tools), user - assert.Len(t, groups[1], 3) // assistant + 2 tool results + assert.Len(t, groups, 3) + assert.Len(t, groups[1], 3) b, _ := json.Marshal(groups) fmt.Println(string(b)) }) diff --git a/internal/im/service.go b/internal/im/service.go index a1601c92..ac2c7b2e 100644 --- a/internal/im/service.go +++ b/internal/im/service.go @@ -57,13 +57,13 @@ const ( // All IM-related Redis keys are defined here for discoverability and to avoid // scattered string literals across multiple files. const ( - RedisKeyLeader = "im:ws:leader:" // + channelID — WebSocket leader election - RedisKeyDedup = "im:dedup:" // + messageID — message deduplication - RedisKeyStop = "im:stop:" // + userKey — cross-instance /stop marker (pre-execution) - RedisKeyInflight = "im:inflight:" // + userKey — maps userKey → sessionID:messageID for cross-instance /stop - RedisKeyQueueUser = "im:queue:user:" // + userKey — global per-user queue counter - RedisKeyRateLimit = "im:ratelimit:" // + key — sliding-window rate limiting - RedisKeyGlobalGate = "im:global:active" // global concurrent worker counter + RedisKeyLeader = "im:ws:leader:" // + channelID — WebSocket leader election + RedisKeyDedup = "im:dedup:" // + messageID — message deduplication + RedisKeyStop = "im:stop:" // + userKey — cross-instance /stop marker (pre-execution) + RedisKeyInflight = "im:inflight:" // + userKey — maps userKey → sessionID:messageID for cross-instance /stop + RedisKeyQueueUser = "im:queue:user:" // + userKey — global per-user queue counter + RedisKeyRateLimit = "im:ratelimit:" // + key — sliding-window rate limiting + RedisKeyGlobalGate = "im:global:active" // global concurrent worker counter ) // channelState holds runtime state for a running IM channel. diff --git a/internal/im/types.go b/internal/im/types.go index 7674f915..bf4ab55c 100644 --- a/internal/im/types.go +++ b/internal/im/types.go @@ -14,20 +14,20 @@ import ( // IMChannel represents an IM channel configuration stored in the database. // Each channel binds to an agent and contains platform-specific credentials. type IMChannel struct { - ID string `json:"id" gorm:"type:varchar(36);primaryKey;default:uuid_generate_v4()"` - TenantID uint64 `json:"tenant_id" gorm:"not null;index:idx_im_channels_tenant"` - AgentID string `json:"agent_id" gorm:"type:varchar(36);not null;index:idx_im_channels_agent"` - Platform string `json:"platform" gorm:"type:varchar(20);not null"` - Name string `json:"name" gorm:"type:varchar(255);not null;default:''"` - Enabled bool `json:"enabled" gorm:"not null;default:true"` - Mode string `json:"mode" gorm:"type:varchar(20);not null;default:'websocket'"` + ID string `json:"id" gorm:"type:varchar(36);primaryKey;default:uuid_generate_v4()"` + TenantID uint64 `json:"tenant_id" gorm:"not null;index:idx_im_channels_tenant"` + AgentID string `json:"agent_id" gorm:"type:varchar(36);not null;index:idx_im_channels_agent"` + Platform string `json:"platform" gorm:"type:varchar(20);not null"` + Name string `json:"name" gorm:"type:varchar(255);not null;default:''"` + Enabled bool `json:"enabled" gorm:"not null;default:true"` + Mode string `json:"mode" gorm:"type:varchar(20);not null;default:'websocket'"` OutputMode string `json:"output_mode" gorm:"type:varchar(20);not null;default:'stream'"` KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);default:''"` BotIdentity string `json:"bot_identity" gorm:"type:varchar(255);not null;default:'';uniqueIndex:idx_im_channels_bot_identity,where:deleted_at IS NULL AND bot_identity != ''"` Credentials types.JSON `json:"credentials" gorm:"type:jsonb;not null;default:'{}'"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"` } func (IMChannel) TableName() string { diff --git a/internal/models/chat/ollama.go b/internal/models/chat/ollama.go index 9ac00b2b..3265abc2 100644 --- a/internal/models/chat/ollama.go +++ b/internal/models/chat/ollama.go @@ -160,11 +160,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt return &types.ChatResponse{ Content: responseContent, ToolCalls: toolCalls, - Usage: struct { - PromptTokens int `json:"prompt_tokens"` - CompletionTokens int `json:"completion_tokens"` - TotalTokens int `json:"total_tokens"` - }{ + Usage: types.TokenUsage{ PromptTokens: promptTokens, CompletionTokens: completionTokens, TotalTokens: promptTokens + completionTokens, @@ -262,9 +258,18 @@ func (c *OllamaChat) ChatStream( } if resp.Done { + var usage *types.TokenUsage + if resp.PromptEvalCount > 0 || resp.EvalCount > 0 { + usage = &types.TokenUsage{ + PromptTokens: resp.PromptEvalCount, + CompletionTokens: resp.EvalCount, + TotalTokens: resp.PromptEvalCount + resp.EvalCount, + } + } streamChan <- types.StreamResponse{ ResponseType: types.ResponseTypeAnswer, Done: true, + Usage: usage, } } diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go index 5641d3b1..446b5dfc 100644 --- a/internal/models/chat/remote_api.go +++ b/internal/models/chat/remote_api.go @@ -152,6 +152,10 @@ func (c *RemoteAPIChat) BuildChatCompletionRequest(messages []Message, opts *Cha Stream: isStream, } + if isStream { + req.StreamOptions = &openai.StreamOptions{IncludeUsage: true} + } + if opts != nil { if opts.Temperature > 0 { req.Temperature = float32(opts.Temperature) @@ -316,11 +320,7 @@ func (c *RemoteAPIChat) parseCompletionResponse(resp *openai.ChatCompletionRespo response := &types.ChatResponse{ Content: content, FinishReason: string(choice.FinishReason), - Usage: struct { - PromptTokens int `json:"prompt_tokens"` - CompletionTokens int `json:"completion_tokens"` - TotalTokens int `json:"total_tokens"` - }{ + Usage: types.TokenUsage{ PromptTokens: resp.Usage.PromptTokens, CompletionTokens: resp.Usage.CompletionTokens, TotalTokens: resp.Usage.TotalTokens, @@ -465,6 +465,7 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo Content: "", Done: true, ToolCalls: state.buildOrderedToolCalls(), + Usage: state.usage, } } else { streamChan <- types.StreamResponse{ @@ -476,6 +477,14 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo return } + if response.Usage != nil { + state.usage = &types.TokenUsage{ + PromptTokens: response.Usage.PromptTokens, + CompletionTokens: response.Usage.CompletionTokens, + TotalTokens: response.Usage.TotalTokens, + } + } + if len(response.Choices) > 0 { c.processStreamDelta(ctx, &response.Choices[0], state, streamChan, response.Choices[0].Delta.ReasoningContent) } @@ -514,6 +523,7 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res Content: "", Done: true, ToolCalls: state.buildOrderedToolCalls(), + Usage: state.usage, } return } @@ -540,6 +550,14 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res continue } + if streamResp.Usage != nil { + state.usage = &types.TokenUsage{ + PromptTokens: streamResp.Usage.PromptTokens, + CompletionTokens: streamResp.Usage.CompletionTokens, + TotalTokens: streamResp.Usage.TotalTokens, + } + } + if len(streamResp.Choices) > 0 { choice := streamResp.Choices[0] // 统一获取逻辑(支持标准和 vLLM 两种路径) @@ -566,6 +584,7 @@ type streamState struct { nameNotified map[int]bool hasThinking bool fieldExtractors map[int]*jsonFieldExtractor // per tool-call-index extractors for streaming field extraction + usage *types.TokenUsage // captured from the final stream chunk when include_usage is enabled } func newStreamState() *streamState { diff --git a/internal/types/chat.go b/internal/types/chat.go index d5502ae4..61441e98 100644 --- a/internal/types/chat.go +++ b/internal/types/chat.go @@ -5,6 +5,13 @@ import ( "encoding/json" ) +// TokenUsage holds token consumption statistics returned by the model API. +type TokenUsage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` +} + // LLMToolCall represents a function/tool call from the LLM type LLMToolCall struct { ID string `json:"id"` @@ -20,20 +27,10 @@ type FunctionCall struct { // ChatResponse chat response type ChatResponse struct { - Content string `json:"content"` - // Tool calls requested by the model - ToolCalls []LLMToolCall `json:"tool_calls,omitempty"` - // Finish reason - FinishReason string `json:"finish_reason,omitempty"` // "stop", "tool_calls", "length", etc. - // Usage information - Usage struct { - // Prompt tokens - PromptTokens int `json:"prompt_tokens"` - // Completion tokens - CompletionTokens int `json:"completion_tokens"` - // Total tokens - TotalTokens int `json:"total_tokens"` - } `json:"usage"` + Content string `json:"content"` + ToolCalls []LLMToolCall `json:"tool_calls,omitempty"` + FinishReason string `json:"finish_reason,omitempty"` + Usage TokenUsage `json:"usage"` } // Response type @@ -64,24 +61,16 @@ const ( // StreamResponse stream response type StreamResponse struct { - // Unique identifier - ID string `json:"id"` - // Response type - ResponseType ResponseType `json:"response_type"` - // Current fragment content - Content string `json:"content"` - // Whether the response is complete - Done bool `json:"done"` - // Knowledge references - KnowledgeReferences References `json:"knowledge_references,omitempty"` - // Session ID (for agent_query event) - SessionID string `json:"session_id,omitempty"` - // Assistant Message ID (for agent_query event) - AssistantMessageID string `json:"assistant_message_id,omitempty"` - // Tool calls for streaming (partial) - ToolCalls []LLMToolCall `json:"tool_calls,omitempty"` - // Additional metadata for enhanced display - Data map[string]interface{} `json:"data,omitempty"` + ID string `json:"id"` + ResponseType ResponseType `json:"response_type"` + Content string `json:"content"` + Done bool `json:"done"` + KnowledgeReferences References `json:"knowledge_references,omitempty"` + SessionID string `json:"session_id,omitempty"` + AssistantMessageID string `json:"assistant_message_id,omitempty"` + ToolCalls []LLMToolCall `json:"tool_calls,omitempty"` + Data map[string]interface{} `json:"data,omitempty"` + Usage *TokenUsage `json:"usage,omitempty"` } // References references