feat(agent): enhance token management and context handling in AgentEngine

- Added token usage tracking to the AgentEngine, allowing for better estimation of current context token counts.
- Implemented a new method to estimate current tokens based on previous usage and newly appended messages, improving context management.
- Updated context window management to utilize estimated token counts for more efficient message consolidation.
- Enhanced streaming methods to include token usage information, providing better insights into LLM interactions.
- Refactored the token estimator to support BPE tokenization, ensuring accurate token counts for messages.
- Added unit tests to validate new token estimation and context management functionalities.
This commit is contained in:
wizardchen
2026-03-22 23:15:33 +08:00
committed by lyingbug
parent 0d4ceb6c96
commit 0bb1a581de
16 changed files with 247 additions and 239 deletions
+2
View File
@@ -114,6 +114,7 @@ require (
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dlclark/regexp2 v1.11.5 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/duckdb/duckdb-go-bindings v0.1.24 // indirect
github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.24 // indirect
@@ -240,6 +241,7 @@ require (
github.com/tidwall/gjson v1.17.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tiktoken-go/tokenizer v0.7.0 // indirect
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
+4
View File
@@ -1562,6 +1562,8 @@ github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
@@ -2434,6 +2436,8 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tiktoken-go/tokenizer v0.7.0 h1:VMu6MPT0bXFDHr7UPh9uii7CNItVt3X9K90omxL54vw=
github.com/tiktoken-go/tokenizer v0.7.0/go.mod h1:6UCYI/DtOallbmL7sSy30p6YQv60qNyU/4aVigPOx6w=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
+32 -6
View File
@@ -36,6 +36,8 @@ type AgentEngine struct {
imageDescriber ImageDescriberFunc // VLM function for describing images in tool results (optional)
tokenEstimator *agenttoken.Estimator // Token estimator for context window management
memoryConsolidator *agentmemory.Consolidator // Memory consolidator for LLM-powered summarization (optional)
lastUsage types.TokenUsage // Token usage from the most recent LLM call
lastSentMsgCount int // Number of messages sent in the most recent LLM call
}
// ImageDescriberFunc generates a text description of an image.
@@ -57,7 +59,10 @@ func NewAgentEngine(
if eventBus == nil {
eventBus = event.NewEventBus()
}
tokenEst := agenttoken.NewEstimator(0)
tokenEst, err := agenttoken.NewEstimator()
if err != nil {
return nil
}
engine := &AgentEngine{
config: config,
toolRegistry: toolRegistry,
@@ -133,6 +138,18 @@ func (e *AgentEngine) GetSkillsManager() *skills.Manager {
return e.skillsManager
}
// estimateCurrentTokens returns the best estimate of the current context token count.
// When API-reported usage from a previous round is available, it uses that as a
// baseline and only BPE-estimates the delta (newly appended messages). Otherwise it
// falls back to a full BPE estimation of all messages.
func (e *AgentEngine) estimateCurrentTokens(messages []chat.Message) int {
if e.lastUsage.TotalTokens > 0 && e.lastSentMsgCount > 0 && e.lastSentMsgCount < len(messages) {
delta := e.tokenEstimator.EstimateMessages(messages[e.lastSentMsgCount:])
return e.lastUsage.TotalTokens + delta
}
return e.tokenEstimator.EstimateMessages(messages)
}
// Execute executes the agent with conversation history and streaming output
// All events are emitted to EventBus and handled by subscribers (like Handler layer)
func (e *AgentEngine) Execute(
@@ -272,11 +289,14 @@ func (e *AgentEngine) executeLoop(
roundStart := time.Now()
// Context window management
messages = e.manageContextWindow(ctx, messages, state.CurrentRound+1)
// Context window management: estimate current token count using
// the API-reported usage from the previous round plus a BPE delta
// for newly appended messages (assistant reply + tool results).
currentTokens := e.estimateCurrentTokens(messages)
messages = e.manageContextWindow(ctx, messages, state.CurrentRound+1, currentTokens)
logger.Infof(ctx, "[Agent][Round-%d/%d] Starting: %d messages, %d tools",
state.CurrentRound+1, e.config.MaxIterations, len(messages), len(tools))
logger.Infof(ctx, "[Agent][Round-%d/%d] Starting: %d messages, %d tools, est_tokens=%d",
state.CurrentRound+1, e.config.MaxIterations, len(messages), len(tools), currentTokens)
common.PipelineInfo(ctx, "Agent", "round_start", map[string]interface{}{
"iteration": state.CurrentRound,
"round": state.CurrentRound + 1,
@@ -286,14 +306,20 @@ func (e *AgentEngine) executeLoop(
})
// 1. Think: Call LLM with function calling (includes retry + graceful degradation)
e.lastSentMsgCount = len(messages)
response, err := e.callLLMWithRetry(ctx, messages, tools, state, query, state.CurrentRound, sessionID)
if err != nil {
return state, err
}
if response == nil {
// Graceful degradation succeeded — state.IsComplete already set
break
}
if response.Usage.TotalTokens > 0 {
e.lastUsage = response.Usage
logger.Debugf(ctx, "[Agent][Round-%d] Usage: prompt=%d, completion=%d, total=%d",
state.CurrentRound+1, response.Usage.PromptTokens,
response.Usage.CompletionTokens, response.Usage.TotalTokens)
}
// Create agent step
step := types.AgentStep{
+3 -2
View File
@@ -87,7 +87,7 @@ Now generate the final answer:`, query)
answerID := generateEventID("answer")
logger.Debugf(ctx, "[Agent][FinalAnswer] AnswerID: %s", answerID)
fullAnswer, _, err := e.streamLLMToEventBus(
llmResult, err := e.streamLLMToEventBus(
ctx,
messages,
&chat.ChatOptions{Temperature: e.config.Temperature, Thinking: e.config.Thinking},
@@ -95,7 +95,7 @@ Now generate the final answer:`, query)
if chunk.Content != "" {
logger.Debugf(ctx, "[Agent][FinalAnswer] Emitting answer chunk: %d chars", len(chunk.Content))
e.eventBus.Emit(ctx, event.Event{
ID: answerID, // Same ID for all chunks in this stream
ID: answerID,
Type: event.EventAgentFinalAnswer,
SessionID: sessionID,
Data: event.AgentFinalAnswerData{
@@ -115,6 +115,7 @@ Now generate the final answer:`, query)
return err
}
fullAnswer := llmResult.Content
logger.Infof(ctx, "[Agent][FinalAnswer] Final answer generated: %d characters", len(fullAnswer))
common.PipelineInfo(ctx, "Agent", "final_answer_done", map[string]interface{}{
"session_id": sessionID,
+5 -5
View File
@@ -58,14 +58,14 @@ func NewConsolidator(
}
}
// ShouldConsolidate checks if the messages need consolidation based on token count.
func (c *Consolidator) ShouldConsolidate(messages []chat.Message) bool {
if c.maxTokens <= 0 || len(messages) <= 3 {
// ShouldConsolidate checks if consolidation is needed based on the caller-provided
// token estimate. The estimate should come from the model API's Usage when available.
func (c *Consolidator) ShouldConsolidate(currentTokens int) bool {
if c.maxTokens <= 0 {
return false
}
tokenCount := c.estimator.EstimateMessages(messages)
triggerAt := int(float64(c.maxTokens) * c.threshold)
return tokenCount > triggerAt
return currentTokens > triggerAt
}
// Consolidate summarizes older messages and returns a compressed message array.
+13 -19
View File
@@ -9,7 +9,8 @@ import (
)
func TestConsolidator_ShouldConsolidate(t *testing.T) {
est := token.NewEstimator(0)
est, err := token.NewEstimator()
assert.NoError(t, err)
t.Run("below threshold returns false", func(t *testing.T) {
c := NewConsolidator(nil, est, 100000, 0)
@@ -19,27 +20,18 @@ func TestConsolidator_ShouldConsolidate(t *testing.T) {
{Role: "assistant", Content: "hi"},
{Role: "user", Content: "current"},
}
assert.False(t, c.ShouldConsolidate(messages))
tokens := est.EstimateMessages(messages)
assert.False(t, c.ShouldConsolidate(tokens))
})
t.Run("too few messages returns false", func(t *testing.T) {
c := NewConsolidator(nil, est, 10, 0)
messages := []chat.Message{
{Role: "system", Content: "system"},
{Role: "user", Content: "hello"},
}
assert.False(t, c.ShouldConsolidate(messages))
t.Run("over threshold returns true", func(t *testing.T) {
c := NewConsolidator(nil, est, 10, 0) // very low threshold
assert.True(t, c.ShouldConsolidate(100))
})
t.Run("disabled returns false", func(t *testing.T) {
c := NewConsolidator(nil, est, 0, 0) // disabled
messages := []chat.Message{
{Role: "system", Content: "system"},
{Role: "user", Content: "hello"},
{Role: "assistant", Content: "hi"},
{Role: "user", Content: "current"},
}
assert.False(t, c.ShouldConsolidate(messages))
c := NewConsolidator(nil, est, 0, 0)
assert.False(t, c.ShouldConsolidate(99999))
})
}
@@ -61,7 +53,8 @@ func TestTruncateForPrompt(t *testing.T) {
}
func TestRawArchive(t *testing.T) {
est := token.NewEstimator(0)
est, err := token.NewEstimator()
assert.NoError(t, err)
c := NewConsolidator(nil, est, 100000, 0)
messages := []chat.Message{
@@ -83,7 +76,8 @@ func TestRawArchive(t *testing.T) {
}
func TestBuildConsolidationPrompt(t *testing.T) {
est := token.NewEstimator(0)
est, err := token.NewEstimator()
assert.NoError(t, err)
c := NewConsolidator(nil, est, 100000, 0)
messages := []chat.Message{
+8 -8
View File
@@ -16,29 +16,29 @@ import (
)
// manageContextWindow consolidates or compresses messages if approaching the token limit.
// It returns the (possibly shortened) message slice.
func (e *AgentEngine) manageContextWindow(ctx context.Context, messages []chat.Message, round int) []chat.Message {
// currentTokens is the caller's best estimate of the current context size (using
// API-reported Usage when available, falling back to BPE estimation).
func (e *AgentEngine) manageContextWindow(ctx context.Context, messages []chat.Message, round, currentTokens int) []chat.Message {
if e.config.MaxContextTokens <= 0 {
return messages
}
beforeLen := len(messages)
// Try LLM-powered consolidation first (preserves more semantic content)
if e.memoryConsolidator != nil && e.memoryConsolidator.ShouldConsolidate(messages) {
logger.Infof(ctx, "[Agent][Round-%d] Token threshold exceeded, consolidating memory",
round)
if e.memoryConsolidator != nil && e.memoryConsolidator.ShouldConsolidate(currentTokens) {
logger.Infof(ctx, "[Agent][Round-%d] Token threshold exceeded (est=%d), consolidating memory",
round, currentTokens)
consolidated, consolidateErr := e.memoryConsolidator.Consolidate(ctx, messages)
if consolidateErr != nil {
logger.Warnf(ctx, "[Agent][Round-%d] Memory consolidation failed: %v, "+
"falling back to simple compression", round, consolidateErr)
} else {
messages = consolidated
currentTokens = e.tokenEstimator.EstimateMessages(messages)
}
}
// Fallback: simple context compression if still over the hard limit (80%)
messages = agenttoken.CompressContext(messages, e.tokenEstimator, e.config.MaxContextTokens)
messages = agenttoken.CompressContext(messages, e.tokenEstimator, e.config.MaxContextTokens, currentTokens)
if len(messages) < beforeLen {
logger.Infof(ctx, "[Agent][Round-%d] Context managed: %d → %d messages (max_tokens=%d)",
+31 -28
View File
@@ -13,58 +13,59 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
// streamLLMResult holds accumulated output from a streaming LLM call.
type streamLLMResult struct {
Content string
ToolCalls []types.LLMToolCall
Usage *types.TokenUsage
}
// streamLLMToEventBus streams LLM response through EventBus (generic method)
// emitFunc: callback to emit each chunk event
// Returns: full accumulated content, tool calls (if any), error
func (e *AgentEngine) streamLLMToEventBus(
ctx context.Context,
messages []chat.Message,
opts *chat.ChatOptions,
emitFunc func(chunk *types.StreamResponse, fullContent string),
) (string, []types.LLMToolCall, error) {
) (*streamLLMResult, error) {
logger.Debugf(ctx, "[Agent][Stream] Starting LLM stream with %d messages", len(messages))
// Create a per-call timeout context so that a single LLM call gets a
// guaranteed time window even when the parent context's deadline is almost
// exhausted after previous iterations. The shorter of the two deadlines wins,
// so the parent pipeline timeout is still respected.
llmCtx, llmCancel := context.WithTimeout(ctx, e.getLLMCallTimeout())
defer llmCancel()
stream, err := e.chatModel.ChatStream(llmCtx, messages, opts)
if err != nil {
logger.Errorf(ctx, "[Agent][Stream] Failed to start LLM stream: %v", err)
return "", nil, err
return nil, err
}
fullContent := ""
var toolCalls []types.LLMToolCall
result := &streamLLMResult{}
chunkCount := 0
for chunk := range stream {
chunkCount++
if chunk.Content != "" {
// Only accumulate LLM's own text output, not content extracted from tool arguments
// (e.g. final_answer's answer or thinking tool's thought are streamed separately)
isExtracted := chunk.Data != nil && chunk.Data["source"] != nil
if !isExtracted {
fullContent += chunk.Content
result.Content += chunk.Content
}
}
// Collect tool calls if present
if len(chunk.ToolCalls) > 0 {
toolCalls = chunk.ToolCalls
result.ToolCalls = chunk.ToolCalls
}
if chunk.Usage != nil {
result.Usage = chunk.Usage
}
// Emit event through callback
if emitFunc != nil {
emitFunc(&chunk, fullContent)
emitFunc(&chunk, result.Content)
}
}
return fullContent, toolCalls, nil
return result, nil
}
// streamReflectionToEventBus streams reflection process through EventBus
@@ -93,14 +94,14 @@ Think:
// Generate a single ID for this entire reflection stream
reflectionID := generateEventID("reflection")
fullReflection, _, err := e.streamLLMToEventBus(
llmResult, err := e.streamLLMToEventBus(
ctx,
messages,
&chat.ChatOptions{Temperature: 0.5},
func(chunk *types.StreamResponse, fullContent string) {
if chunk.Content != "" {
e.eventBus.Emit(ctx, event.Event{
ID: reflectionID, // Same ID for all chunks in this stream
ID: reflectionID,
Type: event.EventAgentReflection,
SessionID: sessionID,
Data: event.AgentReflectionData{
@@ -118,7 +119,7 @@ Think:
return "", err
}
return fullReflection, nil
return llmResult.Content, nil
}
// streamThinkingToEventBus streams the thinking process through EventBus
@@ -145,7 +146,7 @@ func (e *AgentEngine) streamThinkingToEventBus(
thinkingID := generateEventID("thinking")
answerID := generateEventID("answer")
fullContent, toolCalls, err := e.streamLLMToEventBus(
llmResult, err := e.streamLLMToEventBus(
ctx,
messages,
opts,
@@ -230,17 +231,19 @@ func (e *AgentEngine) streamThinkingToEventBus(
}
logger.Debugf(ctx, "[Agent][Thinking] Iteration-%d completed: content=%d chars, tool_calls=%d",
iteration+1, len(fullContent), len(toolCalls))
iteration+1, len(llmResult.Content), len(llmResult.ToolCalls))
// Strip <think>…</think> blocks that some models (DeepSeek, Qwen, etc.) embed in content.
fullContent = agenttools.StripThinkBlocks(fullContent)
fullContent := agenttools.StripThinkBlocks(llmResult.Content)
// Build response
return &types.ChatResponse{
resp := &types.ChatResponse{
Content: fullContent,
ToolCalls: toolCalls,
ToolCalls: llmResult.ToolCalls,
FinishReason: "stop",
}, nil
}
if llmResult.Usage != nil {
resp.Usage = *llmResult.Usage
}
return resp, nil
}
// callLLMWithRetry logs messages, sanitizes them, calls the LLM with retry on transient errors,
+5 -10
View File
@@ -9,39 +9,35 @@ import (
const DefaultContextThresholdRatio = 0.8
// CompressContext trims older messages to bring total token count below the threshold.
// currentTokens is the caller's best estimate of the current context size (from API
// Usage when available, falling back to BPE estimation). The estimator is still used
// internally for per-message-group cost calculation during trimming.
//
// It preserves:
// - The first message (system prompt)
// - The last message (current user query)
// - tool_call / tool_result message pairs (never splits them)
//
// Returns the compressed message slice. If no compression is needed, returns the original.
func CompressContext(
messages []chat.Message,
estimator *Estimator,
maxTokens int,
currentTokens int,
) []chat.Message {
if maxTokens <= 0 || len(messages) <= 2 {
return messages
}
threshold := int(float64(maxTokens) * DefaultContextThresholdRatio)
currentTokens := estimator.EstimateMessages(messages)
if currentTokens <= threshold {
return messages
}
// Strategy: keep system prompt (first) + current query (last),
// trim from the oldest messages inward, respecting tool_call/tool_result boundaries.
systemMsg := messages[0]
lastMsg := messages[len(messages)-1]
middle := messages[1 : len(messages)-1]
// Find groups of messages that form tool_call/tool_result pairs.
// A group = one assistant message with tool_calls + all following tool result messages.
groups := groupToolMessages(middle)
// Remove groups from the front until we're under the threshold.
tokensToFree := currentTokens - threshold
freed := 0
removeUpTo := 0
@@ -58,7 +54,6 @@ func CompressContext(
}
}
// Rebuild messages: system + remaining groups + current query
remaining := make([]chat.Message, 0, len(messages))
remaining = append(remaining, systemMsg)
for i := removeUpTo; i < len(groups); i++ {
+49 -78
View File
@@ -1,116 +1,87 @@
// Package token provides token estimation for LLM context window management.
//
// The primary source of truth for token counts is the model API's Usage field
// (types.TokenUsage), returned with every LLM response. This package serves as
// a supplementary estimator used in two scenarios:
//
// 1. Delta estimation — after an LLM call, new messages (assistant reply +
// tool results) are appended before the next call. The Estimator computes
// the token cost of these new messages so the engine can decide whether
// context compression is needed without making an extra API round-trip.
//
// 2. First-round fallback — on the very first round of a session, no prior
// API Usage is available, so the Estimator provides a full estimate.
//
// The encoding used (cl100k_base) is an approximation. Different model families
// use different tokenizers, so the numbers will not be exact for non-OpenAI
// models. This is acceptable because the estimate only needs to be close enough
// to trigger compression at roughly the right time; over- or under-estimating
// by a small margin is corrected on the next API call.
package token
import (
"unicode"
"fmt"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/tiktoken-go/tokenizer"
)
const (
// DefaultCharsPerToken is a blended ratio for mixed English/CJK text.
// English ≈ 4 chars/token, CJK ≈ 1.5 chars/token, blended ≈ 3.5.
DefaultCharsPerToken = 3.5
// perMessageOverhead is the estimated token overhead per message
// (role label, formatting, separator tokens).
perMessageOverhead = 4
perMessageOverhead = 3
perConversationTail = 3
)
// Estimator estimates token counts for messages and strings.
// Uses a character-based heuristic that accounts for CJK characters.
// Estimator counts tokens for messages and strings using BPE tokenization.
// It is intended for incremental (delta) estimation between LLM calls; the
// authoritative token count comes from the API's Usage response.
type Estimator struct {
charsPerToken float64
codec tokenizer.Codec
}
// NewEstimator creates a token estimator.
// If charsPerToken <= 0, DefaultCharsPerToken is used.
func NewEstimator(charsPerToken float64) *Estimator {
if charsPerToken <= 0 {
charsPerToken = DefaultCharsPerToken
// NewEstimator creates a token estimator using the cl100k_base encoding.
func NewEstimator() (*Estimator, error) {
codec, err := tokenizer.Get(tokenizer.Cl100kBase)
if err != nil {
return nil, fmt.Errorf("token: failed to initialize tokenizer: %w", err)
}
return &Estimator{charsPerToken: charsPerToken}
return &Estimator{codec: codec}, nil
}
// EstimateString estimates the token count for a string.
// It uses a CJK-aware heuristic: CJK characters are counted at ~1.5 chars/token,
// while ASCII/Latin characters use the configured charsPerToken ratio.
func (e *Estimator) EstimateString(s string) int {
if len(s) == 0 {
return 0
}
var cjkRunes, otherRunes int
for _, r := range s {
if isCJK(r) {
cjkRunes++
} else {
otherRunes++
}
}
// CJK characters are roughly 1.5 chars per token
cjkTokens := float64(cjkRunes) / 1.5
otherTokens := float64(otherRunes) / e.charsPerToken
return int(cjkTokens+otherTokens) + 1 // +1 to avoid underestimation
}
// EstimateMessages estimates the total token count for a slice of chat messages.
// EstimateMessages returns the estimated token count for a slice of messages.
// Prefer using API Usage for the full context and this method only for deltas.
func (e *Estimator) EstimateMessages(messages []chat.Message) int {
total := 0
for i := range messages {
total += e.EstimateMessage(&messages[i])
}
total += perConversationTail
return total
}
// EstimateMessage estimates the token count for a single message.
// EstimateString returns the token count for a string using BPE tokenization.
func (e *Estimator) EstimateString(s string) int {
if len(s) == 0 {
return 0
}
ids, _, err := e.codec.Encode(s)
if err != nil {
return (len(s) + 3) / 4
}
return len(ids)
}
// EstimateMessage returns the token count for a single message.
func (e *Estimator) EstimateMessage(msg *chat.Message) int {
tokens := perMessageOverhead
tokens += e.EstimateString(msg.Role)
tokens += e.EstimateString(msg.Content)
tokens += e.EstimateString(msg.Name)
// Tool calls in assistant messages
for _, tc := range msg.ToolCalls {
tokens += e.EstimateString(tc.Function.Name)
tokens += e.EstimateString(tc.Function.Arguments)
tokens += 4 // overhead for tool call structure
tokens += 4
}
return tokens
}
// isCJK checks if a rune is a CJK (Chinese, Japanese, Korean) character.
func isCJK(r rune) bool {
// CJK Unified Ideographs
if r >= 0x4E00 && r <= 0x9FFF {
return true
}
// CJK Unified Ideographs Extension A
if r >= 0x3400 && r <= 0x4DBF {
return true
}
// CJK Unified Ideographs Extension B
if r >= 0x20000 && r <= 0x2A6DF {
return true
}
// CJK Compatibility Ideographs
if r >= 0xF900 && r <= 0xFAFF {
return true
}
// Hiragana + Katakana
if r >= 0x3040 && r <= 0x30FF {
return true
}
// Hangul Syllables
if r >= 0xAC00 && r <= 0xD7AF {
return true
}
// Full-width characters
if unicode.Is(unicode.Han, r) {
return true
}
return false
}
+23 -24
View File
@@ -12,34 +12,34 @@ import (
)
func TestEstimator(t *testing.T) {
e := NewEstimator(0) // use default
e, err := NewEstimator()
if err != nil {
t.Fatalf("Failed to create estimator: %v", err)
}
t.Run("empty string", func(t *testing.T) {
assert.Equal(t, 0, e.EstimateString(""))
})
t.Run("english text", func(t *testing.T) {
// "hello world" = 11 chars ≈ 11/3.5 + 1 ≈ 4 tokens
tokens := e.EstimateString("hello world")
assert.Greater(t, tokens, 0)
assert.Less(t, tokens, 20)
assert.Equal(t, 2, tokens)
})
t.Run("chinese text", func(t *testing.T) {
// 10 Chinese chars at ~1.5 chars/token ≈ 7 tokens
tokens := e.EstimateString("你好世界测试数据中文")
assert.Greater(t, tokens, 3)
assert.Less(t, tokens, 20)
fmt.Println(tokens)
assert.Greater(t, tokens, 0)
})
t.Run("CJK is more tokens per char than latin", func(t *testing.T) {
// Same number of characters, but CJK should estimate more tokens
t.Run("CJK produces more tokens per char than latin", func(t *testing.T) {
latin := strings.Repeat("a", 100)
cjk := strings.Repeat("中", 100)
latinTokens := e.EstimateString(latin)
cjkTokens := e.EstimateString(cjk)
fmt.Println(latinTokens, cjkTokens)
assert.Greater(t, cjkTokens, latinTokens,
"CJK text should estimate more tokens per character")
"CJK text should produce more tokens per character")
})
t.Run("message estimation includes overhead", func(t *testing.T) {
@@ -49,6 +49,7 @@ func TestEstimator(t *testing.T) {
}
tokens := e.EstimateMessage(&msg)
contentTokens := e.EstimateString("hello")
fmt.Println(tokens, contentTokens)
assert.Greater(t, tokens, contentTokens,
"message tokens should include overhead beyond just content")
})
@@ -82,26 +83,28 @@ func TestEstimator(t *testing.T) {
}
func TestCompressContext(t *testing.T) {
e := NewEstimator(0)
e, err := NewEstimator()
require.NoError(t, err)
t.Run("no compression needed", func(t *testing.T) {
messages := []chat.Message{
{Role: "system", Content: "system prompt"},
{Role: "user", Content: "hello"},
}
result := CompressContext(messages, e, 100000)
tokens := e.EstimateMessages(messages)
result := CompressContext(messages, e, 100000, tokens)
assert.Equal(t, messages, result)
})
t.Run("preserves system and last message", func(t *testing.T) {
// Create messages that exceed the token limit
messages := []chat.Message{
{Role: "system", Content: "system prompt"},
{Role: "user", Content: strings.Repeat("old message ", 1000)},
{Role: "assistant", Content: strings.Repeat("old response ", 1000)},
{Role: "user", Content: "current query"},
}
result := CompressContext(messages, e, 100) // very low limit to force compression
tokens := e.EstimateMessages(messages)
result := CompressContext(messages, e, 100, tokens)
require.GreaterOrEqual(t, len(result), 2)
assert.Equal(t, "system", result[0].Role)
assert.Equal(t, "current query", result[len(result)-1].Content)
@@ -110,27 +113,23 @@ func TestCompressContext(t *testing.T) {
t.Run("keeps tool_call and tool_result paired", func(t *testing.T) {
messages := []chat.Message{
{Role: "system", Content: "system prompt"},
// Old conversation
{Role: "user", Content: strings.Repeat("x", 500)},
{Role: "assistant", Content: strings.Repeat("y", 500)},
// Tool interaction that should stay together
{Role: "assistant", Content: "thinking", ToolCalls: []chat.ToolCall{
{ID: "call_1", Function: chat.FunctionCall{Name: "search", Arguments: `{"q":"test"}`}},
}},
{Role: "tool", Content: strings.Repeat("result ", 100), ToolCallID: "call_1", Name: "search"},
// Current query
{Role: "user", Content: "what did you find?"},
}
result := CompressContext(messages, e, 200) // force some compression
tokens := e.EstimateMessages(messages)
result := CompressContext(messages, e, 200, tokens)
b, _ := json.Marshal(result)
fmt.Println(string(b))
// Check that tool_call and tool_result are still paired
for i, msg := range result {
if msg.Role == "tool" {
// The previous message should be an assistant with tool_calls
require.Greater(t, i, 0)
assert.Equal(t, "assistant", result[i-1].Role)
assert.Greater(t, len(result[i-1].ToolCalls), 0)
@@ -143,7 +142,7 @@ func TestCompressContext(t *testing.T) {
{Role: "system", Content: "system"},
{Role: "user", Content: "hello"},
}
result := CompressContext(messages, e, 0)
result := CompressContext(messages, e, 0, 0)
assert.Equal(t, messages, result)
})
@@ -152,7 +151,7 @@ func TestCompressContext(t *testing.T) {
{Role: "system", Content: "system"},
{Role: "user", Content: "hello"},
}
result := CompressContext(messages, e, 1) // even with tiny limit
result := CompressContext(messages, e, 1, 999)
assert.Equal(t, messages, result)
})
}
@@ -179,8 +178,8 @@ func TestGroupToolMessages(t *testing.T) {
{Role: "user", Content: "thanks"},
}
groups := groupToolMessages(messages)
assert.Len(t, groups, 3) // user, (assistant+2 tools), user
assert.Len(t, groups[1], 3) // assistant + 2 tool results
assert.Len(t, groups, 3)
assert.Len(t, groups[1], 3)
b, _ := json.Marshal(groups)
fmt.Println(string(b))
})
+7 -7
View File
@@ -57,13 +57,13 @@ const (
// All IM-related Redis keys are defined here for discoverability and to avoid
// scattered string literals across multiple files.
const (
RedisKeyLeader = "im:ws:leader:" // + channelID — WebSocket leader election
RedisKeyDedup = "im:dedup:" // + messageID — message deduplication
RedisKeyStop = "im:stop:" // + userKey — cross-instance /stop marker (pre-execution)
RedisKeyInflight = "im:inflight:" // + userKey — maps userKey → sessionID:messageID for cross-instance /stop
RedisKeyQueueUser = "im:queue:user:" // + userKey — global per-user queue counter
RedisKeyRateLimit = "im:ratelimit:" // + key — sliding-window rate limiting
RedisKeyGlobalGate = "im:global:active" // global concurrent worker counter
RedisKeyLeader = "im:ws:leader:" // + channelID — WebSocket leader election
RedisKeyDedup = "im:dedup:" // + messageID — message deduplication
RedisKeyStop = "im:stop:" // + userKey — cross-instance /stop marker (pre-execution)
RedisKeyInflight = "im:inflight:" // + userKey — maps userKey → sessionID:messageID for cross-instance /stop
RedisKeyQueueUser = "im:queue:user:" // + userKey — global per-user queue counter
RedisKeyRateLimit = "im:ratelimit:" // + key — sliding-window rate limiting
RedisKeyGlobalGate = "im:global:active" // global concurrent worker counter
)
// channelState holds runtime state for a running IM channel.
+10 -10
View File
@@ -14,20 +14,20 @@ import (
// IMChannel represents an IM channel configuration stored in the database.
// Each channel binds to an agent and contains platform-specific credentials.
type IMChannel struct {
ID string `json:"id" gorm:"type:varchar(36);primaryKey;default:uuid_generate_v4()"`
TenantID uint64 `json:"tenant_id" gorm:"not null;index:idx_im_channels_tenant"`
AgentID string `json:"agent_id" gorm:"type:varchar(36);not null;index:idx_im_channels_agent"`
Platform string `json:"platform" gorm:"type:varchar(20);not null"`
Name string `json:"name" gorm:"type:varchar(255);not null;default:''"`
Enabled bool `json:"enabled" gorm:"not null;default:true"`
Mode string `json:"mode" gorm:"type:varchar(20);not null;default:'websocket'"`
ID string `json:"id" gorm:"type:varchar(36);primaryKey;default:uuid_generate_v4()"`
TenantID uint64 `json:"tenant_id" gorm:"not null;index:idx_im_channels_tenant"`
AgentID string `json:"agent_id" gorm:"type:varchar(36);not null;index:idx_im_channels_agent"`
Platform string `json:"platform" gorm:"type:varchar(20);not null"`
Name string `json:"name" gorm:"type:varchar(255);not null;default:''"`
Enabled bool `json:"enabled" gorm:"not null;default:true"`
Mode string `json:"mode" gorm:"type:varchar(20);not null;default:'websocket'"`
OutputMode string `json:"output_mode" gorm:"type:varchar(20);not null;default:'stream'"`
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);default:''"`
BotIdentity string `json:"bot_identity" gorm:"type:varchar(255);not null;default:'';uniqueIndex:idx_im_channels_bot_identity,where:deleted_at IS NULL AND bot_identity != ''"`
Credentials types.JSON `json:"credentials" gorm:"type:jsonb;not null;default:'{}'"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
}
func (IMChannel) TableName() string {
+10 -5
View File
@@ -160,11 +160,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
return &types.ChatResponse{
Content: responseContent,
ToolCalls: toolCalls,
Usage: struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
}{
Usage: types.TokenUsage{
PromptTokens: promptTokens,
CompletionTokens: completionTokens,
TotalTokens: promptTokens + completionTokens,
@@ -262,9 +258,18 @@ func (c *OllamaChat) ChatStream(
}
if resp.Done {
var usage *types.TokenUsage
if resp.PromptEvalCount > 0 || resp.EvalCount > 0 {
usage = &types.TokenUsage{
PromptTokens: resp.PromptEvalCount,
CompletionTokens: resp.EvalCount,
TotalTokens: resp.PromptEvalCount + resp.EvalCount,
}
}
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Done: true,
Usage: usage,
}
}
+24 -5
View File
@@ -152,6 +152,10 @@ func (c *RemoteAPIChat) BuildChatCompletionRequest(messages []Message, opts *Cha
Stream: isStream,
}
if isStream {
req.StreamOptions = &openai.StreamOptions{IncludeUsage: true}
}
if opts != nil {
if opts.Temperature > 0 {
req.Temperature = float32(opts.Temperature)
@@ -316,11 +320,7 @@ func (c *RemoteAPIChat) parseCompletionResponse(resp *openai.ChatCompletionRespo
response := &types.ChatResponse{
Content: content,
FinishReason: string(choice.FinishReason),
Usage: struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
}{
Usage: types.TokenUsage{
PromptTokens: resp.Usage.PromptTokens,
CompletionTokens: resp.Usage.CompletionTokens,
TotalTokens: resp.Usage.TotalTokens,
@@ -465,6 +465,7 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo
Content: "",
Done: true,
ToolCalls: state.buildOrderedToolCalls(),
Usage: state.usage,
}
} else {
streamChan <- types.StreamResponse{
@@ -476,6 +477,14 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo
return
}
if response.Usage != nil {
state.usage = &types.TokenUsage{
PromptTokens: response.Usage.PromptTokens,
CompletionTokens: response.Usage.CompletionTokens,
TotalTokens: response.Usage.TotalTokens,
}
}
if len(response.Choices) > 0 {
c.processStreamDelta(ctx, &response.Choices[0], state, streamChan, response.Choices[0].Delta.ReasoningContent)
}
@@ -514,6 +523,7 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res
Content: "",
Done: true,
ToolCalls: state.buildOrderedToolCalls(),
Usage: state.usage,
}
return
}
@@ -540,6 +550,14 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res
continue
}
if streamResp.Usage != nil {
state.usage = &types.TokenUsage{
PromptTokens: streamResp.Usage.PromptTokens,
CompletionTokens: streamResp.Usage.CompletionTokens,
TotalTokens: streamResp.Usage.TotalTokens,
}
}
if len(streamResp.Choices) > 0 {
choice := streamResp.Choices[0]
// 统一获取逻辑(支持标准和 vLLM 两种路径)
@@ -566,6 +584,7 @@ type streamState struct {
nameNotified map[int]bool
hasThinking bool
fieldExtractors map[int]*jsonFieldExtractor // per tool-call-index extractors for streaming field extraction
usage *types.TokenUsage // captured from the final stream chunk when include_usage is enabled
}
func newStreamState() *streamState {
+21 -32
View File
@@ -5,6 +5,13 @@ import (
"encoding/json"
)
// TokenUsage holds token consumption statistics returned by the model API.
type TokenUsage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
}
// LLMToolCall represents a function/tool call from the LLM
type LLMToolCall struct {
ID string `json:"id"`
@@ -20,20 +27,10 @@ type FunctionCall struct {
// ChatResponse chat response
type ChatResponse struct {
Content string `json:"content"`
// Tool calls requested by the model
ToolCalls []LLMToolCall `json:"tool_calls,omitempty"`
// Finish reason
FinishReason string `json:"finish_reason,omitempty"` // "stop", "tool_calls", "length", etc.
// Usage information
Usage struct {
// Prompt tokens
PromptTokens int `json:"prompt_tokens"`
// Completion tokens
CompletionTokens int `json:"completion_tokens"`
// Total tokens
TotalTokens int `json:"total_tokens"`
} `json:"usage"`
Content string `json:"content"`
ToolCalls []LLMToolCall `json:"tool_calls,omitempty"`
FinishReason string `json:"finish_reason,omitempty"`
Usage TokenUsage `json:"usage"`
}
// Response type
@@ -64,24 +61,16 @@ const (
// StreamResponse stream response
type StreamResponse struct {
// Unique identifier
ID string `json:"id"`
// Response type
ResponseType ResponseType `json:"response_type"`
// Current fragment content
Content string `json:"content"`
// Whether the response is complete
Done bool `json:"done"`
// Knowledge references
KnowledgeReferences References `json:"knowledge_references,omitempty"`
// Session ID (for agent_query event)
SessionID string `json:"session_id,omitempty"`
// Assistant Message ID (for agent_query event)
AssistantMessageID string `json:"assistant_message_id,omitempty"`
// Tool calls for streaming (partial)
ToolCalls []LLMToolCall `json:"tool_calls,omitempty"`
// Additional metadata for enhanced display
Data map[string]interface{} `json:"data,omitempty"`
ID string `json:"id"`
ResponseType ResponseType `json:"response_type"`
Content string `json:"content"`
Done bool `json:"done"`
KnowledgeReferences References `json:"knowledge_references,omitempty"`
SessionID string `json:"session_id,omitempty"`
AssistantMessageID string `json:"assistant_message_id,omitempty"`
ToolCalls []LLMToolCall `json:"tool_calls,omitempty"`
Data map[string]interface{} `json:"data,omitempty"`
Usage *TokenUsage `json:"usage,omitempty"`
}
// References references