Files
Enterprise-Knowledge-Base/internal/agent/token/compress.go
T
wizardchen 0bb1a581de feat(agent): enhance token management and context handling in AgentEngine
- Added token usage tracking to the AgentEngine, allowing for better estimation of current context token counts.
- Implemented a new method to estimate current tokens based on previous usage and newly appended messages, improving context management.
- Updated context window management to utilize estimated token counts for more efficient message consolidation.
- Enhanced streaming methods to include token usage information, providing better insights into LLM interactions.
- Refactored the token estimator to support BPE tokenization, ensuring accurate token counts for messages.
- Added unit tests to validate new token estimation and context management functionalities.
2026-03-25 22:08:29 +08:00

95 lines
2.7 KiB
Go

package token
import (
"github.com/Tencent/WeKnora/internal/models/chat"
)
// DefaultContextThresholdRatio is the ratio of context window usage that triggers compression.
// When message tokens exceed MaxContextTokens * threshold, old messages are trimmed.
const DefaultContextThresholdRatio = 0.8
// CompressContext trims older messages to bring total token count below the threshold.
// currentTokens is the caller's best estimate of the current context size (from API
// Usage when available, falling back to BPE estimation). The estimator is still used
// internally for per-message-group cost calculation during trimming.
//
// It preserves:
// - The first message (system prompt)
// - The last message (current user query)
// - tool_call / tool_result message pairs (never splits them)
func CompressContext(
messages []chat.Message,
estimator *Estimator,
maxTokens int,
currentTokens int,
) []chat.Message {
if maxTokens <= 0 || len(messages) <= 2 {
return messages
}
threshold := int(float64(maxTokens) * DefaultContextThresholdRatio)
if currentTokens <= threshold {
return messages
}
systemMsg := messages[0]
lastMsg := messages[len(messages)-1]
middle := messages[1 : len(messages)-1]
groups := groupToolMessages(middle)
tokensToFree := currentTokens - threshold
freed := 0
removeUpTo := 0
for i, group := range groups {
groupTokens := 0
for _, msg := range group {
groupTokens += estimator.EstimateMessage(&msg)
}
freed += groupTokens
removeUpTo = i + 1
if freed >= tokensToFree {
break
}
}
remaining := make([]chat.Message, 0, len(messages))
remaining = append(remaining, systemMsg)
for i := removeUpTo; i < len(groups); i++ {
remaining = append(remaining, groups[i]...)
}
remaining = append(remaining, lastMsg)
return remaining
}
// groupToolMessages groups middle messages into logical units:
// - An assistant message with tool_calls + its corresponding tool result messages = one group
// - A standalone message (user, assistant without tool_calls) = one group
//
// This ensures tool_call/tool_result pairs are never split during compression.
func groupToolMessages(messages []chat.Message) [][]chat.Message {
var groups [][]chat.Message
i := 0
for i < len(messages) {
msg := messages[i]
// If this is an assistant message with tool_calls, group it with following tool results
if msg.Role == "assistant" && len(msg.ToolCalls) > 0 {
group := []chat.Message{msg}
i++
// Collect all following tool result messages
for i < len(messages) && messages[i].Role == "tool" {
group = append(group, messages[i])
i++
}
groups = append(groups, group)
} else {
groups = append(groups, []chat.Message{msg})
i++
}
}
return groups
}