0bb1a581de
- Added token usage tracking to the AgentEngine, allowing for better estimation of current context token counts. - Implemented a new method to estimate current tokens based on previous usage and newly appended messages, improving context management. - Updated context window management to utilize estimated token counts for more efficient message consolidation. - Enhanced streaming methods to include token usage information, providing better insights into LLM interactions. - Refactored the token estimator to support BPE tokenization, ensuring accurate token counts for messages. - Added unit tests to validate new token estimation and context management functionalities.
95 lines
2.7 KiB
Go
95 lines
2.7 KiB
Go
package token
|
|
|
|
import (
|
|
"github.com/Tencent/WeKnora/internal/models/chat"
|
|
)
|
|
|
|
// DefaultContextThresholdRatio is the ratio of context window usage that triggers compression.
|
|
// When message tokens exceed MaxContextTokens * threshold, old messages are trimmed.
|
|
const DefaultContextThresholdRatio = 0.8
|
|
|
|
// CompressContext trims older messages to bring total token count below the threshold.
|
|
// currentTokens is the caller's best estimate of the current context size (from API
|
|
// Usage when available, falling back to BPE estimation). The estimator is still used
|
|
// internally for per-message-group cost calculation during trimming.
|
|
//
|
|
// It preserves:
|
|
// - The first message (system prompt)
|
|
// - The last message (current user query)
|
|
// - tool_call / tool_result message pairs (never splits them)
|
|
func CompressContext(
|
|
messages []chat.Message,
|
|
estimator *Estimator,
|
|
maxTokens int,
|
|
currentTokens int,
|
|
) []chat.Message {
|
|
if maxTokens <= 0 || len(messages) <= 2 {
|
|
return messages
|
|
}
|
|
|
|
threshold := int(float64(maxTokens) * DefaultContextThresholdRatio)
|
|
if currentTokens <= threshold {
|
|
return messages
|
|
}
|
|
|
|
systemMsg := messages[0]
|
|
lastMsg := messages[len(messages)-1]
|
|
middle := messages[1 : len(messages)-1]
|
|
|
|
groups := groupToolMessages(middle)
|
|
|
|
tokensToFree := currentTokens - threshold
|
|
freed := 0
|
|
removeUpTo := 0
|
|
|
|
for i, group := range groups {
|
|
groupTokens := 0
|
|
for _, msg := range group {
|
|
groupTokens += estimator.EstimateMessage(&msg)
|
|
}
|
|
freed += groupTokens
|
|
removeUpTo = i + 1
|
|
if freed >= tokensToFree {
|
|
break
|
|
}
|
|
}
|
|
|
|
remaining := make([]chat.Message, 0, len(messages))
|
|
remaining = append(remaining, systemMsg)
|
|
for i := removeUpTo; i < len(groups); i++ {
|
|
remaining = append(remaining, groups[i]...)
|
|
}
|
|
remaining = append(remaining, lastMsg)
|
|
|
|
return remaining
|
|
}
|
|
|
|
// groupToolMessages groups middle messages into logical units:
|
|
// - An assistant message with tool_calls + its corresponding tool result messages = one group
|
|
// - A standalone message (user, assistant without tool_calls) = one group
|
|
//
|
|
// This ensures tool_call/tool_result pairs are never split during compression.
|
|
func groupToolMessages(messages []chat.Message) [][]chat.Message {
|
|
var groups [][]chat.Message
|
|
i := 0
|
|
for i < len(messages) {
|
|
msg := messages[i]
|
|
|
|
// If this is an assistant message with tool_calls, group it with following tool results
|
|
if msg.Role == "assistant" && len(msg.ToolCalls) > 0 {
|
|
group := []chat.Message{msg}
|
|
i++
|
|
// Collect all following tool result messages
|
|
for i < len(messages) && messages[i].Role == "tool" {
|
|
group = append(group, messages[i])
|
|
i++
|
|
}
|
|
groups = append(groups, group)
|
|
} else {
|
|
groups = append(groups, []chat.Message{msg})
|
|
i++
|
|
}
|
|
}
|
|
return groups
|
|
}
|