feat(agent): enhance token management and context handling in AgentEngine
- Added token usage tracking to the AgentEngine, allowing for better estimation of current context token counts. - Implemented a new method to estimate current tokens based on previous usage and newly appended messages, improving context management. - Updated context window management to utilize estimated token counts for more efficient message consolidation. - Enhanced streaming methods to include token usage information, providing better insights into LLM interactions. - Refactored the token estimator to support BPE tokenization, ensuring accurate token counts for messages. - Added unit tests to validate new token estimation and context management functionalities.
This commit is contained in:
@@ -114,6 +114,7 @@ require (
|
||||
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dlclark/regexp2 v1.11.5 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/duckdb/duckdb-go-bindings v0.1.24 // indirect
|
||||
github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.24 // indirect
|
||||
@@ -240,6 +241,7 @@ require (
|
||||
github.com/tidwall/gjson v1.17.1 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tiktoken-go/tokenizer v0.7.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
|
||||
|
||||
@@ -1562,6 +1562,8 @@ github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
|
||||
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
|
||||
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
|
||||
github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94=
|
||||
@@ -2434,6 +2436,8 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tiktoken-go/tokenizer v0.7.0 h1:VMu6MPT0bXFDHr7UPh9uii7CNItVt3X9K90omxL54vw=
|
||||
github.com/tiktoken-go/tokenizer v0.7.0/go.mod h1:6UCYI/DtOallbmL7sSy30p6YQv60qNyU/4aVigPOx6w=
|
||||
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
|
||||
github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
|
||||
@@ -36,6 +36,8 @@ type AgentEngine struct {
|
||||
imageDescriber ImageDescriberFunc // VLM function for describing images in tool results (optional)
|
||||
tokenEstimator *agenttoken.Estimator // Token estimator for context window management
|
||||
memoryConsolidator *agentmemory.Consolidator // Memory consolidator for LLM-powered summarization (optional)
|
||||
lastUsage types.TokenUsage // Token usage from the most recent LLM call
|
||||
lastSentMsgCount int // Number of messages sent in the most recent LLM call
|
||||
}
|
||||
|
||||
// ImageDescriberFunc generates a text description of an image.
|
||||
@@ -57,7 +59,10 @@ func NewAgentEngine(
|
||||
if eventBus == nil {
|
||||
eventBus = event.NewEventBus()
|
||||
}
|
||||
tokenEst := agenttoken.NewEstimator(0)
|
||||
tokenEst, err := agenttoken.NewEstimator()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
engine := &AgentEngine{
|
||||
config: config,
|
||||
toolRegistry: toolRegistry,
|
||||
@@ -133,6 +138,18 @@ func (e *AgentEngine) GetSkillsManager() *skills.Manager {
|
||||
return e.skillsManager
|
||||
}
|
||||
|
||||
// estimateCurrentTokens returns the best estimate of the current context token count.
|
||||
// When API-reported usage from a previous round is available, it uses that as a
|
||||
// baseline and only BPE-estimates the delta (newly appended messages). Otherwise it
|
||||
// falls back to a full BPE estimation of all messages.
|
||||
func (e *AgentEngine) estimateCurrentTokens(messages []chat.Message) int {
|
||||
if e.lastUsage.TotalTokens > 0 && e.lastSentMsgCount > 0 && e.lastSentMsgCount < len(messages) {
|
||||
delta := e.tokenEstimator.EstimateMessages(messages[e.lastSentMsgCount:])
|
||||
return e.lastUsage.TotalTokens + delta
|
||||
}
|
||||
return e.tokenEstimator.EstimateMessages(messages)
|
||||
}
|
||||
|
||||
// Execute executes the agent with conversation history and streaming output
|
||||
// All events are emitted to EventBus and handled by subscribers (like Handler layer)
|
||||
func (e *AgentEngine) Execute(
|
||||
@@ -272,11 +289,14 @@ func (e *AgentEngine) executeLoop(
|
||||
|
||||
roundStart := time.Now()
|
||||
|
||||
// Context window management
|
||||
messages = e.manageContextWindow(ctx, messages, state.CurrentRound+1)
|
||||
// Context window management: estimate current token count using
|
||||
// the API-reported usage from the previous round plus a BPE delta
|
||||
// for newly appended messages (assistant reply + tool results).
|
||||
currentTokens := e.estimateCurrentTokens(messages)
|
||||
messages = e.manageContextWindow(ctx, messages, state.CurrentRound+1, currentTokens)
|
||||
|
||||
logger.Infof(ctx, "[Agent][Round-%d/%d] Starting: %d messages, %d tools",
|
||||
state.CurrentRound+1, e.config.MaxIterations, len(messages), len(tools))
|
||||
logger.Infof(ctx, "[Agent][Round-%d/%d] Starting: %d messages, %d tools, est_tokens=%d",
|
||||
state.CurrentRound+1, e.config.MaxIterations, len(messages), len(tools), currentTokens)
|
||||
common.PipelineInfo(ctx, "Agent", "round_start", map[string]interface{}{
|
||||
"iteration": state.CurrentRound,
|
||||
"round": state.CurrentRound + 1,
|
||||
@@ -286,14 +306,20 @@ func (e *AgentEngine) executeLoop(
|
||||
})
|
||||
|
||||
// 1. Think: Call LLM with function calling (includes retry + graceful degradation)
|
||||
e.lastSentMsgCount = len(messages)
|
||||
response, err := e.callLLMWithRetry(ctx, messages, tools, state, query, state.CurrentRound, sessionID)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
if response == nil {
|
||||
// Graceful degradation succeeded — state.IsComplete already set
|
||||
break
|
||||
}
|
||||
if response.Usage.TotalTokens > 0 {
|
||||
e.lastUsage = response.Usage
|
||||
logger.Debugf(ctx, "[Agent][Round-%d] Usage: prompt=%d, completion=%d, total=%d",
|
||||
state.CurrentRound+1, response.Usage.PromptTokens,
|
||||
response.Usage.CompletionTokens, response.Usage.TotalTokens)
|
||||
}
|
||||
|
||||
// Create agent step
|
||||
step := types.AgentStep{
|
||||
|
||||
@@ -87,7 +87,7 @@ Now generate the final answer:`, query)
|
||||
answerID := generateEventID("answer")
|
||||
logger.Debugf(ctx, "[Agent][FinalAnswer] AnswerID: %s", answerID)
|
||||
|
||||
fullAnswer, _, err := e.streamLLMToEventBus(
|
||||
llmResult, err := e.streamLLMToEventBus(
|
||||
ctx,
|
||||
messages,
|
||||
&chat.ChatOptions{Temperature: e.config.Temperature, Thinking: e.config.Thinking},
|
||||
@@ -95,7 +95,7 @@ Now generate the final answer:`, query)
|
||||
if chunk.Content != "" {
|
||||
logger.Debugf(ctx, "[Agent][FinalAnswer] Emitting answer chunk: %d chars", len(chunk.Content))
|
||||
e.eventBus.Emit(ctx, event.Event{
|
||||
ID: answerID, // Same ID for all chunks in this stream
|
||||
ID: answerID,
|
||||
Type: event.EventAgentFinalAnswer,
|
||||
SessionID: sessionID,
|
||||
Data: event.AgentFinalAnswerData{
|
||||
@@ -115,6 +115,7 @@ Now generate the final answer:`, query)
|
||||
return err
|
||||
}
|
||||
|
||||
fullAnswer := llmResult.Content
|
||||
logger.Infof(ctx, "[Agent][FinalAnswer] Final answer generated: %d characters", len(fullAnswer))
|
||||
common.PipelineInfo(ctx, "Agent", "final_answer_done", map[string]interface{}{
|
||||
"session_id": sessionID,
|
||||
|
||||
@@ -58,14 +58,14 @@ func NewConsolidator(
|
||||
}
|
||||
}
|
||||
|
||||
// ShouldConsolidate checks if the messages need consolidation based on token count.
|
||||
func (c *Consolidator) ShouldConsolidate(messages []chat.Message) bool {
|
||||
if c.maxTokens <= 0 || len(messages) <= 3 {
|
||||
// ShouldConsolidate checks if consolidation is needed based on the caller-provided
|
||||
// token estimate. The estimate should come from the model API's Usage when available.
|
||||
func (c *Consolidator) ShouldConsolidate(currentTokens int) bool {
|
||||
if c.maxTokens <= 0 {
|
||||
return false
|
||||
}
|
||||
tokenCount := c.estimator.EstimateMessages(messages)
|
||||
triggerAt := int(float64(c.maxTokens) * c.threshold)
|
||||
return tokenCount > triggerAt
|
||||
return currentTokens > triggerAt
|
||||
}
|
||||
|
||||
// Consolidate summarizes older messages and returns a compressed message array.
|
||||
|
||||
@@ -9,7 +9,8 @@ import (
|
||||
)
|
||||
|
||||
func TestConsolidator_ShouldConsolidate(t *testing.T) {
|
||||
est := token.NewEstimator(0)
|
||||
est, err := token.NewEstimator()
|
||||
assert.NoError(t, err)
|
||||
|
||||
t.Run("below threshold returns false", func(t *testing.T) {
|
||||
c := NewConsolidator(nil, est, 100000, 0)
|
||||
@@ -19,27 +20,18 @@ func TestConsolidator_ShouldConsolidate(t *testing.T) {
|
||||
{Role: "assistant", Content: "hi"},
|
||||
{Role: "user", Content: "current"},
|
||||
}
|
||||
assert.False(t, c.ShouldConsolidate(messages))
|
||||
tokens := est.EstimateMessages(messages)
|
||||
assert.False(t, c.ShouldConsolidate(tokens))
|
||||
})
|
||||
|
||||
t.Run("too few messages returns false", func(t *testing.T) {
|
||||
c := NewConsolidator(nil, est, 10, 0)
|
||||
messages := []chat.Message{
|
||||
{Role: "system", Content: "system"},
|
||||
{Role: "user", Content: "hello"},
|
||||
}
|
||||
assert.False(t, c.ShouldConsolidate(messages))
|
||||
t.Run("over threshold returns true", func(t *testing.T) {
|
||||
c := NewConsolidator(nil, est, 10, 0) // very low threshold
|
||||
assert.True(t, c.ShouldConsolidate(100))
|
||||
})
|
||||
|
||||
t.Run("disabled returns false", func(t *testing.T) {
|
||||
c := NewConsolidator(nil, est, 0, 0) // disabled
|
||||
messages := []chat.Message{
|
||||
{Role: "system", Content: "system"},
|
||||
{Role: "user", Content: "hello"},
|
||||
{Role: "assistant", Content: "hi"},
|
||||
{Role: "user", Content: "current"},
|
||||
}
|
||||
assert.False(t, c.ShouldConsolidate(messages))
|
||||
c := NewConsolidator(nil, est, 0, 0)
|
||||
assert.False(t, c.ShouldConsolidate(99999))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -61,7 +53,8 @@ func TestTruncateForPrompt(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRawArchive(t *testing.T) {
|
||||
est := token.NewEstimator(0)
|
||||
est, err := token.NewEstimator()
|
||||
assert.NoError(t, err)
|
||||
c := NewConsolidator(nil, est, 100000, 0)
|
||||
|
||||
messages := []chat.Message{
|
||||
@@ -83,7 +76,8 @@ func TestRawArchive(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBuildConsolidationPrompt(t *testing.T) {
|
||||
est := token.NewEstimator(0)
|
||||
est, err := token.NewEstimator()
|
||||
assert.NoError(t, err)
|
||||
c := NewConsolidator(nil, est, 100000, 0)
|
||||
|
||||
messages := []chat.Message{
|
||||
|
||||
@@ -16,29 +16,29 @@ import (
|
||||
)
|
||||
|
||||
// manageContextWindow consolidates or compresses messages if approaching the token limit.
|
||||
// It returns the (possibly shortened) message slice.
|
||||
func (e *AgentEngine) manageContextWindow(ctx context.Context, messages []chat.Message, round int) []chat.Message {
|
||||
// currentTokens is the caller's best estimate of the current context size (using
|
||||
// API-reported Usage when available, falling back to BPE estimation).
|
||||
func (e *AgentEngine) manageContextWindow(ctx context.Context, messages []chat.Message, round, currentTokens int) []chat.Message {
|
||||
if e.config.MaxContextTokens <= 0 {
|
||||
return messages
|
||||
}
|
||||
|
||||
beforeLen := len(messages)
|
||||
|
||||
// Try LLM-powered consolidation first (preserves more semantic content)
|
||||
if e.memoryConsolidator != nil && e.memoryConsolidator.ShouldConsolidate(messages) {
|
||||
logger.Infof(ctx, "[Agent][Round-%d] Token threshold exceeded, consolidating memory",
|
||||
round)
|
||||
if e.memoryConsolidator != nil && e.memoryConsolidator.ShouldConsolidate(currentTokens) {
|
||||
logger.Infof(ctx, "[Agent][Round-%d] Token threshold exceeded (est=%d), consolidating memory",
|
||||
round, currentTokens)
|
||||
consolidated, consolidateErr := e.memoryConsolidator.Consolidate(ctx, messages)
|
||||
if consolidateErr != nil {
|
||||
logger.Warnf(ctx, "[Agent][Round-%d] Memory consolidation failed: %v, "+
|
||||
"falling back to simple compression", round, consolidateErr)
|
||||
} else {
|
||||
messages = consolidated
|
||||
currentTokens = e.tokenEstimator.EstimateMessages(messages)
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: simple context compression if still over the hard limit (80%)
|
||||
messages = agenttoken.CompressContext(messages, e.tokenEstimator, e.config.MaxContextTokens)
|
||||
messages = agenttoken.CompressContext(messages, e.tokenEstimator, e.config.MaxContextTokens, currentTokens)
|
||||
|
||||
if len(messages) < beforeLen {
|
||||
logger.Infof(ctx, "[Agent][Round-%d] Context managed: %d → %d messages (max_tokens=%d)",
|
||||
|
||||
+31
-28
@@ -13,58 +13,59 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// streamLLMResult holds accumulated output from a streaming LLM call.
|
||||
type streamLLMResult struct {
|
||||
Content string
|
||||
ToolCalls []types.LLMToolCall
|
||||
Usage *types.TokenUsage
|
||||
}
|
||||
|
||||
// streamLLMToEventBus streams LLM response through EventBus (generic method)
|
||||
// emitFunc: callback to emit each chunk event
|
||||
// Returns: full accumulated content, tool calls (if any), error
|
||||
func (e *AgentEngine) streamLLMToEventBus(
|
||||
ctx context.Context,
|
||||
messages []chat.Message,
|
||||
opts *chat.ChatOptions,
|
||||
emitFunc func(chunk *types.StreamResponse, fullContent string),
|
||||
) (string, []types.LLMToolCall, error) {
|
||||
) (*streamLLMResult, error) {
|
||||
logger.Debugf(ctx, "[Agent][Stream] Starting LLM stream with %d messages", len(messages))
|
||||
|
||||
// Create a per-call timeout context so that a single LLM call gets a
|
||||
// guaranteed time window even when the parent context's deadline is almost
|
||||
// exhausted after previous iterations. The shorter of the two deadlines wins,
|
||||
// so the parent pipeline timeout is still respected.
|
||||
llmCtx, llmCancel := context.WithTimeout(ctx, e.getLLMCallTimeout())
|
||||
defer llmCancel()
|
||||
|
||||
stream, err := e.chatModel.ChatStream(llmCtx, messages, opts)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "[Agent][Stream] Failed to start LLM stream: %v", err)
|
||||
return "", nil, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fullContent := ""
|
||||
var toolCalls []types.LLMToolCall
|
||||
result := &streamLLMResult{}
|
||||
chunkCount := 0
|
||||
|
||||
for chunk := range stream {
|
||||
chunkCount++
|
||||
|
||||
if chunk.Content != "" {
|
||||
// Only accumulate LLM's own text output, not content extracted from tool arguments
|
||||
// (e.g. final_answer's answer or thinking tool's thought are streamed separately)
|
||||
isExtracted := chunk.Data != nil && chunk.Data["source"] != nil
|
||||
if !isExtracted {
|
||||
fullContent += chunk.Content
|
||||
result.Content += chunk.Content
|
||||
}
|
||||
}
|
||||
|
||||
// Collect tool calls if present
|
||||
if len(chunk.ToolCalls) > 0 {
|
||||
toolCalls = chunk.ToolCalls
|
||||
result.ToolCalls = chunk.ToolCalls
|
||||
}
|
||||
|
||||
if chunk.Usage != nil {
|
||||
result.Usage = chunk.Usage
|
||||
}
|
||||
|
||||
// Emit event through callback
|
||||
if emitFunc != nil {
|
||||
emitFunc(&chunk, fullContent)
|
||||
emitFunc(&chunk, result.Content)
|
||||
}
|
||||
}
|
||||
|
||||
return fullContent, toolCalls, nil
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// streamReflectionToEventBus streams reflection process through EventBus
|
||||
@@ -93,14 +94,14 @@ Think:
|
||||
// Generate a single ID for this entire reflection stream
|
||||
reflectionID := generateEventID("reflection")
|
||||
|
||||
fullReflection, _, err := e.streamLLMToEventBus(
|
||||
llmResult, err := e.streamLLMToEventBus(
|
||||
ctx,
|
||||
messages,
|
||||
&chat.ChatOptions{Temperature: 0.5},
|
||||
func(chunk *types.StreamResponse, fullContent string) {
|
||||
if chunk.Content != "" {
|
||||
e.eventBus.Emit(ctx, event.Event{
|
||||
ID: reflectionID, // Same ID for all chunks in this stream
|
||||
ID: reflectionID,
|
||||
Type: event.EventAgentReflection,
|
||||
SessionID: sessionID,
|
||||
Data: event.AgentReflectionData{
|
||||
@@ -118,7 +119,7 @@ Think:
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fullReflection, nil
|
||||
return llmResult.Content, nil
|
||||
}
|
||||
|
||||
// streamThinkingToEventBus streams the thinking process through EventBus
|
||||
@@ -145,7 +146,7 @@ func (e *AgentEngine) streamThinkingToEventBus(
|
||||
thinkingID := generateEventID("thinking")
|
||||
answerID := generateEventID("answer")
|
||||
|
||||
fullContent, toolCalls, err := e.streamLLMToEventBus(
|
||||
llmResult, err := e.streamLLMToEventBus(
|
||||
ctx,
|
||||
messages,
|
||||
opts,
|
||||
@@ -230,17 +231,19 @@ func (e *AgentEngine) streamThinkingToEventBus(
|
||||
}
|
||||
|
||||
logger.Debugf(ctx, "[Agent][Thinking] Iteration-%d completed: content=%d chars, tool_calls=%d",
|
||||
iteration+1, len(fullContent), len(toolCalls))
|
||||
iteration+1, len(llmResult.Content), len(llmResult.ToolCalls))
|
||||
|
||||
// Strip <think>…</think> blocks that some models (DeepSeek, Qwen, etc.) embed in content.
|
||||
fullContent = agenttools.StripThinkBlocks(fullContent)
|
||||
fullContent := agenttools.StripThinkBlocks(llmResult.Content)
|
||||
|
||||
// Build response
|
||||
return &types.ChatResponse{
|
||||
resp := &types.ChatResponse{
|
||||
Content: fullContent,
|
||||
ToolCalls: toolCalls,
|
||||
ToolCalls: llmResult.ToolCalls,
|
||||
FinishReason: "stop",
|
||||
}, nil
|
||||
}
|
||||
if llmResult.Usage != nil {
|
||||
resp.Usage = *llmResult.Usage
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// callLLMWithRetry logs messages, sanitizes them, calls the LLM with retry on transient errors,
|
||||
|
||||
@@ -9,39 +9,35 @@ import (
|
||||
const DefaultContextThresholdRatio = 0.8
|
||||
|
||||
// CompressContext trims older messages to bring total token count below the threshold.
|
||||
// currentTokens is the caller's best estimate of the current context size (from API
|
||||
// Usage when available, falling back to BPE estimation). The estimator is still used
|
||||
// internally for per-message-group cost calculation during trimming.
|
||||
//
|
||||
// It preserves:
|
||||
// - The first message (system prompt)
|
||||
// - The last message (current user query)
|
||||
// - tool_call / tool_result message pairs (never splits them)
|
||||
//
|
||||
// Returns the compressed message slice. If no compression is needed, returns the original.
|
||||
func CompressContext(
|
||||
messages []chat.Message,
|
||||
estimator *Estimator,
|
||||
maxTokens int,
|
||||
currentTokens int,
|
||||
) []chat.Message {
|
||||
if maxTokens <= 0 || len(messages) <= 2 {
|
||||
return messages
|
||||
}
|
||||
|
||||
threshold := int(float64(maxTokens) * DefaultContextThresholdRatio)
|
||||
currentTokens := estimator.EstimateMessages(messages)
|
||||
if currentTokens <= threshold {
|
||||
return messages
|
||||
}
|
||||
|
||||
// Strategy: keep system prompt (first) + current query (last),
|
||||
// trim from the oldest messages inward, respecting tool_call/tool_result boundaries.
|
||||
|
||||
systemMsg := messages[0]
|
||||
lastMsg := messages[len(messages)-1]
|
||||
middle := messages[1 : len(messages)-1]
|
||||
|
||||
// Find groups of messages that form tool_call/tool_result pairs.
|
||||
// A group = one assistant message with tool_calls + all following tool result messages.
|
||||
groups := groupToolMessages(middle)
|
||||
|
||||
// Remove groups from the front until we're under the threshold.
|
||||
tokensToFree := currentTokens - threshold
|
||||
freed := 0
|
||||
removeUpTo := 0
|
||||
@@ -58,7 +54,6 @@ func CompressContext(
|
||||
}
|
||||
}
|
||||
|
||||
// Rebuild messages: system + remaining groups + current query
|
||||
remaining := make([]chat.Message, 0, len(messages))
|
||||
remaining = append(remaining, systemMsg)
|
||||
for i := removeUpTo; i < len(groups); i++ {
|
||||
|
||||
@@ -1,116 +1,87 @@
|
||||
// Package token provides token estimation for LLM context window management.
|
||||
//
|
||||
// The primary source of truth for token counts is the model API's Usage field
|
||||
// (types.TokenUsage), returned with every LLM response. This package serves as
|
||||
// a supplementary estimator used in two scenarios:
|
||||
//
|
||||
// 1. Delta estimation — after an LLM call, new messages (assistant reply +
|
||||
// tool results) are appended before the next call. The Estimator computes
|
||||
// the token cost of these new messages so the engine can decide whether
|
||||
// context compression is needed without making an extra API round-trip.
|
||||
//
|
||||
// 2. First-round fallback — on the very first round of a session, no prior
|
||||
// API Usage is available, so the Estimator provides a full estimate.
|
||||
//
|
||||
// The encoding used (cl100k_base) is an approximation. Different model families
|
||||
// use different tokenizers, so the numbers will not be exact for non-OpenAI
|
||||
// models. This is acceptable because the estimate only needs to be close enough
|
||||
// to trigger compression at roughly the right time; over- or under-estimating
|
||||
// by a small margin is corrected on the next API call.
|
||||
package token
|
||||
|
||||
import (
|
||||
"unicode"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/tiktoken-go/tokenizer"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultCharsPerToken is a blended ratio for mixed English/CJK text.
|
||||
// English ≈ 4 chars/token, CJK ≈ 1.5 chars/token, blended ≈ 3.5.
|
||||
DefaultCharsPerToken = 3.5
|
||||
|
||||
// perMessageOverhead is the estimated token overhead per message
|
||||
// (role label, formatting, separator tokens).
|
||||
perMessageOverhead = 4
|
||||
perMessageOverhead = 3
|
||||
perConversationTail = 3
|
||||
)
|
||||
|
||||
// Estimator estimates token counts for messages and strings.
|
||||
// Uses a character-based heuristic that accounts for CJK characters.
|
||||
// Estimator counts tokens for messages and strings using BPE tokenization.
|
||||
// It is intended for incremental (delta) estimation between LLM calls; the
|
||||
// authoritative token count comes from the API's Usage response.
|
||||
type Estimator struct {
|
||||
charsPerToken float64
|
||||
codec tokenizer.Codec
|
||||
}
|
||||
|
||||
// NewEstimator creates a token estimator.
|
||||
// If charsPerToken <= 0, DefaultCharsPerToken is used.
|
||||
func NewEstimator(charsPerToken float64) *Estimator {
|
||||
if charsPerToken <= 0 {
|
||||
charsPerToken = DefaultCharsPerToken
|
||||
// NewEstimator creates a token estimator using the cl100k_base encoding.
|
||||
func NewEstimator() (*Estimator, error) {
|
||||
codec, err := tokenizer.Get(tokenizer.Cl100kBase)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("token: failed to initialize tokenizer: %w", err)
|
||||
}
|
||||
return &Estimator{charsPerToken: charsPerToken}
|
||||
return &Estimator{codec: codec}, nil
|
||||
}
|
||||
|
||||
// EstimateString estimates the token count for a string.
|
||||
// It uses a CJK-aware heuristic: CJK characters are counted at ~1.5 chars/token,
|
||||
// while ASCII/Latin characters use the configured charsPerToken ratio.
|
||||
func (e *Estimator) EstimateString(s string) int {
|
||||
if len(s) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
var cjkRunes, otherRunes int
|
||||
for _, r := range s {
|
||||
if isCJK(r) {
|
||||
cjkRunes++
|
||||
} else {
|
||||
otherRunes++
|
||||
}
|
||||
}
|
||||
|
||||
// CJK characters are roughly 1.5 chars per token
|
||||
cjkTokens := float64(cjkRunes) / 1.5
|
||||
otherTokens := float64(otherRunes) / e.charsPerToken
|
||||
|
||||
return int(cjkTokens+otherTokens) + 1 // +1 to avoid underestimation
|
||||
}
|
||||
|
||||
// EstimateMessages estimates the total token count for a slice of chat messages.
|
||||
// EstimateMessages returns the estimated token count for a slice of messages.
|
||||
// Prefer using API Usage for the full context and this method only for deltas.
|
||||
func (e *Estimator) EstimateMessages(messages []chat.Message) int {
|
||||
total := 0
|
||||
for i := range messages {
|
||||
total += e.EstimateMessage(&messages[i])
|
||||
}
|
||||
total += perConversationTail
|
||||
return total
|
||||
}
|
||||
|
||||
// EstimateMessage estimates the token count for a single message.
|
||||
// EstimateString returns the token count for a string using BPE tokenization.
|
||||
func (e *Estimator) EstimateString(s string) int {
|
||||
if len(s) == 0 {
|
||||
return 0
|
||||
}
|
||||
ids, _, err := e.codec.Encode(s)
|
||||
if err != nil {
|
||||
return (len(s) + 3) / 4
|
||||
}
|
||||
return len(ids)
|
||||
}
|
||||
|
||||
// EstimateMessage returns the token count for a single message.
|
||||
func (e *Estimator) EstimateMessage(msg *chat.Message) int {
|
||||
tokens := perMessageOverhead
|
||||
tokens += e.EstimateString(msg.Role)
|
||||
tokens += e.EstimateString(msg.Content)
|
||||
tokens += e.EstimateString(msg.Name)
|
||||
|
||||
// Tool calls in assistant messages
|
||||
for _, tc := range msg.ToolCalls {
|
||||
tokens += e.EstimateString(tc.Function.Name)
|
||||
tokens += e.EstimateString(tc.Function.Arguments)
|
||||
tokens += 4 // overhead for tool call structure
|
||||
tokens += 4
|
||||
}
|
||||
|
||||
return tokens
|
||||
}
|
||||
|
||||
// isCJK checks if a rune is a CJK (Chinese, Japanese, Korean) character.
|
||||
func isCJK(r rune) bool {
|
||||
// CJK Unified Ideographs
|
||||
if r >= 0x4E00 && r <= 0x9FFF {
|
||||
return true
|
||||
}
|
||||
// CJK Unified Ideographs Extension A
|
||||
if r >= 0x3400 && r <= 0x4DBF {
|
||||
return true
|
||||
}
|
||||
// CJK Unified Ideographs Extension B
|
||||
if r >= 0x20000 && r <= 0x2A6DF {
|
||||
return true
|
||||
}
|
||||
// CJK Compatibility Ideographs
|
||||
if r >= 0xF900 && r <= 0xFAFF {
|
||||
return true
|
||||
}
|
||||
// Hiragana + Katakana
|
||||
if r >= 0x3040 && r <= 0x30FF {
|
||||
return true
|
||||
}
|
||||
// Hangul Syllables
|
||||
if r >= 0xAC00 && r <= 0xD7AF {
|
||||
return true
|
||||
}
|
||||
// Full-width characters
|
||||
if unicode.Is(unicode.Han, r) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -12,34 +12,34 @@ import (
|
||||
)
|
||||
|
||||
func TestEstimator(t *testing.T) {
|
||||
e := NewEstimator(0) // use default
|
||||
e, err := NewEstimator()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create estimator: %v", err)
|
||||
}
|
||||
|
||||
t.Run("empty string", func(t *testing.T) {
|
||||
assert.Equal(t, 0, e.EstimateString(""))
|
||||
})
|
||||
|
||||
t.Run("english text", func(t *testing.T) {
|
||||
// "hello world" = 11 chars ≈ 11/3.5 + 1 ≈ 4 tokens
|
||||
tokens := e.EstimateString("hello world")
|
||||
assert.Greater(t, tokens, 0)
|
||||
assert.Less(t, tokens, 20)
|
||||
assert.Equal(t, 2, tokens)
|
||||
})
|
||||
|
||||
t.Run("chinese text", func(t *testing.T) {
|
||||
// 10 Chinese chars at ~1.5 chars/token ≈ 7 tokens
|
||||
tokens := e.EstimateString("你好世界测试数据中文")
|
||||
assert.Greater(t, tokens, 3)
|
||||
assert.Less(t, tokens, 20)
|
||||
fmt.Println(tokens)
|
||||
assert.Greater(t, tokens, 0)
|
||||
})
|
||||
|
||||
t.Run("CJK is more tokens per char than latin", func(t *testing.T) {
|
||||
// Same number of characters, but CJK should estimate more tokens
|
||||
t.Run("CJK produces more tokens per char than latin", func(t *testing.T) {
|
||||
latin := strings.Repeat("a", 100)
|
||||
cjk := strings.Repeat("中", 100)
|
||||
latinTokens := e.EstimateString(latin)
|
||||
cjkTokens := e.EstimateString(cjk)
|
||||
fmt.Println(latinTokens, cjkTokens)
|
||||
assert.Greater(t, cjkTokens, latinTokens,
|
||||
"CJK text should estimate more tokens per character")
|
||||
"CJK text should produce more tokens per character")
|
||||
})
|
||||
|
||||
t.Run("message estimation includes overhead", func(t *testing.T) {
|
||||
@@ -49,6 +49,7 @@ func TestEstimator(t *testing.T) {
|
||||
}
|
||||
tokens := e.EstimateMessage(&msg)
|
||||
contentTokens := e.EstimateString("hello")
|
||||
fmt.Println(tokens, contentTokens)
|
||||
assert.Greater(t, tokens, contentTokens,
|
||||
"message tokens should include overhead beyond just content")
|
||||
})
|
||||
@@ -82,26 +83,28 @@ func TestEstimator(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCompressContext(t *testing.T) {
|
||||
e := NewEstimator(0)
|
||||
e, err := NewEstimator()
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("no compression needed", func(t *testing.T) {
|
||||
messages := []chat.Message{
|
||||
{Role: "system", Content: "system prompt"},
|
||||
{Role: "user", Content: "hello"},
|
||||
}
|
||||
result := CompressContext(messages, e, 100000)
|
||||
tokens := e.EstimateMessages(messages)
|
||||
result := CompressContext(messages, e, 100000, tokens)
|
||||
assert.Equal(t, messages, result)
|
||||
})
|
||||
|
||||
t.Run("preserves system and last message", func(t *testing.T) {
|
||||
// Create messages that exceed the token limit
|
||||
messages := []chat.Message{
|
||||
{Role: "system", Content: "system prompt"},
|
||||
{Role: "user", Content: strings.Repeat("old message ", 1000)},
|
||||
{Role: "assistant", Content: strings.Repeat("old response ", 1000)},
|
||||
{Role: "user", Content: "current query"},
|
||||
}
|
||||
result := CompressContext(messages, e, 100) // very low limit to force compression
|
||||
tokens := e.EstimateMessages(messages)
|
||||
result := CompressContext(messages, e, 100, tokens)
|
||||
require.GreaterOrEqual(t, len(result), 2)
|
||||
assert.Equal(t, "system", result[0].Role)
|
||||
assert.Equal(t, "current query", result[len(result)-1].Content)
|
||||
@@ -110,27 +113,23 @@ func TestCompressContext(t *testing.T) {
|
||||
t.Run("keeps tool_call and tool_result paired", func(t *testing.T) {
|
||||
messages := []chat.Message{
|
||||
{Role: "system", Content: "system prompt"},
|
||||
// Old conversation
|
||||
{Role: "user", Content: strings.Repeat("x", 500)},
|
||||
{Role: "assistant", Content: strings.Repeat("y", 500)},
|
||||
// Tool interaction that should stay together
|
||||
{Role: "assistant", Content: "thinking", ToolCalls: []chat.ToolCall{
|
||||
{ID: "call_1", Function: chat.FunctionCall{Name: "search", Arguments: `{"q":"test"}`}},
|
||||
}},
|
||||
{Role: "tool", Content: strings.Repeat("result ", 100), ToolCallID: "call_1", Name: "search"},
|
||||
// Current query
|
||||
{Role: "user", Content: "what did you find?"},
|
||||
}
|
||||
|
||||
result := CompressContext(messages, e, 200) // force some compression
|
||||
tokens := e.EstimateMessages(messages)
|
||||
result := CompressContext(messages, e, 200, tokens)
|
||||
|
||||
b, _ := json.Marshal(result)
|
||||
fmt.Println(string(b))
|
||||
|
||||
// Check that tool_call and tool_result are still paired
|
||||
for i, msg := range result {
|
||||
if msg.Role == "tool" {
|
||||
// The previous message should be an assistant with tool_calls
|
||||
require.Greater(t, i, 0)
|
||||
assert.Equal(t, "assistant", result[i-1].Role)
|
||||
assert.Greater(t, len(result[i-1].ToolCalls), 0)
|
||||
@@ -143,7 +142,7 @@ func TestCompressContext(t *testing.T) {
|
||||
{Role: "system", Content: "system"},
|
||||
{Role: "user", Content: "hello"},
|
||||
}
|
||||
result := CompressContext(messages, e, 0)
|
||||
result := CompressContext(messages, e, 0, 0)
|
||||
assert.Equal(t, messages, result)
|
||||
})
|
||||
|
||||
@@ -152,7 +151,7 @@ func TestCompressContext(t *testing.T) {
|
||||
{Role: "system", Content: "system"},
|
||||
{Role: "user", Content: "hello"},
|
||||
}
|
||||
result := CompressContext(messages, e, 1) // even with tiny limit
|
||||
result := CompressContext(messages, e, 1, 999)
|
||||
assert.Equal(t, messages, result)
|
||||
})
|
||||
}
|
||||
@@ -179,8 +178,8 @@ func TestGroupToolMessages(t *testing.T) {
|
||||
{Role: "user", Content: "thanks"},
|
||||
}
|
||||
groups := groupToolMessages(messages)
|
||||
assert.Len(t, groups, 3) // user, (assistant+2 tools), user
|
||||
assert.Len(t, groups[1], 3) // assistant + 2 tool results
|
||||
assert.Len(t, groups, 3)
|
||||
assert.Len(t, groups[1], 3)
|
||||
b, _ := json.Marshal(groups)
|
||||
fmt.Println(string(b))
|
||||
})
|
||||
|
||||
@@ -57,13 +57,13 @@ const (
|
||||
// All IM-related Redis keys are defined here for discoverability and to avoid
|
||||
// scattered string literals across multiple files.
|
||||
const (
|
||||
RedisKeyLeader = "im:ws:leader:" // + channelID — WebSocket leader election
|
||||
RedisKeyDedup = "im:dedup:" // + messageID — message deduplication
|
||||
RedisKeyStop = "im:stop:" // + userKey — cross-instance /stop marker (pre-execution)
|
||||
RedisKeyInflight = "im:inflight:" // + userKey — maps userKey → sessionID:messageID for cross-instance /stop
|
||||
RedisKeyQueueUser = "im:queue:user:" // + userKey — global per-user queue counter
|
||||
RedisKeyRateLimit = "im:ratelimit:" // + key — sliding-window rate limiting
|
||||
RedisKeyGlobalGate = "im:global:active" // global concurrent worker counter
|
||||
RedisKeyLeader = "im:ws:leader:" // + channelID — WebSocket leader election
|
||||
RedisKeyDedup = "im:dedup:" // + messageID — message deduplication
|
||||
RedisKeyStop = "im:stop:" // + userKey — cross-instance /stop marker (pre-execution)
|
||||
RedisKeyInflight = "im:inflight:" // + userKey — maps userKey → sessionID:messageID for cross-instance /stop
|
||||
RedisKeyQueueUser = "im:queue:user:" // + userKey — global per-user queue counter
|
||||
RedisKeyRateLimit = "im:ratelimit:" // + key — sliding-window rate limiting
|
||||
RedisKeyGlobalGate = "im:global:active" // global concurrent worker counter
|
||||
)
|
||||
|
||||
// channelState holds runtime state for a running IM channel.
|
||||
|
||||
+10
-10
@@ -14,20 +14,20 @@ import (
|
||||
// IMChannel represents an IM channel configuration stored in the database.
|
||||
// Each channel binds to an agent and contains platform-specific credentials.
|
||||
type IMChannel struct {
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey;default:uuid_generate_v4()"`
|
||||
TenantID uint64 `json:"tenant_id" gorm:"not null;index:idx_im_channels_tenant"`
|
||||
AgentID string `json:"agent_id" gorm:"type:varchar(36);not null;index:idx_im_channels_agent"`
|
||||
Platform string `json:"platform" gorm:"type:varchar(20);not null"`
|
||||
Name string `json:"name" gorm:"type:varchar(255);not null;default:''"`
|
||||
Enabled bool `json:"enabled" gorm:"not null;default:true"`
|
||||
Mode string `json:"mode" gorm:"type:varchar(20);not null;default:'websocket'"`
|
||||
ID string `json:"id" gorm:"type:varchar(36);primaryKey;default:uuid_generate_v4()"`
|
||||
TenantID uint64 `json:"tenant_id" gorm:"not null;index:idx_im_channels_tenant"`
|
||||
AgentID string `json:"agent_id" gorm:"type:varchar(36);not null;index:idx_im_channels_agent"`
|
||||
Platform string `json:"platform" gorm:"type:varchar(20);not null"`
|
||||
Name string `json:"name" gorm:"type:varchar(255);not null;default:''"`
|
||||
Enabled bool `json:"enabled" gorm:"not null;default:true"`
|
||||
Mode string `json:"mode" gorm:"type:varchar(20);not null;default:'websocket'"`
|
||||
OutputMode string `json:"output_mode" gorm:"type:varchar(20);not null;default:'stream'"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"type:varchar(36);default:''"`
|
||||
BotIdentity string `json:"bot_identity" gorm:"type:varchar(255);not null;default:'';uniqueIndex:idx_im_channels_bot_identity,where:deleted_at IS NULL AND bot_identity != ''"`
|
||||
Credentials types.JSON `json:"credentials" gorm:"type:jsonb;not null;default:'{}'"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
|
||||
}
|
||||
|
||||
func (IMChannel) TableName() string {
|
||||
|
||||
@@ -160,11 +160,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
|
||||
return &types.ChatResponse{
|
||||
Content: responseContent,
|
||||
ToolCalls: toolCalls,
|
||||
Usage: struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}{
|
||||
Usage: types.TokenUsage{
|
||||
PromptTokens: promptTokens,
|
||||
CompletionTokens: completionTokens,
|
||||
TotalTokens: promptTokens + completionTokens,
|
||||
@@ -262,9 +258,18 @@ func (c *OllamaChat) ChatStream(
|
||||
}
|
||||
|
||||
if resp.Done {
|
||||
var usage *types.TokenUsage
|
||||
if resp.PromptEvalCount > 0 || resp.EvalCount > 0 {
|
||||
usage = &types.TokenUsage{
|
||||
PromptTokens: resp.PromptEvalCount,
|
||||
CompletionTokens: resp.EvalCount,
|
||||
TotalTokens: resp.PromptEvalCount + resp.EvalCount,
|
||||
}
|
||||
}
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Done: true,
|
||||
Usage: usage,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -152,6 +152,10 @@ func (c *RemoteAPIChat) BuildChatCompletionRequest(messages []Message, opts *Cha
|
||||
Stream: isStream,
|
||||
}
|
||||
|
||||
if isStream {
|
||||
req.StreamOptions = &openai.StreamOptions{IncludeUsage: true}
|
||||
}
|
||||
|
||||
if opts != nil {
|
||||
if opts.Temperature > 0 {
|
||||
req.Temperature = float32(opts.Temperature)
|
||||
@@ -316,11 +320,7 @@ func (c *RemoteAPIChat) parseCompletionResponse(resp *openai.ChatCompletionRespo
|
||||
response := &types.ChatResponse{
|
||||
Content: content,
|
||||
FinishReason: string(choice.FinishReason),
|
||||
Usage: struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}{
|
||||
Usage: types.TokenUsage{
|
||||
PromptTokens: resp.Usage.PromptTokens,
|
||||
CompletionTokens: resp.Usage.CompletionTokens,
|
||||
TotalTokens: resp.Usage.TotalTokens,
|
||||
@@ -465,6 +465,7 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo
|
||||
Content: "",
|
||||
Done: true,
|
||||
ToolCalls: state.buildOrderedToolCalls(),
|
||||
Usage: state.usage,
|
||||
}
|
||||
} else {
|
||||
streamChan <- types.StreamResponse{
|
||||
@@ -476,6 +477,14 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo
|
||||
return
|
||||
}
|
||||
|
||||
if response.Usage != nil {
|
||||
state.usage = &types.TokenUsage{
|
||||
PromptTokens: response.Usage.PromptTokens,
|
||||
CompletionTokens: response.Usage.CompletionTokens,
|
||||
TotalTokens: response.Usage.TotalTokens,
|
||||
}
|
||||
}
|
||||
|
||||
if len(response.Choices) > 0 {
|
||||
c.processStreamDelta(ctx, &response.Choices[0], state, streamChan, response.Choices[0].Delta.ReasoningContent)
|
||||
}
|
||||
@@ -514,6 +523,7 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res
|
||||
Content: "",
|
||||
Done: true,
|
||||
ToolCalls: state.buildOrderedToolCalls(),
|
||||
Usage: state.usage,
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -540,6 +550,14 @@ func (c *RemoteAPIChat) processRawHTTPStream(ctx context.Context, resp *http.Res
|
||||
continue
|
||||
}
|
||||
|
||||
if streamResp.Usage != nil {
|
||||
state.usage = &types.TokenUsage{
|
||||
PromptTokens: streamResp.Usage.PromptTokens,
|
||||
CompletionTokens: streamResp.Usage.CompletionTokens,
|
||||
TotalTokens: streamResp.Usage.TotalTokens,
|
||||
}
|
||||
}
|
||||
|
||||
if len(streamResp.Choices) > 0 {
|
||||
choice := streamResp.Choices[0]
|
||||
// 统一获取逻辑(支持标准和 vLLM 两种路径)
|
||||
@@ -566,6 +584,7 @@ type streamState struct {
|
||||
nameNotified map[int]bool
|
||||
hasThinking bool
|
||||
fieldExtractors map[int]*jsonFieldExtractor // per tool-call-index extractors for streaming field extraction
|
||||
usage *types.TokenUsage // captured from the final stream chunk when include_usage is enabled
|
||||
}
|
||||
|
||||
func newStreamState() *streamState {
|
||||
|
||||
+21
-32
@@ -5,6 +5,13 @@ import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
// TokenUsage holds token consumption statistics returned by the model API.
|
||||
type TokenUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}
|
||||
|
||||
// LLMToolCall represents a function/tool call from the LLM
|
||||
type LLMToolCall struct {
|
||||
ID string `json:"id"`
|
||||
@@ -20,20 +27,10 @@ type FunctionCall struct {
|
||||
|
||||
// ChatResponse chat response
|
||||
type ChatResponse struct {
|
||||
Content string `json:"content"`
|
||||
// Tool calls requested by the model
|
||||
ToolCalls []LLMToolCall `json:"tool_calls,omitempty"`
|
||||
// Finish reason
|
||||
FinishReason string `json:"finish_reason,omitempty"` // "stop", "tool_calls", "length", etc.
|
||||
// Usage information
|
||||
Usage struct {
|
||||
// Prompt tokens
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
// Completion tokens
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
// Total tokens
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
} `json:"usage"`
|
||||
Content string `json:"content"`
|
||||
ToolCalls []LLMToolCall `json:"tool_calls,omitempty"`
|
||||
FinishReason string `json:"finish_reason,omitempty"`
|
||||
Usage TokenUsage `json:"usage"`
|
||||
}
|
||||
|
||||
// Response type
|
||||
@@ -64,24 +61,16 @@ const (
|
||||
|
||||
// StreamResponse stream response
|
||||
type StreamResponse struct {
|
||||
// Unique identifier
|
||||
ID string `json:"id"`
|
||||
// Response type
|
||||
ResponseType ResponseType `json:"response_type"`
|
||||
// Current fragment content
|
||||
Content string `json:"content"`
|
||||
// Whether the response is complete
|
||||
Done bool `json:"done"`
|
||||
// Knowledge references
|
||||
KnowledgeReferences References `json:"knowledge_references,omitempty"`
|
||||
// Session ID (for agent_query event)
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
// Assistant Message ID (for agent_query event)
|
||||
AssistantMessageID string `json:"assistant_message_id,omitempty"`
|
||||
// Tool calls for streaming (partial)
|
||||
ToolCalls []LLMToolCall `json:"tool_calls,omitempty"`
|
||||
// Additional metadata for enhanced display
|
||||
Data map[string]interface{} `json:"data,omitempty"`
|
||||
ID string `json:"id"`
|
||||
ResponseType ResponseType `json:"response_type"`
|
||||
Content string `json:"content"`
|
||||
Done bool `json:"done"`
|
||||
KnowledgeReferences References `json:"knowledge_references,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
AssistantMessageID string `json:"assistant_message_id,omitempty"`
|
||||
ToolCalls []LLMToolCall `json:"tool_calls,omitempty"`
|
||||
Data map[string]interface{} `json:"data,omitempty"`
|
||||
Usage *TokenUsage `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
// References references
|
||||
|
||||
Reference in New Issue
Block a user