fix(agent): prevent empty response when no Knowledge Base is configured
- guard against empty content completion in analyzeResponse/executeLoop with retry (max 2), nudge message, and fallback response - propagate actual finish_reason through streaming pipeline instead of hardcoding "stop" in streamThinkingToEventBus - add FinishReason field to StreamResponse and streamLLMResult - add finish marker chunk in processStreamDelta for empty content scenarios - propagate lastFinishReason through EOF handler in processStream - strengthen PureAgent system prompt with MCP tool guidelines and graceful fallback instructions for unanswerable questions - add unit tests for empty response bug and finish_reason propagation
This commit is contained in:
@@ -25,14 +25,17 @@ templates:
|
||||
### Workflow
|
||||
1. **Analyze:** Understand the user's request.
|
||||
2. **Plan:** If the task is complex, use todo_write to create a plan.
|
||||
3. **Execute:** Use available tools to gather information or perform actions.
|
||||
3. **Execute:** Use available tools (including any connected MCP tools) to gather information or perform actions.
|
||||
After receiving tool results, analyze them and incorporate the findings into your answer.
|
||||
4. **Synthesize:** Call the final_answer tool with your comprehensive answer. You MUST always end by calling final_answer.
|
||||
|
||||
### Tool Guidelines
|
||||
* **MCP Tools:** If external MCP tools are available, use them to fulfill the user's request. Analyze and incorporate their results into your final answer.
|
||||
* **web_search / web_fetch:** Use these if enabled to find information from the internet.
|
||||
* **todo_write:** Use for managing multi-step tasks.
|
||||
* **thinking:** Use to plan and reflect.
|
||||
* **final_answer:** MANDATORY as your final action. Always submit your complete answer through this tool. NEVER end your turn without calling it.
|
||||
If you cannot fully answer, explain what you tried and why. If the question is outside your capabilities, say so politely.
|
||||
|
||||
### User-Friendly Communication
|
||||
In ALL outputs visible to users (including your thinking/reasoning), you MUST:
|
||||
|
||||
@@ -27,6 +27,13 @@ const (
|
||||
|
||||
// maxLLMRetries is the maximum number of retries for transient LLM errors.
|
||||
maxLLMRetries = 2
|
||||
|
||||
// maxEmptyResponseRetries is the maximum number of retries when the LLM
|
||||
// returns an empty content with a natural stop (no tool calls). This guards
|
||||
// against the agent completing with an empty answer when the LLM fails to
|
||||
// produce content (e.g., thinking-only loops without KB).
|
||||
// Trade-off: each retry costs ~2s of LLM latency; 2 retries = max 4s extra.
|
||||
maxEmptyResponseRetries = 2
|
||||
)
|
||||
|
||||
// transientErrorMarkers are substrings that indicate a transient (retryable) error.
|
||||
|
||||
@@ -270,6 +270,7 @@ func (e *AgentEngine) executeLoop(
|
||||
common.PipelineInfo(ctx, "Agent", "loop_start", map[string]interface{}{
|
||||
"max_iterations": e.config.MaxIterations,
|
||||
})
|
||||
emptyRetries := 0
|
||||
for state.CurrentRound < e.config.MaxIterations {
|
||||
// Check for context cancellation (request timeout, user cancel, etc.)
|
||||
select {
|
||||
@@ -336,6 +337,28 @@ func (e *AgentEngine) executeLoop(
|
||||
// 2. Analyze: Check for stop conditions (natural stop or final_answer tool)
|
||||
verdict := e.analyzeResponse(ctx, response, step, state.CurrentRound, sessionID, roundStart)
|
||||
if verdict.isDone {
|
||||
// Guard against empty content: when the LLM stops naturally with no
|
||||
// content and no tool calls (e.g., thinking-only loop without KB),
|
||||
// retry with a nudge message instead of accepting an empty answer.
|
||||
if verdict.emptyContent {
|
||||
emptyRetries++
|
||||
if emptyRetries <= maxEmptyResponseRetries {
|
||||
logger.Warnf(ctx, "[Agent][Round-%d] Empty content with stop - retrying (%d/%d)",
|
||||
state.CurrentRound+1, emptyRetries, maxEmptyResponseRetries)
|
||||
messages = append(messages, chat.Message{
|
||||
Role: "user",
|
||||
Content: "Please provide your answer by calling the final_answer tool.",
|
||||
})
|
||||
continue
|
||||
}
|
||||
// Retries exhausted — use fallback message rather than empty answer
|
||||
logger.Warnf(ctx, "[Agent][Round-%d] Empty content after %d retries - using fallback",
|
||||
state.CurrentRound+1, maxEmptyResponseRetries)
|
||||
state.FinalAnswer = "I'm sorry, I was unable to generate a response. Please try again."
|
||||
state.IsComplete = true
|
||||
state.RoundSteps = append(state.RoundSteps, verdict.step)
|
||||
break
|
||||
}
|
||||
state.FinalAnswer = verdict.finalAnswer
|
||||
state.IsComplete = true
|
||||
state.RoundSteps = append(state.RoundSteps, verdict.step)
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/event"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Mock: chat.Chat
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type mockResponse struct {
|
||||
chunks []types.StreamResponse
|
||||
}
|
||||
|
||||
type mockChat struct {
|
||||
mu sync.Mutex
|
||||
responses []mockResponse
|
||||
callCount int
|
||||
}
|
||||
|
||||
func (m *mockChat) ChatStream(_ context.Context, _ []chat.Message, _ *chat.ChatOptions) (<-chan types.StreamResponse, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.callCount >= len(m.responses) {
|
||||
return nil, fmt.Errorf("unexpected ChatStream call #%d (only %d responses prepared)", m.callCount, len(m.responses))
|
||||
}
|
||||
resp := m.responses[m.callCount]
|
||||
m.callCount++
|
||||
|
||||
ch := make(chan types.StreamResponse, len(resp.chunks))
|
||||
for _, chunk := range resp.chunks {
|
||||
ch <- chunk
|
||||
}
|
||||
close(ch)
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (m *mockChat) Chat(_ context.Context, _ []chat.Message, _ *chat.ChatOptions) (*types.ChatResponse, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (m *mockChat) GetModelName() string { return "mock-model" }
|
||||
func (m *mockChat) GetModelID() string { return "mock-id" }
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type testEngineOption func(*types.AgentConfig)
|
||||
|
||||
func withMaxIterations(n int) testEngineOption {
|
||||
return func(cfg *types.AgentConfig) {
|
||||
cfg.MaxIterations = n
|
||||
}
|
||||
}
|
||||
|
||||
func newTestEngine(t *testing.T, chatModel chat.Chat, opts ...testEngineOption) *AgentEngine {
|
||||
t.Helper()
|
||||
cfg := &types.AgentConfig{
|
||||
MaxIterations: 10,
|
||||
Temperature: 0.7,
|
||||
}
|
||||
for _, opt := range opts {
|
||||
opt(cfg)
|
||||
}
|
||||
engine := NewAgentEngine(
|
||||
cfg,
|
||||
chatModel,
|
||||
nil,
|
||||
event.NewEventBus(),
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
"test-session",
|
||||
"",
|
||||
)
|
||||
require.NotNil(t, engine, "NewAgentEngine returned nil (agenttoken.NewEstimator failed?)")
|
||||
return engine
|
||||
}
|
||||
|
||||
func emptyMessages() []chat.Message {
|
||||
return []chat.Message{
|
||||
{Role: "system", Content: "You are a test agent."},
|
||||
{Role: "user", Content: "test query"},
|
||||
}
|
||||
}
|
||||
|
||||
func emptyTools() []chat.Tool {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC1: Empty content + stop → should NOT complete with empty FinalAnswer
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestExecuteLoop_EmptyContentWithStop_ShouldNotCompleteWithEmpty(t *testing.T) {
|
||||
// Simulate: LLM returns empty content with no tool calls (natural stop).
|
||||
// The stream closes with no content chunks → streamLLMToEventBus returns fullContent="".
|
||||
// streamThinkingToEventBus wraps it as ChatResponse{Content:"", FinishReason:"stop"}.
|
||||
// analyzeResponse() returns verdict{isDone:true, finalAnswer:""} → BUG: empty answer.
|
||||
//
|
||||
// Prepare 3 responses for initial attempt + 2 retries (after fix).
|
||||
mock := &mockChat{
|
||||
responses: []mockResponse{
|
||||
{chunks: []types.StreamResponse{{Done: true}}},
|
||||
{chunks: []types.StreamResponse{{Done: true}}},
|
||||
{chunks: []types.StreamResponse{{Done: true}}},
|
||||
},
|
||||
}
|
||||
|
||||
engine := newTestEngine(t, mock)
|
||||
state := &types.AgentState{}
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, state.IsComplete)
|
||||
assert.NotEmpty(t, state.FinalAnswer,
|
||||
"BUG: FinalAnswer is empty when LLM returns empty content with stop. "+
|
||||
"analyzeResponse() should not allow empty content to be accepted as final answer.")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC2: Non-empty content + stop → normal completion (regression guard)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestExecuteLoop_NonEmptyContentWithStop_ShouldComplete(t *testing.T) {
|
||||
mock := &mockChat{
|
||||
responses: []mockResponse{
|
||||
{chunks: []types.StreamResponse{
|
||||
{Content: "Here is my answer", Done: true},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
engine := newTestEngine(t, mock)
|
||||
state := &types.AgentState{}
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, state.IsComplete)
|
||||
assert.Equal(t, "Here is my answer", state.FinalAnswer)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC4: Empty → retry with nudge → non-empty → success
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestExecuteLoop_EmptyThenNonEmpty_ShouldRetryAndComplete(t *testing.T) {
|
||||
mock := &mockChat{
|
||||
responses: []mockResponse{
|
||||
// Round 1: empty content → triggers retry + nudge
|
||||
{chunks: []types.StreamResponse{{Done: true}}},
|
||||
// Round 2: after nudge, LLM produces answer
|
||||
{chunks: []types.StreamResponse{
|
||||
{Content: "Here is the answer.", Done: true},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
engine := newTestEngine(t, mock)
|
||||
state := &types.AgentState{}
|
||||
ctx := context.Background()
|
||||
|
||||
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, state.IsComplete)
|
||||
assert.Equal(t, "Here is the answer.", state.FinalAnswer)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TC5: FinishReason propagation through streamThinkingToEventBus
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestStreamThinkingToEventBus_PropagatesFinishReason(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
finishReason string
|
||||
wantReason string
|
||||
}{
|
||||
{"stop", "stop", "stop"},
|
||||
{"tool_calls", "tool_calls", "tool_calls"},
|
||||
{"length", "length", "length"},
|
||||
{"empty_fallback", "", "stop"}, // empty FinishReason → fallback to "stop"
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
mock := &mockChat{
|
||||
responses: []mockResponse{
|
||||
{chunks: []types.StreamResponse{
|
||||
{Content: "test content", Done: true, FinishReason: tt.finishReason},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
engine := newTestEngine(t, mock)
|
||||
ctx := context.Background()
|
||||
msgs := []chat.Message{{Role: "user", Content: "test"}}
|
||||
tools := []chat.Tool{}
|
||||
|
||||
resp, err := engine.streamThinkingToEventBus(ctx, msgs, tools, 0, "sess-1")
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.wantReason, resp.FinishReason)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -51,9 +51,10 @@ func (e *AgentEngine) manageContextWindow(ctx context.Context, messages []chat.M
|
||||
// responseVerdict captures the result of analyzing an LLM response to determine
|
||||
// whether the agent loop should stop and what the final answer is (if any).
|
||||
type responseVerdict struct {
|
||||
isDone bool
|
||||
finalAnswer string
|
||||
step types.AgentStep
|
||||
isDone bool
|
||||
finalAnswer string
|
||||
emptyContent bool // LLM returned stop with no tool calls and empty content
|
||||
step types.AgentStep
|
||||
}
|
||||
|
||||
// analyzeResponse inspects the LLM response for stop conditions:
|
||||
@@ -102,9 +103,10 @@ func (e *AgentEngine) analyzeResponse(
|
||||
})
|
||||
|
||||
return responseVerdict{
|
||||
isDone: true,
|
||||
finalAnswer: response.Content,
|
||||
step: step,
|
||||
isDone: true,
|
||||
finalAnswer: response.Content,
|
||||
emptyContent: response.Content == "",
|
||||
step: step,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+17
-4
@@ -15,9 +15,10 @@ import (
|
||||
|
||||
// streamLLMResult holds accumulated output from a streaming LLM call.
|
||||
type streamLLMResult struct {
|
||||
Content string
|
||||
ToolCalls []types.LLMToolCall
|
||||
Usage *types.TokenUsage
|
||||
Content string
|
||||
ToolCalls []types.LLMToolCall
|
||||
Usage *types.TokenUsage
|
||||
FinishReason string // actual finish_reason from LLM (captured from last stream chunk)
|
||||
}
|
||||
|
||||
// streamLLMToEventBus streams LLM response through EventBus (generic method)
|
||||
@@ -66,6 +67,10 @@ func (e *AgentEngine) streamLLMToEventBus(
|
||||
result.Usage = chunk.Usage
|
||||
}
|
||||
|
||||
if chunk.FinishReason != "" {
|
||||
result.FinishReason = chunk.FinishReason
|
||||
}
|
||||
|
||||
if emitFunc != nil {
|
||||
emitFunc(&chunk, result.Content)
|
||||
}
|
||||
@@ -205,10 +210,18 @@ func (e *AgentEngine) streamThinkingToEventBus(
|
||||
|
||||
fullContent := agenttools.StripThinkBlocks(llmResult.Content)
|
||||
|
||||
// Use actual finish_reason from LLM stream instead of hardcoding "stop".
|
||||
// Fallback to "stop" when the stream did not report a finish_reason
|
||||
// (e.g., certain Ollama models or providers that omit the field).
|
||||
finishReason := llmResult.FinishReason
|
||||
if finishReason == "" {
|
||||
finishReason = "stop"
|
||||
}
|
||||
|
||||
resp := &types.ChatResponse{
|
||||
Content: fullContent,
|
||||
ToolCalls: llmResult.ToolCalls,
|
||||
FinishReason: "stop",
|
||||
FinishReason: finishReason,
|
||||
}
|
||||
if llmResult.Usage != nil {
|
||||
resp.Usage = *llmResult.Usage
|
||||
|
||||
@@ -510,6 +510,7 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo
|
||||
Done: true,
|
||||
ToolCalls: state.buildOrderedToolCalls(),
|
||||
Usage: state.usage,
|
||||
FinishReason: state.lastFinishReason,
|
||||
}
|
||||
} else {
|
||||
streamChan <- types.StreamResponse{
|
||||
@@ -633,6 +634,7 @@ type streamState struct {
|
||||
hasThinking bool
|
||||
fieldExtractors map[int]*jsonFieldExtractor // per tool-call-index extractors for streaming field extraction
|
||||
usage *types.TokenUsage // captured from the final stream chunk when include_usage is enabled
|
||||
lastFinishReason string // last observed finish_reason for EOF handler fallback
|
||||
}
|
||||
|
||||
func newStreamState() *streamState {
|
||||
@@ -666,6 +668,11 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C
|
||||
delta := choice.Delta
|
||||
isDone := string(choice.FinishReason) != ""
|
||||
|
||||
// Track finish_reason for EOF handler fallback
|
||||
if isDone {
|
||||
state.lastFinishReason = string(choice.FinishReason)
|
||||
}
|
||||
|
||||
// 处理 tool calls
|
||||
if len(delta.ToolCalls) > 0 {
|
||||
c.processToolCallsDelta(ctx, delta.ToolCalls, state, streamChan)
|
||||
@@ -698,6 +705,7 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C
|
||||
Content: delta.Content,
|
||||
Done: isDone,
|
||||
ToolCalls: state.buildOrderedToolCalls(),
|
||||
FinishReason: string(choice.FinishReason),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -707,6 +715,7 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C
|
||||
Content: "",
|
||||
Done: true,
|
||||
ToolCalls: state.buildOrderedToolCalls(),
|
||||
FinishReason: string(choice.FinishReason),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -720,6 +729,17 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C
|
||||
}
|
||||
state.hasThinking = false
|
||||
}
|
||||
|
||||
// Catch-all: isDone but none of the above branches sent a response with
|
||||
// FinishReason (empty content, no tool calls, no thinking). This prevents
|
||||
// the finish_reason from being lost in the streaming pipeline.
|
||||
if isDone && delta.Content == "" && len(state.toolCallMap) == 0 && !state.hasThinking {
|
||||
streamChan <- types.StreamResponse{
|
||||
ResponseType: types.ResponseTypeAnswer,
|
||||
Done: true,
|
||||
FinishReason: string(choice.FinishReason),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processToolCallsDelta 处理 tool calls 的增量更新
|
||||
|
||||
@@ -71,6 +71,7 @@ type StreamResponse struct {
|
||||
ToolCalls []LLMToolCall `json:"tool_calls,omitempty"`
|
||||
Data map[string]interface{} `json:"data,omitempty"`
|
||||
Usage *TokenUsage `json:"usage,omitempty"`
|
||||
FinishReason string `json:"finish_reason,omitempty"`
|
||||
}
|
||||
|
||||
// References references
|
||||
|
||||
Reference in New Issue
Block a user