fix(agent): prevent empty response when no Knowledge Base is configured

- guard against empty content completion in analyzeResponse/executeLoop
  with retry (max 2), nudge message, and fallback response
- propagate actual finish_reason through streaming pipeline instead of
  hardcoding "stop" in streamThinkingToEventBus
- add FinishReason field to StreamResponse and streamLLMResult
- add finish marker chunk in processStreamDelta for empty content scenarios
- propagate lastFinishReason through EOF handler in processStream
- strengthen PureAgent system prompt with MCP tool guidelines and
  graceful fallback instructions for unanswerable questions
- add unit tests for empty response bug and finish_reason propagation
This commit is contained in:
ochan.kwon
2026-03-27 11:22:08 +09:00
committed by lyingbug
parent 463f40fadc
commit 380f5d9405
8 changed files with 301 additions and 11 deletions
@@ -25,14 +25,17 @@ templates:
### Workflow
1. **Analyze:** Understand the user's request.
2. **Plan:** If the task is complex, use todo_write to create a plan.
3. **Execute:** Use available tools to gather information or perform actions.
3. **Execute:** Use available tools (including any connected MCP tools) to gather information or perform actions.
After receiving tool results, analyze them and incorporate the findings into your answer.
4. **Synthesize:** Call the final_answer tool with your comprehensive answer. You MUST always end by calling final_answer.
### Tool Guidelines
* **MCP Tools:** If external MCP tools are available, use them to fulfill the user's request. Analyze and incorporate their results into your final answer.
* **web_search / web_fetch:** Use these if enabled to find information from the internet.
* **todo_write:** Use for managing multi-step tasks.
* **thinking:** Use to plan and reflect.
* **final_answer:** MANDATORY as your final action. Always submit your complete answer through this tool. NEVER end your turn without calling it.
If you cannot fully answer, explain what you tried and why. If the question is outside your capabilities, say so politely.
### User-Friendly Communication
In ALL outputs visible to users (including your thinking/reasoning), you MUST:
+7
View File
@@ -27,6 +27,13 @@ const (
// maxLLMRetries is the maximum number of retries for transient LLM errors.
maxLLMRetries = 2
// maxEmptyResponseRetries is the maximum number of retries when the LLM
// returns an empty content with a natural stop (no tool calls). This guards
// against the agent completing with an empty answer when the LLM fails to
// produce content (e.g., thinking-only loops without KB).
// Trade-off: each retry costs ~2s of LLM latency; 2 retries = max 4s extra.
maxEmptyResponseRetries = 2
)
// transientErrorMarkers are substrings that indicate a transient (retryable) error.
+23
View File
@@ -270,6 +270,7 @@ func (e *AgentEngine) executeLoop(
common.PipelineInfo(ctx, "Agent", "loop_start", map[string]interface{}{
"max_iterations": e.config.MaxIterations,
})
emptyRetries := 0
for state.CurrentRound < e.config.MaxIterations {
// Check for context cancellation (request timeout, user cancel, etc.)
select {
@@ -336,6 +337,28 @@ func (e *AgentEngine) executeLoop(
// 2. Analyze: Check for stop conditions (natural stop or final_answer tool)
verdict := e.analyzeResponse(ctx, response, step, state.CurrentRound, sessionID, roundStart)
if verdict.isDone {
// Guard against empty content: when the LLM stops naturally with no
// content and no tool calls (e.g., thinking-only loop without KB),
// retry with a nudge message instead of accepting an empty answer.
if verdict.emptyContent {
emptyRetries++
if emptyRetries <= maxEmptyResponseRetries {
logger.Warnf(ctx, "[Agent][Round-%d] Empty content with stop - retrying (%d/%d)",
state.CurrentRound+1, emptyRetries, maxEmptyResponseRetries)
messages = append(messages, chat.Message{
Role: "user",
Content: "Please provide your answer by calling the final_answer tool.",
})
continue
}
// Retries exhausted — use fallback message rather than empty answer
logger.Warnf(ctx, "[Agent][Round-%d] Empty content after %d retries - using fallback",
state.CurrentRound+1, maxEmptyResponseRetries)
state.FinalAnswer = "I'm sorry, I was unable to generate a response. Please try again."
state.IsComplete = true
state.RoundSteps = append(state.RoundSteps, verdict.step)
break
}
state.FinalAnswer = verdict.finalAnswer
state.IsComplete = true
state.RoundSteps = append(state.RoundSteps, verdict.step)
+221
View File
@@ -0,0 +1,221 @@
package agent
import (
"context"
"fmt"
"sync"
"testing"
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// ---------------------------------------------------------------------------
// Mock: chat.Chat
// ---------------------------------------------------------------------------
type mockResponse struct {
chunks []types.StreamResponse
}
type mockChat struct {
mu sync.Mutex
responses []mockResponse
callCount int
}
func (m *mockChat) ChatStream(_ context.Context, _ []chat.Message, _ *chat.ChatOptions) (<-chan types.StreamResponse, error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.callCount >= len(m.responses) {
return nil, fmt.Errorf("unexpected ChatStream call #%d (only %d responses prepared)", m.callCount, len(m.responses))
}
resp := m.responses[m.callCount]
m.callCount++
ch := make(chan types.StreamResponse, len(resp.chunks))
for _, chunk := range resp.chunks {
ch <- chunk
}
close(ch)
return ch, nil
}
func (m *mockChat) Chat(_ context.Context, _ []chat.Message, _ *chat.ChatOptions) (*types.ChatResponse, error) {
return nil, fmt.Errorf("not implemented")
}
func (m *mockChat) GetModelName() string { return "mock-model" }
func (m *mockChat) GetModelID() string { return "mock-id" }
// ---------------------------------------------------------------------------
// Test helpers
// ---------------------------------------------------------------------------
type testEngineOption func(*types.AgentConfig)
func withMaxIterations(n int) testEngineOption {
return func(cfg *types.AgentConfig) {
cfg.MaxIterations = n
}
}
func newTestEngine(t *testing.T, chatModel chat.Chat, opts ...testEngineOption) *AgentEngine {
t.Helper()
cfg := &types.AgentConfig{
MaxIterations: 10,
Temperature: 0.7,
}
for _, opt := range opts {
opt(cfg)
}
engine := NewAgentEngine(
cfg,
chatModel,
nil,
event.NewEventBus(),
nil,
nil,
nil,
"test-session",
"",
)
require.NotNil(t, engine, "NewAgentEngine returned nil (agenttoken.NewEstimator failed?)")
return engine
}
func emptyMessages() []chat.Message {
return []chat.Message{
{Role: "system", Content: "You are a test agent."},
{Role: "user", Content: "test query"},
}
}
func emptyTools() []chat.Tool {
return nil
}
// ---------------------------------------------------------------------------
// TC1: Empty content + stop → should NOT complete with empty FinalAnswer
// ---------------------------------------------------------------------------
func TestExecuteLoop_EmptyContentWithStop_ShouldNotCompleteWithEmpty(t *testing.T) {
// Simulate: LLM returns empty content with no tool calls (natural stop).
// The stream closes with no content chunks → streamLLMToEventBus returns fullContent="".
// streamThinkingToEventBus wraps it as ChatResponse{Content:"", FinishReason:"stop"}.
// analyzeResponse() returns verdict{isDone:true, finalAnswer:""} → BUG: empty answer.
//
// Prepare 3 responses for initial attempt + 2 retries (after fix).
mock := &mockChat{
responses: []mockResponse{
{chunks: []types.StreamResponse{{Done: true}}},
{chunks: []types.StreamResponse{{Done: true}}},
{chunks: []types.StreamResponse{{Done: true}}},
},
}
engine := newTestEngine(t, mock)
state := &types.AgentState{}
ctx := context.Background()
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
assert.NoError(t, err)
assert.True(t, state.IsComplete)
assert.NotEmpty(t, state.FinalAnswer,
"BUG: FinalAnswer is empty when LLM returns empty content with stop. "+
"analyzeResponse() should not allow empty content to be accepted as final answer.")
}
// ---------------------------------------------------------------------------
// TC2: Non-empty content + stop → normal completion (regression guard)
// ---------------------------------------------------------------------------
func TestExecuteLoop_NonEmptyContentWithStop_ShouldComplete(t *testing.T) {
mock := &mockChat{
responses: []mockResponse{
{chunks: []types.StreamResponse{
{Content: "Here is my answer", Done: true},
}},
},
}
engine := newTestEngine(t, mock)
state := &types.AgentState{}
ctx := context.Background()
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
assert.NoError(t, err)
assert.True(t, state.IsComplete)
assert.Equal(t, "Here is my answer", state.FinalAnswer)
}
// ---------------------------------------------------------------------------
// TC4: Empty → retry with nudge → non-empty → success
// ---------------------------------------------------------------------------
func TestExecuteLoop_EmptyThenNonEmpty_ShouldRetryAndComplete(t *testing.T) {
mock := &mockChat{
responses: []mockResponse{
// Round 1: empty content → triggers retry + nudge
{chunks: []types.StreamResponse{{Done: true}}},
// Round 2: after nudge, LLM produces answer
{chunks: []types.StreamResponse{
{Content: "Here is the answer.", Done: true},
}},
},
}
engine := newTestEngine(t, mock)
state := &types.AgentState{}
ctx := context.Background()
_, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1")
assert.NoError(t, err)
assert.True(t, state.IsComplete)
assert.Equal(t, "Here is the answer.", state.FinalAnswer)
}
// ---------------------------------------------------------------------------
// TC5: FinishReason propagation through streamThinkingToEventBus
// ---------------------------------------------------------------------------
func TestStreamThinkingToEventBus_PropagatesFinishReason(t *testing.T) {
tests := []struct {
name string
finishReason string
wantReason string
}{
{"stop", "stop", "stop"},
{"tool_calls", "tool_calls", "tool_calls"},
{"length", "length", "length"},
{"empty_fallback", "", "stop"}, // empty FinishReason → fallback to "stop"
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mock := &mockChat{
responses: []mockResponse{
{chunks: []types.StreamResponse{
{Content: "test content", Done: true, FinishReason: tt.finishReason},
}},
},
}
engine := newTestEngine(t, mock)
ctx := context.Background()
msgs := []chat.Message{{Role: "user", Content: "test"}}
tools := []chat.Tool{}
resp, err := engine.streamThinkingToEventBus(ctx, msgs, tools, 0, "sess-1")
assert.NoError(t, err)
assert.Equal(t, tt.wantReason, resp.FinishReason)
})
}
}
+8 -6
View File
@@ -51,9 +51,10 @@ func (e *AgentEngine) manageContextWindow(ctx context.Context, messages []chat.M
// responseVerdict captures the result of analyzing an LLM response to determine
// whether the agent loop should stop and what the final answer is (if any).
type responseVerdict struct {
isDone bool
finalAnswer string
step types.AgentStep
isDone bool
finalAnswer string
emptyContent bool // LLM returned stop with no tool calls and empty content
step types.AgentStep
}
// analyzeResponse inspects the LLM response for stop conditions:
@@ -102,9 +103,10 @@ func (e *AgentEngine) analyzeResponse(
})
return responseVerdict{
isDone: true,
finalAnswer: response.Content,
step: step,
isDone: true,
finalAnswer: response.Content,
emptyContent: response.Content == "",
step: step,
}
}
+17 -4
View File
@@ -15,9 +15,10 @@ import (
// streamLLMResult holds accumulated output from a streaming LLM call.
type streamLLMResult struct {
Content string
ToolCalls []types.LLMToolCall
Usage *types.TokenUsage
Content string
ToolCalls []types.LLMToolCall
Usage *types.TokenUsage
FinishReason string // actual finish_reason from LLM (captured from last stream chunk)
}
// streamLLMToEventBus streams LLM response through EventBus (generic method)
@@ -66,6 +67,10 @@ func (e *AgentEngine) streamLLMToEventBus(
result.Usage = chunk.Usage
}
if chunk.FinishReason != "" {
result.FinishReason = chunk.FinishReason
}
if emitFunc != nil {
emitFunc(&chunk, result.Content)
}
@@ -205,10 +210,18 @@ func (e *AgentEngine) streamThinkingToEventBus(
fullContent := agenttools.StripThinkBlocks(llmResult.Content)
// Use actual finish_reason from LLM stream instead of hardcoding "stop".
// Fallback to "stop" when the stream did not report a finish_reason
// (e.g., certain Ollama models or providers that omit the field).
finishReason := llmResult.FinishReason
if finishReason == "" {
finishReason = "stop"
}
resp := &types.ChatResponse{
Content: fullContent,
ToolCalls: llmResult.ToolCalls,
FinishReason: "stop",
FinishReason: finishReason,
}
if llmResult.Usage != nil {
resp.Usage = *llmResult.Usage
+20
View File
@@ -510,6 +510,7 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo
Done: true,
ToolCalls: state.buildOrderedToolCalls(),
Usage: state.usage,
FinishReason: state.lastFinishReason,
}
} else {
streamChan <- types.StreamResponse{
@@ -633,6 +634,7 @@ type streamState struct {
hasThinking bool
fieldExtractors map[int]*jsonFieldExtractor // per tool-call-index extractors for streaming field extraction
usage *types.TokenUsage // captured from the final stream chunk when include_usage is enabled
lastFinishReason string // last observed finish_reason for EOF handler fallback
}
func newStreamState() *streamState {
@@ -666,6 +668,11 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C
delta := choice.Delta
isDone := string(choice.FinishReason) != ""
// Track finish_reason for EOF handler fallback
if isDone {
state.lastFinishReason = string(choice.FinishReason)
}
// 处理 tool calls
if len(delta.ToolCalls) > 0 {
c.processToolCallsDelta(ctx, delta.ToolCalls, state, streamChan)
@@ -698,6 +705,7 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C
Content: delta.Content,
Done: isDone,
ToolCalls: state.buildOrderedToolCalls(),
FinishReason: string(choice.FinishReason),
}
}
@@ -707,6 +715,7 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C
Content: "",
Done: true,
ToolCalls: state.buildOrderedToolCalls(),
FinishReason: string(choice.FinishReason),
}
}
@@ -720,6 +729,17 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C
}
state.hasThinking = false
}
// Catch-all: isDone but none of the above branches sent a response with
// FinishReason (empty content, no tool calls, no thinking). This prevents
// the finish_reason from being lost in the streaming pipeline.
if isDone && delta.Content == "" && len(state.toolCallMap) == 0 && !state.hasThinking {
streamChan <- types.StreamResponse{
ResponseType: types.ResponseTypeAnswer,
Done: true,
FinishReason: string(choice.FinishReason),
}
}
}
// processToolCallsDelta 处理 tool calls 的增量更新
+1
View File
@@ -71,6 +71,7 @@ type StreamResponse struct {
ToolCalls []LLMToolCall `json:"tool_calls,omitempty"`
Data map[string]interface{} `json:"data,omitempty"`
Usage *TokenUsage `json:"usage,omitempty"`
FinishReason string `json:"finish_reason,omitempty"`
}
// References references