From 380f5d9405a6677a26b1bd6505e8dff215f4a24a Mon Sep 17 00:00:00 2001 From: "ochan.kwon" Date: Fri, 27 Mar 2026 11:22:08 +0900 Subject: [PATCH] fix(agent): prevent empty response when no Knowledge Base is configured - guard against empty content completion in analyzeResponse/executeLoop with retry (max 2), nudge message, and fallback response - propagate actual finish_reason through streaming pipeline instead of hardcoding "stop" in streamThinkingToEventBus - add FinishReason field to StreamResponse and streamLLMResult - add finish marker chunk in processStreamDelta for empty content scenarios - propagate lastFinishReason through EOF handler in processStream - strengthen PureAgent system prompt with MCP tool guidelines and graceful fallback instructions for unanswerable questions - add unit tests for empty response bug and finish_reason propagation --- .../prompt_templates/agent_system_prompt.yaml | 5 +- internal/agent/const.go | 7 + internal/agent/engine.go | 23 ++ internal/agent/engine_test.go | 221 ++++++++++++++++++ internal/agent/observe.go | 14 +- internal/agent/think.go | 21 +- internal/models/chat/remote_api.go | 20 ++ internal/types/chat.go | 1 + 8 files changed, 301 insertions(+), 11 deletions(-) create mode 100644 internal/agent/engine_test.go diff --git a/config/prompt_templates/agent_system_prompt.yaml b/config/prompt_templates/agent_system_prompt.yaml index 4fd28689..fadc2757 100644 --- a/config/prompt_templates/agent_system_prompt.yaml +++ b/config/prompt_templates/agent_system_prompt.yaml @@ -25,14 +25,17 @@ templates: ### Workflow 1. **Analyze:** Understand the user's request. 2. **Plan:** If the task is complex, use todo_write to create a plan. - 3. **Execute:** Use available tools to gather information or perform actions. + 3. **Execute:** Use available tools (including any connected MCP tools) to gather information or perform actions. + After receiving tool results, analyze them and incorporate the findings into your answer. 4. **Synthesize:** Call the final_answer tool with your comprehensive answer. You MUST always end by calling final_answer. ### Tool Guidelines + * **MCP Tools:** If external MCP tools are available, use them to fulfill the user's request. Analyze and incorporate their results into your final answer. * **web_search / web_fetch:** Use these if enabled to find information from the internet. * **todo_write:** Use for managing multi-step tasks. * **thinking:** Use to plan and reflect. * **final_answer:** MANDATORY as your final action. Always submit your complete answer through this tool. NEVER end your turn without calling it. + If you cannot fully answer, explain what you tried and why. If the question is outside your capabilities, say so politely. ### User-Friendly Communication In ALL outputs visible to users (including your thinking/reasoning), you MUST: diff --git a/internal/agent/const.go b/internal/agent/const.go index 280ab6c7..49f50006 100644 --- a/internal/agent/const.go +++ b/internal/agent/const.go @@ -27,6 +27,13 @@ const ( // maxLLMRetries is the maximum number of retries for transient LLM errors. maxLLMRetries = 2 + + // maxEmptyResponseRetries is the maximum number of retries when the LLM + // returns an empty content with a natural stop (no tool calls). This guards + // against the agent completing with an empty answer when the LLM fails to + // produce content (e.g., thinking-only loops without KB). + // Trade-off: each retry costs ~2s of LLM latency; 2 retries = max 4s extra. + maxEmptyResponseRetries = 2 ) // transientErrorMarkers are substrings that indicate a transient (retryable) error. diff --git a/internal/agent/engine.go b/internal/agent/engine.go index 08488dc6..22111fe1 100644 --- a/internal/agent/engine.go +++ b/internal/agent/engine.go @@ -270,6 +270,7 @@ func (e *AgentEngine) executeLoop( common.PipelineInfo(ctx, "Agent", "loop_start", map[string]interface{}{ "max_iterations": e.config.MaxIterations, }) + emptyRetries := 0 for state.CurrentRound < e.config.MaxIterations { // Check for context cancellation (request timeout, user cancel, etc.) select { @@ -336,6 +337,28 @@ func (e *AgentEngine) executeLoop( // 2. Analyze: Check for stop conditions (natural stop or final_answer tool) verdict := e.analyzeResponse(ctx, response, step, state.CurrentRound, sessionID, roundStart) if verdict.isDone { + // Guard against empty content: when the LLM stops naturally with no + // content and no tool calls (e.g., thinking-only loop without KB), + // retry with a nudge message instead of accepting an empty answer. + if verdict.emptyContent { + emptyRetries++ + if emptyRetries <= maxEmptyResponseRetries { + logger.Warnf(ctx, "[Agent][Round-%d] Empty content with stop - retrying (%d/%d)", + state.CurrentRound+1, emptyRetries, maxEmptyResponseRetries) + messages = append(messages, chat.Message{ + Role: "user", + Content: "Please provide your answer by calling the final_answer tool.", + }) + continue + } + // Retries exhausted — use fallback message rather than empty answer + logger.Warnf(ctx, "[Agent][Round-%d] Empty content after %d retries - using fallback", + state.CurrentRound+1, maxEmptyResponseRetries) + state.FinalAnswer = "I'm sorry, I was unable to generate a response. Please try again." + state.IsComplete = true + state.RoundSteps = append(state.RoundSteps, verdict.step) + break + } state.FinalAnswer = verdict.finalAnswer state.IsComplete = true state.RoundSteps = append(state.RoundSteps, verdict.step) diff --git a/internal/agent/engine_test.go b/internal/agent/engine_test.go new file mode 100644 index 00000000..cd4857b3 --- /dev/null +++ b/internal/agent/engine_test.go @@ -0,0 +1,221 @@ +package agent + +import ( + "context" + "fmt" + "sync" + "testing" + + "github.com/Tencent/WeKnora/internal/event" + "github.com/Tencent/WeKnora/internal/models/chat" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// --------------------------------------------------------------------------- +// Mock: chat.Chat +// --------------------------------------------------------------------------- + +type mockResponse struct { + chunks []types.StreamResponse +} + +type mockChat struct { + mu sync.Mutex + responses []mockResponse + callCount int +} + +func (m *mockChat) ChatStream(_ context.Context, _ []chat.Message, _ *chat.ChatOptions) (<-chan types.StreamResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.callCount >= len(m.responses) { + return nil, fmt.Errorf("unexpected ChatStream call #%d (only %d responses prepared)", m.callCount, len(m.responses)) + } + resp := m.responses[m.callCount] + m.callCount++ + + ch := make(chan types.StreamResponse, len(resp.chunks)) + for _, chunk := range resp.chunks { + ch <- chunk + } + close(ch) + return ch, nil +} + +func (m *mockChat) Chat(_ context.Context, _ []chat.Message, _ *chat.ChatOptions) (*types.ChatResponse, error) { + return nil, fmt.Errorf("not implemented") +} + +func (m *mockChat) GetModelName() string { return "mock-model" } +func (m *mockChat) GetModelID() string { return "mock-id" } + +// --------------------------------------------------------------------------- +// Test helpers +// --------------------------------------------------------------------------- + +type testEngineOption func(*types.AgentConfig) + +func withMaxIterations(n int) testEngineOption { + return func(cfg *types.AgentConfig) { + cfg.MaxIterations = n + } +} + +func newTestEngine(t *testing.T, chatModel chat.Chat, opts ...testEngineOption) *AgentEngine { + t.Helper() + cfg := &types.AgentConfig{ + MaxIterations: 10, + Temperature: 0.7, + } + for _, opt := range opts { + opt(cfg) + } + engine := NewAgentEngine( + cfg, + chatModel, + nil, + event.NewEventBus(), + nil, + nil, + nil, + "test-session", + "", + ) + require.NotNil(t, engine, "NewAgentEngine returned nil (agenttoken.NewEstimator failed?)") + return engine +} + +func emptyMessages() []chat.Message { + return []chat.Message{ + {Role: "system", Content: "You are a test agent."}, + {Role: "user", Content: "test query"}, + } +} + +func emptyTools() []chat.Tool { + return nil +} + +// --------------------------------------------------------------------------- +// TC1: Empty content + stop → should NOT complete with empty FinalAnswer +// --------------------------------------------------------------------------- + +func TestExecuteLoop_EmptyContentWithStop_ShouldNotCompleteWithEmpty(t *testing.T) { + // Simulate: LLM returns empty content with no tool calls (natural stop). + // The stream closes with no content chunks → streamLLMToEventBus returns fullContent="". + // streamThinkingToEventBus wraps it as ChatResponse{Content:"", FinishReason:"stop"}. + // analyzeResponse() returns verdict{isDone:true, finalAnswer:""} → BUG: empty answer. + // + // Prepare 3 responses for initial attempt + 2 retries (after fix). + mock := &mockChat{ + responses: []mockResponse{ + {chunks: []types.StreamResponse{{Done: true}}}, + {chunks: []types.StreamResponse{{Done: true}}}, + {chunks: []types.StreamResponse{{Done: true}}}, + }, + } + + engine := newTestEngine(t, mock) + state := &types.AgentState{} + ctx := context.Background() + + _, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1") + + assert.NoError(t, err) + assert.True(t, state.IsComplete) + assert.NotEmpty(t, state.FinalAnswer, + "BUG: FinalAnswer is empty when LLM returns empty content with stop. "+ + "analyzeResponse() should not allow empty content to be accepted as final answer.") +} + +// --------------------------------------------------------------------------- +// TC2: Non-empty content + stop → normal completion (regression guard) +// --------------------------------------------------------------------------- + +func TestExecuteLoop_NonEmptyContentWithStop_ShouldComplete(t *testing.T) { + mock := &mockChat{ + responses: []mockResponse{ + {chunks: []types.StreamResponse{ + {Content: "Here is my answer", Done: true}, + }}, + }, + } + + engine := newTestEngine(t, mock) + state := &types.AgentState{} + ctx := context.Background() + + _, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1") + + assert.NoError(t, err) + assert.True(t, state.IsComplete) + assert.Equal(t, "Here is my answer", state.FinalAnswer) +} + +// --------------------------------------------------------------------------- +// TC4: Empty → retry with nudge → non-empty → success +// --------------------------------------------------------------------------- + +func TestExecuteLoop_EmptyThenNonEmpty_ShouldRetryAndComplete(t *testing.T) { + mock := &mockChat{ + responses: []mockResponse{ + // Round 1: empty content → triggers retry + nudge + {chunks: []types.StreamResponse{{Done: true}}}, + // Round 2: after nudge, LLM produces answer + {chunks: []types.StreamResponse{ + {Content: "Here is the answer.", Done: true}, + }}, + }, + } + + engine := newTestEngine(t, mock) + state := &types.AgentState{} + ctx := context.Background() + + _, err := engine.executeLoop(ctx, state, "test query", emptyMessages(), emptyTools(), "sess-1", "msg-1") + + assert.NoError(t, err) + assert.True(t, state.IsComplete) + assert.Equal(t, "Here is the answer.", state.FinalAnswer) +} + +// --------------------------------------------------------------------------- +// TC5: FinishReason propagation through streamThinkingToEventBus +// --------------------------------------------------------------------------- + +func TestStreamThinkingToEventBus_PropagatesFinishReason(t *testing.T) { + tests := []struct { + name string + finishReason string + wantReason string + }{ + {"stop", "stop", "stop"}, + {"tool_calls", "tool_calls", "tool_calls"}, + {"length", "length", "length"}, + {"empty_fallback", "", "stop"}, // empty FinishReason → fallback to "stop" + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mock := &mockChat{ + responses: []mockResponse{ + {chunks: []types.StreamResponse{ + {Content: "test content", Done: true, FinishReason: tt.finishReason}, + }}, + }, + } + + engine := newTestEngine(t, mock) + ctx := context.Background() + msgs := []chat.Message{{Role: "user", Content: "test"}} + tools := []chat.Tool{} + + resp, err := engine.streamThinkingToEventBus(ctx, msgs, tools, 0, "sess-1") + + assert.NoError(t, err) + assert.Equal(t, tt.wantReason, resp.FinishReason) + }) + } +} diff --git a/internal/agent/observe.go b/internal/agent/observe.go index f2de5454..d3667352 100644 --- a/internal/agent/observe.go +++ b/internal/agent/observe.go @@ -51,9 +51,10 @@ func (e *AgentEngine) manageContextWindow(ctx context.Context, messages []chat.M // responseVerdict captures the result of analyzing an LLM response to determine // whether the agent loop should stop and what the final answer is (if any). type responseVerdict struct { - isDone bool - finalAnswer string - step types.AgentStep + isDone bool + finalAnswer string + emptyContent bool // LLM returned stop with no tool calls and empty content + step types.AgentStep } // analyzeResponse inspects the LLM response for stop conditions: @@ -102,9 +103,10 @@ func (e *AgentEngine) analyzeResponse( }) return responseVerdict{ - isDone: true, - finalAnswer: response.Content, - step: step, + isDone: true, + finalAnswer: response.Content, + emptyContent: response.Content == "", + step: step, } } diff --git a/internal/agent/think.go b/internal/agent/think.go index 621bf757..2633214c 100644 --- a/internal/agent/think.go +++ b/internal/agent/think.go @@ -15,9 +15,10 @@ import ( // streamLLMResult holds accumulated output from a streaming LLM call. type streamLLMResult struct { - Content string - ToolCalls []types.LLMToolCall - Usage *types.TokenUsage + Content string + ToolCalls []types.LLMToolCall + Usage *types.TokenUsage + FinishReason string // actual finish_reason from LLM (captured from last stream chunk) } // streamLLMToEventBus streams LLM response through EventBus (generic method) @@ -66,6 +67,10 @@ func (e *AgentEngine) streamLLMToEventBus( result.Usage = chunk.Usage } + if chunk.FinishReason != "" { + result.FinishReason = chunk.FinishReason + } + if emitFunc != nil { emitFunc(&chunk, result.Content) } @@ -205,10 +210,18 @@ func (e *AgentEngine) streamThinkingToEventBus( fullContent := agenttools.StripThinkBlocks(llmResult.Content) + // Use actual finish_reason from LLM stream instead of hardcoding "stop". + // Fallback to "stop" when the stream did not report a finish_reason + // (e.g., certain Ollama models or providers that omit the field). + finishReason := llmResult.FinishReason + if finishReason == "" { + finishReason = "stop" + } + resp := &types.ChatResponse{ Content: fullContent, ToolCalls: llmResult.ToolCalls, - FinishReason: "stop", + FinishReason: finishReason, } if llmResult.Usage != nil { resp.Usage = *llmResult.Usage diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go index 829c8b12..44459127 100644 --- a/internal/models/chat/remote_api.go +++ b/internal/models/chat/remote_api.go @@ -510,6 +510,7 @@ func (c *RemoteAPIChat) processStream(ctx context.Context, stream *openai.ChatCo Done: true, ToolCalls: state.buildOrderedToolCalls(), Usage: state.usage, + FinishReason: state.lastFinishReason, } } else { streamChan <- types.StreamResponse{ @@ -633,6 +634,7 @@ type streamState struct { hasThinking bool fieldExtractors map[int]*jsonFieldExtractor // per tool-call-index extractors for streaming field extraction usage *types.TokenUsage // captured from the final stream chunk when include_usage is enabled + lastFinishReason string // last observed finish_reason for EOF handler fallback } func newStreamState() *streamState { @@ -666,6 +668,11 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C delta := choice.Delta isDone := string(choice.FinishReason) != "" + // Track finish_reason for EOF handler fallback + if isDone { + state.lastFinishReason = string(choice.FinishReason) + } + // 处理 tool calls if len(delta.ToolCalls) > 0 { c.processToolCallsDelta(ctx, delta.ToolCalls, state, streamChan) @@ -698,6 +705,7 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C Content: delta.Content, Done: isDone, ToolCalls: state.buildOrderedToolCalls(), + FinishReason: string(choice.FinishReason), } } @@ -707,6 +715,7 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C Content: "", Done: true, ToolCalls: state.buildOrderedToolCalls(), + FinishReason: string(choice.FinishReason), } } @@ -720,6 +729,17 @@ func (c *RemoteAPIChat) processStreamDelta(ctx context.Context, choice *openai.C } state.hasThinking = false } + + // Catch-all: isDone but none of the above branches sent a response with + // FinishReason (empty content, no tool calls, no thinking). This prevents + // the finish_reason from being lost in the streaming pipeline. + if isDone && delta.Content == "" && len(state.toolCallMap) == 0 && !state.hasThinking { + streamChan <- types.StreamResponse{ + ResponseType: types.ResponseTypeAnswer, + Done: true, + FinishReason: string(choice.FinishReason), + } + } } // processToolCallsDelta 处理 tool calls 的增量更新 diff --git a/internal/types/chat.go b/internal/types/chat.go index 61441e98..14d47bf1 100644 --- a/internal/types/chat.go +++ b/internal/types/chat.go @@ -71,6 +71,7 @@ type StreamResponse struct { ToolCalls []LLMToolCall `json:"tool_calls,omitempty"` Data map[string]interface{} `json:"data,omitempty"` Usage *TokenUsage `json:"usage,omitempty"` + FinishReason string `json:"finish_reason,omitempty"` } // References references