diff --git a/internal/agent/observe.go b/internal/agent/observe.go index 9f3a592d..76dfbf95 100644 --- a/internal/agent/observe.go +++ b/internal/agent/observe.go @@ -16,6 +16,13 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) +// finalAnswerParseFallback is the user-visible message surfaced when the LLM +// calls final_answer with arguments we cannot recover into an answer string +// (even after RepairJSON + regex fallback). Terminating the loop with this +// message prevents the agent from re-entering and emitting duplicate answers +// on every subsequent round — the behavior reported in issue #1008. +const finalAnswerParseFallback = "Sorry, the model's final answer could not be parsed due to malformed output. Please try again or rephrase your question." + // manageContextWindow consolidates or compresses messages if approaching the token limit. // currentTokens is the caller's best estimate of the current context size (using // API-reported Usage when available, falling back to BPE estimation). @@ -156,42 +163,87 @@ func (e *AgentEngine) analyzeResponse( } } - // Case 2: final_answer tool call present + // Case 2: final_answer tool call present. + // + // final_answer is always a terminal signal: regardless of whether we can + // parse its arguments, we must end the ReAct loop here. Otherwise the LLM + // will see the tool result in the next round, re-invoke final_answer with + // near-identical content, and surface duplicate answers to the user (see + // issue #1008). Parse with three levels of tolerance: + // + // 1. strict json.Unmarshal + // 2. RepairJSON + Unmarshal + // 3. regex best-effort extraction of the "answer" field + // + // If all three fail, terminate with a user-visible fallback message. if len(response.ToolCalls) > 0 { for _, tc := range response.ToolCalls { - if tc.Function.Name == agenttools.ToolFinalAnswer { - var faArgs struct { - Answer string `json:"answer"` - } - if err := json.Unmarshal([]byte(tc.Function.Arguments), &faArgs); err != nil { - logger.Warnf(ctx, "[Agent][Round-%d] Failed to parse final_answer args: %v", - iteration+1, err) - } else { - logger.Infof(ctx, "[Agent][Round-%d] final_answer tool: answer=%d chars, duration=%dms", - iteration+1, len(faArgs.Answer), time.Since(roundStart).Milliseconds()) + if tc.Function.Name != agenttools.ToolFinalAnswer { + continue + } - e.eventBus.Emit(ctx, event.Event{ - ID: generateEventID("answer-done"), - Type: event.EventAgentFinalAnswer, - SessionID: sessionID, - Data: event.AgentFinalAnswerData{ - Content: "", - Done: true, - }, - }) - common.PipelineInfo(ctx, "Agent", "final_answer_tool", map[string]interface{}{ - "iteration": iteration, - "round": iteration + 1, - "answer_len": len(faArgs.Answer), - }) + rawArgs := tc.Function.Arguments + answer, ok := agenttools.ParseFinalAnswerArgs(rawArgs) + recovered := false + if !ok { + // Could not recover any answer text — fall back to a generic + // message so the user doesn't see a blank response. + logger.Warnf(ctx, "[Agent][Round-%d] Failed to parse final_answer args (args=%q) — "+ + "terminating loop with fallback message", + iteration+1, rawArgs) + answer = finalAnswerParseFallback + } else { + recovered = true + logger.Infof(ctx, "[Agent][Round-%d] final_answer tool: answer=%d chars, duration=%dms", + iteration+1, len(answer), time.Since(roundStart).Milliseconds()) + } - return responseVerdict{ - isDone: true, - finalAnswer: faArgs.Answer, - step: step, - } - } - break + // Always emit the final answer content and Done=true marker to the + // event bus. When strict parsing succeeded earlier in this turn, + // streamThinkingToEventBus already streamed the answer chunks, so + // we only need the Done marker in that common case. When we fell + // back to the generic message, however, the UI has not yet seen + // any answer content — emit both Content and Done to make the + // fallback visible to the user. + answerID := generateEventID("answer-done") + if !recovered { + e.eventBus.Emit(ctx, event.Event{ + ID: answerID, + Type: event.EventAgentFinalAnswer, + SessionID: sessionID, + Data: event.AgentFinalAnswerData{ + Content: answer, + Done: false, + }, + }) + } + e.eventBus.Emit(ctx, event.Event{ + ID: answerID, + Type: event.EventAgentFinalAnswer, + SessionID: sessionID, + Data: event.AgentFinalAnswerData{ + Content: "", + Done: true, + }, + }) + + pipelineFields := map[string]interface{}{ + "iteration": iteration, + "round": iteration + 1, + "answer_len": len(answer), + "recovered": recovered, + } + if recovered { + common.PipelineInfo(ctx, "Agent", "final_answer_tool", pipelineFields) + } else { + pipelineFields["raw_args"] = rawArgs + common.PipelineWarn(ctx, "Agent", "final_answer_tool_parse_failed", pipelineFields) + } + + return responseVerdict{ + isDone: true, + finalAnswer: answer, + step: step, } } } diff --git a/internal/agent/observe_test.go b/internal/agent/observe_test.go new file mode 100644 index 00000000..e827ca51 --- /dev/null +++ b/internal/agent/observe_test.go @@ -0,0 +1,126 @@ +package agent + +import ( + "context" + "testing" + "time" + + agenttools "github.com/Tencent/WeKnora/internal/agent/tools" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/assert" +) + +// newFinalAnswerResponse builds a ChatResponse that carries a single +// final_answer tool call with the given raw JSON arguments. +func newFinalAnswerResponse(rawArgs string) *types.ChatResponse { + return &types.ChatResponse{ + FinishReason: "tool_calls", + ToolCalls: []types.LLMToolCall{ + { + ID: "call-1", + Type: "function", + Function: types.FunctionCall{ + Name: agenttools.ToolFinalAnswer, + Arguments: rawArgs, + }, + }, + }, + } +} + +// TestAnalyzeResponse_FinalAnswer_ValidArgs guards the happy path: well-formed +// arguments must be extracted into the final answer and terminate the loop. +func TestAnalyzeResponse_FinalAnswer_ValidArgs(t *testing.T) { + engine := newTestEngine(t, &mockChat{}) + resp := newFinalAnswerResponse(`{"answer": "Here is the answer."}`) + + verdict := engine.analyzeResponse( + context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(), + ) + + assert.True(t, verdict.isDone, "final_answer must terminate the loop") + assert.Equal(t, "Here is the answer.", verdict.finalAnswer) +} + +// TestAnalyzeResponse_FinalAnswer_MalformedJSON_RecoveredViaRepair covers the +// common case reported in issue #1008: the LLM emits final_answer with a +// trailing comma / missing brace. RepairJSON should recover the answer and +// the loop must still terminate in this single round (not re-invoke +// final_answer in the next round). +func TestAnalyzeResponse_FinalAnswer_MalformedJSON_RecoveredViaRepair(t *testing.T) { + engine := newTestEngine(t, &mockChat{}) + resp := newFinalAnswerResponse(`{"answer": "repaired"`) // missing closing brace + + verdict := engine.analyzeResponse( + context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(), + ) + + assert.True(t, verdict.isDone, + "final_answer must terminate the loop even when JSON repair is needed") + assert.Equal(t, "repaired", verdict.finalAnswer) +} + +// TestAnalyzeResponse_FinalAnswer_UnrecoverableArgs_StillTerminates is the +// direct regression test for issue #1008: when the arguments are so malformed +// that even RepairJSON + regex cannot recover an answer, the loop MUST still +// terminate (with a user-visible fallback message) rather than continuing and +// letting the LLM re-emit final_answer on the next round. +func TestAnalyzeResponse_FinalAnswer_UnrecoverableArgs_StillTerminates(t *testing.T) { + engine := newTestEngine(t, &mockChat{}) + // No `answer` key at all — strict parse succeeds (returns zero-value + // answer), RepairJSON is a no-op on already-valid JSON, regex finds + // nothing. All three tiers fail to recover an answer. + resp := newFinalAnswerResponse(`{"unexpected": "field"}`) + + verdict := engine.analyzeResponse( + context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(), + ) + + assert.True(t, verdict.isDone, + "final_answer must terminate the loop even when args are unrecoverable — "+ + "otherwise the LLM re-emits final_answer and duplicates the answer (issue #1008)") + assert.Equal(t, finalAnswerParseFallback, verdict.finalAnswer, + "unrecoverable final_answer should surface the parse-failure fallback message") +} + +// TestAnalyzeResponse_FinalAnswer_Garbage_StillTerminates exercises the most +// hostile case: completely non-JSON arguments. The loop must still terminate +// — protecting against the duplicate-answer loop reported in issue #1008. +func TestAnalyzeResponse_FinalAnswer_Garbage_StillTerminates(t *testing.T) { + engine := newTestEngine(t, &mockChat{}) + resp := newFinalAnswerResponse(`not json at all`) + + verdict := engine.analyzeResponse( + context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(), + ) + + assert.True(t, verdict.isDone) + assert.Equal(t, finalAnswerParseFallback, verdict.finalAnswer) +} + +// TestAnalyzeResponse_NonFinalAnswerTool_DoesNotTerminate is a regression +// guard: only final_answer is terminal. Other tool calls (e.g. thinking, +// knowledge_search) must keep the loop running. +func TestAnalyzeResponse_NonFinalAnswerTool_DoesNotTerminate(t *testing.T) { + engine := newTestEngine(t, &mockChat{}) + resp := &types.ChatResponse{ + FinishReason: "tool_calls", + ToolCalls: []types.LLMToolCall{ + { + ID: "call-1", + Type: "function", + Function: types.FunctionCall{ + Name: agenttools.ToolKnowledgeSearch, + Arguments: `{"query": "hi"}`, + }, + }, + }, + } + + verdict := engine.analyzeResponse( + context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(), + ) + + assert.False(t, verdict.isDone, + "non-terminal tool calls must keep the loop running") +} diff --git a/internal/agent/tools/final_answer.go b/internal/agent/tools/final_answer.go index 1d83e01a..1f454898 100644 --- a/internal/agent/tools/final_answer.go +++ b/internal/agent/tools/final_answer.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "regexp" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" @@ -62,26 +63,88 @@ func NewFinalAnswerTool() *FinalAnswerTool { func (t *FinalAnswerTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) { logger.Infof(ctx, "[Tool][FinalAnswer] Execute started") - var input FinalAnswerInput - if err := json.Unmarshal(args, &input); err != nil { - logger.Errorf(ctx, "[Tool][FinalAnswer] Failed to parse args: %v", err) + answer, ok := ParseFinalAnswerArgs(string(args)) + if !ok { + logger.Errorf(ctx, "[Tool][FinalAnswer] Failed to parse args (even with repair): %s", string(args)) return &types.ToolResult{ Success: false, - Error: fmt.Sprintf("Failed to parse args: %v", err), - }, err + Error: "Failed to parse final_answer args: malformed JSON and no recoverable answer field", + }, fmt.Errorf("malformed final_answer arguments") } - if input.Answer == "" { + if answer == "" { return &types.ToolResult{ Success: false, Error: "answer must be a non-empty string", }, fmt.Errorf("answer must be a non-empty string") } - logger.Infof(ctx, "[Tool][FinalAnswer] Answer length: %d characters", len(input.Answer)) + logger.Infof(ctx, "[Tool][FinalAnswer] Answer length: %d characters", len(answer)) return &types.ToolResult{ Success: true, - Output: input.Answer, + Output: answer, }, nil } + +// answerRegex best-effort extracts the value of an "answer": "..." field from +// a malformed JSON string. Used as a last resort when both strict parsing and +// RepairJSON fail — this keeps the final_answer tool call terminal so the +// agent loop cannot re-enter and emit duplicate answers. +// +// The pattern handles escaped quotes inside the answer via the non-greedy +// body `(?:\\.|[^"\\])*` which consumes either an escape sequence or any +// non-quote, non-backslash char. +var answerRegex = regexp.MustCompile(`"answer"\s*:\s*"((?:\\.|[^"\\])*)"`) + +// ParseFinalAnswerArgs extracts the `answer` field from the final_answer +// tool's raw arguments. It is intentionally tolerant of malformed JSON that +// LLMs sometimes emit (unescaped quotes inside the answer, trailing commas, +// truncated closing braces, etc.), applying three fallbacks in order: +// +// 1. Strict json.Unmarshal on the raw string. +// 2. RepairJSON (trailing commas, invalid escapes, bracket balance) + Unmarshal. +// 3. Regex best-effort extraction of the `"answer": "..."` field. +// +// Returns the answer string and a bool indicating whether any path succeeded +// with a non-empty answer. Callers should treat ok=false as "unrecoverable" +// and surface a fallback message to the user, but must still treat the +// tool call as terminal to avoid the agent loop re-emitting final_answer. +func ParseFinalAnswerArgs(raw string) (string, bool) { + var input FinalAnswerInput + if err := json.Unmarshal([]byte(raw), &input); err == nil && input.Answer != "" { + return input.Answer, true + } + + repaired := RepairJSON(raw) + if repaired != raw { + if err := json.Unmarshal([]byte(repaired), &input); err == nil && input.Answer != "" { + return input.Answer, true + } + } + + if m := answerRegex.FindStringSubmatch(raw); len(m) == 2 { + if unquoted, err := unquoteJSONString(m[1]); err == nil && unquoted != "" { + return unquoted, true + } + // Unquoting failed: the capture may contain invalid escapes. Fall back + // to returning the raw capture so the user still sees *something*. + if m[1] != "" { + return m[1], true + } + } + + return "", false +} + +// unquoteJSONString decodes a JSON-escaped string body (without surrounding +// quotes) into its literal form. It wraps the body in quotes and leans on +// json.Unmarshal so we get the standard escape semantics (\n, \", \uXXXX, …) +// without reimplementing them. +func unquoteJSONString(body string) (string, error) { + var out string + if err := json.Unmarshal([]byte(`"`+body+`"`), &out); err != nil { + return "", err + } + return out, nil +} diff --git a/internal/agent/tools/final_answer_test.go b/internal/agent/tools/final_answer_test.go new file mode 100644 index 00000000..52ddc8bf --- /dev/null +++ b/internal/agent/tools/final_answer_test.go @@ -0,0 +1,61 @@ +package tools + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestParseFinalAnswerArgs covers the three-tier recovery path used by both +// the final_answer tool and the ReAct loop's terminal detection (issue #1008). +func TestParseFinalAnswerArgs(t *testing.T) { + t.Run("strict JSON is parsed as-is", func(t *testing.T) { + got, ok := ParseFinalAnswerArgs(`{"answer": "Hello, world."}`) + assert.True(t, ok) + assert.Equal(t, "Hello, world.", got) + }) + + t.Run("trailing comma is recovered via RepairJSON", func(t *testing.T) { + got, ok := ParseFinalAnswerArgs(`{"answer": "Hi",}`) + assert.True(t, ok) + assert.Equal(t, "Hi", got) + }) + + t.Run("missing closing brace is recovered via RepairJSON", func(t *testing.T) { + got, ok := ParseFinalAnswerArgs(`{"answer": "Hi"`) + assert.True(t, ok) + assert.Equal(t, "Hi", got) + }) + + t.Run("invalid backslash escape is recovered via RepairJSON", func(t *testing.T) { + // LLM forgot to double-escape the regex metachar — RepairJSON should + // rewrite "\+" to "\\+" so Unmarshal succeeds. + got, ok := ParseFinalAnswerArgs(`{"answer": "C\+\+"}`) + assert.True(t, ok) + assert.Equal(t, `C\+\+`, got) + }) + + t.Run("unescaped inner quote is recovered via regex fallback", func(t *testing.T) { + // Neither strict parse nor RepairJSON can recover this one; the regex + // still captures the well-formed prefix up to the rogue quote. + raw := `{"answer": "She said "hello" to me"}` + got, ok := ParseFinalAnswerArgs(raw) + assert.True(t, ok) + assert.NotEmpty(t, got) + }) + + t.Run("missing answer field returns not ok", func(t *testing.T) { + _, ok := ParseFinalAnswerArgs(`{"other": "value"}`) + assert.False(t, ok) + }) + + t.Run("empty answer returns not ok", func(t *testing.T) { + _, ok := ParseFinalAnswerArgs(`{"answer": ""}`) + assert.False(t, ok) + }) + + t.Run("completely garbled input returns not ok", func(t *testing.T) { + _, ok := ParseFinalAnswerArgs(`not json at all`) + assert.False(t, ok) + }) +}