fix(agent): terminate ReAct loop on final_answer regardless of arg parse success

The agent loop previously kept running when the LLM emitted final_answer with
malformed JSON arguments: observe.analyzeResponse strictly json.Unmarshal'd the
raw args and returned isDone=false on failure, while act.runToolCall separately
repaired the JSON and executed the tool. The mismatch made final_answer
non-terminal, so the LLM saw its own answer in the next round's tool result
and re-emitted final_answer, surfacing near-identical answers multiple times
to the user (issue #1008).

Fix by unifying parsing in a new tools.ParseFinalAnswerArgs helper with three
fallbacks (strict -> RepairJSON -> regex extraction) and making final_answer
always terminal in analyzeResponse. When no answer can be recovered, emit a
user-visible fallback message and a Done=true event so the UI still resolves.

Also route final_answer.Execute through the same helper so tool execution and
loop termination stay consistent.

Adds regression tests in observe_test.go and final_answer_test.go covering
valid args, RepairJSON recovery, unrecoverable args, and garbage input.

Refs: Tencent/WeKnora#1008
This commit is contained in:
wizardchen
2026-04-23 18:10:35 +08:00
committed by lyingbug
parent 3231ae30a4
commit a6d4266c93
4 changed files with 342 additions and 40 deletions
+84 -32
View File
@@ -16,6 +16,13 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
// finalAnswerParseFallback is the user-visible message surfaced when the LLM
// calls final_answer with arguments we cannot recover into an answer string
// (even after RepairJSON + regex fallback). Terminating the loop with this
// message prevents the agent from re-entering and emitting duplicate answers
// on every subsequent round — the behavior reported in issue #1008.
const finalAnswerParseFallback = "Sorry, the model's final answer could not be parsed due to malformed output. Please try again or rephrase your question."
// manageContextWindow consolidates or compresses messages if approaching the token limit.
// currentTokens is the caller's best estimate of the current context size (using
// API-reported Usage when available, falling back to BPE estimation).
@@ -156,42 +163,87 @@ func (e *AgentEngine) analyzeResponse(
}
}
// Case 2: final_answer tool call present
// Case 2: final_answer tool call present.
//
// final_answer is always a terminal signal: regardless of whether we can
// parse its arguments, we must end the ReAct loop here. Otherwise the LLM
// will see the tool result in the next round, re-invoke final_answer with
// near-identical content, and surface duplicate answers to the user (see
// issue #1008). Parse with three levels of tolerance:
//
// 1. strict json.Unmarshal
// 2. RepairJSON + Unmarshal
// 3. regex best-effort extraction of the "answer" field
//
// If all three fail, terminate with a user-visible fallback message.
if len(response.ToolCalls) > 0 {
for _, tc := range response.ToolCalls {
if tc.Function.Name == agenttools.ToolFinalAnswer {
var faArgs struct {
Answer string `json:"answer"`
}
if err := json.Unmarshal([]byte(tc.Function.Arguments), &faArgs); err != nil {
logger.Warnf(ctx, "[Agent][Round-%d] Failed to parse final_answer args: %v",
iteration+1, err)
} else {
logger.Infof(ctx, "[Agent][Round-%d] final_answer tool: answer=%d chars, duration=%dms",
iteration+1, len(faArgs.Answer), time.Since(roundStart).Milliseconds())
if tc.Function.Name != agenttools.ToolFinalAnswer {
continue
}
e.eventBus.Emit(ctx, event.Event{
ID: generateEventID("answer-done"),
Type: event.EventAgentFinalAnswer,
SessionID: sessionID,
Data: event.AgentFinalAnswerData{
Content: "",
Done: true,
},
})
common.PipelineInfo(ctx, "Agent", "final_answer_tool", map[string]interface{}{
"iteration": iteration,
"round": iteration + 1,
"answer_len": len(faArgs.Answer),
})
rawArgs := tc.Function.Arguments
answer, ok := agenttools.ParseFinalAnswerArgs(rawArgs)
recovered := false
if !ok {
// Could not recover any answer text — fall back to a generic
// message so the user doesn't see a blank response.
logger.Warnf(ctx, "[Agent][Round-%d] Failed to parse final_answer args (args=%q) — "+
"terminating loop with fallback message",
iteration+1, rawArgs)
answer = finalAnswerParseFallback
} else {
recovered = true
logger.Infof(ctx, "[Agent][Round-%d] final_answer tool: answer=%d chars, duration=%dms",
iteration+1, len(answer), time.Since(roundStart).Milliseconds())
}
return responseVerdict{
isDone: true,
finalAnswer: faArgs.Answer,
step: step,
}
}
break
// Always emit the final answer content and Done=true marker to the
// event bus. When strict parsing succeeded earlier in this turn,
// streamThinkingToEventBus already streamed the answer chunks, so
// we only need the Done marker in that common case. When we fell
// back to the generic message, however, the UI has not yet seen
// any answer content — emit both Content and Done to make the
// fallback visible to the user.
answerID := generateEventID("answer-done")
if !recovered {
e.eventBus.Emit(ctx, event.Event{
ID: answerID,
Type: event.EventAgentFinalAnswer,
SessionID: sessionID,
Data: event.AgentFinalAnswerData{
Content: answer,
Done: false,
},
})
}
e.eventBus.Emit(ctx, event.Event{
ID: answerID,
Type: event.EventAgentFinalAnswer,
SessionID: sessionID,
Data: event.AgentFinalAnswerData{
Content: "",
Done: true,
},
})
pipelineFields := map[string]interface{}{
"iteration": iteration,
"round": iteration + 1,
"answer_len": len(answer),
"recovered": recovered,
}
if recovered {
common.PipelineInfo(ctx, "Agent", "final_answer_tool", pipelineFields)
} else {
pipelineFields["raw_args"] = rawArgs
common.PipelineWarn(ctx, "Agent", "final_answer_tool_parse_failed", pipelineFields)
}
return responseVerdict{
isDone: true,
finalAnswer: answer,
step: step,
}
}
}
+126
View File
@@ -0,0 +1,126 @@
package agent
import (
"context"
"testing"
"time"
agenttools "github.com/Tencent/WeKnora/internal/agent/tools"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
)
// newFinalAnswerResponse builds a ChatResponse that carries a single
// final_answer tool call with the given raw JSON arguments.
func newFinalAnswerResponse(rawArgs string) *types.ChatResponse {
return &types.ChatResponse{
FinishReason: "tool_calls",
ToolCalls: []types.LLMToolCall{
{
ID: "call-1",
Type: "function",
Function: types.FunctionCall{
Name: agenttools.ToolFinalAnswer,
Arguments: rawArgs,
},
},
},
}
}
// TestAnalyzeResponse_FinalAnswer_ValidArgs guards the happy path: well-formed
// arguments must be extracted into the final answer and terminate the loop.
func TestAnalyzeResponse_FinalAnswer_ValidArgs(t *testing.T) {
engine := newTestEngine(t, &mockChat{})
resp := newFinalAnswerResponse(`{"answer": "Here is the answer."}`)
verdict := engine.analyzeResponse(
context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(),
)
assert.True(t, verdict.isDone, "final_answer must terminate the loop")
assert.Equal(t, "Here is the answer.", verdict.finalAnswer)
}
// TestAnalyzeResponse_FinalAnswer_MalformedJSON_RecoveredViaRepair covers the
// common case reported in issue #1008: the LLM emits final_answer with a
// trailing comma / missing brace. RepairJSON should recover the answer and
// the loop must still terminate in this single round (not re-invoke
// final_answer in the next round).
func TestAnalyzeResponse_FinalAnswer_MalformedJSON_RecoveredViaRepair(t *testing.T) {
engine := newTestEngine(t, &mockChat{})
resp := newFinalAnswerResponse(`{"answer": "repaired"`) // missing closing brace
verdict := engine.analyzeResponse(
context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(),
)
assert.True(t, verdict.isDone,
"final_answer must terminate the loop even when JSON repair is needed")
assert.Equal(t, "repaired", verdict.finalAnswer)
}
// TestAnalyzeResponse_FinalAnswer_UnrecoverableArgs_StillTerminates is the
// direct regression test for issue #1008: when the arguments are so malformed
// that even RepairJSON + regex cannot recover an answer, the loop MUST still
// terminate (with a user-visible fallback message) rather than continuing and
// letting the LLM re-emit final_answer on the next round.
func TestAnalyzeResponse_FinalAnswer_UnrecoverableArgs_StillTerminates(t *testing.T) {
engine := newTestEngine(t, &mockChat{})
// No `answer` key at all — strict parse succeeds (returns zero-value
// answer), RepairJSON is a no-op on already-valid JSON, regex finds
// nothing. All three tiers fail to recover an answer.
resp := newFinalAnswerResponse(`{"unexpected": "field"}`)
verdict := engine.analyzeResponse(
context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(),
)
assert.True(t, verdict.isDone,
"final_answer must terminate the loop even when args are unrecoverable — "+
"otherwise the LLM re-emits final_answer and duplicates the answer (issue #1008)")
assert.Equal(t, finalAnswerParseFallback, verdict.finalAnswer,
"unrecoverable final_answer should surface the parse-failure fallback message")
}
// TestAnalyzeResponse_FinalAnswer_Garbage_StillTerminates exercises the most
// hostile case: completely non-JSON arguments. The loop must still terminate
// — protecting against the duplicate-answer loop reported in issue #1008.
func TestAnalyzeResponse_FinalAnswer_Garbage_StillTerminates(t *testing.T) {
engine := newTestEngine(t, &mockChat{})
resp := newFinalAnswerResponse(`not json at all`)
verdict := engine.analyzeResponse(
context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(),
)
assert.True(t, verdict.isDone)
assert.Equal(t, finalAnswerParseFallback, verdict.finalAnswer)
}
// TestAnalyzeResponse_NonFinalAnswerTool_DoesNotTerminate is a regression
// guard: only final_answer is terminal. Other tool calls (e.g. thinking,
// knowledge_search) must keep the loop running.
func TestAnalyzeResponse_NonFinalAnswerTool_DoesNotTerminate(t *testing.T) {
engine := newTestEngine(t, &mockChat{})
resp := &types.ChatResponse{
FinishReason: "tool_calls",
ToolCalls: []types.LLMToolCall{
{
ID: "call-1",
Type: "function",
Function: types.FunctionCall{
Name: agenttools.ToolKnowledgeSearch,
Arguments: `{"query": "hi"}`,
},
},
},
}
verdict := engine.analyzeResponse(
context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(),
)
assert.False(t, verdict.isDone,
"non-terminal tool calls must keep the loop running")
}
+71 -8
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"regexp"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
@@ -62,26 +63,88 @@ func NewFinalAnswerTool() *FinalAnswerTool {
func (t *FinalAnswerTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) {
logger.Infof(ctx, "[Tool][FinalAnswer] Execute started")
var input FinalAnswerInput
if err := json.Unmarshal(args, &input); err != nil {
logger.Errorf(ctx, "[Tool][FinalAnswer] Failed to parse args: %v", err)
answer, ok := ParseFinalAnswerArgs(string(args))
if !ok {
logger.Errorf(ctx, "[Tool][FinalAnswer] Failed to parse args (even with repair): %s", string(args))
return &types.ToolResult{
Success: false,
Error: fmt.Sprintf("Failed to parse args: %v", err),
}, err
Error: "Failed to parse final_answer args: malformed JSON and no recoverable answer field",
}, fmt.Errorf("malformed final_answer arguments")
}
if input.Answer == "" {
if answer == "" {
return &types.ToolResult{
Success: false,
Error: "answer must be a non-empty string",
}, fmt.Errorf("answer must be a non-empty string")
}
logger.Infof(ctx, "[Tool][FinalAnswer] Answer length: %d characters", len(input.Answer))
logger.Infof(ctx, "[Tool][FinalAnswer] Answer length: %d characters", len(answer))
return &types.ToolResult{
Success: true,
Output: input.Answer,
Output: answer,
}, nil
}
// answerRegex best-effort extracts the value of an "answer": "..." field from
// a malformed JSON string. Used as a last resort when both strict parsing and
// RepairJSON fail — this keeps the final_answer tool call terminal so the
// agent loop cannot re-enter and emit duplicate answers.
//
// The pattern handles escaped quotes inside the answer via the non-greedy
// body `(?:\\.|[^"\\])*` which consumes either an escape sequence or any
// non-quote, non-backslash char.
var answerRegex = regexp.MustCompile(`"answer"\s*:\s*"((?:\\.|[^"\\])*)"`)
// ParseFinalAnswerArgs extracts the `answer` field from the final_answer
// tool's raw arguments. It is intentionally tolerant of malformed JSON that
// LLMs sometimes emit (unescaped quotes inside the answer, trailing commas,
// truncated closing braces, etc.), applying three fallbacks in order:
//
// 1. Strict json.Unmarshal on the raw string.
// 2. RepairJSON (trailing commas, invalid escapes, bracket balance) + Unmarshal.
// 3. Regex best-effort extraction of the `"answer": "..."` field.
//
// Returns the answer string and a bool indicating whether any path succeeded
// with a non-empty answer. Callers should treat ok=false as "unrecoverable"
// and surface a fallback message to the user, but must still treat the
// tool call as terminal to avoid the agent loop re-emitting final_answer.
func ParseFinalAnswerArgs(raw string) (string, bool) {
var input FinalAnswerInput
if err := json.Unmarshal([]byte(raw), &input); err == nil && input.Answer != "" {
return input.Answer, true
}
repaired := RepairJSON(raw)
if repaired != raw {
if err := json.Unmarshal([]byte(repaired), &input); err == nil && input.Answer != "" {
return input.Answer, true
}
}
if m := answerRegex.FindStringSubmatch(raw); len(m) == 2 {
if unquoted, err := unquoteJSONString(m[1]); err == nil && unquoted != "" {
return unquoted, true
}
// Unquoting failed: the capture may contain invalid escapes. Fall back
// to returning the raw capture so the user still sees *something*.
if m[1] != "" {
return m[1], true
}
}
return "", false
}
// unquoteJSONString decodes a JSON-escaped string body (without surrounding
// quotes) into its literal form. It wraps the body in quotes and leans on
// json.Unmarshal so we get the standard escape semantics (\n, \", \uXXXX, …)
// without reimplementing them.
func unquoteJSONString(body string) (string, error) {
var out string
if err := json.Unmarshal([]byte(`"`+body+`"`), &out); err != nil {
return "", err
}
return out, nil
}
+61
View File
@@ -0,0 +1,61 @@
package tools
import (
"testing"
"github.com/stretchr/testify/assert"
)
// TestParseFinalAnswerArgs covers the three-tier recovery path used by both
// the final_answer tool and the ReAct loop's terminal detection (issue #1008).
func TestParseFinalAnswerArgs(t *testing.T) {
t.Run("strict JSON is parsed as-is", func(t *testing.T) {
got, ok := ParseFinalAnswerArgs(`{"answer": "Hello, world."}`)
assert.True(t, ok)
assert.Equal(t, "Hello, world.", got)
})
t.Run("trailing comma is recovered via RepairJSON", func(t *testing.T) {
got, ok := ParseFinalAnswerArgs(`{"answer": "Hi",}`)
assert.True(t, ok)
assert.Equal(t, "Hi", got)
})
t.Run("missing closing brace is recovered via RepairJSON", func(t *testing.T) {
got, ok := ParseFinalAnswerArgs(`{"answer": "Hi"`)
assert.True(t, ok)
assert.Equal(t, "Hi", got)
})
t.Run("invalid backslash escape is recovered via RepairJSON", func(t *testing.T) {
// LLM forgot to double-escape the regex metachar — RepairJSON should
// rewrite "\+" to "\\+" so Unmarshal succeeds.
got, ok := ParseFinalAnswerArgs(`{"answer": "C\+\+"}`)
assert.True(t, ok)
assert.Equal(t, `C\+\+`, got)
})
t.Run("unescaped inner quote is recovered via regex fallback", func(t *testing.T) {
// Neither strict parse nor RepairJSON can recover this one; the regex
// still captures the well-formed prefix up to the rogue quote.
raw := `{"answer": "She said "hello" to me"}`
got, ok := ParseFinalAnswerArgs(raw)
assert.True(t, ok)
assert.NotEmpty(t, got)
})
t.Run("missing answer field returns not ok", func(t *testing.T) {
_, ok := ParseFinalAnswerArgs(`{"other": "value"}`)
assert.False(t, ok)
})
t.Run("empty answer returns not ok", func(t *testing.T) {
_, ok := ParseFinalAnswerArgs(`{"answer": ""}`)
assert.False(t, ok)
})
t.Run("completely garbled input returns not ok", func(t *testing.T) {
_, ok := ParseFinalAnswerArgs(`not json at all`)
assert.False(t, ok)
})
}