fix(agent): terminate ReAct loop on final_answer regardless of arg parse success
The agent loop previously kept running when the LLM emitted final_answer with malformed JSON arguments: observe.analyzeResponse strictly json.Unmarshal'd the raw args and returned isDone=false on failure, while act.runToolCall separately repaired the JSON and executed the tool. The mismatch made final_answer non-terminal, so the LLM saw its own answer in the next round's tool result and re-emitted final_answer, surfacing near-identical answers multiple times to the user (issue #1008). Fix by unifying parsing in a new tools.ParseFinalAnswerArgs helper with three fallbacks (strict -> RepairJSON -> regex extraction) and making final_answer always terminal in analyzeResponse. When no answer can be recovered, emit a user-visible fallback message and a Done=true event so the UI still resolves. Also route final_answer.Execute through the same helper so tool execution and loop termination stay consistent. Adds regression tests in observe_test.go and final_answer_test.go covering valid args, RepairJSON recovery, unrecoverable args, and garbage input. Refs: Tencent/WeKnora#1008
This commit is contained in:
+84
-32
@@ -16,6 +16,13 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// finalAnswerParseFallback is the user-visible message surfaced when the LLM
|
||||
// calls final_answer with arguments we cannot recover into an answer string
|
||||
// (even after RepairJSON + regex fallback). Terminating the loop with this
|
||||
// message prevents the agent from re-entering and emitting duplicate answers
|
||||
// on every subsequent round — the behavior reported in issue #1008.
|
||||
const finalAnswerParseFallback = "Sorry, the model's final answer could not be parsed due to malformed output. Please try again or rephrase your question."
|
||||
|
||||
// manageContextWindow consolidates or compresses messages if approaching the token limit.
|
||||
// currentTokens is the caller's best estimate of the current context size (using
|
||||
// API-reported Usage when available, falling back to BPE estimation).
|
||||
@@ -156,42 +163,87 @@ func (e *AgentEngine) analyzeResponse(
|
||||
}
|
||||
}
|
||||
|
||||
// Case 2: final_answer tool call present
|
||||
// Case 2: final_answer tool call present.
|
||||
//
|
||||
// final_answer is always a terminal signal: regardless of whether we can
|
||||
// parse its arguments, we must end the ReAct loop here. Otherwise the LLM
|
||||
// will see the tool result in the next round, re-invoke final_answer with
|
||||
// near-identical content, and surface duplicate answers to the user (see
|
||||
// issue #1008). Parse with three levels of tolerance:
|
||||
//
|
||||
// 1. strict json.Unmarshal
|
||||
// 2. RepairJSON + Unmarshal
|
||||
// 3. regex best-effort extraction of the "answer" field
|
||||
//
|
||||
// If all three fail, terminate with a user-visible fallback message.
|
||||
if len(response.ToolCalls) > 0 {
|
||||
for _, tc := range response.ToolCalls {
|
||||
if tc.Function.Name == agenttools.ToolFinalAnswer {
|
||||
var faArgs struct {
|
||||
Answer string `json:"answer"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(tc.Function.Arguments), &faArgs); err != nil {
|
||||
logger.Warnf(ctx, "[Agent][Round-%d] Failed to parse final_answer args: %v",
|
||||
iteration+1, err)
|
||||
} else {
|
||||
logger.Infof(ctx, "[Agent][Round-%d] final_answer tool: answer=%d chars, duration=%dms",
|
||||
iteration+1, len(faArgs.Answer), time.Since(roundStart).Milliseconds())
|
||||
if tc.Function.Name != agenttools.ToolFinalAnswer {
|
||||
continue
|
||||
}
|
||||
|
||||
e.eventBus.Emit(ctx, event.Event{
|
||||
ID: generateEventID("answer-done"),
|
||||
Type: event.EventAgentFinalAnswer,
|
||||
SessionID: sessionID,
|
||||
Data: event.AgentFinalAnswerData{
|
||||
Content: "",
|
||||
Done: true,
|
||||
},
|
||||
})
|
||||
common.PipelineInfo(ctx, "Agent", "final_answer_tool", map[string]interface{}{
|
||||
"iteration": iteration,
|
||||
"round": iteration + 1,
|
||||
"answer_len": len(faArgs.Answer),
|
||||
})
|
||||
rawArgs := tc.Function.Arguments
|
||||
answer, ok := agenttools.ParseFinalAnswerArgs(rawArgs)
|
||||
recovered := false
|
||||
if !ok {
|
||||
// Could not recover any answer text — fall back to a generic
|
||||
// message so the user doesn't see a blank response.
|
||||
logger.Warnf(ctx, "[Agent][Round-%d] Failed to parse final_answer args (args=%q) — "+
|
||||
"terminating loop with fallback message",
|
||||
iteration+1, rawArgs)
|
||||
answer = finalAnswerParseFallback
|
||||
} else {
|
||||
recovered = true
|
||||
logger.Infof(ctx, "[Agent][Round-%d] final_answer tool: answer=%d chars, duration=%dms",
|
||||
iteration+1, len(answer), time.Since(roundStart).Milliseconds())
|
||||
}
|
||||
|
||||
return responseVerdict{
|
||||
isDone: true,
|
||||
finalAnswer: faArgs.Answer,
|
||||
step: step,
|
||||
}
|
||||
}
|
||||
break
|
||||
// Always emit the final answer content and Done=true marker to the
|
||||
// event bus. When strict parsing succeeded earlier in this turn,
|
||||
// streamThinkingToEventBus already streamed the answer chunks, so
|
||||
// we only need the Done marker in that common case. When we fell
|
||||
// back to the generic message, however, the UI has not yet seen
|
||||
// any answer content — emit both Content and Done to make the
|
||||
// fallback visible to the user.
|
||||
answerID := generateEventID("answer-done")
|
||||
if !recovered {
|
||||
e.eventBus.Emit(ctx, event.Event{
|
||||
ID: answerID,
|
||||
Type: event.EventAgentFinalAnswer,
|
||||
SessionID: sessionID,
|
||||
Data: event.AgentFinalAnswerData{
|
||||
Content: answer,
|
||||
Done: false,
|
||||
},
|
||||
})
|
||||
}
|
||||
e.eventBus.Emit(ctx, event.Event{
|
||||
ID: answerID,
|
||||
Type: event.EventAgentFinalAnswer,
|
||||
SessionID: sessionID,
|
||||
Data: event.AgentFinalAnswerData{
|
||||
Content: "",
|
||||
Done: true,
|
||||
},
|
||||
})
|
||||
|
||||
pipelineFields := map[string]interface{}{
|
||||
"iteration": iteration,
|
||||
"round": iteration + 1,
|
||||
"answer_len": len(answer),
|
||||
"recovered": recovered,
|
||||
}
|
||||
if recovered {
|
||||
common.PipelineInfo(ctx, "Agent", "final_answer_tool", pipelineFields)
|
||||
} else {
|
||||
pipelineFields["raw_args"] = rawArgs
|
||||
common.PipelineWarn(ctx, "Agent", "final_answer_tool_parse_failed", pipelineFields)
|
||||
}
|
||||
|
||||
return responseVerdict{
|
||||
isDone: true,
|
||||
finalAnswer: answer,
|
||||
step: step,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
agenttools "github.com/Tencent/WeKnora/internal/agent/tools"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// newFinalAnswerResponse builds a ChatResponse that carries a single
|
||||
// final_answer tool call with the given raw JSON arguments.
|
||||
func newFinalAnswerResponse(rawArgs string) *types.ChatResponse {
|
||||
return &types.ChatResponse{
|
||||
FinishReason: "tool_calls",
|
||||
ToolCalls: []types.LLMToolCall{
|
||||
{
|
||||
ID: "call-1",
|
||||
Type: "function",
|
||||
Function: types.FunctionCall{
|
||||
Name: agenttools.ToolFinalAnswer,
|
||||
Arguments: rawArgs,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnalyzeResponse_FinalAnswer_ValidArgs guards the happy path: well-formed
|
||||
// arguments must be extracted into the final answer and terminate the loop.
|
||||
func TestAnalyzeResponse_FinalAnswer_ValidArgs(t *testing.T) {
|
||||
engine := newTestEngine(t, &mockChat{})
|
||||
resp := newFinalAnswerResponse(`{"answer": "Here is the answer."}`)
|
||||
|
||||
verdict := engine.analyzeResponse(
|
||||
context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(),
|
||||
)
|
||||
|
||||
assert.True(t, verdict.isDone, "final_answer must terminate the loop")
|
||||
assert.Equal(t, "Here is the answer.", verdict.finalAnswer)
|
||||
}
|
||||
|
||||
// TestAnalyzeResponse_FinalAnswer_MalformedJSON_RecoveredViaRepair covers the
|
||||
// common case reported in issue #1008: the LLM emits final_answer with a
|
||||
// trailing comma / missing brace. RepairJSON should recover the answer and
|
||||
// the loop must still terminate in this single round (not re-invoke
|
||||
// final_answer in the next round).
|
||||
func TestAnalyzeResponse_FinalAnswer_MalformedJSON_RecoveredViaRepair(t *testing.T) {
|
||||
engine := newTestEngine(t, &mockChat{})
|
||||
resp := newFinalAnswerResponse(`{"answer": "repaired"`) // missing closing brace
|
||||
|
||||
verdict := engine.analyzeResponse(
|
||||
context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(),
|
||||
)
|
||||
|
||||
assert.True(t, verdict.isDone,
|
||||
"final_answer must terminate the loop even when JSON repair is needed")
|
||||
assert.Equal(t, "repaired", verdict.finalAnswer)
|
||||
}
|
||||
|
||||
// TestAnalyzeResponse_FinalAnswer_UnrecoverableArgs_StillTerminates is the
|
||||
// direct regression test for issue #1008: when the arguments are so malformed
|
||||
// that even RepairJSON + regex cannot recover an answer, the loop MUST still
|
||||
// terminate (with a user-visible fallback message) rather than continuing and
|
||||
// letting the LLM re-emit final_answer on the next round.
|
||||
func TestAnalyzeResponse_FinalAnswer_UnrecoverableArgs_StillTerminates(t *testing.T) {
|
||||
engine := newTestEngine(t, &mockChat{})
|
||||
// No `answer` key at all — strict parse succeeds (returns zero-value
|
||||
// answer), RepairJSON is a no-op on already-valid JSON, regex finds
|
||||
// nothing. All three tiers fail to recover an answer.
|
||||
resp := newFinalAnswerResponse(`{"unexpected": "field"}`)
|
||||
|
||||
verdict := engine.analyzeResponse(
|
||||
context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(),
|
||||
)
|
||||
|
||||
assert.True(t, verdict.isDone,
|
||||
"final_answer must terminate the loop even when args are unrecoverable — "+
|
||||
"otherwise the LLM re-emits final_answer and duplicates the answer (issue #1008)")
|
||||
assert.Equal(t, finalAnswerParseFallback, verdict.finalAnswer,
|
||||
"unrecoverable final_answer should surface the parse-failure fallback message")
|
||||
}
|
||||
|
||||
// TestAnalyzeResponse_FinalAnswer_Garbage_StillTerminates exercises the most
|
||||
// hostile case: completely non-JSON arguments. The loop must still terminate
|
||||
// — protecting against the duplicate-answer loop reported in issue #1008.
|
||||
func TestAnalyzeResponse_FinalAnswer_Garbage_StillTerminates(t *testing.T) {
|
||||
engine := newTestEngine(t, &mockChat{})
|
||||
resp := newFinalAnswerResponse(`not json at all`)
|
||||
|
||||
verdict := engine.analyzeResponse(
|
||||
context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(),
|
||||
)
|
||||
|
||||
assert.True(t, verdict.isDone)
|
||||
assert.Equal(t, finalAnswerParseFallback, verdict.finalAnswer)
|
||||
}
|
||||
|
||||
// TestAnalyzeResponse_NonFinalAnswerTool_DoesNotTerminate is a regression
|
||||
// guard: only final_answer is terminal. Other tool calls (e.g. thinking,
|
||||
// knowledge_search) must keep the loop running.
|
||||
func TestAnalyzeResponse_NonFinalAnswerTool_DoesNotTerminate(t *testing.T) {
|
||||
engine := newTestEngine(t, &mockChat{})
|
||||
resp := &types.ChatResponse{
|
||||
FinishReason: "tool_calls",
|
||||
ToolCalls: []types.LLMToolCall{
|
||||
{
|
||||
ID: "call-1",
|
||||
Type: "function",
|
||||
Function: types.FunctionCall{
|
||||
Name: agenttools.ToolKnowledgeSearch,
|
||||
Arguments: `{"query": "hi"}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
verdict := engine.analyzeResponse(
|
||||
context.Background(), resp, types.AgentStep{}, 0, "sess-1", time.Now(),
|
||||
)
|
||||
|
||||
assert.False(t, verdict.isDone,
|
||||
"non-terminal tool calls must keep the loop running")
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
@@ -62,26 +63,88 @@ func NewFinalAnswerTool() *FinalAnswerTool {
|
||||
func (t *FinalAnswerTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) {
|
||||
logger.Infof(ctx, "[Tool][FinalAnswer] Execute started")
|
||||
|
||||
var input FinalAnswerInput
|
||||
if err := json.Unmarshal(args, &input); err != nil {
|
||||
logger.Errorf(ctx, "[Tool][FinalAnswer] Failed to parse args: %v", err)
|
||||
answer, ok := ParseFinalAnswerArgs(string(args))
|
||||
if !ok {
|
||||
logger.Errorf(ctx, "[Tool][FinalAnswer] Failed to parse args (even with repair): %s", string(args))
|
||||
return &types.ToolResult{
|
||||
Success: false,
|
||||
Error: fmt.Sprintf("Failed to parse args: %v", err),
|
||||
}, err
|
||||
Error: "Failed to parse final_answer args: malformed JSON and no recoverable answer field",
|
||||
}, fmt.Errorf("malformed final_answer arguments")
|
||||
}
|
||||
|
||||
if input.Answer == "" {
|
||||
if answer == "" {
|
||||
return &types.ToolResult{
|
||||
Success: false,
|
||||
Error: "answer must be a non-empty string",
|
||||
}, fmt.Errorf("answer must be a non-empty string")
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "[Tool][FinalAnswer] Answer length: %d characters", len(input.Answer))
|
||||
logger.Infof(ctx, "[Tool][FinalAnswer] Answer length: %d characters", len(answer))
|
||||
|
||||
return &types.ToolResult{
|
||||
Success: true,
|
||||
Output: input.Answer,
|
||||
Output: answer,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// answerRegex best-effort extracts the value of an "answer": "..." field from
|
||||
// a malformed JSON string. Used as a last resort when both strict parsing and
|
||||
// RepairJSON fail — this keeps the final_answer tool call terminal so the
|
||||
// agent loop cannot re-enter and emit duplicate answers.
|
||||
//
|
||||
// The pattern handles escaped quotes inside the answer via the non-greedy
|
||||
// body `(?:\\.|[^"\\])*` which consumes either an escape sequence or any
|
||||
// non-quote, non-backslash char.
|
||||
var answerRegex = regexp.MustCompile(`"answer"\s*:\s*"((?:\\.|[^"\\])*)"`)
|
||||
|
||||
// ParseFinalAnswerArgs extracts the `answer` field from the final_answer
|
||||
// tool's raw arguments. It is intentionally tolerant of malformed JSON that
|
||||
// LLMs sometimes emit (unescaped quotes inside the answer, trailing commas,
|
||||
// truncated closing braces, etc.), applying three fallbacks in order:
|
||||
//
|
||||
// 1. Strict json.Unmarshal on the raw string.
|
||||
// 2. RepairJSON (trailing commas, invalid escapes, bracket balance) + Unmarshal.
|
||||
// 3. Regex best-effort extraction of the `"answer": "..."` field.
|
||||
//
|
||||
// Returns the answer string and a bool indicating whether any path succeeded
|
||||
// with a non-empty answer. Callers should treat ok=false as "unrecoverable"
|
||||
// and surface a fallback message to the user, but must still treat the
|
||||
// tool call as terminal to avoid the agent loop re-emitting final_answer.
|
||||
func ParseFinalAnswerArgs(raw string) (string, bool) {
|
||||
var input FinalAnswerInput
|
||||
if err := json.Unmarshal([]byte(raw), &input); err == nil && input.Answer != "" {
|
||||
return input.Answer, true
|
||||
}
|
||||
|
||||
repaired := RepairJSON(raw)
|
||||
if repaired != raw {
|
||||
if err := json.Unmarshal([]byte(repaired), &input); err == nil && input.Answer != "" {
|
||||
return input.Answer, true
|
||||
}
|
||||
}
|
||||
|
||||
if m := answerRegex.FindStringSubmatch(raw); len(m) == 2 {
|
||||
if unquoted, err := unquoteJSONString(m[1]); err == nil && unquoted != "" {
|
||||
return unquoted, true
|
||||
}
|
||||
// Unquoting failed: the capture may contain invalid escapes. Fall back
|
||||
// to returning the raw capture so the user still sees *something*.
|
||||
if m[1] != "" {
|
||||
return m[1], true
|
||||
}
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
// unquoteJSONString decodes a JSON-escaped string body (without surrounding
|
||||
// quotes) into its literal form. It wraps the body in quotes and leans on
|
||||
// json.Unmarshal so we get the standard escape semantics (\n, \", \uXXXX, …)
|
||||
// without reimplementing them.
|
||||
func unquoteJSONString(body string) (string, error) {
|
||||
var out string
|
||||
if err := json.Unmarshal([]byte(`"`+body+`"`), &out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestParseFinalAnswerArgs covers the three-tier recovery path used by both
|
||||
// the final_answer tool and the ReAct loop's terminal detection (issue #1008).
|
||||
func TestParseFinalAnswerArgs(t *testing.T) {
|
||||
t.Run("strict JSON is parsed as-is", func(t *testing.T) {
|
||||
got, ok := ParseFinalAnswerArgs(`{"answer": "Hello, world."}`)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "Hello, world.", got)
|
||||
})
|
||||
|
||||
t.Run("trailing comma is recovered via RepairJSON", func(t *testing.T) {
|
||||
got, ok := ParseFinalAnswerArgs(`{"answer": "Hi",}`)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "Hi", got)
|
||||
})
|
||||
|
||||
t.Run("missing closing brace is recovered via RepairJSON", func(t *testing.T) {
|
||||
got, ok := ParseFinalAnswerArgs(`{"answer": "Hi"`)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, "Hi", got)
|
||||
})
|
||||
|
||||
t.Run("invalid backslash escape is recovered via RepairJSON", func(t *testing.T) {
|
||||
// LLM forgot to double-escape the regex metachar — RepairJSON should
|
||||
// rewrite "\+" to "\\+" so Unmarshal succeeds.
|
||||
got, ok := ParseFinalAnswerArgs(`{"answer": "C\+\+"}`)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, `C\+\+`, got)
|
||||
})
|
||||
|
||||
t.Run("unescaped inner quote is recovered via regex fallback", func(t *testing.T) {
|
||||
// Neither strict parse nor RepairJSON can recover this one; the regex
|
||||
// still captures the well-formed prefix up to the rogue quote.
|
||||
raw := `{"answer": "She said "hello" to me"}`
|
||||
got, ok := ParseFinalAnswerArgs(raw)
|
||||
assert.True(t, ok)
|
||||
assert.NotEmpty(t, got)
|
||||
})
|
||||
|
||||
t.Run("missing answer field returns not ok", func(t *testing.T) {
|
||||
_, ok := ParseFinalAnswerArgs(`{"other": "value"}`)
|
||||
assert.False(t, ok)
|
||||
})
|
||||
|
||||
t.Run("empty answer returns not ok", func(t *testing.T) {
|
||||
_, ok := ParseFinalAnswerArgs(`{"answer": ""}`)
|
||||
assert.False(t, ok)
|
||||
})
|
||||
|
||||
t.Run("completely garbled input returns not ok", func(t *testing.T) {
|
||||
_, ok := ParseFinalAnswerArgs(`not json at all`)
|
||||
assert.False(t, ok)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user