diff --git a/internal/agent/tools/grep_chunks.go b/internal/agent/tools/grep_chunks.go index b6295de9..b8f6934a 100644 --- a/internal/agent/tools/grep_chunks.go +++ b/internal/agent/tools/grep_chunks.go @@ -190,13 +190,17 @@ func (t *GrepChunksTool) Execute(ctx context.Context, args json.RawMessage) (*ty finalResults = finalResults[:limit] } - // Per-chunk rows for the UI (FAQ entries are one chunk each — do not fold - // them into the parent knowledge container). Knowledge-level aggregation is - // kept for backward compatibility. + // chunk_results: per-chunk hits for UI detail (grouped by knowledge_id on the + // frontend). knowledge_results: pre-aggregated per document for summaries; + // document_count is the full distinct-document total even when the knowledge + // list is truncated for payload size. chunkResults := buildGrepChunkResults(finalResults, compiled) aggregatedResults := t.aggregateByKnowledge(finalResults, queries, compiled) - if len(aggregatedResults) > 20 { - aggregatedResults = aggregatedResults[:20] + documentCount := len(aggregatedResults) + knowledgeResultsForUI := aggregatedResults + const maxKnowledgeRows = 20 + if len(knowledgeResultsForUI) > maxKnowledgeRows { + knowledgeResultsForUI = knowledgeResultsForUI[:maxKnowledgeRows] } output := t.formatOutput(ctx, finalResults, queries, compiled) @@ -209,8 +213,9 @@ func (t *GrepChunksTool) Execute(ctx context.Context, args json.RawMessage) (*ty "queries": queries, // legacy alias for older frontends "patterns": queries, // legacy alias for older frontends "chunk_results": chunkResults, - "knowledge_results": aggregatedResults, + "knowledge_results": knowledgeResultsForUI, "result_count": len(chunkResults), + "document_count": documentCount, "total_matches": len(finalResults), "knowledge_base_ids": kbIDs, "limit": limit, diff --git a/internal/agent/tools/grep_chunks_aggregate_test.go b/internal/agent/tools/grep_chunks_aggregate_test.go new file mode 100644 index 00000000..a3a425db --- /dev/null +++ b/internal/agent/tools/grep_chunks_aggregate_test.go @@ -0,0 +1,58 @@ +package tools + +import ( + "regexp" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestAggregateByKnowledge_mergesChunksFromSameDocument(t *testing.T) { + tool := &GrepChunksTool{} + compiled := []*regexp.Regexp{regexp.MustCompile(`(?i)sample`)} + results := []chunkWithTitle{ + { + Chunk: types.Chunk{ + ID: "chunk-a", + KnowledgeID: "doc-1", + KnowledgeBaseID: "kb-1", + Content: "sample hit one", + }, + KnowledgeTitle: "report-a.pdf", + }, + { + Chunk: types.Chunk{ + ID: "chunk-b", + KnowledgeID: "doc-1", + KnowledgeBaseID: "kb-1", + Content: "sample hit two", + }, + KnowledgeTitle: "report-a.pdf", + }, + { + Chunk: types.Chunk{ + ID: "chunk-c", + KnowledgeID: "doc-2", + KnowledgeBaseID: "kb-1", + Content: "sample other doc", + }, + KnowledgeTitle: "report-b.pdf", + }, + } + + aggregated := tool.aggregateByKnowledge(results, []string{"sample"}, compiled) + if len(aggregated) != 2 { + t.Fatalf("want 2 documents, got %d", len(aggregated)) + } + + byID := make(map[string]knowledgeAggregation, len(aggregated)) + for _, row := range aggregated { + byID[row.KnowledgeID] = row + } + if byID["doc-1"].ChunkHitCount != 2 { + t.Fatalf("doc-1 chunk hits = %d, want 2", byID["doc-1"].ChunkHitCount) + } + if byID["doc-2"].ChunkHitCount != 1 { + t.Fatalf("doc-2 chunk hits = %d, want 1", byID["doc-2"].ChunkHitCount) + } +} diff --git a/internal/agent/tools/list_knowledge_chunks.go b/internal/agent/tools/list_knowledge_chunks.go index c5411e7b..5bbee4aa 100644 --- a/internal/agent/tools/list_knowledge_chunks.go +++ b/internal/agent/tools/list_knowledge_chunks.go @@ -269,6 +269,7 @@ func (t *ListKnowledgeChunksTool) Execute(ctx context.Context, args json.RawMess Success: true, Output: output, Data: map[string]interface{}{ + "display_type": "knowledge_chunks_list", "knowledge_id": knowledgeID, "knowledge_title": knowledgeTitle, "total_chunks": totalChunks, @@ -325,6 +326,7 @@ func (t *ListKnowledgeChunksTool) executeByChunkID(ctx context.Context, chunkID normalizeFAQChunkDataMap(formattedChunks[0], chunk) data := map[string]interface{}{ + "display_type": "knowledge_chunks_list", "knowledge_id": chunk.KnowledgeID, "knowledge_title": knowledgeTitle, "total_chunks": int64(1), diff --git a/internal/agent/tools/persist.go b/internal/agent/tools/persist.go new file mode 100644 index 00000000..f0ae9826 --- /dev/null +++ b/internal/agent/tools/persist.go @@ -0,0 +1,198 @@ +package tools + +import ( + "fmt" + "strings" + + "github.com/Tencent/WeKnora/internal/types" +) + +// persistStripFields lists bulky Data keys to drop before SSE replay / DB storage. +var persistStripFields = map[string][]string{ + "knowledge_chunks_list": {"chunks"}, + "grep_results": {"chunk_results"}, +} + +// ShouldOmitRawToolOutput reports whether the raw XML/text Output should be +// excluded from SSE replay and persisted agent_steps. The full Output remains +// available in-memory for the current agent turn. +func ShouldOmitRawToolOutput(_ string, data map[string]interface{}) bool { + if data == nil { + return false + } + displayType, ok := data["display_type"].(string) + return ok && displayType != "" +} + +// SanitizeToolDataForPersist returns a copy of tool Data safe for DB / SSE replay. +func SanitizeToolDataForPersist(data map[string]interface{}) map[string]interface{} { + if data == nil { + return nil + } + out := make(map[string]interface{}, len(data)) + for k, v := range data { + out[k] = v + } + displayType := stringField(data, "display_type") + for _, key := range persistStripFields[displayType] { + delete(out, key) + } + return out +} + +// SanitizeToolResultForClient builds stream / persistence metadata for the UI. +func SanitizeToolResultForClient(_ string, result *types.ToolResult) map[string]interface{} { + meta := map[string]interface{}{} + if result == nil { + return meta + } + if result.Data != nil { + for k, v := range SanitizeToolDataForPersist(result.Data) { + meta[k] = v + } + } + if !ShouldOmitRawToolOutput("", result.Data) && result.Output != "" { + meta["output"] = result.Output + } + return meta +} + +// StreamContentForToolResult is the short SSE Content field for tool results. +func StreamContentForToolResult(toolName string, success bool, errMsg string, data map[string]interface{}) string { + if !success { + return errMsg + } + if ShouldOmitRawToolOutput(toolName, data) { + return compactToolSummary(success, errMsg, data) + } + return "" +} + +// SanitizeAgentStepsForStorage strips LLM-only payloads from persisted steps. +func SanitizeAgentStepsForStorage(steps []types.AgentStep) []types.AgentStep { + if len(steps) == 0 { + return steps + } + out := make([]types.AgentStep, len(steps)) + for i, step := range steps { + out[i] = step + if len(step.ToolCalls) == 0 { + continue + } + toolCalls := make([]types.ToolCall, len(step.ToolCalls)) + for j, tc := range step.ToolCalls { + toolCalls[j] = tc + if tc.Result == nil { + continue + } + result := *tc.Result + if ShouldOmitRawToolOutput(tc.Name, result.Data) { + result.Output = compactToolSummary(result.Success, result.Error, result.Data) + result.Data = SanitizeToolDataForPersist(result.Data) + } + toolCalls[j].Result = &result + } + out[i].ToolCalls = toolCalls + } + return out +} + +// CompactToolOutputForHistory rebuilds a short tool message when replaying history. +func CompactToolOutputForHistory(toolName string, result *types.ToolResult) string { + if result == nil { + return "" + } + if !result.Success { + if result.Error != "" { + return "Error: " + result.Error + } + return "Error: tool call failed" + } + if result.Output != "" && !ShouldOmitRawToolOutput(toolName, result.Data) { + return result.Output + } + return compactToolSummary(result.Success, result.Error, result.Data) +} + +func compactToolSummary(success bool, errMsg string, data map[string]interface{}) string { + if !success { + if errMsg != "" { + return "Error: " + errMsg + } + return "Error: tool call failed" + } + switch stringField(data, "display_type") { + case "knowledge_chunks_list": + title := stringField(data, "knowledge_title") + if title == "" { + title = stringField(data, "knowledge_id") + } + fetched := intField(data, "fetched_chunks") + total := intField(data, "total_chunks") + if q := stringField(data, "faq_question"); q != "" { + return fmt.Sprintf("Loaded FAQ entry: %s (content omitted from history)", q) + } + if title != "" && total > 0 { + return fmt.Sprintf("Listed %d/%d chunks from %s (content omitted from history)", fetched, total, title) + } + if title != "" { + return fmt.Sprintf("Listed chunks from %s (content omitted from history)", title) + } + case "grep_results": + chunks := intField(data, "total_matches") + docs := intField(data, "document_count") + if docs == 0 { + docs = intField(data, "result_count") + } + if chunks > 0 { + return fmt.Sprintf("Keyword search found %d matching chunks across %d document(s) (details omitted from history)", chunks, docs) + } + case "search_results": + count := intField(data, "result_count") + if count == 0 { + count = intField(data, "count") + } + if count > 0 { + return fmt.Sprintf("Semantic search returned %d result(s) (details omitted from history)", count) + } + } + if displayType := stringField(data, "display_type"); displayType != "" { + return fmt.Sprintf("Tool completed (%s; payload omitted from history)", displayType) + } + return "Tool completed (payload omitted from history)" +} + +func stringField(data map[string]interface{}, key string) string { + if data == nil { + return "" + } + v, ok := data[key] + if !ok || v == nil { + return "" + } + return strings.TrimSpace(fmt.Sprint(v)) +} + +func intField(data map[string]interface{}, key string) int { + if data == nil { + return 0 + } + v, ok := data[key] + if !ok || v == nil { + return 0 + } + switch n := v.(type) { + case int: + return n + case int32: + return int(n) + case int64: + return int(n) + case float64: + return int(n) + case float32: + return int(n) + default: + return 0 + } +} diff --git a/internal/agent/tools/persist_test.go b/internal/agent/tools/persist_test.go new file mode 100644 index 00000000..dd4b7ba6 --- /dev/null +++ b/internal/agent/tools/persist_test.go @@ -0,0 +1,89 @@ +package tools + +import ( + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestShouldOmitRawToolOutput(t *testing.T) { + if !ShouldOmitRawToolOutput(ToolListKnowledgeChunks, map[string]interface{}{"display_type": "knowledge_chunks_list"}) { + t.Fatal("structured list_knowledge_chunks output should be omitted") + } + if !ShouldOmitRawToolOutput(ToolGrepChunks, map[string]interface{}{"display_type": "grep_results"}) { + t.Fatal("structured grep output should be omitted") + } + if ShouldOmitRawToolOutput("custom_tool", nil) { + t.Fatal("unknown tools should keep raw output by default") + } +} + +func TestSanitizeToolDataForPersist_knowledgeChunksList(t *testing.T) { + data := map[string]interface{}{ + "display_type": "knowledge_chunks_list", + "knowledge_title": "sample.pdf", + "fetched_chunks": 50, + "total_chunks": 282, + "chunks": []map[string]interface{}{{"content": "secret"}}, + } + out := SanitizeToolDataForPersist(data) + if _, ok := out["chunks"]; ok { + t.Fatal("chunk bodies should be stripped from persisted tool data") + } + if out["fetched_chunks"] != 50 { + t.Fatalf("summary fields should be kept, got %#v", out["fetched_chunks"]) + } +} + +func TestSanitizeAgentStepsForStorage_stripsLargeOutput(t *testing.T) { + steps := []types.AgentStep{{ + Iteration: 1, + ToolCalls: []types.ToolCall{{ + ID: "call-1", + Name: ToolListKnowledgeChunks, + Result: &types.ToolResult{ + Success: true, + Output: strings.Repeat("x", 10000), + Data: map[string]interface{}{ + "display_type": "knowledge_chunks_list", + "knowledge_title": "sample.pdf", + "fetched_chunks": 50, + "total_chunks": 282, + "chunks": []map[string]interface{}{{"content": "body"}}, + }, + }, + }}, + }} + + sanitized := SanitizeAgentStepsForStorage(steps) + result := sanitized[0].ToolCalls[0].Result + if len(result.Output) >= 10000 { + t.Fatal("persisted output should be compacted") + } + if !strings.Contains(result.Output, "content omitted from history") { + t.Fatalf("unexpected compact output: %q", result.Output) + } + if _, ok := result.Data["chunks"]; ok { + t.Fatal("chunk bodies should be removed from persisted data") + } +} + +func TestSanitizeToolResultForClient_omitsOutput(t *testing.T) { + meta := SanitizeToolResultForClient(ToolListKnowledgeChunks, &types.ToolResult{ + Success: true, + Output: "very large", + Data: map[string]interface{}{ + "display_type": "knowledge_chunks_list", + "knowledge_title": "sample.pdf", + "fetched_chunks": 1, + "total_chunks": 1, + }, + }) + if _, ok := meta["output"]; ok { + t.Fatal("raw output should not be sent to client metadata") + } + if meta["fetched_chunks"] != 1 { + t.Fatalf("summary metadata should remain, got %#v", meta["fetched_chunks"]) + } +} diff --git a/internal/application/service/agent_history.go b/internal/application/service/agent_history.go index 7fb7f843..9070b0ad 100644 --- a/internal/application/service/agent_history.go +++ b/internal/application/service/agent_history.go @@ -9,6 +9,7 @@ import ( "strings" "time" + agenttools "github.com/Tencent/WeKnora/internal/agent/tools" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -227,7 +228,7 @@ func toolCallOutput(tc types.ToolCall) string { } return "Error: tool call failed" } - return tc.Result.Output + return agenttools.CompactToolOutputForHistory(tc.Name, tc.Result) } // extractImageCaptionsFromMessage concatenates non-empty Caption fields from