diff --git a/internal/agent/tools/knowledge_search.go b/internal/agent/tools/knowledge_search.go index 26c44a09..b96fc555 100644 --- a/internal/agent/tools/knowledge_search.go +++ b/internal/agent/tools/knowledge_search.go @@ -299,8 +299,8 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args json.RawMessage) var filteredResults []*searchResultWithMeta if (t.rerankModel != nil || t.chatModel != nil) && len(deduplicatedBeforeRerank) > 0 && rerankQuery != "" { - logger.Infof(ctx, "[Tool][KnowledgeSearch] Applying rerank, input: %d results, queries: %v", - len(deduplicatedBeforeRerank), queries) + logger.Infof(ctx, "[Tool][KnowledgeSearch] Applying rerank, input: %d results, threshold: %.2f, queries: %v", + len(deduplicatedBeforeRerank), t.rerankThreshold(), queries) rerankedResults, err := t.rerankResults(ctx, rerankQuery, deduplicatedBeforeRerank) if err != nil { logger.Warnf(ctx, "[Tool][KnowledgeSearch] Rerank failed, using original results: %v", err) @@ -361,12 +361,12 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args json.RawMessage) return deduplicatedResults[i].KnowledgeID < deduplicatedResults[j].KnowledgeID }) - // Log top results + // Log all ranked results (including lower ranks for rerank debugging) if len(deduplicatedResults) > 0 { - for i := 0; i < len(deduplicatedResults) && i < 5; i++ { - r := deduplicatedResults[i] - logger.Infof(ctx, "[Tool][KnowledgeSearch][Top %d] score=%.3f, type=%s, kb=%s, chunk_id=%s", - i+1, r.Score, r.QueryType, r.KnowledgeID, r.ID) + total := len(deduplicatedResults) + for i, r := range deduplicatedResults { + logger.Infof(ctx, "[Tool][KnowledgeSearch][Rank %d/%d] score=%.3f, type=%s, kb=%s, chunk_id=%s", + i+1, total, r.Score, r.QueryType, r.KnowledgeID, r.ID) } } @@ -594,90 +594,50 @@ func (t *KnowledgeSearchTool) concurrentSearchByTargets( return allResults } -// rerankResults applies reranking to search results using LLM prompt scoring or rerank model +// rerankResults applies reranking to all search results (including FAQ entries) +// using the rerank model or LLM fallback, then filters by threshold and applies +// composite scoring so MMR/sorting uses a single score scale. func (t *KnowledgeSearchTool) rerankResults( ctx context.Context, query string, results []*searchResultWithMeta, ) ([]*searchResultWithMeta, error) { - // Separate FAQ and normal results. - // FAQ results keep original scores and bypass reranking model. - faqResults := make([]*searchResultWithMeta, 0) - rerankCandidates := make([]*searchResultWithMeta, 0, len(results)) - - for _, result := range results { - // Skip reranking for FAQ results (they are explicitly matched Q&A pairs) - if result.KnowledgeBaseType == types.KnowledgeBaseTypeFAQ { - faqResults = append(faqResults, result) - } else { - rerankCandidates = append(rerankCandidates, result) - } - } - - // If there are no candidates to rerank, return original list (already all FAQ) - if len(rerankCandidates) == 0 { + if len(results) == 0 { return results, nil } var ( - rerankedCandidates []*searchResultWithMeta - err error + reranked []*searchResultWithMeta + err error ) - // Apply reranking only to candidates - // Try rerankModel first, fallback to chatModel if rerankModel fails or returns no results if t.rerankModel != nil { - rerankedCandidates, err = t.rerankWithModel(ctx, query, rerankCandidates) - // If rerankModel fails or returns no results, fallback to chatModel - if err != nil || len(rerankedCandidates) == 0 { + reranked, err = t.rerankWithModel(ctx, query, results) + if err != nil || len(reranked) == 0 { if err != nil { logger.Warnf(ctx, "[Tool][KnowledgeSearch] Rerank model failed, falling back to chat model: %v", err) } else { - logger.Warnf(ctx, "[Tool][KnowledgeSearch] Rerank model returned no results, falling back to chat model") + logger.Warnf(ctx, "[Tool][KnowledgeSearch] Rerank model returned no results above threshold, falling back to chat model") } - // Reset error to allow fallback err = nil - // Try chatModel if available if t.chatModel != nil { - rerankedCandidates, err = t.rerankWithLLM(ctx, query, rerankCandidates) - } else { - // No fallback available, use original results - rerankedCandidates = rerankCandidates + reranked, err = t.rerankWithLLM(ctx, query, results) + } else if len(reranked) == 0 { + reranked = results } } } else if t.chatModel != nil { - // No rerankModel, use chatModel directly - rerankedCandidates, err = t.rerankWithLLM(ctx, query, rerankCandidates) + reranked, err = t.rerankWithLLM(ctx, query, results) } else { - // No reranking available, use original results - rerankedCandidates = rerankCandidates + return results, nil } if err != nil { return nil, err } - // Apply composite scoring to reranked results - logger.Debugf(ctx, "[Tool][KnowledgeSearch] Applying composite scoring") - - // Store base scores before composite scoring - for _, result := range rerankedCandidates { - baseScore := result.Score - // Apply composite score - result.Score = t.compositeScore(result, result.Score, baseScore) - } - - // Combine FAQ results (with original order) and reranked candidates - combined := make([]*searchResultWithMeta, 0, len(results)) - combined = append(combined, faqResults...) - combined = append(combined, rerankedCandidates...) - - // Sort by score (descending) to keep consistent output order - sort.Slice(combined, func(i, j int) bool { - return combined[i].Score > combined[j].Score - }) - - return combined, nil + logger.Debugf(ctx, "[Tool][KnowledgeSearch] Rerank produced %d results after threshold filter", len(reranked)) + return reranked, nil } func (t *KnowledgeSearchTool) getFAQMetadata( @@ -732,7 +692,6 @@ func (t *KnowledgeSearchTool) rerankWithLLM( // Process in batches allScores := make([]float64, len(results)) - allReranked := make([]*searchResultWithMeta, 0, len(results)) for batchStart := 0; batchStart < len(results); batchStart += batchSize { batchEnd := batchStart + batchSize @@ -862,23 +821,25 @@ Output only the scores, no explanations or additional text.`, } } - // Create reranked results with new scores - for i, result := range results { - newResult := *result - if i < len(allScores) { - newResult.Score = allScores[i] + // Create rerank rank results and apply the same threshold + composite path as the model. + rankResults := make([]rerank.RankResult, 0, len(results)) + for i, score := range allScores { + if i >= len(results) { + break } - allReranked = append(allReranked, &newResult) + rankResults = append(rankResults, rerank.RankResult{ + Index: i, + RelevanceScore: score, + }) } - - // Sort by new scores (descending) - sort.Slice(allReranked, func(i, j int) bool { - return allReranked[i].Score > allReranked[j].Score + sort.Slice(rankResults, func(i, j int) bool { + return rankResults[i].RelevanceScore > rankResults[j].RelevanceScore }) - logger.Infof(ctx, "[Tool][KnowledgeSearch] LLM reranked %d results from %d original results (processed in batches)", - len(allReranked), len(results)) - return allReranked, nil + ranked := t.applyModelRerankScores(results, rankResults, t.rerankThreshold()) + logger.Infof(ctx, "[Tool][KnowledgeSearch] LLM reranked %d/%d results above threshold %.2f", + len(ranked), len(results), t.rerankThreshold()) + return ranked, nil } // parseScoresFromResponse parses scores from LLM response text @@ -951,42 +912,87 @@ func (t *KnowledgeSearchTool) parseScoresFromResponse(responseText string, expec return scores, nil } -// rerankWithModel uses the rerank model for reranking (fallback) +// rerankWithModel uses the rerank model for reranking. func (t *KnowledgeSearchTool) rerankWithModel( ctx context.Context, query string, results []*searchResultWithMeta, ) ([]*searchResultWithMeta, error) { - // Prepare passages for reranking (with enriched content including image info) passages := make([]string, len(results)) for i, result := range results { passages[i] = t.getEnrichedPassage(ctx, result.SearchResult) } - // Call rerank model rerankResp, err := t.rerankModel.Rerank(ctx, query, passages) if err != nil { return nil, fmt.Errorf("rerank call failed: %w", err) } - // Map reranked results back with new scores - reranked := make([]*searchResultWithMeta, 0, len(rerankResp)) - for _, rr := range rerankResp { - if rr.Index >= 0 && rr.Index < len(results) { - // Create new result with reranked score - newResult := *results[rr.Index] - newResult.Score = rr.RelevanceScore - reranked = append(reranked, &newResult) - } - } - + ranked := t.applyModelRerankScores(results, rerankResp, t.rerankThreshold()) logger.Infof( ctx, - "[Tool][KnowledgeSearch] Reranked %d results from %d original results", - len(reranked), + "[Tool][KnowledgeSearch] Reranked %d/%d results above threshold %.2f", + len(ranked), len(results), + t.rerankThreshold(), ) - return reranked, nil + return ranked, nil +} + +func (t *KnowledgeSearchTool) rerankThreshold() float64 { + if t.config != nil && t.config.Conversation != nil && t.config.Conversation.RerankThreshold > 0 { + return t.config.Conversation.RerankThreshold + } + return 0.3 +} + +const agentRerankFallbackMinScore = 0.15 + +func filterRerankRankResults(rankResults []rerank.RankResult, threshold float64) []rerank.RankResult { + if len(rankResults) == 0 { + return nil + } + filtered := make([]rerank.RankResult, 0, len(rankResults)) + for _, r := range rankResults { + if r.RelevanceScore >= threshold { + filtered = append(filtered, r) + } + } + if len(filtered) == 0 { + top := rankResults[0] + for _, r := range rankResults[1:] { + if r.RelevanceScore > top.RelevanceScore { + top = r + } + } + if top.RelevanceScore >= agentRerankFallbackMinScore { + return []rerank.RankResult{top} + } + } + return filtered +} + +func (t *KnowledgeSearchTool) applyModelRerankScores( + originals []*searchResultWithMeta, + rankResults []rerank.RankResult, + threshold float64, +) []*searchResultWithMeta { + filtered := filterRerankRankResults(rankResults, threshold) + out := make([]*searchResultWithMeta, 0, len(filtered)) + for _, rr := range filtered { + if rr.Index < 0 || rr.Index >= len(originals) { + continue + } + newResult := *originals[rr.Index] + baseScore := newResult.Score + modelScore := rr.RelevanceScore + newResult.Score = t.compositeScore(&newResult, modelScore, baseScore) + out = append(out, &newResult) + } + sort.Slice(out, func(i, j int) bool { + return out[i].Score > out[j].Score + }) + return out } // deduplicateResults removes duplicate chunks, keeping the highest score diff --git a/internal/agent/tools/knowledge_search_rerank_test.go b/internal/agent/tools/knowledge_search_rerank_test.go new file mode 100644 index 00000000..9c259111 --- /dev/null +++ b/internal/agent/tools/knowledge_search_rerank_test.go @@ -0,0 +1,77 @@ +package tools + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/models/rerank" + "github.com/Tencent/WeKnora/internal/types" +) + +func TestFilterRerankRankResults_thresholdAndFallback(t *testing.T) { + t.Parallel() + rankResults := []rerank.RankResult{ + {Index: 0, RelevanceScore: 0.05}, + {Index: 1, RelevanceScore: 0.02}, + } + filtered := filterRerankRankResults(rankResults, 0.3) + if len(filtered) != 0 { + t.Fatalf("expected empty filter, got %#v", filtered) + } + + rankResults = []rerank.RankResult{ + {Index: 0, RelevanceScore: 0.05}, + {Index: 1, RelevanceScore: 0.20}, + } + filtered = filterRerankRankResults(rankResults, 0.3) + if len(filtered) != 1 || filtered[0].Index != 1 { + t.Fatalf("expected fallback top score, got %#v", filtered) + } + + rankResults = []rerank.RankResult{ + {Index: 0, RelevanceScore: 0.8}, + {Index: 1, RelevanceScore: 0.4}, + {Index: 2, RelevanceScore: 0.1}, + } + filtered = filterRerankRankResults(rankResults, 0.3) + if len(filtered) != 2 { + t.Fatalf("expected 2 passing scores, got %#v", filtered) + } +} + +func TestApplyModelRerankScores_faqUsesCompositeScale(t *testing.T) { + t.Parallel() + tool := &KnowledgeSearchTool{ + config: &config.Config{ + Conversation: &config.ConversationConfig{RerankThreshold: 0.3}, + }, + } + originals := []*searchResultWithMeta{ + { + SearchResult: &types.SearchResult{ID: "faq-1", Content: "Q: WeKnora", Score: 0.011}, + KnowledgeBaseType: types.KnowledgeBaseTypeFAQ, + }, + { + SearchResult: &types.SearchResult{ID: "doc-1", Content: "swimming club", Score: 0.02}, + }, + } + rankResults := []rerank.RankResult{ + {Index: 0, RelevanceScore: 0.05}, + {Index: 1, RelevanceScore: 0.9}, + } + out := tool.applyModelRerankScores(originals, rankResults, 0.3) + if len(out) != 1 || out[0].ID != "doc-1" { + t.Fatalf("weak FAQ should be filtered out, got %#v", out) + } + if out[0].Score <= 0.011 { + t.Fatalf("composite score should exceed raw retrieval score, got %.4f", out[0].Score) + } +} + +func TestRerankThreshold_default(t *testing.T) { + t.Parallel() + tool := &KnowledgeSearchTool{} + if got := tool.rerankThreshold(); got != 0.3 { + t.Fatalf("default threshold = %v, want 0.3", got) + } +}