feat(agent): rerank all knowledge search hits with threshold filtering

Remove FAQ bypass in agent rerank, apply composite model scores, and
filter low-confidence results using configurable rerank_threshold.
This commit is contained in:
wizardchen
2026-06-16 00:49:34 +08:00
committed by lyingbug
parent de525a4f02
commit 16f2a46465
2 changed files with 177 additions and 94 deletions
+100 -94
View File
@@ -299,8 +299,8 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args json.RawMessage)
var filteredResults []*searchResultWithMeta
if (t.rerankModel != nil || t.chatModel != nil) && len(deduplicatedBeforeRerank) > 0 && rerankQuery != "" {
logger.Infof(ctx, "[Tool][KnowledgeSearch] Applying rerank, input: %d results, queries: %v",
len(deduplicatedBeforeRerank), queries)
logger.Infof(ctx, "[Tool][KnowledgeSearch] Applying rerank, input: %d results, threshold: %.2f, queries: %v",
len(deduplicatedBeforeRerank), t.rerankThreshold(), queries)
rerankedResults, err := t.rerankResults(ctx, rerankQuery, deduplicatedBeforeRerank)
if err != nil {
logger.Warnf(ctx, "[Tool][KnowledgeSearch] Rerank failed, using original results: %v", err)
@@ -361,12 +361,12 @@ func (t *KnowledgeSearchTool) Execute(ctx context.Context, args json.RawMessage)
return deduplicatedResults[i].KnowledgeID < deduplicatedResults[j].KnowledgeID
})
// Log top results
// Log all ranked results (including lower ranks for rerank debugging)
if len(deduplicatedResults) > 0 {
for i := 0; i < len(deduplicatedResults) && i < 5; i++ {
r := deduplicatedResults[i]
logger.Infof(ctx, "[Tool][KnowledgeSearch][Top %d] score=%.3f, type=%s, kb=%s, chunk_id=%s",
i+1, r.Score, r.QueryType, r.KnowledgeID, r.ID)
total := len(deduplicatedResults)
for i, r := range deduplicatedResults {
logger.Infof(ctx, "[Tool][KnowledgeSearch][Rank %d/%d] score=%.3f, type=%s, kb=%s, chunk_id=%s",
i+1, total, r.Score, r.QueryType, r.KnowledgeID, r.ID)
}
}
@@ -594,90 +594,50 @@ func (t *KnowledgeSearchTool) concurrentSearchByTargets(
return allResults
}
// rerankResults applies reranking to search results using LLM prompt scoring or rerank model
// rerankResults applies reranking to all search results (including FAQ entries)
// using the rerank model or LLM fallback, then filters by threshold and applies
// composite scoring so MMR/sorting uses a single score scale.
func (t *KnowledgeSearchTool) rerankResults(
ctx context.Context,
query string,
results []*searchResultWithMeta,
) ([]*searchResultWithMeta, error) {
// Separate FAQ and normal results.
// FAQ results keep original scores and bypass reranking model.
faqResults := make([]*searchResultWithMeta, 0)
rerankCandidates := make([]*searchResultWithMeta, 0, len(results))
for _, result := range results {
// Skip reranking for FAQ results (they are explicitly matched Q&A pairs)
if result.KnowledgeBaseType == types.KnowledgeBaseTypeFAQ {
faqResults = append(faqResults, result)
} else {
rerankCandidates = append(rerankCandidates, result)
}
}
// If there are no candidates to rerank, return original list (already all FAQ)
if len(rerankCandidates) == 0 {
if len(results) == 0 {
return results, nil
}
var (
rerankedCandidates []*searchResultWithMeta
err error
reranked []*searchResultWithMeta
err error
)
// Apply reranking only to candidates
// Try rerankModel first, fallback to chatModel if rerankModel fails or returns no results
if t.rerankModel != nil {
rerankedCandidates, err = t.rerankWithModel(ctx, query, rerankCandidates)
// If rerankModel fails or returns no results, fallback to chatModel
if err != nil || len(rerankedCandidates) == 0 {
reranked, err = t.rerankWithModel(ctx, query, results)
if err != nil || len(reranked) == 0 {
if err != nil {
logger.Warnf(ctx, "[Tool][KnowledgeSearch] Rerank model failed, falling back to chat model: %v", err)
} else {
logger.Warnf(ctx, "[Tool][KnowledgeSearch] Rerank model returned no results, falling back to chat model")
logger.Warnf(ctx, "[Tool][KnowledgeSearch] Rerank model returned no results above threshold, falling back to chat model")
}
// Reset error to allow fallback
err = nil
// Try chatModel if available
if t.chatModel != nil {
rerankedCandidates, err = t.rerankWithLLM(ctx, query, rerankCandidates)
} else {
// No fallback available, use original results
rerankedCandidates = rerankCandidates
reranked, err = t.rerankWithLLM(ctx, query, results)
} else if len(reranked) == 0 {
reranked = results
}
}
} else if t.chatModel != nil {
// No rerankModel, use chatModel directly
rerankedCandidates, err = t.rerankWithLLM(ctx, query, rerankCandidates)
reranked, err = t.rerankWithLLM(ctx, query, results)
} else {
// No reranking available, use original results
rerankedCandidates = rerankCandidates
return results, nil
}
if err != nil {
return nil, err
}
// Apply composite scoring to reranked results
logger.Debugf(ctx, "[Tool][KnowledgeSearch] Applying composite scoring")
// Store base scores before composite scoring
for _, result := range rerankedCandidates {
baseScore := result.Score
// Apply composite score
result.Score = t.compositeScore(result, result.Score, baseScore)
}
// Combine FAQ results (with original order) and reranked candidates
combined := make([]*searchResultWithMeta, 0, len(results))
combined = append(combined, faqResults...)
combined = append(combined, rerankedCandidates...)
// Sort by score (descending) to keep consistent output order
sort.Slice(combined, func(i, j int) bool {
return combined[i].Score > combined[j].Score
})
return combined, nil
logger.Debugf(ctx, "[Tool][KnowledgeSearch] Rerank produced %d results after threshold filter", len(reranked))
return reranked, nil
}
func (t *KnowledgeSearchTool) getFAQMetadata(
@@ -732,7 +692,6 @@ func (t *KnowledgeSearchTool) rerankWithLLM(
// Process in batches
allScores := make([]float64, len(results))
allReranked := make([]*searchResultWithMeta, 0, len(results))
for batchStart := 0; batchStart < len(results); batchStart += batchSize {
batchEnd := batchStart + batchSize
@@ -862,23 +821,25 @@ Output only the scores, no explanations or additional text.`,
}
}
// Create reranked results with new scores
for i, result := range results {
newResult := *result
if i < len(allScores) {
newResult.Score = allScores[i]
// Create rerank rank results and apply the same threshold + composite path as the model.
rankResults := make([]rerank.RankResult, 0, len(results))
for i, score := range allScores {
if i >= len(results) {
break
}
allReranked = append(allReranked, &newResult)
rankResults = append(rankResults, rerank.RankResult{
Index: i,
RelevanceScore: score,
})
}
// Sort by new scores (descending)
sort.Slice(allReranked, func(i, j int) bool {
return allReranked[i].Score > allReranked[j].Score
sort.Slice(rankResults, func(i, j int) bool {
return rankResults[i].RelevanceScore > rankResults[j].RelevanceScore
})
logger.Infof(ctx, "[Tool][KnowledgeSearch] LLM reranked %d results from %d original results (processed in batches)",
len(allReranked), len(results))
return allReranked, nil
ranked := t.applyModelRerankScores(results, rankResults, t.rerankThreshold())
logger.Infof(ctx, "[Tool][KnowledgeSearch] LLM reranked %d/%d results above threshold %.2f",
len(ranked), len(results), t.rerankThreshold())
return ranked, nil
}
// parseScoresFromResponse parses scores from LLM response text
@@ -951,42 +912,87 @@ func (t *KnowledgeSearchTool) parseScoresFromResponse(responseText string, expec
return scores, nil
}
// rerankWithModel uses the rerank model for reranking (fallback)
// rerankWithModel uses the rerank model for reranking.
func (t *KnowledgeSearchTool) rerankWithModel(
ctx context.Context,
query string,
results []*searchResultWithMeta,
) ([]*searchResultWithMeta, error) {
// Prepare passages for reranking (with enriched content including image info)
passages := make([]string, len(results))
for i, result := range results {
passages[i] = t.getEnrichedPassage(ctx, result.SearchResult)
}
// Call rerank model
rerankResp, err := t.rerankModel.Rerank(ctx, query, passages)
if err != nil {
return nil, fmt.Errorf("rerank call failed: %w", err)
}
// Map reranked results back with new scores
reranked := make([]*searchResultWithMeta, 0, len(rerankResp))
for _, rr := range rerankResp {
if rr.Index >= 0 && rr.Index < len(results) {
// Create new result with reranked score
newResult := *results[rr.Index]
newResult.Score = rr.RelevanceScore
reranked = append(reranked, &newResult)
}
}
ranked := t.applyModelRerankScores(results, rerankResp, t.rerankThreshold())
logger.Infof(
ctx,
"[Tool][KnowledgeSearch] Reranked %d results from %d original results",
len(reranked),
"[Tool][KnowledgeSearch] Reranked %d/%d results above threshold %.2f",
len(ranked),
len(results),
t.rerankThreshold(),
)
return reranked, nil
return ranked, nil
}
func (t *KnowledgeSearchTool) rerankThreshold() float64 {
if t.config != nil && t.config.Conversation != nil && t.config.Conversation.RerankThreshold > 0 {
return t.config.Conversation.RerankThreshold
}
return 0.3
}
const agentRerankFallbackMinScore = 0.15
func filterRerankRankResults(rankResults []rerank.RankResult, threshold float64) []rerank.RankResult {
if len(rankResults) == 0 {
return nil
}
filtered := make([]rerank.RankResult, 0, len(rankResults))
for _, r := range rankResults {
if r.RelevanceScore >= threshold {
filtered = append(filtered, r)
}
}
if len(filtered) == 0 {
top := rankResults[0]
for _, r := range rankResults[1:] {
if r.RelevanceScore > top.RelevanceScore {
top = r
}
}
if top.RelevanceScore >= agentRerankFallbackMinScore {
return []rerank.RankResult{top}
}
}
return filtered
}
func (t *KnowledgeSearchTool) applyModelRerankScores(
originals []*searchResultWithMeta,
rankResults []rerank.RankResult,
threshold float64,
) []*searchResultWithMeta {
filtered := filterRerankRankResults(rankResults, threshold)
out := make([]*searchResultWithMeta, 0, len(filtered))
for _, rr := range filtered {
if rr.Index < 0 || rr.Index >= len(originals) {
continue
}
newResult := *originals[rr.Index]
baseScore := newResult.Score
modelScore := rr.RelevanceScore
newResult.Score = t.compositeScore(&newResult, modelScore, baseScore)
out = append(out, &newResult)
}
sort.Slice(out, func(i, j int) bool {
return out[i].Score > out[j].Score
})
return out
}
// deduplicateResults removes duplicate chunks, keeping the highest score
@@ -0,0 +1,77 @@
package tools
import (
"testing"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/types"
)
func TestFilterRerankRankResults_thresholdAndFallback(t *testing.T) {
t.Parallel()
rankResults := []rerank.RankResult{
{Index: 0, RelevanceScore: 0.05},
{Index: 1, RelevanceScore: 0.02},
}
filtered := filterRerankRankResults(rankResults, 0.3)
if len(filtered) != 0 {
t.Fatalf("expected empty filter, got %#v", filtered)
}
rankResults = []rerank.RankResult{
{Index: 0, RelevanceScore: 0.05},
{Index: 1, RelevanceScore: 0.20},
}
filtered = filterRerankRankResults(rankResults, 0.3)
if len(filtered) != 1 || filtered[0].Index != 1 {
t.Fatalf("expected fallback top score, got %#v", filtered)
}
rankResults = []rerank.RankResult{
{Index: 0, RelevanceScore: 0.8},
{Index: 1, RelevanceScore: 0.4},
{Index: 2, RelevanceScore: 0.1},
}
filtered = filterRerankRankResults(rankResults, 0.3)
if len(filtered) != 2 {
t.Fatalf("expected 2 passing scores, got %#v", filtered)
}
}
func TestApplyModelRerankScores_faqUsesCompositeScale(t *testing.T) {
t.Parallel()
tool := &KnowledgeSearchTool{
config: &config.Config{
Conversation: &config.ConversationConfig{RerankThreshold: 0.3},
},
}
originals := []*searchResultWithMeta{
{
SearchResult: &types.SearchResult{ID: "faq-1", Content: "Q: WeKnora", Score: 0.011},
KnowledgeBaseType: types.KnowledgeBaseTypeFAQ,
},
{
SearchResult: &types.SearchResult{ID: "doc-1", Content: "swimming club", Score: 0.02},
},
}
rankResults := []rerank.RankResult{
{Index: 0, RelevanceScore: 0.05},
{Index: 1, RelevanceScore: 0.9},
}
out := tool.applyModelRerankScores(originals, rankResults, 0.3)
if len(out) != 1 || out[0].ID != "doc-1" {
t.Fatalf("weak FAQ should be filtered out, got %#v", out)
}
if out[0].Score <= 0.011 {
t.Fatalf("composite score should exceed raw retrieval score, got %.4f", out[0].Score)
}
}
func TestRerankThreshold_default(t *testing.T) {
t.Parallel()
tool := &KnowledgeSearchTool{}
if got := tool.rerankThreshold(); got != 0.3 {
t.Fatalf("default threshold = %v, want 0.3", got)
}
}