diff --git a/internal/application/service/chat_pipline/rerank.go b/internal/application/service/chat_pipline/rerank.go index 3c355800..3b6cd0c1 100644 --- a/internal/application/service/chat_pipline/rerank.go +++ b/internal/application/service/chat_pipline/rerank.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "math" + "regexp" "strings" "github.com/Tencent/WeKnora/internal/models/rerank" @@ -249,32 +250,34 @@ func (p *PluginRerank) rerank(ctx context.Context, } } - // Filter results based on threshold with special handling for history matches + // Filter results based on threshold rankFilter := []rerank.RankResult{} for _, result := range rerankResp { if result.Index >= len(candidates) { continue } - th := chatManage.RerankThreshold - matchType := candidates[result.Index].MatchType - if matchType == types.MatchTypeHistory { - th = math.Max(th-0.1, 0.5) // Lower threshold for history matches - } - if result.RelevanceScore > th { + if result.RelevanceScore >= chatManage.RerankThreshold { rankFilter = append(rankFilter, result) } } - // Fallback: if threshold filtering removed all results, keep top-N as safety net - // This prevents returning empty results when all scores are below threshold - if len(rankFilter) == 0 && len(rerankResp) > 0 { - fallbackN := min(3, len(rerankResp)) - rankFilter = rerankResp[:fallbackN] - pipelineInfo(ctx, "Rerank", "fallback_topn", map[string]interface{}{ - "reason": "all_below_threshold", - "threshold": chatManage.RerankThreshold, - "fallback_n": fallbackN, - "top_score": rerankResp[0].RelevanceScore, + // Fallback: if threshold filtering removed all results but the top candidate + // still has a reasonable score, keep it as a safety net. Skip fallback entirely + // when the best score is too low — forcing irrelevant results is worse than + // returning nothing and letting the caller handle the empty-result case. + const fallbackMinScore = 0.15 + if len(rankFilter) == 0 && len(rerankResp) > 0 && rerankResp[0].RelevanceScore >= fallbackMinScore { + rankFilter = rerankResp[:1] + pipelineInfo(ctx, "Rerank", "fallback_top1", map[string]interface{}{ + "reason": "all_below_threshold", + "threshold": chatManage.RerankThreshold, + "top_score": rerankResp[0].RelevanceScore, + }) + } else if len(rankFilter) == 0 { + pipelineInfo(ctx, "Rerank", "fallback_skip", map[string]interface{}{ + "reason": "top_score_too_low", + "threshold": chatManage.RerankThreshold, + "top_score": safeTopScore(rerankResp), }) } @@ -289,6 +292,13 @@ func ensureMetadata(m map[string]string) map[string]string { return m } +func safeTopScore(results []rerank.RankResult) float64 { + if len(results) == 0 { + return 0 + } + return results[0].RelevanceScore +} + // compositeScore calculates the composite score for a search result func compositeScore(sr *types.SearchResult, modelScore, baseScore float64) float64 { sourceWeight := 1.0 @@ -397,9 +407,82 @@ func applyMMR( return selected } +// --- Passage cleaning for rerank --- +// +// Rerank models work on semantic text similarity. Markdown formatting, raw URLs, +// image references, table separators, and other structural syntax are noise that +// can dilute the semantic signal. The functions below strip this noise before +// passages are sent to the rerank model. + +var ( + // reMarkdownImage matches ![alt](url) — the entire construct is noise. + reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`) + // reMarkdownLink matches [text](url) — we keep the text, drop the URL. + reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`) + // reRawURL matches standalone http(s) URLs. + reRawURL = regexp.MustCompile(`https?://[^\s)\]>]+`) + // reCodeBlock matches fenced code blocks (```...```). + reCodeBlock = regexp.MustCompile("(?s)```(?:\\w*)\n?.*?```") + // reLatexBlock matches block-level LaTeX ($$...$$). + reLatexBlock = regexp.MustCompile(`(?s)\$\$.*?\$\$`) + // reTableSep matches table separator rows like |---|---|. + reTableSep = regexp.MustCompile(`(?m)^\s*\|[\s:|-]+\|\s*$`) + // reHeadingPrefix matches leading # markers in headings. + reHeadingPrefix = regexp.MustCompile(`(?m)^#{1,6}\s+`) + // reBlockquote matches leading > markers. + reBlockquote = regexp.MustCompile(`(?m)^>\s?`) + // reBoldItalic3 matches ***text*** wrappers (must come before 2 and 1). + reBoldItalic3 = regexp.MustCompile(`\*{3}(.+?)\*{3}`) + // reBoldItalic2 matches **text** wrappers. + reBoldItalic2 = regexp.MustCompile(`\*{2}(.+?)\*{2}`) + // reBoldItalic1 matches *text* wrappers. + reBoldItalic1 = regexp.MustCompile(`\*(.+?)\*`) + // reExcessiveNewlines collapses 3+ consecutive newlines into 2. + reExcessiveNewlines = regexp.MustCompile(`\n{3,}`) + // reListMarker matches unordered (- , * ) and ordered (1. ) list prefixes. + reListMarker = regexp.MustCompile(`(?m)^[\t ]*(?:[-*+]|\d+\.)\s+`) + // reHTMLTag matches HTML tags like
,
, etc. + reHTMLTag = regexp.MustCompile(`]*>`) +) + +// cleanPassageForRerank strips markdown/structural noise from text to produce +// a clean semantic passage for the rerank model. The cleaning is designed to +// preserve all meaningful natural-language content while removing formatting +// that would confuse text-similarity scoring. +func cleanPassageForRerank(text string) string { + // 1. Remove code blocks (before other patterns to avoid partial matches) + text = reCodeBlock.ReplaceAllString(text, "") + // 2. Remove LaTeX block math + text = reLatexBlock.ReplaceAllString(text, "") + // 3. Remove HTML tags + text = reHTMLTag.ReplaceAllString(text, "") + // 4. Remove markdown image references entirely + text = reMarkdownImage.ReplaceAllString(text, "") + // 5. Convert markdown links to just their display text + text = reMarkdownLink.ReplaceAllString(text, "$1") + // 6. Remove standalone raw URLs + text = reRawURL.ReplaceAllString(text, "") + // 7. Remove table separator rows + text = reTableSep.ReplaceAllString(text, "") + // 8. Strip heading markers but keep heading text + text = reHeadingPrefix.ReplaceAllString(text, "") + // 9. Strip blockquote markers + text = reBlockquote.ReplaceAllString(text, "") + // 10. Unwrap bold/italic markers, keeping inner text (order: *** before ** before *) + text = reBoldItalic3.ReplaceAllString(text, "$1") + text = reBoldItalic2.ReplaceAllString(text, "$1") + text = reBoldItalic1.ReplaceAllString(text, "$1") + // 11. Strip list markers + text = reListMarker.ReplaceAllString(text, "") + // 12. Collapse excessive newlines + text = reExcessiveNewlines.ReplaceAllString(text, "\n\n") + + return strings.TrimSpace(text) +} + // getEnrichedPassage 合并Content、ImageInfo和GeneratedQuestions的文本内容 func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string { - combinedText := result.Content + combinedText := cleanPassageForRerank(result.Content) var enrichments []string // 解析ImageInfo @@ -414,10 +497,10 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string // 提取所有图片的描述和OCR文本 for _, img := range imageInfos { if img.Caption != "" { - enrichments = append(enrichments, fmt.Sprintf("图片描述: %s", img.Caption)) + enrichments = append(enrichments, img.Caption) } if img.OCRText != "" { - enrichments = append(enrichments, fmt.Sprintf("图片文本: %s", img.OCRText)) + enrichments = append(enrichments, img.OCRText) } } } @@ -432,7 +515,7 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string "error": err.Error(), }) } else if questionStrings := docMeta.GetQuestionStrings(); len(questionStrings) > 0 { - enrichments = append(enrichments, fmt.Sprintf("相关问题: %s", strings.Join(questionStrings, "; "))) + enrichments = append(enrichments, strings.Join(questionStrings, "; ")) } } @@ -448,7 +531,7 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string pipelineInfo(ctx, "Rerank", "passage_enrich", map[string]interface{}{ "content_len": len(result.Content), - "enrichment": strings.Join(enrichments, "\n"), + "cleaned_len": len(cleanPassageForRerank(result.Content)), "enrichment_len": len(strings.Join(enrichments, "\n")), }) diff --git a/internal/application/service/chat_pipline/rerank_clean_test.go b/internal/application/service/chat_pipline/rerank_clean_test.go new file mode 100644 index 00000000..aee4ec55 --- /dev/null +++ b/internal/application/service/chat_pipline/rerank_clean_test.go @@ -0,0 +1,104 @@ +package chatpipline + +import ( + "testing" +) + +func TestCleanPassageForRerank(t *testing.T) { + tests := []struct { + name string + input string + expect string + }{ + { + name: "plain text unchanged", + input: "这是一段普通的文本内容", + expect: "这是一段普通的文本内容", + }, + { + name: "remove markdown images", + input: "前文 ![图片说明](https://example.com/img.png) 后文", + expect: "前文 后文", + }, + { + name: "convert markdown links to text", + input: "请参考 [官方文档](https://docs.example.com) 了解详情", + expect: "请参考 官方文档 了解详情", + }, + { + name: "remove standalone URLs", + input: "访问 https://example.com/path?q=1&b=2 获取更多信息", + expect: "访问 获取更多信息", + }, + { + name: "remove code blocks", + input: "示例代码:\n```python\nprint('hello')\n```\n以上是示例", + expect: "示例代码:\n\n以上是示例", + }, + { + name: "remove LaTeX blocks", + input: "公式如下 $$E=mc^2$$ 其中E是能量", + expect: "公式如下 其中E是能量", + }, + { + name: "remove table separator rows", + input: "| 名称 | 值 |\n| --- | --- |\n| A | 1 |", + expect: "| 名称 | 值 |\n\n| A | 1 |", + }, + { + name: "strip heading markers", + input: "## 第二章 概述\n### 2.1 背景", + expect: "第二章 概述\n2.1 背景", + }, + { + name: "strip blockquote markers", + input: "> 这是一段引用\n> 第二行引用", + expect: "这是一段引用\n第二行引用", + }, + { + name: "unwrap bold and italic", + input: "这是 **加粗** 和 *斜体* 以及 ***粗斜体*** 文本", + expect: "这是 加粗 和 斜体 以及 粗斜体 文本", + }, + { + name: "strip list markers", + input: "- 项目一\n- 项目二\n1. 有序一\n2. 有序二", + expect: "项目一\n项目二\n有序一\n有序二", + }, + { + name: "remove HTML tags", + input: "文本
换行
内容
结尾", + expect: "文本换行内容结尾", + }, + { + name: "collapse excessive newlines", + input: "段落一\n\n\n\n\n段落二", + expect: "段落一\n\n段落二", + }, + { + name: "combined real-world passage", + input: `## 产品介绍 + +这是一个 **重要的** 产品。详见 [产品页面](https://example.com/product)。 + +![产品截图](images/product.png) + +> 用户评价:非常好用 + +- 功能一 +- 功能二 + +` + "```json\n{\"key\": \"value\"}\n```", + expect: "产品介绍\n\n这是一个 重要的 产品。详见 产品页面。\n\n用户评价:非常好用\n\n功能一\n功能二", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := cleanPassageForRerank(tt.input) + if got != tt.expect { + t.Errorf("cleanPassageForRerank():\ngot: %q\nexpect: %q", got, tt.expect) + } + }) + } +} diff --git a/internal/application/service/chat_pipline/search_parallel.go b/internal/application/service/chat_pipline/search_parallel.go index 895fbb83..054b7e48 100644 --- a/internal/application/service/chat_pipline/search_parallel.go +++ b/internal/application/service/chat_pipline/search_parallel.go @@ -154,6 +154,7 @@ func (p *PluginSearchParallel) OnEvent(ctx context.Context, // Merge results from both searches (no concurrent access now) chatManage.SearchResult = append(chunkChatManage.SearchResult, entityChatManage.SearchResult...) + chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult) // Log any errors but don't fail the pipeline if at least one search succeeded if chunkSearchErr != nil {