diff --git a/internal/application/service/chat_pipline/rerank.go b/internal/application/service/chat_pipline/rerank.go
index 3c355800..3b6cd0c1 100644
--- a/internal/application/service/chat_pipline/rerank.go
+++ b/internal/application/service/chat_pipline/rerank.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"math"
+ "regexp"
"strings"
"github.com/Tencent/WeKnora/internal/models/rerank"
@@ -249,32 +250,34 @@ func (p *PluginRerank) rerank(ctx context.Context,
}
}
- // Filter results based on threshold with special handling for history matches
+ // Filter results based on threshold
rankFilter := []rerank.RankResult{}
for _, result := range rerankResp {
if result.Index >= len(candidates) {
continue
}
- th := chatManage.RerankThreshold
- matchType := candidates[result.Index].MatchType
- if matchType == types.MatchTypeHistory {
- th = math.Max(th-0.1, 0.5) // Lower threshold for history matches
- }
- if result.RelevanceScore > th {
+ if result.RelevanceScore >= chatManage.RerankThreshold {
rankFilter = append(rankFilter, result)
}
}
- // Fallback: if threshold filtering removed all results, keep top-N as safety net
- // This prevents returning empty results when all scores are below threshold
- if len(rankFilter) == 0 && len(rerankResp) > 0 {
- fallbackN := min(3, len(rerankResp))
- rankFilter = rerankResp[:fallbackN]
- pipelineInfo(ctx, "Rerank", "fallback_topn", map[string]interface{}{
- "reason": "all_below_threshold",
- "threshold": chatManage.RerankThreshold,
- "fallback_n": fallbackN,
- "top_score": rerankResp[0].RelevanceScore,
+ // Fallback: if threshold filtering removed all results but the top candidate
+ // still has a reasonable score, keep it as a safety net. Skip fallback entirely
+ // when the best score is too low — forcing irrelevant results is worse than
+ // returning nothing and letting the caller handle the empty-result case.
+ const fallbackMinScore = 0.15
+ if len(rankFilter) == 0 && len(rerankResp) > 0 && rerankResp[0].RelevanceScore >= fallbackMinScore {
+ rankFilter = rerankResp[:1]
+ pipelineInfo(ctx, "Rerank", "fallback_top1", map[string]interface{}{
+ "reason": "all_below_threshold",
+ "threshold": chatManage.RerankThreshold,
+ "top_score": rerankResp[0].RelevanceScore,
+ })
+ } else if len(rankFilter) == 0 {
+ pipelineInfo(ctx, "Rerank", "fallback_skip", map[string]interface{}{
+ "reason": "top_score_too_low",
+ "threshold": chatManage.RerankThreshold,
+ "top_score": safeTopScore(rerankResp),
})
}
@@ -289,6 +292,13 @@ func ensureMetadata(m map[string]string) map[string]string {
return m
}
+func safeTopScore(results []rerank.RankResult) float64 {
+ if len(results) == 0 {
+ return 0
+ }
+ return results[0].RelevanceScore
+}
+
// compositeScore calculates the composite score for a search result
func compositeScore(sr *types.SearchResult, modelScore, baseScore float64) float64 {
sourceWeight := 1.0
@@ -397,9 +407,82 @@ func applyMMR(
return selected
}
+// --- Passage cleaning for rerank ---
+//
+// Rerank models work on semantic text similarity. Markdown formatting, raw URLs,
+// image references, table separators, and other structural syntax are noise that
+// can dilute the semantic signal. The functions below strip this noise before
+// passages are sent to the rerank model.
+
+var (
+ // reMarkdownImage matches  — the entire construct is noise.
+ reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`)
+ // reMarkdownLink matches [text](url) — we keep the text, drop the URL.
+ reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`)
+ // reRawURL matches standalone http(s) URLs.
+ reRawURL = regexp.MustCompile(`https?://[^\s)\]>]+`)
+ // reCodeBlock matches fenced code blocks (```...```).
+ reCodeBlock = regexp.MustCompile("(?s)```(?:\\w*)\n?.*?```")
+ // reLatexBlock matches block-level LaTeX ($$...$$).
+ reLatexBlock = regexp.MustCompile(`(?s)\$\$.*?\$\$`)
+ // reTableSep matches table separator rows like |---|---|.
+ reTableSep = regexp.MustCompile(`(?m)^\s*\|[\s:|-]+\|\s*$`)
+ // reHeadingPrefix matches leading # markers in headings.
+ reHeadingPrefix = regexp.MustCompile(`(?m)^#{1,6}\s+`)
+ // reBlockquote matches leading > markers.
+ reBlockquote = regexp.MustCompile(`(?m)^>\s?`)
+ // reBoldItalic3 matches ***text*** wrappers (must come before 2 and 1).
+ reBoldItalic3 = regexp.MustCompile(`\*{3}(.+?)\*{3}`)
+ // reBoldItalic2 matches **text** wrappers.
+ reBoldItalic2 = regexp.MustCompile(`\*{2}(.+?)\*{2}`)
+ // reBoldItalic1 matches *text* wrappers.
+ reBoldItalic1 = regexp.MustCompile(`\*(.+?)\*`)
+ // reExcessiveNewlines collapses 3+ consecutive newlines into 2.
+ reExcessiveNewlines = regexp.MustCompile(`\n{3,}`)
+ // reListMarker matches unordered (- , * ) and ordered (1. ) list prefixes.
+ reListMarker = regexp.MustCompile(`(?m)^[\t ]*(?:[-*+]|\d+\.)\s+`)
+ // reHTMLTag matches HTML tags like
,
, etc.
+ reHTMLTag = regexp.MustCompile(`?[a-zA-Z][^>]*>`)
+)
+
+// cleanPassageForRerank strips markdown/structural noise from text to produce
+// a clean semantic passage for the rerank model. The cleaning is designed to
+// preserve all meaningful natural-language content while removing formatting
+// that would confuse text-similarity scoring.
+func cleanPassageForRerank(text string) string {
+ // 1. Remove code blocks (before other patterns to avoid partial matches)
+ text = reCodeBlock.ReplaceAllString(text, "")
+ // 2. Remove LaTeX block math
+ text = reLatexBlock.ReplaceAllString(text, "")
+ // 3. Remove HTML tags
+ text = reHTMLTag.ReplaceAllString(text, "")
+ // 4. Remove markdown image references entirely
+ text = reMarkdownImage.ReplaceAllString(text, "")
+ // 5. Convert markdown links to just their display text
+ text = reMarkdownLink.ReplaceAllString(text, "$1")
+ // 6. Remove standalone raw URLs
+ text = reRawURL.ReplaceAllString(text, "")
+ // 7. Remove table separator rows
+ text = reTableSep.ReplaceAllString(text, "")
+ // 8. Strip heading markers but keep heading text
+ text = reHeadingPrefix.ReplaceAllString(text, "")
+ // 9. Strip blockquote markers
+ text = reBlockquote.ReplaceAllString(text, "")
+ // 10. Unwrap bold/italic markers, keeping inner text (order: *** before ** before *)
+ text = reBoldItalic3.ReplaceAllString(text, "$1")
+ text = reBoldItalic2.ReplaceAllString(text, "$1")
+ text = reBoldItalic1.ReplaceAllString(text, "$1")
+ // 11. Strip list markers
+ text = reListMarker.ReplaceAllString(text, "")
+ // 12. Collapse excessive newlines
+ text = reExcessiveNewlines.ReplaceAllString(text, "\n\n")
+
+ return strings.TrimSpace(text)
+}
+
// getEnrichedPassage 合并Content、ImageInfo和GeneratedQuestions的文本内容
func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string {
- combinedText := result.Content
+ combinedText := cleanPassageForRerank(result.Content)
var enrichments []string
// 解析ImageInfo
@@ -414,10 +497,10 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string
// 提取所有图片的描述和OCR文本
for _, img := range imageInfos {
if img.Caption != "" {
- enrichments = append(enrichments, fmt.Sprintf("图片描述: %s", img.Caption))
+ enrichments = append(enrichments, img.Caption)
}
if img.OCRText != "" {
- enrichments = append(enrichments, fmt.Sprintf("图片文本: %s", img.OCRText))
+ enrichments = append(enrichments, img.OCRText)
}
}
}
@@ -432,7 +515,7 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string
"error": err.Error(),
})
} else if questionStrings := docMeta.GetQuestionStrings(); len(questionStrings) > 0 {
- enrichments = append(enrichments, fmt.Sprintf("相关问题: %s", strings.Join(questionStrings, "; ")))
+ enrichments = append(enrichments, strings.Join(questionStrings, "; "))
}
}
@@ -448,7 +531,7 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string
pipelineInfo(ctx, "Rerank", "passage_enrich", map[string]interface{}{
"content_len": len(result.Content),
- "enrichment": strings.Join(enrichments, "\n"),
+ "cleaned_len": len(cleanPassageForRerank(result.Content)),
"enrichment_len": len(strings.Join(enrichments, "\n")),
})
diff --git a/internal/application/service/chat_pipline/rerank_clean_test.go b/internal/application/service/chat_pipline/rerank_clean_test.go
new file mode 100644
index 00000000..aee4ec55
--- /dev/null
+++ b/internal/application/service/chat_pipline/rerank_clean_test.go
@@ -0,0 +1,104 @@
+package chatpipline
+
+import (
+ "testing"
+)
+
+func TestCleanPassageForRerank(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ expect string
+ }{
+ {
+ name: "plain text unchanged",
+ input: "这是一段普通的文本内容",
+ expect: "这是一段普通的文本内容",
+ },
+ {
+ name: "remove markdown images",
+ input: "前文  后文",
+ expect: "前文 后文",
+ },
+ {
+ name: "convert markdown links to text",
+ input: "请参考 [官方文档](https://docs.example.com) 了解详情",
+ expect: "请参考 官方文档 了解详情",
+ },
+ {
+ name: "remove standalone URLs",
+ input: "访问 https://example.com/path?q=1&b=2 获取更多信息",
+ expect: "访问 获取更多信息",
+ },
+ {
+ name: "remove code blocks",
+ input: "示例代码:\n```python\nprint('hello')\n```\n以上是示例",
+ expect: "示例代码:\n\n以上是示例",
+ },
+ {
+ name: "remove LaTeX blocks",
+ input: "公式如下 $$E=mc^2$$ 其中E是能量",
+ expect: "公式如下 其中E是能量",
+ },
+ {
+ name: "remove table separator rows",
+ input: "| 名称 | 值 |\n| --- | --- |\n| A | 1 |",
+ expect: "| 名称 | 值 |\n\n| A | 1 |",
+ },
+ {
+ name: "strip heading markers",
+ input: "## 第二章 概述\n### 2.1 背景",
+ expect: "第二章 概述\n2.1 背景",
+ },
+ {
+ name: "strip blockquote markers",
+ input: "> 这是一段引用\n> 第二行引用",
+ expect: "这是一段引用\n第二行引用",
+ },
+ {
+ name: "unwrap bold and italic",
+ input: "这是 **加粗** 和 *斜体* 以及 ***粗斜体*** 文本",
+ expect: "这是 加粗 和 斜体 以及 粗斜体 文本",
+ },
+ {
+ name: "strip list markers",
+ input: "- 项目一\n- 项目二\n1. 有序一\n2. 有序二",
+ expect: "项目一\n项目二\n有序一\n有序二",
+ },
+ {
+ name: "remove HTML tags",
+ input: "文本
换行
内容
结尾",
+ expect: "文本换行内容结尾",
+ },
+ {
+ name: "collapse excessive newlines",
+ input: "段落一\n\n\n\n\n段落二",
+ expect: "段落一\n\n段落二",
+ },
+ {
+ name: "combined real-world passage",
+ input: `## 产品介绍
+
+这是一个 **重要的** 产品。详见 [产品页面](https://example.com/product)。
+
+
+
+> 用户评价:非常好用
+
+- 功能一
+- 功能二
+
+` + "```json\n{\"key\": \"value\"}\n```",
+ expect: "产品介绍\n\n这是一个 重要的 产品。详见 产品页面。\n\n用户评价:非常好用\n\n功能一\n功能二",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := cleanPassageForRerank(tt.input)
+ if got != tt.expect {
+ t.Errorf("cleanPassageForRerank():\ngot: %q\nexpect: %q", got, tt.expect)
+ }
+ })
+ }
+}
diff --git a/internal/application/service/chat_pipline/search_parallel.go b/internal/application/service/chat_pipline/search_parallel.go
index 895fbb83..054b7e48 100644
--- a/internal/application/service/chat_pipline/search_parallel.go
+++ b/internal/application/service/chat_pipline/search_parallel.go
@@ -154,6 +154,7 @@ func (p *PluginSearchParallel) OnEvent(ctx context.Context,
// Merge results from both searches (no concurrent access now)
chatManage.SearchResult = append(chunkChatManage.SearchResult, entityChatManage.SearchResult...)
+ chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
// Log any errors but don't fail the pipeline if at least one search succeeded
if chunkSearchErr != nil {