fix: improve markdown table cleaning and filter empty passages in rerank (#752)

Enhanced cleanPassageForRerank() to convert markdown table data rows
(| col1 | col2 |) into plain text ("col1, col2") instead of leaving
them intact. Added filtering to skip empty/whitespace-only passages
before sending to the rerank API, preventing Baidu Qianfan 400 errors
caused by empty document fragments after markdown cleaning.
This commit is contained in:
lyingbug
2026-03-20 22:19:38 +00:00
committed by lyingbug
parent b291381cc2
commit cbee01f5e9
2 changed files with 63 additions and 4 deletions
@@ -82,9 +82,9 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
}
// 合并Content和ImageInfo的文本内容
passage := getEnrichedPassage(ctx, result)
// 过滤空内容,避免Rerank API报错
// Skip passages that become empty after cleaning
if strings.TrimSpace(passage) == "" {
pipelineWarn(ctx, "Rerank", "empty_passage_skip", map[string]interface{}{
pipelineInfo(ctx, "Rerank", "empty_passage_skip", map[string]interface{}{
"chunk_id": result.ID,
})
continue
@@ -209,6 +209,27 @@ func (p *PluginRerank) rerank(ctx context.Context,
"query_variant": query,
"passages": len(passages),
})
// Filter out empty or whitespace-only passages before sending to the API
var cleanPassages []string
var cleanCandidates []*types.SearchResult
for i, p := range passages {
if strings.TrimSpace(p) != "" {
cleanPassages = append(cleanPassages, p)
if i < len(candidates) {
cleanCandidates = append(cleanCandidates, candidates[i])
}
}
}
if len(cleanPassages) == 0 {
pipelineInfo(ctx, "Rerank", "model_call_skip", map[string]interface{}{
"reason": "all_passages_empty",
})
return nil
}
passages = cleanPassages
candidates = cleanCandidates
rerankResp, err := rerankModel.Rerank(ctx, query, passages)
if err != nil {
pipelineError(ctx, "Rerank", "model_call", map[string]interface{}{
@@ -428,6 +449,8 @@ var (
reLatexBlock = regexp.MustCompile(`(?s)\$\$.*?\$\$`)
// reTableSep matches table separator rows like |---|---|.
reTableSep = regexp.MustCompile(`(?m)^\s*\|[\s:|-]+\|\s*$`)
// reTableRow matches markdown table data rows like | col1 | col2 |.
reTableRow = regexp.MustCompile(`(?m)^\s*\|(.+?)\|\s*$`)
// reHeadingPrefix matches leading # markers in headings.
reHeadingPrefix = regexp.MustCompile(`(?m)^#{1,6}\s+`)
// reBlockquote matches leading > markers.
@@ -468,6 +491,22 @@ func cleanPassageForRerank(text string) string {
text = reRawURL.ReplaceAllString(text, "")
// 7. Remove table separator rows
text = reTableSep.ReplaceAllString(text, "")
// 7.5. Convert table data rows to plain text (strip | delimiters)
text = reTableRow.ReplaceAllStringFunc(text, func(match string) string {
inner := reTableRow.FindStringSubmatch(match)
if len(inner) < 2 {
return match
}
cells := strings.Split(inner[1], "|")
var parts []string
for _, cell := range cells {
cell = strings.TrimSpace(cell)
if cell != "" {
parts = append(parts, cell)
}
}
return strings.Join(parts, ", ")
})
// 8. Strip heading markers but keep heading text
text = reHeadingPrefix.ReplaceAllString(text, "")
// 9. Strip blockquote markers
@@ -41,9 +41,9 @@ func TestCleanPassageForRerank(t *testing.T) {
expect: "公式如下 其中E是能量",
},
{
name: "remove table separator rows",
name: "remove table separator rows and convert data rows",
input: "| 名称 | 值 |\n| --- | --- |\n| A | 1 |",
expect: "| 名称 | 值 |\n\n| A | 1 |",
expect: "名称, 值\n\nA, 1",
},
{
name: "strip heading markers",
@@ -91,6 +91,26 @@ func TestCleanPassageForRerank(t *testing.T) {
` + "```json\n{\"key\": \"value\"}\n```",
expect: "产品介绍\n\n这是一个 重要的 产品。详见 产品页面。\n\n用户评价:非常好用\n\n功能一\n功能二",
},
{
name: "convert table data rows to plain text",
input: "| col1 | col2 | col3 |",
expect: "col1, col2, col3",
},
{
name: "multi-row table fully converted",
input: "| Header1 | Header2 |\n| --- | --- |\n| data1 | data2 |\n| data3 | data4 |",
expect: "Header1, Header2\n\ndata1, data2\ndata3, data4",
},
{
name: "table-only passage becomes empty after separator removal",
input: "| --- | --- |",
expect: "",
},
{
name: "whitespace-only after cleaning",
input: " \n\n ",
expect: "",
},
}
for _, tt := range tests {