From cbee01f5e9289afc6f46624e9f4d07a9266d2b9b Mon Sep 17 00:00:00 2001 From: lyingbug Date: Fri, 20 Mar 2026 22:19:38 +0000 Subject: [PATCH] fix: improve markdown table cleaning and filter empty passages in rerank (#752) Enhanced cleanPassageForRerank() to convert markdown table data rows (| col1 | col2 |) into plain text ("col1, col2") instead of leaving them intact. Added filtering to skip empty/whitespace-only passages before sending to the rerank API, preventing Baidu Qianfan 400 errors caused by empty document fragments after markdown cleaning. --- .../service/chat_pipline/rerank.go | 43 ++++++++++++++++++- .../service/chat_pipline/rerank_clean_test.go | 24 ++++++++++- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/internal/application/service/chat_pipline/rerank.go b/internal/application/service/chat_pipline/rerank.go index edb72818..babf372a 100644 --- a/internal/application/service/chat_pipline/rerank.go +++ b/internal/application/service/chat_pipline/rerank.go @@ -82,9 +82,9 @@ func (p *PluginRerank) OnEvent(ctx context.Context, } // 合并Content和ImageInfo的文本内容 passage := getEnrichedPassage(ctx, result) - // 过滤空内容,避免Rerank API报错 + // Skip passages that become empty after cleaning if strings.TrimSpace(passage) == "" { - pipelineWarn(ctx, "Rerank", "empty_passage_skip", map[string]interface{}{ + pipelineInfo(ctx, "Rerank", "empty_passage_skip", map[string]interface{}{ "chunk_id": result.ID, }) continue @@ -209,6 +209,27 @@ func (p *PluginRerank) rerank(ctx context.Context, "query_variant": query, "passages": len(passages), }) + + // Filter out empty or whitespace-only passages before sending to the API + var cleanPassages []string + var cleanCandidates []*types.SearchResult + for i, p := range passages { + if strings.TrimSpace(p) != "" { + cleanPassages = append(cleanPassages, p) + if i < len(candidates) { + cleanCandidates = append(cleanCandidates, candidates[i]) + } + } + } + if len(cleanPassages) == 0 { + pipelineInfo(ctx, "Rerank", "model_call_skip", map[string]interface{}{ + "reason": "all_passages_empty", + }) + return nil + } + passages = cleanPassages + candidates = cleanCandidates + rerankResp, err := rerankModel.Rerank(ctx, query, passages) if err != nil { pipelineError(ctx, "Rerank", "model_call", map[string]interface{}{ @@ -428,6 +449,8 @@ var ( reLatexBlock = regexp.MustCompile(`(?s)\$\$.*?\$\$`) // reTableSep matches table separator rows like |---|---|. reTableSep = regexp.MustCompile(`(?m)^\s*\|[\s:|-]+\|\s*$`) + // reTableRow matches markdown table data rows like | col1 | col2 |. + reTableRow = regexp.MustCompile(`(?m)^\s*\|(.+?)\|\s*$`) // reHeadingPrefix matches leading # markers in headings. reHeadingPrefix = regexp.MustCompile(`(?m)^#{1,6}\s+`) // reBlockquote matches leading > markers. @@ -468,6 +491,22 @@ func cleanPassageForRerank(text string) string { text = reRawURL.ReplaceAllString(text, "") // 7. Remove table separator rows text = reTableSep.ReplaceAllString(text, "") + // 7.5. Convert table data rows to plain text (strip | delimiters) + text = reTableRow.ReplaceAllStringFunc(text, func(match string) string { + inner := reTableRow.FindStringSubmatch(match) + if len(inner) < 2 { + return match + } + cells := strings.Split(inner[1], "|") + var parts []string + for _, cell := range cells { + cell = strings.TrimSpace(cell) + if cell != "" { + parts = append(parts, cell) + } + } + return strings.Join(parts, ", ") + }) // 8. Strip heading markers but keep heading text text = reHeadingPrefix.ReplaceAllString(text, "") // 9. Strip blockquote markers diff --git a/internal/application/service/chat_pipline/rerank_clean_test.go b/internal/application/service/chat_pipline/rerank_clean_test.go index aee4ec55..12eaf3f5 100644 --- a/internal/application/service/chat_pipline/rerank_clean_test.go +++ b/internal/application/service/chat_pipline/rerank_clean_test.go @@ -41,9 +41,9 @@ func TestCleanPassageForRerank(t *testing.T) { expect: "公式如下 其中E是能量", }, { - name: "remove table separator rows", + name: "remove table separator rows and convert data rows", input: "| 名称 | 值 |\n| --- | --- |\n| A | 1 |", - expect: "| 名称 | 值 |\n\n| A | 1 |", + expect: "名称, 值\n\nA, 1", }, { name: "strip heading markers", @@ -91,6 +91,26 @@ func TestCleanPassageForRerank(t *testing.T) { ` + "```json\n{\"key\": \"value\"}\n```", expect: "产品介绍\n\n这是一个 重要的 产品。详见 产品页面。\n\n用户评价:非常好用\n\n功能一\n功能二", }, + { + name: "convert table data rows to plain text", + input: "| col1 | col2 | col3 |", + expect: "col1, col2, col3", + }, + { + name: "multi-row table fully converted", + input: "| Header1 | Header2 |\n| --- | --- |\n| data1 | data2 |\n| data3 | data4 |", + expect: "Header1, Header2\n\ndata1, data2\ndata3, data4", + }, + { + name: "table-only passage becomes empty after separator removal", + input: "| --- | --- |", + expect: "", + }, + { + name: "whitespace-only after cleaning", + input: " \n\n ", + expect: "", + }, } for _, tt := range tests {