fix(docparser): normalize PaddleOCR-VL HTML tables (#1725)
PaddleOCR-VL renders tables as HTML with per-cell text-align styles. These waste tokens on layout markup and bypass the chunker's table-protection logic (which only recognizes Markdown pipe tables), causing large tables to be split mid-row. Convert inline HTML <table> blocks to GFM Markdown tables when possible, and strip presentational attributes (style/class/align/...) for tables using rowspan/colspan that Markdown cannot express. Refs: https://github.com/Tencent/WeKnora/issues/1725
This commit is contained in:
@@ -0,0 +1,77 @@
|
|||||||
|
package docparser
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||||
|
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base"
|
||||||
|
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark"
|
||||||
|
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// htmlTableBlockPattern matches a single (non-nested) <table>...</table>
|
||||||
|
// block, the form OCR/layout engines such as PaddleOCR-VL emit tables in.
|
||||||
|
htmlTableBlockPattern = regexp.MustCompile(`(?is)<table\b[^>]*>.*?</table>`)
|
||||||
|
|
||||||
|
// htmlLayoutAttrPattern matches presentational HTML attributes that carry
|
||||||
|
// no semantic value (text-align styles, CSS classes, sizing). Structural
|
||||||
|
// attributes like rowspan/colspan are intentionally excluded.
|
||||||
|
htmlLayoutAttrPattern = regexp.MustCompile(
|
||||||
|
`(?is)\s+(?:style|class|align|valign|width|height|bgcolor)\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)`,
|
||||||
|
)
|
||||||
|
|
||||||
|
// htmlSpanAttrPattern detects rowspan/colspan, which Markdown tables cannot
|
||||||
|
// represent; such tables keep their HTML form (attributes stripped) instead.
|
||||||
|
htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\b`)
|
||||||
|
|
||||||
|
// markdownTableSeparatorPattern matches the |---|---| delimiter row that a
|
||||||
|
// valid GFM table must contain.
|
||||||
|
markdownTableSeparatorPattern = regexp.MustCompile(`(?m)^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)+\|?\s*$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// normalizeHTMLTables rewrites inline HTML <table> blocks embedded in OCR
|
||||||
|
// markdown output. PaddleOCR-VL emits tables as HTML with per-cell text-align
|
||||||
|
// styles, which (1) waste tokens on layout markup and (2) are not recognized
|
||||||
|
// by the chunker's table-protection logic, so large tables get split mid-row.
|
||||||
|
//
|
||||||
|
// Each table block is converted to a GFM Markdown table when possible. Tables
|
||||||
|
// that use rowspan/colspan (which Markdown cannot express) fall back to having
|
||||||
|
// their presentational attributes stripped so they stay intact as HTML.
|
||||||
|
func normalizeHTMLTables(md string) string {
|
||||||
|
if !strings.Contains(strings.ToLower(md), "<table") {
|
||||||
|
return md
|
||||||
|
}
|
||||||
|
|
||||||
|
conv := converter.NewConverter(
|
||||||
|
converter.WithPlugins(
|
||||||
|
base.NewBasePlugin(),
|
||||||
|
commonmark.NewCommonmarkPlugin(),
|
||||||
|
table.NewTablePlugin(),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
return htmlTableBlockPattern.ReplaceAllStringFunc(md, func(block string) string {
|
||||||
|
if htmlSpanAttrPattern.MatchString(block) {
|
||||||
|
return stripHTMLLayoutAttrs(block)
|
||||||
|
}
|
||||||
|
converted, err := conv.ConvertString(block)
|
||||||
|
if err != nil {
|
||||||
|
return stripHTMLLayoutAttrs(block)
|
||||||
|
}
|
||||||
|
converted = strings.TrimSpace(converted)
|
||||||
|
if converted == "" || !markdownTableSeparatorPattern.MatchString(converted) {
|
||||||
|
return stripHTMLLayoutAttrs(block)
|
||||||
|
}
|
||||||
|
// Pad with blank lines so the Markdown table is a standalone block that
|
||||||
|
// the chunker recognizes (and protects) as a table.
|
||||||
|
return "\n\n" + converted + "\n\n"
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// stripHTMLLayoutAttrs removes presentational attributes from an HTML fragment
|
||||||
|
// while preserving structural attributes (rowspan/colspan) and text content.
|
||||||
|
func stripHTMLLayoutAttrs(html string) string {
|
||||||
|
return htmlLayoutAttrPattern.ReplaceAllString(html, "")
|
||||||
|
}
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package docparser
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizeHTMLTables_ConvertsStyledTableToMarkdown(t *testing.T) {
|
||||||
|
// Mirrors PaddleOCR-VL output: a data table where every cell carries a
|
||||||
|
// text-align style that wastes tokens.
|
||||||
|
input := `# 报告
|
||||||
|
|
||||||
|
<table><tr><td style="text-align:center;">指标</td><td style="text-align:center;">数值</td></tr>` +
|
||||||
|
`<tr><td style="text-align:center;">营收</td><td style="text-align:right;">10亿</td></tr>` +
|
||||||
|
`<tr><td style="text-align:center;">利润</td><td style="text-align:right;">2.3亿</td></tr></table>
|
||||||
|
|
||||||
|
结尾。`
|
||||||
|
|
||||||
|
got := normalizeHTMLTables(input)
|
||||||
|
|
||||||
|
if strings.Contains(got, "<table") {
|
||||||
|
t.Fatalf("expected HTML table to be converted away, got:\n%s", got)
|
||||||
|
}
|
||||||
|
if strings.Contains(got, "text-align") {
|
||||||
|
t.Fatalf("expected style attributes removed, got:\n%s", got)
|
||||||
|
}
|
||||||
|
if !markdownTableSeparatorPattern.MatchString(got) {
|
||||||
|
t.Fatalf("expected a Markdown table separator row, got:\n%s", got)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"指标", "数值", "营收", "10亿", "利润", "2.3亿"} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Fatalf("expected converted table to retain %q, got:\n%s", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, "# 报告") || !strings.Contains(got, "结尾。") {
|
||||||
|
t.Fatalf("expected surrounding markdown to be preserved, got:\n%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeHTMLTables_StripsAttrsOnSpanTables(t *testing.T) {
|
||||||
|
// rowspan/colspan cannot be expressed in Markdown, so the table stays HTML
|
||||||
|
// but its presentational attributes are stripped.
|
||||||
|
input := `<table><tr><td colspan="2" style="text-align:center;" class="hdr">合计</td></tr>` +
|
||||||
|
`<tr><td style="text-align:left;">A</td><td width="80">B</td></tr></table>`
|
||||||
|
|
||||||
|
got := normalizeHTMLTables(input)
|
||||||
|
|
||||||
|
if !strings.Contains(got, "<table") {
|
||||||
|
t.Fatalf("expected span table to remain HTML, got:\n%s", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, `colspan="2"`) {
|
||||||
|
t.Fatalf("expected colspan to be preserved, got:\n%s", got)
|
||||||
|
}
|
||||||
|
for _, banned := range []string{"text-align", "class=", "width="} {
|
||||||
|
if strings.Contains(got, banned) {
|
||||||
|
t.Fatalf("expected %q to be stripped, got:\n%s", banned, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeHTMLTables_NoTableUnchanged(t *testing.T) {
|
||||||
|
input := "# 标题\n\n普通段落,没有表格。\n\n| a | b |\n| --- | --- |\n| 1 | 2 |"
|
||||||
|
if got := normalizeHTMLTables(input); got != input {
|
||||||
|
t.Fatalf("expected content without HTML tables to be unchanged, got:\n%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -58,6 +58,12 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (*
|
|||||||
return nil, fmt.Errorf("PaddleOCR-VL layout-parsing: %w", err)
|
return nil, fmt.Errorf("PaddleOCR-VL layout-parsing: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PaddleOCR-VL renders tables as styled HTML (per-cell text-align), which
|
||||||
|
// wastes tokens and defeats the chunker's table-protection logic. Convert
|
||||||
|
// them to Markdown tables (or strip layout attributes when conversion is
|
||||||
|
// not possible) before downstream processing.
|
||||||
|
mdContent = normalizeHTMLTables(mdContent)
|
||||||
|
|
||||||
imageRefs, mdContent := c.processImages(mdContent, imagesB64)
|
imageRefs, mdContent := c.processImages(mdContent, imagesB64)
|
||||||
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)
|
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user