diff --git a/internal/infrastructure/docparser/html_table_normalizer.go b/internal/infrastructure/docparser/html_table_normalizer.go new file mode 100644 index 00000000..ad55b196 --- /dev/null +++ b/internal/infrastructure/docparser/html_table_normalizer.go @@ -0,0 +1,77 @@ +package docparser + +import ( + "regexp" + "strings" + + "github.com/JohannesKaufmann/html-to-markdown/v2/converter" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table" +) + +var ( + // htmlTableBlockPattern matches a single (non-nested) ...
+ // block, the form OCR/layout engines such as PaddleOCR-VL emit tables in. + htmlTableBlockPattern = regexp.MustCompile(`(?is)]*>.*?`) + + // htmlLayoutAttrPattern matches presentational HTML attributes that carry + // no semantic value (text-align styles, CSS classes, sizing). Structural + // attributes like rowspan/colspan are intentionally excluded. + htmlLayoutAttrPattern = regexp.MustCompile( + `(?is)\s+(?:style|class|align|valign|width|height|bgcolor)\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)`, + ) + + // htmlSpanAttrPattern detects rowspan/colspan, which Markdown tables cannot + // represent; such tables keep their HTML form (attributes stripped) instead. + htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\b`) + + // markdownTableSeparatorPattern matches the |---|---| delimiter row that a + // valid GFM table must contain. + markdownTableSeparatorPattern = regexp.MustCompile(`(?m)^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)+\|?\s*$`) +) + +// normalizeHTMLTables rewrites inline HTML blocks embedded in OCR +// markdown output. PaddleOCR-VL emits tables as HTML with per-cell text-align +// styles, which (1) waste tokens on layout markup and (2) are not recognized +// by the chunker's table-protection logic, so large tables get split mid-row. +// +// Each table block is converted to a GFM Markdown table when possible. Tables +// that use rowspan/colspan (which Markdown cannot express) fall back to having +// their presentational attributes stripped so they stay intact as HTML. +func normalizeHTMLTables(md string) string { + if !strings.Contains(strings.ToLower(md), "` + + `` + + `
指标数值
营收10亿
利润2.3亿
+ +结尾。` + + got := normalizeHTMLTables(input) + + if strings.Contains(got, "合计` + + `AB` + + got := normalizeHTMLTables(input) + + if !strings.Contains(got, "