From d7478094a264d5ea2db983f03b32da1cd844cb82 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 6 May 2026 22:09:49 +0800 Subject: [PATCH] fix(chunker): recognise CN chapter titles and multi-level numeric headings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The heuristic splitter relied on two regexes that were too strict for real-world Chinese technical documents: - ChineseChapterPattern required 第 / numeral / unit to be adjacent, so the very common "第 1 章 引言" form never matched. - NumberedSectionPattern required a trailing dot after the numeral, so multi-level numbering such as "1.1 文档目的" or "2.2.1 用户与权限" was missed. As a result, documents like the CHAPTER_SAMPLE shipped with the chunking debug drawer collected zero heuristic markers, and ProfileDocument fell all the way through to the character-level Legacy tier, producing chunks that ignored the document's explicit chapter structure. - Loosen ChineseChapterPattern to tolerate spaces around 第 / 数字 / 单位. - Allow the multi-level branch of NumberedSectionPattern to drop its trailing dot; keep the single-level / roman branch dot-required to avoid false positives on version strings. - Extend patterns_test.go with positive and negative cases covering the CHAPTER_SAMPLE wording plus deep-nesting and lone-numeral regressions. --- internal/infrastructure/chunker/patterns.go | 11 +++++++---- internal/infrastructure/chunker/patterns_test.go | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/internal/infrastructure/chunker/patterns.go b/internal/infrastructure/chunker/patterns.go index b16ba4a7..2ce6554f 100644 --- a/internal/infrastructure/chunker/patterns.go +++ b/internal/infrastructure/chunker/patterns.go @@ -28,8 +28,10 @@ var MarkdownHeadingPattern = regexp.MustCompile(`(?m)^(#{1,6})\s+(.+?)\s*#*\s*$` var FormFeedPattern = regexp.MustCompile(`\f`) // NumberedSectionPattern matches lines starting with numeric or roman numbering -// followed by a non-empty title, e.g. "1. Intro", "2.3 Methods", "IV. Results". -var NumberedSectionPattern = regexp.MustCompile(`(?m)^[ \t]*(?:\d+(?:\.\d+){0,3}|[IVX]{1,5})\.[ \t]+\S.{0,200}$`) +// followed by a non-empty title, e.g. "1. Intro", "2.3 Methods", "IV. Results", +// "2.2.1 用户与权限". The trailing dot after a multi-level numeral is optional +// because many technical documents write "1.1 Foo" without a closing dot. +var NumberedSectionPattern = regexp.MustCompile(`(?m)^[ \t]*(?:\d+(?:\.\d+){1,3}\.?|(?:\d+|[IVX]{1,5})\.)[ \t]+\S.{0,200}$`) // AllCapsHeadingPattern matches short all-caps lines (likely section titles // rendered without Markdown headings). It requires at least 4 letters and @@ -53,8 +55,9 @@ var GermanChapterPattern = regexp.MustCompile(`(?m)^[ \t]*(?:Kapitel|Abschnitt|T // EnglishChapterPattern matches English chapter / section markers. var EnglishChapterPattern = regexp.MustCompile(`(?m)^[ \t]*(?:Chapter|Section|Part)\s+(?:[0-9]+|[IVX]{1,5})[\.: ].{0,200}$`) -// ChineseChapterPattern matches CJK chapter / section markers like 第一章, 第3节. -var ChineseChapterPattern = regexp.MustCompile(`(?m)^[ \t]*第[一二三四五六七八九十百千零〇0-9]+(?:章|节|節|部分|篇)[ \t]?.{0,200}$`) +// ChineseChapterPattern matches CJK chapter / section markers like 第一章, +// 第3节, 第 1 章 (whitespace between 第 / numeral / unit is tolerated). +var ChineseChapterPattern = regexp.MustCompile(`(?m)^[ \t]*第[ \t]*[一二三四五六七八九十百千零〇0-9]+[ \t]*(?:章|节|節|部分|篇)[ \t]?.{0,200}$`) // SentenceSeparators returns sentence-level separators tuned for the language. // Used for fine-grained sub-splitting when a section is still too large. diff --git a/internal/infrastructure/chunker/patterns_test.go b/internal/infrastructure/chunker/patterns_test.go index c2ec47e2..fd953b27 100644 --- a/internal/infrastructure/chunker/patterns_test.go +++ b/internal/infrastructure/chunker/patterns_test.go @@ -29,11 +29,16 @@ func TestNumberedSectionPattern(t *testing.T) { match bool }{ {"1. Introduction", true}, - {"2.3 Methodology", false}, // requires dot after number + {"2.3 Methodology", true}, // multi-level numerals no longer require trailing dot {"2.3. Methodology", true}, + {"2.2.1 用户与权限", true}, // three-level numbering, Chinese title, no trailing dot + {"3.2.1 单机 Docker Compose", true}, {"IV. Results", true}, - {"1.Introduction", false}, // requires whitespace - {"1.", false}, // requires title + {"1.Introduction", false}, // requires whitespace + {"1.", false}, // requires title + {"1.1", false}, // requires title after numeral + {"1 NoDotSingleLevel", false}, // single-level numerals still need trailing dot + {"1.2.3.4.5 TooDeep", false}, // more than 3 sub-levels not accepted {"plain text", false}, } for _, c := range cases { @@ -89,7 +94,11 @@ func TestChineseChapterPattern(t *testing.T) { {"第一章 引言", true}, {"第3节 方法论", true}, {"第二部分 结果", true}, + {"第 1 章 引言", true}, // space between 第 / numeral / unit + {"第 一 章 引言", true}, + {"第1章 引言", true}, {"Chapter 1", false}, + {"第 章 空数字", false}, // missing numeral } for _, c := range cases { got := ChineseChapterPattern.MatchString(c.in)