fix(chunker): recognise CN chapter titles and multi-level numeric headings

The heuristic splitter relied on two regexes that were too strict for
real-world Chinese technical documents:

- ChineseChapterPattern required 第 / numeral / unit to be adjacent, so
  the very common "第 1 章 引言" form never matched.
- NumberedSectionPattern required a trailing dot after the numeral, so
  multi-level numbering such as "1.1 文档目的" or "2.2.1 用户与权限"
  was missed.

As a result, documents like the CHAPTER_SAMPLE shipped with the chunking
debug drawer collected zero heuristic markers, and ProfileDocument fell
all the way through to the character-level Legacy tier, producing chunks
that ignored the document's explicit chapter structure.

- Loosen ChineseChapterPattern to tolerate spaces around 第 / 数字 / 单位.
- Allow the multi-level branch of NumberedSectionPattern to drop its
  trailing dot; keep the single-level / roman branch dot-required to
  avoid false positives on version strings.
- Extend patterns_test.go with positive and negative cases covering the
  CHAPTER_SAMPLE wording plus deep-nesting and lone-numeral regressions.
This commit is contained in:
wizardchen
2026-05-06 22:09:49 +08:00
committed by lyingbug
parent d703a14745
commit d7478094a2
2 changed files with 19 additions and 7 deletions
+7 -4
View File
@@ -28,8 +28,10 @@ var MarkdownHeadingPattern = regexp.MustCompile(`(?m)^(#{1,6})\s+(.+?)\s*#*\s*$`
var FormFeedPattern = regexp.MustCompile(`\f`)
// NumberedSectionPattern matches lines starting with numeric or roman numbering
// followed by a non-empty title, e.g. "1. Intro", "2.3 Methods", "IV. Results".
var NumberedSectionPattern = regexp.MustCompile(`(?m)^[ \t]*(?:\d+(?:\.\d+){0,3}|[IVX]{1,5})\.[ \t]+\S.{0,200}$`)
// followed by a non-empty title, e.g. "1. Intro", "2.3 Methods", "IV. Results",
// "2.2.1 用户与权限". The trailing dot after a multi-level numeral is optional
// because many technical documents write "1.1 Foo" without a closing dot.
var NumberedSectionPattern = regexp.MustCompile(`(?m)^[ \t]*(?:\d+(?:\.\d+){1,3}\.?|(?:\d+|[IVX]{1,5})\.)[ \t]+\S.{0,200}$`)
// AllCapsHeadingPattern matches short all-caps lines (likely section titles
// rendered without Markdown headings). It requires at least 4 letters and
@@ -53,8 +55,9 @@ var GermanChapterPattern = regexp.MustCompile(`(?m)^[ \t]*(?:Kapitel|Abschnitt|T
// EnglishChapterPattern matches English chapter / section markers.
var EnglishChapterPattern = regexp.MustCompile(`(?m)^[ \t]*(?:Chapter|Section|Part)\s+(?:[0-9]+|[IVX]{1,5})[\.: ].{0,200}$`)
// ChineseChapterPattern matches CJK chapter / section markers like 第一章, 第3节.
var ChineseChapterPattern = regexp.MustCompile(`(?m)^[ \t]*第[一二三四五六七八九十百千零〇0-9]+(?:章|节|節|部分|篇)[ \t]?.{0,200}$`)
// ChineseChapterPattern matches CJK chapter / section markers like 第一章,
// 第3节, 第 1 章 (whitespace between 第 / numeral / unit is tolerated).
var ChineseChapterPattern = regexp.MustCompile(`(?m)^[ \t]*第[ \t]*[一二三四五六七八九十百千零〇0-9]+[ \t]*(?:章|节|節|部分|篇)[ \t]?.{0,200}$`)
// SentenceSeparators returns sentence-level separators tuned for the language.
// Used for fine-grained sub-splitting when a section is still too large.
@@ -29,11 +29,16 @@ func TestNumberedSectionPattern(t *testing.T) {
match bool
}{
{"1. Introduction", true},
{"2.3 Methodology", false}, // requires dot after number
{"2.3 Methodology", true}, // multi-level numerals no longer require trailing dot
{"2.3. Methodology", true},
{"2.2.1 用户与权限", true}, // three-level numbering, Chinese title, no trailing dot
{"3.2.1 单机 Docker Compose", true},
{"IV. Results", true},
{"1.Introduction", false}, // requires whitespace
{"1.", false}, // requires title
{"1.Introduction", false}, // requires whitespace
{"1.", false}, // requires title
{"1.1", false}, // requires title after numeral
{"1 NoDotSingleLevel", false}, // single-level numerals still need trailing dot
{"1.2.3.4.5 TooDeep", false}, // more than 3 sub-levels not accepted
{"plain text", false},
}
for _, c := range cases {
@@ -89,7 +94,11 @@ func TestChineseChapterPattern(t *testing.T) {
{"第一章 引言", true},
{"第3节 方法论", true},
{"第二部分 结果", true},
{"第 1 章 引言", true}, // space between 第 / numeral / unit
{"第 一 章 引言", true},
{"第1章 引言", true},
{"Chapter 1", false},
{"第 章 空数字", false}, // missing numeral
}
for _, c := range cases {
got := ChineseChapterPattern.MatchString(c.in)