Files
Enterprise-Knowledge-Base/internal/infrastructure/chunker/patterns_test.go
T
Claude ae6fde2310 feat(chunker): foundation for adaptive 3-tier chunking
Adds the building blocks for tiered chunking strategies — multilingual
patterns, token approximation, document profiling, validation, and a
strategy resolver that today still falls through to the legacy
SplitText. Default Strategy is empty/legacy so existing knowledge bases
continue to chunk identically.

- tokens.go: char/token approximation per language (en/de/zh/mixed)
  and a coarse DetectLanguage heuristic
- patterns.go: source-of-truth regex bibliothek for headings, numbered
  sections, multilingual chapter markers, visual separators, etc.
- profiler.go: single-pass document scan + SelectStrategy chain
- validator.go: rejects obviously broken chunk-sets to drive fallbacks
- strategy.go: public Split / SplitParentChild entry points; resolves
  StrategyAuto via the profiler, anything else maps to a fixed tier
- splitter.go: SplitterConfig gains Strategy / TokenLimit / Languages
  fields and DefaultChunkSize / DefaultChunkOverlap constants (overlap
  raised to 80 ≈ 15%)
- knowledge.go: switches both call sites to chunker.Split /
  SplitParentChild — behavior unchanged for empty Strategy

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
2026-05-06 17:17:07 +08:00

187 lines
4.5 KiB
Go

package chunker
import "testing"
func TestMarkdownHeadingPattern_BasicLevels(t *testing.T) {
cases := []struct {
in string
match bool
}{
{"# Heading 1", true},
{"## Heading 2", true},
{"###### Heading 6", true},
{"####### Too many", false},
{"#NoSpace", false},
{" # Indented", false},
{"plain text", false},
}
for _, c := range cases {
got := MarkdownHeadingPattern.MatchString(c.in)
if got != c.match {
t.Errorf("MarkdownHeadingPattern(%q): got %v want %v", c.in, got, c.match)
}
}
}
func TestNumberedSectionPattern(t *testing.T) {
cases := []struct {
in string
match bool
}{
{"1. Introduction", true},
{"2.3 Methodology", false}, // requires dot after number
{"2.3. Methodology", true},
{"IV. Results", true},
{"1.Introduction", false}, // requires whitespace
{"1.", false}, // requires title
{"plain text", false},
}
for _, c := range cases {
got := NumberedSectionPattern.MatchString(c.in)
if got != c.match {
t.Errorf("NumberedSectionPattern(%q): got %v want %v", c.in, got, c.match)
}
}
}
func TestGermanChapterPattern(t *testing.T) {
cases := []struct {
in string
match bool
}{
{"Kapitel 1: Einführung", true},
{"Abschnitt 2.3 Methodik", true},
{"Abschnitt 3 Methodik", true},
{"Teil II Ergebnisse", true},
{"chapter 1", false},
}
for _, c := range cases {
got := GermanChapterPattern.MatchString(c.in)
if got != c.match {
t.Errorf("GermanChapterPattern(%q): got %v want %v", c.in, got, c.match)
}
}
}
func TestEnglishChapterPattern(t *testing.T) {
cases := []struct {
in string
match bool
}{
{"Chapter 1: Intro", true},
{"Section 5 Methods", true},
{"Part IV Results", true},
{"Kapitel 1", false},
}
for _, c := range cases {
got := EnglishChapterPattern.MatchString(c.in)
if got != c.match {
t.Errorf("EnglishChapterPattern(%q): got %v want %v", c.in, got, c.match)
}
}
}
func TestChineseChapterPattern(t *testing.T) {
cases := []struct {
in string
match bool
}{
{"第一章 引言", true},
{"第3节 方法论", true},
{"第二部分 结果", true},
{"Chapter 1", false},
}
for _, c := range cases {
got := ChineseChapterPattern.MatchString(c.in)
if got != c.match {
t.Errorf("ChineseChapterPattern(%q): got %v want %v", c.in, got, c.match)
}
}
}
func TestVisualSeparatorPattern(t *testing.T) {
cases := []struct {
in string
match bool
}{
{"---", true},
{"========", true},
{"***", true},
{"____", true},
{"--", false}, // needs at least 3
{"-- text", false},
}
for _, c := range cases {
got := VisualSeparatorPattern.MatchString(c.in)
if got != c.match {
t.Errorf("VisualSeparatorPattern(%q): got %v want %v", c.in, got, c.match)
}
}
}
func TestPageFooterPattern(t *testing.T) {
cases := []struct {
in string
match bool
}{
{"Seite 3 von 24", true},
{"Page 5 of 12", true},
{"page 7", true},
{"Seite 9", true},
{"页 3", true},
{"页码 3 / 12", true},
{"Some text", false},
}
for _, c := range cases {
got := PageFooterPattern.MatchString(c.in)
if got != c.match {
t.Errorf("PageFooterPattern(%q): got %v want %v", c.in, got, c.match)
}
}
}
func TestAllCapsHeadingPattern(t *testing.T) {
cases := []struct {
in string
match bool
}{
{"INTRODUCTION", true},
{"METHODS AND MATERIALS", true},
{"Mixed Case Heading", false},
{"ABC", false}, // <4 chars
}
for _, c := range cases {
got := AllCapsHeadingPattern.MatchString(c.in)
if got != c.match {
t.Errorf("AllCapsHeadingPattern(%q): got %v want %v", c.in, got, c.match)
}
}
}
func TestSentenceSeparators(t *testing.T) {
if got := SentenceSeparators(LangChinese); got[0] != "。" {
t.Errorf("Chinese should start with 。, got %v", got)
}
if got := SentenceSeparators(LangEnglish); got[0] != ". " {
t.Errorf("English should start with '. ', got %v", got)
}
if got := SentenceSeparators("xx"); len(got) < 5 {
t.Errorf("Unknown lang should return mixed (>=5 separators), got %v", got)
}
}
func TestChapterPatternsForLangs(t *testing.T) {
if got := ChapterPatternsForLangs(nil); len(got) != 3 {
t.Errorf("nil langs should return all 3, got %d", len(got))
}
if got := ChapterPatternsForLangs([]string{LangGerman}); len(got) != 1 {
t.Errorf("only DE requested should return 1, got %d", len(got))
}
if got := ChapterPatternsForLangs([]string{LangGerman, LangChinese}); len(got) != 2 {
t.Errorf("DE+ZH requested should return 2, got %d", len(got))
}
if got := ChapterPatternsForLangs([]string{"xx"}); len(got) != 3 {
t.Errorf("unknown lang should fall back to all 3, got %d", len(got))
}
}