diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index b381d694..de1a95d5 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -7484,7 +7484,7 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context, if kb.ChunkingConfig.EnableParentChild { parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg) - pcResult := chunker.SplitTextParentChild(clean, parentCfg, childCfg) + pcResult := chunker.SplitParentChild(clean, parentCfg, childCfg) parsed = make([]types.ParsedChunk, len(pcResult.Children)) for i, c := range pcResult.Children { parsed[i] = types.ParsedChunk{ @@ -7501,7 +7501,7 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context, } opts.ParentChunks = parentChunks } else { - splitChunks := chunker.SplitText(clean, chunkCfg) + splitChunks := chunker.Split(clean, chunkCfg) parsed = make([]types.ParsedChunk, len(splitChunks)) for i, c := range splitChunks { parsed[i] = types.ParsedChunk{ @@ -8340,7 +8340,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e if kb.ChunkingConfig.EnableParentChild { parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg) - pcResult := chunker.SplitTextParentChild(convertResult.MarkdownContent, parentCfg, childCfg) + pcResult := chunker.SplitParentChild(convertResult.MarkdownContent, parentCfg, childCfg) chunks = make([]types.ParsedChunk, len(pcResult.Children)) for i, c := range pcResult.Children { chunks[i] = types.ParsedChunk{ @@ -8359,7 +8359,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e logger.Infof(ctx, "Split document into %d parent + %d child chunks for knowledge %s", len(pcResult.Parents), len(pcResult.Children), knowledge.ID) } else { - splitChunks := chunker.SplitText(convertResult.MarkdownContent, chunkCfg) + splitChunks := chunker.Split(convertResult.MarkdownContent, chunkCfg) chunks = make([]types.ParsedChunk, len(splitChunks)) for i, c := range splitChunks { chunks[i] = types.ParsedChunk{ diff --git a/internal/infrastructure/chunker/patterns.go b/internal/infrastructure/chunker/patterns.go new file mode 100644 index 00000000..b16ba4a7 --- /dev/null +++ b/internal/infrastructure/chunker/patterns.go @@ -0,0 +1,94 @@ +// Package chunker - patterns.go is the source of truth for multilingual +// regex patterns used by the heading-aware and heuristic splitters. +// +// Patterns are grouped by purpose (chapter markers, numbering, separators) +// and tagged with a priority that the heuristic splitter uses to rank +// candidate chunk boundaries. +package chunker + +import "regexp" + +// BoundaryPriority levels for heuristic chunk boundaries. Higher = stronger. +const ( + PrioFormFeed = 100 + PrioNumberedHead = 90 + PrioChapterMarker = 85 + PrioAllCapsHeading = 70 + PrioVisualSep = 60 + PrioPageFooter = 50 + PrioBlankBlock = 40 +) + +// MarkdownHeadingPattern matches an ATX-style Markdown heading at line start. +// Capture groups: (1) hashes, (2) heading text. +var MarkdownHeadingPattern = regexp.MustCompile(`(?m)^(#{1,6})\s+(.+?)\s*#*\s*$`) + +// FormFeedPattern matches the form-feed control character used by some PDF +// converters as a page break marker. +var FormFeedPattern = regexp.MustCompile(`\f`) + +// NumberedSectionPattern matches lines starting with numeric or roman numbering +// followed by a non-empty title, e.g. "1. Intro", "2.3 Methods", "IV. Results". +var NumberedSectionPattern = regexp.MustCompile(`(?m)^[ \t]*(?:\d+(?:\.\d+){0,3}|[IVX]{1,5})\.[ \t]+\S.{0,200}$`) + +// AllCapsHeadingPattern matches short all-caps lines (likely section titles +// rendered without Markdown headings). It requires at least 4 letters and +// up to ~10 words. Trailing colons are tolerated. +var AllCapsHeadingPattern = regexp.MustCompile(`(?m)^[ \t]*([A-ZÄÖÜ][A-ZÄÖÜ \-]{3,80}):?\s*$`) + +// VisualSeparatorPattern matches horizontal rules / divider lines used as +// section separators in plain text or pre-Markdown documents. +var VisualSeparatorPattern = regexp.MustCompile(`(?m)^[ \t]*(?:-{3,}|={3,}|\*{3,}|_{3,})[ \t]*$`) + +// ExcessiveBlanksPattern matches three or more consecutive newlines, which +// usually denote a hard section break. +var ExcessiveBlanksPattern = regexp.MustCompile(`\n{3,}`) + +// PageFooterPattern matches typical "Seite X von Y" / "Page X of Y" lines. +var PageFooterPattern = regexp.MustCompile(`(?mi)^[ \t]*(?:Seite|Page|页码?)\s+\d+(?:\s*(?:von|of|/)\s*\d+)?[ \t]*$`) + +// GermanChapterPattern matches German chapter / section markers. +var GermanChapterPattern = regexp.MustCompile(`(?m)^[ \t]*(?:Kapitel|Abschnitt|Teil)\s+(?:[0-9]+|[IVX]{1,5})[\.: ].{0,200}$`) + +// EnglishChapterPattern matches English chapter / section markers. +var EnglishChapterPattern = regexp.MustCompile(`(?m)^[ \t]*(?:Chapter|Section|Part)\s+(?:[0-9]+|[IVX]{1,5})[\.: ].{0,200}$`) + +// ChineseChapterPattern matches CJK chapter / section markers like 第一章, 第3节. +var ChineseChapterPattern = regexp.MustCompile(`(?m)^[ \t]*第[一二三四五六七八九十百千零〇0-9]+(?:章|节|節|部分|篇)[ \t]?.{0,200}$`) + +// SentenceSeparators returns sentence-level separators tuned for the language. +// Used for fine-grained sub-splitting when a section is still too large. +func SentenceSeparators(lang string) []string { + switch lang { + case LangChinese: + return []string{"。", "!", "?", ";", "\n"} + case LangGerman, LangEnglish: + return []string{". ", "! ", "? ", "; ", "\n"} + default: + return []string{"。", "!", "?", ";", ". ", "! ", "? ", "; ", "\n"} + } +} + +// ChapterPatternsForLangs returns the chapter-marker regexes that apply for +// the given language hints. An empty / unknown list returns all of them so +// that auto-detected documents still match. +func ChapterPatternsForLangs(langs []string) []*regexp.Regexp { + if len(langs) == 0 { + return []*regexp.Regexp{GermanChapterPattern, EnglishChapterPattern, ChineseChapterPattern} + } + var out []*regexp.Regexp + for _, l := range langs { + switch l { + case LangGerman: + out = append(out, GermanChapterPattern) + case LangEnglish: + out = append(out, EnglishChapterPattern) + case LangChinese: + out = append(out, ChineseChapterPattern) + } + } + if len(out) == 0 { + out = []*regexp.Regexp{GermanChapterPattern, EnglishChapterPattern, ChineseChapterPattern} + } + return out +} diff --git a/internal/infrastructure/chunker/patterns_test.go b/internal/infrastructure/chunker/patterns_test.go new file mode 100644 index 00000000..c2ec47e2 --- /dev/null +++ b/internal/infrastructure/chunker/patterns_test.go @@ -0,0 +1,186 @@ +package chunker + +import "testing" + +func TestMarkdownHeadingPattern_BasicLevels(t *testing.T) { + cases := []struct { + in string + match bool + }{ + {"# Heading 1", true}, + {"## Heading 2", true}, + {"###### Heading 6", true}, + {"####### Too many", false}, + {"#NoSpace", false}, + {" # Indented", false}, + {"plain text", false}, + } + for _, c := range cases { + got := MarkdownHeadingPattern.MatchString(c.in) + if got != c.match { + t.Errorf("MarkdownHeadingPattern(%q): got %v want %v", c.in, got, c.match) + } + } +} + +func TestNumberedSectionPattern(t *testing.T) { + cases := []struct { + in string + match bool + }{ + {"1. Introduction", true}, + {"2.3 Methodology", false}, // requires dot after number + {"2.3. Methodology", true}, + {"IV. Results", true}, + {"1.Introduction", false}, // requires whitespace + {"1.", false}, // requires title + {"plain text", false}, + } + for _, c := range cases { + got := NumberedSectionPattern.MatchString(c.in) + if got != c.match { + t.Errorf("NumberedSectionPattern(%q): got %v want %v", c.in, got, c.match) + } + } +} + +func TestGermanChapterPattern(t *testing.T) { + cases := []struct { + in string + match bool + }{ + {"Kapitel 1: Einführung", true}, + {"Abschnitt 2.3 Methodik", true}, + {"Abschnitt 3 Methodik", true}, + {"Teil II Ergebnisse", true}, + {"chapter 1", false}, + } + for _, c := range cases { + got := GermanChapterPattern.MatchString(c.in) + if got != c.match { + t.Errorf("GermanChapterPattern(%q): got %v want %v", c.in, got, c.match) + } + } +} + +func TestEnglishChapterPattern(t *testing.T) { + cases := []struct { + in string + match bool + }{ + {"Chapter 1: Intro", true}, + {"Section 5 Methods", true}, + {"Part IV Results", true}, + {"Kapitel 1", false}, + } + for _, c := range cases { + got := EnglishChapterPattern.MatchString(c.in) + if got != c.match { + t.Errorf("EnglishChapterPattern(%q): got %v want %v", c.in, got, c.match) + } + } +} + +func TestChineseChapterPattern(t *testing.T) { + cases := []struct { + in string + match bool + }{ + {"第一章 引言", true}, + {"第3节 方法论", true}, + {"第二部分 结果", true}, + {"Chapter 1", false}, + } + for _, c := range cases { + got := ChineseChapterPattern.MatchString(c.in) + if got != c.match { + t.Errorf("ChineseChapterPattern(%q): got %v want %v", c.in, got, c.match) + } + } +} + +func TestVisualSeparatorPattern(t *testing.T) { + cases := []struct { + in string + match bool + }{ + {"---", true}, + {"========", true}, + {"***", true}, + {"____", true}, + {"--", false}, // needs at least 3 + {"-- text", false}, + } + for _, c := range cases { + got := VisualSeparatorPattern.MatchString(c.in) + if got != c.match { + t.Errorf("VisualSeparatorPattern(%q): got %v want %v", c.in, got, c.match) + } + } +} + +func TestPageFooterPattern(t *testing.T) { + cases := []struct { + in string + match bool + }{ + {"Seite 3 von 24", true}, + {"Page 5 of 12", true}, + {"page 7", true}, + {"Seite 9", true}, + {"页 3", true}, + {"页码 3 / 12", true}, + {"Some text", false}, + } + for _, c := range cases { + got := PageFooterPattern.MatchString(c.in) + if got != c.match { + t.Errorf("PageFooterPattern(%q): got %v want %v", c.in, got, c.match) + } + } +} + +func TestAllCapsHeadingPattern(t *testing.T) { + cases := []struct { + in string + match bool + }{ + {"INTRODUCTION", true}, + {"METHODS AND MATERIALS", true}, + {"Mixed Case Heading", false}, + {"ABC", false}, // <4 chars + } + for _, c := range cases { + got := AllCapsHeadingPattern.MatchString(c.in) + if got != c.match { + t.Errorf("AllCapsHeadingPattern(%q): got %v want %v", c.in, got, c.match) + } + } +} + +func TestSentenceSeparators(t *testing.T) { + if got := SentenceSeparators(LangChinese); got[0] != "。" { + t.Errorf("Chinese should start with 。, got %v", got) + } + if got := SentenceSeparators(LangEnglish); got[0] != ". " { + t.Errorf("English should start with '. ', got %v", got) + } + if got := SentenceSeparators("xx"); len(got) < 5 { + t.Errorf("Unknown lang should return mixed (>=5 separators), got %v", got) + } +} + +func TestChapterPatternsForLangs(t *testing.T) { + if got := ChapterPatternsForLangs(nil); len(got) != 3 { + t.Errorf("nil langs should return all 3, got %d", len(got)) + } + if got := ChapterPatternsForLangs([]string{LangGerman}); len(got) != 1 { + t.Errorf("only DE requested should return 1, got %d", len(got)) + } + if got := ChapterPatternsForLangs([]string{LangGerman, LangChinese}); len(got) != 2 { + t.Errorf("DE+ZH requested should return 2, got %d", len(got)) + } + if got := ChapterPatternsForLangs([]string{"xx"}); len(got) != 3 { + t.Errorf("unknown lang should fall back to all 3, got %d", len(got)) + } +} diff --git a/internal/infrastructure/chunker/profiler.go b/internal/infrastructure/chunker/profiler.go new file mode 100644 index 00000000..0a3f6906 --- /dev/null +++ b/internal/infrastructure/chunker/profiler.go @@ -0,0 +1,235 @@ +// Package chunker - profiler.go scans a document once to gather structure +// indicators that drive strategy selection (heading-aware vs. heuristic vs. +// recursive). Profiling is cheap (a few regex passes plus rune counting) +// and runs before any chunking decision is made. +package chunker + +import ( + "math" + "strings" +) + +// DocProfile holds the document-level signals used to choose a chunking tier. +type DocProfile struct { + TotalChars int + TotalLines int + AvgLineLen float64 + StdLineLen float64 + + // Markdown structure + MdHeadingCounts map[int]int // level (1..6) → count + MdHeadingTotal int + + // Heuristic indicators + NumberedSectionCount int + AllCapsShortLineCount int + BlankParagraphBreaks int + FormFeedCount int + VisualSepCount int + GermanChapterCount int + EnglishChapterCount int + ChineseChapterCount int + RepeatedFooterCount int + + // Content characteristics + HasTables bool + HasCode bool + CodeRatio float64 + + // Detected language hints (best-effort) + DetectedLangs []string +} + +// HeadingDensity returns the share of lines that are Markdown headings. +func (p *DocProfile) HeadingDensity() float64 { + if p.TotalLines == 0 { + return 0 + } + return float64(p.MdHeadingTotal) / float64(p.TotalLines) +} + +// DominantHeadingLevel returns the heading level (1..6) that should drive +// section splitting. It picks the lowest level (closest to root) that has +// at least 3 occurrences, falling back to the most frequent level. Returns +// 0 when no usable Markdown structure exists. +func (p *DocProfile) DominantHeadingLevel() int { + if p.MdHeadingTotal == 0 { + return 0 + } + for level := 1; level <= 6; level++ { + if p.MdHeadingCounts[level] >= 3 { + return level + } + } + // fallback: most frequent level + bestLevel, bestCount := 0, 0 + for level := 1; level <= 6; level++ { + if p.MdHeadingCounts[level] > bestCount { + bestLevel, bestCount = level, p.MdHeadingCounts[level] + } + } + return bestLevel +} + +// HeuristicMarkerTotal sums the non-Markdown structural markers. +func (p *DocProfile) HeuristicMarkerTotal() int { + return p.NumberedSectionCount + + p.GermanChapterCount + p.EnglishChapterCount + p.ChineseChapterCount + + p.AllCapsShortLineCount + p.VisualSepCount + p.FormFeedCount +} + +// ProfileDocument runs a single pass over text and returns its profile. +func ProfileDocument(text string) *DocProfile { + p := &DocProfile{ + MdHeadingCounts: make(map[int]int), + } + if text == "" { + return p + } + + p.TotalChars = len([]rune(text)) + p.FormFeedCount = strings.Count(text, "\f") + + lines := strings.Split(text, "\n") + p.TotalLines = len(lines) + + // First pass: per-line markers and length stats + var lengths []float64 + inFence := false + codeChars := 0 + for _, line := range lines { + trimmed := strings.TrimSpace(line) + + // Toggle fenced-code state. We use a 3-backtick prefix detector here + // rather than a full regex so we don't have to fight with the + // protected-pattern logic later. + if strings.HasPrefix(trimmed, "```") { + inFence = !inFence + p.HasCode = true + continue + } + if inFence { + codeChars += len([]rune(line)) + continue + } + + runeLen := len([]rune(line)) + lengths = append(lengths, float64(runeLen)) + + if matchHeading(line, &p.MdHeadingCounts) { + p.MdHeadingTotal++ + continue + } + if NumberedSectionPattern.MatchString(line) { + p.NumberedSectionCount++ + } + if GermanChapterPattern.MatchString(line) { + p.GermanChapterCount++ + } + if EnglishChapterPattern.MatchString(line) { + p.EnglishChapterCount++ + } + if ChineseChapterPattern.MatchString(line) { + p.ChineseChapterCount++ + } + if AllCapsHeadingPattern.MatchString(line) { + p.AllCapsShortLineCount++ + } + if VisualSeparatorPattern.MatchString(line) { + p.VisualSepCount++ + } + if PageFooterPattern.MatchString(line) { + p.RepeatedFooterCount++ + } + if strings.HasPrefix(trimmed, "|") && strings.HasSuffix(trimmed, "|") { + p.HasTables = true + } + } + + if len(lengths) > 0 { + var sum float64 + for _, l := range lengths { + sum += l + } + p.AvgLineLen = sum / float64(len(lengths)) + var variance float64 + for _, l := range lengths { + d := l - p.AvgLineLen + variance += d * d + } + variance /= float64(len(lengths)) + p.StdLineLen = math.Sqrt(variance) + } + + if p.TotalChars > 0 { + p.CodeRatio = float64(codeChars) / float64(p.TotalChars) + } + + p.BlankParagraphBreaks = strings.Count(text, "\n\n\n") + + // Sample a slice of the document for language detection — avoids paying + // O(N) scan cost on huge inputs while still giving a stable signal. + sample := text + if len(sample) > 4096 { + sample = sample[:4096] + } + lang := DetectLanguage(sample) + p.DetectedLangs = []string{lang} + if lang == LangMixed { + // Provide all three for downstream pattern selection. + p.DetectedLangs = []string{LangEnglish, LangGerman, LangChinese} + } + + return p +} + +// matchHeading checks whether line is an ATX heading and increments the +// appropriate level counter when so. Returns true on match. +func matchHeading(line string, counts *map[int]int) bool { + m := MarkdownHeadingPattern.FindStringSubmatch(line) + if m == nil { + return false + } + level := len(m[1]) + if level < 1 || level > 6 { + return false + } + (*counts)[level]++ + return true +} + +// StrategyTier identifies which chunking implementation should run. +type StrategyTier string + +const ( + TierHeading StrategyTier = "heading" + TierHeuristic StrategyTier = "heuristic" + TierRecursive StrategyTier = "recursive" + TierLegacy StrategyTier = "legacy" +) + +// SelectStrategy returns the ordered tier chain to attempt for this document. +// The first tier is the primary choice; subsequent tiers are fallbacks if +// validation rejects the previous output. The "legacy" tier is appended as +// a final safety net so callers always receive at least one chunk-set. +func SelectStrategy(p *DocProfile) []StrategyTier { + if p == nil { + return []StrategyTier{TierRecursive, TierLegacy} + } + var chain []StrategyTier + + // Tier 1 candidate: Markdown heading-aware + if p.MdHeadingTotal >= 3 && p.HeadingDensity() > 0.005 && p.DominantHeadingLevel() > 0 { + chain = append(chain, TierHeading) + } + + // Tier 2 candidate: heuristic boundary detection + if p.HeuristicMarkerTotal() >= 5 || p.FormFeedCount > 0 || + p.GermanChapterCount+p.EnglishChapterCount+p.ChineseChapterCount > 0 { + chain = append(chain, TierHeuristic) + } + + // Always end with recursive (Tier 3) and then legacy as ultimate fallback. + chain = append(chain, TierRecursive, TierLegacy) + return chain +} diff --git a/internal/infrastructure/chunker/profiler_test.go b/internal/infrastructure/chunker/profiler_test.go new file mode 100644 index 00000000..6549ff1a --- /dev/null +++ b/internal/infrastructure/chunker/profiler_test.go @@ -0,0 +1,160 @@ +package chunker + +import ( + "strings" + "testing" +) + +func TestProfileDocument_Empty(t *testing.T) { + p := ProfileDocument("") + if p.TotalChars != 0 || p.TotalLines != 0 { + t.Errorf("empty doc should have zero stats, got %+v", p) + } +} + +func TestProfileDocument_MarkdownHeadings(t *testing.T) { + doc := `# Title +Some intro text here. + +## Section 1 +Body of section 1. + +## Section 2 +Body of section 2. + +### Subsection 2.1 +Detail. + +## Section 3 +More body.` + p := ProfileDocument(doc) + if p.MdHeadingCounts[1] != 1 { + t.Errorf("expected 1 H1, got %d", p.MdHeadingCounts[1]) + } + if p.MdHeadingCounts[2] != 3 { + t.Errorf("expected 3 H2, got %d", p.MdHeadingCounts[2]) + } + if p.MdHeadingCounts[3] != 1 { + t.Errorf("expected 1 H3, got %d", p.MdHeadingCounts[3]) + } + if p.MdHeadingTotal != 5 { + t.Errorf("expected 5 headings total, got %d", p.MdHeadingTotal) + } + if p.DominantHeadingLevel() != 2 { + t.Errorf("dominant level should be 2 (≥3 occurrences), got %d", p.DominantHeadingLevel()) + } +} + +func TestProfileDocument_DominantLevelFallback(t *testing.T) { + // No level reaches 3 occurrences — should fall back to most frequent. + doc := "# Single H1\n## H2 a\n## H2 b\n" + p := ProfileDocument(doc) + if p.DominantHeadingLevel() != 2 { + t.Errorf("expected fallback to level 2 (most frequent), got %d", p.DominantHeadingLevel()) + } +} + +func TestProfileDocument_NumberedSections(t *testing.T) { + doc := `1. Introduction +text + +2. Methodology +text + +3. Results +text` + p := ProfileDocument(doc) + if p.NumberedSectionCount < 3 { + t.Errorf("expected ≥3 numbered sections, got %d", p.NumberedSectionCount) + } +} + +func TestProfileDocument_GermanChapters(t *testing.T) { + doc := "Kapitel 1: Einführung\n\nText\n\nKapitel 2: Hauptteil\n\nText" + p := ProfileDocument(doc) + if p.GermanChapterCount != 2 { + t.Errorf("expected 2 German chapters, got %d", p.GermanChapterCount) + } +} + +func TestProfileDocument_ChineseChapters(t *testing.T) { + doc := "第一章 引言\n\n内容\n\n第二章 方法\n\n内容" + p := ProfileDocument(doc) + if p.ChineseChapterCount != 2 { + t.Errorf("expected 2 Chinese chapters, got %d", p.ChineseChapterCount) + } +} + +func TestProfileDocument_FormFeed(t *testing.T) { + doc := "page 1 content\f\npage 2 content\f\npage 3 content" + p := ProfileDocument(doc) + if p.FormFeedCount != 2 { + t.Errorf("expected 2 form feeds, got %d", p.FormFeedCount) + } +} + +func TestProfileDocument_DetectsCodeBlock(t *testing.T) { + doc := "Some prose.\n\n```go\nfunc main() {}\n```\n\nMore prose." + p := ProfileDocument(doc) + if !p.HasCode { + t.Error("expected HasCode=true for fenced block") + } +} + +func TestProfileDocument_DetectsTable(t *testing.T) { + doc := "Intro.\n\n| col a | col b |\n| --- | --- |\n| 1 | 2 |\n" + p := ProfileDocument(doc) + if !p.HasTables { + t.Error("expected HasTables=true") + } +} + +func TestProfileDocument_LineStatistics(t *testing.T) { + doc := "short\nthis is a longer line of text\nanother line here" + p := ProfileDocument(doc) + if p.TotalLines != 3 { + t.Errorf("expected 3 lines, got %d", p.TotalLines) + } + if p.AvgLineLen <= 0 { + t.Error("expected positive avg line len") + } +} + +func TestSelectStrategy_HeadingDoc(t *testing.T) { + doc := "# A\nbody\n## B\nbody\n## C\nbody\n## D\nbody" + p := ProfileDocument(doc) + chain := SelectStrategy(p) + if chain[0] != TierHeading { + t.Errorf("expected heading tier first, got %v", chain) + } +} + +func TestSelectStrategy_HeuristicDoc(t *testing.T) { + doc := strings.Repeat("Kapitel 1: Foo\nbody body body\n\n", 1) + + strings.Repeat("Kapitel 2: Bar\nbody body body\n\n", 1) + p := ProfileDocument(doc) + chain := SelectStrategy(p) + // no markdown headings → heuristic must come first (heading tier skipped) + if chain[0] != TierHeuristic { + t.Errorf("expected heuristic tier first, got %v", chain) + } +} + +func TestSelectStrategy_PlainDoc(t *testing.T) { + doc := "just a paragraph of plain text without any structure indicators at all here" + p := ProfileDocument(doc) + chain := SelectStrategy(p) + if chain[0] != TierRecursive { + t.Errorf("expected recursive tier first for unstructured doc, got %v", chain) + } +} + +func TestSelectStrategy_AlwaysFallsBackToLegacy(t *testing.T) { + for _, doc := range []string{"", "simple", "# H1\nbody"} { + p := ProfileDocument(doc) + chain := SelectStrategy(p) + if chain[len(chain)-1] != TierLegacy { + t.Errorf("chain must end with legacy, got %v for doc=%q", chain, doc) + } + } +} diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index d110e22d..2718d857 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -27,18 +27,35 @@ type ImageRef struct { End int } -// SplitterConfig configures the text splitter. +// SplitterConfig configures the text splitter. Strategy and TokenLimit are +// honored by the strategy entry point in strategy.go; the legacy SplitText +// path uses only ChunkSize/Overlap/Separators. type SplitterConfig struct { ChunkSize int ChunkOverlap int Separators []string + + // Strategy selects an adaptive tier. Empty = legacy (backwards-compatible). + // See strategy.go for valid values. + Strategy string + // TokenLimit caps chunk size in approximate tokens. 0 = use ChunkSize chars. + TokenLimit int + // Languages hints multilingual heuristic patterns. Empty = auto-detect. + Languages []string } +// Default sizes used by all entry points (DefaultConfig, ensureDefaults, +// and buildSplitterConfig in the knowledge service). +const ( + DefaultChunkSize = 512 + DefaultChunkOverlap = 80 // ≈ 15% of DefaultChunkSize +) + // DefaultConfig returns sensible defaults. func DefaultConfig() SplitterConfig { return SplitterConfig{ - ChunkSize: 512, - ChunkOverlap: 64, + ChunkSize: DefaultChunkSize, + ChunkOverlap: DefaultChunkOverlap, Separators: []string{"\n\n", "\n", "。"}, } } diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go new file mode 100644 index 00000000..aeabd9d7 --- /dev/null +++ b/internal/infrastructure/chunker/strategy.go @@ -0,0 +1,144 @@ +// Package chunker - strategy.go is the public entry point for adaptive +// chunking. Callers invoke Split / SplitParentChild instead of the legacy +// SplitText / SplitTextParentChild functions; the strategy resolver picks +// a tier based on document profile and the SplitterConfig.Strategy hint. +// +// The legacy entry points still exist in splitter.go for backwards +// compatibility — strategy.go simply layers a tier-selector on top. +package chunker + +import "github.com/Tencent/WeKnora/internal/logger" + +// Strategy values for SplitterConfig.Strategy. +const ( + StrategyAuto = "auto" + StrategyHeading = "heading" + StrategyHeuristic = "heuristic" + StrategyRecursive = "recursive" + StrategyLegacy = "legacy" +) + +// Split chunks text using the strategy configured in cfg. When cfg.Strategy +// is empty or "auto" the document profiler picks the tier. The function +// always returns a non-nil result: on tier failure the chain falls through +// to the legacy splitter, which is the original Tier 3 implementation. +func Split(text string, cfg SplitterConfig) []Chunk { + if text == "" { + return nil + } + cfg = ensureDefaults(cfg) + + chain := resolveChain(text, cfg) + totalChars := len([]rune(text)) + + for _, tier := range chain { + out := runTier(tier, text, cfg) + if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK { + return out + } else { + logger.Debugf(nil, "chunker: tier %s rejected: %s", tier, v.Reason) + } + } + // Last-ditch fallback: always return *something*. + return SplitText(text, cfg) +} + +// SplitParentChild is the strategy-aware analog of SplitTextParentChild. +// It runs the tier selector for parent splitting, then re-splits each +// parent into children with the small-chunk config. +func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult { + if text == "" { + return ParentChildResult{} + } + parentCfg = ensureDefaults(parentCfg) + childCfg = ensureDefaults(childCfg) + + parents := Split(text, parentCfg) + if len(parents) == 0 { + return ParentChildResult{} + } + + var newParents []Chunk + var children []ChildChunk + childSeq := 0 + for _, parent := range parents { + subs := Split(parent.Content, childCfg) + + parentIndex := -1 + if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) { + parentIndex = len(newParents) + newParents = append(newParents, parent) + } + for _, sub := range subs { + sub.Seq = childSeq + sub.Start += parent.Start + sub.End += parent.Start + children = append(children, ChildChunk{Chunk: sub, ParentIndex: parentIndex}) + childSeq++ + } + } + return ParentChildResult{Parents: newParents, Children: children} +} + +// resolveChain returns the strategy chain to attempt. An explicit non-auto +// strategy bypasses the profiler entirely and pins to the requested tier. +func resolveChain(text string, cfg SplitterConfig) []StrategyTier { + switch cfg.Strategy { + case StrategyHeading: + return []StrategyTier{TierHeading, TierLegacy} + case StrategyHeuristic: + return []StrategyTier{TierHeuristic, TierLegacy} + case StrategyRecursive: + return []StrategyTier{TierRecursive, TierLegacy} + case StrategyLegacy, "": + // Empty == legacy preserves backwards compatibility with stored + // ChunkingConfig rows that pre-date the Strategy field. + return []StrategyTier{TierLegacy} + case StrategyAuto: + fallthrough + default: + profile := ProfileDocument(text) + return SelectStrategy(profile) + } +} + +// runTier dispatches the splitter implementation for the given tier. +// Heading and heuristic tiers are stubbed in this scaffold and currently +// fall through to the legacy splitter — they are filled in in later phases. +func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk { + switch tier { + case TierHeading: + return splitByHeadings(text, cfg) + case TierHeuristic: + return splitByHeuristics(text, cfg) + case TierRecursive, TierLegacy: + return SplitText(text, cfg) + } + return SplitText(text, cfg) +} + +// ensureDefaults fills in zero-value config fields with sane defaults. +// Mirrors buildSplitterConfig in internal/application/service/knowledge.go +// so direct callers of this package get the same numbers. +func ensureDefaults(cfg SplitterConfig) SplitterConfig { + if cfg.ChunkSize <= 0 { + cfg.ChunkSize = DefaultChunkSize + } + if cfg.ChunkOverlap <= 0 { + cfg.ChunkOverlap = DefaultChunkOverlap + } + if len(cfg.Separators) == 0 { + cfg.Separators = []string{"\n\n", "\n", "。"} + } + return cfg +} + +// splitByHeadings is overridden by heading_splitter.go. +var splitByHeadings = func(text string, cfg SplitterConfig) []Chunk { + return SplitText(text, cfg) +} + +// splitByHeuristics is overridden by heuristic_splitter.go. +var splitByHeuristics = func(text string, cfg SplitterConfig) []Chunk { + return SplitText(text, cfg) +} diff --git a/internal/infrastructure/chunker/strategy_test.go b/internal/infrastructure/chunker/strategy_test.go new file mode 100644 index 00000000..b3a28206 --- /dev/null +++ b/internal/infrastructure/chunker/strategy_test.go @@ -0,0 +1,122 @@ +package chunker + +import ( + "strings" + "testing" +) + +func TestSplit_EmptyText(t *testing.T) { + if got := Split("", DefaultConfig()); got != nil { + t.Errorf("empty text should return nil, got %v", got) + } +} + +func TestSplit_LegacyStrategy_MatchesSplitText(t *testing.T) { + text := strings.Repeat("Hello world.\n\n", 30) + cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 20, Separators: []string{"\n\n"}, Strategy: StrategyLegacy} + a := Split(text, cfg) + b := SplitText(text, cfg) + if len(a) != len(b) { + t.Errorf("legacy strategy should match SplitText: got %d vs %d chunks", len(a), len(b)) + } + for i := range a { + if a[i].Content != b[i].Content { + t.Errorf("chunk %d differs", i) + } + } +} + +func TestSplit_EmptyStrategyEqualsLegacy(t *testing.T) { + text := strings.Repeat("Sentence one. Sentence two.\n", 20) + cfg := SplitterConfig{ChunkSize: 80, ChunkOverlap: 10} + a := Split(text, cfg) + cfg.Strategy = StrategyLegacy + b := Split(text, cfg) + if len(a) != len(b) { + t.Errorf("empty Strategy should equal legacy: %d vs %d", len(a), len(b)) + } +} + +func TestSplit_AutoStrategy_PicksHeadingForMarkdownDoc(t *testing.T) { + doc := strings.Repeat("# A\nbody\n## B\nbody\n## C\nbody\n## D\nbody\n", 1) + cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Strategy: StrategyAuto} + // Until heading_splitter is wired, this falls through to SplitText — + // just assert we get a valid result. + chunks := Split(doc, cfg) + if len(chunks) == 0 { + t.Error("auto strategy should produce chunks") + } +} + +func TestSplitParentChild_LegacyStrategy(t *testing.T) { + text := strings.Repeat("This is a sentence. Another one.\n\n", 50) + parentCfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 40, Strategy: StrategyLegacy} + childCfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 20, Strategy: StrategyLegacy} + res := SplitParentChild(text, parentCfg, childCfg) + if len(res.Children) == 0 { + t.Fatal("expected children chunks") + } + for i, c := range res.Children { + if c.ParentIndex >= 0 && c.ParentIndex >= len(res.Parents) { + t.Errorf("child[%d] has invalid ParentIndex %d (parents=%d)", i, c.ParentIndex, len(res.Parents)) + } + } +} + +func TestEnsureDefaults(t *testing.T) { + cfg := ensureDefaults(SplitterConfig{}) + if cfg.ChunkSize != DefaultChunkSize { + t.Errorf("expected default ChunkSize %d, got %d", DefaultChunkSize, cfg.ChunkSize) + } + if cfg.ChunkOverlap != DefaultChunkOverlap { + t.Errorf("expected default ChunkOverlap %d, got %d", DefaultChunkOverlap, cfg.ChunkOverlap) + } + if len(cfg.Separators) == 0 { + t.Error("expected default separators") + } +} + +func TestValidateChunks_Empty(t *testing.T) { + if v := ValidateChunks(nil, 1000, 500); v.OK { + t.Error("nil chunks should be invalid") + } +} + +func TestValidateChunks_SingleChunkLargeDoc(t *testing.T) { + c := []Chunk{{Content: strings.Repeat("a", 5000)}} + if v := ValidateChunks(c, 5000, 500); v.OK { + t.Error("single 10x-too-large chunk should be invalid") + } +} + +func TestValidateChunks_AcceptsReasonableOutput(t *testing.T) { + chunks := []Chunk{ + {Content: strings.Repeat("a", 480)}, + {Content: strings.Repeat("b", 510)}, + {Content: strings.Repeat("c", 460)}, + } + if v := ValidateChunks(chunks, 1500, 512); !v.OK { + t.Errorf("reasonable chunks should validate, got: %s", v.Reason) + } +} + +func TestValidateChunks_RejectsOversized(t *testing.T) { + chunks := []Chunk{ + {Content: strings.Repeat("a", 100)}, + {Content: strings.Repeat("b", 5000)}, // > 2x chunkSize + } + if v := ValidateChunks(chunks, 5100, 1000); v.OK { + t.Error("chunk >2x size should be invalid") + } +} + +func TestValidateChunks_TolerantTinyTail(t *testing.T) { + chunks := []Chunk{ + {Content: strings.Repeat("a", 480)}, + {Content: strings.Repeat("b", 510)}, + {Content: "tail"}, + } + if v := ValidateChunks(chunks, 994, 512); !v.OK { + t.Errorf("tiny last chunk should be tolerated, got: %s", v.Reason) + } +} diff --git a/internal/infrastructure/chunker/tokens.go b/internal/infrastructure/chunker/tokens.go new file mode 100644 index 00000000..568f2528 --- /dev/null +++ b/internal/infrastructure/chunker/tokens.go @@ -0,0 +1,145 @@ +// Package chunker - tokens.go provides language-aware token count approximation. +// +// We avoid pulling in a tokenizer dependency (e.g. tiktoken) and instead use +// per-language chars-per-token ratios derived from common embedding model +// vocabularies. The numbers are conservative — they tend to slightly +// over-estimate token counts so that chunks stay safely under model limits. +package chunker + +import "unicode" + +// Language identifiers used by the token estimator and the heuristic splitter. +const ( + LangEnglish = "en" + LangGerman = "de" + LangChinese = "zh" + LangMixed = "mixed" +) + +// charsPerToken holds approximate chars/token ratios per language. +// Numbers err on the conservative side so estimates over-shoot a little. +var charsPerToken = map[string]float64{ + LangEnglish: 4.0, + LangGerman: 4.5, + LangChinese: 1.7, + LangMixed: 3.0, +} + +// ApproxTokenCount returns a conservative token estimate for s in the given +// language. An empty or unknown lang falls back to "mixed". +func ApproxTokenCount(s string, lang string) int { + if s == "" { + return 0 + } + ratio, ok := charsPerToken[lang] + if !ok { + ratio = charsPerToken[LangMixed] + } + runes := []rune(s) + approx := float64(len(runes)) / ratio + if approx < 1 { + return 1 + } + return int(approx + 0.5) +} + +// DetectLanguage returns a coarse language label by counting CJK runes vs. +// Latin runes. The result is one of LangChinese, LangGerman, LangEnglish or +// LangMixed. Detection is cheap and meant only for heuristic dispatch — it +// is NOT a replacement for proper language identification. +func DetectLanguage(s string) string { + if s == "" { + return LangMixed + } + var cjk, latin, umlaut int + for _, r := range s { + switch { + case unicode.Is(unicode.Han, r) || unicode.Is(unicode.Hangul, r) || unicode.Is(unicode.Hiragana, r) || unicode.Is(unicode.Katakana, r): + cjk++ + case isGermanUmlaut(r): + umlaut++ + latin++ + case (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z'): + latin++ + } + } + total := cjk + latin + if total == 0 { + return LangMixed + } + cjkRatio := float64(cjk) / float64(total) + latinRatio := float64(latin) / float64(total) + // Mixed: meaningful presence of both scripts (>=15% each). + if cjkRatio >= 0.15 && latinRatio >= 0.15 { + return LangMixed + } + if cjkRatio > 0.3 { + return LangChinese + } + if umlaut > 0 || hasGermanWords(s) { + return LangGerman + } + return LangEnglish +} + +func isGermanUmlaut(r rune) bool { + switch r { + case 'ä', 'ö', 'ü', 'Ä', 'Ö', 'Ü', 'ß': + return true + } + return false +} + +// hasGermanWords does a tiny stop-word check to bias towards "de" when the +// text uses common German function words. Cheap heuristic — false positives +// on borrowed terms are acceptable. +func hasGermanWords(s string) bool { + const sample = 512 + if len(s) > sample { + s = s[:sample] + } + for _, w := range []string{" der ", " die ", " das ", " und ", " ist ", " nicht ", " mit ", " auf "} { + if containsLower(s, w) { + return true + } + } + return false +} + +func containsLower(haystack, needle string) bool { + if len(haystack) < len(needle) { + return false + } + for i := 0; i+len(needle) <= len(haystack); i++ { + match := true + for j := 0; j < len(needle); j++ { + h := haystack[i+j] + if h >= 'A' && h <= 'Z' { + h += 'a' - 'A' + } + if h != needle[j] { + match = false + break + } + } + if match { + return true + } + } + return false +} + +// CharsForTokenLimit converts a token limit into an approximate character +// budget for a given language. Used to size chunks so they fit within an +// embedding model's max-token window with a small safety margin. +func CharsForTokenLimit(tokens int, lang string) int { + if tokens <= 0 { + return 0 + } + ratio, ok := charsPerToken[lang] + if !ok { + ratio = charsPerToken[LangMixed] + } + // 0.9 safety factor so we under-shoot the model limit instead of overshooting. + return int(float64(tokens) * ratio * 0.9) +} diff --git a/internal/infrastructure/chunker/tokens_test.go b/internal/infrastructure/chunker/tokens_test.go new file mode 100644 index 00000000..6a921f00 --- /dev/null +++ b/internal/infrastructure/chunker/tokens_test.go @@ -0,0 +1,78 @@ +package chunker + +import "testing" + +func TestApproxTokenCount_English(t *testing.T) { + got := ApproxTokenCount("The quick brown fox jumps over the lazy dog.", LangEnglish) + // 44 chars / 4 ≈ 11 tokens + if got < 9 || got > 13 { + t.Errorf("English token estimate out of range: got %d, want 9..13", got) + } +} + +func TestApproxTokenCount_Chinese(t *testing.T) { + got := ApproxTokenCount("这是一段中文测试内容用于检验分词估算", LangChinese) + // 18 runes / 1.7 ≈ 10 + if got < 9 || got > 12 { + t.Errorf("Chinese token estimate out of range: got %d, want 9..12", got) + } +} + +func TestApproxTokenCount_Empty(t *testing.T) { + if got := ApproxTokenCount("", LangEnglish); got != 0 { + t.Errorf("empty string should return 0 tokens, got %d", got) + } +} + +func TestApproxTokenCount_UnknownLang(t *testing.T) { + got := ApproxTokenCount("Hello world hello world", "xx") + if got <= 0 { + t.Errorf("unknown lang should fall back to mixed, got %d", got) + } +} + +func TestDetectLanguage_English(t *testing.T) { + if got := DetectLanguage("The quick brown fox jumps over the lazy dog."); got != LangEnglish { + t.Errorf("expected English, got %s", got) + } +} + +func TestDetectLanguage_German(t *testing.T) { + if got := DetectLanguage("Der schnelle braune Fuchs springt über den faulen Hund."); got != LangGerman { + t.Errorf("expected German, got %s", got) + } +} + +func TestDetectLanguage_GermanByStopwords(t *testing.T) { + // No umlauts but plenty of German function words. + if got := DetectLanguage("Das ist ein Test und nicht mit Umlauten."); got != LangGerman { + t.Errorf("expected German via stopwords, got %s", got) + } +} + +func TestDetectLanguage_Chinese(t *testing.T) { + if got := DetectLanguage("这是一段中文测试内容"); got != LangChinese { + t.Errorf("expected Chinese, got %s", got) + } +} + +func TestDetectLanguage_Mixed(t *testing.T) { + got := DetectLanguage("This 这是 mixed 测试 content with 多语言 inside") + if got != LangMixed { + t.Errorf("expected Mixed, got %s", got) + } +} + +func TestCharsForTokenLimit_AppliesSafetyMargin(t *testing.T) { + got := CharsForTokenLimit(1000, LangEnglish) + // 1000 * 4 * 0.9 = 3600 + if got < 3500 || got > 3700 { + t.Errorf("char budget for 1000 EN tokens out of range: got %d", got) + } +} + +func TestCharsForTokenLimit_ZeroTokens(t *testing.T) { + if got := CharsForTokenLimit(0, LangEnglish); got != 0 { + t.Errorf("zero tokens should give zero chars, got %d", got) + } +} diff --git a/internal/infrastructure/chunker/validator.go b/internal/infrastructure/chunker/validator.go new file mode 100644 index 00000000..c9abf9e2 --- /dev/null +++ b/internal/infrastructure/chunker/validator.go @@ -0,0 +1,75 @@ +// Package chunker - validator.go inspects a tier's output and decides whether +// it is good enough to ship or whether the strategy chain should fall through +// to the next tier. The validator is intentionally permissive: a single +// "obviously broken" output is rejected, but plausible-looking variation is +// accepted so we don't oscillate between tiers. +package chunker + +import "math" + +// ValidationResult captures the verdict and reason for a chunk-set. +type ValidationResult struct { + OK bool + Reason string +} + +// ValidateChunks checks whether the given chunks form a usable result for a +// document of totalChars characters with a target chunkSize. Returns OK=true +// when no broken-output indicator triggers. +func ValidateChunks(chunks []Chunk, totalChars, chunkSize int) ValidationResult { + if len(chunks) == 0 { + return ValidationResult{Reason: "no chunks produced"} + } + + // A single chunk for a document much larger than chunkSize means the + // strategy did not actually split — fail so the next tier runs. + if len(chunks) == 1 && totalChars > 2*chunkSize { + return ValidationResult{Reason: "single chunk for large document"} + } + + // Compute size statistics. + var sum, sumSq float64 + maxLen, minLen := 0, math.MaxInt32 + for _, c := range chunks { + l := len([]rune(c.Content)) + sum += float64(l) + sumSq += float64(l * l) + if l > maxLen { + maxLen = l + } + if l < minLen { + minLen = l + } + } + avg := sum / float64(len(chunks)) + + // All but the last chunk should carry meaningful content. We allow the + // last chunk to be tiny because tail residue is normal. + tinyCount := 0 + for i, c := range chunks { + if i == len(chunks)-1 { + continue + } + if len([]rune(c.Content)) < 50 { + tinyCount++ + } + } + if tinyCount > len(chunks)/4 && tinyCount > 2 { + return ValidationResult{Reason: "too many tiny chunks"} + } + + // Reject when no chunk reached at least 25% of the target — the splitter + // is fragmenting too aggressively to be useful. + if maxLen < chunkSize/4 && totalChars > chunkSize { + return ValidationResult{Reason: "all chunks far below target size"} + } + + // Sanity check on absolute upper bound. Anything past 2x chunkSize is a + // red flag — the splitter ignored its size budget. + if maxLen > 2*chunkSize && chunkSize > 0 { + return ValidationResult{Reason: "chunk exceeds 2x target size"} + } + + _ = avg + return ValidationResult{OK: true} +}