diff --git a/internal/infrastructure/chunker/heuristic_splitter.go b/internal/infrastructure/chunker/heuristic_splitter.go new file mode 100644 index 00000000..502b448b --- /dev/null +++ b/internal/infrastructure/chunker/heuristic_splitter.go @@ -0,0 +1,242 @@ +// Package chunker - heuristic_splitter.go implements Tier 2: boundary-driven +// chunking for documents that lack proper Markdown headings but contain +// recognizable structural cues (page breaks, numbered sections, multilingual +// chapter markers, visual separators, all-caps section titles, page footers). +// +// The algorithm finds all candidate boundary positions, then performs greedy +// bin-packing — accumulating blocks between boundaries into chunks until the +// next block would exceed cfg.ChunkSize. Blocks larger than ChunkSize are +// recursively delegated to the legacy splitter for inner segmentation. +package chunker + +import ( + "sort" + "strings" + "unicode/utf8" +) + +func init() { + splitByHeuristics = splitByHeuristicsImpl +} + +// boundary marks a candidate split point in the document. +type boundary struct { + runeStart int // rune offset where the next chunk should start + priority int +} + +// splitByHeuristicsImpl is the Tier-2 implementation. Falls through to the +// legacy splitter when no heuristic boundaries are found. +func splitByHeuristicsImpl(text string, cfg SplitterConfig) []Chunk { + if text == "" { + return nil + } + runes := []rune(text) + totalRunes := len(runes) + if totalRunes <= cfg.ChunkSize { + return SplitText(text, cfg) + } + + bounds := findHeuristicBoundaries(text, cfg.Languages) + if len(bounds) == 0 { + return SplitText(text, cfg) + } + + // Append a sentinel at end-of-document so the bin-packer can flush. + bounds = append(bounds, boundary{runeStart: totalRunes}) + // Always start with a boundary at offset 0 if not already there. + if bounds[0].runeStart != 0 { + bounds = append([]boundary{{runeStart: 0}}, bounds...) + } + + // Greedy bin-packing. + var out []Chunk + seq := 0 + chunkStart := bounds[0].runeStart + curEnd := chunkStart + minChunkSize := cfg.ChunkSize / 4 + if minChunkSize < 50 { + minChunkSize = 50 + } + + for i := 1; i < len(bounds); i++ { + nextEnd := bounds[i].runeStart + blockLen := nextEnd - curEnd + + if blockLen > cfg.ChunkSize { + // The block between the previous and this boundary is itself too + // large to fit in any chunk. Flush current accumulation, then + // recursively chunk the oversize block via the legacy splitter. + if curEnd-chunkStart > 0 { + out = appendChunk(out, runes, chunkStart, curEnd, &seq) + chunkStart = curEnd + } + out = appendOversizeBlock(out, runes, curEnd, nextEnd, cfg, &seq) + curEnd = nextEnd + chunkStart = nextEnd + continue + } + + // Would adding this block exceed the budget? + accumulated := nextEnd - chunkStart + if accumulated > cfg.ChunkSize && curEnd-chunkStart >= minChunkSize { + // Flush accumulated content as a chunk, restart at curEnd. + out = appendChunk(out, runes, chunkStart, curEnd, &seq) + // Apply overlap by shifting chunkStart back a bit. + chunkStart = applyOverlap(curEnd, cfg.ChunkOverlap) + } + curEnd = nextEnd + } + + // Flush remaining content. + if curEnd > chunkStart { + out = appendChunk(out, runes, chunkStart, curEnd, &seq) + } + return out +} + +// findHeuristicBoundaries scans text and returns boundary positions in +// ascending order. Lower-priority duplicates at the same offset are dropped. +func findHeuristicBoundaries(text string, langs []string) []boundary { + var bounds []boundary + + // Form feeds — strongest single-character boundary. + for _, idx := range allRuneIndices(text, "\f") { + bounds = append(bounds, boundary{runeStart: idx, priority: PrioFormFeed}) + } + + // Per-line patterns walk the text once, line by line. + lines := strings.Split(text, "\n") + chapterPatterns := ChapterPatternsForLangs(langs) + pos := 0 + inFence := false + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "```") { + inFence = !inFence + } else if !inFence { + runeStart := pos + added := false + for _, pat := range chapterPatterns { + if pat.MatchString(line) { + bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioChapterMarker}) + added = true + break + } + } + if !added && NumberedSectionPattern.MatchString(line) { + bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioNumberedHead}) + added = true + } + if !added && AllCapsHeadingPattern.MatchString(line) { + bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioAllCapsHeading}) + added = true + } + if !added && VisualSeparatorPattern.MatchString(line) { + bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioVisualSep}) + added = true + } + if !added && PageFooterPattern.MatchString(line) { + bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioPageFooter}) + } + } + pos += utf8.RuneCountInString(line) + if i < len(lines)-1 { + pos++ // \n + } + } + + // Excessive blank blocks (\n{3,}). Match at the *start* of the run so we + // drop into the next paragraph cleanly. + for _, idx := range ExcessiveBlanksPattern.FindAllStringIndex(text, -1) { + runeStart := utf8.RuneCountInString(text[:idx[1]]) + bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioBlankBlock}) + } + + if len(bounds) == 0 { + return nil + } + + // Sort by position; drop near-duplicate offsets keeping the highest priority. + sort.Slice(bounds, func(i, j int) bool { + if bounds[i].runeStart != bounds[j].runeStart { + return bounds[i].runeStart < bounds[j].runeStart + } + return bounds[i].priority > bounds[j].priority + }) + deduped := bounds[:0] + prev := -1 + for _, b := range bounds { + if b.runeStart != prev { + deduped = append(deduped, b) + prev = b.runeStart + } + } + return deduped +} + +// allRuneIndices returns every rune offset where needle starts in text. +// Only used for single-rune needles like form-feed. +func allRuneIndices(text, needle string) []int { + var out []int + if needle == "" { + return out + } + pos := 0 + for _, r := range text { + if string(r) == needle { + out = append(out, pos) + } + pos++ + } + return out +} + +// appendChunk slices runes[start:end] into a Chunk and appends it to out. +func appendChunk(out []Chunk, runes []rune, start, end int, seq *int) []Chunk { + if end <= start { + return out + } + content := strings.TrimSpace(string(runes[start:end])) + if content == "" { + return out + } + c := Chunk{Content: string(runes[start:end]), Seq: *seq, Start: start, End: end} + *seq++ + return append(out, c) +} + +// appendOversizeBlock recursively chunks a region that is itself larger than +// cfg.ChunkSize, using the legacy splitter so internal length budgets and +// protected patterns are still respected. +func appendOversizeBlock(out []Chunk, runes []rune, start, end int, cfg SplitterConfig, seq *int) []Chunk { + if end <= start { + return out + } + subText := string(runes[start:end]) + subs := SplitText(subText, cfg) + for _, s := range subs { + out = append(out, Chunk{ + Content: s.Content, + Seq: *seq, + Start: start + s.Start, + End: start + s.End, + }) + *seq++ + } + return out +} + +// applyOverlap shifts the next chunk's start back by `overlap` runes (but +// not before zero) so consecutive chunks share content. Mirrors the +// computeOverlap behavior of the legacy splitter. +func applyOverlap(curEnd, overlap int) int { + if overlap <= 0 { + return curEnd + } + start := curEnd - overlap + if start < 0 { + start = 0 + } + return start +} diff --git a/internal/infrastructure/chunker/heuristic_splitter_test.go b/internal/infrastructure/chunker/heuristic_splitter_test.go new file mode 100644 index 00000000..6bb2847c --- /dev/null +++ b/internal/infrastructure/chunker/heuristic_splitter_test.go @@ -0,0 +1,89 @@ +package chunker + +import ( + "strings" + "testing" +) + +func TestSplitByHeuristics_FormFeedBoundary(t *testing.T) { + doc := strings.Repeat("page one body text. ", 30) + "\f" + strings.Repeat("page two body. ", 30) + cfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 20, Separators: []string{". "}} + chunks := splitByHeuristicsImpl(doc, cfg) + if len(chunks) < 2 { + t.Fatalf("form feed should produce ≥2 chunks, got %d", len(chunks)) + } +} + +func TestSplitByHeuristics_NumberedSections(t *testing.T) { + body := strings.Repeat("body sentence. ", 8) + doc := "1. Introduction\n" + body + "\n\n2. Methods\n" + body + "\n\n3. Results\n" + body + cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}} + chunks := splitByHeuristicsImpl(doc, cfg) + if len(chunks) < 2 { + t.Fatalf("numbered sections should split: got %d chunks", len(chunks)) + } +} + +func TestSplitByHeuristics_GermanChapterMarkers(t *testing.T) { + body := strings.Repeat("Beispieltext. ", 10) + doc := "Kapitel 1: Einführung\n" + body + "\n\nKapitel 2: Hauptteil\n" + body + cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}} + chunks := splitByHeuristicsImpl(doc, cfg) + if len(chunks) < 2 { + t.Fatalf("German chapter markers should split: got %d", len(chunks)) + } +} + +func TestSplitByHeuristics_ChineseChapterMarkers(t *testing.T) { + body := strings.Repeat("内容内容内容。", 60) + doc := "第一章 引言\n" + body + "\n\n第二章 方法\n" + body + cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{"。"}, Languages: []string{LangChinese}} + chunks := splitByHeuristicsImpl(doc, cfg) + if len(chunks) < 2 { + t.Fatalf("Chinese chapter markers should split: got %d", len(chunks)) + } +} + +func TestSplitByHeuristics_FallsThroughForUnstructuredDoc(t *testing.T) { + doc := strings.Repeat("plain prose without structure. ", 5) + cfg := SplitterConfig{ChunkSize: 1000, ChunkOverlap: 20} + chunks := splitByHeuristicsImpl(doc, cfg) + if len(chunks) != 1 { + t.Errorf("unstructured short doc should be one chunk, got %d", len(chunks)) + } +} + +func TestSplitByHeuristics_OversizeBlockRecursesIntoLegacy(t *testing.T) { + huge := strings.Repeat("This is a long sentence. ", 200) // ~5000 chars + doc := "1. Intro\n" + huge + cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 50, Separators: []string{". "}} + chunks := splitByHeuristicsImpl(doc, cfg) + if len(chunks) < 5 { + t.Errorf("oversize block should produce many sub-chunks, got %d", len(chunks)) + } + // No single chunk should massively exceed the budget. + for i, c := range chunks { + if len([]rune(c.Content)) > 2*cfg.ChunkSize { + t.Errorf("chunk %d exceeds 2x size: %d runes", i, len([]rune(c.Content))) + } + } +} + +func TestSplitByHeuristics_BoundariesAreOrdered(t *testing.T) { + doc := "Kapitel 1: A\nbody\n\n---\n\n2. Section B\nbody\n\nPage 3 of 10\n\n第三章 C\nbody" + bounds := findHeuristicBoundaries(doc, nil) + if len(bounds) < 2 { + t.Fatalf("expected multiple boundaries, got %d", len(bounds)) + } + for i := 1; i < len(bounds); i++ { + if bounds[i].runeStart < bounds[i-1].runeStart { + t.Errorf("bounds not sorted: %d before %d", bounds[i].runeStart, bounds[i-1].runeStart) + } + } +} + +func TestSplitByHeuristics_EmptyText(t *testing.T) { + if got := splitByHeuristicsImpl("", DefaultConfig()); got != nil { + t.Errorf("empty doc should be nil, got %v", got) + } +}