feat(chunker): heuristic Tier 2 splitter
splitByHeuristicsImpl handles documents that lack proper Markdown headings but contain other structural cues — form-feeds, numbered sections, multilingual chapter markers (DE/EN/ZH), all-caps section titles, visual separators (---/===), repeated page footers, and runs of blank lines. Boundaries are sorted by position; greedy bin-packing fills chunks up to cfg.ChunkSize and oversize blocks between two boundaries fall back to the legacy splitter for inner segmentation. Boundary detection skips fenced code blocks and de-duplicates overlapping offsets keeping the highest-priority candidate. https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
@@ -0,0 +1,242 @@
|
||||
// Package chunker - heuristic_splitter.go implements Tier 2: boundary-driven
|
||||
// chunking for documents that lack proper Markdown headings but contain
|
||||
// recognizable structural cues (page breaks, numbered sections, multilingual
|
||||
// chapter markers, visual separators, all-caps section titles, page footers).
|
||||
//
|
||||
// The algorithm finds all candidate boundary positions, then performs greedy
|
||||
// bin-packing — accumulating blocks between boundaries into chunks until the
|
||||
// next block would exceed cfg.ChunkSize. Blocks larger than ChunkSize are
|
||||
// recursively delegated to the legacy splitter for inner segmentation.
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
func init() {
|
||||
splitByHeuristics = splitByHeuristicsImpl
|
||||
}
|
||||
|
||||
// boundary marks a candidate split point in the document.
|
||||
type boundary struct {
|
||||
runeStart int // rune offset where the next chunk should start
|
||||
priority int
|
||||
}
|
||||
|
||||
// splitByHeuristicsImpl is the Tier-2 implementation. Falls through to the
|
||||
// legacy splitter when no heuristic boundaries are found.
|
||||
func splitByHeuristicsImpl(text string, cfg SplitterConfig) []Chunk {
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
runes := []rune(text)
|
||||
totalRunes := len(runes)
|
||||
if totalRunes <= cfg.ChunkSize {
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
|
||||
bounds := findHeuristicBoundaries(text, cfg.Languages)
|
||||
if len(bounds) == 0 {
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
|
||||
// Append a sentinel at end-of-document so the bin-packer can flush.
|
||||
bounds = append(bounds, boundary{runeStart: totalRunes})
|
||||
// Always start with a boundary at offset 0 if not already there.
|
||||
if bounds[0].runeStart != 0 {
|
||||
bounds = append([]boundary{{runeStart: 0}}, bounds...)
|
||||
}
|
||||
|
||||
// Greedy bin-packing.
|
||||
var out []Chunk
|
||||
seq := 0
|
||||
chunkStart := bounds[0].runeStart
|
||||
curEnd := chunkStart
|
||||
minChunkSize := cfg.ChunkSize / 4
|
||||
if minChunkSize < 50 {
|
||||
minChunkSize = 50
|
||||
}
|
||||
|
||||
for i := 1; i < len(bounds); i++ {
|
||||
nextEnd := bounds[i].runeStart
|
||||
blockLen := nextEnd - curEnd
|
||||
|
||||
if blockLen > cfg.ChunkSize {
|
||||
// The block between the previous and this boundary is itself too
|
||||
// large to fit in any chunk. Flush current accumulation, then
|
||||
// recursively chunk the oversize block via the legacy splitter.
|
||||
if curEnd-chunkStart > 0 {
|
||||
out = appendChunk(out, runes, chunkStart, curEnd, &seq)
|
||||
chunkStart = curEnd
|
||||
}
|
||||
out = appendOversizeBlock(out, runes, curEnd, nextEnd, cfg, &seq)
|
||||
curEnd = nextEnd
|
||||
chunkStart = nextEnd
|
||||
continue
|
||||
}
|
||||
|
||||
// Would adding this block exceed the budget?
|
||||
accumulated := nextEnd - chunkStart
|
||||
if accumulated > cfg.ChunkSize && curEnd-chunkStart >= minChunkSize {
|
||||
// Flush accumulated content as a chunk, restart at curEnd.
|
||||
out = appendChunk(out, runes, chunkStart, curEnd, &seq)
|
||||
// Apply overlap by shifting chunkStart back a bit.
|
||||
chunkStart = applyOverlap(curEnd, cfg.ChunkOverlap)
|
||||
}
|
||||
curEnd = nextEnd
|
||||
}
|
||||
|
||||
// Flush remaining content.
|
||||
if curEnd > chunkStart {
|
||||
out = appendChunk(out, runes, chunkStart, curEnd, &seq)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// findHeuristicBoundaries scans text and returns boundary positions in
|
||||
// ascending order. Lower-priority duplicates at the same offset are dropped.
|
||||
func findHeuristicBoundaries(text string, langs []string) []boundary {
|
||||
var bounds []boundary
|
||||
|
||||
// Form feeds — strongest single-character boundary.
|
||||
for _, idx := range allRuneIndices(text, "\f") {
|
||||
bounds = append(bounds, boundary{runeStart: idx, priority: PrioFormFeed})
|
||||
}
|
||||
|
||||
// Per-line patterns walk the text once, line by line.
|
||||
lines := strings.Split(text, "\n")
|
||||
chapterPatterns := ChapterPatternsForLangs(langs)
|
||||
pos := 0
|
||||
inFence := false
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "```") {
|
||||
inFence = !inFence
|
||||
} else if !inFence {
|
||||
runeStart := pos
|
||||
added := false
|
||||
for _, pat := range chapterPatterns {
|
||||
if pat.MatchString(line) {
|
||||
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioChapterMarker})
|
||||
added = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !added && NumberedSectionPattern.MatchString(line) {
|
||||
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioNumberedHead})
|
||||
added = true
|
||||
}
|
||||
if !added && AllCapsHeadingPattern.MatchString(line) {
|
||||
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioAllCapsHeading})
|
||||
added = true
|
||||
}
|
||||
if !added && VisualSeparatorPattern.MatchString(line) {
|
||||
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioVisualSep})
|
||||
added = true
|
||||
}
|
||||
if !added && PageFooterPattern.MatchString(line) {
|
||||
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioPageFooter})
|
||||
}
|
||||
}
|
||||
pos += utf8.RuneCountInString(line)
|
||||
if i < len(lines)-1 {
|
||||
pos++ // \n
|
||||
}
|
||||
}
|
||||
|
||||
// Excessive blank blocks (\n{3,}). Match at the *start* of the run so we
|
||||
// drop into the next paragraph cleanly.
|
||||
for _, idx := range ExcessiveBlanksPattern.FindAllStringIndex(text, -1) {
|
||||
runeStart := utf8.RuneCountInString(text[:idx[1]])
|
||||
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioBlankBlock})
|
||||
}
|
||||
|
||||
if len(bounds) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sort by position; drop near-duplicate offsets keeping the highest priority.
|
||||
sort.Slice(bounds, func(i, j int) bool {
|
||||
if bounds[i].runeStart != bounds[j].runeStart {
|
||||
return bounds[i].runeStart < bounds[j].runeStart
|
||||
}
|
||||
return bounds[i].priority > bounds[j].priority
|
||||
})
|
||||
deduped := bounds[:0]
|
||||
prev := -1
|
||||
for _, b := range bounds {
|
||||
if b.runeStart != prev {
|
||||
deduped = append(deduped, b)
|
||||
prev = b.runeStart
|
||||
}
|
||||
}
|
||||
return deduped
|
||||
}
|
||||
|
||||
// allRuneIndices returns every rune offset where needle starts in text.
|
||||
// Only used for single-rune needles like form-feed.
|
||||
func allRuneIndices(text, needle string) []int {
|
||||
var out []int
|
||||
if needle == "" {
|
||||
return out
|
||||
}
|
||||
pos := 0
|
||||
for _, r := range text {
|
||||
if string(r) == needle {
|
||||
out = append(out, pos)
|
||||
}
|
||||
pos++
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// appendChunk slices runes[start:end] into a Chunk and appends it to out.
|
||||
func appendChunk(out []Chunk, runes []rune, start, end int, seq *int) []Chunk {
|
||||
if end <= start {
|
||||
return out
|
||||
}
|
||||
content := strings.TrimSpace(string(runes[start:end]))
|
||||
if content == "" {
|
||||
return out
|
||||
}
|
||||
c := Chunk{Content: string(runes[start:end]), Seq: *seq, Start: start, End: end}
|
||||
*seq++
|
||||
return append(out, c)
|
||||
}
|
||||
|
||||
// appendOversizeBlock recursively chunks a region that is itself larger than
|
||||
// cfg.ChunkSize, using the legacy splitter so internal length budgets and
|
||||
// protected patterns are still respected.
|
||||
func appendOversizeBlock(out []Chunk, runes []rune, start, end int, cfg SplitterConfig, seq *int) []Chunk {
|
||||
if end <= start {
|
||||
return out
|
||||
}
|
||||
subText := string(runes[start:end])
|
||||
subs := SplitText(subText, cfg)
|
||||
for _, s := range subs {
|
||||
out = append(out, Chunk{
|
||||
Content: s.Content,
|
||||
Seq: *seq,
|
||||
Start: start + s.Start,
|
||||
End: start + s.End,
|
||||
})
|
||||
*seq++
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// applyOverlap shifts the next chunk's start back by `overlap` runes (but
|
||||
// not before zero) so consecutive chunks share content. Mirrors the
|
||||
// computeOverlap behavior of the legacy splitter.
|
||||
func applyOverlap(curEnd, overlap int) int {
|
||||
if overlap <= 0 {
|
||||
return curEnd
|
||||
}
|
||||
start := curEnd - overlap
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
return start
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplitByHeuristics_FormFeedBoundary(t *testing.T) {
|
||||
doc := strings.Repeat("page one body text. ", 30) + "\f" + strings.Repeat("page two body. ", 30)
|
||||
cfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 20, Separators: []string{". "}}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("form feed should produce ≥2 chunks, got %d", len(chunks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByHeuristics_NumberedSections(t *testing.T) {
|
||||
body := strings.Repeat("body sentence. ", 8)
|
||||
doc := "1. Introduction\n" + body + "\n\n2. Methods\n" + body + "\n\n3. Results\n" + body
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("numbered sections should split: got %d chunks", len(chunks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByHeuristics_GermanChapterMarkers(t *testing.T) {
|
||||
body := strings.Repeat("Beispieltext. ", 10)
|
||||
doc := "Kapitel 1: Einführung\n" + body + "\n\nKapitel 2: Hauptteil\n" + body
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("German chapter markers should split: got %d", len(chunks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByHeuristics_ChineseChapterMarkers(t *testing.T) {
|
||||
body := strings.Repeat("内容内容内容。", 60)
|
||||
doc := "第一章 引言\n" + body + "\n\n第二章 方法\n" + body
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{"。"}, Languages: []string{LangChinese}}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("Chinese chapter markers should split: got %d", len(chunks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByHeuristics_FallsThroughForUnstructuredDoc(t *testing.T) {
|
||||
doc := strings.Repeat("plain prose without structure. ", 5)
|
||||
cfg := SplitterConfig{ChunkSize: 1000, ChunkOverlap: 20}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
if len(chunks) != 1 {
|
||||
t.Errorf("unstructured short doc should be one chunk, got %d", len(chunks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByHeuristics_OversizeBlockRecursesIntoLegacy(t *testing.T) {
|
||||
huge := strings.Repeat("This is a long sentence. ", 200) // ~5000 chars
|
||||
doc := "1. Intro\n" + huge
|
||||
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 50, Separators: []string{". "}}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
if len(chunks) < 5 {
|
||||
t.Errorf("oversize block should produce many sub-chunks, got %d", len(chunks))
|
||||
}
|
||||
// No single chunk should massively exceed the budget.
|
||||
for i, c := range chunks {
|
||||
if len([]rune(c.Content)) > 2*cfg.ChunkSize {
|
||||
t.Errorf("chunk %d exceeds 2x size: %d runes", i, len([]rune(c.Content)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByHeuristics_BoundariesAreOrdered(t *testing.T) {
|
||||
doc := "Kapitel 1: A\nbody\n\n---\n\n2. Section B\nbody\n\nPage 3 of 10\n\n第三章 C\nbody"
|
||||
bounds := findHeuristicBoundaries(doc, nil)
|
||||
if len(bounds) < 2 {
|
||||
t.Fatalf("expected multiple boundaries, got %d", len(bounds))
|
||||
}
|
||||
for i := 1; i < len(bounds); i++ {
|
||||
if bounds[i].runeStart < bounds[i-1].runeStart {
|
||||
t.Errorf("bounds not sorted: %d before %d", bounds[i].runeStart, bounds[i-1].runeStart)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByHeuristics_EmptyText(t *testing.T) {
|
||||
if got := splitByHeuristicsImpl("", DefaultConfig()); got != nil {
|
||||
t.Errorf("empty doc should be nil, got %v", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user