feat(chunker): heuristic Tier 2 splitter

splitByHeuristicsImpl handles documents that lack proper Markdown
headings but contain other structural cues — form-feeds, numbered
sections, multilingual chapter markers (DE/EN/ZH), all-caps section
titles, visual separators (---/===), repeated page footers, and runs
of blank lines. Boundaries are sorted by position; greedy bin-packing
fills chunks up to cfg.ChunkSize and oversize blocks between two
boundaries fall back to the legacy splitter for inner segmentation.

Boundary detection skips fenced code blocks and de-duplicates
overlapping offsets keeping the highest-priority candidate.

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
Claude
2026-05-02 09:28:18 +00:00
committed by lyingbug
parent cf8789f84c
commit d31acf1063
2 changed files with 331 additions and 0 deletions
@@ -0,0 +1,242 @@
// Package chunker - heuristic_splitter.go implements Tier 2: boundary-driven
// chunking for documents that lack proper Markdown headings but contain
// recognizable structural cues (page breaks, numbered sections, multilingual
// chapter markers, visual separators, all-caps section titles, page footers).
//
// The algorithm finds all candidate boundary positions, then performs greedy
// bin-packing — accumulating blocks between boundaries into chunks until the
// next block would exceed cfg.ChunkSize. Blocks larger than ChunkSize are
// recursively delegated to the legacy splitter for inner segmentation.
package chunker
import (
"sort"
"strings"
"unicode/utf8"
)
func init() {
splitByHeuristics = splitByHeuristicsImpl
}
// boundary marks a candidate split point in the document.
type boundary struct {
runeStart int // rune offset where the next chunk should start
priority int
}
// splitByHeuristicsImpl is the Tier-2 implementation. Falls through to the
// legacy splitter when no heuristic boundaries are found.
func splitByHeuristicsImpl(text string, cfg SplitterConfig) []Chunk {
if text == "" {
return nil
}
runes := []rune(text)
totalRunes := len(runes)
if totalRunes <= cfg.ChunkSize {
return SplitText(text, cfg)
}
bounds := findHeuristicBoundaries(text, cfg.Languages)
if len(bounds) == 0 {
return SplitText(text, cfg)
}
// Append a sentinel at end-of-document so the bin-packer can flush.
bounds = append(bounds, boundary{runeStart: totalRunes})
// Always start with a boundary at offset 0 if not already there.
if bounds[0].runeStart != 0 {
bounds = append([]boundary{{runeStart: 0}}, bounds...)
}
// Greedy bin-packing.
var out []Chunk
seq := 0
chunkStart := bounds[0].runeStart
curEnd := chunkStart
minChunkSize := cfg.ChunkSize / 4
if minChunkSize < 50 {
minChunkSize = 50
}
for i := 1; i < len(bounds); i++ {
nextEnd := bounds[i].runeStart
blockLen := nextEnd - curEnd
if blockLen > cfg.ChunkSize {
// The block between the previous and this boundary is itself too
// large to fit in any chunk. Flush current accumulation, then
// recursively chunk the oversize block via the legacy splitter.
if curEnd-chunkStart > 0 {
out = appendChunk(out, runes, chunkStart, curEnd, &seq)
chunkStart = curEnd
}
out = appendOversizeBlock(out, runes, curEnd, nextEnd, cfg, &seq)
curEnd = nextEnd
chunkStart = nextEnd
continue
}
// Would adding this block exceed the budget?
accumulated := nextEnd - chunkStart
if accumulated > cfg.ChunkSize && curEnd-chunkStart >= minChunkSize {
// Flush accumulated content as a chunk, restart at curEnd.
out = appendChunk(out, runes, chunkStart, curEnd, &seq)
// Apply overlap by shifting chunkStart back a bit.
chunkStart = applyOverlap(curEnd, cfg.ChunkOverlap)
}
curEnd = nextEnd
}
// Flush remaining content.
if curEnd > chunkStart {
out = appendChunk(out, runes, chunkStart, curEnd, &seq)
}
return out
}
// findHeuristicBoundaries scans text and returns boundary positions in
// ascending order. Lower-priority duplicates at the same offset are dropped.
func findHeuristicBoundaries(text string, langs []string) []boundary {
var bounds []boundary
// Form feeds — strongest single-character boundary.
for _, idx := range allRuneIndices(text, "\f") {
bounds = append(bounds, boundary{runeStart: idx, priority: PrioFormFeed})
}
// Per-line patterns walk the text once, line by line.
lines := strings.Split(text, "\n")
chapterPatterns := ChapterPatternsForLangs(langs)
pos := 0
inFence := false
for i, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "```") {
inFence = !inFence
} else if !inFence {
runeStart := pos
added := false
for _, pat := range chapterPatterns {
if pat.MatchString(line) {
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioChapterMarker})
added = true
break
}
}
if !added && NumberedSectionPattern.MatchString(line) {
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioNumberedHead})
added = true
}
if !added && AllCapsHeadingPattern.MatchString(line) {
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioAllCapsHeading})
added = true
}
if !added && VisualSeparatorPattern.MatchString(line) {
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioVisualSep})
added = true
}
if !added && PageFooterPattern.MatchString(line) {
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioPageFooter})
}
}
pos += utf8.RuneCountInString(line)
if i < len(lines)-1 {
pos++ // \n
}
}
// Excessive blank blocks (\n{3,}). Match at the *start* of the run so we
// drop into the next paragraph cleanly.
for _, idx := range ExcessiveBlanksPattern.FindAllStringIndex(text, -1) {
runeStart := utf8.RuneCountInString(text[:idx[1]])
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioBlankBlock})
}
if len(bounds) == 0 {
return nil
}
// Sort by position; drop near-duplicate offsets keeping the highest priority.
sort.Slice(bounds, func(i, j int) bool {
if bounds[i].runeStart != bounds[j].runeStart {
return bounds[i].runeStart < bounds[j].runeStart
}
return bounds[i].priority > bounds[j].priority
})
deduped := bounds[:0]
prev := -1
for _, b := range bounds {
if b.runeStart != prev {
deduped = append(deduped, b)
prev = b.runeStart
}
}
return deduped
}
// allRuneIndices returns every rune offset where needle starts in text.
// Only used for single-rune needles like form-feed.
func allRuneIndices(text, needle string) []int {
var out []int
if needle == "" {
return out
}
pos := 0
for _, r := range text {
if string(r) == needle {
out = append(out, pos)
}
pos++
}
return out
}
// appendChunk slices runes[start:end] into a Chunk and appends it to out.
func appendChunk(out []Chunk, runes []rune, start, end int, seq *int) []Chunk {
if end <= start {
return out
}
content := strings.TrimSpace(string(runes[start:end]))
if content == "" {
return out
}
c := Chunk{Content: string(runes[start:end]), Seq: *seq, Start: start, End: end}
*seq++
return append(out, c)
}
// appendOversizeBlock recursively chunks a region that is itself larger than
// cfg.ChunkSize, using the legacy splitter so internal length budgets and
// protected patterns are still respected.
func appendOversizeBlock(out []Chunk, runes []rune, start, end int, cfg SplitterConfig, seq *int) []Chunk {
if end <= start {
return out
}
subText := string(runes[start:end])
subs := SplitText(subText, cfg)
for _, s := range subs {
out = append(out, Chunk{
Content: s.Content,
Seq: *seq,
Start: start + s.Start,
End: start + s.End,
})
*seq++
}
return out
}
// applyOverlap shifts the next chunk's start back by `overlap` runes (but
// not before zero) so consecutive chunks share content. Mirrors the
// computeOverlap behavior of the legacy splitter.
func applyOverlap(curEnd, overlap int) int {
if overlap <= 0 {
return curEnd
}
start := curEnd - overlap
if start < 0 {
start = 0
}
return start
}
@@ -0,0 +1,89 @@
package chunker
import (
"strings"
"testing"
)
func TestSplitByHeuristics_FormFeedBoundary(t *testing.T) {
doc := strings.Repeat("page one body text. ", 30) + "\f" + strings.Repeat("page two body. ", 30)
cfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 20, Separators: []string{". "}}
chunks := splitByHeuristicsImpl(doc, cfg)
if len(chunks) < 2 {
t.Fatalf("form feed should produce ≥2 chunks, got %d", len(chunks))
}
}
func TestSplitByHeuristics_NumberedSections(t *testing.T) {
body := strings.Repeat("body sentence. ", 8)
doc := "1. Introduction\n" + body + "\n\n2. Methods\n" + body + "\n\n3. Results\n" + body
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}}
chunks := splitByHeuristicsImpl(doc, cfg)
if len(chunks) < 2 {
t.Fatalf("numbered sections should split: got %d chunks", len(chunks))
}
}
func TestSplitByHeuristics_GermanChapterMarkers(t *testing.T) {
body := strings.Repeat("Beispieltext. ", 10)
doc := "Kapitel 1: Einführung\n" + body + "\n\nKapitel 2: Hauptteil\n" + body
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}}
chunks := splitByHeuristicsImpl(doc, cfg)
if len(chunks) < 2 {
t.Fatalf("German chapter markers should split: got %d", len(chunks))
}
}
func TestSplitByHeuristics_ChineseChapterMarkers(t *testing.T) {
body := strings.Repeat("内容内容内容。", 60)
doc := "第一章 引言\n" + body + "\n\n第二章 方法\n" + body
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{"。"}, Languages: []string{LangChinese}}
chunks := splitByHeuristicsImpl(doc, cfg)
if len(chunks) < 2 {
t.Fatalf("Chinese chapter markers should split: got %d", len(chunks))
}
}
func TestSplitByHeuristics_FallsThroughForUnstructuredDoc(t *testing.T) {
doc := strings.Repeat("plain prose without structure. ", 5)
cfg := SplitterConfig{ChunkSize: 1000, ChunkOverlap: 20}
chunks := splitByHeuristicsImpl(doc, cfg)
if len(chunks) != 1 {
t.Errorf("unstructured short doc should be one chunk, got %d", len(chunks))
}
}
func TestSplitByHeuristics_OversizeBlockRecursesIntoLegacy(t *testing.T) {
huge := strings.Repeat("This is a long sentence. ", 200) // ~5000 chars
doc := "1. Intro\n" + huge
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 50, Separators: []string{". "}}
chunks := splitByHeuristicsImpl(doc, cfg)
if len(chunks) < 5 {
t.Errorf("oversize block should produce many sub-chunks, got %d", len(chunks))
}
// No single chunk should massively exceed the budget.
for i, c := range chunks {
if len([]rune(c.Content)) > 2*cfg.ChunkSize {
t.Errorf("chunk %d exceeds 2x size: %d runes", i, len([]rune(c.Content)))
}
}
}
func TestSplitByHeuristics_BoundariesAreOrdered(t *testing.T) {
doc := "Kapitel 1: A\nbody\n\n---\n\n2. Section B\nbody\n\nPage 3 of 10\n\n第三章 C\nbody"
bounds := findHeuristicBoundaries(doc, nil)
if len(bounds) < 2 {
t.Fatalf("expected multiple boundaries, got %d", len(bounds))
}
for i := 1; i < len(bounds); i++ {
if bounds[i].runeStart < bounds[i-1].runeStart {
t.Errorf("bounds not sorted: %d before %d", bounds[i].runeStart, bounds[i-1].runeStart)
}
}
}
func TestSplitByHeuristics_EmptyText(t *testing.T) {
if got := splitByHeuristicsImpl("", DefaultConfig()); got != nil {
t.Errorf("empty doc should be nil, got %v", got)
}
}