7b9a831db3
Heuristic and recursive splitters preserve original byte positions in Chunk.Content (the End-Start == RuneCountInString invariant is required by document-reconstruction code paths), which means a chunk sliced at a boundary often carries leading/trailing newlines from the boundary itself. Feeding that whitespace into the embedding model dilutes the vector and wastes tokens for no benefit. TrimSpace the body inside EmbeddingContent for both chunker.Chunk and the two types.Chunk / types.ParsedChunk mirrors. Inner whitespace is preserved; positions/Content are unchanged. Also clean up appendChunk: the previous code computed a trimmed string for the empty-check then stored a separate untrimmed copy, which read as if it might be intentionally divergent. Replace with an explicit 'raw text, skip if pure whitespace' shape and document the invariant.
314 lines
9.7 KiB
Go
314 lines
9.7 KiB
Go
// Package chunker - heuristic_splitter.go implements Tier 2: boundary-driven
|
|
// chunking for documents that lack proper Markdown headings but contain
|
|
// recognizable structural cues (page breaks, numbered sections, multilingual
|
|
// chapter markers, visual separators, all-caps section titles, page footers).
|
|
//
|
|
// The algorithm finds all candidate boundary positions, then performs greedy
|
|
// bin-packing — accumulating blocks between boundaries into chunks until the
|
|
// next block would exceed cfg.ChunkSize. Blocks larger than ChunkSize are
|
|
// recursively delegated to the legacy splitter for inner segmentation.
|
|
package chunker
|
|
|
|
import (
|
|
"sort"
|
|
"strings"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
func init() {
|
|
splitByHeuristics = splitByHeuristicsImpl
|
|
}
|
|
|
|
// boundary marks a candidate split point in the document.
|
|
type boundary struct {
|
|
runeStart int // rune offset where the next chunk should start
|
|
priority int
|
|
}
|
|
|
|
// splitByHeuristicsImpl is the Tier-2 implementation. Falls through to the
|
|
// legacy splitter when no heuristic boundaries are found.
|
|
//
|
|
// profile is currently unused (this tier scans for boundaries directly) but
|
|
// is accepted to keep the splitByHeadings / splitByHeuristics signatures
|
|
// uniform — see strategy.runTier.
|
|
func splitByHeuristicsImpl(text string, cfg SplitterConfig, _ *DocProfile) []Chunk {
|
|
if text == "" {
|
|
return nil
|
|
}
|
|
runes := []rune(text)
|
|
totalRunes := len(runes)
|
|
if totalRunes <= cfg.ChunkSize {
|
|
return SplitText(text, cfg)
|
|
}
|
|
|
|
bounds := findHeuristicBoundaries(text, cfg.Languages)
|
|
// Drop any boundary that falls strictly inside a protected region (table,
|
|
// fenced code block, LaTeX block, etc.) — splitting there would cut
|
|
// through atomic content. Boundaries on a span edge are kept since they
|
|
// align with the protected region start/end.
|
|
if prot := protectedSpansRune(text, protectedSpans(text)); len(prot) > 0 {
|
|
bounds = dropBoundsInsideSpans(bounds, prot)
|
|
}
|
|
if len(bounds) == 0 {
|
|
return SplitText(text, cfg)
|
|
}
|
|
|
|
// Append a sentinel at end-of-document so the bin-packer can flush.
|
|
bounds = append(bounds, boundary{runeStart: totalRunes})
|
|
// Always start with a boundary at offset 0 if not already there.
|
|
if bounds[0].runeStart != 0 {
|
|
bounds = append([]boundary{{runeStart: 0}}, bounds...)
|
|
}
|
|
|
|
// Greedy bin-packing.
|
|
var out []Chunk
|
|
seq := 0
|
|
chunkStart := bounds[0].runeStart
|
|
curEnd := chunkStart
|
|
minChunkSize := cfg.ChunkSize / 4
|
|
if minChunkSize < 50 {
|
|
minChunkSize = 50
|
|
}
|
|
|
|
for i := 1; i < len(bounds); i++ {
|
|
nextEnd := bounds[i].runeStart
|
|
blockLen := nextEnd - curEnd
|
|
|
|
if blockLen > cfg.ChunkSize {
|
|
// The block between the previous and this boundary is itself too
|
|
// large to fit in any chunk. Flush current accumulation, then
|
|
// recursively chunk the oversize block via the legacy splitter.
|
|
if curEnd-chunkStart > 0 {
|
|
out = appendChunk(out, runes, chunkStart, curEnd, &seq)
|
|
chunkStart = curEnd
|
|
}
|
|
out = appendOversizeBlock(out, runes, curEnd, nextEnd, cfg, &seq)
|
|
curEnd = nextEnd
|
|
chunkStart = nextEnd
|
|
continue
|
|
}
|
|
|
|
// Would adding this block exceed the budget?
|
|
accumulated := nextEnd - chunkStart
|
|
if accumulated > cfg.ChunkSize && curEnd-chunkStart >= minChunkSize {
|
|
// Flush accumulated content as a chunk, restart at curEnd.
|
|
out = appendChunk(out, runes, chunkStart, curEnd, &seq)
|
|
// Snap overlap start to the nearest semantic boundary or line
|
|
// break instead of slicing mid-line / mid-word.
|
|
chunkStart = applyOverlapAligned(runes, curEnd, cfg.ChunkOverlap, bounds)
|
|
}
|
|
curEnd = nextEnd
|
|
}
|
|
|
|
// Flush remaining content.
|
|
if curEnd > chunkStart {
|
|
out = appendChunk(out, runes, chunkStart, curEnd, &seq)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// findHeuristicBoundaries scans text and returns boundary positions in
|
|
// ascending order. Lower-priority duplicates at the same offset are dropped.
|
|
func findHeuristicBoundaries(text string, langs []string) []boundary {
|
|
var bounds []boundary
|
|
|
|
// Form feeds — strongest single-character boundary.
|
|
for _, idx := range allRuneIndices(text, "\f") {
|
|
bounds = append(bounds, boundary{runeStart: idx, priority: PrioFormFeed})
|
|
}
|
|
|
|
// Per-line patterns walk the text once, line by line.
|
|
lines := strings.Split(text, "\n")
|
|
chapterPatterns := ChapterPatternsForLangs(langs)
|
|
pos := 0
|
|
inFence := false
|
|
for i, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
if strings.HasPrefix(trimmed, "```") {
|
|
inFence = !inFence
|
|
} else if !inFence {
|
|
runeStart := pos
|
|
added := false
|
|
for _, pat := range chapterPatterns {
|
|
if pat.MatchString(line) {
|
|
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioChapterMarker})
|
|
added = true
|
|
break
|
|
}
|
|
}
|
|
if !added && NumberedSectionPattern.MatchString(line) {
|
|
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioNumberedHead})
|
|
added = true
|
|
}
|
|
if !added && AllCapsHeadingPattern.MatchString(line) {
|
|
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioAllCapsHeading})
|
|
added = true
|
|
}
|
|
if !added && VisualSeparatorPattern.MatchString(line) {
|
|
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioVisualSep})
|
|
added = true
|
|
}
|
|
if !added && PageFooterPattern.MatchString(line) {
|
|
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioPageFooter})
|
|
}
|
|
}
|
|
pos += utf8.RuneCountInString(line)
|
|
if i < len(lines)-1 {
|
|
pos++ // \n
|
|
}
|
|
}
|
|
|
|
// Excessive blank blocks (\n{3,}). Match at the *start* of the run so we
|
|
// drop into the next paragraph cleanly.
|
|
for _, idx := range ExcessiveBlanksPattern.FindAllStringIndex(text, -1) {
|
|
runeStart := utf8.RuneCountInString(text[:idx[1]])
|
|
bounds = append(bounds, boundary{runeStart: runeStart, priority: PrioBlankBlock})
|
|
}
|
|
|
|
if len(bounds) == 0 {
|
|
return nil
|
|
}
|
|
|
|
// Sort by position; drop near-duplicate offsets keeping the highest priority.
|
|
sort.Slice(bounds, func(i, j int) bool {
|
|
if bounds[i].runeStart != bounds[j].runeStart {
|
|
return bounds[i].runeStart < bounds[j].runeStart
|
|
}
|
|
return bounds[i].priority > bounds[j].priority
|
|
})
|
|
deduped := bounds[:0]
|
|
prev := -1
|
|
for _, b := range bounds {
|
|
if b.runeStart != prev {
|
|
deduped = append(deduped, b)
|
|
prev = b.runeStart
|
|
}
|
|
}
|
|
return deduped
|
|
}
|
|
|
|
// dropBoundsInsideSpans returns bounds with entries that fall strictly
|
|
// inside any of the (rune-offset) protected spans removed. Bounds at a
|
|
// span's start or end are kept — they align with the span edge and don't
|
|
// split protected content. spans must be sorted by start.
|
|
func dropBoundsInsideSpans(bounds []boundary, spans []span) []boundary {
|
|
if len(spans) == 0 {
|
|
return bounds
|
|
}
|
|
out := bounds[:0]
|
|
boundLoop:
|
|
for _, b := range bounds {
|
|
for _, s := range spans {
|
|
if s.start >= b.runeStart {
|
|
break // remaining spans start at or after b — can't contain b
|
|
}
|
|
if b.runeStart < s.end {
|
|
continue boundLoop
|
|
}
|
|
}
|
|
out = append(out, b)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// allRuneIndices returns every rune offset where needle starts in text.
|
|
// Only used for single-rune needles like form-feed.
|
|
func allRuneIndices(text, needle string) []int {
|
|
var out []int
|
|
if needle == "" {
|
|
return out
|
|
}
|
|
pos := 0
|
|
for _, r := range text {
|
|
if string(r) == needle {
|
|
out = append(out, pos)
|
|
}
|
|
pos++
|
|
}
|
|
return out
|
|
}
|
|
|
|
// appendChunk slices runes[start:end] into a Chunk and appends it to out.
|
|
// Pure-whitespace slices are skipped (boundary clustering can occasionally
|
|
// produce them). The Content stored is the raw slice — Start/End rune
|
|
// offsets must match utf8.RuneCountInString(Content) for downstream
|
|
// reconstruction code; whitespace stripping for embedding happens in
|
|
// Chunk.EmbeddingContent.
|
|
func appendChunk(out []Chunk, runes []rune, start, end int, seq *int) []Chunk {
|
|
if end <= start {
|
|
return out
|
|
}
|
|
raw := string(runes[start:end])
|
|
if strings.TrimSpace(raw) == "" {
|
|
return out
|
|
}
|
|
c := Chunk{Content: raw, Seq: *seq, Start: start, End: end}
|
|
*seq++
|
|
return append(out, c)
|
|
}
|
|
|
|
// appendOversizeBlock recursively chunks a region that is itself larger than
|
|
// cfg.ChunkSize, using the legacy splitter so internal length budgets and
|
|
// protected patterns are still respected.
|
|
func appendOversizeBlock(out []Chunk, runes []rune, start, end int, cfg SplitterConfig, seq *int) []Chunk {
|
|
if end <= start {
|
|
return out
|
|
}
|
|
subText := string(runes[start:end])
|
|
subs := SplitText(subText, cfg)
|
|
for _, s := range subs {
|
|
out = append(out, Chunk{
|
|
Content: s.Content,
|
|
Seq: *seq,
|
|
Start: start + s.Start,
|
|
End: start + s.End,
|
|
})
|
|
*seq++
|
|
}
|
|
return out
|
|
}
|
|
|
|
// applyOverlapAligned returns the rune offset where the next chunk should
|
|
// start. The target is `curEnd - overlap`, but we snap to the nearest
|
|
// preceding boundary (within 2x overlap) or, failing that, the previous
|
|
// newline so chunks don't begin mid-line / mid-word. Falls back to the raw
|
|
// target only if neither option is available.
|
|
//
|
|
// curEnd itself is always a boundary (the bin-packer flushes at boundary
|
|
// positions), so we exclude it from the search — picking it would yield
|
|
// zero overlap, defeating the purpose of this function.
|
|
func applyOverlapAligned(runes []rune, curEnd, overlap int, bounds []boundary) int {
|
|
if overlap <= 0 {
|
|
return curEnd
|
|
}
|
|
target := curEnd - overlap
|
|
if target < 0 {
|
|
target = 0
|
|
}
|
|
// Allowed search window: [curEnd - 2*overlap, curEnd)
|
|
windowStart := curEnd - 2*overlap
|
|
if windowStart < 0 {
|
|
windowStart = 0
|
|
}
|
|
|
|
// Prefer a semantic boundary strictly inside the window.
|
|
bestBound := -1
|
|
for _, b := range bounds {
|
|
if b.runeStart >= windowStart && b.runeStart < curEnd && b.runeStart > bestBound {
|
|
bestBound = b.runeStart
|
|
}
|
|
}
|
|
if bestBound >= 0 {
|
|
return bestBound
|
|
}
|
|
|
|
// Fallback: scan backwards from `target` to the previous newline, but
|
|
// not past windowStart so we keep the overlap roughly the right size.
|
|
for i := target; i > windowStart && i < len(runes); i-- {
|
|
if runes[i] == '\n' {
|
|
return i + 1
|
|
}
|
|
}
|
|
return target
|
|
}
|