fix(chunker): heuristic splitter overlap was always zero

applyOverlapAligned searched for the latest boundary in
[curEnd-2*overlap, curEnd], but curEnd itself is always one of the
boundaries (the bin-packer only flushes at boundary positions). The
loop therefore always returned curEnd, producing chunkStart == curEnd
and zero overlap regardless of cfg.ChunkOverlap.

Exclude curEnd from the search window so an earlier boundary can be
picked, restoring the intended overlap behaviour.
This commit is contained in:
wizardchen
2026-05-06 20:54:12 +08:00
committed by lyingbug
parent 49cd392308
commit 5a79a817a1
2 changed files with 67 additions and 3 deletions
@@ -233,6 +233,10 @@ func appendOversizeBlock(out []Chunk, runes []rune, start, end int, cfg Splitter
// preceding boundary (within 2x overlap) or, failing that, the previous
// newline so chunks don't begin mid-line / mid-word. Falls back to the raw
// target only if neither option is available.
//
// curEnd itself is always a boundary (the bin-packer flushes at boundary
// positions), so we exclude it from the search — picking it would yield
// zero overlap, defeating the purpose of this function.
func applyOverlapAligned(runes []rune, curEnd, overlap int, bounds []boundary) int {
if overlap <= 0 {
return curEnd
@@ -241,16 +245,16 @@ func applyOverlapAligned(runes []rune, curEnd, overlap int, bounds []boundary) i
if target < 0 {
target = 0
}
// Allowed search window: [curEnd - 2*overlap, curEnd]
// Allowed search window: [curEnd - 2*overlap, curEnd)
windowStart := curEnd - 2*overlap
if windowStart < 0 {
windowStart = 0
}
// Prefer a semantic boundary inside the window.
// Prefer a semantic boundary strictly inside the window.
bestBound := -1
for _, b := range bounds {
if b.runeStart >= windowStart && b.runeStart <= curEnd && b.runeStart > bestBound {
if b.runeStart >= windowStart && b.runeStart < curEnd && b.runeStart > bestBound {
bestBound = b.runeStart
}
}
@@ -87,3 +87,63 @@ func TestSplitByHeuristics_EmptyText(t *testing.T) {
t.Errorf("empty doc should be nil, got %v", got)
}
}
// Regression: applyOverlapAligned previously included curEnd itself in its
// boundary search, and curEnd is always one of the bounds (the bin-packer
// flushes at boundary positions). The function therefore always returned
// curEnd, producing zero overlap regardless of cfg.ChunkOverlap.
func TestSplitByHeuristics_OverlapActuallyOverlaps(t *testing.T) {
// Build many small numbered sections so the bin-packer flushes mid-doc
// with at least one earlier boundary inside the overlap window.
var sb strings.Builder
for i := 1; i <= 12; i++ {
sb.WriteString("\n\n")
sb.WriteByte(byte('0' + i%10))
sb.WriteString(". ")
sb.WriteString(strings.Repeat("alpha beta gamma. ", 4)) // ~72 chars / section
}
doc := sb.String()
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 80, Separators: []string{". "}}
chunks := splitByHeuristicsImpl(doc, cfg)
if len(chunks) < 2 {
t.Fatalf("need >=2 chunks to test overlap, got %d", len(chunks))
}
// At least one consecutive chunk pair must share a non-trivial suffix /
// prefix. We don't require *every* pair to overlap (oversize blocks
// short-circuit through legacy and reset chunkStart), but at least one
// regular flush boundary should produce real overlap.
saw := false
for i := 1; i < len(chunks); i++ {
prev := strings.TrimSpace(chunks[i-1].Content)
cur := strings.TrimSpace(chunks[i].Content)
// Walk back the longest suffix of prev that prefixes cur.
match := 0
maxScan := len(prev)
if len(cur) < maxScan {
maxScan = len(cur)
}
for n := 1; n <= maxScan; n++ {
if strings.HasPrefix(cur, prev[len(prev)-n:]) {
match = n
}
}
if match >= 20 {
saw = true
break
}
}
if !saw {
t.Fatalf("expected at least one chunk pair to overlap by >=20 chars, none did. chunk sizes: %v",
chunkLengths(chunks))
}
}
func chunkLengths(chunks []Chunk) []int {
out := make([]int, len(chunks))
for i, c := range chunks {
out[i] = len([]rune(c.Content))
}
return out
}