From 5a79a817a1056194d0c172c07577fb72f419aa6f Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 6 May 2026 20:54:12 +0800 Subject: [PATCH] fix(chunker): heuristic splitter overlap was always zero applyOverlapAligned searched for the latest boundary in [curEnd-2*overlap, curEnd], but curEnd itself is always one of the boundaries (the bin-packer only flushes at boundary positions). The loop therefore always returned curEnd, producing chunkStart == curEnd and zero overlap regardless of cfg.ChunkOverlap. Exclude curEnd from the search window so an earlier boundary can be picked, restoring the intended overlap behaviour. --- .../chunker/heuristic_splitter.go | 10 +++- .../chunker/heuristic_splitter_test.go | 60 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/internal/infrastructure/chunker/heuristic_splitter.go b/internal/infrastructure/chunker/heuristic_splitter.go index a9bc74c5..e4012d03 100644 --- a/internal/infrastructure/chunker/heuristic_splitter.go +++ b/internal/infrastructure/chunker/heuristic_splitter.go @@ -233,6 +233,10 @@ func appendOversizeBlock(out []Chunk, runes []rune, start, end int, cfg Splitter // preceding boundary (within 2x overlap) or, failing that, the previous // newline so chunks don't begin mid-line / mid-word. Falls back to the raw // target only if neither option is available. +// +// curEnd itself is always a boundary (the bin-packer flushes at boundary +// positions), so we exclude it from the search — picking it would yield +// zero overlap, defeating the purpose of this function. func applyOverlapAligned(runes []rune, curEnd, overlap int, bounds []boundary) int { if overlap <= 0 { return curEnd @@ -241,16 +245,16 @@ func applyOverlapAligned(runes []rune, curEnd, overlap int, bounds []boundary) i if target < 0 { target = 0 } - // Allowed search window: [curEnd - 2*overlap, curEnd] + // Allowed search window: [curEnd - 2*overlap, curEnd) windowStart := curEnd - 2*overlap if windowStart < 0 { windowStart = 0 } - // Prefer a semantic boundary inside the window. + // Prefer a semantic boundary strictly inside the window. bestBound := -1 for _, b := range bounds { - if b.runeStart >= windowStart && b.runeStart <= curEnd && b.runeStart > bestBound { + if b.runeStart >= windowStart && b.runeStart < curEnd && b.runeStart > bestBound { bestBound = b.runeStart } } diff --git a/internal/infrastructure/chunker/heuristic_splitter_test.go b/internal/infrastructure/chunker/heuristic_splitter_test.go index 6bb2847c..a37f7cd9 100644 --- a/internal/infrastructure/chunker/heuristic_splitter_test.go +++ b/internal/infrastructure/chunker/heuristic_splitter_test.go @@ -87,3 +87,63 @@ func TestSplitByHeuristics_EmptyText(t *testing.T) { t.Errorf("empty doc should be nil, got %v", got) } } + +// Regression: applyOverlapAligned previously included curEnd itself in its +// boundary search, and curEnd is always one of the bounds (the bin-packer +// flushes at boundary positions). The function therefore always returned +// curEnd, producing zero overlap regardless of cfg.ChunkOverlap. +func TestSplitByHeuristics_OverlapActuallyOverlaps(t *testing.T) { + // Build many small numbered sections so the bin-packer flushes mid-doc + // with at least one earlier boundary inside the overlap window. + var sb strings.Builder + for i := 1; i <= 12; i++ { + sb.WriteString("\n\n") + sb.WriteByte(byte('0' + i%10)) + sb.WriteString(". ") + sb.WriteString(strings.Repeat("alpha beta gamma. ", 4)) // ~72 chars / section + } + doc := sb.String() + + cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 80, Separators: []string{". "}} + chunks := splitByHeuristicsImpl(doc, cfg) + if len(chunks) < 2 { + t.Fatalf("need >=2 chunks to test overlap, got %d", len(chunks)) + } + + // At least one consecutive chunk pair must share a non-trivial suffix / + // prefix. We don't require *every* pair to overlap (oversize blocks + // short-circuit through legacy and reset chunkStart), but at least one + // regular flush boundary should produce real overlap. + saw := false + for i := 1; i < len(chunks); i++ { + prev := strings.TrimSpace(chunks[i-1].Content) + cur := strings.TrimSpace(chunks[i].Content) + // Walk back the longest suffix of prev that prefixes cur. + match := 0 + maxScan := len(prev) + if len(cur) < maxScan { + maxScan = len(cur) + } + for n := 1; n <= maxScan; n++ { + if strings.HasPrefix(cur, prev[len(prev)-n:]) { + match = n + } + } + if match >= 20 { + saw = true + break + } + } + if !saw { + t.Fatalf("expected at least one chunk pair to overlap by >=20 chars, none did. chunk sizes: %v", + chunkLengths(chunks)) + } +} + +func chunkLengths(chunks []Chunk) []int { + out := make([]int, len(chunks)) + for i, c := range chunks { + out[i] = len([]rune(c.Content)) + } + return out +}