ae6fde2310
Adds the building blocks for tiered chunking strategies — multilingual patterns, token approximation, document profiling, validation, and a strategy resolver that today still falls through to the legacy SplitText. Default Strategy is empty/legacy so existing knowledge bases continue to chunk identically. - tokens.go: char/token approximation per language (en/de/zh/mixed) and a coarse DetectLanguage heuristic - patterns.go: source-of-truth regex bibliothek for headings, numbered sections, multilingual chapter markers, visual separators, etc. - profiler.go: single-pass document scan + SelectStrategy chain - validator.go: rejects obviously broken chunk-sets to drive fallbacks - strategy.go: public Split / SplitParentChild entry points; resolves StrategyAuto via the profiler, anything else maps to a fixed tier - splitter.go: SplitterConfig gains Strategy / TokenLimit / Languages fields and DefaultChunkSize / DefaultChunkOverlap constants (overlap raised to 80 ≈ 15%) - knowledge.go: switches both call sites to chunker.Split / SplitParentChild — behavior unchanged for empty Strategy https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
123 lines
3.7 KiB
Go
123 lines
3.7 KiB
Go
package chunker
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestSplit_EmptyText(t *testing.T) {
|
|
if got := Split("", DefaultConfig()); got != nil {
|
|
t.Errorf("empty text should return nil, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestSplit_LegacyStrategy_MatchesSplitText(t *testing.T) {
|
|
text := strings.Repeat("Hello world.\n\n", 30)
|
|
cfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 20, Separators: []string{"\n\n"}, Strategy: StrategyLegacy}
|
|
a := Split(text, cfg)
|
|
b := SplitText(text, cfg)
|
|
if len(a) != len(b) {
|
|
t.Errorf("legacy strategy should match SplitText: got %d vs %d chunks", len(a), len(b))
|
|
}
|
|
for i := range a {
|
|
if a[i].Content != b[i].Content {
|
|
t.Errorf("chunk %d differs", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSplit_EmptyStrategyEqualsLegacy(t *testing.T) {
|
|
text := strings.Repeat("Sentence one. Sentence two.\n", 20)
|
|
cfg := SplitterConfig{ChunkSize: 80, ChunkOverlap: 10}
|
|
a := Split(text, cfg)
|
|
cfg.Strategy = StrategyLegacy
|
|
b := Split(text, cfg)
|
|
if len(a) != len(b) {
|
|
t.Errorf("empty Strategy should equal legacy: %d vs %d", len(a), len(b))
|
|
}
|
|
}
|
|
|
|
func TestSplit_AutoStrategy_PicksHeadingForMarkdownDoc(t *testing.T) {
|
|
doc := strings.Repeat("# A\nbody\n## B\nbody\n## C\nbody\n## D\nbody\n", 1)
|
|
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Strategy: StrategyAuto}
|
|
// Until heading_splitter is wired, this falls through to SplitText —
|
|
// just assert we get a valid result.
|
|
chunks := Split(doc, cfg)
|
|
if len(chunks) == 0 {
|
|
t.Error("auto strategy should produce chunks")
|
|
}
|
|
}
|
|
|
|
func TestSplitParentChild_LegacyStrategy(t *testing.T) {
|
|
text := strings.Repeat("This is a sentence. Another one.\n\n", 50)
|
|
parentCfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 40, Strategy: StrategyLegacy}
|
|
childCfg := SplitterConfig{ChunkSize: 100, ChunkOverlap: 20, Strategy: StrategyLegacy}
|
|
res := SplitParentChild(text, parentCfg, childCfg)
|
|
if len(res.Children) == 0 {
|
|
t.Fatal("expected children chunks")
|
|
}
|
|
for i, c := range res.Children {
|
|
if c.ParentIndex >= 0 && c.ParentIndex >= len(res.Parents) {
|
|
t.Errorf("child[%d] has invalid ParentIndex %d (parents=%d)", i, c.ParentIndex, len(res.Parents))
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEnsureDefaults(t *testing.T) {
|
|
cfg := ensureDefaults(SplitterConfig{})
|
|
if cfg.ChunkSize != DefaultChunkSize {
|
|
t.Errorf("expected default ChunkSize %d, got %d", DefaultChunkSize, cfg.ChunkSize)
|
|
}
|
|
if cfg.ChunkOverlap != DefaultChunkOverlap {
|
|
t.Errorf("expected default ChunkOverlap %d, got %d", DefaultChunkOverlap, cfg.ChunkOverlap)
|
|
}
|
|
if len(cfg.Separators) == 0 {
|
|
t.Error("expected default separators")
|
|
}
|
|
}
|
|
|
|
func TestValidateChunks_Empty(t *testing.T) {
|
|
if v := ValidateChunks(nil, 1000, 500); v.OK {
|
|
t.Error("nil chunks should be invalid")
|
|
}
|
|
}
|
|
|
|
func TestValidateChunks_SingleChunkLargeDoc(t *testing.T) {
|
|
c := []Chunk{{Content: strings.Repeat("a", 5000)}}
|
|
if v := ValidateChunks(c, 5000, 500); v.OK {
|
|
t.Error("single 10x-too-large chunk should be invalid")
|
|
}
|
|
}
|
|
|
|
func TestValidateChunks_AcceptsReasonableOutput(t *testing.T) {
|
|
chunks := []Chunk{
|
|
{Content: strings.Repeat("a", 480)},
|
|
{Content: strings.Repeat("b", 510)},
|
|
{Content: strings.Repeat("c", 460)},
|
|
}
|
|
if v := ValidateChunks(chunks, 1500, 512); !v.OK {
|
|
t.Errorf("reasonable chunks should validate, got: %s", v.Reason)
|
|
}
|
|
}
|
|
|
|
func TestValidateChunks_RejectsOversized(t *testing.T) {
|
|
chunks := []Chunk{
|
|
{Content: strings.Repeat("a", 100)},
|
|
{Content: strings.Repeat("b", 5000)}, // > 2x chunkSize
|
|
}
|
|
if v := ValidateChunks(chunks, 5100, 1000); v.OK {
|
|
t.Error("chunk >2x size should be invalid")
|
|
}
|
|
}
|
|
|
|
func TestValidateChunks_TolerantTinyTail(t *testing.T) {
|
|
chunks := []Chunk{
|
|
{Content: strings.Repeat("a", 480)},
|
|
{Content: strings.Repeat("b", 510)},
|
|
{Content: "tail"},
|
|
}
|
|
if v := ValidateChunks(chunks, 994, 512); !v.OK {
|
|
t.Errorf("tiny last chunk should be tolerated, got: %s", v.Reason)
|
|
}
|
|
}
|