Files
Enterprise-Knowledge-Base/internal/infrastructure/chunker/strategy_token_test.go
T
Claude 1585a6f12c feat(chunker): recursive priority splitting + token-aware budget
The Go splitter previously combined all separators into one regex, losing
the priority-based recursive behavior of the Python reference. This pass
walks the separator list in order: the first separator that yields >1
piece wins, mirroring docreader/splitter/splitter.py. Larger semantic
boundaries (\n\n) now win over smaller ones (\n).

Additional cross-cutting changes:
- ChunkingConfig gains Strategy / TokenLimit / Languages fields so the
  per-KB config can pick a tier and pin embedding token budgets.
- buildSplitterConfig in the knowledge service uses the new
  chunker.DefaultChunkSize / DefaultChunkOverlap constants (overlap
  consolidated to 80 ≈ 15%) instead of the previous hardcoded 50.
- ensureDefaults clamps ChunkSize to a per-language character budget
  derived from TokenLimit so chunks stay safe under embedding model
  token caps. Chinese (~1.7 chars/tok) gets a tighter clamp than
  English (~4 chars/tok).

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
2026-05-06 17:17:07 +08:00

36 lines
1.2 KiB
Go

package chunker
import "testing"
func TestEnsureDefaults_TokenLimitClampsChunkSize(t *testing.T) {
cfg := SplitterConfig{
ChunkSize: 10000, // huge
TokenLimit: 100,
Languages: []string{LangEnglish},
}
out := ensureDefaults(cfg)
// 100 tokens * 4 chars/token * 0.9 ≈ 360 chars
if out.ChunkSize >= 1000 {
t.Errorf("expected ChunkSize clamped by TokenLimit, got %d", out.ChunkSize)
}
if out.ChunkOverlap >= out.ChunkSize {
t.Errorf("overlap should be smaller than clamped chunk size: overlap=%d size=%d", out.ChunkOverlap, out.ChunkSize)
}
}
func TestEnsureDefaults_TokenLimitChineseTighter(t *testing.T) {
cfgEN := ensureDefaults(SplitterConfig{TokenLimit: 200, Languages: []string{LangEnglish}})
cfgZH := ensureDefaults(SplitterConfig{TokenLimit: 200, Languages: []string{LangChinese}})
if cfgZH.ChunkSize >= cfgEN.ChunkSize {
t.Errorf("Chinese char budget should be tighter than English: zh=%d en=%d", cfgZH.ChunkSize, cfgEN.ChunkSize)
}
}
func TestEnsureDefaults_NoTokenLimitKeepsChunkSize(t *testing.T) {
cfg := SplitterConfig{ChunkSize: 800}
out := ensureDefaults(cfg)
if out.ChunkSize != 800 {
t.Errorf("ChunkSize should stay 800, got %d", out.ChunkSize)
}
}