feat(chunker): recursive priority splitting + token-aware budget
The Go splitter previously combined all separators into one regex, losing the priority-based recursive behavior of the Python reference. This pass walks the separator list in order: the first separator that yields >1 piece wins, mirroring docreader/splitter/splitter.py. Larger semantic boundaries (\n\n) now win over smaller ones (\n). Additional cross-cutting changes: - ChunkingConfig gains Strategy / TokenLimit / Languages fields so the per-KB config can pick a tier and pin embedding token budgets. - buildSplitterConfig in the knowledge service uses the new chunker.DefaultChunkSize / DefaultChunkOverlap constants (overlap consolidated to 80 ≈ 15%) instead of the previous hardcoded 50. - ensureDefaults clamps ChunkSize to a per-language character budget derived from TokenLimit so chunks stay safe under embedding model token caps. Chinese (~1.7 chars/tok) gets a tighter clamp than English (~4 chars/tok). https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
@@ -1786,17 +1786,23 @@ type ProcessChunksOptions struct {
|
||||
}
|
||||
|
||||
// buildSplitterConfig creates a SplitterConfig with fallbacks from a KnowledgeBase.
|
||||
// Defaults mirror chunker.DefaultChunkSize / DefaultChunkOverlap so behavior is
|
||||
// identical whether callers come through this path or invoke the chunker
|
||||
// directly with a zero-value config.
|
||||
func buildSplitterConfig(kb *types.KnowledgeBase) chunker.SplitterConfig {
|
||||
chunkCfg := chunker.SplitterConfig{
|
||||
ChunkSize: kb.ChunkingConfig.ChunkSize,
|
||||
ChunkOverlap: kb.ChunkingConfig.ChunkOverlap,
|
||||
Separators: kb.ChunkingConfig.Separators,
|
||||
Strategy: kb.ChunkingConfig.Strategy,
|
||||
TokenLimit: kb.ChunkingConfig.TokenLimit,
|
||||
Languages: kb.ChunkingConfig.Languages,
|
||||
}
|
||||
if chunkCfg.ChunkSize <= 0 {
|
||||
chunkCfg.ChunkSize = 512
|
||||
chunkCfg.ChunkSize = chunker.DefaultChunkSize
|
||||
}
|
||||
if chunkCfg.ChunkOverlap <= 0 {
|
||||
chunkCfg.ChunkOverlap = 50
|
||||
chunkCfg.ChunkOverlap = chunker.DefaultChunkOverlap
|
||||
}
|
||||
if len(chunkCfg.Separators) == 0 {
|
||||
chunkCfg.Separators = []string{"\n\n", "\n", "。"}
|
||||
|
||||
@@ -122,33 +122,41 @@ type splitUnit struct {
|
||||
start, end int
|
||||
}
|
||||
|
||||
// splitBySeparators splits text by separators in priority order, keeping separators.
|
||||
// splitBySeparators splits text by separators in priority order, keeping the
|
||||
// separators themselves as standalone units. Walks the separator list in
|
||||
// order: the first separator that actually splits the text wins, the rest
|
||||
// are not applied. Mirrors the recursive priority semantics of the Python
|
||||
// reference splitter (docreader/splitter/splitter.py:_split).
|
||||
func splitBySeparators(text string, separators []string) []string {
|
||||
if len(separators) == 0 || text == "" {
|
||||
return []string{text}
|
||||
}
|
||||
|
||||
// Build regex that captures separators
|
||||
var parts []string
|
||||
for _, sep := range separators {
|
||||
parts = append(parts, regexp.QuoteMeta(sep))
|
||||
}
|
||||
pattern := "(" + strings.Join(parts, "|") + ")"
|
||||
re := regexp.MustCompile(pattern)
|
||||
|
||||
splits := re.Split(text, -1)
|
||||
matches := re.FindAllString(text, -1)
|
||||
|
||||
var result []string
|
||||
for i, s := range splits {
|
||||
if s != "" {
|
||||
result = append(result, s)
|
||||
if sep == "" {
|
||||
continue
|
||||
}
|
||||
if i < len(matches) && matches[i] != "" {
|
||||
result = append(result, matches[i])
|
||||
re := regexp.MustCompile("(" + regexp.QuoteMeta(sep) + ")")
|
||||
splits := re.Split(text, -1)
|
||||
matches := re.FindAllString(text, -1)
|
||||
if len(matches) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var result []string
|
||||
for i, s := range splits {
|
||||
if s != "" {
|
||||
result = append(result, s)
|
||||
}
|
||||
if i < len(matches) && matches[i] != "" {
|
||||
result = append(result, matches[i])
|
||||
}
|
||||
}
|
||||
if len(result) > 1 {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return result
|
||||
return []string{text}
|
||||
}
|
||||
|
||||
// runeLen returns the number of runes in s.
|
||||
|
||||
@@ -120,6 +120,10 @@ func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk {
|
||||
// ensureDefaults fills in zero-value config fields with sane defaults.
|
||||
// Mirrors buildSplitterConfig in internal/application/service/knowledge.go
|
||||
// so direct callers of this package get the same numbers.
|
||||
//
|
||||
// When cfg.TokenLimit is set, ChunkSize is clamped to the character budget
|
||||
// that fits within that token limit (with a 10% safety factor). This makes
|
||||
// chunks safe for embedding APIs that have hard token caps.
|
||||
func ensureDefaults(cfg SplitterConfig) SplitterConfig {
|
||||
if cfg.ChunkSize <= 0 {
|
||||
cfg.ChunkSize = DefaultChunkSize
|
||||
@@ -130,6 +134,19 @@ func ensureDefaults(cfg SplitterConfig) SplitterConfig {
|
||||
if len(cfg.Separators) == 0 {
|
||||
cfg.Separators = []string{"\n\n", "\n", "。"}
|
||||
}
|
||||
if cfg.TokenLimit > 0 {
|
||||
lang := LangMixed
|
||||
if len(cfg.Languages) > 0 {
|
||||
lang = cfg.Languages[0]
|
||||
}
|
||||
charBudget := CharsForTokenLimit(cfg.TokenLimit, lang)
|
||||
if charBudget > 0 && (cfg.ChunkSize == 0 || charBudget < cfg.ChunkSize) {
|
||||
cfg.ChunkSize = charBudget
|
||||
if cfg.ChunkOverlap >= cfg.ChunkSize {
|
||||
cfg.ChunkOverlap = cfg.ChunkSize / 5
|
||||
}
|
||||
}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package chunker
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestEnsureDefaults_TokenLimitClampsChunkSize(t *testing.T) {
|
||||
cfg := SplitterConfig{
|
||||
ChunkSize: 10000, // huge
|
||||
TokenLimit: 100,
|
||||
Languages: []string{LangEnglish},
|
||||
}
|
||||
out := ensureDefaults(cfg)
|
||||
// 100 tokens * 4 chars/token * 0.9 ≈ 360 chars
|
||||
if out.ChunkSize >= 1000 {
|
||||
t.Errorf("expected ChunkSize clamped by TokenLimit, got %d", out.ChunkSize)
|
||||
}
|
||||
if out.ChunkOverlap >= out.ChunkSize {
|
||||
t.Errorf("overlap should be smaller than clamped chunk size: overlap=%d size=%d", out.ChunkOverlap, out.ChunkSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDefaults_TokenLimitChineseTighter(t *testing.T) {
|
||||
cfgEN := ensureDefaults(SplitterConfig{TokenLimit: 200, Languages: []string{LangEnglish}})
|
||||
cfgZH := ensureDefaults(SplitterConfig{TokenLimit: 200, Languages: []string{LangChinese}})
|
||||
if cfgZH.ChunkSize >= cfgEN.ChunkSize {
|
||||
t.Errorf("Chinese char budget should be tighter than English: zh=%d en=%d", cfgZH.ChunkSize, cfgEN.ChunkSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureDefaults_NoTokenLimitKeepsChunkSize(t *testing.T) {
|
||||
cfg := SplitterConfig{ChunkSize: 800}
|
||||
out := ensureDefaults(cfg)
|
||||
if out.ChunkSize != 800 {
|
||||
t.Errorf("ChunkSize should stay 800, got %d", out.ChunkSize)
|
||||
}
|
||||
}
|
||||
@@ -151,6 +151,17 @@ type ChunkingConfig struct {
|
||||
// ChildChunkSize is the size of child chunks used for embedding (default: 384).
|
||||
// Only used when EnableParentChild is true.
|
||||
ChildChunkSize int `yaml:"child_chunk_size,omitempty" json:"child_chunk_size,omitempty"`
|
||||
// Strategy selects the adaptive chunking tier. Empty / "legacy" preserves
|
||||
// the historical recursive splitter; "auto" lets a profiler pick between
|
||||
// heading-aware, heuristic and recursive tiers; "heading" / "heuristic" /
|
||||
// "recursive" pin the tier explicitly.
|
||||
Strategy string `yaml:"strategy,omitempty" json:"strategy,omitempty"`
|
||||
// TokenLimit caps chunk size in approximate tokens. 0 = use ChunkSize
|
||||
// as a character count.
|
||||
TokenLimit int `yaml:"token_limit,omitempty" json:"token_limit,omitempty"`
|
||||
// Languages hints the heuristic patterns. Empty = auto-detect from content.
|
||||
// Examples: ["de"], ["en", "zh"].
|
||||
Languages []string `yaml:"languages,omitempty" json:"languages,omitempty"`
|
||||
}
|
||||
|
||||
// ResolveParserEngine returns the engine name for the given file type
|
||||
|
||||
Reference in New Issue
Block a user