feat(chunker): recursive priority splitting + token-aware budget

The Go splitter previously combined all separators into one regex, losing
the priority-based recursive behavior of the Python reference. This pass
walks the separator list in order: the first separator that yields >1
piece wins, mirroring docreader/splitter/splitter.py. Larger semantic
boundaries (\n\n) now win over smaller ones (\n).

Additional cross-cutting changes:
- ChunkingConfig gains Strategy / TokenLimit / Languages fields so the
  per-KB config can pick a tier and pin embedding token budgets.
- buildSplitterConfig in the knowledge service uses the new
  chunker.DefaultChunkSize / DefaultChunkOverlap constants (overlap
  consolidated to 80 ≈ 15%) instead of the previous hardcoded 50.
- ensureDefaults clamps ChunkSize to a per-language character budget
  derived from TokenLimit so chunks stay safe under embedding model
  token caps. Chinese (~1.7 chars/tok) gets a tighter clamp than
  English (~4 chars/tok).

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
Claude
2026-05-02 09:22:02 +00:00
committed by lyingbug
parent ae6fde2310
commit 1585a6f12c
5 changed files with 97 additions and 20 deletions
+8 -2
View File
@@ -1786,17 +1786,23 @@ type ProcessChunksOptions struct {
}
// buildSplitterConfig creates a SplitterConfig with fallbacks from a KnowledgeBase.
// Defaults mirror chunker.DefaultChunkSize / DefaultChunkOverlap so behavior is
// identical whether callers come through this path or invoke the chunker
// directly with a zero-value config.
func buildSplitterConfig(kb *types.KnowledgeBase) chunker.SplitterConfig {
chunkCfg := chunker.SplitterConfig{
ChunkSize: kb.ChunkingConfig.ChunkSize,
ChunkOverlap: kb.ChunkingConfig.ChunkOverlap,
Separators: kb.ChunkingConfig.Separators,
Strategy: kb.ChunkingConfig.Strategy,
TokenLimit: kb.ChunkingConfig.TokenLimit,
Languages: kb.ChunkingConfig.Languages,
}
if chunkCfg.ChunkSize <= 0 {
chunkCfg.ChunkSize = 512
chunkCfg.ChunkSize = chunker.DefaultChunkSize
}
if chunkCfg.ChunkOverlap <= 0 {
chunkCfg.ChunkOverlap = 50
chunkCfg.ChunkOverlap = chunker.DefaultChunkOverlap
}
if len(chunkCfg.Separators) == 0 {
chunkCfg.Separators = []string{"\n\n", "\n", "。"}
+26 -18
View File
@@ -122,33 +122,41 @@ type splitUnit struct {
start, end int
}
// splitBySeparators splits text by separators in priority order, keeping separators.
// splitBySeparators splits text by separators in priority order, keeping the
// separators themselves as standalone units. Walks the separator list in
// order: the first separator that actually splits the text wins, the rest
// are not applied. Mirrors the recursive priority semantics of the Python
// reference splitter (docreader/splitter/splitter.py:_split).
func splitBySeparators(text string, separators []string) []string {
if len(separators) == 0 || text == "" {
return []string{text}
}
// Build regex that captures separators
var parts []string
for _, sep := range separators {
parts = append(parts, regexp.QuoteMeta(sep))
}
pattern := "(" + strings.Join(parts, "|") + ")"
re := regexp.MustCompile(pattern)
splits := re.Split(text, -1)
matches := re.FindAllString(text, -1)
var result []string
for i, s := range splits {
if s != "" {
result = append(result, s)
if sep == "" {
continue
}
if i < len(matches) && matches[i] != "" {
result = append(result, matches[i])
re := regexp.MustCompile("(" + regexp.QuoteMeta(sep) + ")")
splits := re.Split(text, -1)
matches := re.FindAllString(text, -1)
if len(matches) == 0 {
continue
}
var result []string
for i, s := range splits {
if s != "" {
result = append(result, s)
}
if i < len(matches) && matches[i] != "" {
result = append(result, matches[i])
}
}
if len(result) > 1 {
return result
}
}
return result
return []string{text}
}
// runeLen returns the number of runes in s.
@@ -120,6 +120,10 @@ func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk {
// ensureDefaults fills in zero-value config fields with sane defaults.
// Mirrors buildSplitterConfig in internal/application/service/knowledge.go
// so direct callers of this package get the same numbers.
//
// When cfg.TokenLimit is set, ChunkSize is clamped to the character budget
// that fits within that token limit (with a 10% safety factor). This makes
// chunks safe for embedding APIs that have hard token caps.
func ensureDefaults(cfg SplitterConfig) SplitterConfig {
if cfg.ChunkSize <= 0 {
cfg.ChunkSize = DefaultChunkSize
@@ -130,6 +134,19 @@ func ensureDefaults(cfg SplitterConfig) SplitterConfig {
if len(cfg.Separators) == 0 {
cfg.Separators = []string{"\n\n", "\n", "。"}
}
if cfg.TokenLimit > 0 {
lang := LangMixed
if len(cfg.Languages) > 0 {
lang = cfg.Languages[0]
}
charBudget := CharsForTokenLimit(cfg.TokenLimit, lang)
if charBudget > 0 && (cfg.ChunkSize == 0 || charBudget < cfg.ChunkSize) {
cfg.ChunkSize = charBudget
if cfg.ChunkOverlap >= cfg.ChunkSize {
cfg.ChunkOverlap = cfg.ChunkSize / 5
}
}
}
return cfg
}
@@ -0,0 +1,35 @@
package chunker
import "testing"
func TestEnsureDefaults_TokenLimitClampsChunkSize(t *testing.T) {
cfg := SplitterConfig{
ChunkSize: 10000, // huge
TokenLimit: 100,
Languages: []string{LangEnglish},
}
out := ensureDefaults(cfg)
// 100 tokens * 4 chars/token * 0.9 ≈ 360 chars
if out.ChunkSize >= 1000 {
t.Errorf("expected ChunkSize clamped by TokenLimit, got %d", out.ChunkSize)
}
if out.ChunkOverlap >= out.ChunkSize {
t.Errorf("overlap should be smaller than clamped chunk size: overlap=%d size=%d", out.ChunkOverlap, out.ChunkSize)
}
}
func TestEnsureDefaults_TokenLimitChineseTighter(t *testing.T) {
cfgEN := ensureDefaults(SplitterConfig{TokenLimit: 200, Languages: []string{LangEnglish}})
cfgZH := ensureDefaults(SplitterConfig{TokenLimit: 200, Languages: []string{LangChinese}})
if cfgZH.ChunkSize >= cfgEN.ChunkSize {
t.Errorf("Chinese char budget should be tighter than English: zh=%d en=%d", cfgZH.ChunkSize, cfgEN.ChunkSize)
}
}
func TestEnsureDefaults_NoTokenLimitKeepsChunkSize(t *testing.T) {
cfg := SplitterConfig{ChunkSize: 800}
out := ensureDefaults(cfg)
if out.ChunkSize != 800 {
t.Errorf("ChunkSize should stay 800, got %d", out.ChunkSize)
}
}
+11
View File
@@ -151,6 +151,17 @@ type ChunkingConfig struct {
// ChildChunkSize is the size of child chunks used for embedding (default: 384).
// Only used when EnableParentChild is true.
ChildChunkSize int `yaml:"child_chunk_size,omitempty" json:"child_chunk_size,omitempty"`
// Strategy selects the adaptive chunking tier. Empty / "legacy" preserves
// the historical recursive splitter; "auto" lets a profiler pick between
// heading-aware, heuristic and recursive tiers; "heading" / "heuristic" /
// "recursive" pin the tier explicitly.
Strategy string `yaml:"strategy,omitempty" json:"strategy,omitempty"`
// TokenLimit caps chunk size in approximate tokens. 0 = use ChunkSize
// as a character count.
TokenLimit int `yaml:"token_limit,omitempty" json:"token_limit,omitempty"`
// Languages hints the heuristic patterns. Empty = auto-detect from content.
// Examples: ["de"], ["en", "zh"].
Languages []string `yaml:"languages,omitempty" json:"languages,omitempty"`
}
// ResolveParserEngine returns the engine name for the given file type