From 1585a6f12cb1b75d4c58740e997b2bcbe1eb3baa Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 2 May 2026 09:22:02 +0000 Subject: [PATCH] feat(chunker): recursive priority splitting + token-aware budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go splitter previously combined all separators into one regex, losing the priority-based recursive behavior of the Python reference. This pass walks the separator list in order: the first separator that yields >1 piece wins, mirroring docreader/splitter/splitter.py. Larger semantic boundaries (\n\n) now win over smaller ones (\n). Additional cross-cutting changes: - ChunkingConfig gains Strategy / TokenLimit / Languages fields so the per-KB config can pick a tier and pin embedding token budgets. - buildSplitterConfig in the knowledge service uses the new chunker.DefaultChunkSize / DefaultChunkOverlap constants (overlap consolidated to 80 ≈ 15%) instead of the previous hardcoded 50. - ensureDefaults clamps ChunkSize to a per-language character budget derived from TokenLimit so chunks stay safe under embedding model token caps. Chinese (~1.7 chars/tok) gets a tighter clamp than English (~4 chars/tok). https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k --- internal/application/service/knowledge.go | 10 ++++- internal/infrastructure/chunker/splitter.go | 44 +++++++++++-------- internal/infrastructure/chunker/strategy.go | 17 +++++++ .../chunker/strategy_token_test.go | 35 +++++++++++++++ internal/types/knowledgebase.go | 11 +++++ 5 files changed, 97 insertions(+), 20 deletions(-) create mode 100644 internal/infrastructure/chunker/strategy_token_test.go diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index de1a95d5..1411f86b 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -1786,17 +1786,23 @@ type ProcessChunksOptions struct { } // buildSplitterConfig creates a SplitterConfig with fallbacks from a KnowledgeBase. +// Defaults mirror chunker.DefaultChunkSize / DefaultChunkOverlap so behavior is +// identical whether callers come through this path or invoke the chunker +// directly with a zero-value config. func buildSplitterConfig(kb *types.KnowledgeBase) chunker.SplitterConfig { chunkCfg := chunker.SplitterConfig{ ChunkSize: kb.ChunkingConfig.ChunkSize, ChunkOverlap: kb.ChunkingConfig.ChunkOverlap, Separators: kb.ChunkingConfig.Separators, + Strategy: kb.ChunkingConfig.Strategy, + TokenLimit: kb.ChunkingConfig.TokenLimit, + Languages: kb.ChunkingConfig.Languages, } if chunkCfg.ChunkSize <= 0 { - chunkCfg.ChunkSize = 512 + chunkCfg.ChunkSize = chunker.DefaultChunkSize } if chunkCfg.ChunkOverlap <= 0 { - chunkCfg.ChunkOverlap = 50 + chunkCfg.ChunkOverlap = chunker.DefaultChunkOverlap } if len(chunkCfg.Separators) == 0 { chunkCfg.Separators = []string{"\n\n", "\n", "。"} diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index 2718d857..5e8db351 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -122,33 +122,41 @@ type splitUnit struct { start, end int } -// splitBySeparators splits text by separators in priority order, keeping separators. +// splitBySeparators splits text by separators in priority order, keeping the +// separators themselves as standalone units. Walks the separator list in +// order: the first separator that actually splits the text wins, the rest +// are not applied. Mirrors the recursive priority semantics of the Python +// reference splitter (docreader/splitter/splitter.py:_split). func splitBySeparators(text string, separators []string) []string { if len(separators) == 0 || text == "" { return []string{text} } - // Build regex that captures separators - var parts []string for _, sep := range separators { - parts = append(parts, regexp.QuoteMeta(sep)) - } - pattern := "(" + strings.Join(parts, "|") + ")" - re := regexp.MustCompile(pattern) - - splits := re.Split(text, -1) - matches := re.FindAllString(text, -1) - - var result []string - for i, s := range splits { - if s != "" { - result = append(result, s) + if sep == "" { + continue } - if i < len(matches) && matches[i] != "" { - result = append(result, matches[i]) + re := regexp.MustCompile("(" + regexp.QuoteMeta(sep) + ")") + splits := re.Split(text, -1) + matches := re.FindAllString(text, -1) + if len(matches) == 0 { + continue + } + + var result []string + for i, s := range splits { + if s != "" { + result = append(result, s) + } + if i < len(matches) && matches[i] != "" { + result = append(result, matches[i]) + } + } + if len(result) > 1 { + return result } } - return result + return []string{text} } // runeLen returns the number of runes in s. diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go index aeabd9d7..8b3133d8 100644 --- a/internal/infrastructure/chunker/strategy.go +++ b/internal/infrastructure/chunker/strategy.go @@ -120,6 +120,10 @@ func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk { // ensureDefaults fills in zero-value config fields with sane defaults. // Mirrors buildSplitterConfig in internal/application/service/knowledge.go // so direct callers of this package get the same numbers. +// +// When cfg.TokenLimit is set, ChunkSize is clamped to the character budget +// that fits within that token limit (with a 10% safety factor). This makes +// chunks safe for embedding APIs that have hard token caps. func ensureDefaults(cfg SplitterConfig) SplitterConfig { if cfg.ChunkSize <= 0 { cfg.ChunkSize = DefaultChunkSize @@ -130,6 +134,19 @@ func ensureDefaults(cfg SplitterConfig) SplitterConfig { if len(cfg.Separators) == 0 { cfg.Separators = []string{"\n\n", "\n", "。"} } + if cfg.TokenLimit > 0 { + lang := LangMixed + if len(cfg.Languages) > 0 { + lang = cfg.Languages[0] + } + charBudget := CharsForTokenLimit(cfg.TokenLimit, lang) + if charBudget > 0 && (cfg.ChunkSize == 0 || charBudget < cfg.ChunkSize) { + cfg.ChunkSize = charBudget + if cfg.ChunkOverlap >= cfg.ChunkSize { + cfg.ChunkOverlap = cfg.ChunkSize / 5 + } + } + } return cfg } diff --git a/internal/infrastructure/chunker/strategy_token_test.go b/internal/infrastructure/chunker/strategy_token_test.go new file mode 100644 index 00000000..290802e7 --- /dev/null +++ b/internal/infrastructure/chunker/strategy_token_test.go @@ -0,0 +1,35 @@ +package chunker + +import "testing" + +func TestEnsureDefaults_TokenLimitClampsChunkSize(t *testing.T) { + cfg := SplitterConfig{ + ChunkSize: 10000, // huge + TokenLimit: 100, + Languages: []string{LangEnglish}, + } + out := ensureDefaults(cfg) + // 100 tokens * 4 chars/token * 0.9 ≈ 360 chars + if out.ChunkSize >= 1000 { + t.Errorf("expected ChunkSize clamped by TokenLimit, got %d", out.ChunkSize) + } + if out.ChunkOverlap >= out.ChunkSize { + t.Errorf("overlap should be smaller than clamped chunk size: overlap=%d size=%d", out.ChunkOverlap, out.ChunkSize) + } +} + +func TestEnsureDefaults_TokenLimitChineseTighter(t *testing.T) { + cfgEN := ensureDefaults(SplitterConfig{TokenLimit: 200, Languages: []string{LangEnglish}}) + cfgZH := ensureDefaults(SplitterConfig{TokenLimit: 200, Languages: []string{LangChinese}}) + if cfgZH.ChunkSize >= cfgEN.ChunkSize { + t.Errorf("Chinese char budget should be tighter than English: zh=%d en=%d", cfgZH.ChunkSize, cfgEN.ChunkSize) + } +} + +func TestEnsureDefaults_NoTokenLimitKeepsChunkSize(t *testing.T) { + cfg := SplitterConfig{ChunkSize: 800} + out := ensureDefaults(cfg) + if out.ChunkSize != 800 { + t.Errorf("ChunkSize should stay 800, got %d", out.ChunkSize) + } +} diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index 25ffb149..17fc39b0 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -151,6 +151,17 @@ type ChunkingConfig struct { // ChildChunkSize is the size of child chunks used for embedding (default: 384). // Only used when EnableParentChild is true. ChildChunkSize int `yaml:"child_chunk_size,omitempty" json:"child_chunk_size,omitempty"` + // Strategy selects the adaptive chunking tier. Empty / "legacy" preserves + // the historical recursive splitter; "auto" lets a profiler pick between + // heading-aware, heuristic and recursive tiers; "heading" / "heuristic" / + // "recursive" pin the tier explicitly. + Strategy string `yaml:"strategy,omitempty" json:"strategy,omitempty"` + // TokenLimit caps chunk size in approximate tokens. 0 = use ChunkSize + // as a character count. + TokenLimit int `yaml:"token_limit,omitempty" json:"token_limit,omitempty"` + // Languages hints the heuristic patterns. Empty = auto-detect from content. + // Examples: ["de"], ["en", "zh"]. + Languages []string `yaml:"languages,omitempty" json:"languages,omitempty"` } // ResolveParserEngine returns the engine name for the given file type