feat(chunker): foundation for adaptive 3-tier chunking
Adds the building blocks for tiered chunking strategies — multilingual patterns, token approximation, document profiling, validation, and a strategy resolver that today still falls through to the legacy SplitText. Default Strategy is empty/legacy so existing knowledge bases continue to chunk identically. - tokens.go: char/token approximation per language (en/de/zh/mixed) and a coarse DetectLanguage heuristic - patterns.go: source-of-truth regex bibliothek for headings, numbered sections, multilingual chapter markers, visual separators, etc. - profiler.go: single-pass document scan + SelectStrategy chain - validator.go: rejects obviously broken chunk-sets to drive fallbacks - strategy.go: public Split / SplitParentChild entry points; resolves StrategyAuto via the profiler, anything else maps to a fixed tier - splitter.go: SplitterConfig gains Strategy / TokenLimit / Languages fields and DefaultChunkSize / DefaultChunkOverlap constants (overlap raised to 80 ≈ 15%) - knowledge.go: switches both call sites to chunker.Split / SplitParentChild — behavior unchanged for empty Strategy https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
@@ -7484,7 +7484,7 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
|
||||
if kb.ChunkingConfig.EnableParentChild {
|
||||
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
|
||||
pcResult := chunker.SplitTextParentChild(clean, parentCfg, childCfg)
|
||||
pcResult := chunker.SplitParentChild(clean, parentCfg, childCfg)
|
||||
parsed = make([]types.ParsedChunk, len(pcResult.Children))
|
||||
for i, c := range pcResult.Children {
|
||||
parsed[i] = types.ParsedChunk{
|
||||
@@ -7501,7 +7501,7 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
}
|
||||
opts.ParentChunks = parentChunks
|
||||
} else {
|
||||
splitChunks := chunker.SplitText(clean, chunkCfg)
|
||||
splitChunks := chunker.Split(clean, chunkCfg)
|
||||
parsed = make([]types.ParsedChunk, len(splitChunks))
|
||||
for i, c := range splitChunks {
|
||||
parsed[i] = types.ParsedChunk{
|
||||
@@ -8340,7 +8340,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
|
||||
if kb.ChunkingConfig.EnableParentChild {
|
||||
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
|
||||
pcResult := chunker.SplitTextParentChild(convertResult.MarkdownContent, parentCfg, childCfg)
|
||||
pcResult := chunker.SplitParentChild(convertResult.MarkdownContent, parentCfg, childCfg)
|
||||
chunks = make([]types.ParsedChunk, len(pcResult.Children))
|
||||
for i, c := range pcResult.Children {
|
||||
chunks[i] = types.ParsedChunk{
|
||||
@@ -8359,7 +8359,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
logger.Infof(ctx, "Split document into %d parent + %d child chunks for knowledge %s",
|
||||
len(pcResult.Parents), len(pcResult.Children), knowledge.ID)
|
||||
} else {
|
||||
splitChunks := chunker.SplitText(convertResult.MarkdownContent, chunkCfg)
|
||||
splitChunks := chunker.Split(convertResult.MarkdownContent, chunkCfg)
|
||||
chunks = make([]types.ParsedChunk, len(splitChunks))
|
||||
for i, c := range splitChunks {
|
||||
chunks[i] = types.ParsedChunk{
|
||||
|
||||
Reference in New Issue
Block a user