feat(chunker): foundation for adaptive 3-tier chunking

Adds the building blocks for tiered chunking strategies — multilingual
patterns, token approximation, document profiling, validation, and a
strategy resolver that today still falls through to the legacy
SplitText. Default Strategy is empty/legacy so existing knowledge bases
continue to chunk identically.

- tokens.go: char/token approximation per language (en/de/zh/mixed)
  and a coarse DetectLanguage heuristic
- patterns.go: source-of-truth regex bibliothek for headings, numbered
  sections, multilingual chapter markers, visual separators, etc.
- profiler.go: single-pass document scan + SelectStrategy chain
- validator.go: rejects obviously broken chunk-sets to drive fallbacks
- strategy.go: public Split / SplitParentChild entry points; resolves
  StrategyAuto via the profiler, anything else maps to a fixed tier
- splitter.go: SplitterConfig gains Strategy / TokenLimit / Languages
  fields and DefaultChunkSize / DefaultChunkOverlap constants (overlap
  raised to 80 ≈ 15%)
- knowledge.go: switches both call sites to chunker.Split /
  SplitParentChild — behavior unchanged for empty Strategy

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
Claude
2026-05-02 09:17:57 +00:00
committed by lyingbug
parent 46cab52fb0
commit ae6fde2310
11 changed files with 1263 additions and 7 deletions
+4 -4
View File
@@ -7484,7 +7484,7 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
if kb.ChunkingConfig.EnableParentChild {
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
pcResult := chunker.SplitTextParentChild(clean, parentCfg, childCfg)
pcResult := chunker.SplitParentChild(clean, parentCfg, childCfg)
parsed = make([]types.ParsedChunk, len(pcResult.Children))
for i, c := range pcResult.Children {
parsed[i] = types.ParsedChunk{
@@ -7501,7 +7501,7 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
}
opts.ParentChunks = parentChunks
} else {
splitChunks := chunker.SplitText(clean, chunkCfg)
splitChunks := chunker.Split(clean, chunkCfg)
parsed = make([]types.ParsedChunk, len(splitChunks))
for i, c := range splitChunks {
parsed[i] = types.ParsedChunk{
@@ -8340,7 +8340,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
if kb.ChunkingConfig.EnableParentChild {
parentCfg, childCfg := buildParentChildConfigs(kb.ChunkingConfig, chunkCfg)
pcResult := chunker.SplitTextParentChild(convertResult.MarkdownContent, parentCfg, childCfg)
pcResult := chunker.SplitParentChild(convertResult.MarkdownContent, parentCfg, childCfg)
chunks = make([]types.ParsedChunk, len(pcResult.Children))
for i, c := range pcResult.Children {
chunks[i] = types.ParsedChunk{
@@ -8359,7 +8359,7 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
logger.Infof(ctx, "Split document into %d parent + %d child chunks for knowledge %s",
len(pcResult.Parents), len(pcResult.Children), knowledge.ID)
} else {
splitChunks := chunker.SplitText(convertResult.MarkdownContent, chunkCfg)
splitChunks := chunker.Split(convertResult.MarkdownContent, chunkCfg)
chunks = make([]types.ParsedChunk, len(splitChunks))
for i, c := range splitChunks {
chunks[i] = types.ParsedChunk{