Files
Enterprise-Knowledge-Base/internal/infrastructure/chunker/strategy.go
T
Claude ae6fde2310 feat(chunker): foundation for adaptive 3-tier chunking
Adds the building blocks for tiered chunking strategies — multilingual
patterns, token approximation, document profiling, validation, and a
strategy resolver that today still falls through to the legacy
SplitText. Default Strategy is empty/legacy so existing knowledge bases
continue to chunk identically.

- tokens.go: char/token approximation per language (en/de/zh/mixed)
  and a coarse DetectLanguage heuristic
- patterns.go: source-of-truth regex bibliothek for headings, numbered
  sections, multilingual chapter markers, visual separators, etc.
- profiler.go: single-pass document scan + SelectStrategy chain
- validator.go: rejects obviously broken chunk-sets to drive fallbacks
- strategy.go: public Split / SplitParentChild entry points; resolves
  StrategyAuto via the profiler, anything else maps to a fixed tier
- splitter.go: SplitterConfig gains Strategy / TokenLimit / Languages
  fields and DefaultChunkSize / DefaultChunkOverlap constants (overlap
  raised to 80 ≈ 15%)
- knowledge.go: switches both call sites to chunker.Split /
  SplitParentChild — behavior unchanged for empty Strategy

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
2026-05-06 17:17:07 +08:00

145 lines
4.6 KiB
Go

// Package chunker - strategy.go is the public entry point for adaptive
// chunking. Callers invoke Split / SplitParentChild instead of the legacy
// SplitText / SplitTextParentChild functions; the strategy resolver picks
// a tier based on document profile and the SplitterConfig.Strategy hint.
//
// The legacy entry points still exist in splitter.go for backwards
// compatibility — strategy.go simply layers a tier-selector on top.
package chunker
import "github.com/Tencent/WeKnora/internal/logger"
// Strategy values for SplitterConfig.Strategy.
const (
StrategyAuto = "auto"
StrategyHeading = "heading"
StrategyHeuristic = "heuristic"
StrategyRecursive = "recursive"
StrategyLegacy = "legacy"
)
// Split chunks text using the strategy configured in cfg. When cfg.Strategy
// is empty or "auto" the document profiler picks the tier. The function
// always returns a non-nil result: on tier failure the chain falls through
// to the legacy splitter, which is the original Tier 3 implementation.
func Split(text string, cfg SplitterConfig) []Chunk {
if text == "" {
return nil
}
cfg = ensureDefaults(cfg)
chain := resolveChain(text, cfg)
totalChars := len([]rune(text))
for _, tier := range chain {
out := runTier(tier, text, cfg)
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
return out
} else {
logger.Debugf(nil, "chunker: tier %s rejected: %s", tier, v.Reason)
}
}
// Last-ditch fallback: always return *something*.
return SplitText(text, cfg)
}
// SplitParentChild is the strategy-aware analog of SplitTextParentChild.
// It runs the tier selector for parent splitting, then re-splits each
// parent into children with the small-chunk config.
func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
if text == "" {
return ParentChildResult{}
}
parentCfg = ensureDefaults(parentCfg)
childCfg = ensureDefaults(childCfg)
parents := Split(text, parentCfg)
if len(parents) == 0 {
return ParentChildResult{}
}
var newParents []Chunk
var children []ChildChunk
childSeq := 0
for _, parent := range parents {
subs := Split(parent.Content, childCfg)
parentIndex := -1
if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) {
parentIndex = len(newParents)
newParents = append(newParents, parent)
}
for _, sub := range subs {
sub.Seq = childSeq
sub.Start += parent.Start
sub.End += parent.Start
children = append(children, ChildChunk{Chunk: sub, ParentIndex: parentIndex})
childSeq++
}
}
return ParentChildResult{Parents: newParents, Children: children}
}
// resolveChain returns the strategy chain to attempt. An explicit non-auto
// strategy bypasses the profiler entirely and pins to the requested tier.
func resolveChain(text string, cfg SplitterConfig) []StrategyTier {
switch cfg.Strategy {
case StrategyHeading:
return []StrategyTier{TierHeading, TierLegacy}
case StrategyHeuristic:
return []StrategyTier{TierHeuristic, TierLegacy}
case StrategyRecursive:
return []StrategyTier{TierRecursive, TierLegacy}
case StrategyLegacy, "":
// Empty == legacy preserves backwards compatibility with stored
// ChunkingConfig rows that pre-date the Strategy field.
return []StrategyTier{TierLegacy}
case StrategyAuto:
fallthrough
default:
profile := ProfileDocument(text)
return SelectStrategy(profile)
}
}
// runTier dispatches the splitter implementation for the given tier.
// Heading and heuristic tiers are stubbed in this scaffold and currently
// fall through to the legacy splitter — they are filled in in later phases.
func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk {
switch tier {
case TierHeading:
return splitByHeadings(text, cfg)
case TierHeuristic:
return splitByHeuristics(text, cfg)
case TierRecursive, TierLegacy:
return SplitText(text, cfg)
}
return SplitText(text, cfg)
}
// ensureDefaults fills in zero-value config fields with sane defaults.
// Mirrors buildSplitterConfig in internal/application/service/knowledge.go
// so direct callers of this package get the same numbers.
func ensureDefaults(cfg SplitterConfig) SplitterConfig {
if cfg.ChunkSize <= 0 {
cfg.ChunkSize = DefaultChunkSize
}
if cfg.ChunkOverlap <= 0 {
cfg.ChunkOverlap = DefaultChunkOverlap
}
if len(cfg.Separators) == 0 {
cfg.Separators = []string{"\n\n", "\n", "。"}
}
return cfg
}
// splitByHeadings is overridden by heading_splitter.go.
var splitByHeadings = func(text string, cfg SplitterConfig) []Chunk {
return SplitText(text, cfg)
}
// splitByHeuristics is overridden by heuristic_splitter.go.
var splitByHeuristics = func(text string, cfg SplitterConfig) []Chunk {
return SplitText(text, cfg)
}