ae6fde2310
Adds the building blocks for tiered chunking strategies — multilingual patterns, token approximation, document profiling, validation, and a strategy resolver that today still falls through to the legacy SplitText. Default Strategy is empty/legacy so existing knowledge bases continue to chunk identically. - tokens.go: char/token approximation per language (en/de/zh/mixed) and a coarse DetectLanguage heuristic - patterns.go: source-of-truth regex bibliothek for headings, numbered sections, multilingual chapter markers, visual separators, etc. - profiler.go: single-pass document scan + SelectStrategy chain - validator.go: rejects obviously broken chunk-sets to drive fallbacks - strategy.go: public Split / SplitParentChild entry points; resolves StrategyAuto via the profiler, anything else maps to a fixed tier - splitter.go: SplitterConfig gains Strategy / TokenLimit / Languages fields and DefaultChunkSize / DefaultChunkOverlap constants (overlap raised to 80 ≈ 15%) - knowledge.go: switches both call sites to chunker.Split / SplitParentChild — behavior unchanged for empty Strategy https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
145 lines
4.6 KiB
Go
145 lines
4.6 KiB
Go
// Package chunker - strategy.go is the public entry point for adaptive
|
|
// chunking. Callers invoke Split / SplitParentChild instead of the legacy
|
|
// SplitText / SplitTextParentChild functions; the strategy resolver picks
|
|
// a tier based on document profile and the SplitterConfig.Strategy hint.
|
|
//
|
|
// The legacy entry points still exist in splitter.go for backwards
|
|
// compatibility — strategy.go simply layers a tier-selector on top.
|
|
package chunker
|
|
|
|
import "github.com/Tencent/WeKnora/internal/logger"
|
|
|
|
// Strategy values for SplitterConfig.Strategy.
|
|
const (
|
|
StrategyAuto = "auto"
|
|
StrategyHeading = "heading"
|
|
StrategyHeuristic = "heuristic"
|
|
StrategyRecursive = "recursive"
|
|
StrategyLegacy = "legacy"
|
|
)
|
|
|
|
// Split chunks text using the strategy configured in cfg. When cfg.Strategy
|
|
// is empty or "auto" the document profiler picks the tier. The function
|
|
// always returns a non-nil result: on tier failure the chain falls through
|
|
// to the legacy splitter, which is the original Tier 3 implementation.
|
|
func Split(text string, cfg SplitterConfig) []Chunk {
|
|
if text == "" {
|
|
return nil
|
|
}
|
|
cfg = ensureDefaults(cfg)
|
|
|
|
chain := resolveChain(text, cfg)
|
|
totalChars := len([]rune(text))
|
|
|
|
for _, tier := range chain {
|
|
out := runTier(tier, text, cfg)
|
|
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
|
|
return out
|
|
} else {
|
|
logger.Debugf(nil, "chunker: tier %s rejected: %s", tier, v.Reason)
|
|
}
|
|
}
|
|
// Last-ditch fallback: always return *something*.
|
|
return SplitText(text, cfg)
|
|
}
|
|
|
|
// SplitParentChild is the strategy-aware analog of SplitTextParentChild.
|
|
// It runs the tier selector for parent splitting, then re-splits each
|
|
// parent into children with the small-chunk config.
|
|
func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
|
|
if text == "" {
|
|
return ParentChildResult{}
|
|
}
|
|
parentCfg = ensureDefaults(parentCfg)
|
|
childCfg = ensureDefaults(childCfg)
|
|
|
|
parents := Split(text, parentCfg)
|
|
if len(parents) == 0 {
|
|
return ParentChildResult{}
|
|
}
|
|
|
|
var newParents []Chunk
|
|
var children []ChildChunk
|
|
childSeq := 0
|
|
for _, parent := range parents {
|
|
subs := Split(parent.Content, childCfg)
|
|
|
|
parentIndex := -1
|
|
if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) {
|
|
parentIndex = len(newParents)
|
|
newParents = append(newParents, parent)
|
|
}
|
|
for _, sub := range subs {
|
|
sub.Seq = childSeq
|
|
sub.Start += parent.Start
|
|
sub.End += parent.Start
|
|
children = append(children, ChildChunk{Chunk: sub, ParentIndex: parentIndex})
|
|
childSeq++
|
|
}
|
|
}
|
|
return ParentChildResult{Parents: newParents, Children: children}
|
|
}
|
|
|
|
// resolveChain returns the strategy chain to attempt. An explicit non-auto
|
|
// strategy bypasses the profiler entirely and pins to the requested tier.
|
|
func resolveChain(text string, cfg SplitterConfig) []StrategyTier {
|
|
switch cfg.Strategy {
|
|
case StrategyHeading:
|
|
return []StrategyTier{TierHeading, TierLegacy}
|
|
case StrategyHeuristic:
|
|
return []StrategyTier{TierHeuristic, TierLegacy}
|
|
case StrategyRecursive:
|
|
return []StrategyTier{TierRecursive, TierLegacy}
|
|
case StrategyLegacy, "":
|
|
// Empty == legacy preserves backwards compatibility with stored
|
|
// ChunkingConfig rows that pre-date the Strategy field.
|
|
return []StrategyTier{TierLegacy}
|
|
case StrategyAuto:
|
|
fallthrough
|
|
default:
|
|
profile := ProfileDocument(text)
|
|
return SelectStrategy(profile)
|
|
}
|
|
}
|
|
|
|
// runTier dispatches the splitter implementation for the given tier.
|
|
// Heading and heuristic tiers are stubbed in this scaffold and currently
|
|
// fall through to the legacy splitter — they are filled in in later phases.
|
|
func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk {
|
|
switch tier {
|
|
case TierHeading:
|
|
return splitByHeadings(text, cfg)
|
|
case TierHeuristic:
|
|
return splitByHeuristics(text, cfg)
|
|
case TierRecursive, TierLegacy:
|
|
return SplitText(text, cfg)
|
|
}
|
|
return SplitText(text, cfg)
|
|
}
|
|
|
|
// ensureDefaults fills in zero-value config fields with sane defaults.
|
|
// Mirrors buildSplitterConfig in internal/application/service/knowledge.go
|
|
// so direct callers of this package get the same numbers.
|
|
func ensureDefaults(cfg SplitterConfig) SplitterConfig {
|
|
if cfg.ChunkSize <= 0 {
|
|
cfg.ChunkSize = DefaultChunkSize
|
|
}
|
|
if cfg.ChunkOverlap <= 0 {
|
|
cfg.ChunkOverlap = DefaultChunkOverlap
|
|
}
|
|
if len(cfg.Separators) == 0 {
|
|
cfg.Separators = []string{"\n\n", "\n", "。"}
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
// splitByHeadings is overridden by heading_splitter.go.
|
|
var splitByHeadings = func(text string, cfg SplitterConfig) []Chunk {
|
|
return SplitText(text, cfg)
|
|
}
|
|
|
|
// splitByHeuristics is overridden by heuristic_splitter.go.
|
|
var splitByHeuristics = func(text string, cfg SplitterConfig) []Chunk {
|
|
return SplitText(text, cfg)
|
|
}
|