bc44c31cec
Resolves the issues surfaced by a self-review of the adaptive chunking work. Tests covering each regression were added first to prove the fix. Critical - Heading splitter no longer prepends breadcrumb to Content. The breadcrumb now lives on Chunk.ContextHeader; EmbeddingContent() combines header + content for the embedder. This restores the End-Start == len(Content) invariant relied on by knowledge.go's document-reconstruction path (summary generation, UI highlight) and also eliminates the duplicate-heading regression where the section heading appeared twice in a chunk's body. - splitBySeparators is now genuinely recursive: when one separator leaves a piece > ChunkSize, the next-priority separator is applied inside that piece. Mirrors the Python reference. Without this, a document with one paragraph break followed by a long run of newline-separated lines could emit a single ~1950-rune chunk for ChunkSize=300. Medium - SplitParentChild forces children onto the recursive tier, skipping the per-parent profile pass (was N extra O(N) scans). - Heuristic splitter snaps overlap start to the nearest semantic boundary or newline instead of slicing mid-line / mid-word. - Strategy.Split returns the legacy tier's output directly when the whole chain rejects rather than running SplitText a second time. Low - ensureDefaults caps ChunkOverlap at ChunkSize/2 so pathological configs (TokenLimit clamping ChunkSize tiny while Overlap stays high) cannot produce ~97% overlap chunks. - DefaultChunkOverlap docstring spells out the migration story for KBs that stored ChunkOverlap=0. ContextHeader plumbing - chunker.Chunk and types.ParsedChunk gain ContextHeader plus an EmbeddingContent() helper. - types.Chunk gets a transient (gorm:"-") ContextHeader field so the knowledge service can build embedding inputs as titlePrefix + chunk.EmbeddingContent() without a DB schema change. https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
196 lines
6.6 KiB
Go
196 lines
6.6 KiB
Go
// Package chunker - strategy.go is the public entry point for adaptive
|
|
// chunking. Callers invoke Split / SplitParentChild instead of the legacy
|
|
// SplitText / SplitTextParentChild functions; the strategy resolver picks
|
|
// a tier based on document profile and the SplitterConfig.Strategy hint.
|
|
//
|
|
// The legacy entry points still exist in splitter.go for backwards
|
|
// compatibility — strategy.go simply layers a tier-selector on top.
|
|
package chunker
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
)
|
|
|
|
// Strategy values for SplitterConfig.Strategy.
|
|
const (
|
|
StrategyAuto = "auto"
|
|
StrategyHeading = "heading"
|
|
StrategyHeuristic = "heuristic"
|
|
StrategyRecursive = "recursive"
|
|
StrategyLegacy = "legacy"
|
|
)
|
|
|
|
// Split chunks text using the strategy configured in cfg. When cfg.Strategy
|
|
// is empty or "auto" the document profiler picks the tier. The function
|
|
// always returns a non-nil result: on tier failure the chain falls through
|
|
// to the legacy splitter, which is the original Tier 3 implementation.
|
|
func Split(text string, cfg SplitterConfig) []Chunk {
|
|
if text == "" {
|
|
return nil
|
|
}
|
|
cfg = ensureDefaults(cfg)
|
|
|
|
chain := resolveChain(text, cfg)
|
|
totalChars := len([]rune(text))
|
|
|
|
var lastOut []Chunk
|
|
for i, tier := range chain {
|
|
out := runTier(tier, text, cfg)
|
|
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
|
|
return out
|
|
} else {
|
|
logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason)
|
|
}
|
|
// Remember the legacy tier's output: we'll return it as-is below if
|
|
// every tier rejected — running SplitText again would just produce
|
|
// the same rejected result.
|
|
if tier == TierLegacy && i == len(chain)-1 {
|
|
lastOut = out
|
|
}
|
|
}
|
|
if lastOut != nil {
|
|
return lastOut
|
|
}
|
|
// Defensive last-ditch fallback (only reached if the chain didn't end on TierLegacy).
|
|
return SplitText(text, cfg)
|
|
}
|
|
|
|
// SplitParentChild is the strategy-aware analog of SplitTextParentChild.
|
|
// It runs the tier selector for parent splitting, then re-splits each
|
|
// parent into children with the small-chunk config.
|
|
//
|
|
// Children are forced to recursive splitting — they are sub-pieces of an
|
|
// already-segmented parent, so re-profiling per parent would cost N extra
|
|
// O(N) document scans without any real chance of picking a better tier.
|
|
func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
|
|
if text == "" {
|
|
return ParentChildResult{}
|
|
}
|
|
parentCfg = ensureDefaults(parentCfg)
|
|
childCfg = ensureDefaults(childCfg)
|
|
|
|
parents := Split(text, parentCfg)
|
|
if len(parents) == 0 {
|
|
return ParentChildResult{}
|
|
}
|
|
|
|
// Pin children to the recursive tier so SplitText runs directly without
|
|
// the profiler/strategy chain re-deciding per parent. Preserves Tier-1
|
|
// breadcrumb context (already attached to each parent) since we only
|
|
// re-split parent content, not re-detect headings.
|
|
childCfg.Strategy = StrategyRecursive
|
|
|
|
var newParents []Chunk
|
|
var children []ChildChunk
|
|
childSeq := 0
|
|
for _, parent := range parents {
|
|
subs := Split(parent.Content, childCfg)
|
|
|
|
parentIndex := -1
|
|
if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) {
|
|
parentIndex = len(newParents)
|
|
newParents = append(newParents, parent)
|
|
}
|
|
for _, sub := range subs {
|
|
sub.Seq = childSeq
|
|
sub.Start += parent.Start
|
|
sub.End += parent.Start
|
|
// Children inherit the parent's breadcrumb so embedding still
|
|
// sees the section context — but only if the child itself does
|
|
// not already carry one (sub-splits don't, but defensive).
|
|
if sub.ContextHeader == "" {
|
|
sub.ContextHeader = parent.ContextHeader
|
|
}
|
|
children = append(children, ChildChunk{Chunk: sub, ParentIndex: parentIndex})
|
|
childSeq++
|
|
}
|
|
}
|
|
return ParentChildResult{Parents: newParents, Children: children}
|
|
}
|
|
|
|
// resolveChain returns the strategy chain to attempt. An explicit non-auto
|
|
// strategy bypasses the profiler entirely and pins to the requested tier.
|
|
func resolveChain(text string, cfg SplitterConfig) []StrategyTier {
|
|
switch cfg.Strategy {
|
|
case StrategyHeading:
|
|
return []StrategyTier{TierHeading, TierLegacy}
|
|
case StrategyHeuristic:
|
|
return []StrategyTier{TierHeuristic, TierLegacy}
|
|
case StrategyRecursive:
|
|
return []StrategyTier{TierRecursive, TierLegacy}
|
|
case StrategyLegacy, "":
|
|
// Empty == legacy preserves backwards compatibility with stored
|
|
// ChunkingConfig rows that pre-date the Strategy field.
|
|
return []StrategyTier{TierLegacy}
|
|
case StrategyAuto:
|
|
fallthrough
|
|
default:
|
|
profile := ProfileDocument(text)
|
|
return SelectStrategy(profile)
|
|
}
|
|
}
|
|
|
|
// runTier dispatches the splitter implementation for the given tier.
|
|
// Heading and heuristic tiers are stubbed in this scaffold and currently
|
|
// fall through to the legacy splitter — they are filled in in later phases.
|
|
func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk {
|
|
switch tier {
|
|
case TierHeading:
|
|
return splitByHeadings(text, cfg)
|
|
case TierHeuristic:
|
|
return splitByHeuristics(text, cfg)
|
|
case TierRecursive, TierLegacy:
|
|
return SplitText(text, cfg)
|
|
}
|
|
return SplitText(text, cfg)
|
|
}
|
|
|
|
// ensureDefaults fills in zero-value config fields with sane defaults.
|
|
// Mirrors buildSplitterConfig in internal/application/service/knowledge.go
|
|
// so direct callers of this package get the same numbers.
|
|
//
|
|
// When cfg.TokenLimit is set, ChunkSize is clamped to the character budget
|
|
// that fits within that token limit (with a 10% safety factor). This makes
|
|
// chunks safe for embedding APIs that have hard token caps.
|
|
func ensureDefaults(cfg SplitterConfig) SplitterConfig {
|
|
if cfg.ChunkSize <= 0 {
|
|
cfg.ChunkSize = DefaultChunkSize
|
|
}
|
|
if cfg.ChunkOverlap <= 0 {
|
|
cfg.ChunkOverlap = DefaultChunkOverlap
|
|
}
|
|
if len(cfg.Separators) == 0 {
|
|
cfg.Separators = []string{"\n\n", "\n", "。"}
|
|
}
|
|
if cfg.TokenLimit > 0 {
|
|
lang := LangMixed
|
|
if len(cfg.Languages) > 0 {
|
|
lang = cfg.Languages[0]
|
|
}
|
|
charBudget := CharsForTokenLimit(cfg.TokenLimit, lang)
|
|
if charBudget > 0 && (cfg.ChunkSize == 0 || charBudget < cfg.ChunkSize) {
|
|
cfg.ChunkSize = charBudget
|
|
}
|
|
}
|
|
// Guard against pathological overlap configurations: if Overlap exceeds
|
|
// half of ChunkSize, almost every chunk is duplicate content. Cap it at
|
|
// ChunkSize/2 so Overlap stays a useful smoothing band rather than a
|
|
// near-clone of the previous chunk.
|
|
if cfg.ChunkOverlap > cfg.ChunkSize/2 && cfg.ChunkSize > 0 {
|
|
cfg.ChunkOverlap = cfg.ChunkSize / 2
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
// splitByHeadings is overridden by heading_splitter.go.
|
|
var splitByHeadings = func(text string, cfg SplitterConfig) []Chunk {
|
|
return SplitText(text, cfg)
|
|
}
|
|
|
|
// splitByHeuristics is overridden by heuristic_splitter.go.
|
|
var splitByHeuristics = func(text string, cfg SplitterConfig) []Chunk {
|
|
return SplitText(text, cfg)
|
|
}
|