Files
Enterprise-Knowledge-Base/internal/infrastructure/chunker/strategy.go
T
Claude bc44c31cec fix(chunker): QA audit — position invariant, recursive split, perf
Resolves the issues surfaced by a self-review of the adaptive chunking
work. Tests covering each regression were added first to prove the
fix.

Critical
- Heading splitter no longer prepends breadcrumb to Content. The
  breadcrumb now lives on Chunk.ContextHeader; EmbeddingContent()
  combines header + content for the embedder. This restores the
  End-Start == len(Content) invariant relied on by knowledge.go's
  document-reconstruction path (summary generation, UI highlight) and
  also eliminates the duplicate-heading regression where the section
  heading appeared twice in a chunk's body.
- splitBySeparators is now genuinely recursive: when one separator
  leaves a piece > ChunkSize, the next-priority separator is applied
  inside that piece. Mirrors the Python reference. Without this, a
  document with one paragraph break followed by a long run of
  newline-separated lines could emit a single ~1950-rune chunk for
  ChunkSize=300.

Medium
- SplitParentChild forces children onto the recursive tier, skipping
  the per-parent profile pass (was N extra O(N) scans).
- Heuristic splitter snaps overlap start to the nearest semantic
  boundary or newline instead of slicing mid-line / mid-word.
- Strategy.Split returns the legacy tier's output directly when the
  whole chain rejects rather than running SplitText a second time.

Low
- ensureDefaults caps ChunkOverlap at ChunkSize/2 so pathological
  configs (TokenLimit clamping ChunkSize tiny while Overlap stays
  high) cannot produce ~97% overlap chunks.
- DefaultChunkOverlap docstring spells out the migration story for
  KBs that stored ChunkOverlap=0.

ContextHeader plumbing
- chunker.Chunk and types.ParsedChunk gain ContextHeader plus an
  EmbeddingContent() helper.
- types.Chunk gets a transient (gorm:"-") ContextHeader field so the
  knowledge service can build embedding inputs as
  titlePrefix + chunk.EmbeddingContent() without a DB schema change.

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
2026-05-06 17:17:07 +08:00

196 lines
6.6 KiB
Go

// Package chunker - strategy.go is the public entry point for adaptive
// chunking. Callers invoke Split / SplitParentChild instead of the legacy
// SplitText / SplitTextParentChild functions; the strategy resolver picks
// a tier based on document profile and the SplitterConfig.Strategy hint.
//
// The legacy entry points still exist in splitter.go for backwards
// compatibility — strategy.go simply layers a tier-selector on top.
package chunker
import (
"context"
"github.com/Tencent/WeKnora/internal/logger"
)
// Strategy values for SplitterConfig.Strategy.
const (
StrategyAuto = "auto"
StrategyHeading = "heading"
StrategyHeuristic = "heuristic"
StrategyRecursive = "recursive"
StrategyLegacy = "legacy"
)
// Split chunks text using the strategy configured in cfg. When cfg.Strategy
// is empty or "auto" the document profiler picks the tier. The function
// always returns a non-nil result: on tier failure the chain falls through
// to the legacy splitter, which is the original Tier 3 implementation.
func Split(text string, cfg SplitterConfig) []Chunk {
if text == "" {
return nil
}
cfg = ensureDefaults(cfg)
chain := resolveChain(text, cfg)
totalChars := len([]rune(text))
var lastOut []Chunk
for i, tier := range chain {
out := runTier(tier, text, cfg)
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
return out
} else {
logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason)
}
// Remember the legacy tier's output: we'll return it as-is below if
// every tier rejected — running SplitText again would just produce
// the same rejected result.
if tier == TierLegacy && i == len(chain)-1 {
lastOut = out
}
}
if lastOut != nil {
return lastOut
}
// Defensive last-ditch fallback (only reached if the chain didn't end on TierLegacy).
return SplitText(text, cfg)
}
// SplitParentChild is the strategy-aware analog of SplitTextParentChild.
// It runs the tier selector for parent splitting, then re-splits each
// parent into children with the small-chunk config.
//
// Children are forced to recursive splitting — they are sub-pieces of an
// already-segmented parent, so re-profiling per parent would cost N extra
// O(N) document scans without any real chance of picking a better tier.
func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
if text == "" {
return ParentChildResult{}
}
parentCfg = ensureDefaults(parentCfg)
childCfg = ensureDefaults(childCfg)
parents := Split(text, parentCfg)
if len(parents) == 0 {
return ParentChildResult{}
}
// Pin children to the recursive tier so SplitText runs directly without
// the profiler/strategy chain re-deciding per parent. Preserves Tier-1
// breadcrumb context (already attached to each parent) since we only
// re-split parent content, not re-detect headings.
childCfg.Strategy = StrategyRecursive
var newParents []Chunk
var children []ChildChunk
childSeq := 0
for _, parent := range parents {
subs := Split(parent.Content, childCfg)
parentIndex := -1
if len(subs) > 1 || (len(subs) == 1 && subs[0].Content != parent.Content) {
parentIndex = len(newParents)
newParents = append(newParents, parent)
}
for _, sub := range subs {
sub.Seq = childSeq
sub.Start += parent.Start
sub.End += parent.Start
// Children inherit the parent's breadcrumb so embedding still
// sees the section context — but only if the child itself does
// not already carry one (sub-splits don't, but defensive).
if sub.ContextHeader == "" {
sub.ContextHeader = parent.ContextHeader
}
children = append(children, ChildChunk{Chunk: sub, ParentIndex: parentIndex})
childSeq++
}
}
return ParentChildResult{Parents: newParents, Children: children}
}
// resolveChain returns the strategy chain to attempt. An explicit non-auto
// strategy bypasses the profiler entirely and pins to the requested tier.
func resolveChain(text string, cfg SplitterConfig) []StrategyTier {
switch cfg.Strategy {
case StrategyHeading:
return []StrategyTier{TierHeading, TierLegacy}
case StrategyHeuristic:
return []StrategyTier{TierHeuristic, TierLegacy}
case StrategyRecursive:
return []StrategyTier{TierRecursive, TierLegacy}
case StrategyLegacy, "":
// Empty == legacy preserves backwards compatibility with stored
// ChunkingConfig rows that pre-date the Strategy field.
return []StrategyTier{TierLegacy}
case StrategyAuto:
fallthrough
default:
profile := ProfileDocument(text)
return SelectStrategy(profile)
}
}
// runTier dispatches the splitter implementation for the given tier.
// Heading and heuristic tiers are stubbed in this scaffold and currently
// fall through to the legacy splitter — they are filled in in later phases.
func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk {
switch tier {
case TierHeading:
return splitByHeadings(text, cfg)
case TierHeuristic:
return splitByHeuristics(text, cfg)
case TierRecursive, TierLegacy:
return SplitText(text, cfg)
}
return SplitText(text, cfg)
}
// ensureDefaults fills in zero-value config fields with sane defaults.
// Mirrors buildSplitterConfig in internal/application/service/knowledge.go
// so direct callers of this package get the same numbers.
//
// When cfg.TokenLimit is set, ChunkSize is clamped to the character budget
// that fits within that token limit (with a 10% safety factor). This makes
// chunks safe for embedding APIs that have hard token caps.
func ensureDefaults(cfg SplitterConfig) SplitterConfig {
if cfg.ChunkSize <= 0 {
cfg.ChunkSize = DefaultChunkSize
}
if cfg.ChunkOverlap <= 0 {
cfg.ChunkOverlap = DefaultChunkOverlap
}
if len(cfg.Separators) == 0 {
cfg.Separators = []string{"\n\n", "\n", "。"}
}
if cfg.TokenLimit > 0 {
lang := LangMixed
if len(cfg.Languages) > 0 {
lang = cfg.Languages[0]
}
charBudget := CharsForTokenLimit(cfg.TokenLimit, lang)
if charBudget > 0 && (cfg.ChunkSize == 0 || charBudget < cfg.ChunkSize) {
cfg.ChunkSize = charBudget
}
}
// Guard against pathological overlap configurations: if Overlap exceeds
// half of ChunkSize, almost every chunk is duplicate content. Cap it at
// ChunkSize/2 so Overlap stays a useful smoothing band rather than a
// near-clone of the previous chunk.
if cfg.ChunkOverlap > cfg.ChunkSize/2 && cfg.ChunkSize > 0 {
cfg.ChunkOverlap = cfg.ChunkSize / 2
}
return cfg
}
// splitByHeadings is overridden by heading_splitter.go.
var splitByHeadings = func(text string, cfg SplitterConfig) []Chunk {
return SplitText(text, cfg)
}
// splitByHeuristics is overridden by heuristic_splitter.go.
var splitByHeuristics = func(text string, cfg SplitterConfig) []Chunk {
return SplitText(text, cfg)
}