Claude
ae6fde2310
Adds the building blocks for tiered chunking strategies — multilingual patterns, token approximation, document profiling, validation, and a strategy resolver that today still falls through to the legacy SplitText. Default Strategy is empty/legacy so existing knowledge bases continue to chunk identically. - tokens.go: char/token approximation per language (en/de/zh/mixed) and a coarse DetectLanguage heuristic - patterns.go: source-of-truth regex bibliothek for headings, numbered sections, multilingual chapter markers, visual separators, etc. - profiler.go: single-pass document scan + SelectStrategy chain - validator.go: rejects obviously broken chunk-sets to drive fallbacks - strategy.go: public Split / SplitParentChild entry points; resolves StrategyAuto via the profiler, anything else maps to a fixed tier - splitter.go: SplitterConfig gains Strategy / TokenLimit / Languages fields and DefaultChunkSize / DefaultChunkOverlap constants (overlap raised to 80 ≈ 15%) - knowledge.go: switches both call sites to chunker.Split / SplitParentChild — behavior unchanged for empty Strategy https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
76 lines
2.3 KiB
Go
76 lines
2.3 KiB
Go
// Package chunker - validator.go inspects a tier's output and decides whether
|
|
// it is good enough to ship or whether the strategy chain should fall through
|
|
// to the next tier. The validator is intentionally permissive: a single
|
|
// "obviously broken" output is rejected, but plausible-looking variation is
|
|
// accepted so we don't oscillate between tiers.
|
|
package chunker
|
|
|
|
import "math"
|
|
|
|
// ValidationResult captures the verdict and reason for a chunk-set.
|
|
type ValidationResult struct {
|
|
OK bool
|
|
Reason string
|
|
}
|
|
|
|
// ValidateChunks checks whether the given chunks form a usable result for a
|
|
// document of totalChars characters with a target chunkSize. Returns OK=true
|
|
// when no broken-output indicator triggers.
|
|
func ValidateChunks(chunks []Chunk, totalChars, chunkSize int) ValidationResult {
|
|
if len(chunks) == 0 {
|
|
return ValidationResult{Reason: "no chunks produced"}
|
|
}
|
|
|
|
// A single chunk for a document much larger than chunkSize means the
|
|
// strategy did not actually split — fail so the next tier runs.
|
|
if len(chunks) == 1 && totalChars > 2*chunkSize {
|
|
return ValidationResult{Reason: "single chunk for large document"}
|
|
}
|
|
|
|
// Compute size statistics.
|
|
var sum, sumSq float64
|
|
maxLen, minLen := 0, math.MaxInt32
|
|
for _, c := range chunks {
|
|
l := len([]rune(c.Content))
|
|
sum += float64(l)
|
|
sumSq += float64(l * l)
|
|
if l > maxLen {
|
|
maxLen = l
|
|
}
|
|
if l < minLen {
|
|
minLen = l
|
|
}
|
|
}
|
|
avg := sum / float64(len(chunks))
|
|
|
|
// All but the last chunk should carry meaningful content. We allow the
|
|
// last chunk to be tiny because tail residue is normal.
|
|
tinyCount := 0
|
|
for i, c := range chunks {
|
|
if i == len(chunks)-1 {
|
|
continue
|
|
}
|
|
if len([]rune(c.Content)) < 50 {
|
|
tinyCount++
|
|
}
|
|
}
|
|
if tinyCount > len(chunks)/4 && tinyCount > 2 {
|
|
return ValidationResult{Reason: "too many tiny chunks"}
|
|
}
|
|
|
|
// Reject when no chunk reached at least 25% of the target — the splitter
|
|
// is fragmenting too aggressively to be useful.
|
|
if maxLen < chunkSize/4 && totalChars > chunkSize {
|
|
return ValidationResult{Reason: "all chunks far below target size"}
|
|
}
|
|
|
|
// Sanity check on absolute upper bound. Anything past 2x chunkSize is a
|
|
// red flag — the splitter ignored its size budget.
|
|
if maxLen > 2*chunkSize && chunkSize > 0 {
|
|
return ValidationResult{Reason: "chunk exceeds 2x target size"}
|
|
}
|
|
|
|
_ = avg
|
|
return ValidationResult{OK: true}
|
|
}
|