ae6fde2310
Adds the building blocks for tiered chunking strategies — multilingual patterns, token approximation, document profiling, validation, and a strategy resolver that today still falls through to the legacy SplitText. Default Strategy is empty/legacy so existing knowledge bases continue to chunk identically. - tokens.go: char/token approximation per language (en/de/zh/mixed) and a coarse DetectLanguage heuristic - patterns.go: source-of-truth regex bibliothek for headings, numbered sections, multilingual chapter markers, visual separators, etc. - profiler.go: single-pass document scan + SelectStrategy chain - validator.go: rejects obviously broken chunk-sets to drive fallbacks - strategy.go: public Split / SplitParentChild entry points; resolves StrategyAuto via the profiler, anything else maps to a fixed tier - splitter.go: SplitterConfig gains Strategy / TokenLimit / Languages fields and DefaultChunkSize / DefaultChunkOverlap constants (overlap raised to 80 ≈ 15%) - knowledge.go: switches both call sites to chunker.Split / SplitParentChild — behavior unchanged for empty Strategy https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
236 lines
6.3 KiB
Go
236 lines
6.3 KiB
Go
// Package chunker - profiler.go scans a document once to gather structure
|
|
// indicators that drive strategy selection (heading-aware vs. heuristic vs.
|
|
// recursive). Profiling is cheap (a few regex passes plus rune counting)
|
|
// and runs before any chunking decision is made.
|
|
package chunker
|
|
|
|
import (
|
|
"math"
|
|
"strings"
|
|
)
|
|
|
|
// DocProfile holds the document-level signals used to choose a chunking tier.
|
|
type DocProfile struct {
|
|
TotalChars int
|
|
TotalLines int
|
|
AvgLineLen float64
|
|
StdLineLen float64
|
|
|
|
// Markdown structure
|
|
MdHeadingCounts map[int]int // level (1..6) → count
|
|
MdHeadingTotal int
|
|
|
|
// Heuristic indicators
|
|
NumberedSectionCount int
|
|
AllCapsShortLineCount int
|
|
BlankParagraphBreaks int
|
|
FormFeedCount int
|
|
VisualSepCount int
|
|
GermanChapterCount int
|
|
EnglishChapterCount int
|
|
ChineseChapterCount int
|
|
RepeatedFooterCount int
|
|
|
|
// Content characteristics
|
|
HasTables bool
|
|
HasCode bool
|
|
CodeRatio float64
|
|
|
|
// Detected language hints (best-effort)
|
|
DetectedLangs []string
|
|
}
|
|
|
|
// HeadingDensity returns the share of lines that are Markdown headings.
|
|
func (p *DocProfile) HeadingDensity() float64 {
|
|
if p.TotalLines == 0 {
|
|
return 0
|
|
}
|
|
return float64(p.MdHeadingTotal) / float64(p.TotalLines)
|
|
}
|
|
|
|
// DominantHeadingLevel returns the heading level (1..6) that should drive
|
|
// section splitting. It picks the lowest level (closest to root) that has
|
|
// at least 3 occurrences, falling back to the most frequent level. Returns
|
|
// 0 when no usable Markdown structure exists.
|
|
func (p *DocProfile) DominantHeadingLevel() int {
|
|
if p.MdHeadingTotal == 0 {
|
|
return 0
|
|
}
|
|
for level := 1; level <= 6; level++ {
|
|
if p.MdHeadingCounts[level] >= 3 {
|
|
return level
|
|
}
|
|
}
|
|
// fallback: most frequent level
|
|
bestLevel, bestCount := 0, 0
|
|
for level := 1; level <= 6; level++ {
|
|
if p.MdHeadingCounts[level] > bestCount {
|
|
bestLevel, bestCount = level, p.MdHeadingCounts[level]
|
|
}
|
|
}
|
|
return bestLevel
|
|
}
|
|
|
|
// HeuristicMarkerTotal sums the non-Markdown structural markers.
|
|
func (p *DocProfile) HeuristicMarkerTotal() int {
|
|
return p.NumberedSectionCount +
|
|
p.GermanChapterCount + p.EnglishChapterCount + p.ChineseChapterCount +
|
|
p.AllCapsShortLineCount + p.VisualSepCount + p.FormFeedCount
|
|
}
|
|
|
|
// ProfileDocument runs a single pass over text and returns its profile.
|
|
func ProfileDocument(text string) *DocProfile {
|
|
p := &DocProfile{
|
|
MdHeadingCounts: make(map[int]int),
|
|
}
|
|
if text == "" {
|
|
return p
|
|
}
|
|
|
|
p.TotalChars = len([]rune(text))
|
|
p.FormFeedCount = strings.Count(text, "\f")
|
|
|
|
lines := strings.Split(text, "\n")
|
|
p.TotalLines = len(lines)
|
|
|
|
// First pass: per-line markers and length stats
|
|
var lengths []float64
|
|
inFence := false
|
|
codeChars := 0
|
|
for _, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
|
|
// Toggle fenced-code state. We use a 3-backtick prefix detector here
|
|
// rather than a full regex so we don't have to fight with the
|
|
// protected-pattern logic later.
|
|
if strings.HasPrefix(trimmed, "```") {
|
|
inFence = !inFence
|
|
p.HasCode = true
|
|
continue
|
|
}
|
|
if inFence {
|
|
codeChars += len([]rune(line))
|
|
continue
|
|
}
|
|
|
|
runeLen := len([]rune(line))
|
|
lengths = append(lengths, float64(runeLen))
|
|
|
|
if matchHeading(line, &p.MdHeadingCounts) {
|
|
p.MdHeadingTotal++
|
|
continue
|
|
}
|
|
if NumberedSectionPattern.MatchString(line) {
|
|
p.NumberedSectionCount++
|
|
}
|
|
if GermanChapterPattern.MatchString(line) {
|
|
p.GermanChapterCount++
|
|
}
|
|
if EnglishChapterPattern.MatchString(line) {
|
|
p.EnglishChapterCount++
|
|
}
|
|
if ChineseChapterPattern.MatchString(line) {
|
|
p.ChineseChapterCount++
|
|
}
|
|
if AllCapsHeadingPattern.MatchString(line) {
|
|
p.AllCapsShortLineCount++
|
|
}
|
|
if VisualSeparatorPattern.MatchString(line) {
|
|
p.VisualSepCount++
|
|
}
|
|
if PageFooterPattern.MatchString(line) {
|
|
p.RepeatedFooterCount++
|
|
}
|
|
if strings.HasPrefix(trimmed, "|") && strings.HasSuffix(trimmed, "|") {
|
|
p.HasTables = true
|
|
}
|
|
}
|
|
|
|
if len(lengths) > 0 {
|
|
var sum float64
|
|
for _, l := range lengths {
|
|
sum += l
|
|
}
|
|
p.AvgLineLen = sum / float64(len(lengths))
|
|
var variance float64
|
|
for _, l := range lengths {
|
|
d := l - p.AvgLineLen
|
|
variance += d * d
|
|
}
|
|
variance /= float64(len(lengths))
|
|
p.StdLineLen = math.Sqrt(variance)
|
|
}
|
|
|
|
if p.TotalChars > 0 {
|
|
p.CodeRatio = float64(codeChars) / float64(p.TotalChars)
|
|
}
|
|
|
|
p.BlankParagraphBreaks = strings.Count(text, "\n\n\n")
|
|
|
|
// Sample a slice of the document for language detection — avoids paying
|
|
// O(N) scan cost on huge inputs while still giving a stable signal.
|
|
sample := text
|
|
if len(sample) > 4096 {
|
|
sample = sample[:4096]
|
|
}
|
|
lang := DetectLanguage(sample)
|
|
p.DetectedLangs = []string{lang}
|
|
if lang == LangMixed {
|
|
// Provide all three for downstream pattern selection.
|
|
p.DetectedLangs = []string{LangEnglish, LangGerman, LangChinese}
|
|
}
|
|
|
|
return p
|
|
}
|
|
|
|
// matchHeading checks whether line is an ATX heading and increments the
|
|
// appropriate level counter when so. Returns true on match.
|
|
func matchHeading(line string, counts *map[int]int) bool {
|
|
m := MarkdownHeadingPattern.FindStringSubmatch(line)
|
|
if m == nil {
|
|
return false
|
|
}
|
|
level := len(m[1])
|
|
if level < 1 || level > 6 {
|
|
return false
|
|
}
|
|
(*counts)[level]++
|
|
return true
|
|
}
|
|
|
|
// StrategyTier identifies which chunking implementation should run.
|
|
type StrategyTier string
|
|
|
|
const (
|
|
TierHeading StrategyTier = "heading"
|
|
TierHeuristic StrategyTier = "heuristic"
|
|
TierRecursive StrategyTier = "recursive"
|
|
TierLegacy StrategyTier = "legacy"
|
|
)
|
|
|
|
// SelectStrategy returns the ordered tier chain to attempt for this document.
|
|
// The first tier is the primary choice; subsequent tiers are fallbacks if
|
|
// validation rejects the previous output. The "legacy" tier is appended as
|
|
// a final safety net so callers always receive at least one chunk-set.
|
|
func SelectStrategy(p *DocProfile) []StrategyTier {
|
|
if p == nil {
|
|
return []StrategyTier{TierRecursive, TierLegacy}
|
|
}
|
|
var chain []StrategyTier
|
|
|
|
// Tier 1 candidate: Markdown heading-aware
|
|
if p.MdHeadingTotal >= 3 && p.HeadingDensity() > 0.005 && p.DominantHeadingLevel() > 0 {
|
|
chain = append(chain, TierHeading)
|
|
}
|
|
|
|
// Tier 2 candidate: heuristic boundary detection
|
|
if p.HeuristicMarkerTotal() >= 5 || p.FormFeedCount > 0 ||
|
|
p.GermanChapterCount+p.EnglishChapterCount+p.ChineseChapterCount > 0 {
|
|
chain = append(chain, TierHeuristic)
|
|
}
|
|
|
|
// Always end with recursive (Tier 3) and then legacy as ultimate fallback.
|
|
chain = append(chain, TierRecursive, TierLegacy)
|
|
return chain
|
|
}
|