Files
Enterprise-Knowledge-Base/internal/infrastructure/chunker/profiler.go
T
Claude ae6fde2310 feat(chunker): foundation for adaptive 3-tier chunking
Adds the building blocks for tiered chunking strategies — multilingual
patterns, token approximation, document profiling, validation, and a
strategy resolver that today still falls through to the legacy
SplitText. Default Strategy is empty/legacy so existing knowledge bases
continue to chunk identically.

- tokens.go: char/token approximation per language (en/de/zh/mixed)
  and a coarse DetectLanguage heuristic
- patterns.go: source-of-truth regex bibliothek for headings, numbered
  sections, multilingual chapter markers, visual separators, etc.
- profiler.go: single-pass document scan + SelectStrategy chain
- validator.go: rejects obviously broken chunk-sets to drive fallbacks
- strategy.go: public Split / SplitParentChild entry points; resolves
  StrategyAuto via the profiler, anything else maps to a fixed tier
- splitter.go: SplitterConfig gains Strategy / TokenLimit / Languages
  fields and DefaultChunkSize / DefaultChunkOverlap constants (overlap
  raised to 80 ≈ 15%)
- knowledge.go: switches both call sites to chunker.Split /
  SplitParentChild — behavior unchanged for empty Strategy

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
2026-05-06 17:17:07 +08:00

236 lines
6.3 KiB
Go

// Package chunker - profiler.go scans a document once to gather structure
// indicators that drive strategy selection (heading-aware vs. heuristic vs.
// recursive). Profiling is cheap (a few regex passes plus rune counting)
// and runs before any chunking decision is made.
package chunker
import (
"math"
"strings"
)
// DocProfile holds the document-level signals used to choose a chunking tier.
type DocProfile struct {
TotalChars int
TotalLines int
AvgLineLen float64
StdLineLen float64
// Markdown structure
MdHeadingCounts map[int]int // level (1..6) → count
MdHeadingTotal int
// Heuristic indicators
NumberedSectionCount int
AllCapsShortLineCount int
BlankParagraphBreaks int
FormFeedCount int
VisualSepCount int
GermanChapterCount int
EnglishChapterCount int
ChineseChapterCount int
RepeatedFooterCount int
// Content characteristics
HasTables bool
HasCode bool
CodeRatio float64
// Detected language hints (best-effort)
DetectedLangs []string
}
// HeadingDensity returns the share of lines that are Markdown headings.
func (p *DocProfile) HeadingDensity() float64 {
if p.TotalLines == 0 {
return 0
}
return float64(p.MdHeadingTotal) / float64(p.TotalLines)
}
// DominantHeadingLevel returns the heading level (1..6) that should drive
// section splitting. It picks the lowest level (closest to root) that has
// at least 3 occurrences, falling back to the most frequent level. Returns
// 0 when no usable Markdown structure exists.
func (p *DocProfile) DominantHeadingLevel() int {
if p.MdHeadingTotal == 0 {
return 0
}
for level := 1; level <= 6; level++ {
if p.MdHeadingCounts[level] >= 3 {
return level
}
}
// fallback: most frequent level
bestLevel, bestCount := 0, 0
for level := 1; level <= 6; level++ {
if p.MdHeadingCounts[level] > bestCount {
bestLevel, bestCount = level, p.MdHeadingCounts[level]
}
}
return bestLevel
}
// HeuristicMarkerTotal sums the non-Markdown structural markers.
func (p *DocProfile) HeuristicMarkerTotal() int {
return p.NumberedSectionCount +
p.GermanChapterCount + p.EnglishChapterCount + p.ChineseChapterCount +
p.AllCapsShortLineCount + p.VisualSepCount + p.FormFeedCount
}
// ProfileDocument runs a single pass over text and returns its profile.
func ProfileDocument(text string) *DocProfile {
p := &DocProfile{
MdHeadingCounts: make(map[int]int),
}
if text == "" {
return p
}
p.TotalChars = len([]rune(text))
p.FormFeedCount = strings.Count(text, "\f")
lines := strings.Split(text, "\n")
p.TotalLines = len(lines)
// First pass: per-line markers and length stats
var lengths []float64
inFence := false
codeChars := 0
for _, line := range lines {
trimmed := strings.TrimSpace(line)
// Toggle fenced-code state. We use a 3-backtick prefix detector here
// rather than a full regex so we don't have to fight with the
// protected-pattern logic later.
if strings.HasPrefix(trimmed, "```") {
inFence = !inFence
p.HasCode = true
continue
}
if inFence {
codeChars += len([]rune(line))
continue
}
runeLen := len([]rune(line))
lengths = append(lengths, float64(runeLen))
if matchHeading(line, &p.MdHeadingCounts) {
p.MdHeadingTotal++
continue
}
if NumberedSectionPattern.MatchString(line) {
p.NumberedSectionCount++
}
if GermanChapterPattern.MatchString(line) {
p.GermanChapterCount++
}
if EnglishChapterPattern.MatchString(line) {
p.EnglishChapterCount++
}
if ChineseChapterPattern.MatchString(line) {
p.ChineseChapterCount++
}
if AllCapsHeadingPattern.MatchString(line) {
p.AllCapsShortLineCount++
}
if VisualSeparatorPattern.MatchString(line) {
p.VisualSepCount++
}
if PageFooterPattern.MatchString(line) {
p.RepeatedFooterCount++
}
if strings.HasPrefix(trimmed, "|") && strings.HasSuffix(trimmed, "|") {
p.HasTables = true
}
}
if len(lengths) > 0 {
var sum float64
for _, l := range lengths {
sum += l
}
p.AvgLineLen = sum / float64(len(lengths))
var variance float64
for _, l := range lengths {
d := l - p.AvgLineLen
variance += d * d
}
variance /= float64(len(lengths))
p.StdLineLen = math.Sqrt(variance)
}
if p.TotalChars > 0 {
p.CodeRatio = float64(codeChars) / float64(p.TotalChars)
}
p.BlankParagraphBreaks = strings.Count(text, "\n\n\n")
// Sample a slice of the document for language detection — avoids paying
// O(N) scan cost on huge inputs while still giving a stable signal.
sample := text
if len(sample) > 4096 {
sample = sample[:4096]
}
lang := DetectLanguage(sample)
p.DetectedLangs = []string{lang}
if lang == LangMixed {
// Provide all three for downstream pattern selection.
p.DetectedLangs = []string{LangEnglish, LangGerman, LangChinese}
}
return p
}
// matchHeading checks whether line is an ATX heading and increments the
// appropriate level counter when so. Returns true on match.
func matchHeading(line string, counts *map[int]int) bool {
m := MarkdownHeadingPattern.FindStringSubmatch(line)
if m == nil {
return false
}
level := len(m[1])
if level < 1 || level > 6 {
return false
}
(*counts)[level]++
return true
}
// StrategyTier identifies which chunking implementation should run.
type StrategyTier string
const (
TierHeading StrategyTier = "heading"
TierHeuristic StrategyTier = "heuristic"
TierRecursive StrategyTier = "recursive"
TierLegacy StrategyTier = "legacy"
)
// SelectStrategy returns the ordered tier chain to attempt for this document.
// The first tier is the primary choice; subsequent tiers are fallbacks if
// validation rejects the previous output. The "legacy" tier is appended as
// a final safety net so callers always receive at least one chunk-set.
func SelectStrategy(p *DocProfile) []StrategyTier {
if p == nil {
return []StrategyTier{TierRecursive, TierLegacy}
}
var chain []StrategyTier
// Tier 1 candidate: Markdown heading-aware
if p.MdHeadingTotal >= 3 && p.HeadingDensity() > 0.005 && p.DominantHeadingLevel() > 0 {
chain = append(chain, TierHeading)
}
// Tier 2 candidate: heuristic boundary detection
if p.HeuristicMarkerTotal() >= 5 || p.FormFeedCount > 0 ||
p.GermanChapterCount+p.EnglishChapterCount+p.ChineseChapterCount > 0 {
chain = append(chain, TierHeuristic)
}
// Always end with recursive (Tier 3) and then legacy as ultimate fallback.
chain = append(chain, TierRecursive, TierLegacy)
return chain
}