Files
Enterprise-Knowledge-Base/internal/infrastructure/chunker/heading_splitter.go
T
Claude bc44c31cec fix(chunker): QA audit — position invariant, recursive split, perf
Resolves the issues surfaced by a self-review of the adaptive chunking
work. Tests covering each regression were added first to prove the
fix.

Critical
- Heading splitter no longer prepends breadcrumb to Content. The
  breadcrumb now lives on Chunk.ContextHeader; EmbeddingContent()
  combines header + content for the embedder. This restores the
  End-Start == len(Content) invariant relied on by knowledge.go's
  document-reconstruction path (summary generation, UI highlight) and
  also eliminates the duplicate-heading regression where the section
  heading appeared twice in a chunk's body.
- splitBySeparators is now genuinely recursive: when one separator
  leaves a piece > ChunkSize, the next-priority separator is applied
  inside that piece. Mirrors the Python reference. Without this, a
  document with one paragraph break followed by a long run of
  newline-separated lines could emit a single ~1950-rune chunk for
  ChunkSize=300.

Medium
- SplitParentChild forces children onto the recursive tier, skipping
  the per-parent profile pass (was N extra O(N) scans).
- Heuristic splitter snaps overlap start to the nearest semantic
  boundary or newline instead of slicing mid-line / mid-word.
- Strategy.Split returns the legacy tier's output directly when the
  whole chain rejects rather than running SplitText a second time.

Low
- ensureDefaults caps ChunkOverlap at ChunkSize/2 so pathological
  configs (TokenLimit clamping ChunkSize tiny while Overlap stays
  high) cannot produce ~97% overlap chunks.
- DefaultChunkOverlap docstring spells out the migration story for
  KBs that stored ChunkOverlap=0.

ContextHeader plumbing
- chunker.Chunk and types.ParsedChunk gain ContextHeader plus an
  EmbeddingContent() helper.
- types.Chunk gets a transient (gorm:"-") ContextHeader field so the
  knowledge service can build embedding inputs as
  titlePrefix + chunk.EmbeddingContent() without a DB schema change.

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
2026-05-06 17:17:07 +08:00

191 lines
5.6 KiB
Go

// Package chunker - heading_splitter.go implements Tier 1: Markdown
// heading-aware chunking. Documents with proper heading structure are split
// at heading boundaries and each chunk is prefixed with a breadcrumb of
// active heading context (e.g. "# Chapter 1\n## Section 1.2").
package chunker
import (
"strings"
"unicode/utf8"
)
// init wires this implementation into the strategy resolver.
func init() {
splitByHeadings = splitByHeadingsImpl
}
// headingBoundary marks where a section starts. The first boundary is at
// rune offset 0 (covers any preamble before the first heading), subsequent
// boundaries sit at headings whose level is <= primaryLevel.
type headingBoundary struct {
runeStart int
line string // raw heading line, "" when this is the leading boundary
}
// splitByHeadingsImpl is the Tier-1 implementation. It falls through to the
// legacy splitter when the document has no usable heading structure or when
// the heading split would produce a single section anyway.
func splitByHeadingsImpl(text string, cfg SplitterConfig) []Chunk {
if text == "" {
return nil
}
profile := ProfileDocument(text)
primaryLevel := profile.DominantHeadingLevel()
if primaryLevel == 0 {
return SplitText(text, cfg)
}
bounds := findHeadingBoundaries(text, primaryLevel)
if len(bounds) <= 1 {
return SplitText(text, cfg)
}
runes := []rune(text)
hierarchy := NewHeadingHierarchy()
// Pre-walk every heading (not just primary-level) so the hierarchy
// reflects the full nesting context for each section's start. We only
// snapshot the breadcrumb at section boundaries; deeper sub-headings
// inside a section update the hierarchy but do not change the chunk's
// breadcrumb (chunks within a section share one breadcrumb).
var out []Chunk
seq := 0
for i, b := range bounds {
endRune := len(runes)
if i+1 < len(bounds) {
endRune = bounds[i+1].runeStart
}
if b.line != "" {
hierarchy.Observe(b.line)
}
// Catch sub-headings that occur between this primary boundary and
// the next so the hierarchy stays in sync for subsequent sections.
// We intentionally do this after observing the section header so
// the breadcrumb reflects the section-leading heading.
breadcrumb := hierarchy.BreadcrumbWithHashes()
observeSubHeadings(runes[b.runeStart:endRune], primaryLevel, hierarchy)
sectionRunes := runes[b.runeStart:endRune]
sectionContent := string(sectionRunes)
secLen := len(sectionRunes)
if secLen == 0 {
continue
}
bcLen := utf8.RuneCountInString(breadcrumb)
// Single-chunk section: emit as-is, breadcrumb tracked separately.
// The breadcrumb is delivered via Chunk.ContextHeader (not Content)
// to preserve End-Start == len(Content) invariants relied on by
// document reconstruction (knowledge.go:2278+).
if bcLen+2+secLen <= cfg.ChunkSize {
out = append(out, Chunk{
Content: sectionContent,
ContextHeader: breadcrumb,
Seq: seq,
Start: b.runeStart,
End: endRune,
})
seq++
continue
}
// Section too large: defer to the legacy splitter for inner
// segmentation. Sub-chunks inherit the same breadcrumb via
// ContextHeader. We do NOT shrink the inner ChunkSize budget here
// because the breadcrumb no longer counts against Content size.
subChunks := SplitText(sectionContent, cfg)
for _, sub := range subChunks {
out = append(out, Chunk{
Content: sub.Content,
ContextHeader: breadcrumb,
Seq: seq,
Start: b.runeStart + sub.Start,
End: b.runeStart + sub.End,
})
seq++
}
}
return out
}
// findHeadingBoundaries returns one boundary at offset 0 plus one per
// Markdown heading at level <= primaryLevel that sits outside fenced code
// blocks. Heading detection is line-oriented — a heading must occupy a
// whole line to be recognized.
func findHeadingBoundaries(text string, primaryLevel int) []headingBoundary {
runes := []rune(text)
bounds := []headingBoundary{{runeStart: 0}}
if len(runes) == 0 {
return bounds
}
pos := 0
inFence := false
lines := strings.Split(text, "\n")
for i, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "```") {
inFence = !inFence
pos += utf8.RuneCountInString(line)
if i < len(lines)-1 {
pos++ // newline
}
continue
}
if !inFence {
m := MarkdownHeadingPattern.FindStringSubmatch(line)
if m != nil {
level := len(m[1])
if level >= 1 && level <= primaryLevel && pos > 0 {
bounds = append(bounds, headingBoundary{
runeStart: pos,
line: line,
})
}
if level >= 1 && level <= primaryLevel && pos == 0 {
// First line is a heading — replace the leading boundary
bounds[0].line = line
}
}
}
pos += utf8.RuneCountInString(line)
if i < len(lines)-1 {
pos++ // account for the \n that strings.Split removed
}
}
return bounds
}
// observeSubHeadings walks the section's lines and feeds every Markdown
// heading deeper than primaryLevel into the hierarchy. This keeps the
// hierarchy state correct so the breadcrumb at the next primary section
// reflects the truly active stack.
func observeSubHeadings(runes []rune, primaryLevel int, h *HeadingHierarchy) {
if len(runes) == 0 {
return
}
text := string(runes)
inFence := false
for _, line := range strings.Split(text, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "```") {
inFence = !inFence
continue
}
if inFence {
continue
}
m := MarkdownHeadingPattern.FindStringSubmatch(line)
if m == nil {
continue
}
level := len(m[1])
if level > primaryLevel {
h.Observe(line)
}
}
}