bc44c31cec
Resolves the issues surfaced by a self-review of the adaptive chunking work. Tests covering each regression were added first to prove the fix. Critical - Heading splitter no longer prepends breadcrumb to Content. The breadcrumb now lives on Chunk.ContextHeader; EmbeddingContent() combines header + content for the embedder. This restores the End-Start == len(Content) invariant relied on by knowledge.go's document-reconstruction path (summary generation, UI highlight) and also eliminates the duplicate-heading regression where the section heading appeared twice in a chunk's body. - splitBySeparators is now genuinely recursive: when one separator leaves a piece > ChunkSize, the next-priority separator is applied inside that piece. Mirrors the Python reference. Without this, a document with one paragraph break followed by a long run of newline-separated lines could emit a single ~1950-rune chunk for ChunkSize=300. Medium - SplitParentChild forces children onto the recursive tier, skipping the per-parent profile pass (was N extra O(N) scans). - Heuristic splitter snaps overlap start to the nearest semantic boundary or newline instead of slicing mid-line / mid-word. - Strategy.Split returns the legacy tier's output directly when the whole chain rejects rather than running SplitText a second time. Low - ensureDefaults caps ChunkOverlap at ChunkSize/2 so pathological configs (TokenLimit clamping ChunkSize tiny while Overlap stays high) cannot produce ~97% overlap chunks. - DefaultChunkOverlap docstring spells out the migration story for KBs that stored ChunkOverlap=0. ContextHeader plumbing - chunker.Chunk and types.ParsedChunk gain ContextHeader plus an EmbeddingContent() helper. - types.Chunk gets a transient (gorm:"-") ContextHeader field so the knowledge service can build embedding inputs as titlePrefix + chunk.EmbeddingContent() without a DB schema change. https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
191 lines
5.6 KiB
Go
191 lines
5.6 KiB
Go
// Package chunker - heading_splitter.go implements Tier 1: Markdown
|
|
// heading-aware chunking. Documents with proper heading structure are split
|
|
// at heading boundaries and each chunk is prefixed with a breadcrumb of
|
|
// active heading context (e.g. "# Chapter 1\n## Section 1.2").
|
|
package chunker
|
|
|
|
import (
|
|
"strings"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
// init wires this implementation into the strategy resolver.
|
|
func init() {
|
|
splitByHeadings = splitByHeadingsImpl
|
|
}
|
|
|
|
// headingBoundary marks where a section starts. The first boundary is at
|
|
// rune offset 0 (covers any preamble before the first heading), subsequent
|
|
// boundaries sit at headings whose level is <= primaryLevel.
|
|
type headingBoundary struct {
|
|
runeStart int
|
|
line string // raw heading line, "" when this is the leading boundary
|
|
}
|
|
|
|
// splitByHeadingsImpl is the Tier-1 implementation. It falls through to the
|
|
// legacy splitter when the document has no usable heading structure or when
|
|
// the heading split would produce a single section anyway.
|
|
func splitByHeadingsImpl(text string, cfg SplitterConfig) []Chunk {
|
|
if text == "" {
|
|
return nil
|
|
}
|
|
profile := ProfileDocument(text)
|
|
primaryLevel := profile.DominantHeadingLevel()
|
|
if primaryLevel == 0 {
|
|
return SplitText(text, cfg)
|
|
}
|
|
|
|
bounds := findHeadingBoundaries(text, primaryLevel)
|
|
if len(bounds) <= 1 {
|
|
return SplitText(text, cfg)
|
|
}
|
|
|
|
runes := []rune(text)
|
|
hierarchy := NewHeadingHierarchy()
|
|
|
|
// Pre-walk every heading (not just primary-level) so the hierarchy
|
|
// reflects the full nesting context for each section's start. We only
|
|
// snapshot the breadcrumb at section boundaries; deeper sub-headings
|
|
// inside a section update the hierarchy but do not change the chunk's
|
|
// breadcrumb (chunks within a section share one breadcrumb).
|
|
var out []Chunk
|
|
seq := 0
|
|
|
|
for i, b := range bounds {
|
|
endRune := len(runes)
|
|
if i+1 < len(bounds) {
|
|
endRune = bounds[i+1].runeStart
|
|
}
|
|
if b.line != "" {
|
|
hierarchy.Observe(b.line)
|
|
}
|
|
// Catch sub-headings that occur between this primary boundary and
|
|
// the next so the hierarchy stays in sync for subsequent sections.
|
|
// We intentionally do this after observing the section header so
|
|
// the breadcrumb reflects the section-leading heading.
|
|
breadcrumb := hierarchy.BreadcrumbWithHashes()
|
|
observeSubHeadings(runes[b.runeStart:endRune], primaryLevel, hierarchy)
|
|
|
|
sectionRunes := runes[b.runeStart:endRune]
|
|
sectionContent := string(sectionRunes)
|
|
secLen := len(sectionRunes)
|
|
if secLen == 0 {
|
|
continue
|
|
}
|
|
|
|
bcLen := utf8.RuneCountInString(breadcrumb)
|
|
// Single-chunk section: emit as-is, breadcrumb tracked separately.
|
|
// The breadcrumb is delivered via Chunk.ContextHeader (not Content)
|
|
// to preserve End-Start == len(Content) invariants relied on by
|
|
// document reconstruction (knowledge.go:2278+).
|
|
if bcLen+2+secLen <= cfg.ChunkSize {
|
|
out = append(out, Chunk{
|
|
Content: sectionContent,
|
|
ContextHeader: breadcrumb,
|
|
Seq: seq,
|
|
Start: b.runeStart,
|
|
End: endRune,
|
|
})
|
|
seq++
|
|
continue
|
|
}
|
|
|
|
// Section too large: defer to the legacy splitter for inner
|
|
// segmentation. Sub-chunks inherit the same breadcrumb via
|
|
// ContextHeader. We do NOT shrink the inner ChunkSize budget here
|
|
// because the breadcrumb no longer counts against Content size.
|
|
subChunks := SplitText(sectionContent, cfg)
|
|
for _, sub := range subChunks {
|
|
out = append(out, Chunk{
|
|
Content: sub.Content,
|
|
ContextHeader: breadcrumb,
|
|
Seq: seq,
|
|
Start: b.runeStart + sub.Start,
|
|
End: b.runeStart + sub.End,
|
|
})
|
|
seq++
|
|
}
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
// findHeadingBoundaries returns one boundary at offset 0 plus one per
|
|
// Markdown heading at level <= primaryLevel that sits outside fenced code
|
|
// blocks. Heading detection is line-oriented — a heading must occupy a
|
|
// whole line to be recognized.
|
|
func findHeadingBoundaries(text string, primaryLevel int) []headingBoundary {
|
|
runes := []rune(text)
|
|
bounds := []headingBoundary{{runeStart: 0}}
|
|
if len(runes) == 0 {
|
|
return bounds
|
|
}
|
|
|
|
pos := 0
|
|
inFence := false
|
|
lines := strings.Split(text, "\n")
|
|
for i, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
if strings.HasPrefix(trimmed, "```") {
|
|
inFence = !inFence
|
|
pos += utf8.RuneCountInString(line)
|
|
if i < len(lines)-1 {
|
|
pos++ // newline
|
|
}
|
|
continue
|
|
}
|
|
if !inFence {
|
|
m := MarkdownHeadingPattern.FindStringSubmatch(line)
|
|
if m != nil {
|
|
level := len(m[1])
|
|
if level >= 1 && level <= primaryLevel && pos > 0 {
|
|
bounds = append(bounds, headingBoundary{
|
|
runeStart: pos,
|
|
line: line,
|
|
})
|
|
}
|
|
if level >= 1 && level <= primaryLevel && pos == 0 {
|
|
// First line is a heading — replace the leading boundary
|
|
bounds[0].line = line
|
|
}
|
|
}
|
|
}
|
|
pos += utf8.RuneCountInString(line)
|
|
if i < len(lines)-1 {
|
|
pos++ // account for the \n that strings.Split removed
|
|
}
|
|
}
|
|
return bounds
|
|
}
|
|
|
|
// observeSubHeadings walks the section's lines and feeds every Markdown
|
|
// heading deeper than primaryLevel into the hierarchy. This keeps the
|
|
// hierarchy state correct so the breadcrumb at the next primary section
|
|
// reflects the truly active stack.
|
|
func observeSubHeadings(runes []rune, primaryLevel int, h *HeadingHierarchy) {
|
|
if len(runes) == 0 {
|
|
return
|
|
}
|
|
text := string(runes)
|
|
inFence := false
|
|
for _, line := range strings.Split(text, "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if strings.HasPrefix(trimmed, "```") {
|
|
inFence = !inFence
|
|
continue
|
|
}
|
|
if inFence {
|
|
continue
|
|
}
|
|
m := MarkdownHeadingPattern.FindStringSubmatch(line)
|
|
if m == nil {
|
|
continue
|
|
}
|
|
level := len(m[1])
|
|
if level > primaryLevel {
|
|
h.Observe(line)
|
|
}
|
|
}
|
|
}
|
|
|