feat(chunker): heading-aware Tier 1 splitter
splitByHeadingsImpl detects the dominant Markdown heading level via the profiler and splits the document at headings of that level (or shallower). Each resulting chunk gets a breadcrumb prefix like '# Chapter 1\n## Section 1.2' so downstream embedding sees the section context, not just the body. Sections larger than ChunkSize fall back to the legacy splitter for inner segmentation, with the breadcrumb prepended to every sub-chunk and ChunkSize reduced by the breadcrumb length so the final chunk still fits the budget. - DominantHeadingLevel: prefer the lowest level with ≥3 hits, else the deepest level with ≥1 hit (so small docs with H1+H2 still split at H2) - HeadingHierarchy state is updated for sub-headings within a section so later primary sections see the right active stack - Code-block contents are skipped during heading detection so a `# foo` inside ``` doesn't create a fake boundary - strategy.go uses context.Background() for the fallback log entry (GetLogger panics on nil context) https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
@@ -0,0 +1,204 @@
|
||||
// Package chunker - heading_splitter.go implements Tier 1: Markdown
|
||||
// heading-aware chunking. Documents with proper heading structure are split
|
||||
// at heading boundaries and each chunk is prefixed with a breadcrumb of
|
||||
// active heading context (e.g. "# Chapter 1\n## Section 1.2").
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// init wires this implementation into the strategy resolver.
|
||||
func init() {
|
||||
splitByHeadings = splitByHeadingsImpl
|
||||
}
|
||||
|
||||
// headingBoundary marks where a section starts. The first boundary is at
|
||||
// rune offset 0 (covers any preamble before the first heading), subsequent
|
||||
// boundaries sit at headings whose level is <= primaryLevel.
|
||||
type headingBoundary struct {
|
||||
runeStart int
|
||||
line string // raw heading line, "" when this is the leading boundary
|
||||
}
|
||||
|
||||
// splitByHeadingsImpl is the Tier-1 implementation. It falls through to the
|
||||
// legacy splitter when the document has no usable heading structure or when
|
||||
// the heading split would produce a single section anyway.
|
||||
func splitByHeadingsImpl(text string, cfg SplitterConfig) []Chunk {
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
profile := ProfileDocument(text)
|
||||
primaryLevel := profile.DominantHeadingLevel()
|
||||
if primaryLevel == 0 {
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
|
||||
bounds := findHeadingBoundaries(text, primaryLevel)
|
||||
if len(bounds) <= 1 {
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
|
||||
runes := []rune(text)
|
||||
hierarchy := NewHeadingHierarchy()
|
||||
|
||||
// Pre-walk every heading (not just primary-level) so the hierarchy
|
||||
// reflects the full nesting context for each section's start. We only
|
||||
// snapshot the breadcrumb at section boundaries; deeper sub-headings
|
||||
// inside a section update the hierarchy but do not change the chunk's
|
||||
// breadcrumb (chunks within a section share one breadcrumb).
|
||||
var out []Chunk
|
||||
seq := 0
|
||||
|
||||
for i, b := range bounds {
|
||||
endRune := len(runes)
|
||||
if i+1 < len(bounds) {
|
||||
endRune = bounds[i+1].runeStart
|
||||
}
|
||||
if b.line != "" {
|
||||
hierarchy.Observe(b.line)
|
||||
}
|
||||
// Catch sub-headings that occur between this primary boundary and
|
||||
// the next so the hierarchy stays in sync for subsequent sections.
|
||||
// We intentionally do this after observing the section header so
|
||||
// the breadcrumb reflects the section-leading heading.
|
||||
breadcrumb := hierarchy.BreadcrumbWithHashes()
|
||||
observeSubHeadings(runes[b.runeStart:endRune], primaryLevel, hierarchy)
|
||||
|
||||
sectionRunes := runes[b.runeStart:endRune]
|
||||
sectionContent := string(sectionRunes)
|
||||
secLen := len(sectionRunes)
|
||||
if secLen == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
bcLen := utf8.RuneCountInString(breadcrumb)
|
||||
// Reserve some headroom (breadcrumb + 2 newlines) when fitting a section.
|
||||
if bcLen+2+secLen <= cfg.ChunkSize {
|
||||
content := prependBreadcrumb(sectionContent, breadcrumb)
|
||||
out = append(out, Chunk{
|
||||
Content: content,
|
||||
Seq: seq,
|
||||
Start: b.runeStart,
|
||||
End: endRune,
|
||||
})
|
||||
seq++
|
||||
continue
|
||||
}
|
||||
|
||||
// Section too large for one chunk: defer to the legacy splitter for
|
||||
// inner segmentation, then prepend the breadcrumb to every sub-chunk.
|
||||
// Reduce the inner budget by the breadcrumb length so the final
|
||||
// chunk (incl. breadcrumb) still fits under cfg.ChunkSize.
|
||||
innerCfg := cfg
|
||||
if bcLen > 0 && innerCfg.ChunkSize > bcLen+10 {
|
||||
innerCfg.ChunkSize -= bcLen + 2
|
||||
}
|
||||
subChunks := SplitText(sectionContent, innerCfg)
|
||||
for _, sub := range subChunks {
|
||||
content := prependBreadcrumb(sub.Content, breadcrumb)
|
||||
out = append(out, Chunk{
|
||||
Content: content,
|
||||
Seq: seq,
|
||||
Start: b.runeStart + sub.Start,
|
||||
End: b.runeStart + sub.End,
|
||||
})
|
||||
seq++
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// findHeadingBoundaries returns one boundary at offset 0 plus one per
|
||||
// Markdown heading at level <= primaryLevel that sits outside fenced code
|
||||
// blocks. Heading detection is line-oriented — a heading must occupy a
|
||||
// whole line to be recognized.
|
||||
func findHeadingBoundaries(text string, primaryLevel int) []headingBoundary {
|
||||
runes := []rune(text)
|
||||
bounds := []headingBoundary{{runeStart: 0}}
|
||||
if len(runes) == 0 {
|
||||
return bounds
|
||||
}
|
||||
|
||||
pos := 0
|
||||
inFence := false
|
||||
lines := strings.Split(text, "\n")
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "```") {
|
||||
inFence = !inFence
|
||||
pos += utf8.RuneCountInString(line)
|
||||
if i < len(lines)-1 {
|
||||
pos++ // newline
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !inFence {
|
||||
m := MarkdownHeadingPattern.FindStringSubmatch(line)
|
||||
if m != nil {
|
||||
level := len(m[1])
|
||||
if level >= 1 && level <= primaryLevel && pos > 0 {
|
||||
bounds = append(bounds, headingBoundary{
|
||||
runeStart: pos,
|
||||
line: line,
|
||||
})
|
||||
}
|
||||
if level >= 1 && level <= primaryLevel && pos == 0 {
|
||||
// First line is a heading — replace the leading boundary
|
||||
bounds[0].line = line
|
||||
}
|
||||
}
|
||||
}
|
||||
pos += utf8.RuneCountInString(line)
|
||||
if i < len(lines)-1 {
|
||||
pos++ // account for the \n that strings.Split removed
|
||||
}
|
||||
}
|
||||
return bounds
|
||||
}
|
||||
|
||||
// observeSubHeadings walks the section's lines and feeds every Markdown
|
||||
// heading deeper than primaryLevel into the hierarchy. This keeps the
|
||||
// hierarchy state correct so the breadcrumb at the next primary section
|
||||
// reflects the truly active stack.
|
||||
func observeSubHeadings(runes []rune, primaryLevel int, h *HeadingHierarchy) {
|
||||
if len(runes) == 0 {
|
||||
return
|
||||
}
|
||||
text := string(runes)
|
||||
inFence := false
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "```") {
|
||||
inFence = !inFence
|
||||
continue
|
||||
}
|
||||
if inFence {
|
||||
continue
|
||||
}
|
||||
m := MarkdownHeadingPattern.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
level := len(m[1])
|
||||
if level > primaryLevel {
|
||||
h.Observe(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// prependBreadcrumb attaches the breadcrumb to content unless content
|
||||
// already begins with that exact breadcrumb (avoid duplication when a
|
||||
// section's first line is itself the section heading).
|
||||
func prependBreadcrumb(content, breadcrumb string) string {
|
||||
if breadcrumb == "" {
|
||||
return content
|
||||
}
|
||||
trimmed := strings.TrimLeft(content, " \t\r\n")
|
||||
if strings.HasPrefix(trimmed, breadcrumb) {
|
||||
return content
|
||||
}
|
||||
return breadcrumb + "\n\n" + content
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package chunker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplitByHeadings_BasicSections(t *testing.T) {
|
||||
doc := `# Top
|
||||
preamble.
|
||||
|
||||
## Section A
|
||||
content of A.
|
||||
|
||||
## Section B
|
||||
content of B.
|
||||
|
||||
## Section C
|
||||
content of C.`
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
if len(chunks) < 3 {
|
||||
t.Fatalf("expected ≥3 chunks (one per section), got %d", len(chunks))
|
||||
}
|
||||
|
||||
// Each chunk should contain the H1 + section heading as breadcrumb.
|
||||
for i, c := range chunks {
|
||||
if !strings.Contains(c.Content, "# Top") {
|
||||
t.Errorf("chunk %d missing H1 breadcrumb:\n%s", i, c.Content)
|
||||
}
|
||||
}
|
||||
|
||||
// Section B chunk should mention Section B.
|
||||
found := false
|
||||
for _, c := range chunks {
|
||||
if strings.Contains(c.Content, "Section B") && strings.Contains(c.Content, "content of B") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("no chunk contains Section B with its content")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByHeadings_FallsThroughForUnstructuredDoc(t *testing.T) {
|
||||
doc := "Just a plain paragraph without any headings at all in this text."
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
// no headings → falls through to SplitText, which keeps the whole thing
|
||||
if len(chunks) != 1 {
|
||||
t.Errorf("expected fallthrough single chunk, got %d", len(chunks))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByHeadings_LargeSectionRecursesIntoLegacy(t *testing.T) {
|
||||
body := strings.Repeat("This is a long sentence repeated many times. ", 50)
|
||||
doc := "# Top\n## Big\n" + body
|
||||
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Separators: []string{". "}}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("large section should be sub-split, got %d chunks", len(chunks))
|
||||
}
|
||||
// Every sub-chunk should still carry the breadcrumb.
|
||||
for i, c := range chunks {
|
||||
if !strings.Contains(c.Content, "# Top") {
|
||||
t.Errorf("sub-chunk %d missing H1 breadcrumb", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByHeadings_BreadcrumbReflectsLatestPath(t *testing.T) {
|
||||
doc := `# Chapter 1
|
||||
intro
|
||||
|
||||
## Section A
|
||||
text A
|
||||
|
||||
## Section B
|
||||
text B`
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
if len(chunks) < 3 {
|
||||
t.Fatalf("expected ≥3 chunks, got %d", len(chunks))
|
||||
}
|
||||
// Section B chunk's breadcrumb should NOT mention Section A
|
||||
for _, c := range chunks {
|
||||
if strings.Contains(c.Content, "text B") {
|
||||
if strings.Contains(c.Content, "## Section A") {
|
||||
t.Errorf("Section B chunk should not include Section A in breadcrumb:\n%s", c.Content)
|
||||
}
|
||||
if !strings.Contains(c.Content, "## Section B") {
|
||||
t.Errorf("Section B chunk should include its own heading:\n%s", c.Content)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitByHeadings_IgnoresHeadingsInsideCodeFence(t *testing.T) {
|
||||
doc := "# Real\n\n```\n# Fake heading inside code\n```\n\nbody"
|
||||
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
// The fake heading should not create a section boundary — there's only
|
||||
// one real H1, so we expect either 1 chunk or fall-through.
|
||||
for _, c := range chunks {
|
||||
if strings.Contains(c.Content, "# Real") {
|
||||
// Good — found the real one.
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Error("expected real H1 breadcrumb in some chunk")
|
||||
}
|
||||
|
||||
func TestSplitByHeadings_PreservesPositionRelativeToOriginal(t *testing.T) {
|
||||
doc := "# Top\nintro\n\n## A\nbody A\n\n## B\nbody B"
|
||||
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
for i, c := range chunks {
|
||||
if c.Start < 0 {
|
||||
t.Errorf("chunk %d has negative Start", i)
|
||||
}
|
||||
if c.End < c.Start {
|
||||
t.Errorf("chunk %d End < Start", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -49,9 +49,13 @@ func (p *DocProfile) HeadingDensity() float64 {
|
||||
}
|
||||
|
||||
// DominantHeadingLevel returns the heading level (1..6) that should drive
|
||||
// section splitting. It picks the lowest level (closest to root) that has
|
||||
// at least 3 occurrences, falling back to the most frequent level. Returns
|
||||
// 0 when no usable Markdown structure exists.
|
||||
// section splitting. Preference order:
|
||||
// 1. The lowest level (closest to root) that has at least 3 occurrences —
|
||||
// a "real" structural backbone of the document.
|
||||
// 2. Otherwise the deepest level present at least once — gives finer-grained
|
||||
// boundaries for small documents that just have an H1 + a few H2s.
|
||||
//
|
||||
// Returns 0 when no Markdown headings exist.
|
||||
func (p *DocProfile) DominantHeadingLevel() int {
|
||||
if p.MdHeadingTotal == 0 {
|
||||
return 0
|
||||
@@ -61,14 +65,12 @@ func (p *DocProfile) DominantHeadingLevel() int {
|
||||
return level
|
||||
}
|
||||
}
|
||||
// fallback: most frequent level
|
||||
bestLevel, bestCount := 0, 0
|
||||
for level := 1; level <= 6; level++ {
|
||||
if p.MdHeadingCounts[level] > bestCount {
|
||||
bestLevel, bestCount = level, p.MdHeadingCounts[level]
|
||||
for level := 6; level >= 1; level-- {
|
||||
if p.MdHeadingCounts[level] > 0 {
|
||||
return level
|
||||
}
|
||||
}
|
||||
return bestLevel
|
||||
return 0
|
||||
}
|
||||
|
||||
// HeuristicMarkerTotal sums the non-Markdown structural markers.
|
||||
|
||||
@@ -7,7 +7,11 @@
|
||||
// compatibility — strategy.go simply layers a tier-selector on top.
|
||||
package chunker
|
||||
|
||||
import "github.com/Tencent/WeKnora/internal/logger"
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
// Strategy values for SplitterConfig.Strategy.
|
||||
const (
|
||||
@@ -36,7 +40,7 @@ func Split(text string, cfg SplitterConfig) []Chunk {
|
||||
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
|
||||
return out
|
||||
} else {
|
||||
logger.Debugf(nil, "chunker: tier %s rejected: %s", tier, v.Reason)
|
||||
logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason)
|
||||
}
|
||||
}
|
||||
// Last-ditch fallback: always return *something*.
|
||||
|
||||
Reference in New Issue
Block a user