feat(chunker): heading-aware Tier 1 splitter

splitByHeadingsImpl detects the dominant Markdown heading level via the
profiler and splits the document at headings of that level (or shallower).
Each resulting chunk gets a breadcrumb prefix like
'# Chapter 1\n## Section 1.2' so downstream embedding sees the section
context, not just the body. Sections larger than ChunkSize fall back to
the legacy splitter for inner segmentation, with the breadcrumb prepended
to every sub-chunk and ChunkSize reduced by the breadcrumb length so the
final chunk still fits the budget.

- DominantHeadingLevel: prefer the lowest level with ≥3 hits, else the
  deepest level with ≥1 hit (so small docs with H1+H2 still split at H2)
- HeadingHierarchy state is updated for sub-headings within a section so
  later primary sections see the right active stack
- Code-block contents are skipped during heading detection so a `# foo`
  inside ``` doesn't create a fake boundary
- strategy.go uses context.Background() for the fallback log entry
  (GetLogger panics on nil context)

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
Claude
2026-05-02 09:26:40 +00:00
committed by lyingbug
parent 592d3618dc
commit cf8789f84c
4 changed files with 346 additions and 11 deletions
@@ -0,0 +1,204 @@
// Package chunker - heading_splitter.go implements Tier 1: Markdown
// heading-aware chunking. Documents with proper heading structure are split
// at heading boundaries and each chunk is prefixed with a breadcrumb of
// active heading context (e.g. "# Chapter 1\n## Section 1.2").
package chunker
import (
"strings"
"unicode/utf8"
)
// init wires this implementation into the strategy resolver.
func init() {
splitByHeadings = splitByHeadingsImpl
}
// headingBoundary marks where a section starts. The first boundary is at
// rune offset 0 (covers any preamble before the first heading), subsequent
// boundaries sit at headings whose level is <= primaryLevel.
type headingBoundary struct {
runeStart int
line string // raw heading line, "" when this is the leading boundary
}
// splitByHeadingsImpl is the Tier-1 implementation. It falls through to the
// legacy splitter when the document has no usable heading structure or when
// the heading split would produce a single section anyway.
func splitByHeadingsImpl(text string, cfg SplitterConfig) []Chunk {
if text == "" {
return nil
}
profile := ProfileDocument(text)
primaryLevel := profile.DominantHeadingLevel()
if primaryLevel == 0 {
return SplitText(text, cfg)
}
bounds := findHeadingBoundaries(text, primaryLevel)
if len(bounds) <= 1 {
return SplitText(text, cfg)
}
runes := []rune(text)
hierarchy := NewHeadingHierarchy()
// Pre-walk every heading (not just primary-level) so the hierarchy
// reflects the full nesting context for each section's start. We only
// snapshot the breadcrumb at section boundaries; deeper sub-headings
// inside a section update the hierarchy but do not change the chunk's
// breadcrumb (chunks within a section share one breadcrumb).
var out []Chunk
seq := 0
for i, b := range bounds {
endRune := len(runes)
if i+1 < len(bounds) {
endRune = bounds[i+1].runeStart
}
if b.line != "" {
hierarchy.Observe(b.line)
}
// Catch sub-headings that occur between this primary boundary and
// the next so the hierarchy stays in sync for subsequent sections.
// We intentionally do this after observing the section header so
// the breadcrumb reflects the section-leading heading.
breadcrumb := hierarchy.BreadcrumbWithHashes()
observeSubHeadings(runes[b.runeStart:endRune], primaryLevel, hierarchy)
sectionRunes := runes[b.runeStart:endRune]
sectionContent := string(sectionRunes)
secLen := len(sectionRunes)
if secLen == 0 {
continue
}
bcLen := utf8.RuneCountInString(breadcrumb)
// Reserve some headroom (breadcrumb + 2 newlines) when fitting a section.
if bcLen+2+secLen <= cfg.ChunkSize {
content := prependBreadcrumb(sectionContent, breadcrumb)
out = append(out, Chunk{
Content: content,
Seq: seq,
Start: b.runeStart,
End: endRune,
})
seq++
continue
}
// Section too large for one chunk: defer to the legacy splitter for
// inner segmentation, then prepend the breadcrumb to every sub-chunk.
// Reduce the inner budget by the breadcrumb length so the final
// chunk (incl. breadcrumb) still fits under cfg.ChunkSize.
innerCfg := cfg
if bcLen > 0 && innerCfg.ChunkSize > bcLen+10 {
innerCfg.ChunkSize -= bcLen + 2
}
subChunks := SplitText(sectionContent, innerCfg)
for _, sub := range subChunks {
content := prependBreadcrumb(sub.Content, breadcrumb)
out = append(out, Chunk{
Content: content,
Seq: seq,
Start: b.runeStart + sub.Start,
End: b.runeStart + sub.End,
})
seq++
}
}
return out
}
// findHeadingBoundaries returns one boundary at offset 0 plus one per
// Markdown heading at level <= primaryLevel that sits outside fenced code
// blocks. Heading detection is line-oriented — a heading must occupy a
// whole line to be recognized.
func findHeadingBoundaries(text string, primaryLevel int) []headingBoundary {
runes := []rune(text)
bounds := []headingBoundary{{runeStart: 0}}
if len(runes) == 0 {
return bounds
}
pos := 0
inFence := false
lines := strings.Split(text, "\n")
for i, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "```") {
inFence = !inFence
pos += utf8.RuneCountInString(line)
if i < len(lines)-1 {
pos++ // newline
}
continue
}
if !inFence {
m := MarkdownHeadingPattern.FindStringSubmatch(line)
if m != nil {
level := len(m[1])
if level >= 1 && level <= primaryLevel && pos > 0 {
bounds = append(bounds, headingBoundary{
runeStart: pos,
line: line,
})
}
if level >= 1 && level <= primaryLevel && pos == 0 {
// First line is a heading — replace the leading boundary
bounds[0].line = line
}
}
}
pos += utf8.RuneCountInString(line)
if i < len(lines)-1 {
pos++ // account for the \n that strings.Split removed
}
}
return bounds
}
// observeSubHeadings walks the section's lines and feeds every Markdown
// heading deeper than primaryLevel into the hierarchy. This keeps the
// hierarchy state correct so the breadcrumb at the next primary section
// reflects the truly active stack.
func observeSubHeadings(runes []rune, primaryLevel int, h *HeadingHierarchy) {
if len(runes) == 0 {
return
}
text := string(runes)
inFence := false
for _, line := range strings.Split(text, "\n") {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "```") {
inFence = !inFence
continue
}
if inFence {
continue
}
m := MarkdownHeadingPattern.FindStringSubmatch(line)
if m == nil {
continue
}
level := len(m[1])
if level > primaryLevel {
h.Observe(line)
}
}
}
// prependBreadcrumb attaches the breadcrumb to content unless content
// already begins with that exact breadcrumb (avoid duplication when a
// section's first line is itself the section heading).
func prependBreadcrumb(content, breadcrumb string) string {
if breadcrumb == "" {
return content
}
trimmed := strings.TrimLeft(content, " \t\r\n")
if strings.HasPrefix(trimmed, breadcrumb) {
return content
}
return breadcrumb + "\n\n" + content
}
@@ -0,0 +1,125 @@
package chunker
import (
"strings"
"testing"
)
func TestSplitByHeadings_BasicSections(t *testing.T) {
doc := `# Top
preamble.
## Section A
content of A.
## Section B
content of B.
## Section C
content of C.`
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
if len(chunks) < 3 {
t.Fatalf("expected ≥3 chunks (one per section), got %d", len(chunks))
}
// Each chunk should contain the H1 + section heading as breadcrumb.
for i, c := range chunks {
if !strings.Contains(c.Content, "# Top") {
t.Errorf("chunk %d missing H1 breadcrumb:\n%s", i, c.Content)
}
}
// Section B chunk should mention Section B.
found := false
for _, c := range chunks {
if strings.Contains(c.Content, "Section B") && strings.Contains(c.Content, "content of B") {
found = true
}
}
if !found {
t.Error("no chunk contains Section B with its content")
}
}
func TestSplitByHeadings_FallsThroughForUnstructuredDoc(t *testing.T) {
doc := "Just a plain paragraph without any headings at all in this text."
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
// no headings → falls through to SplitText, which keeps the whole thing
if len(chunks) != 1 {
t.Errorf("expected fallthrough single chunk, got %d", len(chunks))
}
}
func TestSplitByHeadings_LargeSectionRecursesIntoLegacy(t *testing.T) {
body := strings.Repeat("This is a long sentence repeated many times. ", 50)
doc := "# Top\n## Big\n" + body
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Separators: []string{". "}}
chunks := splitByHeadingsImpl(doc, cfg)
if len(chunks) < 2 {
t.Fatalf("large section should be sub-split, got %d chunks", len(chunks))
}
// Every sub-chunk should still carry the breadcrumb.
for i, c := range chunks {
if !strings.Contains(c.Content, "# Top") {
t.Errorf("sub-chunk %d missing H1 breadcrumb", i)
}
}
}
func TestSplitByHeadings_BreadcrumbReflectsLatestPath(t *testing.T) {
doc := `# Chapter 1
intro
## Section A
text A
## Section B
text B`
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
if len(chunks) < 3 {
t.Fatalf("expected ≥3 chunks, got %d", len(chunks))
}
// Section B chunk's breadcrumb should NOT mention Section A
for _, c := range chunks {
if strings.Contains(c.Content, "text B") {
if strings.Contains(c.Content, "## Section A") {
t.Errorf("Section B chunk should not include Section A in breadcrumb:\n%s", c.Content)
}
if !strings.Contains(c.Content, "## Section B") {
t.Errorf("Section B chunk should include its own heading:\n%s", c.Content)
}
}
}
}
func TestSplitByHeadings_IgnoresHeadingsInsideCodeFence(t *testing.T) {
doc := "# Real\n\n```\n# Fake heading inside code\n```\n\nbody"
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
// The fake heading should not create a section boundary — there's only
// one real H1, so we expect either 1 chunk or fall-through.
for _, c := range chunks {
if strings.Contains(c.Content, "# Real") {
// Good — found the real one.
return
}
}
t.Error("expected real H1 breadcrumb in some chunk")
}
func TestSplitByHeadings_PreservesPositionRelativeToOriginal(t *testing.T) {
doc := "# Top\nintro\n\n## A\nbody A\n\n## B\nbody B"
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
for i, c := range chunks {
if c.Start < 0 {
t.Errorf("chunk %d has negative Start", i)
}
if c.End < c.Start {
t.Errorf("chunk %d End < Start", i)
}
}
}
+11 -9
View File
@@ -49,9 +49,13 @@ func (p *DocProfile) HeadingDensity() float64 {
}
// DominantHeadingLevel returns the heading level (1..6) that should drive
// section splitting. It picks the lowest level (closest to root) that has
// at least 3 occurrences, falling back to the most frequent level. Returns
// 0 when no usable Markdown structure exists.
// section splitting. Preference order:
// 1. The lowest level (closest to root) that has at least 3 occurrences —
// a "real" structural backbone of the document.
// 2. Otherwise the deepest level present at least once — gives finer-grained
// boundaries for small documents that just have an H1 + a few H2s.
//
// Returns 0 when no Markdown headings exist.
func (p *DocProfile) DominantHeadingLevel() int {
if p.MdHeadingTotal == 0 {
return 0
@@ -61,14 +65,12 @@ func (p *DocProfile) DominantHeadingLevel() int {
return level
}
}
// fallback: most frequent level
bestLevel, bestCount := 0, 0
for level := 1; level <= 6; level++ {
if p.MdHeadingCounts[level] > bestCount {
bestLevel, bestCount = level, p.MdHeadingCounts[level]
for level := 6; level >= 1; level-- {
if p.MdHeadingCounts[level] > 0 {
return level
}
}
return bestLevel
return 0
}
// HeuristicMarkerTotal sums the non-Markdown structural markers.
+6 -2
View File
@@ -7,7 +7,11 @@
// compatibility — strategy.go simply layers a tier-selector on top.
package chunker
import "github.com/Tencent/WeKnora/internal/logger"
import (
"context"
"github.com/Tencent/WeKnora/internal/logger"
)
// Strategy values for SplitterConfig.Strategy.
const (
@@ -36,7 +40,7 @@ func Split(text string, cfg SplitterConfig) []Chunk {
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
return out
} else {
logger.Debugf(nil, "chunker: tier %s rejected: %s", tier, v.Reason)
logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason)
}
}
// Last-ditch fallback: always return *something*.