fix(chunker): carry deep sub-heading context into large-section chunks

When the dominant heading level repeats and becomes the split backbone,
a long section is sub-split via SplitText and every sub-chunk inherited
the section-level breadcrumb. Deep `###`/`####` headings that define a
sub-chunk's meaning were dropped from Chunk.ContextHeader, so queries
about content nested far below a shallow heading failed to retrieve it.

Track the heading hierarchy across the section and assign each sub-chunk
the breadcrumb in effect at its start offset, so the deepest active
heading reaches the breadcrumb. Adds a regression test.

Fixes #1674

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
adawang
2026-06-16 15:15:52 +08:00
committed by lyingbug
parent 0c3d45d3cd
commit 8d3030d339
2 changed files with 107 additions and 3 deletions
@@ -70,6 +70,7 @@ func splitByHeadingsImpl(text string, cfg SplitterConfig, profile *DocProfile) [
// We intentionally do this after observing the section header so
// the breadcrumb reflects the section-leading heading.
breadcrumb := hierarchy.BreadcrumbWithHashes()
sectionStart := *hierarchy
observeSubHeadings(runes[b.runeStart:endRune], primaryLevel, hierarchy)
sectionRunes := runes[b.runeStart:endRune]
@@ -97,14 +98,17 @@ func splitByHeadingsImpl(text string, cfg SplitterConfig, profile *DocProfile) [
}
// Section too large: defer to the legacy splitter for inner
// segmentation. Sub-chunks inherit the same breadcrumb via
// ContextHeader. We do NOT shrink the inner ChunkSize budget here
// segmentation. We do NOT shrink the inner ChunkSize budget here
// because the breadcrumb no longer counts against Content size.
// Each sub-chunk gets a breadcrumb reflecting the deepest heading
// active at its start, so deep `###`/`####` sub-headings inside a
// long section aren't collapsed to the section-level header.
subBreadcrumbs := sectionBreadcrumbs(sectionRunes, primaryLevel, sectionStart)
subChunks := SplitText(sectionContent, cfg)
for _, sub := range subChunks {
out = append(out, Chunk{
Content: sub.Content,
ContextHeader: breadcrumb,
ContextHeader: breadcrumbAtOffset(subBreadcrumbs, sub.Start, breadcrumb),
Seq: seq,
Start: b.runeStart + sub.Start,
End: b.runeStart + sub.End,
@@ -275,3 +279,64 @@ func observeSubHeadings(runes []rune, primaryLevel int, h *HeadingHierarchy) {
}
}
}
// sectionBreadcrumb pairs a rune offset within a section with the breadcrumb
// in effect from that offset onward.
type sectionBreadcrumb struct {
runeStart int
breadcrumb string
}
// sectionBreadcrumbs walks a section's deeper sub-headings (level >
// primaryLevel) and records, for each, the rune offset where it takes effect
// and the resulting breadcrumb. seed is the hierarchy state at the section's
// start (already including the section heading and its ancestors). The
// returned slice is ordered by runeStart and always begins with the seed
// breadcrumb at offset 0, so a sub-chunk sitting far below a deep heading
// still resolves to that heading's path rather than the section header.
func sectionBreadcrumbs(sectionRunes []rune, primaryLevel int, seed HeadingHierarchy) []sectionBreadcrumb {
h := seed
result := []sectionBreadcrumb{{runeStart: 0, breadcrumb: h.BreadcrumbWithHashes()}}
pos := 0
inFence := false
lines := strings.Split(string(sectionRunes), "\n")
for i, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "```") {
inFence = !inFence
pos += utf8.RuneCountInString(line)
if i < len(lines)-1 {
pos++
}
continue
}
if !inFence {
if m := MarkdownHeadingPattern.FindStringSubmatch(line); m != nil && len(m[1]) > primaryLevel {
h.Observe(line)
result = append(result, sectionBreadcrumb{
runeStart: pos,
breadcrumb: h.BreadcrumbWithHashes(),
})
}
}
pos += utf8.RuneCountInString(line)
if i < len(lines)-1 {
pos++
}
}
return result
}
// breadcrumbAtOffset returns the breadcrumb in effect at the given rune offset
// — the last entry whose runeStart <= offset. fallback covers the (unreachable
// in practice) empty-slice case.
func breadcrumbAtOffset(bcs []sectionBreadcrumb, offset int, fallback string) string {
bc := fallback
for _, e := range bcs {
if e.runeStart > offset {
break
}
bc = e.breadcrumb
}
return bc
}
@@ -321,3 +321,42 @@ body B.`
}
}
}
// TestSplitByHeadings_DeepSubHeadingInLargeSection covers issue #1674: when a
// shallow heading level repeats and therefore becomes the dominant split
// backbone, a long section split into sub-chunks must still carry the deep
// `###`/`####` heading that defines a sub-chunk's meaning, not just the
// section-level header. The marker line sits far below its deep heading, in a
// later sub-chunk that has no heading of its own.
func TestSplitByHeadings_DeepSubHeadingInLargeSection(t *testing.T) {
filler := strings.Repeat("clause body sentence that pads the section out. ", 20)
doc := "# Standard XYZ\n" +
"## Preface\n" + filler + "\n\n" +
"## 5 Classification\n" + filler + "\n\n" +
"### 5.9 Grade Nine\n" + filler + "\n\n" +
"#### 5.9.2 Clause Series\n" + filler + "\n\n" +
"the item users search for is MARKER_ITEM_23 graded here.\n\n" +
"## Appendix A\n" + filler + "\n\n" +
"## Appendix B\n" + filler + "\n\n" +
"## Appendix C\n" + filler
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 0, Separators: []string{". "}}
chunks := splitByHeadingsImpl(doc, cfg, nil)
var marker *Chunk
for i := range chunks {
if strings.Contains(chunks[i].Content, "MARKER_ITEM_23") {
marker = &chunks[i]
break
}
}
if marker == nil {
t.Fatal("no chunk contains the marker line")
}
if !strings.Contains(marker.ContextHeader, "#### 5.9.2 Clause Series") {
t.Errorf("marker chunk lost its deep sub-heading in the breadcrumb:\nContextHeader=%q", marker.ContextHeader)
}
if !strings.Contains(marker.ContextHeader, "## 5 Classification") {
t.Errorf("marker chunk lost its section heading in the breadcrumb:\nContextHeader=%q", marker.ContextHeader)
}
}