fix(chunker): carry deep sub-heading context into large-section chunks
When the dominant heading level repeats and becomes the split backbone, a long section is sub-split via SplitText and every sub-chunk inherited the section-level breadcrumb. Deep `###`/`####` headings that define a sub-chunk's meaning were dropped from Chunk.ContextHeader, so queries about content nested far below a shallow heading failed to retrieve it. Track the heading hierarchy across the section and assign each sub-chunk the breadcrumb in effect at its start offset, so the deepest active heading reaches the breadcrumb. Adds a regression test. Fixes #1674 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -70,6 +70,7 @@ func splitByHeadingsImpl(text string, cfg SplitterConfig, profile *DocProfile) [
|
||||
// We intentionally do this after observing the section header so
|
||||
// the breadcrumb reflects the section-leading heading.
|
||||
breadcrumb := hierarchy.BreadcrumbWithHashes()
|
||||
sectionStart := *hierarchy
|
||||
observeSubHeadings(runes[b.runeStart:endRune], primaryLevel, hierarchy)
|
||||
|
||||
sectionRunes := runes[b.runeStart:endRune]
|
||||
@@ -97,14 +98,17 @@ func splitByHeadingsImpl(text string, cfg SplitterConfig, profile *DocProfile) [
|
||||
}
|
||||
|
||||
// Section too large: defer to the legacy splitter for inner
|
||||
// segmentation. Sub-chunks inherit the same breadcrumb via
|
||||
// ContextHeader. We do NOT shrink the inner ChunkSize budget here
|
||||
// segmentation. We do NOT shrink the inner ChunkSize budget here
|
||||
// because the breadcrumb no longer counts against Content size.
|
||||
// Each sub-chunk gets a breadcrumb reflecting the deepest heading
|
||||
// active at its start, so deep `###`/`####` sub-headings inside a
|
||||
// long section aren't collapsed to the section-level header.
|
||||
subBreadcrumbs := sectionBreadcrumbs(sectionRunes, primaryLevel, sectionStart)
|
||||
subChunks := SplitText(sectionContent, cfg)
|
||||
for _, sub := range subChunks {
|
||||
out = append(out, Chunk{
|
||||
Content: sub.Content,
|
||||
ContextHeader: breadcrumb,
|
||||
ContextHeader: breadcrumbAtOffset(subBreadcrumbs, sub.Start, breadcrumb),
|
||||
Seq: seq,
|
||||
Start: b.runeStart + sub.Start,
|
||||
End: b.runeStart + sub.End,
|
||||
@@ -275,3 +279,64 @@ func observeSubHeadings(runes []rune, primaryLevel int, h *HeadingHierarchy) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sectionBreadcrumb pairs a rune offset within a section with the breadcrumb
|
||||
// in effect from that offset onward.
|
||||
type sectionBreadcrumb struct {
|
||||
runeStart int
|
||||
breadcrumb string
|
||||
}
|
||||
|
||||
// sectionBreadcrumbs walks a section's deeper sub-headings (level >
|
||||
// primaryLevel) and records, for each, the rune offset where it takes effect
|
||||
// and the resulting breadcrumb. seed is the hierarchy state at the section's
|
||||
// start (already including the section heading and its ancestors). The
|
||||
// returned slice is ordered by runeStart and always begins with the seed
|
||||
// breadcrumb at offset 0, so a sub-chunk sitting far below a deep heading
|
||||
// still resolves to that heading's path rather than the section header.
|
||||
func sectionBreadcrumbs(sectionRunes []rune, primaryLevel int, seed HeadingHierarchy) []sectionBreadcrumb {
|
||||
h := seed
|
||||
result := []sectionBreadcrumb{{runeStart: 0, breadcrumb: h.BreadcrumbWithHashes()}}
|
||||
pos := 0
|
||||
inFence := false
|
||||
lines := strings.Split(string(sectionRunes), "\n")
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "```") {
|
||||
inFence = !inFence
|
||||
pos += utf8.RuneCountInString(line)
|
||||
if i < len(lines)-1 {
|
||||
pos++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !inFence {
|
||||
if m := MarkdownHeadingPattern.FindStringSubmatch(line); m != nil && len(m[1]) > primaryLevel {
|
||||
h.Observe(line)
|
||||
result = append(result, sectionBreadcrumb{
|
||||
runeStart: pos,
|
||||
breadcrumb: h.BreadcrumbWithHashes(),
|
||||
})
|
||||
}
|
||||
}
|
||||
pos += utf8.RuneCountInString(line)
|
||||
if i < len(lines)-1 {
|
||||
pos++
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// breadcrumbAtOffset returns the breadcrumb in effect at the given rune offset
|
||||
// — the last entry whose runeStart <= offset. fallback covers the (unreachable
|
||||
// in practice) empty-slice case.
|
||||
func breadcrumbAtOffset(bcs []sectionBreadcrumb, offset int, fallback string) string {
|
||||
bc := fallback
|
||||
for _, e := range bcs {
|
||||
if e.runeStart > offset {
|
||||
break
|
||||
}
|
||||
bc = e.breadcrumb
|
||||
}
|
||||
return bc
|
||||
}
|
||||
|
||||
@@ -321,3 +321,42 @@ body B.`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSplitByHeadings_DeepSubHeadingInLargeSection covers issue #1674: when a
|
||||
// shallow heading level repeats and therefore becomes the dominant split
|
||||
// backbone, a long section split into sub-chunks must still carry the deep
|
||||
// `###`/`####` heading that defines a sub-chunk's meaning, not just the
|
||||
// section-level header. The marker line sits far below its deep heading, in a
|
||||
// later sub-chunk that has no heading of its own.
|
||||
func TestSplitByHeadings_DeepSubHeadingInLargeSection(t *testing.T) {
|
||||
filler := strings.Repeat("clause body sentence that pads the section out. ", 20)
|
||||
doc := "# Standard XYZ\n" +
|
||||
"## Preface\n" + filler + "\n\n" +
|
||||
"## 5 Classification\n" + filler + "\n\n" +
|
||||
"### 5.9 Grade Nine\n" + filler + "\n\n" +
|
||||
"#### 5.9.2 Clause Series\n" + filler + "\n\n" +
|
||||
"the item users search for is MARKER_ITEM_23 graded here.\n\n" +
|
||||
"## Appendix A\n" + filler + "\n\n" +
|
||||
"## Appendix B\n" + filler + "\n\n" +
|
||||
"## Appendix C\n" + filler
|
||||
|
||||
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 0, Separators: []string{". "}}
|
||||
chunks := splitByHeadingsImpl(doc, cfg, nil)
|
||||
|
||||
var marker *Chunk
|
||||
for i := range chunks {
|
||||
if strings.Contains(chunks[i].Content, "MARKER_ITEM_23") {
|
||||
marker = &chunks[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if marker == nil {
|
||||
t.Fatal("no chunk contains the marker line")
|
||||
}
|
||||
if !strings.Contains(marker.ContextHeader, "#### 5.9.2 Clause Series") {
|
||||
t.Errorf("marker chunk lost its deep sub-heading in the breadcrumb:\nContextHeader=%q", marker.ContextHeader)
|
||||
}
|
||||
if !strings.Contains(marker.ContextHeader, "## 5 Classification") {
|
||||
t.Errorf("marker chunk lost its section heading in the breadcrumb:\nContextHeader=%q", marker.ContextHeader)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user