feat(chunker): heading hierarchy tracker

Adds HeadingHierarchy, a level-stack that tracks Markdown heading
nesting (#, ##, ###, ...). When a level-N heading is observed, all
deeper entries are popped — so Breadcrumb() always reflects the active
context path. Used by the upcoming heading-aware splitter to prepend a
context breadcrumb (e.g. '# Chapter 1\n## Section 1.2') to every chunk
inside a section, lifting embedding quality for non-trivial documents.

The existing header_tracker (table headers) uses a binary start/end hook
model that does not fit headings — a heading does not have an explicit
end, it is replaced by the next heading of equal or shallower depth.
A separate level-stack is the right shape.

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
Claude
2026-05-02 09:23:10 +00:00
committed by lyingbug
parent 1585a6f12c
commit 592d3618dc
2 changed files with 199 additions and 0 deletions
@@ -0,0 +1,111 @@
// Package chunker - heading_hierarchy.go tracks Markdown heading nesting so
// that the heading-aware splitter (heading_splitter.go) can prepend a
// breadcrumb like "# Top > ## Section > ### Subsection" to each chunk.
//
// Conceptually similar to header_tracker.go (which tracks table headers via
// start/end hooks) but a Markdown heading does not have an explicit end —
// it is ended by the next heading of equal or shallower depth. The hook
// abstraction in header_tracker.go does not fit that pattern, so we model
// it as an explicit level-stack.
package chunker
import "strings"
// HeadingHierarchy maintains a stack of active Markdown headings indexed by
// level (1..6). Pushing a level-N heading pops every entry of level >= N
// because the previous siblings/descendants are no longer in scope.
type HeadingHierarchy struct {
// stack[i] holds the heading text for level i+1 (so stack[0] = H1).
// Entries beyond the deepest active level are empty strings.
stack [6]string
depth int // current deepest active level (0 if no active heading)
}
// NewHeadingHierarchy returns an empty hierarchy.
func NewHeadingHierarchy() *HeadingHierarchy {
return &HeadingHierarchy{}
}
// Observe parses line and updates the hierarchy if line is a Markdown
// heading. Returns the (level, headingText) when a heading was recognized,
// or (0, "") otherwise. Lines that look like headings inside fenced code
// blocks are NOT detected here — callers must avoid feeding code-block
// content to Observe (the heading splitter does so).
func (h *HeadingHierarchy) Observe(line string) (int, string) {
m := MarkdownHeadingPattern.FindStringSubmatch(line)
if m == nil {
return 0, ""
}
level := len(m[1])
if level < 1 || level > 6 {
return 0, ""
}
heading := strings.TrimSpace(m[2])
// Replace this level and clear deeper ones — siblings/descendants of
// the previous heading at this level are no longer in scope.
h.stack[level-1] = heading
for i := level; i < 6; i++ {
h.stack[i] = ""
}
if level > h.depth {
h.depth = level
} else {
// Recompute depth: it might shrink if we just pushed a shallower heading.
h.depth = 0
for i := 0; i < 6; i++ {
if h.stack[i] != "" {
h.depth = i + 1
}
}
}
return level, heading
}
// Breadcrumb returns the current heading path joined by " > ", e.g.
// "Chapter 1 > Section 2 > Subsection a". Returns "" when no headings
// are active.
func (h *HeadingHierarchy) Breadcrumb() string {
if h.depth == 0 {
return ""
}
parts := make([]string, 0, h.depth)
for i := 0; i < h.depth; i++ {
if h.stack[i] != "" {
parts = append(parts, h.stack[i])
}
}
return strings.Join(parts, " > ")
}
// BreadcrumbWithHashes returns the path with the original `#` prefixes,
// suitable for embedding back into chunk content as a context header.
// Example: "# Chapter 1\n## Section 2\n### Subsection a"
func (h *HeadingHierarchy) BreadcrumbWithHashes() string {
if h.depth == 0 {
return ""
}
var sb strings.Builder
for i := 0; i < h.depth; i++ {
if h.stack[i] == "" {
continue
}
if sb.Len() > 0 {
sb.WriteByte('\n')
}
sb.WriteString(strings.Repeat("#", i+1))
sb.WriteByte(' ')
sb.WriteString(h.stack[i])
}
return sb.String()
}
// Depth returns the current deepest active heading level.
func (h *HeadingHierarchy) Depth() int { return h.depth }
// Reset clears all state.
func (h *HeadingHierarchy) Reset() {
for i := range h.stack {
h.stack[i] = ""
}
h.depth = 0
}
@@ -0,0 +1,88 @@
package chunker
import "testing"
func TestHeadingHierarchy_LinearNesting(t *testing.T) {
h := NewHeadingHierarchy()
h.Observe("# Chapter 1")
h.Observe("## Section 1.1")
h.Observe("### Subsection 1.1.1")
if got := h.Breadcrumb(); got != "Chapter 1 > Section 1.1 > Subsection 1.1.1" {
t.Errorf("breadcrumb mismatch: %q", got)
}
}
func TestHeadingHierarchy_PopsDeeperOnSiblingHeading(t *testing.T) {
h := NewHeadingHierarchy()
h.Observe("# Chapter 1")
h.Observe("## Section 1.1")
h.Observe("### Subsection 1.1.1")
h.Observe("## Section 1.2") // pops the H3
if got := h.Breadcrumb(); got != "Chapter 1 > Section 1.2" {
t.Errorf("breadcrumb after sibling: %q", got)
}
}
func TestHeadingHierarchy_PopsAllOnNewTopLevel(t *testing.T) {
h := NewHeadingHierarchy()
h.Observe("# Chapter 1")
h.Observe("## Section A")
h.Observe("### Sub")
h.Observe("# Chapter 2")
if got := h.Breadcrumb(); got != "Chapter 2" {
t.Errorf("breadcrumb after new H1: %q", got)
}
if h.Depth() != 1 {
t.Errorf("depth should be 1, got %d", h.Depth())
}
}
func TestHeadingHierarchy_NonHeadingIgnored(t *testing.T) {
h := NewHeadingHierarchy()
h.Observe("# Title")
level, txt := h.Observe("just a paragraph")
if level != 0 || txt != "" {
t.Errorf("non-heading should not register: %d %q", level, txt)
}
if got := h.Breadcrumb(); got != "Title" {
t.Errorf("breadcrumb after non-heading: %q", got)
}
}
func TestHeadingHierarchy_BreadcrumbWithHashes(t *testing.T) {
h := NewHeadingHierarchy()
h.Observe("# A")
h.Observe("## B")
got := h.BreadcrumbWithHashes()
want := "# A\n## B"
if got != want {
t.Errorf("hashes breadcrumb:\n got: %q\n want: %q", got, want)
}
}
func TestHeadingHierarchy_EmptyState(t *testing.T) {
h := NewHeadingHierarchy()
if h.Breadcrumb() != "" || h.BreadcrumbWithHashes() != "" || h.Depth() != 0 {
t.Error("empty hierarchy should have no breadcrumb / depth=0")
}
}
func TestHeadingHierarchy_Reset(t *testing.T) {
h := NewHeadingHierarchy()
h.Observe("# A")
h.Observe("## B")
h.Reset()
if h.Breadcrumb() != "" || h.Depth() != 0 {
t.Error("Reset did not clear state")
}
}
func TestHeadingHierarchy_SkipLevels(t *testing.T) {
h := NewHeadingHierarchy()
// Document jumps from H1 directly to H3, skipping H2.
h.Observe("# Top")
h.Observe("### Deep")
if got := h.Breadcrumb(); got != "Top > Deep" {
t.Errorf("skipped-level breadcrumb: %q", got)
}
}