From 592d3618dcce324790eb2366e5f078ceedaf9597 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 2 May 2026 09:23:10 +0000 Subject: [PATCH] feat(chunker): heading hierarchy tracker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds HeadingHierarchy, a level-stack that tracks Markdown heading nesting (#, ##, ###, ...). When a level-N heading is observed, all deeper entries are popped — so Breadcrumb() always reflects the active context path. Used by the upcoming heading-aware splitter to prepend a context breadcrumb (e.g. '# Chapter 1\n## Section 1.2') to every chunk inside a section, lifting embedding quality for non-trivial documents. The existing header_tracker (table headers) uses a binary start/end hook model that does not fit headings — a heading does not have an explicit end, it is replaced by the next heading of equal or shallower depth. A separate level-stack is the right shape. https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k --- .../chunker/heading_hierarchy.go | 111 ++++++++++++++++++ .../chunker/heading_hierarchy_test.go | 88 ++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 internal/infrastructure/chunker/heading_hierarchy.go create mode 100644 internal/infrastructure/chunker/heading_hierarchy_test.go diff --git a/internal/infrastructure/chunker/heading_hierarchy.go b/internal/infrastructure/chunker/heading_hierarchy.go new file mode 100644 index 00000000..739c691b --- /dev/null +++ b/internal/infrastructure/chunker/heading_hierarchy.go @@ -0,0 +1,111 @@ +// Package chunker - heading_hierarchy.go tracks Markdown heading nesting so +// that the heading-aware splitter (heading_splitter.go) can prepend a +// breadcrumb like "# Top > ## Section > ### Subsection" to each chunk. +// +// Conceptually similar to header_tracker.go (which tracks table headers via +// start/end hooks) but a Markdown heading does not have an explicit end — +// it is ended by the next heading of equal or shallower depth. The hook +// abstraction in header_tracker.go does not fit that pattern, so we model +// it as an explicit level-stack. +package chunker + +import "strings" + +// HeadingHierarchy maintains a stack of active Markdown headings indexed by +// level (1..6). Pushing a level-N heading pops every entry of level >= N +// because the previous siblings/descendants are no longer in scope. +type HeadingHierarchy struct { + // stack[i] holds the heading text for level i+1 (so stack[0] = H1). + // Entries beyond the deepest active level are empty strings. + stack [6]string + depth int // current deepest active level (0 if no active heading) +} + +// NewHeadingHierarchy returns an empty hierarchy. +func NewHeadingHierarchy() *HeadingHierarchy { + return &HeadingHierarchy{} +} + +// Observe parses line and updates the hierarchy if line is a Markdown +// heading. Returns the (level, headingText) when a heading was recognized, +// or (0, "") otherwise. Lines that look like headings inside fenced code +// blocks are NOT detected here — callers must avoid feeding code-block +// content to Observe (the heading splitter does so). +func (h *HeadingHierarchy) Observe(line string) (int, string) { + m := MarkdownHeadingPattern.FindStringSubmatch(line) + if m == nil { + return 0, "" + } + level := len(m[1]) + if level < 1 || level > 6 { + return 0, "" + } + heading := strings.TrimSpace(m[2]) + // Replace this level and clear deeper ones — siblings/descendants of + // the previous heading at this level are no longer in scope. + h.stack[level-1] = heading + for i := level; i < 6; i++ { + h.stack[i] = "" + } + if level > h.depth { + h.depth = level + } else { + // Recompute depth: it might shrink if we just pushed a shallower heading. + h.depth = 0 + for i := 0; i < 6; i++ { + if h.stack[i] != "" { + h.depth = i + 1 + } + } + } + return level, heading +} + +// Breadcrumb returns the current heading path joined by " > ", e.g. +// "Chapter 1 > Section 2 > Subsection a". Returns "" when no headings +// are active. +func (h *HeadingHierarchy) Breadcrumb() string { + if h.depth == 0 { + return "" + } + parts := make([]string, 0, h.depth) + for i := 0; i < h.depth; i++ { + if h.stack[i] != "" { + parts = append(parts, h.stack[i]) + } + } + return strings.Join(parts, " > ") +} + +// BreadcrumbWithHashes returns the path with the original `#` prefixes, +// suitable for embedding back into chunk content as a context header. +// Example: "# Chapter 1\n## Section 2\n### Subsection a" +func (h *HeadingHierarchy) BreadcrumbWithHashes() string { + if h.depth == 0 { + return "" + } + var sb strings.Builder + for i := 0; i < h.depth; i++ { + if h.stack[i] == "" { + continue + } + if sb.Len() > 0 { + sb.WriteByte('\n') + } + sb.WriteString(strings.Repeat("#", i+1)) + sb.WriteByte(' ') + sb.WriteString(h.stack[i]) + } + return sb.String() +} + +// Depth returns the current deepest active heading level. +func (h *HeadingHierarchy) Depth() int { return h.depth } + +// Reset clears all state. +func (h *HeadingHierarchy) Reset() { + for i := range h.stack { + h.stack[i] = "" + } + h.depth = 0 +} diff --git a/internal/infrastructure/chunker/heading_hierarchy_test.go b/internal/infrastructure/chunker/heading_hierarchy_test.go new file mode 100644 index 00000000..80eae4cb --- /dev/null +++ b/internal/infrastructure/chunker/heading_hierarchy_test.go @@ -0,0 +1,88 @@ +package chunker + +import "testing" + +func TestHeadingHierarchy_LinearNesting(t *testing.T) { + h := NewHeadingHierarchy() + h.Observe("# Chapter 1") + h.Observe("## Section 1.1") + h.Observe("### Subsection 1.1.1") + if got := h.Breadcrumb(); got != "Chapter 1 > Section 1.1 > Subsection 1.1.1" { + t.Errorf("breadcrumb mismatch: %q", got) + } +} + +func TestHeadingHierarchy_PopsDeeperOnSiblingHeading(t *testing.T) { + h := NewHeadingHierarchy() + h.Observe("# Chapter 1") + h.Observe("## Section 1.1") + h.Observe("### Subsection 1.1.1") + h.Observe("## Section 1.2") // pops the H3 + if got := h.Breadcrumb(); got != "Chapter 1 > Section 1.2" { + t.Errorf("breadcrumb after sibling: %q", got) + } +} + +func TestHeadingHierarchy_PopsAllOnNewTopLevel(t *testing.T) { + h := NewHeadingHierarchy() + h.Observe("# Chapter 1") + h.Observe("## Section A") + h.Observe("### Sub") + h.Observe("# Chapter 2") + if got := h.Breadcrumb(); got != "Chapter 2" { + t.Errorf("breadcrumb after new H1: %q", got) + } + if h.Depth() != 1 { + t.Errorf("depth should be 1, got %d", h.Depth()) + } +} + +func TestHeadingHierarchy_NonHeadingIgnored(t *testing.T) { + h := NewHeadingHierarchy() + h.Observe("# Title") + level, txt := h.Observe("just a paragraph") + if level != 0 || txt != "" { + t.Errorf("non-heading should not register: %d %q", level, txt) + } + if got := h.Breadcrumb(); got != "Title" { + t.Errorf("breadcrumb after non-heading: %q", got) + } +} + +func TestHeadingHierarchy_BreadcrumbWithHashes(t *testing.T) { + h := NewHeadingHierarchy() + h.Observe("# A") + h.Observe("## B") + got := h.BreadcrumbWithHashes() + want := "# A\n## B" + if got != want { + t.Errorf("hashes breadcrumb:\n got: %q\n want: %q", got, want) + } +} + +func TestHeadingHierarchy_EmptyState(t *testing.T) { + h := NewHeadingHierarchy() + if h.Breadcrumb() != "" || h.BreadcrumbWithHashes() != "" || h.Depth() != 0 { + t.Error("empty hierarchy should have no breadcrumb / depth=0") + } +} + +func TestHeadingHierarchy_Reset(t *testing.T) { + h := NewHeadingHierarchy() + h.Observe("# A") + h.Observe("## B") + h.Reset() + if h.Breadcrumb() != "" || h.Depth() != 0 { + t.Error("Reset did not clear state") + } +} + +func TestHeadingHierarchy_SkipLevels(t *testing.T) { + h := NewHeadingHierarchy() + // Document jumps from H1 directly to H3, skipping H2. + h.Observe("# Top") + h.Observe("### Deep") + if got := h.Breadcrumb(); got != "Top > Deep" { + t.Errorf("skipped-level breadcrumb: %q", got) + } +}