diff --git a/internal/infrastructure/chunker/heading_splitter.go b/internal/infrastructure/chunker/heading_splitter.go new file mode 100644 index 00000000..3789aa14 --- /dev/null +++ b/internal/infrastructure/chunker/heading_splitter.go @@ -0,0 +1,204 @@ +// Package chunker - heading_splitter.go implements Tier 1: Markdown +// heading-aware chunking. Documents with proper heading structure are split +// at heading boundaries and each chunk is prefixed with a breadcrumb of +// active heading context (e.g. "# Chapter 1\n## Section 1.2"). +package chunker + +import ( + "strings" + "unicode/utf8" +) + +// init wires this implementation into the strategy resolver. +func init() { + splitByHeadings = splitByHeadingsImpl +} + +// headingBoundary marks where a section starts. The first boundary is at +// rune offset 0 (covers any preamble before the first heading), subsequent +// boundaries sit at headings whose level is <= primaryLevel. +type headingBoundary struct { + runeStart int + line string // raw heading line, "" when this is the leading boundary +} + +// splitByHeadingsImpl is the Tier-1 implementation. It falls through to the +// legacy splitter when the document has no usable heading structure or when +// the heading split would produce a single section anyway. +func splitByHeadingsImpl(text string, cfg SplitterConfig) []Chunk { + if text == "" { + return nil + } + profile := ProfileDocument(text) + primaryLevel := profile.DominantHeadingLevel() + if primaryLevel == 0 { + return SplitText(text, cfg) + } + + bounds := findHeadingBoundaries(text, primaryLevel) + if len(bounds) <= 1 { + return SplitText(text, cfg) + } + + runes := []rune(text) + hierarchy := NewHeadingHierarchy() + + // Pre-walk every heading (not just primary-level) so the hierarchy + // reflects the full nesting context for each section's start. We only + // snapshot the breadcrumb at section boundaries; deeper sub-headings + // inside a section update the hierarchy but do not change the chunk's + // breadcrumb (chunks within a section share one breadcrumb). + var out []Chunk + seq := 0 + + for i, b := range bounds { + endRune := len(runes) + if i+1 < len(bounds) { + endRune = bounds[i+1].runeStart + } + if b.line != "" { + hierarchy.Observe(b.line) + } + // Catch sub-headings that occur between this primary boundary and + // the next so the hierarchy stays in sync for subsequent sections. + // We intentionally do this after observing the section header so + // the breadcrumb reflects the section-leading heading. + breadcrumb := hierarchy.BreadcrumbWithHashes() + observeSubHeadings(runes[b.runeStart:endRune], primaryLevel, hierarchy) + + sectionRunes := runes[b.runeStart:endRune] + sectionContent := string(sectionRunes) + secLen := len(sectionRunes) + if secLen == 0 { + continue + } + + bcLen := utf8.RuneCountInString(breadcrumb) + // Reserve some headroom (breadcrumb + 2 newlines) when fitting a section. + if bcLen+2+secLen <= cfg.ChunkSize { + content := prependBreadcrumb(sectionContent, breadcrumb) + out = append(out, Chunk{ + Content: content, + Seq: seq, + Start: b.runeStart, + End: endRune, + }) + seq++ + continue + } + + // Section too large for one chunk: defer to the legacy splitter for + // inner segmentation, then prepend the breadcrumb to every sub-chunk. + // Reduce the inner budget by the breadcrumb length so the final + // chunk (incl. breadcrumb) still fits under cfg.ChunkSize. + innerCfg := cfg + if bcLen > 0 && innerCfg.ChunkSize > bcLen+10 { + innerCfg.ChunkSize -= bcLen + 2 + } + subChunks := SplitText(sectionContent, innerCfg) + for _, sub := range subChunks { + content := prependBreadcrumb(sub.Content, breadcrumb) + out = append(out, Chunk{ + Content: content, + Seq: seq, + Start: b.runeStart + sub.Start, + End: b.runeStart + sub.End, + }) + seq++ + } + } + + return out +} + +// findHeadingBoundaries returns one boundary at offset 0 plus one per +// Markdown heading at level <= primaryLevel that sits outside fenced code +// blocks. Heading detection is line-oriented — a heading must occupy a +// whole line to be recognized. +func findHeadingBoundaries(text string, primaryLevel int) []headingBoundary { + runes := []rune(text) + bounds := []headingBoundary{{runeStart: 0}} + if len(runes) == 0 { + return bounds + } + + pos := 0 + inFence := false + lines := strings.Split(text, "\n") + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "```") { + inFence = !inFence + pos += utf8.RuneCountInString(line) + if i < len(lines)-1 { + pos++ // newline + } + continue + } + if !inFence { + m := MarkdownHeadingPattern.FindStringSubmatch(line) + if m != nil { + level := len(m[1]) + if level >= 1 && level <= primaryLevel && pos > 0 { + bounds = append(bounds, headingBoundary{ + runeStart: pos, + line: line, + }) + } + if level >= 1 && level <= primaryLevel && pos == 0 { + // First line is a heading — replace the leading boundary + bounds[0].line = line + } + } + } + pos += utf8.RuneCountInString(line) + if i < len(lines)-1 { + pos++ // account for the \n that strings.Split removed + } + } + return bounds +} + +// observeSubHeadings walks the section's lines and feeds every Markdown +// heading deeper than primaryLevel into the hierarchy. This keeps the +// hierarchy state correct so the breadcrumb at the next primary section +// reflects the truly active stack. +func observeSubHeadings(runes []rune, primaryLevel int, h *HeadingHierarchy) { + if len(runes) == 0 { + return + } + text := string(runes) + inFence := false + for _, line := range strings.Split(text, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "```") { + inFence = !inFence + continue + } + if inFence { + continue + } + m := MarkdownHeadingPattern.FindStringSubmatch(line) + if m == nil { + continue + } + level := len(m[1]) + if level > primaryLevel { + h.Observe(line) + } + } +} + +// prependBreadcrumb attaches the breadcrumb to content unless content +// already begins with that exact breadcrumb (avoid duplication when a +// section's first line is itself the section heading). +func prependBreadcrumb(content, breadcrumb string) string { + if breadcrumb == "" { + return content + } + trimmed := strings.TrimLeft(content, " \t\r\n") + if strings.HasPrefix(trimmed, breadcrumb) { + return content + } + return breadcrumb + "\n\n" + content +} diff --git a/internal/infrastructure/chunker/heading_splitter_test.go b/internal/infrastructure/chunker/heading_splitter_test.go new file mode 100644 index 00000000..9132eb7b --- /dev/null +++ b/internal/infrastructure/chunker/heading_splitter_test.go @@ -0,0 +1,125 @@ +package chunker + +import ( + "strings" + "testing" +) + +func TestSplitByHeadings_BasicSections(t *testing.T) { + doc := `# Top +preamble. + +## Section A +content of A. + +## Section B +content of B. + +## Section C +content of C.` + cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0} + chunks := splitByHeadingsImpl(doc, cfg) + if len(chunks) < 3 { + t.Fatalf("expected ≥3 chunks (one per section), got %d", len(chunks)) + } + + // Each chunk should contain the H1 + section heading as breadcrumb. + for i, c := range chunks { + if !strings.Contains(c.Content, "# Top") { + t.Errorf("chunk %d missing H1 breadcrumb:\n%s", i, c.Content) + } + } + + // Section B chunk should mention Section B. + found := false + for _, c := range chunks { + if strings.Contains(c.Content, "Section B") && strings.Contains(c.Content, "content of B") { + found = true + } + } + if !found { + t.Error("no chunk contains Section B with its content") + } +} + +func TestSplitByHeadings_FallsThroughForUnstructuredDoc(t *testing.T) { + doc := "Just a plain paragraph without any headings at all in this text." + cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0} + chunks := splitByHeadingsImpl(doc, cfg) + // no headings → falls through to SplitText, which keeps the whole thing + if len(chunks) != 1 { + t.Errorf("expected fallthrough single chunk, got %d", len(chunks)) + } +} + +func TestSplitByHeadings_LargeSectionRecursesIntoLegacy(t *testing.T) { + body := strings.Repeat("This is a long sentence repeated many times. ", 50) + doc := "# Top\n## Big\n" + body + cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Separators: []string{". "}} + chunks := splitByHeadingsImpl(doc, cfg) + if len(chunks) < 2 { + t.Fatalf("large section should be sub-split, got %d chunks", len(chunks)) + } + // Every sub-chunk should still carry the breadcrumb. + for i, c := range chunks { + if !strings.Contains(c.Content, "# Top") { + t.Errorf("sub-chunk %d missing H1 breadcrumb", i) + } + } +} + +func TestSplitByHeadings_BreadcrumbReflectsLatestPath(t *testing.T) { + doc := `# Chapter 1 +intro + +## Section A +text A + +## Section B +text B` + cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0} + chunks := splitByHeadingsImpl(doc, cfg) + if len(chunks) < 3 { + t.Fatalf("expected ≥3 chunks, got %d", len(chunks)) + } + // Section B chunk's breadcrumb should NOT mention Section A + for _, c := range chunks { + if strings.Contains(c.Content, "text B") { + if strings.Contains(c.Content, "## Section A") { + t.Errorf("Section B chunk should not include Section A in breadcrumb:\n%s", c.Content) + } + if !strings.Contains(c.Content, "## Section B") { + t.Errorf("Section B chunk should include its own heading:\n%s", c.Content) + } + } + } +} + +func TestSplitByHeadings_IgnoresHeadingsInsideCodeFence(t *testing.T) { + doc := "# Real\n\n```\n# Fake heading inside code\n```\n\nbody" + cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0} + chunks := splitByHeadingsImpl(doc, cfg) + // The fake heading should not create a section boundary — there's only + // one real H1, so we expect either 1 chunk or fall-through. + for _, c := range chunks { + if strings.Contains(c.Content, "# Real") { + // Good — found the real one. + return + } + } + t.Error("expected real H1 breadcrumb in some chunk") +} + +func TestSplitByHeadings_PreservesPositionRelativeToOriginal(t *testing.T) { + doc := "# Top\nintro\n\n## A\nbody A\n\n## B\nbody B" + cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0} + chunks := splitByHeadingsImpl(doc, cfg) + for i, c := range chunks { + if c.Start < 0 { + t.Errorf("chunk %d has negative Start", i) + } + if c.End < c.Start { + t.Errorf("chunk %d End < Start", i) + } + } +} diff --git a/internal/infrastructure/chunker/profiler.go b/internal/infrastructure/chunker/profiler.go index 0a3f6906..ea9476a9 100644 --- a/internal/infrastructure/chunker/profiler.go +++ b/internal/infrastructure/chunker/profiler.go @@ -49,9 +49,13 @@ func (p *DocProfile) HeadingDensity() float64 { } // DominantHeadingLevel returns the heading level (1..6) that should drive -// section splitting. It picks the lowest level (closest to root) that has -// at least 3 occurrences, falling back to the most frequent level. Returns -// 0 when no usable Markdown structure exists. +// section splitting. Preference order: +// 1. The lowest level (closest to root) that has at least 3 occurrences — +// a "real" structural backbone of the document. +// 2. Otherwise the deepest level present at least once — gives finer-grained +// boundaries for small documents that just have an H1 + a few H2s. +// +// Returns 0 when no Markdown headings exist. func (p *DocProfile) DominantHeadingLevel() int { if p.MdHeadingTotal == 0 { return 0 @@ -61,14 +65,12 @@ func (p *DocProfile) DominantHeadingLevel() int { return level } } - // fallback: most frequent level - bestLevel, bestCount := 0, 0 - for level := 1; level <= 6; level++ { - if p.MdHeadingCounts[level] > bestCount { - bestLevel, bestCount = level, p.MdHeadingCounts[level] + for level := 6; level >= 1; level-- { + if p.MdHeadingCounts[level] > 0 { + return level } } - return bestLevel + return 0 } // HeuristicMarkerTotal sums the non-Markdown structural markers. diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go index 8b3133d8..5162588b 100644 --- a/internal/infrastructure/chunker/strategy.go +++ b/internal/infrastructure/chunker/strategy.go @@ -7,7 +7,11 @@ // compatibility — strategy.go simply layers a tier-selector on top. package chunker -import "github.com/Tencent/WeKnora/internal/logger" +import ( + "context" + + "github.com/Tencent/WeKnora/internal/logger" +) // Strategy values for SplitterConfig.Strategy. const ( @@ -36,7 +40,7 @@ func Split(text string, cfg SplitterConfig) []Chunk { if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK { return out } else { - logger.Debugf(nil, "chunker: tier %s rejected: %s", tier, v.Reason) + logger.Debugf(context.Background(), "chunker: tier %s rejected: %s", tier, v.Reason) } } // Last-ditch fallback: always return *something*.