perf(chunker): thread DocProfile through splitter dispatch

Strategy.Split / SplitWithDiagnostics already run ProfileDocument once
when the auto strategy resolves the chain. splitByHeadingsImpl was
rerunning the same O(N) pass on entry, so every auto-mode call paid
2x scan cost; SplitParentChild paid 2x per parent.

Pass the profile down through runTier and let splitters compute their
own only when called outside the auto path (where profile == nil).
This commit is contained in:
wizardchen
2026-05-06 20:56:01 +08:00
committed by lyingbug
parent 5a79a817a1
commit 151e999db1
5 changed files with 48 additions and 32 deletions
@@ -25,11 +25,17 @@ type headingBoundary struct {
// splitByHeadingsImpl is the Tier-1 implementation. It falls through to the
// legacy splitter when the document has no usable heading structure or when
// the heading split would produce a single section anyway.
func splitByHeadingsImpl(text string, cfg SplitterConfig) []Chunk {
//
// profile may be nil; we compute one on demand. When the strategy resolver
// already ran the profiler (auto strategy), the same profile is threaded
// through here so we don't re-scan the entire document.
func splitByHeadingsImpl(text string, cfg SplitterConfig, profile *DocProfile) []Chunk {
if text == "" {
return nil
}
profile := ProfileDocument(text)
if profile == nil {
profile = ProfileDocument(text)
}
primaryLevel := profile.DominantHeadingLevel()
if primaryLevel == 0 {
return SplitText(text, cfg)
@@ -13,7 +13,7 @@ func TestSplitByHeadings_BasicSections(t *testing.T) {
body := strings.Repeat("Lorem ipsum dolor sit amet consectetur adipiscing elit. ", 4)
doc := "# Top\n" + body + "\n\n## Section A\n" + body + "\n\n## Section B\n" + body + "\n\n## Section C\n" + body
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
chunks := splitByHeadingsImpl(doc, cfg, nil)
if len(chunks) < 3 {
t.Fatalf("expected ≥3 chunks (one per section), got %d", len(chunks))
}
@@ -43,7 +43,7 @@ func TestSplitByHeadings_BasicSections(t *testing.T) {
func TestSplitByHeadings_FallsThroughForUnstructuredDoc(t *testing.T) {
doc := "Just a plain paragraph without any headings at all in this text."
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
chunks := splitByHeadingsImpl(doc, cfg, nil)
// no headings → falls through to SplitText, which keeps the whole thing
if len(chunks) != 1 {
t.Errorf("expected fallthrough single chunk, got %d", len(chunks))
@@ -54,7 +54,7 @@ func TestSplitByHeadings_LargeSectionRecursesIntoLegacy(t *testing.T) {
body := strings.Repeat("This is a long sentence repeated many times. ", 50)
doc := "# Top\n## Big\n" + body
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Separators: []string{". "}}
chunks := splitByHeadingsImpl(doc, cfg)
chunks := splitByHeadingsImpl(doc, cfg, nil)
if len(chunks) < 2 {
t.Fatalf("large section should be sub-split, got %d chunks", len(chunks))
}
@@ -73,7 +73,7 @@ func TestSplitByHeadings_BreadcrumbReflectsLatestPath(t *testing.T) {
body := strings.Repeat("Lorem ipsum dolor sit amet consectetur adipiscing elit. ", 4)
doc := "# Chapter 1\n" + body + "\n\n## Section A\n" + body + "\n\n## Section B\n" + body
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
chunks := splitByHeadingsImpl(doc, cfg, nil)
if len(chunks) < 3 {
t.Fatalf("expected ≥3 chunks, got %d", len(chunks))
}
@@ -92,7 +92,7 @@ func TestSplitByHeadings_BreadcrumbReflectsLatestPath(t *testing.T) {
func TestSplitByHeadings_IgnoresHeadingsInsideCodeFence(t *testing.T) {
doc := "# Real\n\n```\n# Fake heading inside code\n```\n\nbody"
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
chunks := splitByHeadingsImpl(doc, cfg, nil)
for _, c := range chunks {
if strings.Contains(c.ContextHeader, "# Real") || strings.Contains(c.Content, "# Real") {
return
@@ -104,7 +104,7 @@ func TestSplitByHeadings_IgnoresHeadingsInsideCodeFence(t *testing.T) {
func TestSplitByHeadings_PreservesPositionRelativeToOriginal(t *testing.T) {
doc := "# Top\nintro\n\n## A\nbody A\n\n## B\nbody B"
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
chunks := splitByHeadingsImpl(doc, cfg, nil)
for i, c := range chunks {
if c.Start < 0 {
t.Errorf("chunk %d has negative Start", i)
@@ -131,7 +131,7 @@ content of B here.
## Section C
content of C here.`
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20}
chunks := splitByHeadingsImpl(doc, cfg)
chunks := splitByHeadingsImpl(doc, cfg, nil)
if len(chunks) == 0 {
t.Fatal("expected chunks")
}
@@ -171,7 +171,7 @@ ERROR: column missing.
## 解析失败
embedding 表缺列。`
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
chunks := splitByHeadingsImpl(doc, cfg, nil)
if len(chunks) == 0 {
t.Fatal("expected at least one chunk")
}
@@ -219,7 +219,7 @@ short B.
## C
short C.`
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
chunks := splitByHeadingsImpl(doc, cfg, nil)
docRunes := []rune(doc)
for i, c := range chunks {
contentRuneLen := len([]rune(c.Content))
@@ -248,7 +248,7 @@ func TestSplitByHeadings_CoalesceRespectsChunkSize(t *testing.T) {
sb.WriteString("\nshort body line.\n")
}
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(sb.String(), cfg)
chunks := splitByHeadingsImpl(sb.String(), cfg, nil)
for i, c := range chunks {
if l := len([]rune(c.Content)); l > cfg.ChunkSize {
t.Errorf("chunk %d exceeds ChunkSize: %d > %d", i, l, cfg.ChunkSize)
@@ -288,7 +288,7 @@ body A.
## Section B
body B.`
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
chunks := splitByHeadingsImpl(doc, cfg)
chunks := splitByHeadingsImpl(doc, cfg, nil)
for i, c := range chunks {
// Count occurrences of "## Section A" / "## Section B"
for _, heading := range []string{"## Section A", "## Section B"} {
@@ -27,7 +27,11 @@ type boundary struct {
// splitByHeuristicsImpl is the Tier-2 implementation. Falls through to the
// legacy splitter when no heuristic boundaries are found.
func splitByHeuristicsImpl(text string, cfg SplitterConfig) []Chunk {
//
// profile is currently unused (this tier scans for boundaries directly) but
// is accepted to keep the splitByHeadings / splitByHeuristics signatures
// uniform — see strategy.runTier.
func splitByHeuristicsImpl(text string, cfg SplitterConfig, _ *DocProfile) []Chunk {
if text == "" {
return nil
}
@@ -8,7 +8,7 @@ import (
func TestSplitByHeuristics_FormFeedBoundary(t *testing.T) {
doc := strings.Repeat("page one body text. ", 30) + "\f" + strings.Repeat("page two body. ", 30)
cfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 20, Separators: []string{". "}}
chunks := splitByHeuristicsImpl(doc, cfg)
chunks := splitByHeuristicsImpl(doc, cfg, nil)
if len(chunks) < 2 {
t.Fatalf("form feed should produce ≥2 chunks, got %d", len(chunks))
}
@@ -18,7 +18,7 @@ func TestSplitByHeuristics_NumberedSections(t *testing.T) {
body := strings.Repeat("body sentence. ", 8)
doc := "1. Introduction\n" + body + "\n\n2. Methods\n" + body + "\n\n3. Results\n" + body
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}}
chunks := splitByHeuristicsImpl(doc, cfg)
chunks := splitByHeuristicsImpl(doc, cfg, nil)
if len(chunks) < 2 {
t.Fatalf("numbered sections should split: got %d chunks", len(chunks))
}
@@ -28,7 +28,7 @@ func TestSplitByHeuristics_GermanChapterMarkers(t *testing.T) {
body := strings.Repeat("Beispieltext. ", 10)
doc := "Kapitel 1: Einführung\n" + body + "\n\nKapitel 2: Hauptteil\n" + body
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}}
chunks := splitByHeuristicsImpl(doc, cfg)
chunks := splitByHeuristicsImpl(doc, cfg, nil)
if len(chunks) < 2 {
t.Fatalf("German chapter markers should split: got %d", len(chunks))
}
@@ -38,7 +38,7 @@ func TestSplitByHeuristics_ChineseChapterMarkers(t *testing.T) {
body := strings.Repeat("内容内容内容。", 60)
doc := "第一章 引言\n" + body + "\n\n第二章 方法\n" + body
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{"。"}, Languages: []string{LangChinese}}
chunks := splitByHeuristicsImpl(doc, cfg)
chunks := splitByHeuristicsImpl(doc, cfg, nil)
if len(chunks) < 2 {
t.Fatalf("Chinese chapter markers should split: got %d", len(chunks))
}
@@ -47,7 +47,7 @@ func TestSplitByHeuristics_ChineseChapterMarkers(t *testing.T) {
func TestSplitByHeuristics_FallsThroughForUnstructuredDoc(t *testing.T) {
doc := strings.Repeat("plain prose without structure. ", 5)
cfg := SplitterConfig{ChunkSize: 1000, ChunkOverlap: 20}
chunks := splitByHeuristicsImpl(doc, cfg)
chunks := splitByHeuristicsImpl(doc, cfg, nil)
if len(chunks) != 1 {
t.Errorf("unstructured short doc should be one chunk, got %d", len(chunks))
}
@@ -57,7 +57,7 @@ func TestSplitByHeuristics_OversizeBlockRecursesIntoLegacy(t *testing.T) {
huge := strings.Repeat("This is a long sentence. ", 200) // ~5000 chars
doc := "1. Intro\n" + huge
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 50, Separators: []string{". "}}
chunks := splitByHeuristicsImpl(doc, cfg)
chunks := splitByHeuristicsImpl(doc, cfg, nil)
if len(chunks) < 5 {
t.Errorf("oversize block should produce many sub-chunks, got %d", len(chunks))
}
@@ -83,7 +83,7 @@ func TestSplitByHeuristics_BoundariesAreOrdered(t *testing.T) {
}
func TestSplitByHeuristics_EmptyText(t *testing.T) {
if got := splitByHeuristicsImpl("", DefaultConfig()); got != nil {
if got := splitByHeuristicsImpl("", DefaultConfig(), nil); got != nil {
t.Errorf("empty doc should be nil, got %v", got)
}
}
@@ -105,7 +105,7 @@ func TestSplitByHeuristics_OverlapActuallyOverlaps(t *testing.T) {
doc := sb.String()
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 80, Separators: []string{". "}}
chunks := splitByHeuristicsImpl(doc, cfg)
chunks := splitByHeuristicsImpl(doc, cfg, nil)
if len(chunks) < 2 {
t.Fatalf("need >=2 chunks to test overlap, got %d", len(chunks))
}
+16 -10
View File
@@ -35,12 +35,12 @@ func Split(text string, cfg SplitterConfig) []Chunk {
return nil
}
cfg = ensureDefaults(cfg)
chain, _ := resolveChainWithProfile(text, cfg)
chain, profile := resolveChainWithProfile(text, cfg)
totalChars := len([]rune(text))
var lastOut []Chunk
for i, tier := range chain {
out := runTier(tier, text, cfg)
out := runTier(tier, text, cfg, profile)
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
return out
} else {
@@ -100,7 +100,7 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic
var lastOut []Chunk
var lastTier StrategyTier
for i, tier := range chain {
out := runTier(tier, text, cfg)
out := runTier(tier, text, cfg, profile)
v := ValidateChunks(out, totalChars, cfg.ChunkSize)
if v.OK {
diag.SelectedTier = tier
@@ -205,12 +205,16 @@ func resolveChainWithProfile(text string, cfg SplitterConfig) ([]StrategyTier, *
// from heading_splitter.go / heuristic_splitter.go via init(); legacy
// runs SplitText. The default branch is defensive for future
// StrategyTier additions.
func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk {
//
// profile may be nil when the caller did not run the document profiler
// (explicit non-auto strategies skip profiling); splitters that need a
// profile compute one on demand.
func runTier(tier StrategyTier, text string, cfg SplitterConfig, profile *DocProfile) []Chunk {
switch tier {
case TierHeading:
return splitByHeadings(text, cfg)
return splitByHeadings(text, cfg, profile)
case TierHeuristic:
return splitByHeuristics(text, cfg)
return splitByHeuristics(text, cfg, profile)
case TierLegacy:
return SplitText(text, cfg)
}
@@ -254,12 +258,14 @@ func ensureDefaults(cfg SplitterConfig) SplitterConfig {
return cfg
}
// splitByHeadings is overridden by heading_splitter.go.
var splitByHeadings = func(text string, cfg SplitterConfig) []Chunk {
// splitByHeadings is overridden by heading_splitter.go. The default no-op
// fallback ignores the profile (profile may be nil; the override computes
// one on demand when so).
var splitByHeadings = func(text string, cfg SplitterConfig, _ *DocProfile) []Chunk {
return SplitText(text, cfg)
}
// splitByHeuristics is overridden by heuristic_splitter.go.
var splitByHeuristics = func(text string, cfg SplitterConfig) []Chunk {
// splitByHeuristics is overridden by heuristic_splitter.go. profile may be nil.
var splitByHeuristics = func(text string, cfg SplitterConfig, _ *DocProfile) []Chunk {
return SplitText(text, cfg)
}