perf(chunker): thread DocProfile through splitter dispatch
Strategy.Split / SplitWithDiagnostics already run ProfileDocument once when the auto strategy resolves the chain. splitByHeadingsImpl was rerunning the same O(N) pass on entry, so every auto-mode call paid 2x scan cost; SplitParentChild paid 2x per parent. Pass the profile down through runTier and let splitters compute their own only when called outside the auto path (where profile == nil).
This commit is contained in:
@@ -25,11 +25,17 @@ type headingBoundary struct {
|
||||
// splitByHeadingsImpl is the Tier-1 implementation. It falls through to the
|
||||
// legacy splitter when the document has no usable heading structure or when
|
||||
// the heading split would produce a single section anyway.
|
||||
func splitByHeadingsImpl(text string, cfg SplitterConfig) []Chunk {
|
||||
//
|
||||
// profile may be nil; we compute one on demand. When the strategy resolver
|
||||
// already ran the profiler (auto strategy), the same profile is threaded
|
||||
// through here so we don't re-scan the entire document.
|
||||
func splitByHeadingsImpl(text string, cfg SplitterConfig, profile *DocProfile) []Chunk {
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
profile := ProfileDocument(text)
|
||||
if profile == nil {
|
||||
profile = ProfileDocument(text)
|
||||
}
|
||||
primaryLevel := profile.DominantHeadingLevel()
|
||||
if primaryLevel == 0 {
|
||||
return SplitText(text, cfg)
|
||||
|
||||
@@ -13,7 +13,7 @@ func TestSplitByHeadings_BasicSections(t *testing.T) {
|
||||
body := strings.Repeat("Lorem ipsum dolor sit amet consectetur adipiscing elit. ", 4)
|
||||
doc := "# Top\n" + body + "\n\n## Section A\n" + body + "\n\n## Section B\n" + body + "\n\n## Section C\n" + body
|
||||
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
chunks := splitByHeadingsImpl(doc, cfg, nil)
|
||||
if len(chunks) < 3 {
|
||||
t.Fatalf("expected ≥3 chunks (one per section), got %d", len(chunks))
|
||||
}
|
||||
@@ -43,7 +43,7 @@ func TestSplitByHeadings_BasicSections(t *testing.T) {
|
||||
func TestSplitByHeadings_FallsThroughForUnstructuredDoc(t *testing.T) {
|
||||
doc := "Just a plain paragraph without any headings at all in this text."
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
chunks := splitByHeadingsImpl(doc, cfg, nil)
|
||||
// no headings → falls through to SplitText, which keeps the whole thing
|
||||
if len(chunks) != 1 {
|
||||
t.Errorf("expected fallthrough single chunk, got %d", len(chunks))
|
||||
@@ -54,7 +54,7 @@ func TestSplitByHeadings_LargeSectionRecursesIntoLegacy(t *testing.T) {
|
||||
body := strings.Repeat("This is a long sentence repeated many times. ", 50)
|
||||
doc := "# Top\n## Big\n" + body
|
||||
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Separators: []string{". "}}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
chunks := splitByHeadingsImpl(doc, cfg, nil)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("large section should be sub-split, got %d chunks", len(chunks))
|
||||
}
|
||||
@@ -73,7 +73,7 @@ func TestSplitByHeadings_BreadcrumbReflectsLatestPath(t *testing.T) {
|
||||
body := strings.Repeat("Lorem ipsum dolor sit amet consectetur adipiscing elit. ", 4)
|
||||
doc := "# Chapter 1\n" + body + "\n\n## Section A\n" + body + "\n\n## Section B\n" + body
|
||||
cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
chunks := splitByHeadingsImpl(doc, cfg, nil)
|
||||
if len(chunks) < 3 {
|
||||
t.Fatalf("expected ≥3 chunks, got %d", len(chunks))
|
||||
}
|
||||
@@ -92,7 +92,7 @@ func TestSplitByHeadings_BreadcrumbReflectsLatestPath(t *testing.T) {
|
||||
func TestSplitByHeadings_IgnoresHeadingsInsideCodeFence(t *testing.T) {
|
||||
doc := "# Real\n\n```\n# Fake heading inside code\n```\n\nbody"
|
||||
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
chunks := splitByHeadingsImpl(doc, cfg, nil)
|
||||
for _, c := range chunks {
|
||||
if strings.Contains(c.ContextHeader, "# Real") || strings.Contains(c.Content, "# Real") {
|
||||
return
|
||||
@@ -104,7 +104,7 @@ func TestSplitByHeadings_IgnoresHeadingsInsideCodeFence(t *testing.T) {
|
||||
func TestSplitByHeadings_PreservesPositionRelativeToOriginal(t *testing.T) {
|
||||
doc := "# Top\nintro\n\n## A\nbody A\n\n## B\nbody B"
|
||||
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
chunks := splitByHeadingsImpl(doc, cfg, nil)
|
||||
for i, c := range chunks {
|
||||
if c.Start < 0 {
|
||||
t.Errorf("chunk %d has negative Start", i)
|
||||
@@ -131,7 +131,7 @@ content of B here.
|
||||
## Section C
|
||||
content of C here.`
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
chunks := splitByHeadingsImpl(doc, cfg, nil)
|
||||
if len(chunks) == 0 {
|
||||
t.Fatal("expected chunks")
|
||||
}
|
||||
@@ -171,7 +171,7 @@ ERROR: column missing.
|
||||
## 解析失败
|
||||
embedding 表缺列。`
|
||||
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
chunks := splitByHeadingsImpl(doc, cfg, nil)
|
||||
if len(chunks) == 0 {
|
||||
t.Fatal("expected at least one chunk")
|
||||
}
|
||||
@@ -219,7 +219,7 @@ short B.
|
||||
## C
|
||||
short C.`
|
||||
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
chunks := splitByHeadingsImpl(doc, cfg, nil)
|
||||
docRunes := []rune(doc)
|
||||
for i, c := range chunks {
|
||||
contentRuneLen := len([]rune(c.Content))
|
||||
@@ -248,7 +248,7 @@ func TestSplitByHeadings_CoalesceRespectsChunkSize(t *testing.T) {
|
||||
sb.WriteString("\nshort body line.\n")
|
||||
}
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(sb.String(), cfg)
|
||||
chunks := splitByHeadingsImpl(sb.String(), cfg, nil)
|
||||
for i, c := range chunks {
|
||||
if l := len([]rune(c.Content)); l > cfg.ChunkSize {
|
||||
t.Errorf("chunk %d exceeds ChunkSize: %d > %d", i, l, cfg.ChunkSize)
|
||||
@@ -288,7 +288,7 @@ body A.
|
||||
## Section B
|
||||
body B.`
|
||||
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0}
|
||||
chunks := splitByHeadingsImpl(doc, cfg)
|
||||
chunks := splitByHeadingsImpl(doc, cfg, nil)
|
||||
for i, c := range chunks {
|
||||
// Count occurrences of "## Section A" / "## Section B"
|
||||
for _, heading := range []string{"## Section A", "## Section B"} {
|
||||
|
||||
@@ -27,7 +27,11 @@ type boundary struct {
|
||||
|
||||
// splitByHeuristicsImpl is the Tier-2 implementation. Falls through to the
|
||||
// legacy splitter when no heuristic boundaries are found.
|
||||
func splitByHeuristicsImpl(text string, cfg SplitterConfig) []Chunk {
|
||||
//
|
||||
// profile is currently unused (this tier scans for boundaries directly) but
|
||||
// is accepted to keep the splitByHeadings / splitByHeuristics signatures
|
||||
// uniform — see strategy.runTier.
|
||||
func splitByHeuristicsImpl(text string, cfg SplitterConfig, _ *DocProfile) []Chunk {
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
func TestSplitByHeuristics_FormFeedBoundary(t *testing.T) {
|
||||
doc := strings.Repeat("page one body text. ", 30) + "\f" + strings.Repeat("page two body. ", 30)
|
||||
cfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 20, Separators: []string{". "}}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
chunks := splitByHeuristicsImpl(doc, cfg, nil)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("form feed should produce ≥2 chunks, got %d", len(chunks))
|
||||
}
|
||||
@@ -18,7 +18,7 @@ func TestSplitByHeuristics_NumberedSections(t *testing.T) {
|
||||
body := strings.Repeat("body sentence. ", 8)
|
||||
doc := "1. Introduction\n" + body + "\n\n2. Methods\n" + body + "\n\n3. Results\n" + body
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
chunks := splitByHeuristicsImpl(doc, cfg, nil)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("numbered sections should split: got %d chunks", len(chunks))
|
||||
}
|
||||
@@ -28,7 +28,7 @@ func TestSplitByHeuristics_GermanChapterMarkers(t *testing.T) {
|
||||
body := strings.Repeat("Beispieltext. ", 10)
|
||||
doc := "Kapitel 1: Einführung\n" + body + "\n\nKapitel 2: Hauptteil\n" + body
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
chunks := splitByHeuristicsImpl(doc, cfg, nil)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("German chapter markers should split: got %d", len(chunks))
|
||||
}
|
||||
@@ -38,7 +38,7 @@ func TestSplitByHeuristics_ChineseChapterMarkers(t *testing.T) {
|
||||
body := strings.Repeat("内容内容内容。", 60)
|
||||
doc := "第一章 引言\n" + body + "\n\n第二章 方法\n" + body
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{"。"}, Languages: []string{LangChinese}}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
chunks := splitByHeuristicsImpl(doc, cfg, nil)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("Chinese chapter markers should split: got %d", len(chunks))
|
||||
}
|
||||
@@ -47,7 +47,7 @@ func TestSplitByHeuristics_ChineseChapterMarkers(t *testing.T) {
|
||||
func TestSplitByHeuristics_FallsThroughForUnstructuredDoc(t *testing.T) {
|
||||
doc := strings.Repeat("plain prose without structure. ", 5)
|
||||
cfg := SplitterConfig{ChunkSize: 1000, ChunkOverlap: 20}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
chunks := splitByHeuristicsImpl(doc, cfg, nil)
|
||||
if len(chunks) != 1 {
|
||||
t.Errorf("unstructured short doc should be one chunk, got %d", len(chunks))
|
||||
}
|
||||
@@ -57,7 +57,7 @@ func TestSplitByHeuristics_OversizeBlockRecursesIntoLegacy(t *testing.T) {
|
||||
huge := strings.Repeat("This is a long sentence. ", 200) // ~5000 chars
|
||||
doc := "1. Intro\n" + huge
|
||||
cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 50, Separators: []string{". "}}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
chunks := splitByHeuristicsImpl(doc, cfg, nil)
|
||||
if len(chunks) < 5 {
|
||||
t.Errorf("oversize block should produce many sub-chunks, got %d", len(chunks))
|
||||
}
|
||||
@@ -83,7 +83,7 @@ func TestSplitByHeuristics_BoundariesAreOrdered(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSplitByHeuristics_EmptyText(t *testing.T) {
|
||||
if got := splitByHeuristicsImpl("", DefaultConfig()); got != nil {
|
||||
if got := splitByHeuristicsImpl("", DefaultConfig(), nil); got != nil {
|
||||
t.Errorf("empty doc should be nil, got %v", got)
|
||||
}
|
||||
}
|
||||
@@ -105,7 +105,7 @@ func TestSplitByHeuristics_OverlapActuallyOverlaps(t *testing.T) {
|
||||
doc := sb.String()
|
||||
|
||||
cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 80, Separators: []string{". "}}
|
||||
chunks := splitByHeuristicsImpl(doc, cfg)
|
||||
chunks := splitByHeuristicsImpl(doc, cfg, nil)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("need >=2 chunks to test overlap, got %d", len(chunks))
|
||||
}
|
||||
|
||||
@@ -35,12 +35,12 @@ func Split(text string, cfg SplitterConfig) []Chunk {
|
||||
return nil
|
||||
}
|
||||
cfg = ensureDefaults(cfg)
|
||||
chain, _ := resolveChainWithProfile(text, cfg)
|
||||
chain, profile := resolveChainWithProfile(text, cfg)
|
||||
totalChars := len([]rune(text))
|
||||
|
||||
var lastOut []Chunk
|
||||
for i, tier := range chain {
|
||||
out := runTier(tier, text, cfg)
|
||||
out := runTier(tier, text, cfg, profile)
|
||||
if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK {
|
||||
return out
|
||||
} else {
|
||||
@@ -100,7 +100,7 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic
|
||||
var lastOut []Chunk
|
||||
var lastTier StrategyTier
|
||||
for i, tier := range chain {
|
||||
out := runTier(tier, text, cfg)
|
||||
out := runTier(tier, text, cfg, profile)
|
||||
v := ValidateChunks(out, totalChars, cfg.ChunkSize)
|
||||
if v.OK {
|
||||
diag.SelectedTier = tier
|
||||
@@ -205,12 +205,16 @@ func resolveChainWithProfile(text string, cfg SplitterConfig) ([]StrategyTier, *
|
||||
// from heading_splitter.go / heuristic_splitter.go via init(); legacy
|
||||
// runs SplitText. The default branch is defensive for future
|
||||
// StrategyTier additions.
|
||||
func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk {
|
||||
//
|
||||
// profile may be nil when the caller did not run the document profiler
|
||||
// (explicit non-auto strategies skip profiling); splitters that need a
|
||||
// profile compute one on demand.
|
||||
func runTier(tier StrategyTier, text string, cfg SplitterConfig, profile *DocProfile) []Chunk {
|
||||
switch tier {
|
||||
case TierHeading:
|
||||
return splitByHeadings(text, cfg)
|
||||
return splitByHeadings(text, cfg, profile)
|
||||
case TierHeuristic:
|
||||
return splitByHeuristics(text, cfg)
|
||||
return splitByHeuristics(text, cfg, profile)
|
||||
case TierLegacy:
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
@@ -254,12 +258,14 @@ func ensureDefaults(cfg SplitterConfig) SplitterConfig {
|
||||
return cfg
|
||||
}
|
||||
|
||||
// splitByHeadings is overridden by heading_splitter.go.
|
||||
var splitByHeadings = func(text string, cfg SplitterConfig) []Chunk {
|
||||
// splitByHeadings is overridden by heading_splitter.go. The default no-op
|
||||
// fallback ignores the profile (profile may be nil; the override computes
|
||||
// one on demand when so).
|
||||
var splitByHeadings = func(text string, cfg SplitterConfig, _ *DocProfile) []Chunk {
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
|
||||
// splitByHeuristics is overridden by heuristic_splitter.go.
|
||||
var splitByHeuristics = func(text string, cfg SplitterConfig) []Chunk {
|
||||
// splitByHeuristics is overridden by heuristic_splitter.go. profile may be nil.
|
||||
var splitByHeuristics = func(text string, cfg SplitterConfig, _ *DocProfile) []Chunk {
|
||||
return SplitText(text, cfg)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user