From 151e999db1fdb05f164972b6d02ef108ea9407c3 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 6 May 2026 20:56:01 +0800 Subject: [PATCH] perf(chunker): thread DocProfile through splitter dispatch Strategy.Split / SplitWithDiagnostics already run ProfileDocument once when the auto strategy resolves the chain. splitByHeadingsImpl was rerunning the same O(N) pass on entry, so every auto-mode call paid 2x scan cost; SplitParentChild paid 2x per parent. Pass the profile down through runTier and let splitters compute their own only when called outside the auto path (where profile == nil). --- .../chunker/heading_splitter.go | 10 +++++-- .../chunker/heading_splitter_test.go | 22 ++++++++-------- .../chunker/heuristic_splitter.go | 6 ++++- .../chunker/heuristic_splitter_test.go | 16 ++++++------ internal/infrastructure/chunker/strategy.go | 26 ++++++++++++------- 5 files changed, 48 insertions(+), 32 deletions(-) diff --git a/internal/infrastructure/chunker/heading_splitter.go b/internal/infrastructure/chunker/heading_splitter.go index 4ea46dd7..c9242d91 100644 --- a/internal/infrastructure/chunker/heading_splitter.go +++ b/internal/infrastructure/chunker/heading_splitter.go @@ -25,11 +25,17 @@ type headingBoundary struct { // splitByHeadingsImpl is the Tier-1 implementation. It falls through to the // legacy splitter when the document has no usable heading structure or when // the heading split would produce a single section anyway. -func splitByHeadingsImpl(text string, cfg SplitterConfig) []Chunk { +// +// profile may be nil; we compute one on demand. When the strategy resolver +// already ran the profiler (auto strategy), the same profile is threaded +// through here so we don't re-scan the entire document. +func splitByHeadingsImpl(text string, cfg SplitterConfig, profile *DocProfile) []Chunk { if text == "" { return nil } - profile := ProfileDocument(text) + if profile == nil { + profile = ProfileDocument(text) + } primaryLevel := profile.DominantHeadingLevel() if primaryLevel == 0 { return SplitText(text, cfg) diff --git a/internal/infrastructure/chunker/heading_splitter_test.go b/internal/infrastructure/chunker/heading_splitter_test.go index 1efcbd24..82e1509d 100644 --- a/internal/infrastructure/chunker/heading_splitter_test.go +++ b/internal/infrastructure/chunker/heading_splitter_test.go @@ -13,7 +13,7 @@ func TestSplitByHeadings_BasicSections(t *testing.T) { body := strings.Repeat("Lorem ipsum dolor sit amet consectetur adipiscing elit. ", 4) doc := "# Top\n" + body + "\n\n## Section A\n" + body + "\n\n## Section B\n" + body + "\n\n## Section C\n" + body cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 0} - chunks := splitByHeadingsImpl(doc, cfg) + chunks := splitByHeadingsImpl(doc, cfg, nil) if len(chunks) < 3 { t.Fatalf("expected ≥3 chunks (one per section), got %d", len(chunks)) } @@ -43,7 +43,7 @@ func TestSplitByHeadings_BasicSections(t *testing.T) { func TestSplitByHeadings_FallsThroughForUnstructuredDoc(t *testing.T) { doc := "Just a plain paragraph without any headings at all in this text." cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0} - chunks := splitByHeadingsImpl(doc, cfg) + chunks := splitByHeadingsImpl(doc, cfg, nil) // no headings → falls through to SplitText, which keeps the whole thing if len(chunks) != 1 { t.Errorf("expected fallthrough single chunk, got %d", len(chunks)) @@ -54,7 +54,7 @@ func TestSplitByHeadings_LargeSectionRecursesIntoLegacy(t *testing.T) { body := strings.Repeat("This is a long sentence repeated many times. ", 50) doc := "# Top\n## Big\n" + body cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 30, Separators: []string{". "}} - chunks := splitByHeadingsImpl(doc, cfg) + chunks := splitByHeadingsImpl(doc, cfg, nil) if len(chunks) < 2 { t.Fatalf("large section should be sub-split, got %d chunks", len(chunks)) } @@ -73,7 +73,7 @@ func TestSplitByHeadings_BreadcrumbReflectsLatestPath(t *testing.T) { body := strings.Repeat("Lorem ipsum dolor sit amet consectetur adipiscing elit. ", 4) doc := "# Chapter 1\n" + body + "\n\n## Section A\n" + body + "\n\n## Section B\n" + body cfg := SplitterConfig{ChunkSize: 300, ChunkOverlap: 0} - chunks := splitByHeadingsImpl(doc, cfg) + chunks := splitByHeadingsImpl(doc, cfg, nil) if len(chunks) < 3 { t.Fatalf("expected ≥3 chunks, got %d", len(chunks)) } @@ -92,7 +92,7 @@ func TestSplitByHeadings_BreadcrumbReflectsLatestPath(t *testing.T) { func TestSplitByHeadings_IgnoresHeadingsInsideCodeFence(t *testing.T) { doc := "# Real\n\n```\n# Fake heading inside code\n```\n\nbody" cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0} - chunks := splitByHeadingsImpl(doc, cfg) + chunks := splitByHeadingsImpl(doc, cfg, nil) for _, c := range chunks { if strings.Contains(c.ContextHeader, "# Real") || strings.Contains(c.Content, "# Real") { return @@ -104,7 +104,7 @@ func TestSplitByHeadings_IgnoresHeadingsInsideCodeFence(t *testing.T) { func TestSplitByHeadings_PreservesPositionRelativeToOriginal(t *testing.T) { doc := "# Top\nintro\n\n## A\nbody A\n\n## B\nbody B" cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0} - chunks := splitByHeadingsImpl(doc, cfg) + chunks := splitByHeadingsImpl(doc, cfg, nil) for i, c := range chunks { if c.Start < 0 { t.Errorf("chunk %d has negative Start", i) @@ -131,7 +131,7 @@ content of B here. ## Section C content of C here.` cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20} - chunks := splitByHeadingsImpl(doc, cfg) + chunks := splitByHeadingsImpl(doc, cfg, nil) if len(chunks) == 0 { t.Fatal("expected chunks") } @@ -171,7 +171,7 @@ ERROR: column missing. ## 解析失败 embedding 表缺列。` cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0} - chunks := splitByHeadingsImpl(doc, cfg) + chunks := splitByHeadingsImpl(doc, cfg, nil) if len(chunks) == 0 { t.Fatal("expected at least one chunk") } @@ -219,7 +219,7 @@ short B. ## C short C.` cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0} - chunks := splitByHeadingsImpl(doc, cfg) + chunks := splitByHeadingsImpl(doc, cfg, nil) docRunes := []rune(doc) for i, c := range chunks { contentRuneLen := len([]rune(c.Content)) @@ -248,7 +248,7 @@ func TestSplitByHeadings_CoalesceRespectsChunkSize(t *testing.T) { sb.WriteString("\nshort body line.\n") } cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 0} - chunks := splitByHeadingsImpl(sb.String(), cfg) + chunks := splitByHeadingsImpl(sb.String(), cfg, nil) for i, c := range chunks { if l := len([]rune(c.Content)); l > cfg.ChunkSize { t.Errorf("chunk %d exceeds ChunkSize: %d > %d", i, l, cfg.ChunkSize) @@ -288,7 +288,7 @@ body A. ## Section B body B.` cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 0} - chunks := splitByHeadingsImpl(doc, cfg) + chunks := splitByHeadingsImpl(doc, cfg, nil) for i, c := range chunks { // Count occurrences of "## Section A" / "## Section B" for _, heading := range []string{"## Section A", "## Section B"} { diff --git a/internal/infrastructure/chunker/heuristic_splitter.go b/internal/infrastructure/chunker/heuristic_splitter.go index e4012d03..5e642ccd 100644 --- a/internal/infrastructure/chunker/heuristic_splitter.go +++ b/internal/infrastructure/chunker/heuristic_splitter.go @@ -27,7 +27,11 @@ type boundary struct { // splitByHeuristicsImpl is the Tier-2 implementation. Falls through to the // legacy splitter when no heuristic boundaries are found. -func splitByHeuristicsImpl(text string, cfg SplitterConfig) []Chunk { +// +// profile is currently unused (this tier scans for boundaries directly) but +// is accepted to keep the splitByHeadings / splitByHeuristics signatures +// uniform — see strategy.runTier. +func splitByHeuristicsImpl(text string, cfg SplitterConfig, _ *DocProfile) []Chunk { if text == "" { return nil } diff --git a/internal/infrastructure/chunker/heuristic_splitter_test.go b/internal/infrastructure/chunker/heuristic_splitter_test.go index a37f7cd9..c26854d5 100644 --- a/internal/infrastructure/chunker/heuristic_splitter_test.go +++ b/internal/infrastructure/chunker/heuristic_splitter_test.go @@ -8,7 +8,7 @@ import ( func TestSplitByHeuristics_FormFeedBoundary(t *testing.T) { doc := strings.Repeat("page one body text. ", 30) + "\f" + strings.Repeat("page two body. ", 30) cfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 20, Separators: []string{". "}} - chunks := splitByHeuristicsImpl(doc, cfg) + chunks := splitByHeuristicsImpl(doc, cfg, nil) if len(chunks) < 2 { t.Fatalf("form feed should produce ≥2 chunks, got %d", len(chunks)) } @@ -18,7 +18,7 @@ func TestSplitByHeuristics_NumberedSections(t *testing.T) { body := strings.Repeat("body sentence. ", 8) doc := "1. Introduction\n" + body + "\n\n2. Methods\n" + body + "\n\n3. Results\n" + body cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}} - chunks := splitByHeuristicsImpl(doc, cfg) + chunks := splitByHeuristicsImpl(doc, cfg, nil) if len(chunks) < 2 { t.Fatalf("numbered sections should split: got %d chunks", len(chunks)) } @@ -28,7 +28,7 @@ func TestSplitByHeuristics_GermanChapterMarkers(t *testing.T) { body := strings.Repeat("Beispieltext. ", 10) doc := "Kapitel 1: Einführung\n" + body + "\n\nKapitel 2: Hauptteil\n" + body cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{". "}} - chunks := splitByHeuristicsImpl(doc, cfg) + chunks := splitByHeuristicsImpl(doc, cfg, nil) if len(chunks) < 2 { t.Fatalf("German chapter markers should split: got %d", len(chunks)) } @@ -38,7 +38,7 @@ func TestSplitByHeuristics_ChineseChapterMarkers(t *testing.T) { body := strings.Repeat("内容内容内容。", 60) doc := "第一章 引言\n" + body + "\n\n第二章 方法\n" + body cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Separators: []string{"。"}, Languages: []string{LangChinese}} - chunks := splitByHeuristicsImpl(doc, cfg) + chunks := splitByHeuristicsImpl(doc, cfg, nil) if len(chunks) < 2 { t.Fatalf("Chinese chapter markers should split: got %d", len(chunks)) } @@ -47,7 +47,7 @@ func TestSplitByHeuristics_ChineseChapterMarkers(t *testing.T) { func TestSplitByHeuristics_FallsThroughForUnstructuredDoc(t *testing.T) { doc := strings.Repeat("plain prose without structure. ", 5) cfg := SplitterConfig{ChunkSize: 1000, ChunkOverlap: 20} - chunks := splitByHeuristicsImpl(doc, cfg) + chunks := splitByHeuristicsImpl(doc, cfg, nil) if len(chunks) != 1 { t.Errorf("unstructured short doc should be one chunk, got %d", len(chunks)) } @@ -57,7 +57,7 @@ func TestSplitByHeuristics_OversizeBlockRecursesIntoLegacy(t *testing.T) { huge := strings.Repeat("This is a long sentence. ", 200) // ~5000 chars doc := "1. Intro\n" + huge cfg := SplitterConfig{ChunkSize: 500, ChunkOverlap: 50, Separators: []string{". "}} - chunks := splitByHeuristicsImpl(doc, cfg) + chunks := splitByHeuristicsImpl(doc, cfg, nil) if len(chunks) < 5 { t.Errorf("oversize block should produce many sub-chunks, got %d", len(chunks)) } @@ -83,7 +83,7 @@ func TestSplitByHeuristics_BoundariesAreOrdered(t *testing.T) { } func TestSplitByHeuristics_EmptyText(t *testing.T) { - if got := splitByHeuristicsImpl("", DefaultConfig()); got != nil { + if got := splitByHeuristicsImpl("", DefaultConfig(), nil); got != nil { t.Errorf("empty doc should be nil, got %v", got) } } @@ -105,7 +105,7 @@ func TestSplitByHeuristics_OverlapActuallyOverlaps(t *testing.T) { doc := sb.String() cfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 80, Separators: []string{". "}} - chunks := splitByHeuristicsImpl(doc, cfg) + chunks := splitByHeuristicsImpl(doc, cfg, nil) if len(chunks) < 2 { t.Fatalf("need >=2 chunks to test overlap, got %d", len(chunks)) } diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go index 671d8038..b1a6cf10 100644 --- a/internal/infrastructure/chunker/strategy.go +++ b/internal/infrastructure/chunker/strategy.go @@ -35,12 +35,12 @@ func Split(text string, cfg SplitterConfig) []Chunk { return nil } cfg = ensureDefaults(cfg) - chain, _ := resolveChainWithProfile(text, cfg) + chain, profile := resolveChainWithProfile(text, cfg) totalChars := len([]rune(text)) var lastOut []Chunk for i, tier := range chain { - out := runTier(tier, text, cfg) + out := runTier(tier, text, cfg, profile) if v := ValidateChunks(out, totalChars, cfg.ChunkSize); v.OK { return out } else { @@ -100,7 +100,7 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic var lastOut []Chunk var lastTier StrategyTier for i, tier := range chain { - out := runTier(tier, text, cfg) + out := runTier(tier, text, cfg, profile) v := ValidateChunks(out, totalChars, cfg.ChunkSize) if v.OK { diag.SelectedTier = tier @@ -205,12 +205,16 @@ func resolveChainWithProfile(text string, cfg SplitterConfig) ([]StrategyTier, * // from heading_splitter.go / heuristic_splitter.go via init(); legacy // runs SplitText. The default branch is defensive for future // StrategyTier additions. -func runTier(tier StrategyTier, text string, cfg SplitterConfig) []Chunk { +// +// profile may be nil when the caller did not run the document profiler +// (explicit non-auto strategies skip profiling); splitters that need a +// profile compute one on demand. +func runTier(tier StrategyTier, text string, cfg SplitterConfig, profile *DocProfile) []Chunk { switch tier { case TierHeading: - return splitByHeadings(text, cfg) + return splitByHeadings(text, cfg, profile) case TierHeuristic: - return splitByHeuristics(text, cfg) + return splitByHeuristics(text, cfg, profile) case TierLegacy: return SplitText(text, cfg) } @@ -254,12 +258,14 @@ func ensureDefaults(cfg SplitterConfig) SplitterConfig { return cfg } -// splitByHeadings is overridden by heading_splitter.go. -var splitByHeadings = func(text string, cfg SplitterConfig) []Chunk { +// splitByHeadings is overridden by heading_splitter.go. The default no-op +// fallback ignores the profile (profile may be nil; the override computes +// one on demand when so). +var splitByHeadings = func(text string, cfg SplitterConfig, _ *DocProfile) []Chunk { return SplitText(text, cfg) } -// splitByHeuristics is overridden by heuristic_splitter.go. -var splitByHeuristics = func(text string, cfg SplitterConfig) []Chunk { +// splitByHeuristics is overridden by heuristic_splitter.go. profile may be nil. +var splitByHeuristics = func(text string, cfg SplitterConfig, _ *DocProfile) []Chunk { return SplitText(text, cfg) }