diff --git a/internal/infrastructure/chunker/strategy.go b/internal/infrastructure/chunker/strategy.go index b1a6cf10..014f99c0 100644 --- a/internal/infrastructure/chunker/strategy.go +++ b/internal/infrastructure/chunker/strategy.go @@ -9,6 +9,7 @@ package chunker import ( "context" + "strings" "github.com/Tencent/WeKnora/internal/logger" ) @@ -125,9 +126,12 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic // It runs the tier selector for parent splitting, then re-splits each // parent into children with the small-chunk config. // -// Children are forced to recursive splitting — they are sub-pieces of an -// already-segmented parent, so re-profiling per parent would cost N extra -// O(N) document scans without any real chance of picking a better tier. +// Child splitting honours childCfg.Strategy. If it is empty/auto and a +// parent has its own internal structure (sub-headings, numbered sub- +// sections), the appropriate tier picks it up so child chunks carry a +// finer-grained breadcrumb than the parent's. Re-profiling each parent +// is bounded by O(sum(parent_size)) ≈ O(N) total, which is the same +// order as the original parent profiling pass. func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult { if text == "" { return ParentChildResult{} @@ -140,12 +144,6 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi return ParentChildResult{} } - // Pin children to the recursive tier so SplitText runs directly without - // the profiler/strategy chain re-deciding per parent. Preserves Tier-1 - // breadcrumb context (already attached to each parent) since we only - // re-split parent content, not re-detect headings. - childCfg.Strategy = StrategyRecursive - var newParents []Chunk var children []ChildChunk childSeq := 0 @@ -161,12 +159,7 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi sub.Seq = childSeq sub.Start += parent.Start sub.End += parent.Start - // Children inherit the parent's breadcrumb so embedding still - // sees the section context — but only if the child itself does - // not already carry one (sub-splits don't, but defensive). - if sub.ContextHeader == "" { - sub.ContextHeader = parent.ContextHeader - } + sub.ContextHeader = mergeBreadcrumbs(parent.ContextHeader, sub.ContextHeader) children = append(children, ChildChunk{Chunk: sub, ParentIndex: parentIndex}) childSeq++ } @@ -174,6 +167,30 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi return ParentChildResult{Parents: newParents, Children: children} } +// mergeBreadcrumbs combines the parent and child heading breadcrumbs into a +// single ContextHeader. When the child re-runs heading detection on parent +// content, its first breadcrumb line typically duplicates the parent's last +// line (the parent's leading heading sits at the top of the child's input); +// drop that duplicate so the embedding context isn't redundant. +func mergeBreadcrumbs(parent, child string) string { + if parent == "" { + return child + } + if child == "" { + return parent + } + parentLines := strings.Split(parent, "\n") + childLines := strings.Split(child, "\n") + if len(parentLines) > 0 && len(childLines) > 0 && + strings.TrimSpace(parentLines[len(parentLines)-1]) == strings.TrimSpace(childLines[0]) { + childLines = childLines[1:] + } + if len(childLines) == 0 { + return parent + } + return parent + "\n" + strings.Join(childLines, "\n") +} + // resolveChainWithProfile returns the strategy chain to attempt and, when // the chain was selected by the profiler (auto strategy), the DocProfile // that drove the selection. Profile is nil for explicit non-auto strategies diff --git a/internal/infrastructure/chunker/strategy_test.go b/internal/infrastructure/chunker/strategy_test.go index 662a9828..48a8803e 100644 --- a/internal/infrastructure/chunker/strategy_test.go +++ b/internal/infrastructure/chunker/strategy_test.go @@ -91,6 +91,90 @@ func TestSplit_PreservesPositionInvariantAcrossTiers(t *testing.T) { } } +// Children should pick up sub-heading context when childCfg uses auto +// strategy on a heading-rich document. Previously child splitting was +// pinned to the recursive tier, so sub-chunks all shared the parent's +// top-level breadcrumb regardless of inner H2/H3 structure. +func TestSplitParentChild_AutoStrategyEnrichesChildBreadcrumbs(t *testing.T) { + body := strings.Repeat("Lorem ipsum dolor sit amet. ", 40) + doc := "# Chapter\n" + body + + "\n\n## Section A\n" + body + + "\n\n## Section B\n" + body + + "\n\n## Section C\n" + body + seps := []string{"\n\n", "\n", ". "} + parentCfg := SplitterConfig{ChunkSize: 800, ChunkOverlap: 80, Strategy: StrategyAuto, Separators: seps} + childCfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Strategy: StrategyAuto, Separators: seps} + res := SplitParentChild(doc, parentCfg, childCfg) + if len(res.Children) == 0 { + t.Fatal("expected children") + } + + // At least one child should carry a breadcrumb that includes a + // sub-heading (## Section …) — that's only possible if the child + // splitter re-detected headings inside the parent. + saw := false + for _, c := range res.Children { + if strings.Contains(c.ContextHeader, "## Section") { + saw = true + break + } + } + if !saw { + t.Errorf("no child carries a sub-heading breadcrumb. samples:\n %q\n %q", + firstHeader(res.Children), lastHeader(res.Children)) + } + + // And no breadcrumb should contain the same line twice in a row + // (mergeBreadcrumbs deduplicates the parent/child seam). + for i, c := range res.Children { + lines := strings.Split(c.ContextHeader, "\n") + for j := 1; j < len(lines); j++ { + if strings.TrimSpace(lines[j]) != "" && lines[j] == lines[j-1] { + t.Errorf("child[%d] has duplicate breadcrumb lines: %q", i, c.ContextHeader) + break + } + } + } +} + +func firstHeader(cs []ChildChunk) string { + if len(cs) == 0 { + return "" + } + return cs[0].ContextHeader +} + +func lastHeader(cs []ChildChunk) string { + if len(cs) == 0 { + return "" + } + return cs[len(cs)-1].ContextHeader +} + +func TestMergeBreadcrumbs(t *testing.T) { + cases := []struct { + name string + parent, child string + want string + }{ + {"empty parent", "", "## Sub", "## Sub"}, + {"empty child", "# Top", "", "# Top"}, + {"both empty", "", "", ""}, + {"disjoint", "# Top", "## Other", "# Top\n## Other"}, + {"duplicate seam", "# Top\n## A", "## A\n### A1", "# Top\n## A\n### A1"}, + {"deep duplicate", "# Top", "# Top", "# Top"}, + {"only whitespace differs", "# Top\n## A", " ## A \n### A1", "# Top\n## A\n### A1"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := mergeBreadcrumbs(tc.parent, tc.child) + if got != tc.want { + t.Errorf("mergeBreadcrumbs(%q, %q) = %q, want %q", tc.parent, tc.child, got, tc.want) + } + }) + } +} + func TestSplitParentChild_LegacyStrategy(t *testing.T) { text := strings.Repeat("This is a sentence. Another one.\n\n", 50) parentCfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 40, Strategy: StrategyLegacy}