feat(chunker): SplitParentChild children honour configured strategy

Children were forced to StrategyRecursive on the assumption that
re-profiling each parent would be too expensive. In practice profiling
each parent is bounded by O(sum(parent_size)) ≈ O(N) total, the same
order as the original parent profiling pass — and the gain is real:
when a parent (e.g. an H1 chapter) contains its own sub-headings, the
heading splitter on child input now picks them up and generates a
finer-grained breadcrumb instead of every child sharing the parent's
top-level breadcrumb.

Add mergeBreadcrumbs to combine the parent ContextHeader with the
child's freshly-derived one, dropping the duplicated seam line that
appears when the child's first heading equals the parent's last.

Tests cover (a) sub-headings now appearing in child breadcrumbs and
(b) the merge dedup against duplicated seam lines.
This commit is contained in:
wizardchen
2026-05-06 21:04:51 +08:00
committed by lyingbug
parent 7b9a831db3
commit d703a14745
2 changed files with 116 additions and 15 deletions
+32 -15
View File
@@ -9,6 +9,7 @@ package chunker
import (
"context"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
)
@@ -125,9 +126,12 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic
// It runs the tier selector for parent splitting, then re-splits each
// parent into children with the small-chunk config.
//
// Children are forced to recursive splitting — they are sub-pieces of an
// already-segmented parent, so re-profiling per parent would cost N extra
// O(N) document scans without any real chance of picking a better tier.
// Child splitting honours childCfg.Strategy. If it is empty/auto and a
// parent has its own internal structure (sub-headings, numbered sub-
// sections), the appropriate tier picks it up so child chunks carry a
// finer-grained breadcrumb than the parent's. Re-profiling each parent
// is bounded by O(sum(parent_size)) ≈ O(N) total, which is the same
// order as the original parent profiling pass.
func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
if text == "" {
return ParentChildResult{}
@@ -140,12 +144,6 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi
return ParentChildResult{}
}
// Pin children to the recursive tier so SplitText runs directly without
// the profiler/strategy chain re-deciding per parent. Preserves Tier-1
// breadcrumb context (already attached to each parent) since we only
// re-split parent content, not re-detect headings.
childCfg.Strategy = StrategyRecursive
var newParents []Chunk
var children []ChildChunk
childSeq := 0
@@ -161,12 +159,7 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi
sub.Seq = childSeq
sub.Start += parent.Start
sub.End += parent.Start
// Children inherit the parent's breadcrumb so embedding still
// sees the section context — but only if the child itself does
// not already carry one (sub-splits don't, but defensive).
if sub.ContextHeader == "" {
sub.ContextHeader = parent.ContextHeader
}
sub.ContextHeader = mergeBreadcrumbs(parent.ContextHeader, sub.ContextHeader)
children = append(children, ChildChunk{Chunk: sub, ParentIndex: parentIndex})
childSeq++
}
@@ -174,6 +167,30 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi
return ParentChildResult{Parents: newParents, Children: children}
}
// mergeBreadcrumbs combines the parent and child heading breadcrumbs into a
// single ContextHeader. When the child re-runs heading detection on parent
// content, its first breadcrumb line typically duplicates the parent's last
// line (the parent's leading heading sits at the top of the child's input);
// drop that duplicate so the embedding context isn't redundant.
func mergeBreadcrumbs(parent, child string) string {
if parent == "" {
return child
}
if child == "" {
return parent
}
parentLines := strings.Split(parent, "\n")
childLines := strings.Split(child, "\n")
if len(parentLines) > 0 && len(childLines) > 0 &&
strings.TrimSpace(parentLines[len(parentLines)-1]) == strings.TrimSpace(childLines[0]) {
childLines = childLines[1:]
}
if len(childLines) == 0 {
return parent
}
return parent + "\n" + strings.Join(childLines, "\n")
}
// resolveChainWithProfile returns the strategy chain to attempt and, when
// the chain was selected by the profiler (auto strategy), the DocProfile
// that drove the selection. Profile is nil for explicit non-auto strategies
@@ -91,6 +91,90 @@ func TestSplit_PreservesPositionInvariantAcrossTiers(t *testing.T) {
}
}
// Children should pick up sub-heading context when childCfg uses auto
// strategy on a heading-rich document. Previously child splitting was
// pinned to the recursive tier, so sub-chunks all shared the parent's
// top-level breadcrumb regardless of inner H2/H3 structure.
func TestSplitParentChild_AutoStrategyEnrichesChildBreadcrumbs(t *testing.T) {
body := strings.Repeat("Lorem ipsum dolor sit amet. ", 40)
doc := "# Chapter\n" + body +
"\n\n## Section A\n" + body +
"\n\n## Section B\n" + body +
"\n\n## Section C\n" + body
seps := []string{"\n\n", "\n", ". "}
parentCfg := SplitterConfig{ChunkSize: 800, ChunkOverlap: 80, Strategy: StrategyAuto, Separators: seps}
childCfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Strategy: StrategyAuto, Separators: seps}
res := SplitParentChild(doc, parentCfg, childCfg)
if len(res.Children) == 0 {
t.Fatal("expected children")
}
// At least one child should carry a breadcrumb that includes a
// sub-heading (## Section …) — that's only possible if the child
// splitter re-detected headings inside the parent.
saw := false
for _, c := range res.Children {
if strings.Contains(c.ContextHeader, "## Section") {
saw = true
break
}
}
if !saw {
t.Errorf("no child carries a sub-heading breadcrumb. samples:\n %q\n %q",
firstHeader(res.Children), lastHeader(res.Children))
}
// And no breadcrumb should contain the same line twice in a row
// (mergeBreadcrumbs deduplicates the parent/child seam).
for i, c := range res.Children {
lines := strings.Split(c.ContextHeader, "\n")
for j := 1; j < len(lines); j++ {
if strings.TrimSpace(lines[j]) != "" && lines[j] == lines[j-1] {
t.Errorf("child[%d] has duplicate breadcrumb lines: %q", i, c.ContextHeader)
break
}
}
}
}
func firstHeader(cs []ChildChunk) string {
if len(cs) == 0 {
return ""
}
return cs[0].ContextHeader
}
func lastHeader(cs []ChildChunk) string {
if len(cs) == 0 {
return ""
}
return cs[len(cs)-1].ContextHeader
}
func TestMergeBreadcrumbs(t *testing.T) {
cases := []struct {
name string
parent, child string
want string
}{
{"empty parent", "", "## Sub", "## Sub"},
{"empty child", "# Top", "", "# Top"},
{"both empty", "", "", ""},
{"disjoint", "# Top", "## Other", "# Top\n## Other"},
{"duplicate seam", "# Top\n## A", "## A\n### A1", "# Top\n## A\n### A1"},
{"deep duplicate", "# Top", "# Top", "# Top"},
{"only whitespace differs", "# Top\n## A", " ## A \n### A1", "# Top\n## A\n### A1"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := mergeBreadcrumbs(tc.parent, tc.child)
if got != tc.want {
t.Errorf("mergeBreadcrumbs(%q, %q) = %q, want %q", tc.parent, tc.child, got, tc.want)
}
})
}
}
func TestSplitParentChild_LegacyStrategy(t *testing.T) {
text := strings.Repeat("This is a sentence. Another one.\n\n", 50)
parentCfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 40, Strategy: StrategyLegacy}