feat(chunker): SplitParentChild children honour configured strategy
Children were forced to StrategyRecursive on the assumption that re-profiling each parent would be too expensive. In practice profiling each parent is bounded by O(sum(parent_size)) ≈ O(N) total, the same order as the original parent profiling pass — and the gain is real: when a parent (e.g. an H1 chapter) contains its own sub-headings, the heading splitter on child input now picks them up and generates a finer-grained breadcrumb instead of every child sharing the parent's top-level breadcrumb. Add mergeBreadcrumbs to combine the parent ContextHeader with the child's freshly-derived one, dropping the duplicated seam line that appears when the child's first heading equals the parent's last. Tests cover (a) sub-headings now appearing in child breadcrumbs and (b) the merge dedup against duplicated seam lines.
This commit is contained in:
@@ -9,6 +9,7 @@ package chunker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
@@ -125,9 +126,12 @@ func SplitWithDiagnostics(text string, cfg SplitterConfig) ([]Chunk, *Diagnostic
|
||||
// It runs the tier selector for parent splitting, then re-splits each
|
||||
// parent into children with the small-chunk config.
|
||||
//
|
||||
// Children are forced to recursive splitting — they are sub-pieces of an
|
||||
// already-segmented parent, so re-profiling per parent would cost N extra
|
||||
// O(N) document scans without any real chance of picking a better tier.
|
||||
// Child splitting honours childCfg.Strategy. If it is empty/auto and a
|
||||
// parent has its own internal structure (sub-headings, numbered sub-
|
||||
// sections), the appropriate tier picks it up so child chunks carry a
|
||||
// finer-grained breadcrumb than the parent's. Re-profiling each parent
|
||||
// is bounded by O(sum(parent_size)) ≈ O(N) total, which is the same
|
||||
// order as the original parent profiling pass.
|
||||
func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChildResult {
|
||||
if text == "" {
|
||||
return ParentChildResult{}
|
||||
@@ -140,12 +144,6 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi
|
||||
return ParentChildResult{}
|
||||
}
|
||||
|
||||
// Pin children to the recursive tier so SplitText runs directly without
|
||||
// the profiler/strategy chain re-deciding per parent. Preserves Tier-1
|
||||
// breadcrumb context (already attached to each parent) since we only
|
||||
// re-split parent content, not re-detect headings.
|
||||
childCfg.Strategy = StrategyRecursive
|
||||
|
||||
var newParents []Chunk
|
||||
var children []ChildChunk
|
||||
childSeq := 0
|
||||
@@ -161,12 +159,7 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi
|
||||
sub.Seq = childSeq
|
||||
sub.Start += parent.Start
|
||||
sub.End += parent.Start
|
||||
// Children inherit the parent's breadcrumb so embedding still
|
||||
// sees the section context — but only if the child itself does
|
||||
// not already carry one (sub-splits don't, but defensive).
|
||||
if sub.ContextHeader == "" {
|
||||
sub.ContextHeader = parent.ContextHeader
|
||||
}
|
||||
sub.ContextHeader = mergeBreadcrumbs(parent.ContextHeader, sub.ContextHeader)
|
||||
children = append(children, ChildChunk{Chunk: sub, ParentIndex: parentIndex})
|
||||
childSeq++
|
||||
}
|
||||
@@ -174,6 +167,30 @@ func SplitParentChild(text string, parentCfg, childCfg SplitterConfig) ParentChi
|
||||
return ParentChildResult{Parents: newParents, Children: children}
|
||||
}
|
||||
|
||||
// mergeBreadcrumbs combines the parent and child heading breadcrumbs into a
|
||||
// single ContextHeader. When the child re-runs heading detection on parent
|
||||
// content, its first breadcrumb line typically duplicates the parent's last
|
||||
// line (the parent's leading heading sits at the top of the child's input);
|
||||
// drop that duplicate so the embedding context isn't redundant.
|
||||
func mergeBreadcrumbs(parent, child string) string {
|
||||
if parent == "" {
|
||||
return child
|
||||
}
|
||||
if child == "" {
|
||||
return parent
|
||||
}
|
||||
parentLines := strings.Split(parent, "\n")
|
||||
childLines := strings.Split(child, "\n")
|
||||
if len(parentLines) > 0 && len(childLines) > 0 &&
|
||||
strings.TrimSpace(parentLines[len(parentLines)-1]) == strings.TrimSpace(childLines[0]) {
|
||||
childLines = childLines[1:]
|
||||
}
|
||||
if len(childLines) == 0 {
|
||||
return parent
|
||||
}
|
||||
return parent + "\n" + strings.Join(childLines, "\n")
|
||||
}
|
||||
|
||||
// resolveChainWithProfile returns the strategy chain to attempt and, when
|
||||
// the chain was selected by the profiler (auto strategy), the DocProfile
|
||||
// that drove the selection. Profile is nil for explicit non-auto strategies
|
||||
|
||||
@@ -91,6 +91,90 @@ func TestSplit_PreservesPositionInvariantAcrossTiers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Children should pick up sub-heading context when childCfg uses auto
|
||||
// strategy on a heading-rich document. Previously child splitting was
|
||||
// pinned to the recursive tier, so sub-chunks all shared the parent's
|
||||
// top-level breadcrumb regardless of inner H2/H3 structure.
|
||||
func TestSplitParentChild_AutoStrategyEnrichesChildBreadcrumbs(t *testing.T) {
|
||||
body := strings.Repeat("Lorem ipsum dolor sit amet. ", 40)
|
||||
doc := "# Chapter\n" + body +
|
||||
"\n\n## Section A\n" + body +
|
||||
"\n\n## Section B\n" + body +
|
||||
"\n\n## Section C\n" + body
|
||||
seps := []string{"\n\n", "\n", ". "}
|
||||
parentCfg := SplitterConfig{ChunkSize: 800, ChunkOverlap: 80, Strategy: StrategyAuto, Separators: seps}
|
||||
childCfg := SplitterConfig{ChunkSize: 200, ChunkOverlap: 20, Strategy: StrategyAuto, Separators: seps}
|
||||
res := SplitParentChild(doc, parentCfg, childCfg)
|
||||
if len(res.Children) == 0 {
|
||||
t.Fatal("expected children")
|
||||
}
|
||||
|
||||
// At least one child should carry a breadcrumb that includes a
|
||||
// sub-heading (## Section …) — that's only possible if the child
|
||||
// splitter re-detected headings inside the parent.
|
||||
saw := false
|
||||
for _, c := range res.Children {
|
||||
if strings.Contains(c.ContextHeader, "## Section") {
|
||||
saw = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !saw {
|
||||
t.Errorf("no child carries a sub-heading breadcrumb. samples:\n %q\n %q",
|
||||
firstHeader(res.Children), lastHeader(res.Children))
|
||||
}
|
||||
|
||||
// And no breadcrumb should contain the same line twice in a row
|
||||
// (mergeBreadcrumbs deduplicates the parent/child seam).
|
||||
for i, c := range res.Children {
|
||||
lines := strings.Split(c.ContextHeader, "\n")
|
||||
for j := 1; j < len(lines); j++ {
|
||||
if strings.TrimSpace(lines[j]) != "" && lines[j] == lines[j-1] {
|
||||
t.Errorf("child[%d] has duplicate breadcrumb lines: %q", i, c.ContextHeader)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func firstHeader(cs []ChildChunk) string {
|
||||
if len(cs) == 0 {
|
||||
return ""
|
||||
}
|
||||
return cs[0].ContextHeader
|
||||
}
|
||||
|
||||
func lastHeader(cs []ChildChunk) string {
|
||||
if len(cs) == 0 {
|
||||
return ""
|
||||
}
|
||||
return cs[len(cs)-1].ContextHeader
|
||||
}
|
||||
|
||||
func TestMergeBreadcrumbs(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
parent, child string
|
||||
want string
|
||||
}{
|
||||
{"empty parent", "", "## Sub", "## Sub"},
|
||||
{"empty child", "# Top", "", "# Top"},
|
||||
{"both empty", "", "", ""},
|
||||
{"disjoint", "# Top", "## Other", "# Top\n## Other"},
|
||||
{"duplicate seam", "# Top\n## A", "## A\n### A1", "# Top\n## A\n### A1"},
|
||||
{"deep duplicate", "# Top", "# Top", "# Top"},
|
||||
{"only whitespace differs", "# Top\n## A", " ## A \n### A1", "# Top\n## A\n### A1"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := mergeBreadcrumbs(tc.parent, tc.child)
|
||||
if got != tc.want {
|
||||
t.Errorf("mergeBreadcrumbs(%q, %q) = %q, want %q", tc.parent, tc.child, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitParentChild_LegacyStrategy(t *testing.T) {
|
||||
text := strings.Repeat("This is a sentence. Another one.\n\n", 50)
|
||||
parentCfg := SplitterConfig{ChunkSize: 400, ChunkOverlap: 40, Strategy: StrategyLegacy}
|
||||
|
||||
Reference in New Issue
Block a user