fix(chunker): QA audit — position invariant, recursive split, perf

Resolves the issues surfaced by a self-review of the adaptive chunking
work. Tests covering each regression were added first to prove the
fix.

Critical
- Heading splitter no longer prepends breadcrumb to Content. The
  breadcrumb now lives on Chunk.ContextHeader; EmbeddingContent()
  combines header + content for the embedder. This restores the
  End-Start == len(Content) invariant relied on by knowledge.go's
  document-reconstruction path (summary generation, UI highlight) and
  also eliminates the duplicate-heading regression where the section
  heading appeared twice in a chunk's body.
- splitBySeparators is now genuinely recursive: when one separator
  leaves a piece > ChunkSize, the next-priority separator is applied
  inside that piece. Mirrors the Python reference. Without this, a
  document with one paragraph break followed by a long run of
  newline-separated lines could emit a single ~1950-rune chunk for
  ChunkSize=300.

Medium
- SplitParentChild forces children onto the recursive tier, skipping
  the per-parent profile pass (was N extra O(N) scans).
- Heuristic splitter snaps overlap start to the nearest semantic
  boundary or newline instead of slicing mid-line / mid-word.
- Strategy.Split returns the legacy tier's output directly when the
  whole chain rejects rather than running SplitText a second time.

Low
- ensureDefaults caps ChunkOverlap at ChunkSize/2 so pathological
  configs (TokenLimit clamping ChunkSize tiny while Overlap stays
  high) cannot produce ~97% overlap chunks.
- DefaultChunkOverlap docstring spells out the migration story for
  KBs that stored ChunkOverlap=0.

ContextHeader plumbing
- chunker.Chunk and types.ParsedChunk gain ContextHeader plus an
  EmbeddingContent() helper.
- types.Chunk gets a transient (gorm:"-") ContextHeader field so the
  knowledge service can build embedding inputs as
  titlePrefix + chunk.EmbeddingContent() without a DB schema change.

https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
Claude
2026-05-02 10:06:53 +00:00
committed by lyingbug
parent b0fbb8da9f
commit bc44c31cec
10 changed files with 392 additions and 117 deletions
+27 -19
View File
@@ -2021,6 +2021,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
KnowledgeID: knowledge.ID,
KnowledgeBaseID: knowledge.KnowledgeBaseID,
Content: chunkData.Content,
ContextHeader: chunkData.ContextHeader,
ChunkIndex: int(chunkData.Seq),
IsEnabled: true,
CreatedAt: time.Now(),
@@ -2102,7 +2103,10 @@ func (s *knowledgeService) processChunks(ctx context.Context,
titlePrefix = t + "\n"
}
for _, chunk := range textChunks {
indexContent := titlePrefix + chunk.Content
// chunk.EmbeddingContent prepends ContextHeader (heading breadcrumb)
// when the chunker populated it during Tier-1 splitting; falls back
// to plain Content otherwise. Title prefix sits outermost.
indexContent := titlePrefix + chunk.EmbeddingContent()
indexInfoList = append(indexInfoList, &types.IndexInfo{
Content: indexContent,
SourceID: chunk.ID,
@@ -7494,11 +7498,12 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
parsed = make([]types.ParsedChunk, len(pcResult.Children))
for i, c := range pcResult.Children {
parsed[i] = types.ParsedChunk{
Content: c.Content,
Seq: c.Seq,
Start: c.Start,
End: c.End,
ParentIndex: c.ParentIndex,
Content: c.Content,
ContextHeader: c.ContextHeader,
Seq: c.Seq,
Start: c.Start,
End: c.End,
ParentIndex: c.ParentIndex,
}
}
parentChunks := make([]types.ParsedParentChunk, len(pcResult.Parents))
@@ -7511,10 +7516,11 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
parsed = make([]types.ParsedChunk, len(splitChunks))
for i, c := range splitChunks {
parsed[i] = types.ParsedChunk{
Content: c.Content,
Seq: c.Seq,
Start: c.Start,
End: c.End,
Content: c.Content,
ContextHeader: c.ContextHeader,
Seq: c.Seq,
Start: c.Start,
End: c.End,
}
}
}
@@ -8350,11 +8356,12 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
chunks = make([]types.ParsedChunk, len(pcResult.Children))
for i, c := range pcResult.Children {
chunks[i] = types.ParsedChunk{
Content: c.Content,
Seq: c.Seq,
Start: c.Start,
End: c.End,
ParentIndex: c.ParentIndex,
Content: c.Content,
ContextHeader: c.ContextHeader,
Seq: c.Seq,
Start: c.Start,
End: c.End,
ParentIndex: c.ParentIndex,
}
}
parentChunks := make([]types.ParsedParentChunk, len(pcResult.Parents))
@@ -8369,10 +8376,11 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
chunks = make([]types.ParsedChunk, len(splitChunks))
for i, c := range splitChunks {
chunks[i] = types.ParsedChunk{
Content: c.Content,
Seq: c.Seq,
Start: c.Start,
End: c.End,
Content: c.Content,
ContextHeader: c.ContextHeader,
Seq: c.Seq,
Start: c.Start,
End: c.End,
}
}
logger.Infof(ctx, "Split document into %d chunks for knowledge %s", len(chunks), knowledge.ID)