fix(chunker): QA audit — position invariant, recursive split, perf
Resolves the issues surfaced by a self-review of the adaptive chunking work. Tests covering each regression were added first to prove the fix. Critical - Heading splitter no longer prepends breadcrumb to Content. The breadcrumb now lives on Chunk.ContextHeader; EmbeddingContent() combines header + content for the embedder. This restores the End-Start == len(Content) invariant relied on by knowledge.go's document-reconstruction path (summary generation, UI highlight) and also eliminates the duplicate-heading regression where the section heading appeared twice in a chunk's body. - splitBySeparators is now genuinely recursive: when one separator leaves a piece > ChunkSize, the next-priority separator is applied inside that piece. Mirrors the Python reference. Without this, a document with one paragraph break followed by a long run of newline-separated lines could emit a single ~1950-rune chunk for ChunkSize=300. Medium - SplitParentChild forces children onto the recursive tier, skipping the per-parent profile pass (was N extra O(N) scans). - Heuristic splitter snaps overlap start to the nearest semantic boundary or newline instead of slicing mid-line / mid-word. - Strategy.Split returns the legacy tier's output directly when the whole chain rejects rather than running SplitText a second time. Low - ensureDefaults caps ChunkOverlap at ChunkSize/2 so pathological configs (TokenLimit clamping ChunkSize tiny while Overlap stays high) cannot produce ~97% overlap chunks. - DefaultChunkOverlap docstring spells out the migration story for KBs that stored ChunkOverlap=0. ContextHeader plumbing - chunker.Chunk and types.ParsedChunk gain ContextHeader plus an EmbeddingContent() helper. - types.Chunk gets a transient (gorm:"-") ContextHeader field so the knowledge service can build embedding inputs as titlePrefix + chunk.EmbeddingContent() without a DB schema change. https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
This commit is contained in:
@@ -2021,6 +2021,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
KnowledgeID: knowledge.ID,
|
||||
KnowledgeBaseID: knowledge.KnowledgeBaseID,
|
||||
Content: chunkData.Content,
|
||||
ContextHeader: chunkData.ContextHeader,
|
||||
ChunkIndex: int(chunkData.Seq),
|
||||
IsEnabled: true,
|
||||
CreatedAt: time.Now(),
|
||||
@@ -2102,7 +2103,10 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
titlePrefix = t + "\n"
|
||||
}
|
||||
for _, chunk := range textChunks {
|
||||
indexContent := titlePrefix + chunk.Content
|
||||
// chunk.EmbeddingContent prepends ContextHeader (heading breadcrumb)
|
||||
// when the chunker populated it during Tier-1 splitting; falls back
|
||||
// to plain Content otherwise. Title prefix sits outermost.
|
||||
indexContent := titlePrefix + chunk.EmbeddingContent()
|
||||
indexInfoList = append(indexInfoList, &types.IndexInfo{
|
||||
Content: indexContent,
|
||||
SourceID: chunk.ID,
|
||||
@@ -7494,11 +7498,12 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
parsed = make([]types.ParsedChunk, len(pcResult.Children))
|
||||
for i, c := range pcResult.Children {
|
||||
parsed[i] = types.ParsedChunk{
|
||||
Content: c.Content,
|
||||
Seq: c.Seq,
|
||||
Start: c.Start,
|
||||
End: c.End,
|
||||
ParentIndex: c.ParentIndex,
|
||||
Content: c.Content,
|
||||
ContextHeader: c.ContextHeader,
|
||||
Seq: c.Seq,
|
||||
Start: c.Start,
|
||||
End: c.End,
|
||||
ParentIndex: c.ParentIndex,
|
||||
}
|
||||
}
|
||||
parentChunks := make([]types.ParsedParentChunk, len(pcResult.Parents))
|
||||
@@ -7511,10 +7516,11 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
parsed = make([]types.ParsedChunk, len(splitChunks))
|
||||
for i, c := range splitChunks {
|
||||
parsed[i] = types.ParsedChunk{
|
||||
Content: c.Content,
|
||||
Seq: c.Seq,
|
||||
Start: c.Start,
|
||||
End: c.End,
|
||||
Content: c.Content,
|
||||
ContextHeader: c.ContextHeader,
|
||||
Seq: c.Seq,
|
||||
Start: c.Start,
|
||||
End: c.End,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8350,11 +8356,12 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
chunks = make([]types.ParsedChunk, len(pcResult.Children))
|
||||
for i, c := range pcResult.Children {
|
||||
chunks[i] = types.ParsedChunk{
|
||||
Content: c.Content,
|
||||
Seq: c.Seq,
|
||||
Start: c.Start,
|
||||
End: c.End,
|
||||
ParentIndex: c.ParentIndex,
|
||||
Content: c.Content,
|
||||
ContextHeader: c.ContextHeader,
|
||||
Seq: c.Seq,
|
||||
Start: c.Start,
|
||||
End: c.End,
|
||||
ParentIndex: c.ParentIndex,
|
||||
}
|
||||
}
|
||||
parentChunks := make([]types.ParsedParentChunk, len(pcResult.Parents))
|
||||
@@ -8369,10 +8376,11 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
|
||||
chunks = make([]types.ParsedChunk, len(splitChunks))
|
||||
for i, c := range splitChunks {
|
||||
chunks[i] = types.ParsedChunk{
|
||||
Content: c.Content,
|
||||
Seq: c.Seq,
|
||||
Start: c.Start,
|
||||
End: c.End,
|
||||
Content: c.Content,
|
||||
ContextHeader: c.ContextHeader,
|
||||
Seq: c.Seq,
|
||||
Start: c.Start,
|
||||
End: c.End,
|
||||
}
|
||||
}
|
||||
logger.Infof(ctx, "Split document into %d chunks for knowledge %s", len(chunks), knowledge.ID)
|
||||
|
||||
Reference in New Issue
Block a user