feat: add ListChunksByParentIDs method and enhance chunk merging logic

- Implemented ListChunksByParentIDs in chunkRepository to retrieve chunks based on multiple parent IDs, improving data retrieval efficiency.
- Enhanced the chunk merging logic in PluginMerge to better handle overlapping chunks and collect merged ImageInfo for parent chunks.
- Updated the ChunkRepository interface to include the new ListChunksByParentIDs method, ensuring consistency across the application.

These changes optimize chunk management and improve the handling of image information during merges, enhancing overall application functionality.
This commit is contained in:
wizardchen
2026-03-05 17:25:07 +08:00
committed by lyingbug
parent 4ddc468fea
commit 927e5d4750
3 changed files with 136 additions and 16 deletions
+17
View File
@@ -246,6 +246,23 @@ func (r *chunkRepository) ListChunkByParentID(
return chunks, nil
}
func (r *chunkRepository) ListChunksByParentIDs(
ctx context.Context,
tenantID uint64,
parentIDs []string,
) ([]*types.Chunk, error) {
if len(parentIDs) == 0 {
return nil, nil
}
var chunks []*types.Chunk
if err := r.db.WithContext(ctx).
Where("tenant_id = ? AND parent_chunk_id IN ?", tenantID, parentIDs).
Find(&chunks).Error; err != nil {
return nil, err
}
return chunks, nil
}
// UpdateChunk updates a chunk using GORM Save, which updates ALL fields
// except SeqID (auto-increment, must not be overwritten).
// Make sure the chunk object is complete (e.g., fetched from DB) before calling this method.
@@ -132,25 +132,35 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
knowledgeMergedChunks = append(knowledgeMergedChunks, chunks[i])
continue
}
// Merge overlapping chunks
if chunks[i].EndAt > lastChunk.EndAt {
contentRunes := []rune(chunks[i].Content)
offset := len(contentRunes) - (chunks[i].EndAt - lastChunk.EndAt)
lastChunk.Content = lastChunk.Content + string(contentRunes[offset:])
lastChunk.EndAt = chunks[i].EndAt
lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID)
// Merge overlapping chunks
if chunks[i].EndAt > lastChunk.EndAt {
contentRunes := []rune(chunks[i].Content)
offset := len(contentRunes) - (chunks[i].EndAt - lastChunk.EndAt)
lastChunk.Content = lastChunk.Content + string(contentRunes[offset:])
lastChunk.EndAt = chunks[i].EndAt
lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID)
// 合并 ImageInfo
if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil {
pipelineWarn(ctx, "Merge", "image_merge", map[string]interface{}{
"knowledge_id": knowledgeID,
"error": err.Error(),
})
}
if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil {
pipelineWarn(ctx, "Merge", "image_merge", map[string]interface{}{
"knowledge_id": knowledgeID,
"error": err.Error(),
})
}
if chunks[i].Score > lastChunk.Score {
lastChunk.Score = chunks[i].Score
} else {
// Fully contained: track the subsumed chunk and merge its ImageInfo
if !containsID(lastChunk.SubChunkID, chunks[i].ID) {
lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID)
}
if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil {
pipelineWarn(ctx, "Merge", "image_merge_contained", map[string]interface{}{
"knowledge_id": knowledgeID,
"error": err.Error(),
})
}
}
if chunks[i].Score > lastChunk.Score {
lastChunk.Score = chunks[i].Score
}
}
pipelineInfo(ctx, "Merge", "group_output", map[string]interface{}{
@@ -231,6 +241,11 @@ func (p *PluginMerge) resolveParentChunks(
parentMap[c.ID] = c
}
// Collect merged ImageInfo for each parent by fetching ALL sibling
// child chunks. Individual child chunks only carry ImageInfo for images
// within their own range, but the parent content spans all children.
parentImageInfoMap := p.collectParentImageInfo(ctx, tenantID, ids)
// Replace child content with parent content
for _, r := range results {
if r.ParentChunkID == "" {
@@ -249,6 +264,9 @@ func (p *PluginMerge) resolveParentChunks(
r.Content = parent.Content
r.StartAt = parent.StartAt
r.EndAt = parent.EndAt
if mergedImageInfo, ok := parentImageInfoMap[r.ParentChunkID]; ok && mergedImageInfo != "" {
r.ImageInfo = mergedImageInfo
}
// Track the original child as a sub-chunk
if !containsID(r.SubChunkID, r.ID) {
r.SubChunkID = append(r.SubChunkID, r.ID)
@@ -258,6 +276,89 @@ func (p *PluginMerge) resolveParentChunks(
return results
}
// collectParentImageInfo batch-fetches all child chunks for the given parents
// and merges their ImageInfo into a single JSON string per parent. This ensures
// that when child content is replaced with parent content, the complete set of
// image descriptions across all sibling chunks is preserved.
func (p *PluginMerge) collectParentImageInfo(
ctx context.Context,
tenantID uint64,
parentIDs []string,
) map[string]string {
result := make(map[string]string, len(parentIDs))
allChildren, err := p.chunkRepo.ListChunksByParentIDs(ctx, tenantID, parentIDs)
if err != nil {
pipelineWarn(ctx, "Merge", "parent_imageinfo_fetch_failed", map[string]interface{}{
"parent_cnt": len(parentIDs),
"error": err.Error(),
})
return result
}
// Group children by parent chunk ID, collecting unique ImageInfo entries
type parentAgg struct {
imageInfos []types.ImageInfo
uniqueURLs map[string]bool
siblingCnt int
}
aggMap := make(map[string]*parentAgg, len(parentIDs))
for _, child := range allChildren {
agg, ok := aggMap[child.ParentChunkID]
if !ok {
agg = &parentAgg{uniqueURLs: make(map[string]bool)}
aggMap[child.ParentChunkID] = agg
}
agg.siblingCnt++
if child.ImageInfo == "" {
continue
}
var infos []types.ImageInfo
if err := json.Unmarshal([]byte(child.ImageInfo), &infos); err != nil {
pipelineWarn(ctx, "Merge", "parent_imageinfo_parse", map[string]interface{}{
"chunk_id": child.ID,
"error": err.Error(),
})
continue
}
for _, info := range infos {
key := info.URL
if key == "" {
key = info.OriginalURL
}
if key != "" && !agg.uniqueURLs[key] {
agg.uniqueURLs[key] = true
agg.imageInfos = append(agg.imageInfos, info)
}
}
}
for parentID, agg := range aggMap {
if len(agg.imageInfos) == 0 {
continue
}
merged, err := json.Marshal(agg.imageInfos)
if err != nil {
pipelineWarn(ctx, "Merge", "parent_imageinfo_marshal", map[string]interface{}{
"parent_id": parentID,
"error": err.Error(),
})
continue
}
result[parentID] = string(merged)
pipelineInfo(ctx, "Merge", "parent_imageinfo_collected", map[string]interface{}{
"parent_id": parentID,
"sibling_cnt": agg.siblingCnt,
"image_cnt": len(agg.imageInfos),
})
}
return result
}
// filterHistoryResults retrieves history references and filters them by
// textual similarity to the current query. Only references that are above
// a Jaccard similarity threshold are kept, and their scores are discounted
+2
View File
@@ -44,6 +44,8 @@ type ChunkRepository interface {
knowledgeType string,
) ([]*types.Chunk, int64, error)
ListChunkByParentID(ctx context.Context, tenantID uint64, parentID string) ([]*types.Chunk, error)
// ListChunksByParentIDs lists chunks whose parent_chunk_id is in the given list
ListChunksByParentIDs(ctx context.Context, tenantID uint64, parentIDs []string) ([]*types.Chunk, error)
// UpdateChunk updates a chunk
UpdateChunk(ctx context.Context, chunk *types.Chunk) error
// UpdateChunks updates chunks in batch