Files
Enterprise-Knowledge-Base/internal/application/service/chat_pipeline/merge_overlap.go
T
wizardchen f92302cc64 fix(knowledge): stitch chunks by text overlap, not by position
Reconstructing full document content from chunks relied on position math
(offset = content length - (EndAt - lastEndAt)) that assumes
len([]rune(Content)) == EndAt - StartAt. Two upstream behaviors break that
invariant and caused the merged "全文" view to drop or duplicate text:

- The parent-child chunker prepends a synthetic table header to split
  tables; that header is zero-width in position space (start == end), so
  Content is longer than EndAt - StartAt.
- Chunk Content can retain HTML entities (", > ...), inflating the
  rune count relative to the source span.

Replace position-based trimming with text-overlap matching: find where the
accumulated text's suffix reappears at the head of the next chunk and join
there. Positions are only used to size the search window. This naturally
skips the prepended header and is immune to entity length drift.

Extract the logic into searchutil.AppendWithOverlap / MergeTextChunks and
reuse it across the three backend stitching sites (wiki_ingest
reconstructContent, graph mergeChunkContents, chat_pipeline
mergeOverlappingChunks). Mirror the same algorithm in the frontend
doc-content merge. Also fix the markdown hr/heading styles (thin solid
divider, primary heading color).
2026-05-30 17:53:47 +08:00

135 lines
4.0 KiB
Go

package chatpipeline
import (
"context"
"encoding/json"
"sort"
"github.com/Tencent/WeKnora/internal/searchutil"
"github.com/Tencent/WeKnora/internal/types"
)
// mergeOverlappingChunks merges chunks with overlapping or adjacent StartAt/EndAt
// ranges within a single knowledge source group. Chunks MUST be pre-sorted by
// StartAt ascending, EndAt ascending. The highest score among merged chunks is kept.
func (p *PluginMerge) mergeOverlappingChunks(
ctx context.Context,
knowledgeID string,
chunks []*types.SearchResult,
) []*types.SearchResult {
if len(chunks) == 0 {
return nil
}
merged := []*types.SearchResult{chunks[0]}
for i := 1; i < len(chunks); i++ {
lastChunk := merged[len(merged)-1]
// Non-overlapping: add as a new entry
if chunks[i].StartAt > lastChunk.EndAt {
merged = append(merged, chunks[i])
continue
}
// Partial overlap: append the non-overlapping suffix.
//
// 重叠去重统一交给 searchutil.AppendWithOverlap(按文本匹配),它能
// 兼容父子分块器补写的零宽表头,以及 HTML 实体导致的 content 长度与
// EndAt-StartAt 不一致——这两种情况都会让按位置裁剪错位、丢字或重复。
// StartAt/EndAt 仅用于估算搜索窗口大小。
if chunks[i].EndAt > lastChunk.EndAt {
lastChunk.Content = searchutil.AppendWithOverlap(
lastChunk.Content, chunks[i].Content, lastChunk.EndAt-chunks[i].StartAt,
)
lastChunk.EndAt = chunks[i].EndAt
lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID)
if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil {
pipelineWarn(ctx, "Merge", "image_merge", map[string]interface{}{
"knowledge_id": knowledgeID,
"error": err.Error(),
})
}
} else {
// Fully contained: track the subsumed chunk and merge its ImageInfo
if !containsID(lastChunk.SubChunkID, chunks[i].ID) {
lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID)
}
if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil {
pipelineWarn(ctx, "Merge", "image_merge_contained", map[string]interface{}{
"knowledge_id": knowledgeID,
"error": err.Error(),
})
}
}
// Keep the higher score
if chunks[i].Score > lastChunk.Score {
lastChunk.Score = chunks[i].Score
}
}
// Sort merged chunks by score (highest first)
sort.Slice(merged, func(i, j int) bool {
return merged[i].Score > merged[j].Score
})
return merged
}
// mergeImageInfo merges ImageInfo from source into target, deduplicating by URL.
func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *types.SearchResult) error {
if source.ImageInfo == "" {
return nil
}
var sourceImageInfos []types.ImageInfo
if err := json.Unmarshal([]byte(source.ImageInfo), &sourceImageInfos); err != nil {
pipelineWarn(ctx, "Merge", "image_unmarshal_source", map[string]interface{}{
"error": err.Error(),
})
return err
}
if len(sourceImageInfos) == 0 {
return nil
}
var targetImageInfos []types.ImageInfo
if target.ImageInfo != "" {
if err := json.Unmarshal([]byte(target.ImageInfo), &targetImageInfos); err != nil {
pipelineWarn(ctx, "Merge", "image_unmarshal_target", map[string]interface{}{
"error": err.Error(),
})
target.ImageInfo = source.ImageInfo
return nil
}
}
targetImageInfos = append(targetImageInfos, sourceImageInfos...)
uniqueMap := make(map[string]bool)
uniqueImageInfos := make([]types.ImageInfo, 0, len(targetImageInfos))
for _, imgInfo := range targetImageInfos {
if imgInfo.URL != "" && !uniqueMap[imgInfo.URL] {
uniqueMap[imgInfo.URL] = true
uniqueImageInfos = append(uniqueImageInfos, imgInfo)
}
}
mergedImageInfoJSON, err := json.Marshal(uniqueImageInfos)
if err != nil {
pipelineWarn(ctx, "Merge", "image_marshal", map[string]interface{}{
"error": err.Error(),
})
return err
}
target.ImageInfo = string(mergedImageInfoJSON)
pipelineInfo(ctx, "Merge", "image_merged", map[string]interface{}{
"image_refs": len(uniqueImageInfos),
})
return nil
}