Files
Enterprise-Knowledge-Base/internal/application/service/chat_pipeline/merge_overlap.go
T
wizardchen 052be53c42 feat(prompts): update assistant descriptions and add intent-specific templates
- Updated the assistant descriptions in various prompt templates to specify that they are developed by Tencent.
- Added new intent-specific system prompt templates for greeting, chitchat, follow-up, image analysis, summarization, and web search unavailability scenarios.
- Enhanced citation rules in the system prompts to ensure proper inline citation formatting.
- Refactored the rewrite prompt to classify user intent more accurately and output structured JSON responses.
2026-03-25 22:08:29 +08:00

129 lines
3.6 KiB
Go

package chatpipeline
import (
"context"
"encoding/json"
"sort"
"github.com/Tencent/WeKnora/internal/types"
)
// mergeOverlappingChunks merges chunks with overlapping or adjacent StartAt/EndAt
// ranges within a single knowledge source group. Chunks MUST be pre-sorted by
// StartAt ascending, EndAt ascending. The highest score among merged chunks is kept.
func (p *PluginMerge) mergeOverlappingChunks(
ctx context.Context,
knowledgeID string,
chunks []*types.SearchResult,
) []*types.SearchResult {
if len(chunks) == 0 {
return nil
}
merged := []*types.SearchResult{chunks[0]}
for i := 1; i < len(chunks); i++ {
lastChunk := merged[len(merged)-1]
// Non-overlapping: add as a new entry
if chunks[i].StartAt > lastChunk.EndAt {
merged = append(merged, chunks[i])
continue
}
// Partial overlap: append the non-overlapping suffix
if chunks[i].EndAt > lastChunk.EndAt {
contentRunes := []rune(chunks[i].Content)
offset := len(contentRunes) - (chunks[i].EndAt - lastChunk.EndAt)
lastChunk.Content = lastChunk.Content + string(contentRunes[offset:])
lastChunk.EndAt = chunks[i].EndAt
lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID)
if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil {
pipelineWarn(ctx, "Merge", "image_merge", map[string]interface{}{
"knowledge_id": knowledgeID,
"error": err.Error(),
})
}
} else {
// Fully contained: track the subsumed chunk and merge its ImageInfo
if !containsID(lastChunk.SubChunkID, chunks[i].ID) {
lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID)
}
if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil {
pipelineWarn(ctx, "Merge", "image_merge_contained", map[string]interface{}{
"knowledge_id": knowledgeID,
"error": err.Error(),
})
}
}
// Keep the higher score
if chunks[i].Score > lastChunk.Score {
lastChunk.Score = chunks[i].Score
}
}
// Sort merged chunks by score (highest first)
sort.Slice(merged, func(i, j int) bool {
return merged[i].Score > merged[j].Score
})
return merged
}
// mergeImageInfo merges ImageInfo from source into target, deduplicating by URL.
func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *types.SearchResult) error {
if source.ImageInfo == "" {
return nil
}
var sourceImageInfos []types.ImageInfo
if err := json.Unmarshal([]byte(source.ImageInfo), &sourceImageInfos); err != nil {
pipelineWarn(ctx, "Merge", "image_unmarshal_source", map[string]interface{}{
"error": err.Error(),
})
return err
}
if len(sourceImageInfos) == 0 {
return nil
}
var targetImageInfos []types.ImageInfo
if target.ImageInfo != "" {
if err := json.Unmarshal([]byte(target.ImageInfo), &targetImageInfos); err != nil {
pipelineWarn(ctx, "Merge", "image_unmarshal_target", map[string]interface{}{
"error": err.Error(),
})
target.ImageInfo = source.ImageInfo
return nil
}
}
targetImageInfos = append(targetImageInfos, sourceImageInfos...)
uniqueMap := make(map[string]bool)
uniqueImageInfos := make([]types.ImageInfo, 0, len(targetImageInfos))
for _, imgInfo := range targetImageInfos {
if imgInfo.URL != "" && !uniqueMap[imgInfo.URL] {
uniqueMap[imgInfo.URL] = true
uniqueImageInfos = append(uniqueImageInfos, imgInfo)
}
}
mergedImageInfoJSON, err := json.Marshal(uniqueImageInfos)
if err != nil {
pipelineWarn(ctx, "Merge", "image_marshal", map[string]interface{}{
"error": err.Error(),
})
return err
}
target.ImageInfo = string(mergedImageInfoJSON)
pipelineInfo(ctx, "Merge", "image_merged", map[string]interface{}{
"image_refs": len(uniqueImageInfos),
})
return nil
}