From 47397464ef04291394167a0c9da6450060d22f6c Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 23 Mar 2026 20:24:35 +0800 Subject: [PATCH] refactor(chat_pipeline): enhance merging logic and add partial overlap removal - Updated the merging process in the chat pipeline to include a new step for re-merging overlapping ranges introduced by context expansion. - Implemented a new function, removePartialOverlaps, to eliminate chunks that are largely contained within higher-scored chunks, improving deduplication accuracy. - Enhanced normalization and overlap checking utilities to support the new deduplication logic. --- .../service/chat_pipeline/merge.go | 9 +- .../service/chat_pipeline/search.go | 92 +++++++++++++++++++ internal/searchutil/textutil.go | 49 ++++++++++ 3 files changed, 148 insertions(+), 2 deletions(-) diff --git a/internal/application/service/chat_pipeline/merge.go b/internal/application/service/chat_pipeline/merge.go index a9f3507e..01b77a84 100644 --- a/internal/application/service/chat_pipeline/merge.go +++ b/internal/application/service/chat_pipeline/merge.go @@ -39,7 +39,8 @@ func (p *PluginMerge) ActivationEvents() []types.EventType { // 5. Group by knowledge source + chunk type, merge overlapping ranges // 6. Populate FAQ answers // 7. Expand short contexts with neighboring chunks -// 8. Final deduplication (catches duplicates introduced by steps 4-7) +// 7.5. Re-merge overlapping ranges introduced by expansion +// 8. Final deduplication (ID + signature + partial content overlap) func (p *PluginMerge) OnEvent(ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError, ) *PluginError { @@ -84,8 +85,12 @@ func (p *PluginMerge) OnEvent(ctx context.Context, // Step 7: Expand short contexts mergedChunks = p.expandShortContextWithNeighbors(ctx, chatManage, mergedChunks) - // Step 8: Final dedup (catches duplicates from parent resolution / expansion) + // Step 7.5: Re-merge overlapping ranges introduced by expansion + mergedChunks = p.groupAndMergeOverlapping(ctx, mergedChunks) + + // Step 8: Final dedup — catches exact duplicates plus partial content overlaps mergedChunks = p.dedup(ctx, "final_dedup", mergedChunks) + mergedChunks = removePartialOverlaps(ctx, mergedChunks) chatManage.MergeResult = mergedChunks return next() diff --git a/internal/application/service/chat_pipeline/search.go b/internal/application/service/chat_pipeline/search.go index 900650a8..919ff1eb 100644 --- a/internal/application/service/chat_pipeline/search.go +++ b/internal/application/service/chat_pipeline/search.go @@ -188,6 +188,98 @@ func buildContentSignature(content string) string { return searchutil.BuildContentSignature(content) } +// removePartialOverlaps drops chunks whose content is largely contained within +// a higher-scored chunk, even across different knowledge sources. This catches +// cross-KB duplicates and near-duplicates that exact-signature dedup misses. +// +// Two thresholds are used: +// - Substring containment: if the normalized short text is a literal substring +// of the normalized long text, the shorter chunk is removed. +// - Token overlap coefficient >= 0.85: if 85%+ of the smaller chunk's tokens +// appear in the larger chunk, the smaller one is redundant. +// +// The input slice MUST already be deduplicated by ID/signature. Within each +// pair the chunk with the lower score is the candidate for removal; ties are +// broken by content length (longer wins). +func removePartialOverlaps(ctx context.Context, results []*types.SearchResult) []*types.SearchResult { + const overlapThreshold = 0.85 + + if len(results) <= 1 { + return results + } + + type normEntry struct { + norm string + result *types.SearchResult + } + + entries := make([]normEntry, 0, len(results)) + for _, r := range results { + entries = append(entries, normEntry{ + norm: searchutil.NormalizeContent(r.Content), + result: r, + }) + } + + removed := make(map[int]bool) + + for i := 0; i < len(entries); i++ { + if removed[i] { + continue + } + for j := i + 1; j < len(entries); j++ { + if removed[j] { + continue + } + + a, b := entries[i], entries[j] + + shortIdx, longIdx := i, j + if len(a.norm) > len(b.norm) { + shortIdx, longIdx = j, i + } + + contained := searchutil.IsContentContained( + entries[shortIdx].norm, entries[longIdx].norm, + ) + + if !contained { + ratio := searchutil.ContentOverlapRatio( + entries[shortIdx].result.Content, + entries[longIdx].result.Content, + ) + if ratio < overlapThreshold { + continue + } + } + + victim := shortIdx + if entries[shortIdx].result.Score > entries[longIdx].result.Score { + victim = longIdx + } + removed[victim] = true + + keptIdx := i + if victim == i { + keptIdx = j + } + pipelineInfo(ctx, "Merge", "partial_overlap_drop", map[string]interface{}{ + "kept_id": entries[keptIdx].result.ID, + "dropped_id": entries[victim].result.ID, + "contained": contained, + }) + } + } + + out := make([]*types.SearchResult, 0, len(results)-len(removed)) + for i, e := range entries { + if !removed[i] { + out = append(out, e.result) + } + } + return out +} + func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) { const maxLogRows = 8 limit := min(maxLogRows, len(results)) diff --git a/internal/searchutil/textutil.go b/internal/searchutil/textutil.go index 5e6b4ade..eb927aa0 100644 --- a/internal/searchutil/textutil.go +++ b/internal/searchutil/textutil.go @@ -101,6 +101,55 @@ func Jaccard(a, b map[string]struct{}) float64 { return float64(inter) / float64(union) } +// NormalizeContent returns a lowercased, whitespace-collapsed version of s +// suitable for containment and overlap checks. +func NormalizeContent(s string) string { + c := strings.ToLower(strings.TrimSpace(s)) + if c == "" { + return "" + } + return strings.Join(strings.Fields(c), " ") +} + +// IsContentContained reports whether the normalized form of short is a +// substring of the normalized form of long. Both inputs must already be +// normalized via NormalizeContent. +func IsContentContained(normalizedShort, normalizedLong string) bool { + if normalizedShort == "" || normalizedLong == "" { + return false + } + if len(normalizedShort) > len(normalizedLong) { + return false + } + return strings.Contains(normalizedLong, normalizedShort) +} + +// ContentOverlapRatio estimates how much of a's content overlaps with b by +// comparing their token sets (Jaccard-like but using overlap coefficient: +// |intersection| / |smaller set|). Both inputs should be raw content strings. +// Returns a value in [0, 1] where 1 means the smaller set is fully contained +// in the larger set. +func ContentOverlapRatio(a, b string) float64 { + tokA := TokenizeSimple(a) + tokB := TokenizeSimple(b) + if len(tokA) == 0 || len(tokB) == 0 { + return 0 + } + + small, large := tokA, tokB + if len(tokA) > len(tokB) { + small, large = tokB, tokA + } + + inter := 0 + for k := range small { + if _, ok := large[k]; ok { + inter++ + } + } + return float64(inter) / float64(len(small)) +} + // ClampFloat clamps a float value to the specified range [minV, maxV]. func ClampFloat(v, minV, maxV float64) float64 { if v < minV {