refactor(chat_pipeline): enhance merging logic and add partial overlap removal

- Updated the merging process in the chat pipeline to include a new step for re-merging overlapping ranges introduced by context expansion.
- Implemented a new function, removePartialOverlaps, to eliminate chunks that are largely contained within higher-scored chunks, improving deduplication accuracy.
- Enhanced normalization and overlap checking utilities to support the new deduplication logic.
This commit is contained in:
wizardchen
2026-03-23 20:24:35 +08:00
committed by lyingbug
parent 15ebfe0b4c
commit 47397464ef
3 changed files with 148 additions and 2 deletions
@@ -39,7 +39,8 @@ func (p *PluginMerge) ActivationEvents() []types.EventType {
// 5. Group by knowledge source + chunk type, merge overlapping ranges
// 6. Populate FAQ answers
// 7. Expand short contexts with neighboring chunks
// 8. Final deduplication (catches duplicates introduced by steps 4-7)
// 7.5. Re-merge overlapping ranges introduced by expansion
// 8. Final deduplication (ID + signature + partial content overlap)
func (p *PluginMerge) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
@@ -84,8 +85,12 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
// Step 7: Expand short contexts
mergedChunks = p.expandShortContextWithNeighbors(ctx, chatManage, mergedChunks)
// Step 8: Final dedup (catches duplicates from parent resolution / expansion)
// Step 7.5: Re-merge overlapping ranges introduced by expansion
mergedChunks = p.groupAndMergeOverlapping(ctx, mergedChunks)
// Step 8: Final dedup — catches exact duplicates plus partial content overlaps
mergedChunks = p.dedup(ctx, "final_dedup", mergedChunks)
mergedChunks = removePartialOverlaps(ctx, mergedChunks)
chatManage.MergeResult = mergedChunks
return next()
@@ -188,6 +188,98 @@ func buildContentSignature(content string) string {
return searchutil.BuildContentSignature(content)
}
// removePartialOverlaps drops chunks whose content is largely contained within
// a higher-scored chunk, even across different knowledge sources. This catches
// cross-KB duplicates and near-duplicates that exact-signature dedup misses.
//
// Two thresholds are used:
// - Substring containment: if the normalized short text is a literal substring
// of the normalized long text, the shorter chunk is removed.
// - Token overlap coefficient >= 0.85: if 85%+ of the smaller chunk's tokens
// appear in the larger chunk, the smaller one is redundant.
//
// The input slice MUST already be deduplicated by ID/signature. Within each
// pair the chunk with the lower score is the candidate for removal; ties are
// broken by content length (longer wins).
func removePartialOverlaps(ctx context.Context, results []*types.SearchResult) []*types.SearchResult {
const overlapThreshold = 0.85
if len(results) <= 1 {
return results
}
type normEntry struct {
norm string
result *types.SearchResult
}
entries := make([]normEntry, 0, len(results))
for _, r := range results {
entries = append(entries, normEntry{
norm: searchutil.NormalizeContent(r.Content),
result: r,
})
}
removed := make(map[int]bool)
for i := 0; i < len(entries); i++ {
if removed[i] {
continue
}
for j := i + 1; j < len(entries); j++ {
if removed[j] {
continue
}
a, b := entries[i], entries[j]
shortIdx, longIdx := i, j
if len(a.norm) > len(b.norm) {
shortIdx, longIdx = j, i
}
contained := searchutil.IsContentContained(
entries[shortIdx].norm, entries[longIdx].norm,
)
if !contained {
ratio := searchutil.ContentOverlapRatio(
entries[shortIdx].result.Content,
entries[longIdx].result.Content,
)
if ratio < overlapThreshold {
continue
}
}
victim := shortIdx
if entries[shortIdx].result.Score > entries[longIdx].result.Score {
victim = longIdx
}
removed[victim] = true
keptIdx := i
if victim == i {
keptIdx = j
}
pipelineInfo(ctx, "Merge", "partial_overlap_drop", map[string]interface{}{
"kept_id": entries[keptIdx].result.ID,
"dropped_id": entries[victim].result.ID,
"contained": contained,
})
}
}
out := make([]*types.SearchResult, 0, len(results)-len(removed))
for i, e := range entries {
if !removed[i] {
out = append(out, e.result)
}
}
return out
}
func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) {
const maxLogRows = 8
limit := min(maxLogRows, len(results))
+49
View File
@@ -101,6 +101,55 @@ func Jaccard(a, b map[string]struct{}) float64 {
return float64(inter) / float64(union)
}
// NormalizeContent returns a lowercased, whitespace-collapsed version of s
// suitable for containment and overlap checks.
func NormalizeContent(s string) string {
c := strings.ToLower(strings.TrimSpace(s))
if c == "" {
return ""
}
return strings.Join(strings.Fields(c), " ")
}
// IsContentContained reports whether the normalized form of short is a
// substring of the normalized form of long. Both inputs must already be
// normalized via NormalizeContent.
func IsContentContained(normalizedShort, normalizedLong string) bool {
if normalizedShort == "" || normalizedLong == "" {
return false
}
if len(normalizedShort) > len(normalizedLong) {
return false
}
return strings.Contains(normalizedLong, normalizedShort)
}
// ContentOverlapRatio estimates how much of a's content overlaps with b by
// comparing their token sets (Jaccard-like but using overlap coefficient:
// |intersection| / |smaller set|). Both inputs should be raw content strings.
// Returns a value in [0, 1] where 1 means the smaller set is fully contained
// in the larger set.
func ContentOverlapRatio(a, b string) float64 {
tokA := TokenizeSimple(a)
tokB := TokenizeSimple(b)
if len(tokA) == 0 || len(tokB) == 0 {
return 0
}
small, large := tokA, tokB
if len(tokA) > len(tokB) {
small, large = tokB, tokA
}
inter := 0
for k := range small {
if _, ok := large[k]; ok {
inter++
}
}
return float64(inter) / float64(len(small))
}
// ClampFloat clamps a float value to the specified range [minV, maxV].
func ClampFloat(v, minV, maxV float64) float64 {
if v < minV {