refactor(chat_pipeline): enhance merging logic and add partial overlap removal
- Updated the merging process in the chat pipeline to include a new step for re-merging overlapping ranges introduced by context expansion. - Implemented a new function, removePartialOverlaps, to eliminate chunks that are largely contained within higher-scored chunks, improving deduplication accuracy. - Enhanced normalization and overlap checking utilities to support the new deduplication logic.
This commit is contained in:
@@ -39,7 +39,8 @@ func (p *PluginMerge) ActivationEvents() []types.EventType {
|
||||
// 5. Group by knowledge source + chunk type, merge overlapping ranges
|
||||
// 6. Populate FAQ answers
|
||||
// 7. Expand short contexts with neighboring chunks
|
||||
// 8. Final deduplication (catches duplicates introduced by steps 4-7)
|
||||
// 7.5. Re-merge overlapping ranges introduced by expansion
|
||||
// 8. Final deduplication (ID + signature + partial content overlap)
|
||||
func (p *PluginMerge) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
@@ -84,8 +85,12 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
|
||||
// Step 7: Expand short contexts
|
||||
mergedChunks = p.expandShortContextWithNeighbors(ctx, chatManage, mergedChunks)
|
||||
|
||||
// Step 8: Final dedup (catches duplicates from parent resolution / expansion)
|
||||
// Step 7.5: Re-merge overlapping ranges introduced by expansion
|
||||
mergedChunks = p.groupAndMergeOverlapping(ctx, mergedChunks)
|
||||
|
||||
// Step 8: Final dedup — catches exact duplicates plus partial content overlaps
|
||||
mergedChunks = p.dedup(ctx, "final_dedup", mergedChunks)
|
||||
mergedChunks = removePartialOverlaps(ctx, mergedChunks)
|
||||
|
||||
chatManage.MergeResult = mergedChunks
|
||||
return next()
|
||||
|
||||
@@ -188,6 +188,98 @@ func buildContentSignature(content string) string {
|
||||
return searchutil.BuildContentSignature(content)
|
||||
}
|
||||
|
||||
// removePartialOverlaps drops chunks whose content is largely contained within
|
||||
// a higher-scored chunk, even across different knowledge sources. This catches
|
||||
// cross-KB duplicates and near-duplicates that exact-signature dedup misses.
|
||||
//
|
||||
// Two thresholds are used:
|
||||
// - Substring containment: if the normalized short text is a literal substring
|
||||
// of the normalized long text, the shorter chunk is removed.
|
||||
// - Token overlap coefficient >= 0.85: if 85%+ of the smaller chunk's tokens
|
||||
// appear in the larger chunk, the smaller one is redundant.
|
||||
//
|
||||
// The input slice MUST already be deduplicated by ID/signature. Within each
|
||||
// pair the chunk with the lower score is the candidate for removal; ties are
|
||||
// broken by content length (longer wins).
|
||||
func removePartialOverlaps(ctx context.Context, results []*types.SearchResult) []*types.SearchResult {
|
||||
const overlapThreshold = 0.85
|
||||
|
||||
if len(results) <= 1 {
|
||||
return results
|
||||
}
|
||||
|
||||
type normEntry struct {
|
||||
norm string
|
||||
result *types.SearchResult
|
||||
}
|
||||
|
||||
entries := make([]normEntry, 0, len(results))
|
||||
for _, r := range results {
|
||||
entries = append(entries, normEntry{
|
||||
norm: searchutil.NormalizeContent(r.Content),
|
||||
result: r,
|
||||
})
|
||||
}
|
||||
|
||||
removed := make(map[int]bool)
|
||||
|
||||
for i := 0; i < len(entries); i++ {
|
||||
if removed[i] {
|
||||
continue
|
||||
}
|
||||
for j := i + 1; j < len(entries); j++ {
|
||||
if removed[j] {
|
||||
continue
|
||||
}
|
||||
|
||||
a, b := entries[i], entries[j]
|
||||
|
||||
shortIdx, longIdx := i, j
|
||||
if len(a.norm) > len(b.norm) {
|
||||
shortIdx, longIdx = j, i
|
||||
}
|
||||
|
||||
contained := searchutil.IsContentContained(
|
||||
entries[shortIdx].norm, entries[longIdx].norm,
|
||||
)
|
||||
|
||||
if !contained {
|
||||
ratio := searchutil.ContentOverlapRatio(
|
||||
entries[shortIdx].result.Content,
|
||||
entries[longIdx].result.Content,
|
||||
)
|
||||
if ratio < overlapThreshold {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
victim := shortIdx
|
||||
if entries[shortIdx].result.Score > entries[longIdx].result.Score {
|
||||
victim = longIdx
|
||||
}
|
||||
removed[victim] = true
|
||||
|
||||
keptIdx := i
|
||||
if victim == i {
|
||||
keptIdx = j
|
||||
}
|
||||
pipelineInfo(ctx, "Merge", "partial_overlap_drop", map[string]interface{}{
|
||||
"kept_id": entries[keptIdx].result.ID,
|
||||
"dropped_id": entries[victim].result.ID,
|
||||
"contained": contained,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]*types.SearchResult, 0, len(results)-len(removed))
|
||||
for i, e := range entries {
|
||||
if !removed[i] {
|
||||
out = append(out, e.result)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) {
|
||||
const maxLogRows = 8
|
||||
limit := min(maxLogRows, len(results))
|
||||
|
||||
@@ -101,6 +101,55 @@ func Jaccard(a, b map[string]struct{}) float64 {
|
||||
return float64(inter) / float64(union)
|
||||
}
|
||||
|
||||
// NormalizeContent returns a lowercased, whitespace-collapsed version of s
|
||||
// suitable for containment and overlap checks.
|
||||
func NormalizeContent(s string) string {
|
||||
c := strings.ToLower(strings.TrimSpace(s))
|
||||
if c == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(strings.Fields(c), " ")
|
||||
}
|
||||
|
||||
// IsContentContained reports whether the normalized form of short is a
|
||||
// substring of the normalized form of long. Both inputs must already be
|
||||
// normalized via NormalizeContent.
|
||||
func IsContentContained(normalizedShort, normalizedLong string) bool {
|
||||
if normalizedShort == "" || normalizedLong == "" {
|
||||
return false
|
||||
}
|
||||
if len(normalizedShort) > len(normalizedLong) {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(normalizedLong, normalizedShort)
|
||||
}
|
||||
|
||||
// ContentOverlapRatio estimates how much of a's content overlaps with b by
|
||||
// comparing their token sets (Jaccard-like but using overlap coefficient:
|
||||
// |intersection| / |smaller set|). Both inputs should be raw content strings.
|
||||
// Returns a value in [0, 1] where 1 means the smaller set is fully contained
|
||||
// in the larger set.
|
||||
func ContentOverlapRatio(a, b string) float64 {
|
||||
tokA := TokenizeSimple(a)
|
||||
tokB := TokenizeSimple(b)
|
||||
if len(tokA) == 0 || len(tokB) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
small, large := tokA, tokB
|
||||
if len(tokA) > len(tokB) {
|
||||
small, large = tokB, tokA
|
||||
}
|
||||
|
||||
inter := 0
|
||||
for k := range small {
|
||||
if _, ok := large[k]; ok {
|
||||
inter++
|
||||
}
|
||||
}
|
||||
return float64(inter) / float64(len(small))
|
||||
}
|
||||
|
||||
// ClampFloat clamps a float value to the specified range [minV, maxV].
|
||||
func ClampFloat(v, minV, maxV float64) float64 {
|
||||
if v < minV {
|
||||
|
||||
Reference in New Issue
Block a user