diff --git a/internal/agent/tools/grep_chunks.go b/internal/agent/tools/grep_chunks.go index a0444168..913e3b74 100644 --- a/internal/agent/tools/grep_chunks.go +++ b/internal/agent/tools/grep_chunks.go @@ -650,6 +650,8 @@ func (t *GrepChunksTool) applyMMR( lambda, k, len(results)) selected := make([]chunkWithTitle, 0, k) + selectedTokenSets := make([]map[string]struct{}, 0, k) // cache of token sets + candidates := make([]chunkWithTitle, len(results)) copy(candidates, results) @@ -669,9 +671,8 @@ func (t *GrepChunksTool) applyMMR( redundancy := 0.0 // Calculate maximum redundancy with already selected results - for _, s := range selected { - selectedTokens := t.tokenizeSimple(s.Content) - redundancy = math.Max(redundancy, t.jaccard(tokenSets[i], selectedTokens)) + for _, selectedTS := range selectedTokenSets { + redundancy = math.Max(redundancy, t.jaccard(tokenSets[i], selectedTS)) } // MMR score: balance relevance and diversity @@ -684,6 +685,8 @@ func (t *GrepChunksTool) applyMMR( // Add best candidate to selected and remove from candidates selected = append(selected, candidates[bestIdx]) + selectedTokenSets = append(selectedTokenSets, tokenSets[bestIdx]) + candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...) // Remove corresponding token set tokenSets = append(tokenSets[:bestIdx], tokenSets[bestIdx+1:]...) @@ -695,9 +698,7 @@ func (t *GrepChunksTool) applyMMR( pairs := 0 for i := 0; i < len(selected); i++ { for j := i + 1; j < len(selected); j++ { - si := t.tokenizeSimple(selected[i].Content) - sj := t.tokenizeSimple(selected[j].Content) - avgRed += t.jaccard(si, sj) + avgRed += t.jaccard(selectedTokenSets[i], selectedTokenSets[j]) // read token from cache pairs++ } }