perf: cache tokens instead of recomputing each time
This commit is contained in:
@@ -650,6 +650,8 @@ func (t *GrepChunksTool) applyMMR(
|
||||
lambda, k, len(results))
|
||||
|
||||
selected := make([]chunkWithTitle, 0, k)
|
||||
selectedTokenSets := make([]map[string]struct{}, 0, k) // cache of token sets
|
||||
|
||||
candidates := make([]chunkWithTitle, len(results))
|
||||
copy(candidates, results)
|
||||
|
||||
@@ -669,9 +671,8 @@ func (t *GrepChunksTool) applyMMR(
|
||||
redundancy := 0.0
|
||||
|
||||
// Calculate maximum redundancy with already selected results
|
||||
for _, s := range selected {
|
||||
selectedTokens := t.tokenizeSimple(s.Content)
|
||||
redundancy = math.Max(redundancy, t.jaccard(tokenSets[i], selectedTokens))
|
||||
for _, selectedTS := range selectedTokenSets {
|
||||
redundancy = math.Max(redundancy, t.jaccard(tokenSets[i], selectedTS))
|
||||
}
|
||||
|
||||
// MMR score: balance relevance and diversity
|
||||
@@ -684,6 +685,8 @@ func (t *GrepChunksTool) applyMMR(
|
||||
|
||||
// Add best candidate to selected and remove from candidates
|
||||
selected = append(selected, candidates[bestIdx])
|
||||
selectedTokenSets = append(selectedTokenSets, tokenSets[bestIdx])
|
||||
|
||||
candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...)
|
||||
// Remove corresponding token set
|
||||
tokenSets = append(tokenSets[:bestIdx], tokenSets[bestIdx+1:]...)
|
||||
@@ -695,9 +698,7 @@ func (t *GrepChunksTool) applyMMR(
|
||||
pairs := 0
|
||||
for i := 0; i < len(selected); i++ {
|
||||
for j := i + 1; j < len(selected); j++ {
|
||||
si := t.tokenizeSimple(selected[i].Content)
|
||||
sj := t.tokenizeSimple(selected[j].Content)
|
||||
avgRed += t.jaccard(si, sj)
|
||||
avgRed += t.jaccard(selectedTokenSets[i], selectedTokenSets[j]) // read token from cache
|
||||
pairs++
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user