feat(repository): update keyword retrieval logic to use ParadeDB's matching operator
- Modified the SQL clause in the KeywordsRetrieve function to utilize ParadeDB's ||| operator for improved token matching. - Enhanced the handling of retrieval results in the fuseOrDeduplicate function to maintain original scores for keyword-only results, ensuring better performance in FAQ scenarios. This update optimizes the keyword retrieval process and improves the accuracy of search results.
This commit is contained in:
@@ -193,10 +193,13 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
|
||||
Values: common.ToInterfaceSlice(params.TagIDs),
|
||||
})
|
||||
}
|
||||
|
||||
// Use ParadeDB's ||| operator for matching any token
|
||||
conds = append(conds, clause.Expr{
|
||||
SQL: "id @@@ paradedb.match(field => 'content', value => ?, distance => 1)",
|
||||
SQL: "content ||| ?",
|
||||
Vars: []interface{}{params.Query},
|
||||
})
|
||||
|
||||
// Filter by is_enabled = true or NULL (NULL means enabled for historical data)
|
||||
conds = append(conds, clause.Expr{
|
||||
SQL: "(is_enabled IS NULL OR is_enabled = ?)",
|
||||
|
||||
@@ -3,9 +3,10 @@ package service
|
||||
import (
|
||||
"context"
|
||||
|
||||
"slices"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"slices"
|
||||
)
|
||||
|
||||
// classifyRetrievalResults separates retrieval results by retriever type (vector vs keyword).
|
||||
@@ -35,6 +36,12 @@ func fuseOrDeduplicate(ctx context.Context, vectorResults, keywordResults []*typ
|
||||
logger.Infof(ctx, "Result count after deduplication: %d", len(result))
|
||||
return result
|
||||
}
|
||||
if len(vectorResults) == 0 {
|
||||
// Keyword-only: keep original scores (important for FAQ)
|
||||
result := deduplicateByScore(keywordResults)
|
||||
logger.Infof(ctx, "Result count after deduplication: %d", len(result))
|
||||
return result
|
||||
}
|
||||
// Hybrid: use RRF fusion to merge vector + keyword results
|
||||
result := fuseWithRRF(ctx, vectorResults, keywordResults)
|
||||
logger.Infof(ctx, "Result count after RRF fusion: %d", len(result))
|
||||
|
||||
Reference in New Issue
Block a user