feat(repository): update keyword retrieval logic to use ParadeDB's matching operator

- Modified the SQL clause in the KeywordsRetrieve function to utilize ParadeDB's ||| operator for improved token matching.
- Enhanced the handling of retrieval results in the fuseOrDeduplicate function to maintain original scores for keyword-only results, ensuring better performance in FAQ scenarios.

This update optimizes the keyword retrieval process and improves the accuracy of search results.
This commit is contained in:
wizardchen
2026-04-15 19:55:59 +08:00
committed by lyingbug
parent 8609c09a63
commit 9bb4a12562
2 changed files with 12 additions and 2 deletions
@@ -193,10 +193,13 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
Values: common.ToInterfaceSlice(params.TagIDs),
})
}
// Use ParadeDB's ||| operator for matching any token
conds = append(conds, clause.Expr{
SQL: "id @@@ paradedb.match(field => 'content', value => ?, distance => 1)",
SQL: "content ||| ?",
Vars: []interface{}{params.Query},
})
// Filter by is_enabled = true or NULL (NULL means enabled for historical data)
conds = append(conds, clause.Expr{
SQL: "(is_enabled IS NULL OR is_enabled = ?)",
@@ -3,9 +3,10 @@ package service
import (
"context"
"slices"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"slices"
)
// classifyRetrievalResults separates retrieval results by retriever type (vector vs keyword).
@@ -35,6 +36,12 @@ func fuseOrDeduplicate(ctx context.Context, vectorResults, keywordResults []*typ
logger.Infof(ctx, "Result count after deduplication: %d", len(result))
return result
}
if len(vectorResults) == 0 {
// Keyword-only: keep original scores (important for FAQ)
result := deduplicateByScore(keywordResults)
logger.Infof(ctx, "Result count after deduplication: %d", len(result))
return result
}
// Hybrid: use RRF fusion to merge vector + keyword results
result := fuseWithRRF(ctx, vectorResults, keywordResults)
logger.Infof(ctx, "Result count after RRF fusion: %d", len(result))