feat: 支持FAQ两级优先级标签过滤
添加FAQ搜索支持两级优先级标签过滤功能,新增first_priority_tag_ids和second_priority_tag_ids字段,更新各检索器实现以支持按标签ID过滤,优化搜索结果排序确保优先级标签结果优先展示。
This commit is contained in:
+5
-3
@@ -85,9 +85,11 @@ type FAQDeleteRequest struct {
|
||||
|
||||
// FAQSearchRequest represents the hybrid FAQ search request.
|
||||
type FAQSearchRequest struct {
|
||||
QueryText string `json:"query_text"`
|
||||
VectorThreshold float64 `json:"vector_threshold"`
|
||||
MatchCount int `json:"match_count"`
|
||||
QueryText string `json:"query_text"`
|
||||
VectorThreshold float64 `json:"vector_threshold"`
|
||||
MatchCount int `json:"match_count"`
|
||||
FirstPriorityTagIDs []string `json:"first_priority_tag_ids"` // First priority tag IDs, highest priority
|
||||
SecondPriorityTagIDs []string `json:"second_priority_tag_ids"` // Second priority tag IDs, lower than first
|
||||
}
|
||||
|
||||
// FAQEntriesPage contains paginated FAQ results.
|
||||
|
||||
@@ -379,6 +379,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
|
||||
},
|
||||
})
|
||||
}
|
||||
// Filter by tag IDs if specified
|
||||
if len(params.TagIDs) > 0 {
|
||||
must = append(must, map[string]interface{}{
|
||||
"terms": map[string]interface{}{
|
||||
"tag_id.keyword": params.TagIDs,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Build MUST_NOT conditions (negative filters)
|
||||
mustNot := make([]map[string]interface{}, 0)
|
||||
|
||||
@@ -258,6 +258,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams)
|
||||
},
|
||||
}})
|
||||
}
|
||||
// Filter by tag IDs if specified
|
||||
if len(params.TagIDs) > 0 {
|
||||
must = append(must, types.Query{Terms: &types.TermsQuery{
|
||||
TermsQuery: map[string]types.TermsQueryField{
|
||||
"tag_id.keyword": params.TagIDs,
|
||||
},
|
||||
}})
|
||||
}
|
||||
|
||||
mustNot := make([]types.Query, 0)
|
||||
// Exclude disabled chunks (is_enabled = false)
|
||||
|
||||
@@ -185,6 +185,14 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
|
||||
Values: common.ToInterfaceSlice(params.KnowledgeIDs),
|
||||
})
|
||||
}
|
||||
// Filter by tag IDs if specified
|
||||
if len(params.TagIDs) > 0 {
|
||||
logger.GetLogger(ctx).Debugf("[Postgres] Filtering by tag IDs: %v", params.TagIDs)
|
||||
conds = append(conds, clause.IN{
|
||||
Column: "tag_id",
|
||||
Values: common.ToInterfaceSlice(params.TagIDs),
|
||||
})
|
||||
}
|
||||
conds = append(conds, clause.Expr{
|
||||
SQL: "id @@@ paradedb.match(field => 'content', value => ?, distance => 1)",
|
||||
Vars: []interface{}{params.Query},
|
||||
@@ -209,6 +217,7 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
|
||||
"chunk_id",
|
||||
"knowledge_id",
|
||||
"knowledge_base_id",
|
||||
"tag_id",
|
||||
}).
|
||||
Limit(int(params.TopK)).
|
||||
Find(&embeddingDBList).Error
|
||||
@@ -293,6 +302,21 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
|
||||
whereParts = append(whereParts, fmt.Sprintf("knowledge_id IN (%s)",
|
||||
strings.Join(placeholders, ", ")))
|
||||
}
|
||||
// Filter by tag IDs if specified
|
||||
if len(params.TagIDs) > 0 {
|
||||
logger.GetLogger(ctx).Debugf(
|
||||
"[Postgres] Filtering vector search by tag IDs: %v",
|
||||
params.TagIDs,
|
||||
)
|
||||
placeholders := make([]string, len(params.TagIDs))
|
||||
paramStart := len(allVars) + 1
|
||||
for i := range params.TagIDs {
|
||||
placeholders[i] = fmt.Sprintf("$%d", paramStart+i)
|
||||
allVars = append(allVars, params.TagIDs[i])
|
||||
}
|
||||
whereParts = append(whereParts, fmt.Sprintf("tag_id IN (%s)",
|
||||
strings.Join(placeholders, ", ")))
|
||||
}
|
||||
|
||||
// is_enabled filter
|
||||
whereParts = append(whereParts, fmt.Sprintf("(is_enabled IS NULL OR is_enabled = $%d)", len(allVars)+1))
|
||||
@@ -323,11 +347,11 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
|
||||
|
||||
querySQL := fmt.Sprintf(`
|
||||
SELECT
|
||||
id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id,
|
||||
id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, tag_id,
|
||||
(1 - distance) as score
|
||||
FROM (
|
||||
SELECT
|
||||
id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id,
|
||||
id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, tag_id,
|
||||
embedding::halfvec(%d) <=> $1::halfvec as distance
|
||||
FROM embeddings
|
||||
%s
|
||||
|
||||
@@ -21,6 +21,7 @@ type pgVector struct {
|
||||
ChunkID string `json:"chunk_id" gorm:"column:chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"`
|
||||
TagID string `json:"tag_id" gorm:"column:tag_id;index"`
|
||||
Content string `json:"content" gorm:"column:content;not null"`
|
||||
Dimension int `json:"dimension" gorm:"column:dimension;not null"`
|
||||
Embedding pgvector.HalfVector `json:"embedding" gorm:"column:embedding;not null"`
|
||||
@@ -37,6 +38,7 @@ type pgVectorWithScore struct {
|
||||
ChunkID string `json:"chunk_id" gorm:"column:chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"`
|
||||
TagID string `json:"tag_id" gorm:"column:tag_id;index"`
|
||||
Content string `json:"content" gorm:"column:content;not null"`
|
||||
Dimension int `json:"dimension" gorm:"column:dimension;not null"`
|
||||
Embedding pgvector.HalfVector `json:"embedding" gorm:"column:embedding;not null"`
|
||||
@@ -62,6 +64,7 @@ func toDBVectorEmbedding(indexInfo *types.IndexInfo, additionalParams map[string
|
||||
ChunkID: indexInfo.ChunkID,
|
||||
KnowledgeID: indexInfo.KnowledgeID,
|
||||
KnowledgeBaseID: indexInfo.KnowledgeBaseID,
|
||||
TagID: indexInfo.TagID,
|
||||
Content: common.CleanInvalidUTF8(indexInfo.Content),
|
||||
IsEnabled: true, // Default to enabled
|
||||
}
|
||||
@@ -92,6 +95,7 @@ func fromDBVectorEmbeddingWithScore(embedding *pgVectorWithScore, matchType type
|
||||
ChunkID: embedding.ChunkID,
|
||||
KnowledgeID: embedding.KnowledgeID,
|
||||
KnowledgeBaseID: embedding.KnowledgeBaseID,
|
||||
TagID: embedding.TagID,
|
||||
Content: embedding.Content,
|
||||
Score: embedding.Score,
|
||||
MatchType: matchType,
|
||||
|
||||
@@ -25,6 +25,7 @@ const (
|
||||
fieldChunkID = "chunk_id"
|
||||
fieldKnowledgeID = "knowledge_id"
|
||||
fieldKnowledgeBaseID = "knowledge_base_id"
|
||||
fieldTagID = "tag_id"
|
||||
fieldEmbedding = "embedding"
|
||||
fieldIsEnabled = "is_enabled"
|
||||
)
|
||||
@@ -443,6 +444,10 @@ func (q *qdrantRepository) getBaseFilter(params types.RetrieveParams) *qdrant.Fi
|
||||
if len(params.KnowledgeIDs) > 0 {
|
||||
must = append(must, qdrant.NewMatchKeywords(fieldKnowledgeID, params.KnowledgeIDs...))
|
||||
}
|
||||
// Filter by tag IDs if specified
|
||||
if len(params.TagIDs) > 0 {
|
||||
must = append(must, qdrant.NewMatchKeywords(fieldTagID, params.TagIDs...))
|
||||
}
|
||||
|
||||
if len(params.ExcludeKnowledgeIDs) > 0 {
|
||||
mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldKnowledgeID, params.ExcludeKnowledgeIDs...))
|
||||
@@ -531,6 +536,7 @@ func (q *qdrantRepository) VectorRetrieve(ctx context.Context,
|
||||
ChunkID: payload[fieldChunkID].GetStringValue(),
|
||||
KnowledgeID: payload[fieldKnowledgeID].GetStringValue(),
|
||||
KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(),
|
||||
TagID: payload[fieldTagID].GetStringValue(),
|
||||
},
|
||||
Score: float64(point.Score),
|
||||
}
|
||||
@@ -623,6 +629,7 @@ func (q *qdrantRepository) KeywordsRetrieve(ctx context.Context,
|
||||
ChunkID: payload[fieldChunkID].GetStringValue(),
|
||||
KnowledgeID: payload[fieldKnowledgeID].GetStringValue(),
|
||||
KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(),
|
||||
TagID: payload[fieldTagID].GetStringValue(),
|
||||
},
|
||||
Score: 1.0,
|
||||
}
|
||||
@@ -805,6 +812,7 @@ func createPayload(embedding *QdrantVectorEmbedding) map[string]*qdrant.Value {
|
||||
fieldChunkID: embedding.ChunkID,
|
||||
fieldKnowledgeID: embedding.KnowledgeID,
|
||||
fieldKnowledgeBaseID: embedding.KnowledgeBaseID,
|
||||
fieldTagID: embedding.TagID,
|
||||
fieldIsEnabled: embedding.IsEnabled,
|
||||
}
|
||||
return qdrant.NewValueMap(payload)
|
||||
@@ -862,6 +870,7 @@ func toQdrantVectorEmbedding(embedding *types.IndexInfo, additionalParams map[st
|
||||
ChunkID: embedding.ChunkID,
|
||||
KnowledgeID: embedding.KnowledgeID,
|
||||
KnowledgeBaseID: embedding.KnowledgeBaseID,
|
||||
TagID: embedding.TagID,
|
||||
IsEnabled: true, // Default to enabled
|
||||
}
|
||||
if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), fieldEmbedding) {
|
||||
@@ -884,6 +893,7 @@ func fromQdrantVectorEmbedding(id string,
|
||||
ChunkID: embedding.ChunkID,
|
||||
KnowledgeID: embedding.KnowledgeID,
|
||||
KnowledgeBaseID: embedding.KnowledgeBaseID,
|
||||
TagID: embedding.TagID,
|
||||
Content: embedding.Content,
|
||||
Score: embedding.Score,
|
||||
MatchType: matchType,
|
||||
|
||||
@@ -20,6 +20,7 @@ type QdrantVectorEmbedding struct {
|
||||
ChunkID string `json:"chunk_id"`
|
||||
KnowledgeID string `json:"knowledge_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
TagID string `json:"tag_id"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
IsEnabled bool `json:"is_enabled"`
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/docreader/client"
|
||||
@@ -3915,18 +3916,101 @@ func (s *knowledgeService) SearchFAQEntries(ctx context.Context,
|
||||
req.MatchCount = 50
|
||||
}
|
||||
|
||||
// Prepare search parameters
|
||||
searchParams := types.SearchParams{
|
||||
QueryText: secutils.SanitizeForLog(req.QueryText),
|
||||
VectorThreshold: req.VectorThreshold,
|
||||
MatchCount: req.MatchCount,
|
||||
DisableKeywordsMatch: true,
|
||||
// Build priority tag sets for sorting
|
||||
hasFirstPriority := len(req.FirstPriorityTagIDs) > 0
|
||||
hasSecondPriority := len(req.SecondPriorityTagIDs) > 0
|
||||
hasPriorityFilter := hasFirstPriority || hasSecondPriority
|
||||
|
||||
firstPrioritySet := make(map[string]struct{}, len(req.FirstPriorityTagIDs))
|
||||
for _, tagID := range req.FirstPriorityTagIDs {
|
||||
firstPrioritySet[tagID] = struct{}{}
|
||||
}
|
||||
secondPrioritySet := make(map[string]struct{}, len(req.SecondPriorityTagIDs))
|
||||
for _, tagID := range req.SecondPriorityTagIDs {
|
||||
secondPrioritySet[tagID] = struct{}{}
|
||||
}
|
||||
|
||||
// Call HybridSearch
|
||||
searchResults, err := s.kbService.HybridSearch(ctx, kbID, searchParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
// Perform separate searches for each priority level to ensure FirstPriority results
|
||||
// are not crowded out by higher-scoring SecondPriority results in TopK truncation
|
||||
var searchResults []*types.SearchResult
|
||||
|
||||
if hasPriorityFilter {
|
||||
// Use goroutines to search both priority levels concurrently
|
||||
var (
|
||||
firstResults []*types.SearchResult
|
||||
secondResults []*types.SearchResult
|
||||
firstErr error
|
||||
secondErr error
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
|
||||
if hasFirstPriority {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
firstParams := types.SearchParams{
|
||||
QueryText: secutils.SanitizeForLog(req.QueryText),
|
||||
VectorThreshold: req.VectorThreshold,
|
||||
MatchCount: req.MatchCount,
|
||||
DisableKeywordsMatch: true,
|
||||
TagIDs: req.FirstPriorityTagIDs,
|
||||
}
|
||||
firstResults, firstErr = s.kbService.HybridSearch(ctx, kbID, firstParams)
|
||||
}()
|
||||
}
|
||||
|
||||
if hasSecondPriority {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
secondParams := types.SearchParams{
|
||||
QueryText: secutils.SanitizeForLog(req.QueryText),
|
||||
VectorThreshold: req.VectorThreshold,
|
||||
MatchCount: req.MatchCount,
|
||||
DisableKeywordsMatch: true,
|
||||
TagIDs: req.SecondPriorityTagIDs,
|
||||
}
|
||||
secondResults, secondErr = s.kbService.HybridSearch(ctx, kbID, secondParams)
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// Check errors
|
||||
if firstErr != nil {
|
||||
return nil, firstErr
|
||||
}
|
||||
if secondErr != nil {
|
||||
return nil, secondErr
|
||||
}
|
||||
|
||||
// Merge results: FirstPriority first, then SecondPriority (deduplicated)
|
||||
seenChunkIDs := make(map[string]struct{})
|
||||
for _, result := range firstResults {
|
||||
if _, exists := seenChunkIDs[result.ID]; !exists {
|
||||
seenChunkIDs[result.ID] = struct{}{}
|
||||
searchResults = append(searchResults, result)
|
||||
}
|
||||
}
|
||||
for _, result := range secondResults {
|
||||
if _, exists := seenChunkIDs[result.ID]; !exists {
|
||||
seenChunkIDs[result.ID] = struct{}{}
|
||||
searchResults = append(searchResults, result)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No priority filter, search all
|
||||
searchParams := types.SearchParams{
|
||||
QueryText: secutils.SanitizeForLog(req.QueryText),
|
||||
VectorThreshold: req.VectorThreshold,
|
||||
MatchCount: req.MatchCount,
|
||||
DisableKeywordsMatch: true,
|
||||
}
|
||||
var err error
|
||||
searchResults, err = s.kbService.HybridSearch(ctx, kbID, searchParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if len(searchResults) == 0 {
|
||||
@@ -3982,9 +4066,52 @@ func (s *knowledgeService) SearchFAQEntries(ctx context.Context,
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
slices.SortFunc(entries, func(a, b *types.FAQEntry) int {
|
||||
return int(b.Score - a.Score)
|
||||
})
|
||||
// Sort entries with two-level priority tag support
|
||||
if hasPriorityFilter {
|
||||
// getPriorityLevel returns: 0 = first priority, 1 = second priority, 2 = no priority
|
||||
getPriorityLevel := func(tagID string) int {
|
||||
if _, ok := firstPrioritySet[tagID]; ok {
|
||||
return 0
|
||||
}
|
||||
if _, ok := secondPrioritySet[tagID]; ok {
|
||||
return 1
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
slices.SortFunc(entries, func(a, b *types.FAQEntry) int {
|
||||
aPriority := getPriorityLevel(a.TagID)
|
||||
bPriority := getPriorityLevel(b.TagID)
|
||||
|
||||
// Compare by priority level first
|
||||
if aPriority != bPriority {
|
||||
return aPriority - bPriority // Lower level = higher priority
|
||||
}
|
||||
|
||||
// Same priority level, sort by score descending
|
||||
if b.Score > a.Score {
|
||||
return 1
|
||||
} else if b.Score < a.Score {
|
||||
return -1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
} else {
|
||||
// No priority tags, sort by score only
|
||||
slices.SortFunc(entries, func(a, b *types.FAQEntry) int {
|
||||
if b.Score > a.Score {
|
||||
return 1
|
||||
} else if b.Score < a.Score {
|
||||
return -1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
}
|
||||
|
||||
// Limit results to requested match count
|
||||
if len(entries) > req.MatchCount {
|
||||
entries = entries[:req.MatchCount]
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
@@ -4504,6 +4631,7 @@ func (s *knowledgeService) buildFAQIndexInfoList(
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
||||
KnowledgeType: types.KnowledgeTypeFAQ,
|
||||
TagID: chunk.TagID,
|
||||
IsEnabled: chunk.IsEnabled,
|
||||
},
|
||||
}, nil
|
||||
@@ -4531,6 +4659,7 @@ func (s *knowledgeService) buildFAQIndexInfoList(
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
||||
KnowledgeType: types.KnowledgeTypeFAQ,
|
||||
TagID: chunk.TagID,
|
||||
IsEnabled: chunk.IsEnabled,
|
||||
})
|
||||
|
||||
@@ -4555,6 +4684,7 @@ func (s *knowledgeService) buildFAQIndexInfoList(
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
||||
KnowledgeType: types.KnowledgeTypeFAQ,
|
||||
TagID: chunk.TagID,
|
||||
IsEnabled: chunk.IsEnabled,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -542,6 +542,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
Threshold: params.VectorThreshold,
|
||||
RetrieverType: types.VectorRetrieverType,
|
||||
KnowledgeIDs: params.KnowledgeIDs,
|
||||
TagIDs: params.TagIDs,
|
||||
}
|
||||
|
||||
// For FAQ knowledge base, use FAQ index
|
||||
@@ -564,6 +565,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
Threshold: params.KeywordThreshold,
|
||||
RetrieverType: types.KeywordsRetrieverType,
|
||||
KnowledgeIDs: params.KnowledgeIDs,
|
||||
TagIDs: params.TagIDs,
|
||||
})
|
||||
logger.Info(ctx, "Keyword retrieval parameters setup completed")
|
||||
}
|
||||
@@ -618,7 +620,8 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
logger.Info(ctx, "Only vector results, keeping original embedding scores")
|
||||
chunkInfoMap := make(map[string]*types.IndexWithScore)
|
||||
for _, r := range vectorResults {
|
||||
if _, exists := chunkInfoMap[r.ChunkID]; !exists {
|
||||
// Keep the highest score for each chunk (FAQ may have multiple similar questions)
|
||||
if existing, exists := chunkInfoMap[r.ChunkID]; !exists || r.Score > existing.Score {
|
||||
chunkInfoMap[r.ChunkID] = r
|
||||
}
|
||||
}
|
||||
@@ -656,16 +659,17 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
}
|
||||
|
||||
// Collect all unique chunks and compute RRF scores
|
||||
// Keep the highest score for each chunk from each retriever
|
||||
chunkInfoMap := make(map[string]*types.IndexWithScore)
|
||||
rrfScores := make(map[string]float64)
|
||||
|
||||
// Process vector results
|
||||
// Process vector results - keep highest score per chunk
|
||||
for _, r := range vectorResults {
|
||||
if _, exists := chunkInfoMap[r.ChunkID]; !exists {
|
||||
if existing, exists := chunkInfoMap[r.ChunkID]; !exists || r.Score > existing.Score {
|
||||
chunkInfoMap[r.ChunkID] = r
|
||||
}
|
||||
}
|
||||
// Process keyword results
|
||||
// Process keyword results - only add if not already from vector
|
||||
for _, r := range keywordResults {
|
||||
if _, exists := chunkInfoMap[r.ChunkID]; !exists {
|
||||
chunkInfoMap[r.ChunkID] = r
|
||||
@@ -717,9 +721,8 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
|
||||
// Check if we need iterative retrieval for FAQ with separate indexing
|
||||
// Only use iterative retrieval if we don't have enough unique chunks after first deduplication
|
||||
totalRetrieved := len(vectorResults) + len(keywordResults)
|
||||
needsIterativeRetrieval := len(deduplicatedChunks) < params.MatchCount &&
|
||||
kb.Type == types.KnowledgeBaseTypeFAQ && totalRetrieved == matchCount*2
|
||||
kb.Type == types.KnowledgeBaseTypeFAQ && len(vectorResults) == matchCount
|
||||
if needsIterativeRetrieval {
|
||||
logger.Info(ctx, "Not enough unique chunks, using iterative retrieval for FAQ")
|
||||
// Use iterative retrieval to get more unique chunks (with negative question filtering inside)
|
||||
@@ -746,7 +749,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
|
||||
// iterativeRetrieveWithDeduplication performs iterative retrieval until enough unique chunks are found
|
||||
// This is used for FAQ knowledge bases with separate indexing mode
|
||||
// Negative question filtering is applied after each iteration to ensure we have enough valid chunks
|
||||
// Negative question filtering is applied after each iteration with chunk data caching
|
||||
func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Context,
|
||||
retrieveEngine *retriever.CompositeRetrieveEngine,
|
||||
retrieveParams []types.RetrieveParams,
|
||||
@@ -754,8 +757,17 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co
|
||||
queryText string,
|
||||
) []*types.IndexWithScore {
|
||||
maxIterations := 5
|
||||
currentTopK := matchCount
|
||||
// Start with a larger TopK since we're called when first retrieval wasn't enough
|
||||
// The first retrieval already used matchCount*3, so start from there
|
||||
currentTopK := matchCount * 3
|
||||
uniqueChunks := make(map[string]*types.IndexWithScore)
|
||||
// Cache chunk data to avoid repeated DB queries across iterations
|
||||
chunkDataCache := make(map[string]*types.Chunk)
|
||||
// Track chunks that have been filtered out by negative questions
|
||||
filteredOutChunks := make(map[string]struct{})
|
||||
|
||||
queryTextLower := strings.ToLower(strings.TrimSpace(queryText))
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
|
||||
for i := 0; i < maxIterations; i++ {
|
||||
// Update TopK in retrieve params
|
||||
@@ -783,42 +795,59 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co
|
||||
break
|
||||
}
|
||||
|
||||
// Check if we got fewer results than requested - means no more results available
|
||||
totalRetrieved := len(iterationResults)
|
||||
if totalRetrieved < currentTopK {
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Retrieved %d results (less than TopK %d), no more results available",
|
||||
totalRetrieved,
|
||||
currentTopK,
|
||||
)
|
||||
|
||||
// Collect new chunk IDs that need to be fetched from DB
|
||||
newChunkIDs := make([]string, 0)
|
||||
for _, result := range iterationResults {
|
||||
if _, cached := chunkDataCache[result.ChunkID]; !cached {
|
||||
if _, filtered := filteredOutChunks[result.ChunkID]; !filtered {
|
||||
newChunkIDs = append(newChunkIDs, result.ChunkID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate and merge (keep highest score for each chunk)
|
||||
// Multiple similar questions hitting the same chunk will keep the highest score
|
||||
// Batch fetch only new chunks
|
||||
if len(newChunkIDs) > 0 {
|
||||
newChunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, newChunkIDs)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to fetch chunks at iteration %d: %v", i+1, err)
|
||||
} else {
|
||||
for _, chunk := range newChunks {
|
||||
chunkDataCache[chunk.ID] = chunk
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate, merge, and filter in one pass
|
||||
for _, result := range iterationResults {
|
||||
// Skip if already filtered out
|
||||
if _, filtered := filteredOutChunks[result.ChunkID]; filtered {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check negative questions using cached data
|
||||
if chunkData, ok := chunkDataCache[result.ChunkID]; ok {
|
||||
if chunkData.ChunkType == types.ChunkTypeFAQ {
|
||||
if meta, err := chunkData.FAQMetadata(); err == nil && meta != nil {
|
||||
if s.matchesNegativeQuestions(queryTextLower, meta.NegativeQuestions) {
|
||||
filteredOutChunks[result.ChunkID] = struct{}{}
|
||||
delete(uniqueChunks, result.ChunkID)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Keep highest score for each chunk
|
||||
if existing, ok := uniqueChunks[result.ChunkID]; !ok || result.Score > existing.Score {
|
||||
uniqueChunks[result.ChunkID] = result
|
||||
}
|
||||
}
|
||||
|
||||
// Convert to slice for filtering
|
||||
chunksSlice := make([]*types.IndexWithScore, 0, len(uniqueChunks))
|
||||
for _, chunk := range uniqueChunks {
|
||||
chunksSlice = append(chunksSlice, chunk)
|
||||
}
|
||||
|
||||
// Filter by negative questions
|
||||
chunksSlice = s.filterByNegativeQuestions(ctx, chunksSlice, queryText)
|
||||
// Update uniqueChunks map with filtered results
|
||||
uniqueChunks = make(map[string]*types.IndexWithScore, len(chunksSlice))
|
||||
for _, chunk := range chunksSlice {
|
||||
uniqueChunks[chunk.ChunkID] = chunk
|
||||
}
|
||||
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"After iteration %d: retrieved %d results, found %d unique chunks after filtering (target: %d)",
|
||||
"After iteration %d: retrieved %d results, found %d valid unique chunks (target: %d)",
|
||||
i+1,
|
||||
totalRetrieved,
|
||||
len(uniqueChunks),
|
||||
@@ -833,7 +862,7 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co
|
||||
|
||||
// Early stop: If we got fewer results than TopK, there are no more results to retrieve
|
||||
if totalRetrieved < currentTopK {
|
||||
logger.Infof(ctx, "No more results available, stopping iteration")
|
||||
logger.Infof(ctx, "No more results available (got %d < %d), stopping iteration", totalRetrieved, currentTopK)
|
||||
break
|
||||
}
|
||||
|
||||
|
||||
@@ -285,7 +285,7 @@ func (h *FAQHandler) DeleteEntries(c *gin.Context) {
|
||||
|
||||
// SearchFAQ godoc
|
||||
// @Summary 搜索FAQ
|
||||
// @Description 使用混合搜索在FAQ中搜索
|
||||
// @Description 使用混合搜索在FAQ中搜索,支持两级优先级标签召回:first_priority_tag_ids优先级最高,second_priority_tag_ids次之
|
||||
// @Tags FAQ管理
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
|
||||
@@ -35,5 +35,6 @@ type IndexInfo struct {
|
||||
KnowledgeID string // ID of the knowledge
|
||||
KnowledgeBaseID string // ID of the knowledge base
|
||||
KnowledgeType string // Type of the knowledge (e.g., "faq", "manual")
|
||||
TagID string // Tag ID for categorization (used for FAQ priority filtering)
|
||||
IsEnabled bool // Whether the chunk is enabled for retrieval
|
||||
}
|
||||
|
||||
@@ -223,9 +223,11 @@ type FAQBatchUpsertPayload struct {
|
||||
|
||||
// FAQSearchRequest FAQ检索请求参数
|
||||
type FAQSearchRequest struct {
|
||||
QueryText string `json:"query_text" binding:"required"`
|
||||
VectorThreshold float64 `json:"vector_threshold"`
|
||||
MatchCount int `json:"match_count"`
|
||||
QueryText string `json:"query_text" binding:"required"`
|
||||
VectorThreshold float64 `json:"vector_threshold"`
|
||||
MatchCount int `json:"match_count"`
|
||||
FirstPriorityTagIDs []string `json:"first_priority_tag_ids"` // 第一优先级标签ID列表,限定命中范围,优先级最高
|
||||
SecondPriorityTagIDs []string `json:"second_priority_tag_ids"` // 第二优先级标签ID列表,限定命中范围,优先级低于第一优先级
|
||||
}
|
||||
|
||||
// UntaggedTagID is the special tag ID representing uncategorized entries
|
||||
|
||||
@@ -32,6 +32,8 @@ type RetrieveParams struct {
|
||||
KnowledgeBaseIDs []string
|
||||
// Knowledge IDs
|
||||
KnowledgeIDs []string
|
||||
// Tag IDs for filtering (used for FAQ priority filtering)
|
||||
TagIDs []string
|
||||
// Excluded knowledge IDs
|
||||
ExcludeKnowledgeIDs []string
|
||||
// Excluded chunk IDs
|
||||
@@ -72,6 +74,8 @@ type IndexWithScore struct {
|
||||
KnowledgeID string
|
||||
// Knowledge base ID
|
||||
KnowledgeBaseID string
|
||||
// Tag ID
|
||||
TagID string
|
||||
// Score
|
||||
Score float64
|
||||
// Match type
|
||||
|
||||
@@ -98,6 +98,7 @@ type SearchParams struct {
|
||||
DisableKeywordsMatch bool `json:"disable_keywords_match"`
|
||||
DisableVectorMatch bool `json:"disable_vector_match"`
|
||||
KnowledgeIDs []string `json:"knowledge_ids"`
|
||||
TagIDs []string `json:"tag_ids"` // Tag IDs for filtering (used for FAQ priority filtering)
|
||||
}
|
||||
|
||||
// Value implements the driver.Valuer interface, used to convert SearchResult to database value
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Remove tag_id column from embeddings table
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'embeddings' AND column_name = 'tag_id'
|
||||
) THEN
|
||||
DROP INDEX IF EXISTS idx_embeddings_tag_id;
|
||||
ALTER TABLE embeddings DROP COLUMN tag_id;
|
||||
RAISE NOTICE '[Migration 000007 Rollback] Removed tag_id column from embeddings table';
|
||||
END IF;
|
||||
END $$;
|
||||
@@ -0,0 +1,15 @@
|
||||
-- Add tag_id column to embeddings table for FAQ priority filtering
|
||||
DO $$
|
||||
BEGIN
|
||||
-- Add tag_id column if not exists
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_name = 'embeddings' AND column_name = 'tag_id'
|
||||
) THEN
|
||||
ALTER TABLE embeddings ADD COLUMN tag_id VARCHAR(36);
|
||||
CREATE INDEX IF NOT EXISTS idx_embeddings_tag_id ON embeddings(tag_id);
|
||||
RAISE NOTICE '[Migration 000007] Added tag_id column and index to embeddings table';
|
||||
ELSE
|
||||
RAISE NOTICE '[Migration 000007] tag_id column already exists in embeddings table, skipping';
|
||||
END IF;
|
||||
END $$;
|
||||
Reference in New Issue
Block a user