From ac00206e77fbf71df648b4976b32f129b071f9a4 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 30 Dec 2025 16:15:09 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81FAQ=E4=B8=A4=E7=BA=A7?= =?UTF-8?q?=E4=BC=98=E5=85=88=E7=BA=A7=E6=A0=87=E7=AD=BE=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 添加FAQ搜索支持两级优先级标签过滤功能,新增first_priority_tag_ids和second_priority_tag_ids字段,更新各检索器实现以支持按标签ID过滤,优化搜索结果排序确保优先级标签结果优先展示。 --- client/faq.go | 8 +- .../retriever/elasticsearch/v7/repository.go | 8 + .../retriever/elasticsearch/v8/repository.go | 8 + .../retriever/postgres/repository.go | 28 +++- .../repository/retriever/postgres/structs.go | 4 + .../repository/retriever/qdrant/repository.go | 10 ++ .../repository/retriever/qdrant/structs.go | 1 + internal/application/service/knowledge.go | 156 ++++++++++++++++-- internal/application/service/knowledgebase.go | 97 +++++++---- internal/handler/faq.go | 2 +- internal/types/embedding.go | 1 + internal/types/faq.go | 8 +- internal/types/retriever.go | 4 + internal/types/search.go | 1 + .../000007_embeddings_tag_id.down.sql | 12 ++ .../versioned/000007_embeddings_tag_id.up.sql | 15 ++ 16 files changed, 307 insertions(+), 56 deletions(-) create mode 100644 migrations/versioned/000007_embeddings_tag_id.down.sql create mode 100644 migrations/versioned/000007_embeddings_tag_id.up.sql diff --git a/client/faq.go b/client/faq.go index 55b50264..98d05fac 100644 --- a/client/faq.go +++ b/client/faq.go @@ -85,9 +85,11 @@ type FAQDeleteRequest struct { // FAQSearchRequest represents the hybrid FAQ search request. type FAQSearchRequest struct { - QueryText string `json:"query_text"` - VectorThreshold float64 `json:"vector_threshold"` - MatchCount int `json:"match_count"` + QueryText string `json:"query_text"` + VectorThreshold float64 `json:"vector_threshold"` + MatchCount int `json:"match_count"` + FirstPriorityTagIDs []string `json:"first_priority_tag_ids"` // First priority tag IDs, highest priority + SecondPriorityTagIDs []string `json:"second_priority_tag_ids"` // Second priority tag IDs, lower than first } // FAQEntriesPage contains paginated FAQ results. diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.go index df624674..779ceff5 100644 --- a/internal/application/repository/retriever/elasticsearch/v7/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go @@ -379,6 +379,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) }, }) } + // Filter by tag IDs if specified + if len(params.TagIDs) > 0 { + must = append(must, map[string]interface{}{ + "terms": map[string]interface{}{ + "tag_id.keyword": params.TagIDs, + }, + }) + } // Build MUST_NOT conditions (negative filters) mustNot := make([]map[string]interface{}, 0) diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.go index adfe13dc..f3f89fd1 100644 --- a/internal/application/repository/retriever/elasticsearch/v8/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go @@ -258,6 +258,14 @@ func (e *elasticsearchRepository) getBaseConds(params typesLocal.RetrieveParams) }, }}) } + // Filter by tag IDs if specified + if len(params.TagIDs) > 0 { + must = append(must, types.Query{Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{ + "tag_id.keyword": params.TagIDs, + }, + }}) + } mustNot := make([]types.Query, 0) // Exclude disabled chunks (is_enabled = false) diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.go index b974fd6e..fb55de8e 100644 --- a/internal/application/repository/retriever/postgres/repository.go +++ b/internal/application/repository/retriever/postgres/repository.go @@ -185,6 +185,14 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context, Values: common.ToInterfaceSlice(params.KnowledgeIDs), }) } + // Filter by tag IDs if specified + if len(params.TagIDs) > 0 { + logger.GetLogger(ctx).Debugf("[Postgres] Filtering by tag IDs: %v", params.TagIDs) + conds = append(conds, clause.IN{ + Column: "tag_id", + Values: common.ToInterfaceSlice(params.TagIDs), + }) + } conds = append(conds, clause.Expr{ SQL: "id @@@ paradedb.match(field => 'content', value => ?, distance => 1)", Vars: []interface{}{params.Query}, @@ -209,6 +217,7 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context, "chunk_id", "knowledge_id", "knowledge_base_id", + "tag_id", }). Limit(int(params.TopK)). Find(&embeddingDBList).Error @@ -293,6 +302,21 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context, whereParts = append(whereParts, fmt.Sprintf("knowledge_id IN (%s)", strings.Join(placeholders, ", "))) } + // Filter by tag IDs if specified + if len(params.TagIDs) > 0 { + logger.GetLogger(ctx).Debugf( + "[Postgres] Filtering vector search by tag IDs: %v", + params.TagIDs, + ) + placeholders := make([]string, len(params.TagIDs)) + paramStart := len(allVars) + 1 + for i := range params.TagIDs { + placeholders[i] = fmt.Sprintf("$%d", paramStart+i) + allVars = append(allVars, params.TagIDs[i]) + } + whereParts = append(whereParts, fmt.Sprintf("tag_id IN (%s)", + strings.Join(placeholders, ", "))) + } // is_enabled filter whereParts = append(whereParts, fmt.Sprintf("(is_enabled IS NULL OR is_enabled = $%d)", len(allVars)+1)) @@ -323,11 +347,11 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context, querySQL := fmt.Sprintf(` SELECT - id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, + id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, tag_id, (1 - distance) as score FROM ( SELECT - id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, + id, content, source_id, source_type, chunk_id, knowledge_id, knowledge_base_id, tag_id, embedding::halfvec(%d) <=> $1::halfvec as distance FROM embeddings %s diff --git a/internal/application/repository/retriever/postgres/structs.go b/internal/application/repository/retriever/postgres/structs.go index b673e555..940ae081 100644 --- a/internal/application/repository/retriever/postgres/structs.go +++ b/internal/application/repository/retriever/postgres/structs.go @@ -21,6 +21,7 @@ type pgVector struct { ChunkID string `json:"chunk_id" gorm:"column:chunk_id"` KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"` KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"` + TagID string `json:"tag_id" gorm:"column:tag_id;index"` Content string `json:"content" gorm:"column:content;not null"` Dimension int `json:"dimension" gorm:"column:dimension;not null"` Embedding pgvector.HalfVector `json:"embedding" gorm:"column:embedding;not null"` @@ -37,6 +38,7 @@ type pgVectorWithScore struct { ChunkID string `json:"chunk_id" gorm:"column:chunk_id"` KnowledgeID string `json:"knowledge_id" gorm:"column:knowledge_id"` KnowledgeBaseID string `json:"knowledge_base_id" gorm:"column:knowledge_base_id"` + TagID string `json:"tag_id" gorm:"column:tag_id;index"` Content string `json:"content" gorm:"column:content;not null"` Dimension int `json:"dimension" gorm:"column:dimension;not null"` Embedding pgvector.HalfVector `json:"embedding" gorm:"column:embedding;not null"` @@ -62,6 +64,7 @@ func toDBVectorEmbedding(indexInfo *types.IndexInfo, additionalParams map[string ChunkID: indexInfo.ChunkID, KnowledgeID: indexInfo.KnowledgeID, KnowledgeBaseID: indexInfo.KnowledgeBaseID, + TagID: indexInfo.TagID, Content: common.CleanInvalidUTF8(indexInfo.Content), IsEnabled: true, // Default to enabled } @@ -92,6 +95,7 @@ func fromDBVectorEmbeddingWithScore(embedding *pgVectorWithScore, matchType type ChunkID: embedding.ChunkID, KnowledgeID: embedding.KnowledgeID, KnowledgeBaseID: embedding.KnowledgeBaseID, + TagID: embedding.TagID, Content: embedding.Content, Score: embedding.Score, MatchType: matchType, diff --git a/internal/application/repository/retriever/qdrant/repository.go b/internal/application/repository/retriever/qdrant/repository.go index 7ca72cf9..d5fee479 100644 --- a/internal/application/repository/retriever/qdrant/repository.go +++ b/internal/application/repository/retriever/qdrant/repository.go @@ -25,6 +25,7 @@ const ( fieldChunkID = "chunk_id" fieldKnowledgeID = "knowledge_id" fieldKnowledgeBaseID = "knowledge_base_id" + fieldTagID = "tag_id" fieldEmbedding = "embedding" fieldIsEnabled = "is_enabled" ) @@ -443,6 +444,10 @@ func (q *qdrantRepository) getBaseFilter(params types.RetrieveParams) *qdrant.Fi if len(params.KnowledgeIDs) > 0 { must = append(must, qdrant.NewMatchKeywords(fieldKnowledgeID, params.KnowledgeIDs...)) } + // Filter by tag IDs if specified + if len(params.TagIDs) > 0 { + must = append(must, qdrant.NewMatchKeywords(fieldTagID, params.TagIDs...)) + } if len(params.ExcludeKnowledgeIDs) > 0 { mustNot = append(mustNot, qdrant.NewMatchKeywords(fieldKnowledgeID, params.ExcludeKnowledgeIDs...)) @@ -531,6 +536,7 @@ func (q *qdrantRepository) VectorRetrieve(ctx context.Context, ChunkID: payload[fieldChunkID].GetStringValue(), KnowledgeID: payload[fieldKnowledgeID].GetStringValue(), KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(), + TagID: payload[fieldTagID].GetStringValue(), }, Score: float64(point.Score), } @@ -623,6 +629,7 @@ func (q *qdrantRepository) KeywordsRetrieve(ctx context.Context, ChunkID: payload[fieldChunkID].GetStringValue(), KnowledgeID: payload[fieldKnowledgeID].GetStringValue(), KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(), + TagID: payload[fieldTagID].GetStringValue(), }, Score: 1.0, } @@ -805,6 +812,7 @@ func createPayload(embedding *QdrantVectorEmbedding) map[string]*qdrant.Value { fieldChunkID: embedding.ChunkID, fieldKnowledgeID: embedding.KnowledgeID, fieldKnowledgeBaseID: embedding.KnowledgeBaseID, + fieldTagID: embedding.TagID, fieldIsEnabled: embedding.IsEnabled, } return qdrant.NewValueMap(payload) @@ -862,6 +870,7 @@ func toQdrantVectorEmbedding(embedding *types.IndexInfo, additionalParams map[st ChunkID: embedding.ChunkID, KnowledgeID: embedding.KnowledgeID, KnowledgeBaseID: embedding.KnowledgeBaseID, + TagID: embedding.TagID, IsEnabled: true, // Default to enabled } if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), fieldEmbedding) { @@ -884,6 +893,7 @@ func fromQdrantVectorEmbedding(id string, ChunkID: embedding.ChunkID, KnowledgeID: embedding.KnowledgeID, KnowledgeBaseID: embedding.KnowledgeBaseID, + TagID: embedding.TagID, Content: embedding.Content, Score: embedding.Score, MatchType: matchType, diff --git a/internal/application/repository/retriever/qdrant/structs.go b/internal/application/repository/retriever/qdrant/structs.go index 9bae0d57..01a87ac7 100644 --- a/internal/application/repository/retriever/qdrant/structs.go +++ b/internal/application/repository/retriever/qdrant/structs.go @@ -20,6 +20,7 @@ type QdrantVectorEmbedding struct { ChunkID string `json:"chunk_id"` KnowledgeID string `json:"knowledge_id"` KnowledgeBaseID string `json:"knowledge_base_id"` + TagID string `json:"tag_id"` Embedding []float32 `json:"embedding"` IsEnabled bool `json:"is_enabled"` } diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index e2fb53e4..694b76a6 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -14,6 +14,7 @@ import ( "slices" "sort" "strings" + "sync" "time" "github.com/Tencent/WeKnora/docreader/client" @@ -3915,18 +3916,101 @@ func (s *knowledgeService) SearchFAQEntries(ctx context.Context, req.MatchCount = 50 } - // Prepare search parameters - searchParams := types.SearchParams{ - QueryText: secutils.SanitizeForLog(req.QueryText), - VectorThreshold: req.VectorThreshold, - MatchCount: req.MatchCount, - DisableKeywordsMatch: true, + // Build priority tag sets for sorting + hasFirstPriority := len(req.FirstPriorityTagIDs) > 0 + hasSecondPriority := len(req.SecondPriorityTagIDs) > 0 + hasPriorityFilter := hasFirstPriority || hasSecondPriority + + firstPrioritySet := make(map[string]struct{}, len(req.FirstPriorityTagIDs)) + for _, tagID := range req.FirstPriorityTagIDs { + firstPrioritySet[tagID] = struct{}{} + } + secondPrioritySet := make(map[string]struct{}, len(req.SecondPriorityTagIDs)) + for _, tagID := range req.SecondPriorityTagIDs { + secondPrioritySet[tagID] = struct{}{} } - // Call HybridSearch - searchResults, err := s.kbService.HybridSearch(ctx, kbID, searchParams) - if err != nil { - return nil, err + // Perform separate searches for each priority level to ensure FirstPriority results + // are not crowded out by higher-scoring SecondPriority results in TopK truncation + var searchResults []*types.SearchResult + + if hasPriorityFilter { + // Use goroutines to search both priority levels concurrently + var ( + firstResults []*types.SearchResult + secondResults []*types.SearchResult + firstErr error + secondErr error + wg sync.WaitGroup + ) + + if hasFirstPriority { + wg.Add(1) + go func() { + defer wg.Done() + firstParams := types.SearchParams{ + QueryText: secutils.SanitizeForLog(req.QueryText), + VectorThreshold: req.VectorThreshold, + MatchCount: req.MatchCount, + DisableKeywordsMatch: true, + TagIDs: req.FirstPriorityTagIDs, + } + firstResults, firstErr = s.kbService.HybridSearch(ctx, kbID, firstParams) + }() + } + + if hasSecondPriority { + wg.Add(1) + go func() { + defer wg.Done() + secondParams := types.SearchParams{ + QueryText: secutils.SanitizeForLog(req.QueryText), + VectorThreshold: req.VectorThreshold, + MatchCount: req.MatchCount, + DisableKeywordsMatch: true, + TagIDs: req.SecondPriorityTagIDs, + } + secondResults, secondErr = s.kbService.HybridSearch(ctx, kbID, secondParams) + }() + } + + wg.Wait() + + // Check errors + if firstErr != nil { + return nil, firstErr + } + if secondErr != nil { + return nil, secondErr + } + + // Merge results: FirstPriority first, then SecondPriority (deduplicated) + seenChunkIDs := make(map[string]struct{}) + for _, result := range firstResults { + if _, exists := seenChunkIDs[result.ID]; !exists { + seenChunkIDs[result.ID] = struct{}{} + searchResults = append(searchResults, result) + } + } + for _, result := range secondResults { + if _, exists := seenChunkIDs[result.ID]; !exists { + seenChunkIDs[result.ID] = struct{}{} + searchResults = append(searchResults, result) + } + } + } else { + // No priority filter, search all + searchParams := types.SearchParams{ + QueryText: secutils.SanitizeForLog(req.QueryText), + VectorThreshold: req.VectorThreshold, + MatchCount: req.MatchCount, + DisableKeywordsMatch: true, + } + var err error + searchResults, err = s.kbService.HybridSearch(ctx, kbID, searchParams) + if err != nil { + return nil, err + } } if len(searchResults) == 0 { @@ -3982,9 +4066,52 @@ func (s *knowledgeService) SearchFAQEntries(ctx context.Context, entries = append(entries, entry) } - slices.SortFunc(entries, func(a, b *types.FAQEntry) int { - return int(b.Score - a.Score) - }) + // Sort entries with two-level priority tag support + if hasPriorityFilter { + // getPriorityLevel returns: 0 = first priority, 1 = second priority, 2 = no priority + getPriorityLevel := func(tagID string) int { + if _, ok := firstPrioritySet[tagID]; ok { + return 0 + } + if _, ok := secondPrioritySet[tagID]; ok { + return 1 + } + return 2 + } + + slices.SortFunc(entries, func(a, b *types.FAQEntry) int { + aPriority := getPriorityLevel(a.TagID) + bPriority := getPriorityLevel(b.TagID) + + // Compare by priority level first + if aPriority != bPriority { + return aPriority - bPriority // Lower level = higher priority + } + + // Same priority level, sort by score descending + if b.Score > a.Score { + return 1 + } else if b.Score < a.Score { + return -1 + } + return 0 + }) + } else { + // No priority tags, sort by score only + slices.SortFunc(entries, func(a, b *types.FAQEntry) int { + if b.Score > a.Score { + return 1 + } else if b.Score < a.Score { + return -1 + } + return 0 + }) + } + + // Limit results to requested match count + if len(entries) > req.MatchCount { + entries = entries[:req.MatchCount] + } return entries, nil } @@ -4504,6 +4631,7 @@ func (s *knowledgeService) buildFAQIndexInfoList( KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, KnowledgeType: types.KnowledgeTypeFAQ, + TagID: chunk.TagID, IsEnabled: chunk.IsEnabled, }, }, nil @@ -4531,6 +4659,7 @@ func (s *knowledgeService) buildFAQIndexInfoList( KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, KnowledgeType: types.KnowledgeTypeFAQ, + TagID: chunk.TagID, IsEnabled: chunk.IsEnabled, }) @@ -4555,6 +4684,7 @@ func (s *knowledgeService) buildFAQIndexInfoList( KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, KnowledgeType: types.KnowledgeTypeFAQ, + TagID: chunk.TagID, IsEnabled: chunk.IsEnabled, }) } diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index efdea99a..6dfea6d0 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -542,6 +542,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, Threshold: params.VectorThreshold, RetrieverType: types.VectorRetrieverType, KnowledgeIDs: params.KnowledgeIDs, + TagIDs: params.TagIDs, } // For FAQ knowledge base, use FAQ index @@ -564,6 +565,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, Threshold: params.KeywordThreshold, RetrieverType: types.KeywordsRetrieverType, KnowledgeIDs: params.KnowledgeIDs, + TagIDs: params.TagIDs, }) logger.Info(ctx, "Keyword retrieval parameters setup completed") } @@ -618,7 +620,8 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, logger.Info(ctx, "Only vector results, keeping original embedding scores") chunkInfoMap := make(map[string]*types.IndexWithScore) for _, r := range vectorResults { - if _, exists := chunkInfoMap[r.ChunkID]; !exists { + // Keep the highest score for each chunk (FAQ may have multiple similar questions) + if existing, exists := chunkInfoMap[r.ChunkID]; !exists || r.Score > existing.Score { chunkInfoMap[r.ChunkID] = r } } @@ -656,16 +659,17 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, } // Collect all unique chunks and compute RRF scores + // Keep the highest score for each chunk from each retriever chunkInfoMap := make(map[string]*types.IndexWithScore) rrfScores := make(map[string]float64) - // Process vector results + // Process vector results - keep highest score per chunk for _, r := range vectorResults { - if _, exists := chunkInfoMap[r.ChunkID]; !exists { + if existing, exists := chunkInfoMap[r.ChunkID]; !exists || r.Score > existing.Score { chunkInfoMap[r.ChunkID] = r } } - // Process keyword results + // Process keyword results - only add if not already from vector for _, r := range keywordResults { if _, exists := chunkInfoMap[r.ChunkID]; !exists { chunkInfoMap[r.ChunkID] = r @@ -717,9 +721,8 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, // Check if we need iterative retrieval for FAQ with separate indexing // Only use iterative retrieval if we don't have enough unique chunks after first deduplication - totalRetrieved := len(vectorResults) + len(keywordResults) needsIterativeRetrieval := len(deduplicatedChunks) < params.MatchCount && - kb.Type == types.KnowledgeBaseTypeFAQ && totalRetrieved == matchCount*2 + kb.Type == types.KnowledgeBaseTypeFAQ && len(vectorResults) == matchCount if needsIterativeRetrieval { logger.Info(ctx, "Not enough unique chunks, using iterative retrieval for FAQ") // Use iterative retrieval to get more unique chunks (with negative question filtering inside) @@ -746,7 +749,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, // iterativeRetrieveWithDeduplication performs iterative retrieval until enough unique chunks are found // This is used for FAQ knowledge bases with separate indexing mode -// Negative question filtering is applied after each iteration to ensure we have enough valid chunks +// Negative question filtering is applied after each iteration with chunk data caching func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Context, retrieveEngine *retriever.CompositeRetrieveEngine, retrieveParams []types.RetrieveParams, @@ -754,8 +757,17 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co queryText string, ) []*types.IndexWithScore { maxIterations := 5 - currentTopK := matchCount + // Start with a larger TopK since we're called when first retrieval wasn't enough + // The first retrieval already used matchCount*3, so start from there + currentTopK := matchCount * 3 uniqueChunks := make(map[string]*types.IndexWithScore) + // Cache chunk data to avoid repeated DB queries across iterations + chunkDataCache := make(map[string]*types.Chunk) + // Track chunks that have been filtered out by negative questions + filteredOutChunks := make(map[string]struct{}) + + queryTextLower := strings.ToLower(strings.TrimSpace(queryText)) + tenantID := ctx.Value(types.TenantIDContextKey).(uint64) for i := 0; i < maxIterations; i++ { // Update TopK in retrieve params @@ -783,42 +795,59 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co break } - // Check if we got fewer results than requested - means no more results available totalRetrieved := len(iterationResults) - if totalRetrieved < currentTopK { - logger.Infof( - ctx, - "Retrieved %d results (less than TopK %d), no more results available", - totalRetrieved, - currentTopK, - ) + + // Collect new chunk IDs that need to be fetched from DB + newChunkIDs := make([]string, 0) + for _, result := range iterationResults { + if _, cached := chunkDataCache[result.ChunkID]; !cached { + if _, filtered := filteredOutChunks[result.ChunkID]; !filtered { + newChunkIDs = append(newChunkIDs, result.ChunkID) + } + } } - // Deduplicate and merge (keep highest score for each chunk) - // Multiple similar questions hitting the same chunk will keep the highest score + // Batch fetch only new chunks + if len(newChunkIDs) > 0 { + newChunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, newChunkIDs) + if err != nil { + logger.Warnf(ctx, "Failed to fetch chunks at iteration %d: %v", i+1, err) + } else { + for _, chunk := range newChunks { + chunkDataCache[chunk.ID] = chunk + } + } + } + + // Deduplicate, merge, and filter in one pass for _, result := range iterationResults { + // Skip if already filtered out + if _, filtered := filteredOutChunks[result.ChunkID]; filtered { + continue + } + + // Check negative questions using cached data + if chunkData, ok := chunkDataCache[result.ChunkID]; ok { + if chunkData.ChunkType == types.ChunkTypeFAQ { + if meta, err := chunkData.FAQMetadata(); err == nil && meta != nil { + if s.matchesNegativeQuestions(queryTextLower, meta.NegativeQuestions) { + filteredOutChunks[result.ChunkID] = struct{}{} + delete(uniqueChunks, result.ChunkID) + continue + } + } + } + } + + // Keep highest score for each chunk if existing, ok := uniqueChunks[result.ChunkID]; !ok || result.Score > existing.Score { uniqueChunks[result.ChunkID] = result } } - // Convert to slice for filtering - chunksSlice := make([]*types.IndexWithScore, 0, len(uniqueChunks)) - for _, chunk := range uniqueChunks { - chunksSlice = append(chunksSlice, chunk) - } - - // Filter by negative questions - chunksSlice = s.filterByNegativeQuestions(ctx, chunksSlice, queryText) - // Update uniqueChunks map with filtered results - uniqueChunks = make(map[string]*types.IndexWithScore, len(chunksSlice)) - for _, chunk := range chunksSlice { - uniqueChunks[chunk.ChunkID] = chunk - } - logger.Infof( ctx, - "After iteration %d: retrieved %d results, found %d unique chunks after filtering (target: %d)", + "After iteration %d: retrieved %d results, found %d valid unique chunks (target: %d)", i+1, totalRetrieved, len(uniqueChunks), @@ -833,7 +862,7 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co // Early stop: If we got fewer results than TopK, there are no more results to retrieve if totalRetrieved < currentTopK { - logger.Infof(ctx, "No more results available, stopping iteration") + logger.Infof(ctx, "No more results available (got %d < %d), stopping iteration", totalRetrieved, currentTopK) break } diff --git a/internal/handler/faq.go b/internal/handler/faq.go index a481648f..d2542201 100644 --- a/internal/handler/faq.go +++ b/internal/handler/faq.go @@ -285,7 +285,7 @@ func (h *FAQHandler) DeleteEntries(c *gin.Context) { // SearchFAQ godoc // @Summary 搜索FAQ -// @Description 使用混合搜索在FAQ中搜索 +// @Description 使用混合搜索在FAQ中搜索,支持两级优先级标签召回:first_priority_tag_ids优先级最高,second_priority_tag_ids次之 // @Tags FAQ管理 // @Accept json // @Produce json diff --git a/internal/types/embedding.go b/internal/types/embedding.go index 897b1def..ac080fb4 100644 --- a/internal/types/embedding.go +++ b/internal/types/embedding.go @@ -35,5 +35,6 @@ type IndexInfo struct { KnowledgeID string // ID of the knowledge KnowledgeBaseID string // ID of the knowledge base KnowledgeType string // Type of the knowledge (e.g., "faq", "manual") + TagID string // Tag ID for categorization (used for FAQ priority filtering) IsEnabled bool // Whether the chunk is enabled for retrieval } diff --git a/internal/types/faq.go b/internal/types/faq.go index 2fa4cbea..fd942474 100644 --- a/internal/types/faq.go +++ b/internal/types/faq.go @@ -223,9 +223,11 @@ type FAQBatchUpsertPayload struct { // FAQSearchRequest FAQ检索请求参数 type FAQSearchRequest struct { - QueryText string `json:"query_text" binding:"required"` - VectorThreshold float64 `json:"vector_threshold"` - MatchCount int `json:"match_count"` + QueryText string `json:"query_text" binding:"required"` + VectorThreshold float64 `json:"vector_threshold"` + MatchCount int `json:"match_count"` + FirstPriorityTagIDs []string `json:"first_priority_tag_ids"` // 第一优先级标签ID列表,限定命中范围,优先级最高 + SecondPriorityTagIDs []string `json:"second_priority_tag_ids"` // 第二优先级标签ID列表,限定命中范围,优先级低于第一优先级 } // UntaggedTagID is the special tag ID representing uncategorized entries diff --git a/internal/types/retriever.go b/internal/types/retriever.go index d486cc3d..e2a11f60 100644 --- a/internal/types/retriever.go +++ b/internal/types/retriever.go @@ -32,6 +32,8 @@ type RetrieveParams struct { KnowledgeBaseIDs []string // Knowledge IDs KnowledgeIDs []string + // Tag IDs for filtering (used for FAQ priority filtering) + TagIDs []string // Excluded knowledge IDs ExcludeKnowledgeIDs []string // Excluded chunk IDs @@ -72,6 +74,8 @@ type IndexWithScore struct { KnowledgeID string // Knowledge base ID KnowledgeBaseID string + // Tag ID + TagID string // Score Score float64 // Match type diff --git a/internal/types/search.go b/internal/types/search.go index 95617b3a..a925b173 100644 --- a/internal/types/search.go +++ b/internal/types/search.go @@ -98,6 +98,7 @@ type SearchParams struct { DisableKeywordsMatch bool `json:"disable_keywords_match"` DisableVectorMatch bool `json:"disable_vector_match"` KnowledgeIDs []string `json:"knowledge_ids"` + TagIDs []string `json:"tag_ids"` // Tag IDs for filtering (used for FAQ priority filtering) } // Value implements the driver.Valuer interface, used to convert SearchResult to database value diff --git a/migrations/versioned/000007_embeddings_tag_id.down.sql b/migrations/versioned/000007_embeddings_tag_id.down.sql new file mode 100644 index 00000000..59122db4 --- /dev/null +++ b/migrations/versioned/000007_embeddings_tag_id.down.sql @@ -0,0 +1,12 @@ +-- Remove tag_id column from embeddings table +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'embeddings' AND column_name = 'tag_id' + ) THEN + DROP INDEX IF EXISTS idx_embeddings_tag_id; + ALTER TABLE embeddings DROP COLUMN tag_id; + RAISE NOTICE '[Migration 000007 Rollback] Removed tag_id column from embeddings table'; + END IF; +END $$; diff --git a/migrations/versioned/000007_embeddings_tag_id.up.sql b/migrations/versioned/000007_embeddings_tag_id.up.sql new file mode 100644 index 00000000..082624cd --- /dev/null +++ b/migrations/versioned/000007_embeddings_tag_id.up.sql @@ -0,0 +1,15 @@ +-- Add tag_id column to embeddings table for FAQ priority filtering +DO $$ +BEGIN + -- Add tag_id column if not exists + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'embeddings' AND column_name = 'tag_id' + ) THEN + ALTER TABLE embeddings ADD COLUMN tag_id VARCHAR(36); + CREATE INDEX IF NOT EXISTS idx_embeddings_tag_id ON embeddings(tag_id); + RAISE NOTICE '[Migration 000007] Added tag_id column and index to embeddings table'; + ELSE + RAISE NOTICE '[Migration 000007] tag_id column already exists in embeddings table, skipping'; + END IF; +END $$;