diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.go index 5e205a54..1497a9bc 100644 --- a/internal/application/repository/retriever/elasticsearch/v7/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go @@ -287,18 +287,18 @@ func (e *elasticsearchRepository) countBulkErrors(ctx context.Context, } // DeleteByChunkIDList Delete indices by chunk ID list -func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error { +func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error { return e.deleteByFieldList(ctx, "chunk_id.keyword", chunkIDList) } // DeleteBySourceIDList Delete indices by source ID list -func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error { +func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error { return e.deleteByFieldList(ctx, "source_id.keyword", sourceIDList) } // DeleteByKnowledgeIDList Delete indices by knowledge ID list func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context, - knowledgeIDList []string, dimension int, + knowledgeIDList []string, dimension int, knowledgeType string, ) error { return e.deleteByFieldList(ctx, "knowledge_id.keyword", knowledgeIDList) } @@ -816,6 +816,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context, sourceToTargetChunkIDMap map[string]string, targetKnowledgeBaseID string, dimension int, + knowledgeType string, ) error { log := logger.GetLogger(ctx) log.Infof( diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.go index a6c025f5..acd305c3 100644 --- a/internal/application/repository/retriever/elasticsearch/v8/repository.go +++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go @@ -163,7 +163,7 @@ func (e *elasticsearchRepository) BatchSave(ctx context.Context, // DeleteByChunkIDList removes documents from the index based on chunk IDs // Returns an error if the delete operation fails -func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error { +func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error { log := logger.GetLogger(ctx) if len(chunkIDList) == 0 { log.Warn("[Elasticsearch] Empty chunk ID list provided for deletion, skipping") @@ -186,7 +186,7 @@ func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunk // DeleteBySourceIDList removes documents from the index based on source IDs // Returns an error if the delete operation fails -func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error { +func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error { log := logger.GetLogger(ctx) if len(sourceIDList) == 0 { log.Warn("[Elasticsearch] Empty source ID list provided for deletion, skipping") @@ -210,7 +210,7 @@ func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sour // DeleteByKnowledgeIDList removes documents from the index based on knowledge IDs // Returns an error if the delete operation fails func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context, - knowledgeIDList []string, dimension int, + knowledgeIDList []string, dimension int, knowledgeType string, ) error { log := logger.GetLogger(ctx) if len(knowledgeIDList) == 0 { @@ -447,6 +447,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context, sourceToTargetChunkIDMap map[string]string, targetKnowledgeBaseID string, dimension int, + knowledgeType string, ) error { log := logger.GetLogger(ctx) log.Infof( diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.go index ad53480f..3c493714 100644 --- a/internal/application/repository/retriever/postgres/repository.go +++ b/internal/application/repository/retriever/postgres/repository.go @@ -107,7 +107,7 @@ func (g *pgRepository) BatchSave( } // DeleteByChunkIDList deletes indices by chunk IDs -func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error { +func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error { logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by chunk IDs, count: %d", len(chunkIDList)) result := g.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&pgVector{}) if result.Error != nil { @@ -119,7 +119,7 @@ func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []st } // DeleteBySourceIDList deletes indices by source IDs -func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error { +func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error { if len(sourceIDList) == 0 { return nil } @@ -134,7 +134,7 @@ func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList [] } // DeleteByKnowledgeIDList deletes indices by knowledge IDs -func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error { +func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string) error { logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList)) result := g.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&pgVector{}) if result.Error != nil { @@ -357,6 +357,7 @@ func (g *pgRepository) CopyIndices(ctx context.Context, sourceToTargetChunkIDMap map[string]string, targetKnowledgeBaseID string, dimension int, + knowledgeType string, ) error { logger.GetLogger(ctx).Infof( "[Postgres] Copying indices, source knowledge base: %s, target knowledge base: %s, mapping count: %d", diff --git a/internal/application/repository/retriever/qdrant/repository.go b/internal/application/repository/retriever/qdrant/repository.go index e615f7fa..c971a1c4 100644 --- a/internal/application/repository/retriever/qdrant/repository.go +++ b/internal/application/repository/retriever/qdrant/repository.go @@ -264,7 +264,7 @@ func (q *qdrantRepository) BatchSave(ctx context.Context, } // DeleteByChunkIDList removes points from the collection based on chunk IDs -func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error { +func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error { log := logger.GetLogger(ctx) if len(chunkIDList) == 0 { log.Warn("[Qdrant] Empty chunk ID list provided for deletion, skipping") @@ -293,7 +293,7 @@ func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList // DeleteByKnowledgeIDList removes points from the collection based on knowledge IDs func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context, - knowledgeIDList []string, dimension int, + knowledgeIDList []string, dimension int, knowledgeType string, ) error { log := logger.GetLogger(ctx) if len(knowledgeIDList) == 0 { @@ -323,7 +323,7 @@ func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context, // DeleteBySourceIDList removes points from the collection based on source IDs func (q *qdrantRepository) DeleteBySourceIDList(ctx context.Context, - sourceIDList []string, dimension int, + sourceIDList []string, dimension int, knowledgeType string, ) error { log := logger.GetLogger(ctx) if len(sourceIDList) == 0 { @@ -644,6 +644,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, sourceToTargetChunkIDMap map[string]string, targetKnowledgeBaseID string, dimension int, + knowledgeType string, ) error { log := logger.GetLogger(ctx) log.Infof( diff --git a/internal/application/service/chunk.go b/internal/application/service/chunk.go index dd0a9751..938232e2 100644 --- a/internal/application/service/chunk.go +++ b/internal/application/service/chunk.go @@ -406,7 +406,7 @@ func (s *chunkService) DeleteGeneratedQuestion(ctx context.Context, chunkID stri } // Delete the vector index by source ID - if err := retrieveEngine.DeleteBySourceIDList(ctx, []string{sourceID}, embeddingModel.GetDimensions()); err != nil { + if err := retrieveEngine.DeleteBySourceIDList(ctx, []string{sourceID}, embeddingModel.GetDimensions(), kb.Type); err != nil { logger.Warnf(ctx, "Failed to delete vector index for question (may not exist): %v", err) // Continue even if vector deletion fails - the question might not have been indexed } diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 5b21a751..42ed3b5b 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -755,7 +755,7 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } - if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { + if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } @@ -839,17 +839,23 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") return err } - group := map[string][]string{} - for _, knowledge := range knowledgeList { - group[knowledge.EmbeddingModelID] = append(group[knowledge.EmbeddingModelID], knowledge.ID) + // Group by EmbeddingModelID and Type + type groupKey struct { + EmbeddingModelID string + Type string } - for embeddingModelID, knowledgeList := range group { - embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, embeddingModelID) + group := map[groupKey][]string{} + for _, knowledge := range knowledgeList { + key := groupKey{EmbeddingModelID: knowledge.EmbeddingModelID, Type: knowledge.Type} + group[key] = append(group[key], knowledge.ID) + } + for key, knowledgeIDs := range group { + embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, key.EmbeddingModelID) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge get embedding model failed") return err } - if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeList, embeddingModel.GetDimensions()); err != nil { + if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeIDs, embeddingModel.GetDimensions(), key.Type); err != nil { logger.GetLogger(ctx). WithField("error", err). Errorf("DeleteKnowledge delete knowledge embedding failed") @@ -1056,7 +1062,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err == nil { - if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { + if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil { logger.Warnf(ctx, "Failed to delete existing index data (may not exist): %v", err) // 不返回错误,继续处理(可能没有旧数据) } else { @@ -1348,7 +1354,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, // delete index if err := retrieveEngine.DeleteByKnowledgeIDList( - ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), + ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), kb.Type, ); err != nil { logger.Errorf(ctx, "Delete index failed: %v", err) } @@ -1375,7 +1381,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil { logger.Warnf(ctx, "Failed to cleanup chunks after deletion detected: %v", err) } - if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { + if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), kb.Type); err != nil { logger.Warnf(ctx, "Failed to cleanup index after deletion detected: %v", err) } span.AddEvent("aborted: knowledge was deleted during processing") @@ -2330,7 +2336,7 @@ func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, c } // Delete old vector representation of the chunk - err = retrieveEngine.DeleteByChunkIDList(ctx, ids, embeddingModel.GetDimensions()) + err = retrieveEngine.DeleteByChunkIDList(ctx, ids, embeddingModel.GetDimensions(), sourceKB.Type) if err != nil { return err } @@ -2607,6 +2613,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl map[string]string{src.ID: dst.ID}, srcTodst, embeddingModel.GetDimensions(), + dst.Type, ); err != nil { return err } @@ -4372,6 +4379,8 @@ func (s *knowledgeService) buildFAQIndexInfoList( ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, + KnowledgeType: types.KnowledgeTypeFAQ, + IsEnabled: chunk.IsEnabled, }, }, nil } @@ -4397,6 +4406,8 @@ func (s *knowledgeService) buildFAQIndexInfoList( ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, + KnowledgeType: types.KnowledgeTypeFAQ, + IsEnabled: chunk.IsEnabled, }) // 每个相似问创建一个索引项 @@ -4419,6 +4430,8 @@ func (s *knowledgeService) buildFAQIndexInfoList( ChunkID: chunk.ID, KnowledgeID: chunk.KnowledgeID, KnowledgeBaseID: chunk.KnowledgeBaseID, + KnowledgeType: types.KnowledgeTypeFAQ, + IsEnabled: chunk.IsEnabled, }) } @@ -4478,7 +4491,7 @@ func (s *knowledgeService) indexFAQChunks(ctx context.Context, var deleteDuration time.Duration if needDelete { deleteStartTime := time.Now() - if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil { + if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil { logger.Warnf(ctx, "Delete FAQ vectors failed: %v", err) } deleteDuration = time.Since(deleteStartTime) @@ -4561,7 +4574,7 @@ func (s *knowledgeService) deleteFAQChunkVectors(ctx context.Context, } size := retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfo) - if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil { + if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil { return err } if size > 0 { @@ -4696,7 +4709,7 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle logger.GetLogger(ctx).WithField("error", modelErr).Error("Failed to get embedding model during cleanup") cleanupErr = errors.Join(cleanupErr, modelErr) } else { - if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { + if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge index") cleanupErr = errors.Join(cleanupErr, err) } @@ -5119,7 +5132,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) for _, chunk := range chunksDeleted { chunkIDs = append(chunkIDs, chunk.ID) } - if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil { + if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil { logger.Warnf(ctx, "Failed to delete index data for chunks (may not exist): %v", err) } else { logger.Infof(ctx, "Successfully deleted index data for %d chunks", len(chunksDeleted)) @@ -5420,7 +5433,7 @@ func (s *knowledgeService) cloneFAQKnowledgeBase( // Delete FAQ chunks that don't exist in source if len(chunksToDelete) > 0 { // Delete from vector store - if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions()); err != nil { + if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil { logger.Errorf(ctx, "Failed to delete FAQ chunks from vector store: %v", err) handleError(progress, err, "Failed to delete FAQ entries from vector store") return err diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index 3bcfc41a..d35bdde9 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -254,20 +254,25 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin if err != nil { logger.Warnf(ctx, "Failed to create retrieve engine: %v", err) } else { - // Group knowledge by embedding model - embeddingGroups := make(map[string][]string) + // Group knowledge by embedding model and type + type groupKey struct { + EmbeddingModelID string + Type string + } + embeddingGroups := make(map[groupKey][]string) for _, knowledge := range knowledgeList { - embeddingGroups[knowledge.EmbeddingModelID] = append(embeddingGroups[knowledge.EmbeddingModelID], knowledge.ID) + key := groupKey{EmbeddingModelID: knowledge.EmbeddingModelID, Type: knowledge.Type} + embeddingGroups[key] = append(embeddingGroups[key], knowledge.ID) } - for embeddingModelID, knowledgeGroup := range embeddingGroups { - embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, embeddingModelID) + for key, knowledgeGroup := range embeddingGroups { + embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, key.EmbeddingModelID) if err != nil { - logger.Warnf(ctx, "Failed to get embedding model %s: %v", embeddingModelID, err) + logger.Warnf(ctx, "Failed to get embedding model %s: %v", key.EmbeddingModelID, err) continue } - if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeGroup, embeddingModel.GetDimensions()); err != nil { - logger.Warnf(ctx, "Failed to delete embeddings for model %s: %v", embeddingModelID, err) + if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeGroup, embeddingModel.GetDimensions(), key.Type); err != nil { + logger.Warnf(ctx, "Failed to delete embeddings for model %s: %v", key.EmbeddingModelID, err) } } } @@ -479,14 +484,21 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, } logger.Infof(ctx, "Query embedding generated successfully, embedding vector length: %d", len(queryEmbedding)) - retrieveParams = append(retrieveParams, types.RetrieveParams{ + vectorParams := types.RetrieveParams{ Query: params.QueryText, Embedding: queryEmbedding, KnowledgeBaseIDs: []string{id}, TopK: matchCount, Threshold: params.VectorThreshold, RetrieverType: types.VectorRetrieverType, - }) + } + + // For FAQ knowledge base, use FAQ index + if kb.Type == types.KnowledgeBaseTypeFAQ { + vectorParams.KnowledgeType = types.KnowledgeTypeFAQ + } + + retrieveParams = append(retrieveParams, vectorParams) logger.Info(ctx, "Vector retrieval parameters setup completed") } diff --git a/internal/application/service/retriever/composite.go b/internal/application/service/retriever/composite.go index 532b1feb..959a622c 100644 --- a/internal/application/service/retriever/composite.go +++ b/internal/application/service/retriever/composite.go @@ -234,10 +234,10 @@ func (c *CompositeRetrieveEngine) BatchIndex(ctx context.Context, // DeleteByChunkIDList deletes vector embeddings by chunk ID list from all registered repositories func (c *CompositeRetrieveEngine) DeleteByChunkIDList(ctx context.Context, - chunkIDList []string, dimension int, + chunkIDList []string, dimension int, knowledgeType string, ) error { return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error { - if err := engineInfo.retrieveEngine.DeleteByChunkIDList(ctx, chunkIDList, dimension); err != nil { + if err := engineInfo.retrieveEngine.DeleteByChunkIDList(ctx, chunkIDList, dimension, knowledgeType); err != nil { logger.GetLogger(ctx).Errorf("Repository %s failed to delete chunk ID list: %v", engineInfo.retrieveEngine.EngineType(), err) return err @@ -248,10 +248,10 @@ func (c *CompositeRetrieveEngine) DeleteByChunkIDList(ctx context.Context, // DeleteBySourceIDList deletes vector embeddings by source ID list from all registered repositories func (c *CompositeRetrieveEngine) DeleteBySourceIDList(ctx context.Context, - sourceIDList []string, dimension int, + sourceIDList []string, dimension int, knowledgeType string, ) error { return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error { - if err := engineInfo.retrieveEngine.DeleteBySourceIDList(ctx, sourceIDList, dimension); err != nil { + if err := engineInfo.retrieveEngine.DeleteBySourceIDList(ctx, sourceIDList, dimension, knowledgeType); err != nil { logger.GetLogger(ctx).Errorf("Repository %s failed to delete source ID list: %v", engineInfo.retrieveEngine.EngineType(), err) return err @@ -268,6 +268,7 @@ func (c *CompositeRetrieveEngine) CopyIndices( sourceToTargetKBIDMap map[string]string, sourceToTargetChunkIDMap map[string]string, dimension int, + knowledgeType string, ) error { return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error { if err := engineInfo.retrieveEngine.CopyIndices( @@ -277,6 +278,7 @@ func (c *CompositeRetrieveEngine) CopyIndices( sourceToTargetChunkIDMap, targetKnowledgeBaseID, dimension, + knowledgeType, ); err != nil { logger.Errorf(ctx, "Repository %s failed to copy indices: %v", engineInfo.retrieveEngine.EngineType(), err) return err @@ -287,10 +289,10 @@ func (c *CompositeRetrieveEngine) CopyIndices( // DeleteByKnowledgeIDList deletes vector embeddings by knowledge ID list from all registered repositories func (c *CompositeRetrieveEngine) DeleteByKnowledgeIDList(ctx context.Context, - knowledgeIDList []string, dimension int, + knowledgeIDList []string, dimension int, knowledgeType string, ) error { return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error { - if err := engineInfo.retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension); err != nil { + if err := engineInfo.retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension, knowledgeType); err != nil { logger.GetLogger(ctx).Errorf("Repository %s failed to delete knowledge ID list: %v", engineInfo.retrieveEngine.EngineType(), err) return err diff --git a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go index 1cbe355f..226c3555 100644 --- a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go +++ b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go @@ -65,7 +65,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte if len(indexInfoList) == 0 { return nil } - params := make(map[string]any) + if slices.Contains(retrieverTypes, types.VectorRetrieverType) { var contentList []string for _, indexInfo := range indexInfoList { @@ -87,6 +87,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte } batchSize := 40 for i, indexChunk := range utils.ChunkSlice(indexInfoList, batchSize) { + params := make(map[string]any) embeddingMap := make(map[string][]float32) for j, indexInfo := range indexChunk { embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j] @@ -101,6 +102,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte } var err error for _, indexChunk := range utils.ChunkSlice(indexInfoList, 10) { + params := make(map[string]any) err = v.indexRepository.BatchSave(ctx, indexChunk, params) if err != nil { return err @@ -111,23 +113,23 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte // DeleteByChunkIDList deletes vectors by their chunk IDs func (v *KeywordsVectorHybridRetrieveEngineService) DeleteByChunkIDList(ctx context.Context, - indexIDList []string, dimension int, + indexIDList []string, dimension int, knowledgeType string, ) error { - return v.indexRepository.DeleteByChunkIDList(ctx, indexIDList, dimension) + return v.indexRepository.DeleteByChunkIDList(ctx, indexIDList, dimension, knowledgeType) } // DeleteBySourceIDList deletes vectors by their source IDs func (v *KeywordsVectorHybridRetrieveEngineService) DeleteBySourceIDList(ctx context.Context, - sourceIDList []string, dimension int, + sourceIDList []string, dimension int, knowledgeType string, ) error { - return v.indexRepository.DeleteBySourceIDList(ctx, sourceIDList, dimension) + return v.indexRepository.DeleteBySourceIDList(ctx, sourceIDList, dimension, knowledgeType) } // DeleteByKnowledgeIDList deletes vectors by their knowledge IDs func (v *KeywordsVectorHybridRetrieveEngineService) DeleteByKnowledgeIDList(ctx context.Context, - knowledgeIDList []string, dimension int, + knowledgeIDList []string, dimension int, knowledgeType string, ) error { - return v.indexRepository.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension) + return v.indexRepository.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension, knowledgeType) } // Support returns the retriever types supported by this engine @@ -162,12 +164,13 @@ func (v *KeywordsVectorHybridRetrieveEngineService) CopyIndices( sourceToTargetChunkIDMap map[string]string, targetKnowledgeBaseID string, dimension int, + knowledgeType string, ) error { logger.Infof(ctx, "Copy indices from knowledge base %s to %s, mapping relation count: %d", sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap), ) return v.indexRepository.CopyIndices( - ctx, sourceKnowledgeBaseID, sourceToTargetKBIDMap, sourceToTargetChunkIDMap, targetKnowledgeBaseID, dimension, + ctx, sourceKnowledgeBaseID, sourceToTargetKBIDMap, sourceToTargetChunkIDMap, targetKnowledgeBaseID, dimension, knowledgeType, ) } diff --git a/internal/types/embedding.go b/internal/types/embedding.go index 51b50f9e..ffc320ad 100644 --- a/internal/types/embedding.go +++ b/internal/types/embedding.go @@ -32,4 +32,6 @@ type IndexInfo struct { ChunkID string // ID of the text chunk KnowledgeID string // ID of the knowledge KnowledgeBaseID string // ID of the knowledge base + KnowledgeType string // Type of the knowledge (e.g., "faq", "manual") + IsEnabled bool // Whether the chunk is enabled for retrieval } diff --git a/internal/types/interfaces/retriever.go b/internal/types/interfaces/retriever.go index 0c8cc3c3..549dd180 100644 --- a/internal/types/interfaces/retriever.go +++ b/internal/types/interfaces/retriever.go @@ -31,9 +31,9 @@ type RetrieveEngineRepository interface { EstimateStorageSize(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) int64 // DeleteByChunkIDList deletes the index info by chunk id list - DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int) error + DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int, knowledgeType string) error // DeleteBySourceIDList deletes the index info by source id list - DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error + DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error // 复制索引数据 // sourceKnowledgeBaseID: 源知识库ID // sourceToTargetChunkIDMap: 源分块ID到目标分块ID的映射关系 @@ -46,10 +46,11 @@ type RetrieveEngineRepository interface { sourceToTargetChunkIDMap map[string]string, targetKnowledgeBaseID string, dimension int, + knowledgeType string, ) error // DeleteByKnowledgeIDList deletes the index info by knowledge id list - DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error + DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string) error // BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch // chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled) @@ -102,16 +103,17 @@ type RetrieveEngineService interface { sourceToTargetChunkIDMap map[string]string, targetKnowledgeBaseID string, dimension int, + knowledgeType string, ) error // DeleteByChunkIDList deletes the index info by chunk id list - DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int) error + DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int, knowledgeType string) error // DeleteBySourceIDList deletes the index info by source id list - DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error + DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error // DeleteByKnowledgeIDList deletes the index info by knowledge id list - DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error + DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string) error // BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch // chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled) diff --git a/internal/types/retriever.go b/internal/types/retriever.go index 2f2b3100..c42e5940 100644 --- a/internal/types/retriever.go +++ b/internal/types/retriever.go @@ -38,6 +38,8 @@ type RetrieveParams struct { TopK int // Similarity threshold Threshold float64 + // Knowledge type (e.g., "faq", "manual") - determines which index to use + KnowledgeType string // Additional parameters, different retrievers may require different parameters AdditionalParams map[string]interface{} // Retriever type @@ -72,6 +74,8 @@ type IndexWithScore struct { Score float64 // Match type MatchType MatchType + // IsEnabled + IsEnabled bool } // GetScore returns the score for ScoreComparable interface