feat: 支持FAQ索引类型分离,新增is_enabled字段过滤和批量更新优化

This commit is contained in:
wizardchen
2025-12-16 21:17:55 +08:00
committed by lyingbug
parent 509582a4d5
commit b70450ddd8
12 changed files with 101 additions and 59 deletions
@@ -287,18 +287,18 @@ func (e *elasticsearchRepository) countBulkErrors(ctx context.Context,
}
// DeleteByChunkIDList Delete indices by chunk ID list
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
return e.deleteByFieldList(ctx, "chunk_id.keyword", chunkIDList)
}
// DeleteBySourceIDList Delete indices by source ID list
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error {
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
return e.deleteByFieldList(ctx, "source_id.keyword", sourceIDList)
}
// DeleteByKnowledgeIDList Delete indices by knowledge ID list
func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
knowledgeIDList []string, dimension int,
knowledgeIDList []string, dimension int, knowledgeType string,
) error {
return e.deleteByFieldList(ctx, "knowledge_id.keyword", knowledgeIDList)
}
@@ -816,6 +816,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error {
log := logger.GetLogger(ctx)
log.Infof(
@@ -163,7 +163,7 @@ func (e *elasticsearchRepository) BatchSave(ctx context.Context,
// DeleteByChunkIDList removes documents from the index based on chunk IDs
// Returns an error if the delete operation fails
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
log := logger.GetLogger(ctx)
if len(chunkIDList) == 0 {
log.Warn("[Elasticsearch] Empty chunk ID list provided for deletion, skipping")
@@ -186,7 +186,7 @@ func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunk
// DeleteBySourceIDList removes documents from the index based on source IDs
// Returns an error if the delete operation fails
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error {
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
log := logger.GetLogger(ctx)
if len(sourceIDList) == 0 {
log.Warn("[Elasticsearch] Empty source ID list provided for deletion, skipping")
@@ -210,7 +210,7 @@ func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sour
// DeleteByKnowledgeIDList removes documents from the index based on knowledge IDs
// Returns an error if the delete operation fails
func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
knowledgeIDList []string, dimension int,
knowledgeIDList []string, dimension int, knowledgeType string,
) error {
log := logger.GetLogger(ctx)
if len(knowledgeIDList) == 0 {
@@ -447,6 +447,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error {
log := logger.GetLogger(ctx)
log.Infof(
@@ -107,7 +107,7 @@ func (g *pgRepository) BatchSave(
}
// DeleteByChunkIDList deletes indices by chunk IDs
func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
result := g.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&pgVector{})
if result.Error != nil {
@@ -119,7 +119,7 @@ func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []st
}
// DeleteBySourceIDList deletes indices by source IDs
func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error {
func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
if len(sourceIDList) == 0 {
return nil
}
@@ -134,7 +134,7 @@ func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []
}
// DeleteByKnowledgeIDList deletes indices by knowledge IDs
func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error {
func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string) error {
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
result := g.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&pgVector{})
if result.Error != nil {
@@ -357,6 +357,7 @@ func (g *pgRepository) CopyIndices(ctx context.Context,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error {
logger.GetLogger(ctx).Infof(
"[Postgres] Copying indices, source knowledge base: %s, target knowledge base: %s, mapping count: %d",
@@ -264,7 +264,7 @@ func (q *qdrantRepository) BatchSave(ctx context.Context,
}
// DeleteByChunkIDList removes points from the collection based on chunk IDs
func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
log := logger.GetLogger(ctx)
if len(chunkIDList) == 0 {
log.Warn("[Qdrant] Empty chunk ID list provided for deletion, skipping")
@@ -293,7 +293,7 @@ func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList
// DeleteByKnowledgeIDList removes points from the collection based on knowledge IDs
func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context,
knowledgeIDList []string, dimension int,
knowledgeIDList []string, dimension int, knowledgeType string,
) error {
log := logger.GetLogger(ctx)
if len(knowledgeIDList) == 0 {
@@ -323,7 +323,7 @@ func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context,
// DeleteBySourceIDList removes points from the collection based on source IDs
func (q *qdrantRepository) DeleteBySourceIDList(ctx context.Context,
sourceIDList []string, dimension int,
sourceIDList []string, dimension int, knowledgeType string,
) error {
log := logger.GetLogger(ctx)
if len(sourceIDList) == 0 {
@@ -644,6 +644,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error {
log := logger.GetLogger(ctx)
log.Infof(
+1 -1
View File
@@ -406,7 +406,7 @@ func (s *chunkService) DeleteGeneratedQuestion(ctx context.Context, chunkID stri
}
// Delete the vector index by source ID
if err := retrieveEngine.DeleteBySourceIDList(ctx, []string{sourceID}, embeddingModel.GetDimensions()); err != nil {
if err := retrieveEngine.DeleteBySourceIDList(ctx, []string{sourceID}, embeddingModel.GetDimensions(), kb.Type); err != nil {
logger.Warnf(ctx, "Failed to delete vector index for question (may not exist): %v", err)
// Continue even if vector deletion fails - the question might not have been indexed
}
+29 -16
View File
@@ -755,7 +755,7 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
return err
}
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil {
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
return err
}
@@ -839,17 +839,23 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
return err
}
group := map[string][]string{}
for _, knowledge := range knowledgeList {
group[knowledge.EmbeddingModelID] = append(group[knowledge.EmbeddingModelID], knowledge.ID)
// Group by EmbeddingModelID and Type
type groupKey struct {
EmbeddingModelID string
Type string
}
for embeddingModelID, knowledgeList := range group {
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, embeddingModelID)
group := map[groupKey][]string{}
for _, knowledge := range knowledgeList {
key := groupKey{EmbeddingModelID: knowledge.EmbeddingModelID, Type: knowledge.Type}
group[key] = append(group[key], knowledge.ID)
}
for key, knowledgeIDs := range group {
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, key.EmbeddingModelID)
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge get embedding model failed")
return err
}
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeList, embeddingModel.GetDimensions()); err != nil {
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeIDs, embeddingModel.GetDimensions(), key.Type); err != nil {
logger.GetLogger(ctx).
WithField("error", err).
Errorf("DeleteKnowledge delete knowledge embedding failed")
@@ -1056,7 +1062,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
if err == nil {
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil {
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil {
logger.Warnf(ctx, "Failed to delete existing index data (may not exist): %v", err)
// 不返回错误,继续处理(可能没有旧数据)
} else {
@@ -1348,7 +1354,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
// delete index
if err := retrieveEngine.DeleteByKnowledgeIDList(
ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(),
ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), kb.Type,
); err != nil {
logger.Errorf(ctx, "Delete index failed: %v", err)
}
@@ -1375,7 +1381,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil {
logger.Warnf(ctx, "Failed to cleanup chunks after deletion detected: %v", err)
}
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil {
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), kb.Type); err != nil {
logger.Warnf(ctx, "Failed to cleanup index after deletion detected: %v", err)
}
span.AddEvent("aborted: knowledge was deleted during processing")
@@ -2330,7 +2336,7 @@ func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, c
}
// Delete old vector representation of the chunk
err = retrieveEngine.DeleteByChunkIDList(ctx, ids, embeddingModel.GetDimensions())
err = retrieveEngine.DeleteByChunkIDList(ctx, ids, embeddingModel.GetDimensions(), sourceKB.Type)
if err != nil {
return err
}
@@ -2607,6 +2613,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
map[string]string{src.ID: dst.ID},
srcTodst,
embeddingModel.GetDimensions(),
dst.Type,
); err != nil {
return err
}
@@ -4372,6 +4379,8 @@ func (s *knowledgeService) buildFAQIndexInfoList(
ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
KnowledgeType: types.KnowledgeTypeFAQ,
IsEnabled: chunk.IsEnabled,
},
}, nil
}
@@ -4397,6 +4406,8 @@ func (s *knowledgeService) buildFAQIndexInfoList(
ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
KnowledgeType: types.KnowledgeTypeFAQ,
IsEnabled: chunk.IsEnabled,
})
// 每个相似问创建一个索引项
@@ -4419,6 +4430,8 @@ func (s *knowledgeService) buildFAQIndexInfoList(
ChunkID: chunk.ID,
KnowledgeID: chunk.KnowledgeID,
KnowledgeBaseID: chunk.KnowledgeBaseID,
KnowledgeType: types.KnowledgeTypeFAQ,
IsEnabled: chunk.IsEnabled,
})
}
@@ -4478,7 +4491,7 @@ func (s *knowledgeService) indexFAQChunks(ctx context.Context,
var deleteDuration time.Duration
if needDelete {
deleteStartTime := time.Now()
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil {
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil {
logger.Warnf(ctx, "Delete FAQ vectors failed: %v", err)
}
deleteDuration = time.Since(deleteStartTime)
@@ -4561,7 +4574,7 @@ func (s *knowledgeService) deleteFAQChunkVectors(ctx context.Context,
}
size := retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfo)
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil {
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil {
return err
}
if size > 0 {
@@ -4696,7 +4709,7 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle
logger.GetLogger(ctx).WithField("error", modelErr).Error("Failed to get embedding model during cleanup")
cleanupErr = errors.Join(cleanupErr, modelErr)
} else {
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil {
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil {
logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge index")
cleanupErr = errors.Join(cleanupErr, err)
}
@@ -5119,7 +5132,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
for _, chunk := range chunksDeleted {
chunkIDs = append(chunkIDs, chunk.ID)
}
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil {
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil {
logger.Warnf(ctx, "Failed to delete index data for chunks (may not exist): %v", err)
} else {
logger.Infof(ctx, "Successfully deleted index data for %d chunks", len(chunksDeleted))
@@ -5420,7 +5433,7 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
// Delete FAQ chunks that don't exist in source
if len(chunksToDelete) > 0 {
// Delete from vector store
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions()); err != nil {
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil {
logger.Errorf(ctx, "Failed to delete FAQ chunks from vector store: %v", err)
handleError(progress, err, "Failed to delete FAQ entries from vector store")
return err
+22 -10
View File
@@ -254,20 +254,25 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
if err != nil {
logger.Warnf(ctx, "Failed to create retrieve engine: %v", err)
} else {
// Group knowledge by embedding model
embeddingGroups := make(map[string][]string)
// Group knowledge by embedding model and type
type groupKey struct {
EmbeddingModelID string
Type string
}
embeddingGroups := make(map[groupKey][]string)
for _, knowledge := range knowledgeList {
embeddingGroups[knowledge.EmbeddingModelID] = append(embeddingGroups[knowledge.EmbeddingModelID], knowledge.ID)
key := groupKey{EmbeddingModelID: knowledge.EmbeddingModelID, Type: knowledge.Type}
embeddingGroups[key] = append(embeddingGroups[key], knowledge.ID)
}
for embeddingModelID, knowledgeGroup := range embeddingGroups {
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, embeddingModelID)
for key, knowledgeGroup := range embeddingGroups {
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, key.EmbeddingModelID)
if err != nil {
logger.Warnf(ctx, "Failed to get embedding model %s: %v", embeddingModelID, err)
logger.Warnf(ctx, "Failed to get embedding model %s: %v", key.EmbeddingModelID, err)
continue
}
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeGroup, embeddingModel.GetDimensions()); err != nil {
logger.Warnf(ctx, "Failed to delete embeddings for model %s: %v", embeddingModelID, err)
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeGroup, embeddingModel.GetDimensions(), key.Type); err != nil {
logger.Warnf(ctx, "Failed to delete embeddings for model %s: %v", key.EmbeddingModelID, err)
}
}
}
@@ -479,14 +484,21 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
}
logger.Infof(ctx, "Query embedding generated successfully, embedding vector length: %d", len(queryEmbedding))
retrieveParams = append(retrieveParams, types.RetrieveParams{
vectorParams := types.RetrieveParams{
Query: params.QueryText,
Embedding: queryEmbedding,
KnowledgeBaseIDs: []string{id},
TopK: matchCount,
Threshold: params.VectorThreshold,
RetrieverType: types.VectorRetrieverType,
})
}
// For FAQ knowledge base, use FAQ index
if kb.Type == types.KnowledgeBaseTypeFAQ {
vectorParams.KnowledgeType = types.KnowledgeTypeFAQ
}
retrieveParams = append(retrieveParams, vectorParams)
logger.Info(ctx, "Vector retrieval parameters setup completed")
}
@@ -234,10 +234,10 @@ func (c *CompositeRetrieveEngine) BatchIndex(ctx context.Context,
// DeleteByChunkIDList deletes vector embeddings by chunk ID list from all registered repositories
func (c *CompositeRetrieveEngine) DeleteByChunkIDList(ctx context.Context,
chunkIDList []string, dimension int,
chunkIDList []string, dimension int, knowledgeType string,
) error {
return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error {
if err := engineInfo.retrieveEngine.DeleteByChunkIDList(ctx, chunkIDList, dimension); err != nil {
if err := engineInfo.retrieveEngine.DeleteByChunkIDList(ctx, chunkIDList, dimension, knowledgeType); err != nil {
logger.GetLogger(ctx).Errorf("Repository %s failed to delete chunk ID list: %v",
engineInfo.retrieveEngine.EngineType(), err)
return err
@@ -248,10 +248,10 @@ func (c *CompositeRetrieveEngine) DeleteByChunkIDList(ctx context.Context,
// DeleteBySourceIDList deletes vector embeddings by source ID list from all registered repositories
func (c *CompositeRetrieveEngine) DeleteBySourceIDList(ctx context.Context,
sourceIDList []string, dimension int,
sourceIDList []string, dimension int, knowledgeType string,
) error {
return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error {
if err := engineInfo.retrieveEngine.DeleteBySourceIDList(ctx, sourceIDList, dimension); err != nil {
if err := engineInfo.retrieveEngine.DeleteBySourceIDList(ctx, sourceIDList, dimension, knowledgeType); err != nil {
logger.GetLogger(ctx).Errorf("Repository %s failed to delete source ID list: %v",
engineInfo.retrieveEngine.EngineType(), err)
return err
@@ -268,6 +268,7 @@ func (c *CompositeRetrieveEngine) CopyIndices(
sourceToTargetKBIDMap map[string]string,
sourceToTargetChunkIDMap map[string]string,
dimension int,
knowledgeType string,
) error {
return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error {
if err := engineInfo.retrieveEngine.CopyIndices(
@@ -277,6 +278,7 @@ func (c *CompositeRetrieveEngine) CopyIndices(
sourceToTargetChunkIDMap,
targetKnowledgeBaseID,
dimension,
knowledgeType,
); err != nil {
logger.Errorf(ctx, "Repository %s failed to copy indices: %v", engineInfo.retrieveEngine.EngineType(), err)
return err
@@ -287,10 +289,10 @@ func (c *CompositeRetrieveEngine) CopyIndices(
// DeleteByKnowledgeIDList deletes vector embeddings by knowledge ID list from all registered repositories
func (c *CompositeRetrieveEngine) DeleteByKnowledgeIDList(ctx context.Context,
knowledgeIDList []string, dimension int,
knowledgeIDList []string, dimension int, knowledgeType string,
) error {
return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error {
if err := engineInfo.retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension); err != nil {
if err := engineInfo.retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension, knowledgeType); err != nil {
logger.GetLogger(ctx).Errorf("Repository %s failed to delete knowledge ID list: %v",
engineInfo.retrieveEngine.EngineType(), err)
return err
@@ -65,7 +65,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
if len(indexInfoList) == 0 {
return nil
}
params := make(map[string]any)
if slices.Contains(retrieverTypes, types.VectorRetrieverType) {
var contentList []string
for _, indexInfo := range indexInfoList {
@@ -87,6 +87,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
}
batchSize := 40
for i, indexChunk := range utils.ChunkSlice(indexInfoList, batchSize) {
params := make(map[string]any)
embeddingMap := make(map[string][]float32)
for j, indexInfo := range indexChunk {
embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j]
@@ -101,6 +102,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
}
var err error
for _, indexChunk := range utils.ChunkSlice(indexInfoList, 10) {
params := make(map[string]any)
err = v.indexRepository.BatchSave(ctx, indexChunk, params)
if err != nil {
return err
@@ -111,23 +113,23 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
// DeleteByChunkIDList deletes vectors by their chunk IDs
func (v *KeywordsVectorHybridRetrieveEngineService) DeleteByChunkIDList(ctx context.Context,
indexIDList []string, dimension int,
indexIDList []string, dimension int, knowledgeType string,
) error {
return v.indexRepository.DeleteByChunkIDList(ctx, indexIDList, dimension)
return v.indexRepository.DeleteByChunkIDList(ctx, indexIDList, dimension, knowledgeType)
}
// DeleteBySourceIDList deletes vectors by their source IDs
func (v *KeywordsVectorHybridRetrieveEngineService) DeleteBySourceIDList(ctx context.Context,
sourceIDList []string, dimension int,
sourceIDList []string, dimension int, knowledgeType string,
) error {
return v.indexRepository.DeleteBySourceIDList(ctx, sourceIDList, dimension)
return v.indexRepository.DeleteBySourceIDList(ctx, sourceIDList, dimension, knowledgeType)
}
// DeleteByKnowledgeIDList deletes vectors by their knowledge IDs
func (v *KeywordsVectorHybridRetrieveEngineService) DeleteByKnowledgeIDList(ctx context.Context,
knowledgeIDList []string, dimension int,
knowledgeIDList []string, dimension int, knowledgeType string,
) error {
return v.indexRepository.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension)
return v.indexRepository.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension, knowledgeType)
}
// Support returns the retriever types supported by this engine
@@ -162,12 +164,13 @@ func (v *KeywordsVectorHybridRetrieveEngineService) CopyIndices(
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error {
logger.Infof(ctx, "Copy indices from knowledge base %s to %s, mapping relation count: %d",
sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap),
)
return v.indexRepository.CopyIndices(
ctx, sourceKnowledgeBaseID, sourceToTargetKBIDMap, sourceToTargetChunkIDMap, targetKnowledgeBaseID, dimension,
ctx, sourceKnowledgeBaseID, sourceToTargetKBIDMap, sourceToTargetChunkIDMap, targetKnowledgeBaseID, dimension, knowledgeType,
)
}
+2
View File
@@ -32,4 +32,6 @@ type IndexInfo struct {
ChunkID string // ID of the text chunk
KnowledgeID string // ID of the knowledge
KnowledgeBaseID string // ID of the knowledge base
KnowledgeType string // Type of the knowledge (e.g., "faq", "manual")
IsEnabled bool // Whether the chunk is enabled for retrieval
}
+8 -6
View File
@@ -31,9 +31,9 @@ type RetrieveEngineRepository interface {
EstimateStorageSize(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) int64
// DeleteByChunkIDList deletes the index info by chunk id list
DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int) error
DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int, knowledgeType string) error
// DeleteBySourceIDList deletes the index info by source id list
DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error
DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error
// 复制索引数据
// sourceKnowledgeBaseID: 源知识库ID
// sourceToTargetChunkIDMap: 源分块ID到目标分块ID的映射关系
@@ -46,10 +46,11 @@ type RetrieveEngineRepository interface {
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error
// DeleteByKnowledgeIDList deletes the index info by knowledge id list
DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error
DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string) error
// BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch
// chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled)
@@ -102,16 +103,17 @@ type RetrieveEngineService interface {
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
dimension int,
knowledgeType string,
) error
// DeleteByChunkIDList deletes the index info by chunk id list
DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int) error
DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int, knowledgeType string) error
// DeleteBySourceIDList deletes the index info by source id list
DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error
DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error
// DeleteByKnowledgeIDList deletes the index info by knowledge id list
DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error
DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string) error
// BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch
// chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled)
+4
View File
@@ -38,6 +38,8 @@ type RetrieveParams struct {
TopK int
// Similarity threshold
Threshold float64
// Knowledge type (e.g., "faq", "manual") - determines which index to use
KnowledgeType string
// Additional parameters, different retrievers may require different parameters
AdditionalParams map[string]interface{}
// Retriever type
@@ -72,6 +74,8 @@ type IndexWithScore struct {
Score float64
// Match type
MatchType MatchType
// IsEnabled
IsEnabled bool
}
// GetScore returns the score for ScoreComparable interface