feat: 支持FAQ索引类型分离,新增is_enabled字段过滤和批量更新优化
This commit is contained in:
@@ -287,18 +287,18 @@ func (e *elasticsearchRepository) countBulkErrors(ctx context.Context,
|
||||
}
|
||||
|
||||
// DeleteByChunkIDList Delete indices by chunk ID list
|
||||
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
|
||||
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
|
||||
return e.deleteByFieldList(ctx, "chunk_id.keyword", chunkIDList)
|
||||
}
|
||||
|
||||
// DeleteBySourceIDList Delete indices by source ID list
|
||||
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error {
|
||||
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
|
||||
return e.deleteByFieldList(ctx, "source_id.keyword", sourceIDList)
|
||||
}
|
||||
|
||||
// DeleteByKnowledgeIDList Delete indices by knowledge ID list
|
||||
func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
knowledgeIDList []string, dimension int,
|
||||
knowledgeIDList []string, dimension int, knowledgeType string,
|
||||
) error {
|
||||
return e.deleteByFieldList(ctx, "knowledge_id.keyword", knowledgeIDList)
|
||||
}
|
||||
@@ -816,6 +816,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
dimension int,
|
||||
knowledgeType string,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Infof(
|
||||
|
||||
@@ -163,7 +163,7 @@ func (e *elasticsearchRepository) BatchSave(ctx context.Context,
|
||||
|
||||
// DeleteByChunkIDList removes documents from the index based on chunk IDs
|
||||
// Returns an error if the delete operation fails
|
||||
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
|
||||
func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(chunkIDList) == 0 {
|
||||
log.Warn("[Elasticsearch] Empty chunk ID list provided for deletion, skipping")
|
||||
@@ -186,7 +186,7 @@ func (e *elasticsearchRepository) DeleteByChunkIDList(ctx context.Context, chunk
|
||||
|
||||
// DeleteBySourceIDList removes documents from the index based on source IDs
|
||||
// Returns an error if the delete operation fails
|
||||
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error {
|
||||
func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(sourceIDList) == 0 {
|
||||
log.Warn("[Elasticsearch] Empty source ID list provided for deletion, skipping")
|
||||
@@ -210,7 +210,7 @@ func (e *elasticsearchRepository) DeleteBySourceIDList(ctx context.Context, sour
|
||||
// DeleteByKnowledgeIDList removes documents from the index based on knowledge IDs
|
||||
// Returns an error if the delete operation fails
|
||||
func (e *elasticsearchRepository) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
knowledgeIDList []string, dimension int,
|
||||
knowledgeIDList []string, dimension int, knowledgeType string,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(knowledgeIDList) == 0 {
|
||||
@@ -447,6 +447,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
dimension int,
|
||||
knowledgeType string,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Infof(
|
||||
|
||||
@@ -107,7 +107,7 @@ func (g *pgRepository) BatchSave(
|
||||
}
|
||||
|
||||
// DeleteByChunkIDList deletes indices by chunk IDs
|
||||
func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
|
||||
func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
|
||||
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by chunk IDs, count: %d", len(chunkIDList))
|
||||
result := g.db.WithContext(ctx).Where("chunk_id IN ?", chunkIDList).Delete(&pgVector{})
|
||||
if result.Error != nil {
|
||||
@@ -119,7 +119,7 @@ func (g *pgRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []st
|
||||
}
|
||||
|
||||
// DeleteBySourceIDList deletes indices by source IDs
|
||||
func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error {
|
||||
func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error {
|
||||
if len(sourceIDList) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -134,7 +134,7 @@ func (g *pgRepository) DeleteBySourceIDList(ctx context.Context, sourceIDList []
|
||||
}
|
||||
|
||||
// DeleteByKnowledgeIDList deletes indices by knowledge IDs
|
||||
func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error {
|
||||
func (g *pgRepository) DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string) error {
|
||||
logger.GetLogger(ctx).Infof("[Postgres] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList))
|
||||
result := g.db.WithContext(ctx).Where("knowledge_id IN ?", knowledgeIDList).Delete(&pgVector{})
|
||||
if result.Error != nil {
|
||||
@@ -357,6 +357,7 @@ func (g *pgRepository) CopyIndices(ctx context.Context,
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
dimension int,
|
||||
knowledgeType string,
|
||||
) error {
|
||||
logger.GetLogger(ctx).Infof(
|
||||
"[Postgres] Copying indices, source knowledge base: %s, target knowledge base: %s, mapping count: %d",
|
||||
|
||||
@@ -264,7 +264,7 @@ func (q *qdrantRepository) BatchSave(ctx context.Context,
|
||||
}
|
||||
|
||||
// DeleteByChunkIDList removes points from the collection based on chunk IDs
|
||||
func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int) error {
|
||||
func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList []string, dimension int, knowledgeType string) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(chunkIDList) == 0 {
|
||||
log.Warn("[Qdrant] Empty chunk ID list provided for deletion, skipping")
|
||||
@@ -293,7 +293,7 @@ func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList
|
||||
|
||||
// DeleteByKnowledgeIDList removes points from the collection based on knowledge IDs
|
||||
func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
knowledgeIDList []string, dimension int,
|
||||
knowledgeIDList []string, dimension int, knowledgeType string,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(knowledgeIDList) == 0 {
|
||||
@@ -323,7 +323,7 @@ func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
|
||||
// DeleteBySourceIDList removes points from the collection based on source IDs
|
||||
func (q *qdrantRepository) DeleteBySourceIDList(ctx context.Context,
|
||||
sourceIDList []string, dimension int,
|
||||
sourceIDList []string, dimension int, knowledgeType string,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
if len(sourceIDList) == 0 {
|
||||
@@ -644,6 +644,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context,
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
dimension int,
|
||||
knowledgeType string,
|
||||
) error {
|
||||
log := logger.GetLogger(ctx)
|
||||
log.Infof(
|
||||
|
||||
@@ -406,7 +406,7 @@ func (s *chunkService) DeleteGeneratedQuestion(ctx context.Context, chunkID stri
|
||||
}
|
||||
|
||||
// Delete the vector index by source ID
|
||||
if err := retrieveEngine.DeleteBySourceIDList(ctx, []string{sourceID}, embeddingModel.GetDimensions()); err != nil {
|
||||
if err := retrieveEngine.DeleteBySourceIDList(ctx, []string{sourceID}, embeddingModel.GetDimensions(), kb.Type); err != nil {
|
||||
logger.Warnf(ctx, "Failed to delete vector index for question (may not exist): %v", err)
|
||||
// Continue even if vector deletion fails - the question might not have been indexed
|
||||
}
|
||||
|
||||
@@ -755,7 +755,7 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
|
||||
return err
|
||||
}
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil {
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
|
||||
return err
|
||||
}
|
||||
@@ -839,17 +839,23 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
|
||||
return err
|
||||
}
|
||||
group := map[string][]string{}
|
||||
for _, knowledge := range knowledgeList {
|
||||
group[knowledge.EmbeddingModelID] = append(group[knowledge.EmbeddingModelID], knowledge.ID)
|
||||
// Group by EmbeddingModelID and Type
|
||||
type groupKey struct {
|
||||
EmbeddingModelID string
|
||||
Type string
|
||||
}
|
||||
for embeddingModelID, knowledgeList := range group {
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, embeddingModelID)
|
||||
group := map[groupKey][]string{}
|
||||
for _, knowledge := range knowledgeList {
|
||||
key := groupKey{EmbeddingModelID: knowledge.EmbeddingModelID, Type: knowledge.Type}
|
||||
group[key] = append(group[key], knowledge.ID)
|
||||
}
|
||||
for key, knowledgeIDs := range group {
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, key.EmbeddingModelID)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge get embedding model failed")
|
||||
return err
|
||||
}
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeList, embeddingModel.GetDimensions()); err != nil {
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeIDs, embeddingModel.GetDimensions(), key.Type); err != nil {
|
||||
logger.GetLogger(ctx).
|
||||
WithField("error", err).
|
||||
Errorf("DeleteKnowledge delete knowledge embedding failed")
|
||||
@@ -1056,7 +1062,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
if err == nil {
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil {
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil {
|
||||
logger.Warnf(ctx, "Failed to delete existing index data (may not exist): %v", err)
|
||||
// 不返回错误,继续处理(可能没有旧数据)
|
||||
} else {
|
||||
@@ -1348,7 +1354,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
|
||||
// delete index
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(
|
||||
ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(),
|
||||
ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), kb.Type,
|
||||
); err != nil {
|
||||
logger.Errorf(ctx, "Delete index failed: %v", err)
|
||||
}
|
||||
@@ -1375,7 +1381,7 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
if err := s.chunkService.DeleteChunksByKnowledgeID(ctx, knowledge.ID); err != nil {
|
||||
logger.Warnf(ctx, "Failed to cleanup chunks after deletion detected: %v", err)
|
||||
}
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil {
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), kb.Type); err != nil {
|
||||
logger.Warnf(ctx, "Failed to cleanup index after deletion detected: %v", err)
|
||||
}
|
||||
span.AddEvent("aborted: knowledge was deleted during processing")
|
||||
@@ -2330,7 +2336,7 @@ func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, c
|
||||
}
|
||||
|
||||
// Delete old vector representation of the chunk
|
||||
err = retrieveEngine.DeleteByChunkIDList(ctx, ids, embeddingModel.GetDimensions())
|
||||
err = retrieveEngine.DeleteByChunkIDList(ctx, ids, embeddingModel.GetDimensions(), sourceKB.Type)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2607,6 +2613,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
|
||||
map[string]string{src.ID: dst.ID},
|
||||
srcTodst,
|
||||
embeddingModel.GetDimensions(),
|
||||
dst.Type,
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -4372,6 +4379,8 @@ func (s *knowledgeService) buildFAQIndexInfoList(
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
||||
KnowledgeType: types.KnowledgeTypeFAQ,
|
||||
IsEnabled: chunk.IsEnabled,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -4397,6 +4406,8 @@ func (s *knowledgeService) buildFAQIndexInfoList(
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
||||
KnowledgeType: types.KnowledgeTypeFAQ,
|
||||
IsEnabled: chunk.IsEnabled,
|
||||
})
|
||||
|
||||
// 每个相似问创建一个索引项
|
||||
@@ -4419,6 +4430,8 @@ func (s *knowledgeService) buildFAQIndexInfoList(
|
||||
ChunkID: chunk.ID,
|
||||
KnowledgeID: chunk.KnowledgeID,
|
||||
KnowledgeBaseID: chunk.KnowledgeBaseID,
|
||||
KnowledgeType: types.KnowledgeTypeFAQ,
|
||||
IsEnabled: chunk.IsEnabled,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4478,7 +4491,7 @@ func (s *knowledgeService) indexFAQChunks(ctx context.Context,
|
||||
var deleteDuration time.Duration
|
||||
if needDelete {
|
||||
deleteStartTime := time.Now()
|
||||
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil {
|
||||
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil {
|
||||
logger.Warnf(ctx, "Delete FAQ vectors failed: %v", err)
|
||||
}
|
||||
deleteDuration = time.Since(deleteStartTime)
|
||||
@@ -4561,7 +4574,7 @@ func (s *knowledgeService) deleteFAQChunkVectors(ctx context.Context,
|
||||
}
|
||||
|
||||
size := retrieveEngine.EstimateStorageSize(ctx, embeddingModel, indexInfo)
|
||||
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil {
|
||||
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil {
|
||||
return err
|
||||
}
|
||||
if size > 0 {
|
||||
@@ -4696,7 +4709,7 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle
|
||||
logger.GetLogger(ctx).WithField("error", modelErr).Error("Failed to get embedding model during cleanup")
|
||||
cleanupErr = errors.Join(cleanupErr, modelErr)
|
||||
} else {
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil {
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Error("Failed to delete manual knowledge index")
|
||||
cleanupErr = errors.Join(cleanupErr, err)
|
||||
}
|
||||
@@ -5119,7 +5132,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
for _, chunk := range chunksDeleted {
|
||||
chunkIDs = append(chunkIDs, chunk.ID)
|
||||
}
|
||||
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions()); err != nil {
|
||||
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunkIDs, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil {
|
||||
logger.Warnf(ctx, "Failed to delete index data for chunks (may not exist): %v", err)
|
||||
} else {
|
||||
logger.Infof(ctx, "Successfully deleted index data for %d chunks", len(chunksDeleted))
|
||||
@@ -5420,7 +5433,7 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
|
||||
// Delete FAQ chunks that don't exist in source
|
||||
if len(chunksToDelete) > 0 {
|
||||
// Delete from vector store
|
||||
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions()); err != nil {
|
||||
if err := retrieveEngine.DeleteByChunkIDList(ctx, chunksToDelete, embeddingModel.GetDimensions(), types.KnowledgeTypeFAQ); err != nil {
|
||||
logger.Errorf(ctx, "Failed to delete FAQ chunks from vector store: %v", err)
|
||||
handleError(progress, err, "Failed to delete FAQ entries from vector store")
|
||||
return err
|
||||
|
||||
@@ -254,20 +254,25 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to create retrieve engine: %v", err)
|
||||
} else {
|
||||
// Group knowledge by embedding model
|
||||
embeddingGroups := make(map[string][]string)
|
||||
// Group knowledge by embedding model and type
|
||||
type groupKey struct {
|
||||
EmbeddingModelID string
|
||||
Type string
|
||||
}
|
||||
embeddingGroups := make(map[groupKey][]string)
|
||||
for _, knowledge := range knowledgeList {
|
||||
embeddingGroups[knowledge.EmbeddingModelID] = append(embeddingGroups[knowledge.EmbeddingModelID], knowledge.ID)
|
||||
key := groupKey{EmbeddingModelID: knowledge.EmbeddingModelID, Type: knowledge.Type}
|
||||
embeddingGroups[key] = append(embeddingGroups[key], knowledge.ID)
|
||||
}
|
||||
|
||||
for embeddingModelID, knowledgeGroup := range embeddingGroups {
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, embeddingModelID)
|
||||
for key, knowledgeGroup := range embeddingGroups {
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, key.EmbeddingModelID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to get embedding model %s: %v", embeddingModelID, err)
|
||||
logger.Warnf(ctx, "Failed to get embedding model %s: %v", key.EmbeddingModelID, err)
|
||||
continue
|
||||
}
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeGroup, embeddingModel.GetDimensions()); err != nil {
|
||||
logger.Warnf(ctx, "Failed to delete embeddings for model %s: %v", embeddingModelID, err)
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeGroup, embeddingModel.GetDimensions(), key.Type); err != nil {
|
||||
logger.Warnf(ctx, "Failed to delete embeddings for model %s: %v", key.EmbeddingModelID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -479,14 +484,21 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
}
|
||||
logger.Infof(ctx, "Query embedding generated successfully, embedding vector length: %d", len(queryEmbedding))
|
||||
|
||||
retrieveParams = append(retrieveParams, types.RetrieveParams{
|
||||
vectorParams := types.RetrieveParams{
|
||||
Query: params.QueryText,
|
||||
Embedding: queryEmbedding,
|
||||
KnowledgeBaseIDs: []string{id},
|
||||
TopK: matchCount,
|
||||
Threshold: params.VectorThreshold,
|
||||
RetrieverType: types.VectorRetrieverType,
|
||||
})
|
||||
}
|
||||
|
||||
// For FAQ knowledge base, use FAQ index
|
||||
if kb.Type == types.KnowledgeBaseTypeFAQ {
|
||||
vectorParams.KnowledgeType = types.KnowledgeTypeFAQ
|
||||
}
|
||||
|
||||
retrieveParams = append(retrieveParams, vectorParams)
|
||||
logger.Info(ctx, "Vector retrieval parameters setup completed")
|
||||
}
|
||||
|
||||
|
||||
@@ -234,10 +234,10 @@ func (c *CompositeRetrieveEngine) BatchIndex(ctx context.Context,
|
||||
|
||||
// DeleteByChunkIDList deletes vector embeddings by chunk ID list from all registered repositories
|
||||
func (c *CompositeRetrieveEngine) DeleteByChunkIDList(ctx context.Context,
|
||||
chunkIDList []string, dimension int,
|
||||
chunkIDList []string, dimension int, knowledgeType string,
|
||||
) error {
|
||||
return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error {
|
||||
if err := engineInfo.retrieveEngine.DeleteByChunkIDList(ctx, chunkIDList, dimension); err != nil {
|
||||
if err := engineInfo.retrieveEngine.DeleteByChunkIDList(ctx, chunkIDList, dimension, knowledgeType); err != nil {
|
||||
logger.GetLogger(ctx).Errorf("Repository %s failed to delete chunk ID list: %v",
|
||||
engineInfo.retrieveEngine.EngineType(), err)
|
||||
return err
|
||||
@@ -248,10 +248,10 @@ func (c *CompositeRetrieveEngine) DeleteByChunkIDList(ctx context.Context,
|
||||
|
||||
// DeleteBySourceIDList deletes vector embeddings by source ID list from all registered repositories
|
||||
func (c *CompositeRetrieveEngine) DeleteBySourceIDList(ctx context.Context,
|
||||
sourceIDList []string, dimension int,
|
||||
sourceIDList []string, dimension int, knowledgeType string,
|
||||
) error {
|
||||
return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error {
|
||||
if err := engineInfo.retrieveEngine.DeleteBySourceIDList(ctx, sourceIDList, dimension); err != nil {
|
||||
if err := engineInfo.retrieveEngine.DeleteBySourceIDList(ctx, sourceIDList, dimension, knowledgeType); err != nil {
|
||||
logger.GetLogger(ctx).Errorf("Repository %s failed to delete source ID list: %v",
|
||||
engineInfo.retrieveEngine.EngineType(), err)
|
||||
return err
|
||||
@@ -268,6 +268,7 @@ func (c *CompositeRetrieveEngine) CopyIndices(
|
||||
sourceToTargetKBIDMap map[string]string,
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
dimension int,
|
||||
knowledgeType string,
|
||||
) error {
|
||||
return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error {
|
||||
if err := engineInfo.retrieveEngine.CopyIndices(
|
||||
@@ -277,6 +278,7 @@ func (c *CompositeRetrieveEngine) CopyIndices(
|
||||
sourceToTargetChunkIDMap,
|
||||
targetKnowledgeBaseID,
|
||||
dimension,
|
||||
knowledgeType,
|
||||
); err != nil {
|
||||
logger.Errorf(ctx, "Repository %s failed to copy indices: %v", engineInfo.retrieveEngine.EngineType(), err)
|
||||
return err
|
||||
@@ -287,10 +289,10 @@ func (c *CompositeRetrieveEngine) CopyIndices(
|
||||
|
||||
// DeleteByKnowledgeIDList deletes vector embeddings by knowledge ID list from all registered repositories
|
||||
func (c *CompositeRetrieveEngine) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
knowledgeIDList []string, dimension int,
|
||||
knowledgeIDList []string, dimension int, knowledgeType string,
|
||||
) error {
|
||||
return c.concurrentExecWithError(ctx, func(ctx context.Context, engineInfo *engineInfo) error {
|
||||
if err := engineInfo.retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension); err != nil {
|
||||
if err := engineInfo.retrieveEngine.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension, knowledgeType); err != nil {
|
||||
logger.GetLogger(ctx).Errorf("Repository %s failed to delete knowledge ID list: %v",
|
||||
engineInfo.retrieveEngine.EngineType(), err)
|
||||
return err
|
||||
|
||||
@@ -65,7 +65,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
|
||||
if len(indexInfoList) == 0 {
|
||||
return nil
|
||||
}
|
||||
params := make(map[string]any)
|
||||
|
||||
if slices.Contains(retrieverTypes, types.VectorRetrieverType) {
|
||||
var contentList []string
|
||||
for _, indexInfo := range indexInfoList {
|
||||
@@ -87,6 +87,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
|
||||
}
|
||||
batchSize := 40
|
||||
for i, indexChunk := range utils.ChunkSlice(indexInfoList, batchSize) {
|
||||
params := make(map[string]any)
|
||||
embeddingMap := make(map[string][]float32)
|
||||
for j, indexInfo := range indexChunk {
|
||||
embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j]
|
||||
@@ -101,6 +102,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
|
||||
}
|
||||
var err error
|
||||
for _, indexChunk := range utils.ChunkSlice(indexInfoList, 10) {
|
||||
params := make(map[string]any)
|
||||
err = v.indexRepository.BatchSave(ctx, indexChunk, params)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -111,23 +113,23 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
|
||||
|
||||
// DeleteByChunkIDList deletes vectors by their chunk IDs
|
||||
func (v *KeywordsVectorHybridRetrieveEngineService) DeleteByChunkIDList(ctx context.Context,
|
||||
indexIDList []string, dimension int,
|
||||
indexIDList []string, dimension int, knowledgeType string,
|
||||
) error {
|
||||
return v.indexRepository.DeleteByChunkIDList(ctx, indexIDList, dimension)
|
||||
return v.indexRepository.DeleteByChunkIDList(ctx, indexIDList, dimension, knowledgeType)
|
||||
}
|
||||
|
||||
// DeleteBySourceIDList deletes vectors by their source IDs
|
||||
func (v *KeywordsVectorHybridRetrieveEngineService) DeleteBySourceIDList(ctx context.Context,
|
||||
sourceIDList []string, dimension int,
|
||||
sourceIDList []string, dimension int, knowledgeType string,
|
||||
) error {
|
||||
return v.indexRepository.DeleteBySourceIDList(ctx, sourceIDList, dimension)
|
||||
return v.indexRepository.DeleteBySourceIDList(ctx, sourceIDList, dimension, knowledgeType)
|
||||
}
|
||||
|
||||
// DeleteByKnowledgeIDList deletes vectors by their knowledge IDs
|
||||
func (v *KeywordsVectorHybridRetrieveEngineService) DeleteByKnowledgeIDList(ctx context.Context,
|
||||
knowledgeIDList []string, dimension int,
|
||||
knowledgeIDList []string, dimension int, knowledgeType string,
|
||||
) error {
|
||||
return v.indexRepository.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension)
|
||||
return v.indexRepository.DeleteByKnowledgeIDList(ctx, knowledgeIDList, dimension, knowledgeType)
|
||||
}
|
||||
|
||||
// Support returns the retriever types supported by this engine
|
||||
@@ -162,12 +164,13 @@ func (v *KeywordsVectorHybridRetrieveEngineService) CopyIndices(
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
dimension int,
|
||||
knowledgeType string,
|
||||
) error {
|
||||
logger.Infof(ctx, "Copy indices from knowledge base %s to %s, mapping relation count: %d",
|
||||
sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap),
|
||||
)
|
||||
return v.indexRepository.CopyIndices(
|
||||
ctx, sourceKnowledgeBaseID, sourceToTargetKBIDMap, sourceToTargetChunkIDMap, targetKnowledgeBaseID, dimension,
|
||||
ctx, sourceKnowledgeBaseID, sourceToTargetKBIDMap, sourceToTargetChunkIDMap, targetKnowledgeBaseID, dimension, knowledgeType,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -32,4 +32,6 @@ type IndexInfo struct {
|
||||
ChunkID string // ID of the text chunk
|
||||
KnowledgeID string // ID of the knowledge
|
||||
KnowledgeBaseID string // ID of the knowledge base
|
||||
KnowledgeType string // Type of the knowledge (e.g., "faq", "manual")
|
||||
IsEnabled bool // Whether the chunk is enabled for retrieval
|
||||
}
|
||||
|
||||
@@ -31,9 +31,9 @@ type RetrieveEngineRepository interface {
|
||||
EstimateStorageSize(ctx context.Context, indexInfoList []*types.IndexInfo, params map[string]any) int64
|
||||
|
||||
// DeleteByChunkIDList deletes the index info by chunk id list
|
||||
DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int) error
|
||||
DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int, knowledgeType string) error
|
||||
// DeleteBySourceIDList deletes the index info by source id list
|
||||
DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error
|
||||
DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error
|
||||
// 复制索引数据
|
||||
// sourceKnowledgeBaseID: 源知识库ID
|
||||
// sourceToTargetChunkIDMap: 源分块ID到目标分块ID的映射关系
|
||||
@@ -46,10 +46,11 @@ type RetrieveEngineRepository interface {
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
dimension int,
|
||||
knowledgeType string,
|
||||
) error
|
||||
|
||||
// DeleteByKnowledgeIDList deletes the index info by knowledge id list
|
||||
DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error
|
||||
DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string) error
|
||||
|
||||
// BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch
|
||||
// chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled)
|
||||
@@ -102,16 +103,17 @@ type RetrieveEngineService interface {
|
||||
sourceToTargetChunkIDMap map[string]string,
|
||||
targetKnowledgeBaseID string,
|
||||
dimension int,
|
||||
knowledgeType string,
|
||||
) error
|
||||
|
||||
// DeleteByChunkIDList deletes the index info by chunk id list
|
||||
DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int) error
|
||||
DeleteByChunkIDList(ctx context.Context, indexIDList []string, dimension int, knowledgeType string) error
|
||||
|
||||
// DeleteBySourceIDList deletes the index info by source id list
|
||||
DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int) error
|
||||
DeleteBySourceIDList(ctx context.Context, sourceIDList []string, dimension int, knowledgeType string) error
|
||||
|
||||
// DeleteByKnowledgeIDList deletes the index info by knowledge id list
|
||||
DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int) error
|
||||
DeleteByKnowledgeIDList(ctx context.Context, knowledgeIDList []string, dimension int, knowledgeType string) error
|
||||
|
||||
// BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch
|
||||
// chunkStatusMap: map of chunk ID to enabled status (true = enabled, false = disabled)
|
||||
|
||||
@@ -38,6 +38,8 @@ type RetrieveParams struct {
|
||||
TopK int
|
||||
// Similarity threshold
|
||||
Threshold float64
|
||||
// Knowledge type (e.g., "faq", "manual") - determines which index to use
|
||||
KnowledgeType string
|
||||
// Additional parameters, different retrievers may require different parameters
|
||||
AdditionalParams map[string]interface{}
|
||||
// Retriever type
|
||||
@@ -72,6 +74,8 @@ type IndexWithScore struct {
|
||||
Score float64
|
||||
// Match type
|
||||
MatchType MatchType
|
||||
// IsEnabled
|
||||
IsEnabled bool
|
||||
}
|
||||
|
||||
// GetScore returns the score for ScoreComparable interface
|
||||
|
||||
Reference in New Issue
Block a user