feat: Add chunk enabled status support in Qdrant repository with batch update and deletion by source ID functionality

This commit is contained in:
wizardchen
2025-12-05 14:31:16 +08:00
parent b1c79ccd4d
commit d23b917667
2 changed files with 98 additions and 0 deletions
@@ -25,6 +25,7 @@ const (
fieldKnowledgeID = "knowledge_id"
fieldKnowledgeBaseID = "knowledge_base_id"
fieldEmbedding = "embedding"
fieldIsEnabled = "is_enabled"
)
// NewQdrantRetrieveEngineRepository creates and initializes a new Qdrant repository
@@ -208,10 +209,104 @@ func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context,
return nil
}
// DeleteBySourceIDList removes points from the collection based on source IDs
func (q *qdrantRepository) DeleteBySourceIDList(ctx context.Context,
sourceIDList []string, dimension int,
) error {
log := logger.GetLogger(ctx)
if len(sourceIDList) == 0 {
log.Warn("[Qdrant] Empty source ID list provided for deletion, skipping")
return nil
}
log.Infof("[Qdrant] Deleting indices by source IDs, count: %d", len(sourceIDList))
_, err := q.client.Delete(ctx, &qdrant.DeletePoints{
CollectionName: q.collectionName,
Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{
Must: []*qdrant.Condition{
qdrant.NewMatchKeywords(fieldSourceID, sourceIDList...),
},
}),
})
if err != nil {
log.Errorf("[Qdrant] Failed to delete by source IDs: %v", err)
return fmt.Errorf("failed to delete by source IDs: %w", err)
}
log.Infof("[Qdrant] Successfully deleted documents by source IDs")
return nil
}
// BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch
func (q *qdrantRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error {
log := logger.GetLogger(ctx)
if len(chunkStatusMap) == 0 {
log.Warn("[Qdrant] Empty chunk status map provided, skipping")
return nil
}
log.Infof("[Qdrant] Batch updating chunk enabled status, count: %d", len(chunkStatusMap))
// Group chunks by enabled status for batch updates
enabledChunkIDs := make([]string, 0)
disabledChunkIDs := make([]string, 0)
for chunkID, enabled := range chunkStatusMap {
if enabled {
enabledChunkIDs = append(enabledChunkIDs, chunkID)
} else {
disabledChunkIDs = append(disabledChunkIDs, chunkID)
}
}
// Update enabled chunks
if len(enabledChunkIDs) > 0 {
_, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{
CollectionName: q.collectionName,
Payload: qdrant.NewValueMap(map[string]any{fieldIsEnabled: true}),
PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{
Must: []*qdrant.Condition{
qdrant.NewMatchKeywords(fieldChunkID, enabledChunkIDs...),
},
}),
})
if err != nil {
log.Errorf("[Qdrant] Failed to update enabled chunks: %v", err)
return fmt.Errorf("failed to update enabled chunks: %w", err)
}
log.Infof("[Qdrant] Successfully enabled %d chunks", len(enabledChunkIDs))
}
// Update disabled chunks
if len(disabledChunkIDs) > 0 {
_, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{
CollectionName: q.collectionName,
Payload: qdrant.NewValueMap(map[string]any{fieldIsEnabled: false}),
PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{
Must: []*qdrant.Condition{
qdrant.NewMatchKeywords(fieldChunkID, disabledChunkIDs...),
},
}),
})
if err != nil {
log.Errorf("[Qdrant] Failed to update disabled chunks: %v", err)
return fmt.Errorf("failed to update disabled chunks: %w", err)
}
log.Infof("[Qdrant] Successfully disabled %d chunks", len(disabledChunkIDs))
}
log.Infof("[Qdrant] Batch update chunk enabled status completed")
return nil
}
func (q *qdrantRepository) getBaseFilter(params typesLocal.RetrieveParams) *qdrant.Filter {
must := make([]*qdrant.Condition, 0)
mustNot := make([]*qdrant.Condition, 0)
// Only retrieve enabled chunks
must = append(must, qdrant.NewMatchBool(fieldIsEnabled, true))
if len(params.KnowledgeBaseIDs) > 0 {
must = append(must, qdrant.NewMatchKeywords(fieldKnowledgeBaseID, params.KnowledgeBaseIDs...))
}
@@ -488,6 +583,7 @@ func createPayload(embedding *QdrantVectorEmbedding) map[string]*qdrant.Value {
fieldChunkID: embedding.ChunkID,
fieldKnowledgeID: embedding.KnowledgeID,
fieldKnowledgeBaseID: embedding.KnowledgeBaseID,
fieldIsEnabled: embedding.IsEnabled,
}
return qdrant.NewValueMap(payload)
}
@@ -544,6 +640,7 @@ func toQdrantVectorEmbedding(embedding *types.IndexInfo, additionalParams map[st
ChunkID: embedding.ChunkID,
KnowledgeID: embedding.KnowledgeID,
KnowledgeBaseID: embedding.KnowledgeBaseID,
IsEnabled: true, // Default to enabled
}
if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), fieldEmbedding) {
if embeddingMap, ok := additionalParams[fieldEmbedding].(map[string][]float32); ok {
@@ -15,6 +15,7 @@ type QdrantVectorEmbedding struct {
KnowledgeID string `json:"knowledge_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
Embedding []float32 `json:"embedding"`
IsEnabled bool `json:"is_enabled"`
}
type QdrantVectorEmbeddingWithScore struct {