From 3645477dba8ea115c7f5dd3b3aa4c593bf76ed48 Mon Sep 17 00:00:00 2001 From: Sheep44044 <3290120053@qq.com> Date: Thu, 28 May 2026 07:39:41 +0800 Subject: [PATCH] refactor(qdrant): optimize batch save, map iteration, and error wrapping (#1508) --- .../repository/retriever/qdrant/repository.go | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/internal/application/repository/retriever/qdrant/repository.go b/internal/application/repository/retriever/qdrant/repository.go index ee5f474b..6a4ba84d 100644 --- a/internal/application/repository/retriever/qdrant/repository.go +++ b/internal/application/repository/retriever/qdrant/repository.go @@ -3,8 +3,6 @@ package qdrant import ( "context" "fmt" - "maps" - "slices" "strings" "unicode/utf8" @@ -196,7 +194,7 @@ func (q *qdrantRepository) Save(ctx context.Context, }) if err != nil { log.Errorf("[Qdrant] Failed to save index: %v", err) - return err + return fmt.Errorf("failed to save index for chunk ID %s: %w", embedding.ChunkID, err) } log.Infof("[Qdrant] Successfully saved index for chunk ID: %s, point ID: %s", embedding.ChunkID, pointID) @@ -242,19 +240,28 @@ func (q *qdrantRepository) BatchSave(ctx context.Context, // Save points to each dimension-specific collection totalSaved := 0 + const batchSize = 100 for dimension, points := range pointsByDimension { if err := q.ensureCollection(ctx, dimension); err != nil { return err } collectionName := q.getCollectionName(dimension) - _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ - CollectionName: collectionName, - Points: points, - }) - if err != nil { - log.Errorf("[Qdrant] Failed to execute batch operation for dimension %d: %v", dimension, err) - return fmt.Errorf("failed to batch save (dimension %d): %w", dimension, err) + + for i := 0; i < len(points); i += batchSize { + end := i + batchSize + if end > len(points) { + end = len(points) + } + batch := points[i:end] + + _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ + CollectionName: collectionName, + Points: batch, + }) + if err != nil { + return fmt.Errorf("failed to upsert batch: %w", err) + } } totalSaved += len(points) log.Infof("[Qdrant] Saved %d points to collection %s", len(points), collectionName) @@ -840,7 +847,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, }) if err != nil { log.Errorf("[Qdrant] Failed to batch upsert target points: %v", err) - return err + return fmt.Errorf("failed to batch upsert target points during copy: %w", err) } totalCopied += len(targetPoints) @@ -931,9 +938,11 @@ func toQdrantVectorEmbedding(embedding *types.IndexInfo, additionalParams map[st TagID: embedding.TagID, IsEnabled: embedding.IsEnabled, } - if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), fieldEmbedding) { - if embeddingMap, ok := additionalParams[fieldEmbedding].(map[string][]float32); ok { - vector.Embedding = embeddingMap[embedding.SourceID] + if additionalParams != nil { + if val, exists := additionalParams[fieldEmbedding]; exists { + if embeddingMap, ok := val.(map[string][]float32); ok { + vector.Embedding = embeddingMap[embedding.SourceID] + } } } return vector