refactor(qdrant): optimize batch save, map iteration, and error wrapping (#1508)
This commit is contained in:
@@ -3,8 +3,6 @@ package qdrant
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
@@ -196,7 +194,7 @@ func (q *qdrantRepository) Save(ctx context.Context,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to save index: %v", err)
|
||||
return err
|
||||
return fmt.Errorf("failed to save index for chunk ID %s: %w", embedding.ChunkID, err)
|
||||
}
|
||||
|
||||
log.Infof("[Qdrant] Successfully saved index for chunk ID: %s, point ID: %s", embedding.ChunkID, pointID)
|
||||
@@ -242,19 +240,28 @@ func (q *qdrantRepository) BatchSave(ctx context.Context,
|
||||
|
||||
// Save points to each dimension-specific collection
|
||||
totalSaved := 0
|
||||
const batchSize = 100
|
||||
for dimension, points := range pointsByDimension {
|
||||
if err := q.ensureCollection(ctx, dimension); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
collectionName := q.getCollectionName(dimension)
|
||||
_, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: points,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to execute batch operation for dimension %d: %v", dimension, err)
|
||||
return fmt.Errorf("failed to batch save (dimension %d): %w", dimension, err)
|
||||
|
||||
for i := 0; i < len(points); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(points) {
|
||||
end = len(points)
|
||||
}
|
||||
batch := points[i:end]
|
||||
|
||||
_, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{
|
||||
CollectionName: collectionName,
|
||||
Points: batch,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upsert batch: %w", err)
|
||||
}
|
||||
}
|
||||
totalSaved += len(points)
|
||||
log.Infof("[Qdrant] Saved %d points to collection %s", len(points), collectionName)
|
||||
@@ -840,7 +847,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context,
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("[Qdrant] Failed to batch upsert target points: %v", err)
|
||||
return err
|
||||
return fmt.Errorf("failed to batch upsert target points during copy: %w", err)
|
||||
}
|
||||
|
||||
totalCopied += len(targetPoints)
|
||||
@@ -931,9 +938,11 @@ func toQdrantVectorEmbedding(embedding *types.IndexInfo, additionalParams map[st
|
||||
TagID: embedding.TagID,
|
||||
IsEnabled: embedding.IsEnabled,
|
||||
}
|
||||
if additionalParams != nil && slices.Contains(slices.Collect(maps.Keys(additionalParams)), fieldEmbedding) {
|
||||
if embeddingMap, ok := additionalParams[fieldEmbedding].(map[string][]float32); ok {
|
||||
vector.Embedding = embeddingMap[embedding.SourceID]
|
||||
if additionalParams != nil {
|
||||
if val, exists := additionalParams[fieldEmbedding]; exists {
|
||||
if embeddingMap, ok := val.(map[string][]float32); ok {
|
||||
vector.Embedding = embeddingMap[embedding.SourceID]
|
||||
}
|
||||
}
|
||||
}
|
||||
return vector
|
||||
|
||||
Reference in New Issue
Block a user