perf: 为批量索引添加并发保存优化

This commit is contained in:
wizardchen
2026-01-08 15:40:44 +08:00
parent 5a93be8dbd
commit 17604c9b94
@@ -10,6 +10,7 @@ import (
"github.com/Tencent/WeKnora/internal/models/utils"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"golang.org/x/sync/errgroup"
)
// KeywordsVectorHybridRetrieveEngineService implements a hybrid retrieval engine
@@ -58,7 +59,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) Index(ctx context.Context,
}
// BatchIndex creates embeddings for multiple content items and saves them to the repository
// in batches for efficiency
// in batches for efficiency. Uses concurrent batch saving to improve performance.
func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Context,
embedder embedding.Embedder, indexInfoList []*types.IndexInfo, retrieverTypes []types.RetrieverType,
) error {
@@ -85,30 +86,123 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte
if err != nil {
return err
}
batchSize := 40
for i, indexChunk := range utils.ChunkSlice(indexInfoList, batchSize) {
chunks := utils.ChunkSlice(indexInfoList, batchSize)
// Use concurrent batch saving for better performance
// Limit concurrency to avoid overwhelming the backend
const maxConcurrency = 5
if len(chunks) <= maxConcurrency {
// For small number of batches, use simple concurrency
return v.concurrentBatchSave(ctx, chunks, embeddings, batchSize)
}
// For large number of batches, use bounded concurrency
return v.boundedConcurrentBatchSave(ctx, chunks, embeddings, batchSize, maxConcurrency)
}
// For non-vector retrieval, use concurrent batch saving as well
chunks := utils.ChunkSlice(indexInfoList, 10)
const maxConcurrency = 5
if len(chunks) <= maxConcurrency {
return v.concurrentBatchSaveNoEmbedding(ctx, chunks)
}
return v.boundedConcurrentBatchSaveNoEmbedding(ctx, chunks, maxConcurrency)
}
// concurrentBatchSave saves all batches concurrently without concurrency limit
func (v *KeywordsVectorHybridRetrieveEngineService) concurrentBatchSave(
ctx context.Context,
chunks [][]*types.IndexInfo,
embeddings [][]float32,
batchSize int,
) error {
g, ctx := errgroup.WithContext(ctx)
for i, indexChunk := range chunks {
g.Go(func() error {
params := make(map[string]any)
embeddingMap := make(map[string][]float32)
for j, indexInfo := range indexChunk {
embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j]
}
params["embedding"] = embeddingMap
err = v.indexRepository.BatchSave(ctx, indexChunk, params)
if err != nil {
return err
return v.indexRepository.BatchSave(ctx, indexChunk, params)
})
}
return g.Wait()
}
// boundedConcurrentBatchSave saves batches with bounded concurrency using semaphore pattern
func (v *KeywordsVectorHybridRetrieveEngineService) boundedConcurrentBatchSave(
ctx context.Context,
chunks [][]*types.IndexInfo,
embeddings [][]float32,
batchSize int,
maxConcurrency int,
) error {
g, ctx := errgroup.WithContext(ctx)
sem := make(chan struct{}, maxConcurrency)
for i, indexChunk := range chunks {
g.Go(func() error {
select {
case sem <- struct{}{}:
defer func() { <-sem }()
case <-ctx.Done():
return ctx.Err()
}
}
return nil
params := make(map[string]any)
embeddingMap := make(map[string][]float32)
for j, indexInfo := range indexChunk {
embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j]
}
params["embedding"] = embeddingMap
return v.indexRepository.BatchSave(ctx, indexChunk, params)
})
}
var err error
for _, indexChunk := range utils.ChunkSlice(indexInfoList, 10) {
params := make(map[string]any)
err = v.indexRepository.BatchSave(ctx, indexChunk, params)
if err != nil {
return err
}
return g.Wait()
}
// concurrentBatchSaveNoEmbedding saves all batches concurrently without embeddings
func (v *KeywordsVectorHybridRetrieveEngineService) concurrentBatchSaveNoEmbedding(
ctx context.Context,
chunks [][]*types.IndexInfo,
) error {
g, ctx := errgroup.WithContext(ctx)
for _, indexChunk := range chunks {
g.Go(func() error {
params := make(map[string]any)
return v.indexRepository.BatchSave(ctx, indexChunk, params)
})
}
return nil
return g.Wait()
}
// boundedConcurrentBatchSaveNoEmbedding saves batches with bounded concurrency without embeddings
func (v *KeywordsVectorHybridRetrieveEngineService) boundedConcurrentBatchSaveNoEmbedding(
ctx context.Context,
chunks [][]*types.IndexInfo,
maxConcurrency int,
) error {
g, ctx := errgroup.WithContext(ctx)
sem := make(chan struct{}, maxConcurrency)
for _, indexChunk := range chunks {
g.Go(func() error {
select {
case sem <- struct{}{}:
defer func() { <-sem }()
case <-ctx.Done():
return ctx.Err()
}
params := make(map[string]any)
return v.indexRepository.BatchSave(ctx, indexChunk, params)
})
}
return g.Wait()
}
// DeleteByChunkIDList deletes vectors by their chunk IDs