From 17604c9b94fe180a741f3f95b97e62f3344a72fd Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 8 Jan 2026 15:40:44 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20=E4=B8=BA=E6=89=B9=E9=87=8F=E7=B4=A2?= =?UTF-8?q?=E5=BC=95=E6=B7=BB=E5=8A=A0=E5=B9=B6=E5=8F=91=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../keywords_vector_hybrid_indexer.go | 124 +++++++++++++++--- 1 file changed, 109 insertions(+), 15 deletions(-) diff --git a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go index e803558f..45bba2b5 100644 --- a/internal/application/service/retriever/keywords_vector_hybrid_indexer.go +++ b/internal/application/service/retriever/keywords_vector_hybrid_indexer.go @@ -10,6 +10,7 @@ import ( "github.com/Tencent/WeKnora/internal/models/utils" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "golang.org/x/sync/errgroup" ) // KeywordsVectorHybridRetrieveEngineService implements a hybrid retrieval engine @@ -58,7 +59,7 @@ func (v *KeywordsVectorHybridRetrieveEngineService) Index(ctx context.Context, } // BatchIndex creates embeddings for multiple content items and saves them to the repository -// in batches for efficiency +// in batches for efficiency. Uses concurrent batch saving to improve performance. func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Context, embedder embedding.Embedder, indexInfoList []*types.IndexInfo, retrieverTypes []types.RetrieverType, ) error { @@ -85,30 +86,123 @@ func (v *KeywordsVectorHybridRetrieveEngineService) BatchIndex(ctx context.Conte if err != nil { return err } + batchSize := 40 - for i, indexChunk := range utils.ChunkSlice(indexInfoList, batchSize) { + chunks := utils.ChunkSlice(indexInfoList, batchSize) + + // Use concurrent batch saving for better performance + // Limit concurrency to avoid overwhelming the backend + const maxConcurrency = 5 + if len(chunks) <= maxConcurrency { + // For small number of batches, use simple concurrency + return v.concurrentBatchSave(ctx, chunks, embeddings, batchSize) + } + + // For large number of batches, use bounded concurrency + return v.boundedConcurrentBatchSave(ctx, chunks, embeddings, batchSize, maxConcurrency) + } + + // For non-vector retrieval, use concurrent batch saving as well + chunks := utils.ChunkSlice(indexInfoList, 10) + const maxConcurrency = 5 + if len(chunks) <= maxConcurrency { + return v.concurrentBatchSaveNoEmbedding(ctx, chunks) + } + return v.boundedConcurrentBatchSaveNoEmbedding(ctx, chunks, maxConcurrency) +} + +// concurrentBatchSave saves all batches concurrently without concurrency limit +func (v *KeywordsVectorHybridRetrieveEngineService) concurrentBatchSave( + ctx context.Context, + chunks [][]*types.IndexInfo, + embeddings [][]float32, + batchSize int, +) error { + g, ctx := errgroup.WithContext(ctx) + for i, indexChunk := range chunks { + g.Go(func() error { params := make(map[string]any) embeddingMap := make(map[string][]float32) for j, indexInfo := range indexChunk { embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j] } params["embedding"] = embeddingMap - err = v.indexRepository.BatchSave(ctx, indexChunk, params) - if err != nil { - return err + return v.indexRepository.BatchSave(ctx, indexChunk, params) + }) + } + return g.Wait() +} + +// boundedConcurrentBatchSave saves batches with bounded concurrency using semaphore pattern +func (v *KeywordsVectorHybridRetrieveEngineService) boundedConcurrentBatchSave( + ctx context.Context, + chunks [][]*types.IndexInfo, + embeddings [][]float32, + batchSize int, + maxConcurrency int, +) error { + g, ctx := errgroup.WithContext(ctx) + sem := make(chan struct{}, maxConcurrency) + + for i, indexChunk := range chunks { + g.Go(func() error { + select { + case sem <- struct{}{}: + defer func() { <-sem }() + case <-ctx.Done(): + return ctx.Err() } - } - return nil + + params := make(map[string]any) + embeddingMap := make(map[string][]float32) + for j, indexInfo := range indexChunk { + embeddingMap[indexInfo.SourceID] = embeddings[i*batchSize+j] + } + params["embedding"] = embeddingMap + return v.indexRepository.BatchSave(ctx, indexChunk, params) + }) } - var err error - for _, indexChunk := range utils.ChunkSlice(indexInfoList, 10) { - params := make(map[string]any) - err = v.indexRepository.BatchSave(ctx, indexChunk, params) - if err != nil { - return err - } + return g.Wait() +} + +// concurrentBatchSaveNoEmbedding saves all batches concurrently without embeddings +func (v *KeywordsVectorHybridRetrieveEngineService) concurrentBatchSaveNoEmbedding( + ctx context.Context, + chunks [][]*types.IndexInfo, +) error { + g, ctx := errgroup.WithContext(ctx) + for _, indexChunk := range chunks { + g.Go(func() error { + params := make(map[string]any) + return v.indexRepository.BatchSave(ctx, indexChunk, params) + }) } - return nil + return g.Wait() +} + +// boundedConcurrentBatchSaveNoEmbedding saves batches with bounded concurrency without embeddings +func (v *KeywordsVectorHybridRetrieveEngineService) boundedConcurrentBatchSaveNoEmbedding( + ctx context.Context, + chunks [][]*types.IndexInfo, + maxConcurrency int, +) error { + g, ctx := errgroup.WithContext(ctx) + sem := make(chan struct{}, maxConcurrency) + + for _, indexChunk := range chunks { + g.Go(func() error { + select { + case sem <- struct{}{}: + defer func() { <-sem }() + case <-ctx.Done(): + return ctx.Err() + } + + params := make(map[string]any) + return v.indexRepository.BatchSave(ctx, indexChunk, params) + }) + } + return g.Wait() } // DeleteByChunkIDList deletes vectors by their chunk IDs