From 6bc8e9ed49febb32ff77e7389c58609bbd29c184 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cjiangyijun=E2=80=9D?= Date: Thu, 11 Jun 2026 17:07:43 +0800 Subject: [PATCH] =?UTF-8?q?fix(vectordb):=20=E4=BF=AE=E5=A4=8D=20question?= =?UTF-8?q?=20=E6=95=B0=E6=8D=AE=E5=A4=8D=E5=88=B6=E5=88=B0=E5=90=91?= =?UTF-8?q?=E9=87=8F=E6=95=B0=E6=8D=AE=E5=BA=93=E7=9A=84=20ID=20=E6=98=A0?= =?UTF-8?q?=E5=B0=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 主要改进: 1. CopyIndices 新增分页查询逻辑(每页500条),避免大批量数据查询失败 2. 修复 ID 字段优先级:优先使用 IndexInfo.ID,兜底使用 ChunkID 3. 新增 translateSourceID 函数,智能处理 question 类型数据的 SourceID 映射: - 当 SourceID == ChunkID 时,直接映射为目标 ChunkID - 当 SourceID 为 "ChunkID-QuestionID" 格式时,保留 QuestionID 部分 - 其他情况使用 SHA256 生成稳定的映射 ID 影响范围: - 修改文件:internal/application/repository/retriever/tencentvectordb/repository.go - 新增函数:copySourceQueryParams、remapCopiedEmbeddings、translateSourceID --- .../retriever/tencentvectordb/repository.go | 108 +++++++++++++----- 1 file changed, 82 insertions(+), 26 deletions(-) diff --git a/internal/application/repository/retriever/tencentvectordb/repository.go b/internal/application/repository/retriever/tencentvectordb/repository.go index fdbf2f2e..3bca58cf 100644 --- a/internal/application/repository/retriever/tencentvectordb/repository.go +++ b/internal/application/repository/retriever/tencentvectordb/repository.go @@ -2,6 +2,8 @@ package tencentvectordb import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" "maps" "os" @@ -17,6 +19,8 @@ import ( "github.com/tencent/vectordatabase-sdk-go/tcvectordb" ) +const copyIndicesQueryPageSize int64 = 500 + // NewTencentVectorDBRetrieveEngineRepository creates a Tencent VectorDB-backed retrieve repository. func NewTencentVectorDBRetrieveEngineRepository( client *tcvectordb.RpcClient, @@ -139,34 +143,26 @@ func (r *repository) CopyIndices( } collectionName := r.collectionName(dimension) ids := slices.Collect(maps.Keys(sourceToTargetChunkIDMap)) - query, err := r.client.Database(r.databaseName).Collection(collectionName).Query( - ctx, - nil, - &tcvectordb.QueryDocumentParams{ - Filter: tcvectordb.NewFilter(tcvectordb.In(fieldChunkID, ids)), - RetrieveVector: true, - OutputFields: outputFields(), - Limit: int64(len(ids)), - }, - ) - if err != nil { - return fmt.Errorf("tencent vectordb query source indices: %w", err) - } - embeddings := make([]*vectorEmbedding, 0, len(query.Documents)) - for _, doc := range query.Documents { - embedding := fromDocument(doc) - targetChunkID := sourceToTargetChunkIDMap[embedding.ChunkID] - if targetChunkID == "" { - continue + var embeddings []*vectorEmbedding + for offset := int64(0); ; offset += copyIndicesQueryPageSize { + query, err := r.client.Database(r.databaseName).Collection(collectionName).Query( + ctx, + nil, + copySourceQueryParams(sourceKnowledgeBaseID, ids, offset), + ) + if err != nil { + return fmt.Errorf("tencent vectordb query source indices: %w", err) } - embedding.ID = targetChunkID - embedding.ChunkID = targetChunkID - embedding.KnowledgeBaseID = targetKnowledgeBaseID - if targetKBID := sourceToTargetKBIDMap[embedding.KnowledgeID]; targetKBID != "" { - embedding.KnowledgeID = targetKBID + embeddings = append(embeddings, remapCopiedEmbeddings( + query.Documents, + sourceToTargetKBIDMap, + sourceToTargetChunkIDMap, + targetKnowledgeBaseID, + )...) + if len(query.Documents) < int(copyIndicesQueryPageSize) { + break } - embeddings = append(embeddings, embedding) } if len(embeddings) == 0 { return nil @@ -563,7 +559,7 @@ func (r *repository) toDocumentsWithSparseVectors( func toVectorEmbedding(indexInfo *types.IndexInfo, params map[string]any) *vectorEmbedding { embedding := &vectorEmbedding{ - ID: indexInfo.ChunkID, + ID: indexInfo.ID, Content: cleanInvalidUTF8(indexInfo.Content), SourceID: indexInfo.SourceID, SourceType: int(indexInfo.SourceType), @@ -576,6 +572,9 @@ func toVectorEmbedding(indexInfo *types.IndexInfo, params map[string]any) *vecto if embedding.ID == "" { embedding.ID = indexInfo.SourceID } + if embedding.ID == "" { + embedding.ID = indexInfo.ChunkID + } if params != nil && slices.Contains(slices.Collect(maps.Keys(params)), fieldVector) { if embeddingMap, ok := params[fieldVector].(map[string][]float32); ok { embedding.Embedding = lookupEmbedding(embeddingMap, indexInfo) @@ -596,6 +595,63 @@ func lookupEmbedding(embeddingMap map[string][]float32, indexInfo *types.IndexIn return embeddingMap[indexInfo.ChunkID] } +func copySourceQueryParams(sourceKnowledgeBaseID string, chunkIDs []string, offset int64) *tcvectordb.QueryDocumentParams { + conditions := []string{tcvectordb.In(fieldChunkID, chunkIDs)} + if sourceKnowledgeBaseID != "" { + conditions = append([]string{tcvectordb.In(fieldKnowledgeBaseID, []string{sourceKnowledgeBaseID})}, conditions...) + } + return &tcvectordb.QueryDocumentParams{ + Filter: tcvectordb.NewFilter(strings.Join(conditions, " and ")), + RetrieveVector: true, + OutputFields: outputFields(), + Offset: offset, + Limit: copyIndicesQueryPageSize, + } +} + +func remapCopiedEmbeddings( + docs []tcvectordb.Document, + sourceToTargetKBIDMap map[string]string, + sourceToTargetChunkIDMap map[string]string, + targetKnowledgeBaseID string, +) []*vectorEmbedding { + embeddings := make([]*vectorEmbedding, 0, len(docs)) + for _, doc := range docs { + embedding := fromDocument(doc) + targetChunkID := sourceToTargetChunkIDMap[embedding.ChunkID] + if targetChunkID == "" { + continue + } + originalSourceID := embedding.SourceID + if originalSourceID == "" { + originalSourceID = embedding.ID + } + targetSourceID := translateSourceID(originalSourceID, embedding.ChunkID, targetChunkID) + embedding.ID = targetSourceID + embedding.SourceID = targetSourceID + embedding.ChunkID = targetChunkID + embedding.KnowledgeBaseID = targetKnowledgeBaseID + if targetKBID := sourceToTargetKBIDMap[embedding.KnowledgeID]; targetKBID != "" { + embedding.KnowledgeID = targetKBID + } + embeddings = append(embeddings, embedding) + } + return embeddings +} + +func translateSourceID(originalSourceID, sourceChunkID, targetChunkID string) string { + switch { + case originalSourceID == sourceChunkID: + return targetChunkID + case strings.HasPrefix(originalSourceID, sourceChunkID+"-"): + questionID := strings.TrimPrefix(originalSourceID, sourceChunkID+"-") + return fmt.Sprintf("%s-%s", targetChunkID, questionID) + default: + sum := sha256.Sum256([]byte(targetChunkID + "\x00" + sourceChunkID + "\x00" + originalSourceID)) + return fmt.Sprintf("%s-%s", targetChunkID, hex.EncodeToString(sum[:])[:16]) + } +} + func cleanInvalidUTF8(s string) string { var b strings.Builder b.Grow(len(s))