fix(vectordb): 修复 question 数据复制到向量数据库的 ID 映射问题

主要改进:
1. CopyIndices 新增分页查询逻辑(每页500条),避免大批量数据查询失败
2. 修复 ID 字段优先级:优先使用 IndexInfo.ID,兜底使用 ChunkID
3. 新增 translateSourceID 函数,智能处理 question 类型数据的 SourceID 映射:
   - 当 SourceID == ChunkID 时,直接映射为目标 ChunkID
   - 当 SourceID 为 "ChunkID-QuestionID" 格式时,保留 QuestionID 部分
   - 其他情况使用 SHA256 生成稳定的映射 ID

影响范围:
- 修改文件:internal/application/repository/retriever/tencentvectordb/repository.go
- 新增函数:copySourceQueryParams、remapCopiedEmbeddings、translateSourceID
This commit is contained in:
“jiangyijun”
2026-06-11 17:07:43 +08:00
committed by lyingbug
parent 408364c6c9
commit 6bc8e9ed49
@@ -2,6 +2,8 @@ package tencentvectordb
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"maps"
"os"
@@ -17,6 +19,8 @@ import (
"github.com/tencent/vectordatabase-sdk-go/tcvectordb"
)
const copyIndicesQueryPageSize int64 = 500
// NewTencentVectorDBRetrieveEngineRepository creates a Tencent VectorDB-backed retrieve repository.
func NewTencentVectorDBRetrieveEngineRepository(
client *tcvectordb.RpcClient,
@@ -139,34 +143,26 @@ func (r *repository) CopyIndices(
}
collectionName := r.collectionName(dimension)
ids := slices.Collect(maps.Keys(sourceToTargetChunkIDMap))
query, err := r.client.Database(r.databaseName).Collection(collectionName).Query(
ctx,
nil,
&tcvectordb.QueryDocumentParams{
Filter: tcvectordb.NewFilter(tcvectordb.In(fieldChunkID, ids)),
RetrieveVector: true,
OutputFields: outputFields(),
Limit: int64(len(ids)),
},
)
if err != nil {
return fmt.Errorf("tencent vectordb query source indices: %w", err)
}
embeddings := make([]*vectorEmbedding, 0, len(query.Documents))
for _, doc := range query.Documents {
embedding := fromDocument(doc)
targetChunkID := sourceToTargetChunkIDMap[embedding.ChunkID]
if targetChunkID == "" {
continue
var embeddings []*vectorEmbedding
for offset := int64(0); ; offset += copyIndicesQueryPageSize {
query, err := r.client.Database(r.databaseName).Collection(collectionName).Query(
ctx,
nil,
copySourceQueryParams(sourceKnowledgeBaseID, ids, offset),
)
if err != nil {
return fmt.Errorf("tencent vectordb query source indices: %w", err)
}
embedding.ID = targetChunkID
embedding.ChunkID = targetChunkID
embedding.KnowledgeBaseID = targetKnowledgeBaseID
if targetKBID := sourceToTargetKBIDMap[embedding.KnowledgeID]; targetKBID != "" {
embedding.KnowledgeID = targetKBID
embeddings = append(embeddings, remapCopiedEmbeddings(
query.Documents,
sourceToTargetKBIDMap,
sourceToTargetChunkIDMap,
targetKnowledgeBaseID,
)...)
if len(query.Documents) < int(copyIndicesQueryPageSize) {
break
}
embeddings = append(embeddings, embedding)
}
if len(embeddings) == 0 {
return nil
@@ -563,7 +559,7 @@ func (r *repository) toDocumentsWithSparseVectors(
func toVectorEmbedding(indexInfo *types.IndexInfo, params map[string]any) *vectorEmbedding {
embedding := &vectorEmbedding{
ID: indexInfo.ChunkID,
ID: indexInfo.ID,
Content: cleanInvalidUTF8(indexInfo.Content),
SourceID: indexInfo.SourceID,
SourceType: int(indexInfo.SourceType),
@@ -576,6 +572,9 @@ func toVectorEmbedding(indexInfo *types.IndexInfo, params map[string]any) *vecto
if embedding.ID == "" {
embedding.ID = indexInfo.SourceID
}
if embedding.ID == "" {
embedding.ID = indexInfo.ChunkID
}
if params != nil && slices.Contains(slices.Collect(maps.Keys(params)), fieldVector) {
if embeddingMap, ok := params[fieldVector].(map[string][]float32); ok {
embedding.Embedding = lookupEmbedding(embeddingMap, indexInfo)
@@ -596,6 +595,63 @@ func lookupEmbedding(embeddingMap map[string][]float32, indexInfo *types.IndexIn
return embeddingMap[indexInfo.ChunkID]
}
func copySourceQueryParams(sourceKnowledgeBaseID string, chunkIDs []string, offset int64) *tcvectordb.QueryDocumentParams {
conditions := []string{tcvectordb.In(fieldChunkID, chunkIDs)}
if sourceKnowledgeBaseID != "" {
conditions = append([]string{tcvectordb.In(fieldKnowledgeBaseID, []string{sourceKnowledgeBaseID})}, conditions...)
}
return &tcvectordb.QueryDocumentParams{
Filter: tcvectordb.NewFilter(strings.Join(conditions, " and ")),
RetrieveVector: true,
OutputFields: outputFields(),
Offset: offset,
Limit: copyIndicesQueryPageSize,
}
}
func remapCopiedEmbeddings(
docs []tcvectordb.Document,
sourceToTargetKBIDMap map[string]string,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
) []*vectorEmbedding {
embeddings := make([]*vectorEmbedding, 0, len(docs))
for _, doc := range docs {
embedding := fromDocument(doc)
targetChunkID := sourceToTargetChunkIDMap[embedding.ChunkID]
if targetChunkID == "" {
continue
}
originalSourceID := embedding.SourceID
if originalSourceID == "" {
originalSourceID = embedding.ID
}
targetSourceID := translateSourceID(originalSourceID, embedding.ChunkID, targetChunkID)
embedding.ID = targetSourceID
embedding.SourceID = targetSourceID
embedding.ChunkID = targetChunkID
embedding.KnowledgeBaseID = targetKnowledgeBaseID
if targetKBID := sourceToTargetKBIDMap[embedding.KnowledgeID]; targetKBID != "" {
embedding.KnowledgeID = targetKBID
}
embeddings = append(embeddings, embedding)
}
return embeddings
}
func translateSourceID(originalSourceID, sourceChunkID, targetChunkID string) string {
switch {
case originalSourceID == sourceChunkID:
return targetChunkID
case strings.HasPrefix(originalSourceID, sourceChunkID+"-"):
questionID := strings.TrimPrefix(originalSourceID, sourceChunkID+"-")
return fmt.Sprintf("%s-%s", targetChunkID, questionID)
default:
sum := sha256.Sum256([]byte(targetChunkID + "\x00" + sourceChunkID + "\x00" + originalSourceID))
return fmt.Sprintf("%s-%s", targetChunkID, hex.EncodeToString(sum[:])[:16])
}
}
func cleanInvalidUTF8(s string) string {
var b strings.Builder
b.Grow(len(s))