diff --git a/docs/api/vector-store.md b/docs/api/vector-store.md index ccf94503..2ad6ef0e 100644 --- a/docs/api/vector-store.md +++ b/docs/api/vector-store.md @@ -110,7 +110,7 @@ curl --location 'http://localhost:8080/api/v1/vector-stores/test' \ 创建一个新的向量存储配置。同一 endpoint + index 组合不允许重复注册(包括环境变量配置的存储)。 -Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 为必填,`database` 可选;`index_config.collection_name` 表示集合名前缀,实际集合会按向量维度追加后缀,例如 `weknora_embeddings_768`。 +Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 为必填,`database` 可选;`index_config.collection_name` 表示集合名前缀,实际集合会按向量维度追加后缀,例如 `weknora_embeddings_768`。Tencent VectorDB 适配器同时支持向量检索和基于 BM25 sparse vector 的关键词检索;旧版本已创建且没有 `sparse_vector` 索引的集合,需要重建并重新导入数据后才能启用关键词检索。 **请求**: diff --git a/docs/wiki/集成扩展/集成向量数据库.md b/docs/wiki/集成扩展/集成向量数据库.md index e22b9978..ee778528 100644 --- a/docs/wiki/集成扩展/集成向量数据库.md +++ b/docs/wiki/集成扩展/集成向量数据库.md @@ -90,7 +90,7 @@ DORIS_TABLE_PREFIX=weknora_embeddings ## Tencent VectorDB -WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索和索引管理,关键词检索不在当前适配器范围内。 +WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索、基于 BM25 sparse vector 的关键词检索和索引管理,可参与 WeKnora 上层混合检索。 ```env RETRIEVE_DRIVER=tencent_vectordb @@ -103,6 +103,8 @@ TENCENT_VECTORDB_COLLECTION=weknora_embeddings `TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合,例如 `weknora_embeddings_768`。 +关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引;旧版本已创建的向量集合如果没有该索引,需要重建集合并重新导入知识库数据后才能启用关键词检索。 + ## 相关主题 - [添加网络搜索引擎](../集成扩展/添加网络搜索引擎.md) — 同类扩展开发模式(接口实现 + 注册 + DI) diff --git a/docs/使用其他向量数据库.md b/docs/使用其他向量数据库.md index 6422e8f2..d21ba338 100644 --- a/docs/使用其他向量数据库.md +++ b/docs/使用其他向量数据库.md @@ -278,7 +278,7 @@ docker exec -it WeKnora-doris-fe mysql -h 127.0.0.1 -P 9030 -uroot \ ## Tencent VectorDB -WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索和索引管理,关键词检索不在当前适配器范围内。 +WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索、基于 BM25 sparse vector 的关键词检索和索引管理,可参与 WeKnora 上层混合检索。 ### 环境变量示例 @@ -293,3 +293,4 @@ TENCENT_VECTORDB_COLLECTION=weknora_embeddings `TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合,例如 `weknora_embeddings_768`,用于隔离不同 embedding 模型维度的数据。 +关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引;旧版本已创建的向量集合如果没有该索引,需要重建集合并重新导入知识库数据后才能启用关键词检索。 diff --git a/internal/application/repository/retriever/tencentvectordb/repository.go b/internal/application/repository/retriever/tencentvectordb/repository.go index 64f1e63a..067b16f6 100644 --- a/internal/application/repository/retriever/tencentvectordb/repository.go +++ b/internal/application/repository/retriever/tencentvectordb/repository.go @@ -6,12 +6,14 @@ import ( "maps" "os" "slices" + "sort" "strings" "unicode/utf8" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/tencent/vectordatabase-sdk-go/tcvdbtext/encoder" "github.com/tencent/vectordatabase-sdk-go/tcvectordb" ) @@ -43,7 +45,7 @@ func (r *repository) EngineType() types.RetrieverEngineType { } func (r *repository) Support() []types.RetrieverType { - return []types.RetrieverType{types.VectorRetrieverType} + return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType} } func (r *repository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error { @@ -56,7 +58,7 @@ func (r *repository) BatchSave(ctx context.Context, indexInfoList []*types.Index return nil } - docsByDimension := make(map[int][]tcvectordb.Document) + embeddingsByDimension := make(map[int][]*vectorEmbedding) for _, indexInfo := range indexInfoList { embedding := toVectorEmbedding(indexInfo, params) if len(embedding.Embedding) == 0 { @@ -64,19 +66,28 @@ func (r *repository) BatchSave(ctx context.Context, indexInfoList []*types.Index continue } dim := len(embedding.Embedding) - docsByDimension[dim] = append(docsByDimension[dim], toDocument(embedding)) + embeddingsByDimension[dim] = append(embeddingsByDimension[dim], embedding) } - if len(docsByDimension) == 0 { + if len(embeddingsByDimension) == 0 { return nil } + bm25, err := r.bm25Encoder() + if err != nil { + return err + } + buildIndex := true - for dim, docs := range docsByDimension { + for dim, embeddings := range embeddingsByDimension { + docs, err := r.toDocumentsWithSparseVectors(bm25, embeddings) + if err != nil { + return err + } if err := r.ensureCollection(ctx, dim); err != nil { return err } collectionName := r.collectionName(dim) - _, err := r.client.Database(r.databaseName).Collection(collectionName).Upsert( + _, err = r.client.Database(r.databaseName).Collection(collectionName).Upsert( ctx, docs, &tcvectordb.UpsertDocumentParams{BuildIndex: &buildIndex}, @@ -94,6 +105,7 @@ func (r *repository) EstimateStorageSize(ctx context.Context, indexInfoList []*t embedding := toVectorEmbedding(indexInfo, params) total += int64(len(embedding.Content)) total += int64(len(embedding.Embedding) * 4) + total += int64(len(embedding.Content) * 2) total += int64(len(embedding.SourceID) + len(embedding.ChunkID) + len(embedding.KnowledgeID) + len(embedding.KnowledgeBaseID) + 256) } logger.GetLogger(ctx).Infof("[TencentVectorDB] estimated storage size for %d indices: %d bytes", len(indexInfoList), total) @@ -140,7 +152,7 @@ func (r *repository) CopyIndices( return fmt.Errorf("tencent vectordb query source indices: %w", err) } - docs := make([]tcvectordb.Document, 0, len(query.Documents)) + embeddings := make([]*vectorEmbedding, 0, len(query.Documents)) for _, doc := range query.Documents { embedding := fromDocument(doc) targetChunkID := sourceToTargetChunkIDMap[embedding.ChunkID] @@ -153,12 +165,21 @@ func (r *repository) CopyIndices( if targetKBID := sourceToTargetKBIDMap[embedding.KnowledgeID]; targetKBID != "" { embedding.KnowledgeID = targetKBID } - docs = append(docs, toDocument(embedding)) + embeddings = append(embeddings, embedding) } - if len(docs) == 0 { + if len(embeddings) == 0 { return nil } + bm25, err := r.bm25Encoder() + if err != nil { + return err + } + docs, err := r.toDocumentsWithSparseVectors(bm25, embeddings) + if err != nil { + return err + } + buildIndex := true _, err = r.client.Database(r.databaseName).Collection(collectionName).Upsert( ctx, @@ -208,7 +229,7 @@ func (r *repository) Retrieve(ctx context.Context, params types.RetrieveParams) case types.VectorRetrieverType: return r.VectorRetrieve(ctx, params) case types.KeywordsRetrieverType: - return nil, fmt.Errorf("tencent vectordb keyword retrieval is not supported") + return r.KeywordsRetrieve(ctx, params) default: return nil, fmt.Errorf("invalid retriever type: %s", params.RetrieverType) } @@ -261,6 +282,82 @@ func (r *repository) VectorRetrieve(ctx context.Context, params types.RetrievePa return r.retrieveResult(results, types.VectorRetrieverType), nil } +func (r *repository) KeywordsRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) { + log := logger.GetLogger(ctx) + query := strings.TrimSpace(params.Query) + if query == "" { + return r.retrieveResult(nil, types.KeywordsRetrieverType), nil + } + + bm25, err := r.bm25Encoder() + if err != nil { + return nil, err + } + queryVectors, err := bm25.EncodeQueries([]string{query}) + if err != nil { + return nil, fmt.Errorf("tencent vectordb encode keyword query: %w", err) + } + if len(queryVectors) == 0 || len(queryVectors[0]) == 0 { + return r.retrieveResult(nil, types.KeywordsRetrieverType), nil + } + + collections, err := r.client.Database(r.databaseName).ListCollection(ctx) + if err != nil { + return nil, fmt.Errorf("tencent vectordb list collections: %w", err) + } + + limit := params.TopK + if limit <= 0 { + limit = 10 + } + results := make([]*types.IndexWithScore, 0, limit) + matchedCollections := 0 + failedCollections := 0 + for _, collection := range collections.Collections { + if !strings.HasPrefix(collection.CollectionName, r.collectionBaseName+"_") { + continue + } + matchedCollections++ + + search, err := r.client.Database(r.databaseName).Collection(collection.CollectionName).FullTextSearch( + ctx, + tcvectordb.FullTextSearchParams{ + Filter: r.baseFilter(params), + RetrieveVector: false, + OutputFields: outputFields(), + Limit: &limit, + Match: &tcvectordb.FullTextSearchMatchOption{ + FieldName: fieldSparseVector, + Data: queryVectors, + }, + }, + ) + if err != nil { + failedCollections++ + log.Warnf("[TencentVectorDB] keyword search failed in %s: %v", collection.CollectionName, err) + continue + } + if len(search.Documents) == 0 { + continue + } + for _, doc := range search.Documents[0] { + embedding := fromDocument(doc) + results = append(results, toIndexWithScore(embedding, types.MatchTypeKeywords)) + } + } + if matchedCollections > 0 && failedCollections == matchedCollections { + return nil, fmt.Errorf("tencent vectordb keyword search failed in all matched collections; ensure collections have the %q sparse vector index and reimport data if they were created before keyword support", fieldSparseVector) + } + + sort.SliceStable(results, func(i, j int) bool { + return results[i].Score > results[j].Score + }) + if len(results) > limit { + results = results[:limit] + } + return r.retrieveResult(results, types.KeywordsRetrieverType), nil +} + func (r *repository) ensureCollection(ctx context.Context, dimension int) error { if _, ok := r.initialized.Load(dimension); ok { return nil @@ -296,6 +393,14 @@ func (r *repository) ensureCollection(ctx context.Context, dimension int) error }, }, }, + SparseVectorIndex: []tcvectordb.SparseVectorIndex{ + { + FieldName: fieldSparseVector, + FieldType: tcvectordb.SparseVector, + IndexType: tcvectordb.SPARSE_INVERTED, + MetricType: tcvectordb.IP, + }, + }, FilterIndex: []tcvectordb.FilterIndex{ {FieldName: fieldID, FieldType: tcvectordb.String, IndexType: tcvectordb.PRIMARY}, {FieldName: fieldContent, FieldType: tcvectordb.String, IndexType: tcvectordb.FILTER}, @@ -393,6 +498,41 @@ func (r *repository) retrieveResult(results []*types.IndexWithScore, retrieverTy } } +func (r *repository) bm25Encoder() (encoder.SparseEncoder, error) { + r.bm25Once.Do(func() { + r.bm25, r.bm25Err = encoder.NewBM25Encoder(&encoder.BM25EncoderParams{ + Bm25Language: encoder.BM25_ZH_CONTENT, + }) + if r.bm25Err != nil { + r.bm25Err = fmt.Errorf("tencent vectordb init BM25 encoder: %w", r.bm25Err) + } + }) + return r.bm25, r.bm25Err +} + +func (r *repository) toDocumentsWithSparseVectors( + bm25 encoder.SparseEncoder, + embeddings []*vectorEmbedding, +) ([]tcvectordb.Document, error) { + texts := make([]string, 0, len(embeddings)) + for _, embedding := range embeddings { + texts = append(texts, embedding.Content) + } + sparseVectors, err := bm25.EncodeTexts(texts) + if err != nil { + return nil, fmt.Errorf("tencent vectordb encode BM25 sparse vectors: %w", err) + } + docs := make([]tcvectordb.Document, 0, len(embeddings)) + for i, embedding := range embeddings { + if i >= len(sparseVectors) { + return nil, fmt.Errorf("tencent vectordb encoded sparse vector count mismatch: got %d, want %d", len(sparseVectors), len(embeddings)) + } + embedding.SparseVector = sparseVectors[i] + docs = append(docs, toDocument(embedding)) + } + return docs, nil +} + func toVectorEmbedding(indexInfo *types.IndexInfo, params map[string]any) *vectorEmbedding { embedding := &vectorEmbedding{ ID: indexInfo.ChunkID, @@ -450,8 +590,9 @@ func cleanInvalidUTF8(s string) string { func toDocument(embedding *vectorEmbedding) tcvectordb.Document { return tcvectordb.Document{ - Id: embedding.ID, - Vector: embedding.Embedding, + Id: embedding.ID, + Vector: embedding.Embedding, + SparseVector: embedding.SparseVector, Fields: map[string]tcvectordb.Field{ fieldContent: {Val: embedding.Content}, fieldSourceID: {Val: embedding.SourceID}, @@ -476,6 +617,7 @@ func fromDocument(doc tcvectordb.Document) *vectorEmbedding { KnowledgeBaseID: fieldString(doc, fieldKnowledgeBaseID), TagID: fieldString(doc, fieldTagID), Embedding: doc.Vector, + SparseVector: doc.SparseVector, IsEnabled: fieldUint64(doc, fieldIsEnabled) == 1, Score: float64(doc.Score), } diff --git a/internal/application/repository/retriever/tencentvectordb/repository_test.go b/internal/application/repository/retriever/tencentvectordb/repository_test.go new file mode 100644 index 00000000..e28f33ae --- /dev/null +++ b/internal/application/repository/retriever/tencentvectordb/repository_test.go @@ -0,0 +1,70 @@ +package tencentvectordb + +import ( + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/assert" + "github.com/tencent/vectordatabase-sdk-go/tcvdbtext/encoder" +) + +func TestSupportIncludesKeywordAndVectorRetrieval(t *testing.T) { + repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", nil) + + supports := repo.Support() + + assert.Contains(t, supports, types.KeywordsRetrieverType) + assert.Contains(t, supports, types.VectorRetrieverType) +} + +func TestToDocumentIncludesSparseVector(t *testing.T) { + embedding := &vectorEmbedding{ + ID: "chunk-1", + Content: "腾讯云向量数据库支持关键词检索", + SourceID: "source-1", + SourceType: int(types.ChunkSourceType), + ChunkID: "chunk-1", + KnowledgeID: "knowledge-1", + KnowledgeBaseID: "kb-1", + TagID: "tag-1", + Embedding: []float32{0.1, 0.2}, + SparseVector: []encoder.SparseVecItem{ + {TermId: 10, Score: 0.3}, + {TermId: 20, Score: 0.7}, + }, + IsEnabled: true, + } + + doc := toDocument(embedding) + + assert.Equal(t, embedding.ID, doc.Id) + assert.Equal(t, embedding.Embedding, doc.Vector) + assert.Equal(t, embedding.SparseVector, doc.SparseVector) + assert.Equal(t, "腾讯云向量数据库支持关键词检索", doc.Fields[fieldContent].String()) + assert.Equal(t, uint64(1), doc.Fields[fieldIsEnabled].Uint64()) +} + +func TestBaseFilterBuildsTencentVectorDBCondition(t *testing.T) { + repo := &repository{} + + filter := repo.baseFilter(types.RetrieveParams{ + KnowledgeBaseIDs: []string{"kb-1"}, + KnowledgeIDs: []string{"knowledge-1", "knowledge-2"}, + TagIDs: []string{"tag-1"}, + ExcludeKnowledgeIDs: []string{"knowledge-9"}, + ExcludeChunkIDs: []string{"chunk-9"}, + }) + cond := filter.Cond() + + for _, want := range []string{ + "is_enabled=1", + "knowledge_base_id in (\"kb-1\")", + "knowledge_id in (\"knowledge-1\",\"knowledge-2\")", + "tag_id in (\"tag-1\")", + "knowledge_id not in (\"knowledge-9\")", + "chunk_id not in (\"chunk-9\")", + } { + assert.True(t, strings.Contains(cond, want), "condition %q should contain %q", cond, want) + } +} diff --git a/internal/application/repository/retriever/tencentvectordb/structs.go b/internal/application/repository/retriever/tencentvectordb/structs.go index 7ff249a4..7672dc80 100644 --- a/internal/application/repository/retriever/tencentvectordb/structs.go +++ b/internal/application/repository/retriever/tencentvectordb/structs.go @@ -3,6 +3,7 @@ package tencentvectordb import ( "sync" + "github.com/tencent/vectordatabase-sdk-go/tcvdbtext/encoder" "github.com/tencent/vectordatabase-sdk-go/tcvectordb" ) @@ -14,6 +15,7 @@ const ( fieldID = "id" fieldVector = "vector" + fieldSparseVector = "sparse_vector" fieldContent = "content" fieldSourceID = "source_id" fieldSourceType = "source_type" @@ -31,6 +33,9 @@ type repository struct { shardsNum int replicasNum int initialized sync.Map + bm25Once sync.Once + bm25 encoder.SparseEncoder + bm25Err error } type vectorEmbedding struct { @@ -43,6 +48,7 @@ type vectorEmbedding struct { KnowledgeBaseID string TagID string Embedding []float32 + SparseVector []encoder.SparseVecItem IsEnabled bool Score float64 } diff --git a/internal/types/tenant.go b/internal/types/tenant.go index 077cb80a..d6cd9cc7 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -45,6 +45,10 @@ var retrieverEngineMapping = map[string][]RetrieverEngineParams{ {RetrieverType: KeywordsRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType}, {RetrieverType: VectorRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType}, }, + "tencent_vectordb": { + {RetrieverType: KeywordsRetrieverType, RetrieverEngineType: TencentVectorDBRetrieverEngineType}, + {RetrieverType: VectorRetrieverType, RetrieverEngineType: TencentVectorDBRetrieverEngineType}, + }, } // GetRetrieverEngineMapping returns the retriever engine mapping diff --git a/internal/types/tenant_test.go b/internal/types/tenant_test.go new file mode 100644 index 00000000..b2a81a36 --- /dev/null +++ b/internal/types/tenant_test.go @@ -0,0 +1,20 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRetrieverEngineMappingIncludesTencentVectorDBHybridCapabilities(t *testing.T) { + mapping := GetRetrieverEngineMapping() + + assert.Contains(t, mapping["tencent_vectordb"], RetrieverEngineParams{ + RetrieverType: KeywordsRetrieverType, + RetrieverEngineType: TencentVectorDBRetrieverEngineType, + }) + assert.Contains(t, mapping["tencent_vectordb"], RetrieverEngineParams{ + RetrieverType: VectorRetrieverType, + RetrieverEngineType: TencentVectorDBRetrieverEngineType, + }) +}