feat: support Tencent VectorDB keyword retrieval

This commit is contained in:
langcaiye
2026-05-09 16:32:31 +08:00
committed by lyingbug
parent 82947c726e
commit 02be05c095
8 changed files with 260 additions and 15 deletions
+1 -1
View File
@@ -110,7 +110,7 @@ curl --location 'http://localhost:8080/api/v1/vector-stores/test' \
创建一个新的向量存储配置。同一 endpoint + index 组合不允许重复注册(包括环境变量配置的存储)。
Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 为必填,`database` 可选;`index_config.collection_name` 表示集合名前缀,实际集合会按向量维度追加后缀,例如 `weknora_embeddings_768`。
Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 为必填,`database` 可选;`index_config.collection_name` 表示集合名前缀,实际集合会按向量维度追加后缀,例如 `weknora_embeddings_768`。Tencent VectorDB 适配器同时支持向量检索和基于 BM25 sparse vector 的关键词检索;旧版本已创建且没有 `sparse_vector` 索引的集合,需要重建并重新导入数据后才能启用关键词检索。
**请求**:
@@ -90,7 +90,7 @@ DORIS_TABLE_PREFIX=weknora_embeddings
## Tencent VectorDB
WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索和索引管理,关键词检索不在当前适配器范围内。
WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索、基于 BM25 sparse vector 的关键词检索和索引管理,可参与 WeKnora 上层混合检索。
```env
RETRIEVE_DRIVER=tencent_vectordb
@@ -103,6 +103,8 @@ TENCENT_VECTORDB_COLLECTION=weknora_embeddings
`TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合,例如 `weknora_embeddings_768`。
关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引;旧版本已创建的向量集合如果没有该索引,需要重建集合并重新导入知识库数据后才能启用关键词检索。
## 相关主题
- [添加网络搜索引擎](../集成扩展/添加网络搜索引擎.md) — 同类扩展开发模式(接口实现 + 注册 + DI)
+2 -1
View File
@@ -278,7 +278,7 @@ docker exec -it WeKnora-doris-fe mysql -h 127.0.0.1 -P 9030 -uroot \
## Tencent VectorDB
WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索和索引管理,关键词检索不在当前适配器范围内。
WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索、基于 BM25 sparse vector 的关键词检索和索引管理,可参与 WeKnora 上层混合检索。
### 环境变量示例
@@ -293,3 +293,4 @@ TENCENT_VECTORDB_COLLECTION=weknora_embeddings
`TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合,例如 `weknora_embeddings_768`,用于隔离不同 embedding 模型维度的数据。
关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引;旧版本已创建的向量集合如果没有该索引,需要重建集合并重新导入知识库数据后才能启用关键词检索。
@@ -6,12 +6,14 @@ import (
"maps"
"os"
"slices"
"sort"
"strings"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/tencent/vectordatabase-sdk-go/tcvdbtext/encoder"
"github.com/tencent/vectordatabase-sdk-go/tcvectordb"
)
@@ -43,7 +45,7 @@ func (r *repository) EngineType() types.RetrieverEngineType {
}
func (r *repository) Support() []types.RetrieverType {
return []types.RetrieverType{types.VectorRetrieverType}
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
}
func (r *repository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error {
@@ -56,7 +58,7 @@ func (r *repository) BatchSave(ctx context.Context, indexInfoList []*types.Index
return nil
}
docsByDimension := make(map[int][]tcvectordb.Document)
embeddingsByDimension := make(map[int][]*vectorEmbedding)
for _, indexInfo := range indexInfoList {
embedding := toVectorEmbedding(indexInfo, params)
if len(embedding.Embedding) == 0 {
@@ -64,19 +66,28 @@ func (r *repository) BatchSave(ctx context.Context, indexInfoList []*types.Index
continue
}
dim := len(embedding.Embedding)
docsByDimension[dim] = append(docsByDimension[dim], toDocument(embedding))
embeddingsByDimension[dim] = append(embeddingsByDimension[dim], embedding)
}
if len(docsByDimension) == 0 {
if len(embeddingsByDimension) == 0 {
return nil
}
bm25, err := r.bm25Encoder()
if err != nil {
return err
}
buildIndex := true
for dim, docs := range docsByDimension {
for dim, embeddings := range embeddingsByDimension {
docs, err := r.toDocumentsWithSparseVectors(bm25, embeddings)
if err != nil {
return err
}
if err := r.ensureCollection(ctx, dim); err != nil {
return err
}
collectionName := r.collectionName(dim)
_, err := r.client.Database(r.databaseName).Collection(collectionName).Upsert(
_, err = r.client.Database(r.databaseName).Collection(collectionName).Upsert(
ctx,
docs,
&tcvectordb.UpsertDocumentParams{BuildIndex: &buildIndex},
@@ -94,6 +105,7 @@ func (r *repository) EstimateStorageSize(ctx context.Context, indexInfoList []*t
embedding := toVectorEmbedding(indexInfo, params)
total += int64(len(embedding.Content))
total += int64(len(embedding.Embedding) * 4)
total += int64(len(embedding.Content) * 2)
total += int64(len(embedding.SourceID) + len(embedding.ChunkID) + len(embedding.KnowledgeID) + len(embedding.KnowledgeBaseID) + 256)
}
logger.GetLogger(ctx).Infof("[TencentVectorDB] estimated storage size for %d indices: %d bytes", len(indexInfoList), total)
@@ -140,7 +152,7 @@ func (r *repository) CopyIndices(
return fmt.Errorf("tencent vectordb query source indices: %w", err)
}
docs := make([]tcvectordb.Document, 0, len(query.Documents))
embeddings := make([]*vectorEmbedding, 0, len(query.Documents))
for _, doc := range query.Documents {
embedding := fromDocument(doc)
targetChunkID := sourceToTargetChunkIDMap[embedding.ChunkID]
@@ -153,12 +165,21 @@ func (r *repository) CopyIndices(
if targetKBID := sourceToTargetKBIDMap[embedding.KnowledgeID]; targetKBID != "" {
embedding.KnowledgeID = targetKBID
}
docs = append(docs, toDocument(embedding))
embeddings = append(embeddings, embedding)
}
if len(docs) == 0 {
if len(embeddings) == 0 {
return nil
}
bm25, err := r.bm25Encoder()
if err != nil {
return err
}
docs, err := r.toDocumentsWithSparseVectors(bm25, embeddings)
if err != nil {
return err
}
buildIndex := true
_, err = r.client.Database(r.databaseName).Collection(collectionName).Upsert(
ctx,
@@ -208,7 +229,7 @@ func (r *repository) Retrieve(ctx context.Context, params types.RetrieveParams)
case types.VectorRetrieverType:
return r.VectorRetrieve(ctx, params)
case types.KeywordsRetrieverType:
return nil, fmt.Errorf("tencent vectordb keyword retrieval is not supported")
return r.KeywordsRetrieve(ctx, params)
default:
return nil, fmt.Errorf("invalid retriever type: %s", params.RetrieverType)
}
@@ -261,6 +282,82 @@ func (r *repository) VectorRetrieve(ctx context.Context, params types.RetrievePa
return r.retrieveResult(results, types.VectorRetrieverType), nil
}
func (r *repository) KeywordsRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
log := logger.GetLogger(ctx)
query := strings.TrimSpace(params.Query)
if query == "" {
return r.retrieveResult(nil, types.KeywordsRetrieverType), nil
}
bm25, err := r.bm25Encoder()
if err != nil {
return nil, err
}
queryVectors, err := bm25.EncodeQueries([]string{query})
if err != nil {
return nil, fmt.Errorf("tencent vectordb encode keyword query: %w", err)
}
if len(queryVectors) == 0 || len(queryVectors[0]) == 0 {
return r.retrieveResult(nil, types.KeywordsRetrieverType), nil
}
collections, err := r.client.Database(r.databaseName).ListCollection(ctx)
if err != nil {
return nil, fmt.Errorf("tencent vectordb list collections: %w", err)
}
limit := params.TopK
if limit <= 0 {
limit = 10
}
results := make([]*types.IndexWithScore, 0, limit)
matchedCollections := 0
failedCollections := 0
for _, collection := range collections.Collections {
if !strings.HasPrefix(collection.CollectionName, r.collectionBaseName+"_") {
continue
}
matchedCollections++
search, err := r.client.Database(r.databaseName).Collection(collection.CollectionName).FullTextSearch(
ctx,
tcvectordb.FullTextSearchParams{
Filter: r.baseFilter(params),
RetrieveVector: false,
OutputFields: outputFields(),
Limit: &limit,
Match: &tcvectordb.FullTextSearchMatchOption{
FieldName: fieldSparseVector,
Data: queryVectors,
},
},
)
if err != nil {
failedCollections++
log.Warnf("[TencentVectorDB] keyword search failed in %s: %v", collection.CollectionName, err)
continue
}
if len(search.Documents) == 0 {
continue
}
for _, doc := range search.Documents[0] {
embedding := fromDocument(doc)
results = append(results, toIndexWithScore(embedding, types.MatchTypeKeywords))
}
}
if matchedCollections > 0 && failedCollections == matchedCollections {
return nil, fmt.Errorf("tencent vectordb keyword search failed in all matched collections; ensure collections have the %q sparse vector index and reimport data if they were created before keyword support", fieldSparseVector)
}
sort.SliceStable(results, func(i, j int) bool {
return results[i].Score > results[j].Score
})
if len(results) > limit {
results = results[:limit]
}
return r.retrieveResult(results, types.KeywordsRetrieverType), nil
}
func (r *repository) ensureCollection(ctx context.Context, dimension int) error {
if _, ok := r.initialized.Load(dimension); ok {
return nil
@@ -296,6 +393,14 @@ func (r *repository) ensureCollection(ctx context.Context, dimension int) error
},
},
},
SparseVectorIndex: []tcvectordb.SparseVectorIndex{
{
FieldName: fieldSparseVector,
FieldType: tcvectordb.SparseVector,
IndexType: tcvectordb.SPARSE_INVERTED,
MetricType: tcvectordb.IP,
},
},
FilterIndex: []tcvectordb.FilterIndex{
{FieldName: fieldID, FieldType: tcvectordb.String, IndexType: tcvectordb.PRIMARY},
{FieldName: fieldContent, FieldType: tcvectordb.String, IndexType: tcvectordb.FILTER},
@@ -393,6 +498,41 @@ func (r *repository) retrieveResult(results []*types.IndexWithScore, retrieverTy
}
}
func (r *repository) bm25Encoder() (encoder.SparseEncoder, error) {
r.bm25Once.Do(func() {
r.bm25, r.bm25Err = encoder.NewBM25Encoder(&encoder.BM25EncoderParams{
Bm25Language: encoder.BM25_ZH_CONTENT,
})
if r.bm25Err != nil {
r.bm25Err = fmt.Errorf("tencent vectordb init BM25 encoder: %w", r.bm25Err)
}
})
return r.bm25, r.bm25Err
}
func (r *repository) toDocumentsWithSparseVectors(
bm25 encoder.SparseEncoder,
embeddings []*vectorEmbedding,
) ([]tcvectordb.Document, error) {
texts := make([]string, 0, len(embeddings))
for _, embedding := range embeddings {
texts = append(texts, embedding.Content)
}
sparseVectors, err := bm25.EncodeTexts(texts)
if err != nil {
return nil, fmt.Errorf("tencent vectordb encode BM25 sparse vectors: %w", err)
}
docs := make([]tcvectordb.Document, 0, len(embeddings))
for i, embedding := range embeddings {
if i >= len(sparseVectors) {
return nil, fmt.Errorf("tencent vectordb encoded sparse vector count mismatch: got %d, want %d", len(sparseVectors), len(embeddings))
}
embedding.SparseVector = sparseVectors[i]
docs = append(docs, toDocument(embedding))
}
return docs, nil
}
func toVectorEmbedding(indexInfo *types.IndexInfo, params map[string]any) *vectorEmbedding {
embedding := &vectorEmbedding{
ID: indexInfo.ChunkID,
@@ -450,8 +590,9 @@ func cleanInvalidUTF8(s string) string {
func toDocument(embedding *vectorEmbedding) tcvectordb.Document {
return tcvectordb.Document{
Id: embedding.ID,
Vector: embedding.Embedding,
Id: embedding.ID,
Vector: embedding.Embedding,
SparseVector: embedding.SparseVector,
Fields: map[string]tcvectordb.Field{
fieldContent: {Val: embedding.Content},
fieldSourceID: {Val: embedding.SourceID},
@@ -476,6 +617,7 @@ func fromDocument(doc tcvectordb.Document) *vectorEmbedding {
KnowledgeBaseID: fieldString(doc, fieldKnowledgeBaseID),
TagID: fieldString(doc, fieldTagID),
Embedding: doc.Vector,
SparseVector: doc.SparseVector,
IsEnabled: fieldUint64(doc, fieldIsEnabled) == 1,
Score: float64(doc.Score),
}
@@ -0,0 +1,70 @@
package tencentvectordb
import (
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/stretchr/testify/assert"
"github.com/tencent/vectordatabase-sdk-go/tcvdbtext/encoder"
)
func TestSupportIncludesKeywordAndVectorRetrieval(t *testing.T) {
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", nil)
supports := repo.Support()
assert.Contains(t, supports, types.KeywordsRetrieverType)
assert.Contains(t, supports, types.VectorRetrieverType)
}
func TestToDocumentIncludesSparseVector(t *testing.T) {
embedding := &vectorEmbedding{
ID: "chunk-1",
Content: "腾讯云向量数据库支持关键词检索",
SourceID: "source-1",
SourceType: int(types.ChunkSourceType),
ChunkID: "chunk-1",
KnowledgeID: "knowledge-1",
KnowledgeBaseID: "kb-1",
TagID: "tag-1",
Embedding: []float32{0.1, 0.2},
SparseVector: []encoder.SparseVecItem{
{TermId: 10, Score: 0.3},
{TermId: 20, Score: 0.7},
},
IsEnabled: true,
}
doc := toDocument(embedding)
assert.Equal(t, embedding.ID, doc.Id)
assert.Equal(t, embedding.Embedding, doc.Vector)
assert.Equal(t, embedding.SparseVector, doc.SparseVector)
assert.Equal(t, "腾讯云向量数据库支持关键词检索", doc.Fields[fieldContent].String())
assert.Equal(t, uint64(1), doc.Fields[fieldIsEnabled].Uint64())
}
func TestBaseFilterBuildsTencentVectorDBCondition(t *testing.T) {
repo := &repository{}
filter := repo.baseFilter(types.RetrieveParams{
KnowledgeBaseIDs: []string{"kb-1"},
KnowledgeIDs: []string{"knowledge-1", "knowledge-2"},
TagIDs: []string{"tag-1"},
ExcludeKnowledgeIDs: []string{"knowledge-9"},
ExcludeChunkIDs: []string{"chunk-9"},
})
cond := filter.Cond()
for _, want := range []string{
"is_enabled=1",
"knowledge_base_id in (\"kb-1\")",
"knowledge_id in (\"knowledge-1\",\"knowledge-2\")",
"tag_id in (\"tag-1\")",
"knowledge_id not in (\"knowledge-9\")",
"chunk_id not in (\"chunk-9\")",
} {
assert.True(t, strings.Contains(cond, want), "condition %q should contain %q", cond, want)
}
}
@@ -3,6 +3,7 @@ package tencentvectordb
import (
"sync"
"github.com/tencent/vectordatabase-sdk-go/tcvdbtext/encoder"
"github.com/tencent/vectordatabase-sdk-go/tcvectordb"
)
@@ -14,6 +15,7 @@ const (
fieldID = "id"
fieldVector = "vector"
fieldSparseVector = "sparse_vector"
fieldContent = "content"
fieldSourceID = "source_id"
fieldSourceType = "source_type"
@@ -31,6 +33,9 @@ type repository struct {
shardsNum int
replicasNum int
initialized sync.Map
bm25Once sync.Once
bm25 encoder.SparseEncoder
bm25Err error
}
type vectorEmbedding struct {
@@ -43,6 +48,7 @@ type vectorEmbedding struct {
KnowledgeBaseID string
TagID string
Embedding []float32
SparseVector []encoder.SparseVecItem
IsEnabled bool
Score float64
}
+4
View File
@@ -45,6 +45,10 @@ var retrieverEngineMapping = map[string][]RetrieverEngineParams{
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType},
{RetrieverType: VectorRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType},
},
"tencent_vectordb": {
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: TencentVectorDBRetrieverEngineType},
{RetrieverType: VectorRetrieverType, RetrieverEngineType: TencentVectorDBRetrieverEngineType},
},
}
// GetRetrieverEngineMapping returns the retriever engine mapping
+20
View File
@@ -0,0 +1,20 @@
package types
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestRetrieverEngineMappingIncludesTencentVectorDBHybridCapabilities(t *testing.T) {
mapping := GetRetrieverEngineMapping()
assert.Contains(t, mapping["tencent_vectordb"], RetrieverEngineParams{
RetrieverType: KeywordsRetrieverType,
RetrieverEngineType: TencentVectorDBRetrieverEngineType,
})
assert.Contains(t, mapping["tencent_vectordb"], RetrieverEngineParams{
RetrieverType: VectorRetrieverType,
RetrieverEngineType: TencentVectorDBRetrieverEngineType,
})
}