feat: support Tencent VectorDB keyword retrieval
This commit is contained in:
@@ -110,7 +110,7 @@ curl --location 'http://localhost:8080/api/v1/vector-stores/test' \
|
||||
|
||||
创建一个新的向量存储配置。同一 endpoint + index 组合不允许重复注册(包括环境变量配置的存储)。
|
||||
|
||||
Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 为必填,`database` 可选;`index_config.collection_name` 表示集合名前缀,实际集合会按向量维度追加后缀,例如 `weknora_embeddings_768`。
|
||||
Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 为必填,`database` 可选;`index_config.collection_name` 表示集合名前缀,实际集合会按向量维度追加后缀,例如 `weknora_embeddings_768`。Tencent VectorDB 适配器同时支持向量检索和基于 BM25 sparse vector 的关键词检索;旧版本已创建且没有 `sparse_vector` 索引的集合,需要重建并重新导入数据后才能启用关键词检索。
|
||||
|
||||
**请求**:
|
||||
|
||||
|
||||
@@ -90,7 +90,7 @@ DORIS_TABLE_PREFIX=weknora_embeddings
|
||||
|
||||
## Tencent VectorDB
|
||||
|
||||
WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索和索引管理,关键词检索不在当前适配器范围内。
|
||||
WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索、基于 BM25 sparse vector 的关键词检索和索引管理,可参与 WeKnora 上层混合检索。
|
||||
|
||||
```env
|
||||
RETRIEVE_DRIVER=tencent_vectordb
|
||||
@@ -103,6 +103,8 @@ TENCENT_VECTORDB_COLLECTION=weknora_embeddings
|
||||
|
||||
`TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合,例如 `weknora_embeddings_768`。
|
||||
|
||||
关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引;旧版本已创建的向量集合如果没有该索引,需要重建集合并重新导入知识库数据后才能启用关键词检索。
|
||||
|
||||
## 相关主题
|
||||
|
||||
- [添加网络搜索引擎](../集成扩展/添加网络搜索引擎.md) — 同类扩展开发模式(接口实现 + 注册 + DI)
|
||||
|
||||
+2
-1
@@ -278,7 +278,7 @@ docker exec -it WeKnora-doris-fe mysql -h 127.0.0.1 -P 9030 -uroot \
|
||||
|
||||
## Tencent VectorDB
|
||||
|
||||
WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索和索引管理,关键词检索不在当前适配器范围内。
|
||||
WeKnora 内置 Tencent VectorDB 适配器,驱动名为 `tencent_vectordb`。该适配器支持向量检索、基于 BM25 sparse vector 的关键词检索和索引管理,可参与 WeKnora 上层混合检索。
|
||||
|
||||
### 环境变量示例
|
||||
|
||||
@@ -293,3 +293,4 @@ TENCENT_VECTORDB_COLLECTION=weknora_embeddings
|
||||
|
||||
`TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合,例如 `weknora_embeddings_768`,用于隔离不同 embedding 模型维度的数据。
|
||||
|
||||
关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引;旧版本已创建的向量集合如果没有该索引,需要重建集合并重新导入知识库数据后才能启用关键词检索。
|
||||
|
||||
@@ -6,12 +6,14 @@ import (
|
||||
"maps"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/tencent/vectordatabase-sdk-go/tcvdbtext/encoder"
|
||||
"github.com/tencent/vectordatabase-sdk-go/tcvectordb"
|
||||
)
|
||||
|
||||
@@ -43,7 +45,7 @@ func (r *repository) EngineType() types.RetrieverEngineType {
|
||||
}
|
||||
|
||||
func (r *repository) Support() []types.RetrieverType {
|
||||
return []types.RetrieverType{types.VectorRetrieverType}
|
||||
return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType}
|
||||
}
|
||||
|
||||
func (r *repository) Save(ctx context.Context, indexInfo *types.IndexInfo, params map[string]any) error {
|
||||
@@ -56,7 +58,7 @@ func (r *repository) BatchSave(ctx context.Context, indexInfoList []*types.Index
|
||||
return nil
|
||||
}
|
||||
|
||||
docsByDimension := make(map[int][]tcvectordb.Document)
|
||||
embeddingsByDimension := make(map[int][]*vectorEmbedding)
|
||||
for _, indexInfo := range indexInfoList {
|
||||
embedding := toVectorEmbedding(indexInfo, params)
|
||||
if len(embedding.Embedding) == 0 {
|
||||
@@ -64,19 +66,28 @@ func (r *repository) BatchSave(ctx context.Context, indexInfoList []*types.Index
|
||||
continue
|
||||
}
|
||||
dim := len(embedding.Embedding)
|
||||
docsByDimension[dim] = append(docsByDimension[dim], toDocument(embedding))
|
||||
embeddingsByDimension[dim] = append(embeddingsByDimension[dim], embedding)
|
||||
}
|
||||
if len(docsByDimension) == 0 {
|
||||
if len(embeddingsByDimension) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
bm25, err := r.bm25Encoder()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
buildIndex := true
|
||||
for dim, docs := range docsByDimension {
|
||||
for dim, embeddings := range embeddingsByDimension {
|
||||
docs, err := r.toDocumentsWithSparseVectors(bm25, embeddings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := r.ensureCollection(ctx, dim); err != nil {
|
||||
return err
|
||||
}
|
||||
collectionName := r.collectionName(dim)
|
||||
_, err := r.client.Database(r.databaseName).Collection(collectionName).Upsert(
|
||||
_, err = r.client.Database(r.databaseName).Collection(collectionName).Upsert(
|
||||
ctx,
|
||||
docs,
|
||||
&tcvectordb.UpsertDocumentParams{BuildIndex: &buildIndex},
|
||||
@@ -94,6 +105,7 @@ func (r *repository) EstimateStorageSize(ctx context.Context, indexInfoList []*t
|
||||
embedding := toVectorEmbedding(indexInfo, params)
|
||||
total += int64(len(embedding.Content))
|
||||
total += int64(len(embedding.Embedding) * 4)
|
||||
total += int64(len(embedding.Content) * 2)
|
||||
total += int64(len(embedding.SourceID) + len(embedding.ChunkID) + len(embedding.KnowledgeID) + len(embedding.KnowledgeBaseID) + 256)
|
||||
}
|
||||
logger.GetLogger(ctx).Infof("[TencentVectorDB] estimated storage size for %d indices: %d bytes", len(indexInfoList), total)
|
||||
@@ -140,7 +152,7 @@ func (r *repository) CopyIndices(
|
||||
return fmt.Errorf("tencent vectordb query source indices: %w", err)
|
||||
}
|
||||
|
||||
docs := make([]tcvectordb.Document, 0, len(query.Documents))
|
||||
embeddings := make([]*vectorEmbedding, 0, len(query.Documents))
|
||||
for _, doc := range query.Documents {
|
||||
embedding := fromDocument(doc)
|
||||
targetChunkID := sourceToTargetChunkIDMap[embedding.ChunkID]
|
||||
@@ -153,12 +165,21 @@ func (r *repository) CopyIndices(
|
||||
if targetKBID := sourceToTargetKBIDMap[embedding.KnowledgeID]; targetKBID != "" {
|
||||
embedding.KnowledgeID = targetKBID
|
||||
}
|
||||
docs = append(docs, toDocument(embedding))
|
||||
embeddings = append(embeddings, embedding)
|
||||
}
|
||||
if len(docs) == 0 {
|
||||
if len(embeddings) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
bm25, err := r.bm25Encoder()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
docs, err := r.toDocumentsWithSparseVectors(bm25, embeddings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
buildIndex := true
|
||||
_, err = r.client.Database(r.databaseName).Collection(collectionName).Upsert(
|
||||
ctx,
|
||||
@@ -208,7 +229,7 @@ func (r *repository) Retrieve(ctx context.Context, params types.RetrieveParams)
|
||||
case types.VectorRetrieverType:
|
||||
return r.VectorRetrieve(ctx, params)
|
||||
case types.KeywordsRetrieverType:
|
||||
return nil, fmt.Errorf("tencent vectordb keyword retrieval is not supported")
|
||||
return r.KeywordsRetrieve(ctx, params)
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid retriever type: %s", params.RetrieverType)
|
||||
}
|
||||
@@ -261,6 +282,82 @@ func (r *repository) VectorRetrieve(ctx context.Context, params types.RetrievePa
|
||||
return r.retrieveResult(results, types.VectorRetrieverType), nil
|
||||
}
|
||||
|
||||
func (r *repository) KeywordsRetrieve(ctx context.Context, params types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
||||
log := logger.GetLogger(ctx)
|
||||
query := strings.TrimSpace(params.Query)
|
||||
if query == "" {
|
||||
return r.retrieveResult(nil, types.KeywordsRetrieverType), nil
|
||||
}
|
||||
|
||||
bm25, err := r.bm25Encoder()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
queryVectors, err := bm25.EncodeQueries([]string{query})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tencent vectordb encode keyword query: %w", err)
|
||||
}
|
||||
if len(queryVectors) == 0 || len(queryVectors[0]) == 0 {
|
||||
return r.retrieveResult(nil, types.KeywordsRetrieverType), nil
|
||||
}
|
||||
|
||||
collections, err := r.client.Database(r.databaseName).ListCollection(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tencent vectordb list collections: %w", err)
|
||||
}
|
||||
|
||||
limit := params.TopK
|
||||
if limit <= 0 {
|
||||
limit = 10
|
||||
}
|
||||
results := make([]*types.IndexWithScore, 0, limit)
|
||||
matchedCollections := 0
|
||||
failedCollections := 0
|
||||
for _, collection := range collections.Collections {
|
||||
if !strings.HasPrefix(collection.CollectionName, r.collectionBaseName+"_") {
|
||||
continue
|
||||
}
|
||||
matchedCollections++
|
||||
|
||||
search, err := r.client.Database(r.databaseName).Collection(collection.CollectionName).FullTextSearch(
|
||||
ctx,
|
||||
tcvectordb.FullTextSearchParams{
|
||||
Filter: r.baseFilter(params),
|
||||
RetrieveVector: false,
|
||||
OutputFields: outputFields(),
|
||||
Limit: &limit,
|
||||
Match: &tcvectordb.FullTextSearchMatchOption{
|
||||
FieldName: fieldSparseVector,
|
||||
Data: queryVectors,
|
||||
},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
failedCollections++
|
||||
log.Warnf("[TencentVectorDB] keyword search failed in %s: %v", collection.CollectionName, err)
|
||||
continue
|
||||
}
|
||||
if len(search.Documents) == 0 {
|
||||
continue
|
||||
}
|
||||
for _, doc := range search.Documents[0] {
|
||||
embedding := fromDocument(doc)
|
||||
results = append(results, toIndexWithScore(embedding, types.MatchTypeKeywords))
|
||||
}
|
||||
}
|
||||
if matchedCollections > 0 && failedCollections == matchedCollections {
|
||||
return nil, fmt.Errorf("tencent vectordb keyword search failed in all matched collections; ensure collections have the %q sparse vector index and reimport data if they were created before keyword support", fieldSparseVector)
|
||||
}
|
||||
|
||||
sort.SliceStable(results, func(i, j int) bool {
|
||||
return results[i].Score > results[j].Score
|
||||
})
|
||||
if len(results) > limit {
|
||||
results = results[:limit]
|
||||
}
|
||||
return r.retrieveResult(results, types.KeywordsRetrieverType), nil
|
||||
}
|
||||
|
||||
func (r *repository) ensureCollection(ctx context.Context, dimension int) error {
|
||||
if _, ok := r.initialized.Load(dimension); ok {
|
||||
return nil
|
||||
@@ -296,6 +393,14 @@ func (r *repository) ensureCollection(ctx context.Context, dimension int) error
|
||||
},
|
||||
},
|
||||
},
|
||||
SparseVectorIndex: []tcvectordb.SparseVectorIndex{
|
||||
{
|
||||
FieldName: fieldSparseVector,
|
||||
FieldType: tcvectordb.SparseVector,
|
||||
IndexType: tcvectordb.SPARSE_INVERTED,
|
||||
MetricType: tcvectordb.IP,
|
||||
},
|
||||
},
|
||||
FilterIndex: []tcvectordb.FilterIndex{
|
||||
{FieldName: fieldID, FieldType: tcvectordb.String, IndexType: tcvectordb.PRIMARY},
|
||||
{FieldName: fieldContent, FieldType: tcvectordb.String, IndexType: tcvectordb.FILTER},
|
||||
@@ -393,6 +498,41 @@ func (r *repository) retrieveResult(results []*types.IndexWithScore, retrieverTy
|
||||
}
|
||||
}
|
||||
|
||||
func (r *repository) bm25Encoder() (encoder.SparseEncoder, error) {
|
||||
r.bm25Once.Do(func() {
|
||||
r.bm25, r.bm25Err = encoder.NewBM25Encoder(&encoder.BM25EncoderParams{
|
||||
Bm25Language: encoder.BM25_ZH_CONTENT,
|
||||
})
|
||||
if r.bm25Err != nil {
|
||||
r.bm25Err = fmt.Errorf("tencent vectordb init BM25 encoder: %w", r.bm25Err)
|
||||
}
|
||||
})
|
||||
return r.bm25, r.bm25Err
|
||||
}
|
||||
|
||||
func (r *repository) toDocumentsWithSparseVectors(
|
||||
bm25 encoder.SparseEncoder,
|
||||
embeddings []*vectorEmbedding,
|
||||
) ([]tcvectordb.Document, error) {
|
||||
texts := make([]string, 0, len(embeddings))
|
||||
for _, embedding := range embeddings {
|
||||
texts = append(texts, embedding.Content)
|
||||
}
|
||||
sparseVectors, err := bm25.EncodeTexts(texts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tencent vectordb encode BM25 sparse vectors: %w", err)
|
||||
}
|
||||
docs := make([]tcvectordb.Document, 0, len(embeddings))
|
||||
for i, embedding := range embeddings {
|
||||
if i >= len(sparseVectors) {
|
||||
return nil, fmt.Errorf("tencent vectordb encoded sparse vector count mismatch: got %d, want %d", len(sparseVectors), len(embeddings))
|
||||
}
|
||||
embedding.SparseVector = sparseVectors[i]
|
||||
docs = append(docs, toDocument(embedding))
|
||||
}
|
||||
return docs, nil
|
||||
}
|
||||
|
||||
func toVectorEmbedding(indexInfo *types.IndexInfo, params map[string]any) *vectorEmbedding {
|
||||
embedding := &vectorEmbedding{
|
||||
ID: indexInfo.ChunkID,
|
||||
@@ -450,8 +590,9 @@ func cleanInvalidUTF8(s string) string {
|
||||
|
||||
func toDocument(embedding *vectorEmbedding) tcvectordb.Document {
|
||||
return tcvectordb.Document{
|
||||
Id: embedding.ID,
|
||||
Vector: embedding.Embedding,
|
||||
Id: embedding.ID,
|
||||
Vector: embedding.Embedding,
|
||||
SparseVector: embedding.SparseVector,
|
||||
Fields: map[string]tcvectordb.Field{
|
||||
fieldContent: {Val: embedding.Content},
|
||||
fieldSourceID: {Val: embedding.SourceID},
|
||||
@@ -476,6 +617,7 @@ func fromDocument(doc tcvectordb.Document) *vectorEmbedding {
|
||||
KnowledgeBaseID: fieldString(doc, fieldKnowledgeBaseID),
|
||||
TagID: fieldString(doc, fieldTagID),
|
||||
Embedding: doc.Vector,
|
||||
SparseVector: doc.SparseVector,
|
||||
IsEnabled: fieldUint64(doc, fieldIsEnabled) == 1,
|
||||
Score: float64(doc.Score),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package tencentvectordb
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/tencent/vectordatabase-sdk-go/tcvdbtext/encoder"
|
||||
)
|
||||
|
||||
func TestSupportIncludesKeywordAndVectorRetrieval(t *testing.T) {
|
||||
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", nil)
|
||||
|
||||
supports := repo.Support()
|
||||
|
||||
assert.Contains(t, supports, types.KeywordsRetrieverType)
|
||||
assert.Contains(t, supports, types.VectorRetrieverType)
|
||||
}
|
||||
|
||||
func TestToDocumentIncludesSparseVector(t *testing.T) {
|
||||
embedding := &vectorEmbedding{
|
||||
ID: "chunk-1",
|
||||
Content: "腾讯云向量数据库支持关键词检索",
|
||||
SourceID: "source-1",
|
||||
SourceType: int(types.ChunkSourceType),
|
||||
ChunkID: "chunk-1",
|
||||
KnowledgeID: "knowledge-1",
|
||||
KnowledgeBaseID: "kb-1",
|
||||
TagID: "tag-1",
|
||||
Embedding: []float32{0.1, 0.2},
|
||||
SparseVector: []encoder.SparseVecItem{
|
||||
{TermId: 10, Score: 0.3},
|
||||
{TermId: 20, Score: 0.7},
|
||||
},
|
||||
IsEnabled: true,
|
||||
}
|
||||
|
||||
doc := toDocument(embedding)
|
||||
|
||||
assert.Equal(t, embedding.ID, doc.Id)
|
||||
assert.Equal(t, embedding.Embedding, doc.Vector)
|
||||
assert.Equal(t, embedding.SparseVector, doc.SparseVector)
|
||||
assert.Equal(t, "腾讯云向量数据库支持关键词检索", doc.Fields[fieldContent].String())
|
||||
assert.Equal(t, uint64(1), doc.Fields[fieldIsEnabled].Uint64())
|
||||
}
|
||||
|
||||
func TestBaseFilterBuildsTencentVectorDBCondition(t *testing.T) {
|
||||
repo := &repository{}
|
||||
|
||||
filter := repo.baseFilter(types.RetrieveParams{
|
||||
KnowledgeBaseIDs: []string{"kb-1"},
|
||||
KnowledgeIDs: []string{"knowledge-1", "knowledge-2"},
|
||||
TagIDs: []string{"tag-1"},
|
||||
ExcludeKnowledgeIDs: []string{"knowledge-9"},
|
||||
ExcludeChunkIDs: []string{"chunk-9"},
|
||||
})
|
||||
cond := filter.Cond()
|
||||
|
||||
for _, want := range []string{
|
||||
"is_enabled=1",
|
||||
"knowledge_base_id in (\"kb-1\")",
|
||||
"knowledge_id in (\"knowledge-1\",\"knowledge-2\")",
|
||||
"tag_id in (\"tag-1\")",
|
||||
"knowledge_id not in (\"knowledge-9\")",
|
||||
"chunk_id not in (\"chunk-9\")",
|
||||
} {
|
||||
assert.True(t, strings.Contains(cond, want), "condition %q should contain %q", cond, want)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package tencentvectordb
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/tencent/vectordatabase-sdk-go/tcvdbtext/encoder"
|
||||
"github.com/tencent/vectordatabase-sdk-go/tcvectordb"
|
||||
)
|
||||
|
||||
@@ -14,6 +15,7 @@ const (
|
||||
|
||||
fieldID = "id"
|
||||
fieldVector = "vector"
|
||||
fieldSparseVector = "sparse_vector"
|
||||
fieldContent = "content"
|
||||
fieldSourceID = "source_id"
|
||||
fieldSourceType = "source_type"
|
||||
@@ -31,6 +33,9 @@ type repository struct {
|
||||
shardsNum int
|
||||
replicasNum int
|
||||
initialized sync.Map
|
||||
bm25Once sync.Once
|
||||
bm25 encoder.SparseEncoder
|
||||
bm25Err error
|
||||
}
|
||||
|
||||
type vectorEmbedding struct {
|
||||
@@ -43,6 +48,7 @@ type vectorEmbedding struct {
|
||||
KnowledgeBaseID string
|
||||
TagID string
|
||||
Embedding []float32
|
||||
SparseVector []encoder.SparseVecItem
|
||||
IsEnabled bool
|
||||
Score float64
|
||||
}
|
||||
|
||||
@@ -45,6 +45,10 @@ var retrieverEngineMapping = map[string][]RetrieverEngineParams{
|
||||
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType},
|
||||
{RetrieverType: VectorRetrieverType, RetrieverEngineType: SQLiteRetrieverEngineType},
|
||||
},
|
||||
"tencent_vectordb": {
|
||||
{RetrieverType: KeywordsRetrieverType, RetrieverEngineType: TencentVectorDBRetrieverEngineType},
|
||||
{RetrieverType: VectorRetrieverType, RetrieverEngineType: TencentVectorDBRetrieverEngineType},
|
||||
},
|
||||
}
|
||||
|
||||
// GetRetrieverEngineMapping returns the retriever engine mapping
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestRetrieverEngineMappingIncludesTencentVectorDBHybridCapabilities(t *testing.T) {
|
||||
mapping := GetRetrieverEngineMapping()
|
||||
|
||||
assert.Contains(t, mapping["tencent_vectordb"], RetrieverEngineParams{
|
||||
RetrieverType: KeywordsRetrieverType,
|
||||
RetrieverEngineType: TencentVectorDBRetrieverEngineType,
|
||||
})
|
||||
assert.Contains(t, mapping["tencent_vectordb"], RetrieverEngineParams{
|
||||
RetrieverType: VectorRetrieverType,
|
||||
RetrieverEngineType: TencentVectorDBRetrieverEngineType,
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user