From 3eb49da7461d56c5a65524903bc2507e4bc220c4 Mon Sep 17 00:00:00 2001 From: langcaiye Date: Mon, 22 Jun 2026 19:34:23 +0800 Subject: [PATCH] feat(retriever): configure Tencent VectorDB replicas via env --- .env.example | 3 ++ docs/api/vector-store.md | 7 ++-- docs/wiki/集成扩展/集成向量数据库.md | 2 ++ docs/使用其他向量数据库.md | 2 ++ .../retriever/tencentvectordb/repository.go | 16 +++++++++- .../tencentvectordb/repository_test.go | 15 +++++++++ .../retriever/tencentvectordb/structs.go | 2 ++ internal/types/vectorstore.go | 32 ++++++++++++++++--- internal/types/vectorstore_test.go | 20 ++++++++++++ 9 files changed, 91 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 2a5e63fb..b49f589e 100644 --- a/.env.example +++ b/.env.example @@ -337,6 +337,9 @@ CONCURRENCY_POOL_SIZE=5 # Tencent VectorDB 集合名前缀（可选，默认 weknora_embeddings；实际集合会按向量维度追加后缀） # TENCENT_VECTORDB_COLLECTION=weknora_embeddings +# Tencent VectorDB 集合副本数（可选，默认 1；单节点 QA 环境可设为 0） +# TENCENT_VECTORDB_REPLICA_NUMBER=1 + # 如果使用MinIO作为文件存储，需要配置以下参数 # MinIO访问端点（host:port），连接外部MinIO时需修改 # diff --git a/docs/api/vector-store.md b/docs/api/vector-store.md index 3aa7342f..02b32cab 100644 --- a/docs/api/vector-store.md +++ b/docs/api/vector-store.md @@ -120,7 +120,7 @@ curl --location --request POST 'http://localhost:8080/api/v1/vector-stores/test' | connection_config | object | 是 | 连接配置（与所选引擎的 `connection_fields` 对应） | | index_config | object | 否 | 索引配置（与所选引擎的 `index_fields` 对应） | -> Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 必填，`database` 可选；`index_config.collection_name` 表示集合名前缀，实际集合会按向量维度追加后缀（例如 `weknora_embeddings_768`）。该适配器同时支持向量检索和基于 BM25 sparse vector 的关键词检索；旧版本已创建且没有 `sparse_vector` 索引的集合需要重建并重新导入数据后才能启用关键词检索。 +> Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 必填，`database` 可选；`index_config.collection_name` 表示集合名前缀，实际集合会按向量维度追加后缀（例如 `weknora_embeddings_768`）；`index_config.replica_number` 表示创建集合时使用的副本数。该适配器同时支持向量检索和基于 BM25 sparse vector 的关键词检索；旧版本已创建且没有 `sparse_vector` 索引的集合需要重建并重新导入数据后才能启用关键词检索。 **请求**: @@ -158,7 +158,8 @@ curl --location 'http://localhost:8080/api/v1/vector-stores' \ "database": "weknora" }, "index_config": { - "collection_name": "weknora_embeddings" + "collection_name": "weknora_embeddings", + "replica_number": 1 } }' ``` @@ -419,6 +420,8 @@ curl --location --request POST 'http://localhost:8080/api/v1/vector-stores/550e8 - **可测试连通性**：`POST /vector-stores/:id/test` 正常工作 - **被知识库绑定时**：未指定 `vector_store_id` 创建的知识库默认使用环境变量存储；这种知识库在响应中显示为 `vector_store_name="System default"` + `vector_store_source="env"`。 +Tencent VectorDB 环境变量存储可通过 `TENCENT_VECTORDB_REPLICA_NUMBER` 覆盖默认集合副本数。默认值为 `1`；单节点 QA 环境可设为 `0`，生产环境可按 Tencent VectorDB 集群规模调整。 + ## 错误码 | HTTP 状态码 | code | 含义 | diff --git a/docs/wiki/集成扩展/集成向量数据库.md b/docs/wiki/集成扩展/集成向量数据库.md index ee778528..dfd5b620 100644 --- a/docs/wiki/集成扩展/集成向量数据库.md +++ b/docs/wiki/集成扩展/集成向量数据库.md @@ -99,9 +99,11 @@ TENCENT_VECTORDB_USERNAME=root TENCENT_VECTORDB_API_KEY=your_tencent_vectordb_api_key TENCENT_VECTORDB_DATABASE=weknora TENCENT_VECTORDB_COLLECTION=weknora_embeddings +TENCENT_VECTORDB_REPLICA_NUMBER=1 ``` `TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合，例如 `weknora_embeddings_768`。 +`TENCENT_VECTORDB_REPLICA_NUMBER` 是创建集合时使用的副本数，默认 `1`；单节点 QA 环境可设为 `0`，生产环境可按 Tencent VectorDB 集群规模调整。 关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引；旧版本已创建的向量集合如果没有该索引，需要重建集合并重新导入知识库数据后才能启用关键词检索。 diff --git a/docs/使用其他向量数据库.md b/docs/使用其他向量数据库.md index d21ba338..f941810b 100644 --- a/docs/使用其他向量数据库.md +++ b/docs/使用其他向量数据库.md @@ -289,8 +289,10 @@ TENCENT_VECTORDB_USERNAME=root TENCENT_VECTORDB_API_KEY=your_tencent_vectordb_api_key TENCENT_VECTORDB_DATABASE=weknora TENCENT_VECTORDB_COLLECTION=weknora_embeddings +TENCENT_VECTORDB_REPLICA_NUMBER=1 ``` `TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合，例如 `weknora_embeddings_768`，用于隔离不同 embedding 模型维度的数据。 +`TENCENT_VECTORDB_REPLICA_NUMBER` 是创建集合时使用的副本数，默认 `1`；单节点 QA 环境可设为 `0`，生产环境可按 Tencent VectorDB 集群规模调整。 关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引；旧版本已创建的向量集合如果没有该索引，需要重建集合并重新导入知识库数据后才能启用关键词检索。 diff --git a/internal/application/repository/retriever/tencentvectordb/repository.go b/internal/application/repository/retriever/tencentvectordb/repository.go index 3bca58cf..54f6a19f 100644 --- a/internal/application/repository/retriever/tencentvectordb/repository.go +++ b/internal/application/repository/retriever/tencentvectordb/repository.go @@ -9,6 +9,7 @@ import ( "os" "slices" "sort" + "strconv" "strings" "unicode/utf8" @@ -41,10 +42,23 @@ func NewTencentVectorDBRetrieveEngineRepository( collectionBaseName: collectionBaseName, useDimensionSuffix: shouldUseDimensionSuffix(indexCfg), shardsNum: defaultIfZero(indexCfg.GetShardsNum(1), 1), - replicasNum: defaultIfZero(indexCfg.GetReplicaNumber(1), 1), + replicasNum: resolveReplicaNumber(indexCfg), } } +func resolveReplicaNumber(indexCfg *types.IndexConfig) int { + if indexCfg != nil && indexCfg.ReplicaNumber > 0 { + return indexCfg.ReplicaNumber + } + if raw := strings.TrimSpace(os.Getenv(envTencentVectorDBReplicaNum)); raw != "" { + replicas, err := strconv.Atoi(raw) + if err == nil && replicas >= 0 { + return replicas + } + } + return defaultReplicaNumber +} + func (r *repository) EngineType() types.RetrieverEngineType { return types.TencentVectorDBRetrieverEngineType } diff --git a/internal/application/repository/retriever/tencentvectordb/repository_test.go b/internal/application/repository/retriever/tencentvectordb/repository_test.go index 556c1753..deb56520 100644 --- a/internal/application/repository/retriever/tencentvectordb/repository_test.go +++ b/internal/application/repository/retriever/tencentvectordb/repository_test.go @@ -76,7 +76,22 @@ func TestTencentVectorDBDefaultsToReplicaNumberOne(t *testing.T) { assert.Equal(t, 1, repo.replicasNum) } +func TestTencentVectorDBUsesEnvReplicaNumber(t *testing.T) { + t.Setenv(envTencentVectorDBReplicaNum, "0") + repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", nil).(*repository) + + assert.Equal(t, 0, repo.replicasNum) +} + +func TestTencentVectorDBUsesPositiveEnvReplicaNumber(t *testing.T) { + t.Setenv(envTencentVectorDBReplicaNum, "3") + repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", nil).(*repository) + + assert.Equal(t, 3, repo.replicasNum) +} + func TestTencentVectorDBUsesConfiguredReplicaNumber(t *testing.T) { + t.Setenv(envTencentVectorDBReplicaNum, "0") repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", &types.IndexConfig{ ReplicaNumber: 2, }).(*repository) diff --git a/internal/application/repository/retriever/tencentvectordb/structs.go b/internal/application/repository/retriever/tencentvectordb/structs.go index 51c0e9ed..c15a0480 100644 --- a/internal/application/repository/retriever/tencentvectordb/structs.go +++ b/internal/application/repository/retriever/tencentvectordb/structs.go @@ -10,8 +10,10 @@ import ( const ( envTencentVectorDBDatabase = "TENCENT_VECTORDB_DATABASE" envTencentVectorDBCollection = "TENCENT_VECTORDB_COLLECTION" + envTencentVectorDBReplicaNum = "TENCENT_VECTORDB_REPLICA_NUMBER" defaultDatabaseName = "weknora" defaultCollectionName = "weknora_embeddings" + defaultReplicaNumber = 1 fieldID = "id" fieldVector = "vector" diff --git a/internal/types/vectorstore.go b/internal/types/vectorstore.go index 0c758dbe..2cd74794 100644 --- a/internal/types/vectorstore.go +++ b/internal/types/vectorstore.go @@ -19,6 +19,11 @@ import ( // EnvStoreIDPrefix is the prefix for virtual env store IDs. const EnvStoreIDPrefix = "__env_" +const ( + envTencentVectorDBReplicaNumber = "TENCENT_VECTORDB_REPLICA_NUMBER" + defaultTencentVectorDBReplicaNumber = 1 +) + // IsEnvStoreID checks if the given ID is an env store virtual ID. func IsEnvStoreID(id string) bool { return strings.HasPrefix(id, EnvStoreIDPrefix) @@ -248,7 +253,7 @@ type IndexConfig struct { ShardNumber int `yaml:"shard_number" json:"shard_number,omitempty"` // Qdrant: number of shards per collection ReplicationFactor int `yaml:"replication_factor" json:"replication_factor,omitempty"` // Qdrant, Weaviate: number of replicas ShardsNum int `yaml:"shards_num" json:"shards_num,omitempty"` // Milvus: number of shards per collection (CreateCollection) - ReplicaNumber int `yaml:"replica_number" json:"replica_number,omitempty"` // Milvus: in-memory replica count (LoadCollection) + ReplicaNumber int `yaml:"replica_number" json:"replica_number,omitempty"` // Milvus LoadCollection / Tencent VectorDB CreateCollection replicas DesiredShardCount int `yaml:"desired_shard_count" json:"desired_shard_count,omitempty"` // Weaviate: number of shards per collection BucketsNum int `yaml:"buckets_num" json:"buckets_num,omitempty"` // Doris: number of buckets per table (DISTRIBUTED BY HASH ... BUCKETS N) ReplicationNum int `yaml:"replication_num" json:"replication_num,omitempty"` // Doris: replication_num PROPERTIES @@ -370,9 +375,9 @@ func (c *IndexConfig) GetShardsNum(def int) int { return def } -// GetReplicaNumber returns the configured replica_number (Milvus in-memory replicas), or def if unset/zero. -// Milvus replicas are set at LoadCollection time, not CreateCollection. -// They control how many query nodes hold the data in memory for read HA/throughput. +// GetReplicaNumber returns the configured replica_number, or def if unset/zero. +// Milvus applies it at LoadCollection time; Tencent VectorDB applies it at +// CreateCollection time. It controls read HA/throughput replicas. func (c *IndexConfig) GetReplicaNumber(def int) int { if c != nil && c.ReplicaNumber > 0 { return c.ReplicaNumber @@ -610,6 +615,21 @@ type VectorStoreTypeInfo struct { IndexFields []VectorStoreFieldInfo `json:"index_fields,omitempty"` } +func resolveTencentVectorDBReplicaNumber(lookup EnvLookupFunc) int { + if lookup == nil { + lookup = os.Getenv + } + raw := strings.TrimSpace(lookup(envTencentVectorDBReplicaNumber)) + if raw == "" { + return defaultTencentVectorDBReplicaNumber + } + replicas, err := strconv.Atoi(raw) + if err != nil || replicas < 0 { + return defaultTencentVectorDBReplicaNumber + } + return replicas +} + // VectorStoreFieldInfo describes a single configuration field exposed // by /api/v1/vector-stores/types for the registration UI. The optional // validation hints (`Immutable`, `Min`, `Max`, `Enum`) are used by both @@ -644,6 +664,8 @@ type VectorStoreFieldInfo struct { // GetVectorStoreTypes returns metadata for all supported engine types. func GetVectorStoreTypes() []VectorStoreTypeInfo { + tencentVectorDBReplicaNumber := resolveTencentVectorDBReplicaNumber(os.Getenv) + return []VectorStoreTypeInfo{ { Type: "elasticsearch", @@ -704,7 +726,7 @@ func GetVectorStoreTypes() []VectorStoreTypeInfo { IndexFields: []VectorStoreFieldInfo{ {Name: "collection_name", Type: "string", Required: false, Description: "Collection Name", Default: "weknora_embeddings"}, {Name: "shards_num", Type: "number", Required: false, Description: "Shards", Default: 1}, - {Name: "replica_number", Type: "number", Required: false, Description: "Replicas", Default: 1}, + {Name: "replica_number", Type: "number", Required: false, Description: "Replicas", Default: tencentVectorDBReplicaNumber}, }, }, { diff --git a/internal/types/vectorstore_test.go b/internal/types/vectorstore_test.go index aaf70286..f21ad2e5 100644 --- a/internal/types/vectorstore_test.go +++ b/internal/types/vectorstore_test.go @@ -344,6 +344,26 @@ func TestGetVectorStoreTypes(t *testing.T) { assert.Equal(t, 1, seen["replica_number"].Default) }) + t.Run("tencent vectordb replica default follows env", func(t *testing.T) { + t.Setenv(envTencentVectorDBReplicaNumber, "0") + types := GetVectorStoreTypes() + + var tencentType VectorStoreTypeInfo + for _, typ := range types { + if typ.Type == "tencent_vectordb" { + tencentType = typ + break + } + } + require.NotEmpty(t, tencentType.IndexFields) + + seen := map[string]VectorStoreFieldInfo{} + for _, f := range tencentType.IndexFields { + seen[f.Name] = f + } + assert.Equal(t, 0, seen["replica_number"].Default) + }) + t.Run("display names have no parenthetical suffix", func(t *testing.T) { for _, typ := range types { assert.NotContains(t, typ.DisplayName, "(", "display_name should not contain parenthetical suffix: %s", typ.DisplayName)