feat(retriever): configure Tencent VectorDB replicas via env

This commit is contained in:
langcaiye
2026-06-22 19:34:23 +08:00
committed by lyingbug
parent 7f3f219f58
commit 3eb49da746
9 changed files with 91 additions and 8 deletions
+3
View File
@@ -337,6 +337,9 @@ CONCURRENCY_POOL_SIZE=5
# Tencent VectorDB 集合名前缀(可选,默认 weknora_embeddings;实际集合会按向量维度追加后缀)
# TENCENT_VECTORDB_COLLECTION=weknora_embeddings
# Tencent VectorDB 集合副本数(可选,默认 1;单节点 QA 环境可设为 0)
# TENCENT_VECTORDB_REPLICA_NUMBER=1
# 如果使用MinIO作为文件存储,需要配置以下参数
# MinIO访问端点(host:port),连接外部MinIO时需修改
#
+5 -2
View File
@@ -120,7 +120,7 @@ curl --location --request POST 'http://localhost:8080/api/v1/vector-stores/test'
| connection_config | object | 是 | 连接配置(与所选引擎的 `connection_fields` 对应) |
| index_config | object | 否 | 索引配置(与所选引擎的 `index_fields` 对应) |
> Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 必填,`database` 可选;`index_config.collection_name` 表示集合名前缀,实际集合会按向量维度追加后缀(例如 `weknora_embeddings_768`)。该适配器同时支持向量检索和基于 BM25 sparse vector 的关键词检索;旧版本已创建且没有 `sparse_vector` 索引的集合需要重建并重新导入数据后才能启用关键词检索。
> Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 必填,`database` 可选;`index_config.collection_name` 表示集合名前缀,实际集合会按向量维度追加后缀(例如 `weknora_embeddings_768`);`index_config.replica_number` 表示创建集合时使用的副本数。该适配器同时支持向量检索和基于 BM25 sparse vector 的关键词检索;旧版本已创建且没有 `sparse_vector` 索引的集合需要重建并重新导入数据后才能启用关键词检索。
**请求**:
@@ -158,7 +158,8 @@ curl --location 'http://localhost:8080/api/v1/vector-stores' \
"database": "weknora"
},
"index_config": {
"collection_name": "weknora_embeddings"
"collection_name": "weknora_embeddings",
"replica_number": 1
}
}'
```
@@ -419,6 +420,8 @@ curl --location --request POST 'http://localhost:8080/api/v1/vector-stores/550e8
- **可测试连通性**:`POST /vector-stores/:id/test` 正常工作
- **被知识库绑定时**:未指定 `vector_store_id` 创建的知识库默认使用环境变量存储;这种知识库在响应中显示为 `vector_store_name="System default"` + `vector_store_source="env"`。
Tencent VectorDB 环境变量存储可通过 `TENCENT_VECTORDB_REPLICA_NUMBER` 覆盖默认集合副本数。默认值为 `1`;单节点 QA 环境可设为 `0`,生产环境可按 Tencent VectorDB 集群规模调整。
## 错误码
| HTTP 状态码 | code | 含义 |
@@ -99,9 +99,11 @@ TENCENT_VECTORDB_USERNAME=root
TENCENT_VECTORDB_API_KEY=your_tencent_vectordb_api_key
TENCENT_VECTORDB_DATABASE=weknora
TENCENT_VECTORDB_COLLECTION=weknora_embeddings
TENCENT_VECTORDB_REPLICA_NUMBER=1
```
`TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合,例如 `weknora_embeddings_768`。
`TENCENT_VECTORDB_REPLICA_NUMBER` 是创建集合时使用的副本数,默认 `1`;单节点 QA 环境可设为 `0`,生产环境可按 Tencent VectorDB 集群规模调整。
关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引;旧版本已创建的向量集合如果没有该索引,需要重建集合并重新导入知识库数据后才能启用关键词检索。
+2
View File
@@ -289,8 +289,10 @@ TENCENT_VECTORDB_USERNAME=root
TENCENT_VECTORDB_API_KEY=your_tencent_vectordb_api_key
TENCENT_VECTORDB_DATABASE=weknora
TENCENT_VECTORDB_COLLECTION=weknora_embeddings
TENCENT_VECTORDB_REPLICA_NUMBER=1
```
`TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合,例如 `weknora_embeddings_768`,用于隔离不同 embedding 模型维度的数据。
`TENCENT_VECTORDB_REPLICA_NUMBER` 是创建集合时使用的副本数,默认 `1`;单节点 QA 环境可设为 `0`,生产环境可按 Tencent VectorDB 集群规模调整。
关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引;旧版本已创建的向量集合如果没有该索引,需要重建集合并重新导入知识库数据后才能启用关键词检索。
@@ -9,6 +9,7 @@ import (
"os"
"slices"
"sort"
"strconv"
"strings"
"unicode/utf8"
@@ -41,10 +42,23 @@ func NewTencentVectorDBRetrieveEngineRepository(
collectionBaseName: collectionBaseName,
useDimensionSuffix: shouldUseDimensionSuffix(indexCfg),
shardsNum: defaultIfZero(indexCfg.GetShardsNum(1), 1),
replicasNum: defaultIfZero(indexCfg.GetReplicaNumber(1), 1),
replicasNum: resolveReplicaNumber(indexCfg),
}
}
func resolveReplicaNumber(indexCfg *types.IndexConfig) int {
if indexCfg != nil && indexCfg.ReplicaNumber > 0 {
return indexCfg.ReplicaNumber
}
if raw := strings.TrimSpace(os.Getenv(envTencentVectorDBReplicaNum)); raw != "" {
replicas, err := strconv.Atoi(raw)
if err == nil && replicas >= 0 {
return replicas
}
}
return defaultReplicaNumber
}
func (r *repository) EngineType() types.RetrieverEngineType {
return types.TencentVectorDBRetrieverEngineType
}
@@ -76,7 +76,22 @@ func TestTencentVectorDBDefaultsToReplicaNumberOne(t *testing.T) {
assert.Equal(t, 1, repo.replicasNum)
}
func TestTencentVectorDBUsesEnvReplicaNumber(t *testing.T) {
t.Setenv(envTencentVectorDBReplicaNum, "0")
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", nil).(*repository)
assert.Equal(t, 0, repo.replicasNum)
}
func TestTencentVectorDBUsesPositiveEnvReplicaNumber(t *testing.T) {
t.Setenv(envTencentVectorDBReplicaNum, "3")
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", nil).(*repository)
assert.Equal(t, 3, repo.replicasNum)
}
func TestTencentVectorDBUsesConfiguredReplicaNumber(t *testing.T) {
t.Setenv(envTencentVectorDBReplicaNum, "0")
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", &types.IndexConfig{
ReplicaNumber: 2,
}).(*repository)
@@ -10,8 +10,10 @@ import (
const (
envTencentVectorDBDatabase = "TENCENT_VECTORDB_DATABASE"
envTencentVectorDBCollection = "TENCENT_VECTORDB_COLLECTION"
envTencentVectorDBReplicaNum = "TENCENT_VECTORDB_REPLICA_NUMBER"
defaultDatabaseName = "weknora"
defaultCollectionName = "weknora_embeddings"
defaultReplicaNumber = 1
fieldID = "id"
fieldVector = "vector"
+27 -5
View File
@@ -19,6 +19,11 @@ import (
// EnvStoreIDPrefix is the prefix for virtual env store IDs.
const EnvStoreIDPrefix = "__env_"
const (
envTencentVectorDBReplicaNumber = "TENCENT_VECTORDB_REPLICA_NUMBER"
defaultTencentVectorDBReplicaNumber = 1
)
// IsEnvStoreID checks if the given ID is an env store virtual ID.
func IsEnvStoreID(id string) bool {
return strings.HasPrefix(id, EnvStoreIDPrefix)
@@ -248,7 +253,7 @@ type IndexConfig struct {
ShardNumber int `yaml:"shard_number" json:"shard_number,omitempty"` // Qdrant: number of shards per collection
ReplicationFactor int `yaml:"replication_factor" json:"replication_factor,omitempty"` // Qdrant, Weaviate: number of replicas
ShardsNum int `yaml:"shards_num" json:"shards_num,omitempty"` // Milvus: number of shards per collection (CreateCollection)
ReplicaNumber int `yaml:"replica_number" json:"replica_number,omitempty"` // Milvus: in-memory replica count (LoadCollection)
ReplicaNumber int `yaml:"replica_number" json:"replica_number,omitempty"` // Milvus LoadCollection / Tencent VectorDB CreateCollection replicas
DesiredShardCount int `yaml:"desired_shard_count" json:"desired_shard_count,omitempty"` // Weaviate: number of shards per collection
BucketsNum int `yaml:"buckets_num" json:"buckets_num,omitempty"` // Doris: number of buckets per table (DISTRIBUTED BY HASH ... BUCKETS N)
ReplicationNum int `yaml:"replication_num" json:"replication_num,omitempty"` // Doris: replication_num PROPERTIES
@@ -370,9 +375,9 @@ func (c *IndexConfig) GetShardsNum(def int) int {
return def
}
// GetReplicaNumber returns the configured replica_number (Milvus in-memory replicas), or def if unset/zero.
// Milvus replicas are set at LoadCollection time, not CreateCollection.
// They control how many query nodes hold the data in memory for read HA/throughput.
// GetReplicaNumber returns the configured replica_number, or def if unset/zero.
// Milvus applies it at LoadCollection time; Tencent VectorDB applies it at
// CreateCollection time. It controls read HA/throughput replicas.
func (c *IndexConfig) GetReplicaNumber(def int) int {
if c != nil && c.ReplicaNumber > 0 {
return c.ReplicaNumber
@@ -610,6 +615,21 @@ type VectorStoreTypeInfo struct {
IndexFields []VectorStoreFieldInfo `json:"index_fields,omitempty"`
}
func resolveTencentVectorDBReplicaNumber(lookup EnvLookupFunc) int {
if lookup == nil {
lookup = os.Getenv
}
raw := strings.TrimSpace(lookup(envTencentVectorDBReplicaNumber))
if raw == "" {
return defaultTencentVectorDBReplicaNumber
}
replicas, err := strconv.Atoi(raw)
if err != nil || replicas < 0 {
return defaultTencentVectorDBReplicaNumber
}
return replicas
}
// VectorStoreFieldInfo describes a single configuration field exposed
// by /api/v1/vector-stores/types for the registration UI. The optional
// validation hints (`Immutable`, `Min`, `Max`, `Enum`) are used by both
@@ -644,6 +664,8 @@ type VectorStoreFieldInfo struct {
// GetVectorStoreTypes returns metadata for all supported engine types.
func GetVectorStoreTypes() []VectorStoreTypeInfo {
tencentVectorDBReplicaNumber := resolveTencentVectorDBReplicaNumber(os.Getenv)
return []VectorStoreTypeInfo{
{
Type: "elasticsearch",
@@ -704,7 +726,7 @@ func GetVectorStoreTypes() []VectorStoreTypeInfo {
IndexFields: []VectorStoreFieldInfo{
{Name: "collection_name", Type: "string", Required: false, Description: "Collection Name", Default: "weknora_embeddings"},
{Name: "shards_num", Type: "number", Required: false, Description: "Shards", Default: 1},
{Name: "replica_number", Type: "number", Required: false, Description: "Replicas", Default: 1},
{Name: "replica_number", Type: "number", Required: false, Description: "Replicas", Default: tencentVectorDBReplicaNumber},
},
},
{
+20
View File
@@ -344,6 +344,26 @@ func TestGetVectorStoreTypes(t *testing.T) {
assert.Equal(t, 1, seen["replica_number"].Default)
})
t.Run("tencent vectordb replica default follows env", func(t *testing.T) {
t.Setenv(envTencentVectorDBReplicaNumber, "0")
types := GetVectorStoreTypes()
var tencentType VectorStoreTypeInfo
for _, typ := range types {
if typ.Type == "tencent_vectordb" {
tencentType = typ
break
}
}
require.NotEmpty(t, tencentType.IndexFields)
seen := map[string]VectorStoreFieldInfo{}
for _, f := range tencentType.IndexFields {
seen[f.Name] = f
}
assert.Equal(t, 0, seen["replica_number"].Default)
})
t.Run("display names have no parenthetical suffix", func(t *testing.T) {
for _, typ := range types {
assert.NotContains(t, typ.DisplayName, "(", "display_name should not contain parenthetical suffix: %s", typ.DisplayName)