feat(retriever): configure Tencent VectorDB replicas via env
This commit is contained in:
@@ -337,6 +337,9 @@ CONCURRENCY_POOL_SIZE=5
|
||||
# Tencent VectorDB 集合名前缀(可选,默认 weknora_embeddings;实际集合会按向量维度追加后缀)
|
||||
# TENCENT_VECTORDB_COLLECTION=weknora_embeddings
|
||||
|
||||
# Tencent VectorDB 集合副本数(可选,默认 1;单节点 QA 环境可设为 0)
|
||||
# TENCENT_VECTORDB_REPLICA_NUMBER=1
|
||||
|
||||
# 如果使用MinIO作为文件存储,需要配置以下参数
|
||||
# MinIO访问端点(host:port),连接外部MinIO时需修改
|
||||
#
|
||||
|
||||
@@ -120,7 +120,7 @@ curl --location --request POST 'http://localhost:8080/api/v1/vector-stores/test'
|
||||
| connection_config | object | 是 | 连接配置(与所选引擎的 `connection_fields` 对应) |
|
||||
| index_config | object | 否 | 索引配置(与所选引擎的 `index_fields` 对应) |
|
||||
|
||||
> Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 必填,`database` 可选;`index_config.collection_name` 表示集合名前缀,实际集合会按向量维度追加后缀(例如 `weknora_embeddings_768`)。该适配器同时支持向量检索和基于 BM25 sparse vector 的关键词检索;旧版本已创建且没有 `sparse_vector` 索引的集合需要重建并重新导入数据后才能启用关键词检索。
|
||||
> Tencent VectorDB 使用 `engine_type: "tencent_vectordb"`。`connection_config` 中 `addr`、`username`、`api_key` 必填,`database` 可选;`index_config.collection_name` 表示集合名前缀,实际集合会按向量维度追加后缀(例如 `weknora_embeddings_768`);`index_config.replica_number` 表示创建集合时使用的副本数。该适配器同时支持向量检索和基于 BM25 sparse vector 的关键词检索;旧版本已创建且没有 `sparse_vector` 索引的集合需要重建并重新导入数据后才能启用关键词检索。
|
||||
|
||||
**请求**:
|
||||
|
||||
@@ -158,7 +158,8 @@ curl --location 'http://localhost:8080/api/v1/vector-stores' \
|
||||
"database": "weknora"
|
||||
},
|
||||
"index_config": {
|
||||
"collection_name": "weknora_embeddings"
|
||||
"collection_name": "weknora_embeddings",
|
||||
"replica_number": 1
|
||||
}
|
||||
}'
|
||||
```
|
||||
@@ -419,6 +420,8 @@ curl --location --request POST 'http://localhost:8080/api/v1/vector-stores/550e8
|
||||
- **可测试连通性**:`POST /vector-stores/:id/test` 正常工作
|
||||
- **被知识库绑定时**:未指定 `vector_store_id` 创建的知识库默认使用环境变量存储;这种知识库在响应中显示为 `vector_store_name="System default"` + `vector_store_source="env"`。
|
||||
|
||||
Tencent VectorDB 环境变量存储可通过 `TENCENT_VECTORDB_REPLICA_NUMBER` 覆盖默认集合副本数。默认值为 `1`;单节点 QA 环境可设为 `0`,生产环境可按 Tencent VectorDB 集群规模调整。
|
||||
|
||||
## 错误码
|
||||
|
||||
| HTTP 状态码 | code | 含义 |
|
||||
|
||||
@@ -99,9 +99,11 @@ TENCENT_VECTORDB_USERNAME=root
|
||||
TENCENT_VECTORDB_API_KEY=your_tencent_vectordb_api_key
|
||||
TENCENT_VECTORDB_DATABASE=weknora
|
||||
TENCENT_VECTORDB_COLLECTION=weknora_embeddings
|
||||
TENCENT_VECTORDB_REPLICA_NUMBER=1
|
||||
```
|
||||
|
||||
`TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合,例如 `weknora_embeddings_768`。
|
||||
`TENCENT_VECTORDB_REPLICA_NUMBER` 是创建集合时使用的副本数,默认 `1`;单节点 QA 环境可设为 `0`,生产环境可按 Tencent VectorDB 集群规模调整。
|
||||
|
||||
关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引;旧版本已创建的向量集合如果没有该索引,需要重建集合并重新导入知识库数据后才能启用关键词检索。
|
||||
|
||||
|
||||
@@ -289,8 +289,10 @@ TENCENT_VECTORDB_USERNAME=root
|
||||
TENCENT_VECTORDB_API_KEY=your_tencent_vectordb_api_key
|
||||
TENCENT_VECTORDB_DATABASE=weknora
|
||||
TENCENT_VECTORDB_COLLECTION=weknora_embeddings
|
||||
TENCENT_VECTORDB_REPLICA_NUMBER=1
|
||||
```
|
||||
|
||||
`TENCENT_VECTORDB_COLLECTION` 是集合名前缀。WeKnora 会按向量维度创建实际集合,例如 `weknora_embeddings_768`,用于隔离不同 embedding 模型维度的数据。
|
||||
`TENCENT_VECTORDB_REPLICA_NUMBER` 是创建集合时使用的副本数,默认 `1`;单节点 QA 环境可设为 `0`,生产环境可按 Tencent VectorDB 集群规模调整。
|
||||
|
||||
关键词检索依赖 Tencent VectorDB sparse vector 索引。新建集合会自动创建 `sparse_vector` 索引;旧版本已创建的向量集合如果没有该索引,需要重建集合并重新导入知识库数据后才能启用关键词检索。
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
@@ -41,10 +42,23 @@ func NewTencentVectorDBRetrieveEngineRepository(
|
||||
collectionBaseName: collectionBaseName,
|
||||
useDimensionSuffix: shouldUseDimensionSuffix(indexCfg),
|
||||
shardsNum: defaultIfZero(indexCfg.GetShardsNum(1), 1),
|
||||
replicasNum: defaultIfZero(indexCfg.GetReplicaNumber(1), 1),
|
||||
replicasNum: resolveReplicaNumber(indexCfg),
|
||||
}
|
||||
}
|
||||
|
||||
func resolveReplicaNumber(indexCfg *types.IndexConfig) int {
|
||||
if indexCfg != nil && indexCfg.ReplicaNumber > 0 {
|
||||
return indexCfg.ReplicaNumber
|
||||
}
|
||||
if raw := strings.TrimSpace(os.Getenv(envTencentVectorDBReplicaNum)); raw != "" {
|
||||
replicas, err := strconv.Atoi(raw)
|
||||
if err == nil && replicas >= 0 {
|
||||
return replicas
|
||||
}
|
||||
}
|
||||
return defaultReplicaNumber
|
||||
}
|
||||
|
||||
func (r *repository) EngineType() types.RetrieverEngineType {
|
||||
return types.TencentVectorDBRetrieverEngineType
|
||||
}
|
||||
|
||||
@@ -76,7 +76,22 @@ func TestTencentVectorDBDefaultsToReplicaNumberOne(t *testing.T) {
|
||||
assert.Equal(t, 1, repo.replicasNum)
|
||||
}
|
||||
|
||||
func TestTencentVectorDBUsesEnvReplicaNumber(t *testing.T) {
|
||||
t.Setenv(envTencentVectorDBReplicaNum, "0")
|
||||
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", nil).(*repository)
|
||||
|
||||
assert.Equal(t, 0, repo.replicasNum)
|
||||
}
|
||||
|
||||
func TestTencentVectorDBUsesPositiveEnvReplicaNumber(t *testing.T) {
|
||||
t.Setenv(envTencentVectorDBReplicaNum, "3")
|
||||
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", nil).(*repository)
|
||||
|
||||
assert.Equal(t, 3, repo.replicasNum)
|
||||
}
|
||||
|
||||
func TestTencentVectorDBUsesConfiguredReplicaNumber(t *testing.T) {
|
||||
t.Setenv(envTencentVectorDBReplicaNum, "0")
|
||||
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", &types.IndexConfig{
|
||||
ReplicaNumber: 2,
|
||||
}).(*repository)
|
||||
|
||||
@@ -10,8 +10,10 @@ import (
|
||||
const (
|
||||
envTencentVectorDBDatabase = "TENCENT_VECTORDB_DATABASE"
|
||||
envTencentVectorDBCollection = "TENCENT_VECTORDB_COLLECTION"
|
||||
envTencentVectorDBReplicaNum = "TENCENT_VECTORDB_REPLICA_NUMBER"
|
||||
defaultDatabaseName = "weknora"
|
||||
defaultCollectionName = "weknora_embeddings"
|
||||
defaultReplicaNumber = 1
|
||||
|
||||
fieldID = "id"
|
||||
fieldVector = "vector"
|
||||
|
||||
@@ -19,6 +19,11 @@ import (
|
||||
// EnvStoreIDPrefix is the prefix for virtual env store IDs.
|
||||
const EnvStoreIDPrefix = "__env_"
|
||||
|
||||
const (
|
||||
envTencentVectorDBReplicaNumber = "TENCENT_VECTORDB_REPLICA_NUMBER"
|
||||
defaultTencentVectorDBReplicaNumber = 1
|
||||
)
|
||||
|
||||
// IsEnvStoreID checks if the given ID is an env store virtual ID.
|
||||
func IsEnvStoreID(id string) bool {
|
||||
return strings.HasPrefix(id, EnvStoreIDPrefix)
|
||||
@@ -248,7 +253,7 @@ type IndexConfig struct {
|
||||
ShardNumber int `yaml:"shard_number" json:"shard_number,omitempty"` // Qdrant: number of shards per collection
|
||||
ReplicationFactor int `yaml:"replication_factor" json:"replication_factor,omitempty"` // Qdrant, Weaviate: number of replicas
|
||||
ShardsNum int `yaml:"shards_num" json:"shards_num,omitempty"` // Milvus: number of shards per collection (CreateCollection)
|
||||
ReplicaNumber int `yaml:"replica_number" json:"replica_number,omitempty"` // Milvus: in-memory replica count (LoadCollection)
|
||||
ReplicaNumber int `yaml:"replica_number" json:"replica_number,omitempty"` // Milvus LoadCollection / Tencent VectorDB CreateCollection replicas
|
||||
DesiredShardCount int `yaml:"desired_shard_count" json:"desired_shard_count,omitempty"` // Weaviate: number of shards per collection
|
||||
BucketsNum int `yaml:"buckets_num" json:"buckets_num,omitempty"` // Doris: number of buckets per table (DISTRIBUTED BY HASH ... BUCKETS N)
|
||||
ReplicationNum int `yaml:"replication_num" json:"replication_num,omitempty"` // Doris: replication_num PROPERTIES
|
||||
@@ -370,9 +375,9 @@ func (c *IndexConfig) GetShardsNum(def int) int {
|
||||
return def
|
||||
}
|
||||
|
||||
// GetReplicaNumber returns the configured replica_number (Milvus in-memory replicas), or def if unset/zero.
|
||||
// Milvus replicas are set at LoadCollection time, not CreateCollection.
|
||||
// They control how many query nodes hold the data in memory for read HA/throughput.
|
||||
// GetReplicaNumber returns the configured replica_number, or def if unset/zero.
|
||||
// Milvus applies it at LoadCollection time; Tencent VectorDB applies it at
|
||||
// CreateCollection time. It controls read HA/throughput replicas.
|
||||
func (c *IndexConfig) GetReplicaNumber(def int) int {
|
||||
if c != nil && c.ReplicaNumber > 0 {
|
||||
return c.ReplicaNumber
|
||||
@@ -610,6 +615,21 @@ type VectorStoreTypeInfo struct {
|
||||
IndexFields []VectorStoreFieldInfo `json:"index_fields,omitempty"`
|
||||
}
|
||||
|
||||
func resolveTencentVectorDBReplicaNumber(lookup EnvLookupFunc) int {
|
||||
if lookup == nil {
|
||||
lookup = os.Getenv
|
||||
}
|
||||
raw := strings.TrimSpace(lookup(envTencentVectorDBReplicaNumber))
|
||||
if raw == "" {
|
||||
return defaultTencentVectorDBReplicaNumber
|
||||
}
|
||||
replicas, err := strconv.Atoi(raw)
|
||||
if err != nil || replicas < 0 {
|
||||
return defaultTencentVectorDBReplicaNumber
|
||||
}
|
||||
return replicas
|
||||
}
|
||||
|
||||
// VectorStoreFieldInfo describes a single configuration field exposed
|
||||
// by /api/v1/vector-stores/types for the registration UI. The optional
|
||||
// validation hints (`Immutable`, `Min`, `Max`, `Enum`) are used by both
|
||||
@@ -644,6 +664,8 @@ type VectorStoreFieldInfo struct {
|
||||
|
||||
// GetVectorStoreTypes returns metadata for all supported engine types.
|
||||
func GetVectorStoreTypes() []VectorStoreTypeInfo {
|
||||
tencentVectorDBReplicaNumber := resolveTencentVectorDBReplicaNumber(os.Getenv)
|
||||
|
||||
return []VectorStoreTypeInfo{
|
||||
{
|
||||
Type: "elasticsearch",
|
||||
@@ -704,7 +726,7 @@ func GetVectorStoreTypes() []VectorStoreTypeInfo {
|
||||
IndexFields: []VectorStoreFieldInfo{
|
||||
{Name: "collection_name", Type: "string", Required: false, Description: "Collection Name", Default: "weknora_embeddings"},
|
||||
{Name: "shards_num", Type: "number", Required: false, Description: "Shards", Default: 1},
|
||||
{Name: "replica_number", Type: "number", Required: false, Description: "Replicas", Default: 1},
|
||||
{Name: "replica_number", Type: "number", Required: false, Description: "Replicas", Default: tencentVectorDBReplicaNumber},
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -344,6 +344,26 @@ func TestGetVectorStoreTypes(t *testing.T) {
|
||||
assert.Equal(t, 1, seen["replica_number"].Default)
|
||||
})
|
||||
|
||||
t.Run("tencent vectordb replica default follows env", func(t *testing.T) {
|
||||
t.Setenv(envTencentVectorDBReplicaNumber, "0")
|
||||
types := GetVectorStoreTypes()
|
||||
|
||||
var tencentType VectorStoreTypeInfo
|
||||
for _, typ := range types {
|
||||
if typ.Type == "tencent_vectordb" {
|
||||
tencentType = typ
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotEmpty(t, tencentType.IndexFields)
|
||||
|
||||
seen := map[string]VectorStoreFieldInfo{}
|
||||
for _, f := range tencentType.IndexFields {
|
||||
seen[f.Name] = f
|
||||
}
|
||||
assert.Equal(t, 0, seen["replica_number"].Default)
|
||||
})
|
||||
|
||||
t.Run("display names have no parenthetical suffix", func(t *testing.T) {
|
||||
for _, typ := range types {
|
||||
assert.NotContains(t, typ.DisplayName, "(", "display_name should not contain parenthetical suffix: %s", typ.DisplayName)
|
||||
|
||||
Reference in New Issue
Block a user