fix(tcvdb): improve collection compatibility
This commit is contained in:
@@ -35,8 +35,9 @@ func NewTencentVectorDBRetrieveEngineRepository(
|
||||
client: client,
|
||||
databaseName: databaseName,
|
||||
collectionBaseName: collectionBaseName,
|
||||
useDimensionSuffix: shouldUseDimensionSuffix(indexCfg),
|
||||
shardsNum: defaultIfZero(indexCfg.GetShardsNum(1), 1),
|
||||
replicasNum: defaultIfZero(indexCfg.GetReplicaNumber(1), 1),
|
||||
replicasNum: indexCfg.GetReplicaNumber(0),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -314,7 +315,7 @@ func (r *repository) KeywordsRetrieve(ctx context.Context, params types.Retrieve
|
||||
matchedCollections := 0
|
||||
failedCollections := 0
|
||||
for _, collection := range collections.Collections {
|
||||
if !strings.HasPrefix(collection.CollectionName, r.collectionBaseName+"_") {
|
||||
if !r.matchesCollection(collection.CollectionName) {
|
||||
continue
|
||||
}
|
||||
matchedCollections++
|
||||
@@ -422,6 +423,11 @@ func (r *repository) ensureCollection(ctx context.Context, dimension int) error
|
||||
indexes,
|
||||
)
|
||||
if err != nil {
|
||||
if isCollectionAlreadyExistsErr(err) {
|
||||
logger.GetLogger(ctx).Infof("[TencentVectorDB] collection %s already exists, skip create", collectionName)
|
||||
r.initialized.Store(dimension, true)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("tencent vectordb create collection %s: %w", collectionName, err)
|
||||
}
|
||||
|
||||
@@ -429,6 +435,14 @@ func (r *repository) ensureCollection(ctx context.Context, dimension int) error
|
||||
return nil
|
||||
}
|
||||
|
||||
func isCollectionAlreadyExistsErr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "code: 15202") || strings.Contains(msg, "already exist")
|
||||
}
|
||||
|
||||
func (r *repository) deleteByFilter(ctx context.Context, dimension int, cond string) error {
|
||||
if cond == "" {
|
||||
return nil
|
||||
@@ -450,7 +464,7 @@ func (r *repository) updateChunkFields(ctx context.Context, chunkIDs []string, f
|
||||
}
|
||||
|
||||
for _, collection := range collections.Collections {
|
||||
if !strings.HasPrefix(collection.CollectionName, r.collectionBaseName+"_") {
|
||||
if !r.matchesCollection(collection.CollectionName) {
|
||||
continue
|
||||
}
|
||||
_, err := r.client.Database(r.databaseName).Collection(collection.CollectionName).Update(ctx, tcvectordb.UpdateDocumentParams{
|
||||
@@ -465,9 +479,23 @@ func (r *repository) updateChunkFields(ctx context.Context, chunkIDs []string, f
|
||||
}
|
||||
|
||||
func (r *repository) collectionName(dimension int) string {
|
||||
if !r.useDimensionSuffix {
|
||||
return r.collectionBaseName
|
||||
}
|
||||
return fmt.Sprintf("%s_%d", r.collectionBaseName, dimension)
|
||||
}
|
||||
|
||||
func (r *repository) matchesCollection(collectionName string) bool {
|
||||
if !r.useDimensionSuffix {
|
||||
return collectionName == r.collectionBaseName
|
||||
}
|
||||
return strings.HasPrefix(collectionName, r.collectionBaseName+"_")
|
||||
}
|
||||
|
||||
func shouldUseDimensionSuffix(indexCfg *types.IndexConfig) bool {
|
||||
return indexCfg == nil || indexCfg.CollectionName == ""
|
||||
}
|
||||
|
||||
func (r *repository) baseFilter(params types.RetrieveParams) *tcvectordb.Filter {
|
||||
conditions := []string{fmt.Sprintf("%s=1", fieldIsEnabled)}
|
||||
if len(params.KnowledgeBaseIDs) > 0 {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package tencentvectordb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -68,3 +69,42 @@ func TestBaseFilterBuildsTencentVectorDBCondition(t *testing.T) {
|
||||
assert.True(t, strings.Contains(cond, want), "condition %q should contain %q", cond, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTencentVectorDBDefaultsToServerlessReplicaNumber(t *testing.T) {
|
||||
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", nil).(*repository)
|
||||
|
||||
assert.Equal(t, 0, repo.replicasNum)
|
||||
}
|
||||
|
||||
func TestTencentVectorDBUsesConfiguredReplicaNumber(t *testing.T) {
|
||||
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", &types.IndexConfig{
|
||||
ReplicaNumber: 2,
|
||||
}).(*repository)
|
||||
|
||||
assert.Equal(t, 2, repo.replicasNum)
|
||||
}
|
||||
|
||||
func TestCollectionNameUsesDimensionSuffixByDefault(t *testing.T) {
|
||||
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", nil).(*repository)
|
||||
|
||||
assert.Equal(t, "weknora_embeddings_1024", repo.collectionName(1024))
|
||||
assert.True(t, repo.matchesCollection("weknora_embeddings_1024"))
|
||||
assert.False(t, repo.matchesCollection("weknora_embeddings"))
|
||||
}
|
||||
|
||||
func TestCollectionNameRespectsExplicitCollectionName(t *testing.T) {
|
||||
repo := NewTencentVectorDBRetrieveEngineRepository(nil, "", &types.IndexConfig{
|
||||
CollectionName: "custom_collection",
|
||||
}).(*repository)
|
||||
|
||||
assert.Equal(t, "custom_collection", repo.collectionName(1024))
|
||||
assert.True(t, repo.matchesCollection("custom_collection"))
|
||||
assert.False(t, repo.matchesCollection("custom_collection_1024"))
|
||||
}
|
||||
|
||||
func TestCollectionAlreadyExistsErrorDetection(t *testing.T) {
|
||||
assert.True(t, isCollectionAlreadyExistsErr(errors.New("code: 15202, collection already exists")))
|
||||
assert.True(t, isCollectionAlreadyExistsErr(errors.New("Collection Already Exist")))
|
||||
assert.False(t, isCollectionAlreadyExistsErr(errors.New("permission denied")))
|
||||
assert.False(t, isCollectionAlreadyExistsErr(nil))
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ type repository struct {
|
||||
client *tcvectordb.RpcClient
|
||||
databaseName string
|
||||
collectionBaseName string
|
||||
useDimensionSuffix bool
|
||||
shardsNum int
|
||||
replicasNum int
|
||||
initialized sync.Map
|
||||
|
||||
@@ -704,7 +704,7 @@ func GetVectorStoreTypes() []VectorStoreTypeInfo {
|
||||
IndexFields: []VectorStoreFieldInfo{
|
||||
{Name: "collection_name", Type: "string", Required: false, Description: "Collection Name", Default: "weknora_embeddings"},
|
||||
{Name: "shards_num", Type: "number", Required: false, Description: "Shards", Default: 1},
|
||||
{Name: "replica_number", Type: "number", Required: false, Description: "Replicas", Default: 1},
|
||||
{Name: "replica_number", Type: "number", Required: false, Description: "Replicas", Default: 0},
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -327,6 +327,23 @@ func TestGetVectorStoreTypes(t *testing.T) {
|
||||
assert.True(t, passwordField.Sensitive)
|
||||
})
|
||||
|
||||
t.Run("tencent vectordb defaults to serverless replicas", func(t *testing.T) {
|
||||
var tencentType VectorStoreTypeInfo
|
||||
for _, typ := range types {
|
||||
if typ.Type == "tencent_vectordb" {
|
||||
tencentType = typ
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotEmpty(t, tencentType.IndexFields)
|
||||
|
||||
seen := map[string]VectorStoreFieldInfo{}
|
||||
for _, f := range tencentType.IndexFields {
|
||||
seen[f.Name] = f
|
||||
}
|
||||
assert.Equal(t, 0, seen["replica_number"].Default)
|
||||
})
|
||||
|
||||
t.Run("display names have no parenthetical suffix", func(t *testing.T) {
|
||||
for _, typ := range types {
|
||||
assert.NotContains(t, typ.DisplayName, "(", "display_name should not contain parenthetical suffix: %s", typ.DisplayName)
|
||||
|
||||
Reference in New Issue
Block a user