feat: add VectorStore service layer with types and connection testing

Add VectorStoreService with CRUD validation, duplicate checking (DB + env
stores), and engine-specific health checks for 6 vector database types.
Include VectorStoreResponse DTO, env store builder, engine type metadata,
and comprehensive unit tests.
This commit is contained in:
ochan.kwon
2026-04-10 09:32:36 +09:00
committed by lyingbug
parent 30f349d06a
commit 9edf0839ed
7 changed files with 1406 additions and 0 deletions
+18
View File
@@ -6,6 +6,22 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
// VectorStoreService defines the service interface for vector store management.
// Tenant isolation is enforced by the handler layer (getOwnedStore pattern).
type VectorStoreService interface {
// CreateStore validates and creates a new vector store.
CreateStore(ctx context.Context, store *types.VectorStore) error
// UpdateStore updates an existing vector store (name only).
UpdateStore(ctx context.Context, store *types.VectorStore) error
// DeleteStore deletes a vector store by tenant + id.
DeleteStore(ctx context.Context, tenantID uint64, id string) error
// TestConnection tests connectivity to a vector database.
// Returns the detected server version on success (e.g., "7.10.1"), empty string if unknown.
TestConnection(ctx context.Context, engineType types.RetrieverEngineType, config types.ConnectionConfig) (string, error)
// SaveDetectedVersion updates the connection_config.version for a stored vector store.
SaveDetectedVersion(ctx context.Context, store *types.VectorStore, version string) error
}
// VectorStoreRepository defines the repository interface for VectorStore CRUD.
type VectorStoreRepository interface {
// Create creates a new vector store
@@ -16,6 +32,8 @@ type VectorStoreRepository interface {
List(ctx context.Context, tenantID uint64) ([]*types.VectorStore, error)
// Update updates a vector store (only mutable fields: name)
Update(ctx context.Context, store *types.VectorStore) error
// UpdateConnectionConfig updates only the connection_config column
UpdateConnectionConfig(ctx context.Context, store *types.VectorStore) error
// Delete soft-deletes a vector store
Delete(ctx context.Context, tenantID uint64, id string) error
// ExistsByEndpointAndIndex checks if a store with the same endpoint and index already exists
+266
View File
@@ -4,6 +4,7 @@ import (
"database/sql/driver"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/errors"
@@ -12,6 +13,18 @@ import (
"gorm.io/gorm"
)
// EnvStoreIDPrefix is the prefix for virtual env store IDs.
const EnvStoreIDPrefix = "__env_"
// IsEnvStoreID checks if the given ID is an env store virtual ID.
func IsEnvStoreID(id string) bool {
return strings.HasPrefix(id, EnvStoreIDPrefix)
}
// EnvLookupFunc is a function type for looking up environment variables.
// In production: os.Getenv, in tests: custom lookup function.
type EnvLookupFunc func(string) string
// VectorStore represents a configured vector database instance for a tenant.
// Each tenant can register multiple VectorStore entries (even of the same engine type)
// to support multi-store scenarios (e.g., ES-hot + ES-warm clusters).
@@ -100,6 +113,9 @@ type ConnectionConfig struct {
Scheme string `yaml:"scheme" json:"scheme,omitempty"`
// Postgres
UseDefaultConnection bool `yaml:"use_default_connection" json:"use_default_connection,omitempty"`
// Version is the detected server version (e.g., "7.10.1", "16.2", "1.12.6").
// Auto-populated by TestConnection on successful connectivity check.
Version string `yaml:"version" json:"version,omitempty"`
}
// Value implements the driver.Valuer interface.
@@ -237,3 +253,253 @@ func (c IndexConfig) GetIndexNameOrDefault(engineType RetrieverEngineType) strin
return c.IndexName
}
}
// ---------------------------------------------------------------------------
// VectorStoreResponse — API response DTO
// ---------------------------------------------------------------------------
// VectorStoreResponse is the API response DTO for vector store.
// Wraps VectorStore with additional metadata (source, readonly).
type VectorStoreResponse struct {
VectorStore
Source string `json:"source"` // "env" or "user"
ReadOnly bool `json:"readonly"` // env stores are read-only
}
// NewVectorStoreResponse creates a response DTO from a VectorStore
// with sensitive fields masked.
func NewVectorStoreResponse(store *VectorStore, source string, readonly bool) VectorStoreResponse {
masked := *store
masked.ConnectionConfig = store.ConnectionConfig.MaskSensitiveFields()
return VectorStoreResponse{
VectorStore: masked,
Source: source,
ReadOnly: readonly,
}
}
// ---------------------------------------------------------------------------
// VectorStore type metadata — for /types endpoint
// ---------------------------------------------------------------------------
// VectorStoreTypeInfo describes a supported engine type and its configuration schema.
type VectorStoreTypeInfo struct {
Type string `json:"type"`
DisplayName string `json:"display_name"`
ConnectionFields []VectorStoreFieldInfo `json:"connection_fields"`
IndexFields []VectorStoreFieldInfo `json:"index_fields,omitempty"`
}
// VectorStoreFieldInfo describes a single configuration field.
type VectorStoreFieldInfo struct {
Name string `json:"name"`
Type string `json:"type"` // "string", "number", "boolean"
Required bool `json:"required"`
Sensitive bool `json:"sensitive,omitempty"`
Default any `json:"default,omitempty"`
Description string `json:"description,omitempty"`
}
// GetVectorStoreTypes returns metadata for all supported engine types.
func GetVectorStoreTypes() []VectorStoreTypeInfo {
return []VectorStoreTypeInfo{
{
Type: "elasticsearch",
DisplayName: "Elasticsearch (Keywords + Vector)",
ConnectionFields: []VectorStoreFieldInfo{
{Name: "addr", Type: "string", Required: true, Description: "Elasticsearch URL (e.g., http://localhost:9200)"},
{Name: "username", Type: "string", Required: false},
{Name: "password", Type: "string", Required: false, Sensitive: true},
},
IndexFields: []VectorStoreFieldInfo{
{Name: "index_name", Type: "string", Required: false, Default: "xwrag_default"},
{Name: "number_of_shards", Type: "number", Required: false},
{Name: "number_of_replicas", Type: "number", Required: false},
},
},
{
Type: "postgres",
DisplayName: "PostgreSQL (Keywords + Vector)",
ConnectionFields: []VectorStoreFieldInfo{
{Name: "use_default_connection", Type: "boolean", Required: false, Default: true,
Description: "Use the application's default database connection"},
{Name: "addr", Type: "string", Required: false,
Description: "PostgreSQL connection string (required if use_default_connection is false)"},
{Name: "username", Type: "string", Required: false},
{Name: "password", Type: "string", Required: false, Sensitive: true},
},
},
{
Type: "qdrant",
DisplayName: "Qdrant (Keywords + Vector)",
ConnectionFields: []VectorStoreFieldInfo{
{Name: "host", Type: "string", Required: true, Description: "Qdrant host"},
{Name: "port", Type: "number", Required: false, Default: 6334},
{Name: "api_key", Type: "string", Required: false, Sensitive: true},
{Name: "use_tls", Type: "boolean", Required: false, Default: false},
},
IndexFields: []VectorStoreFieldInfo{
{Name: "collection_prefix", Type: "string", Required: false, Default: "weknora_embeddings"},
},
},
{
Type: "milvus",
DisplayName: "Milvus (Keywords + Vector)",
ConnectionFields: []VectorStoreFieldInfo{
{Name: "addr", Type: "string", Required: true, Description: "Milvus address (e.g., localhost:19530)"},
{Name: "username", Type: "string", Required: false},
{Name: "password", Type: "string", Required: false, Sensitive: true},
},
IndexFields: []VectorStoreFieldInfo{
{Name: "collection_name", Type: "string", Required: false, Default: "weknora_embeddings"},
},
},
{
Type: "weaviate",
DisplayName: "Weaviate (Keywords + Vector)",
ConnectionFields: []VectorStoreFieldInfo{
{Name: "host", Type: "string", Required: true, Description: "Weaviate host (e.g., weaviate:8080)"},
{Name: "grpc_address", Type: "string", Required: false, Default: "weaviate:50051"},
{Name: "scheme", Type: "string", Required: false, Default: "http"},
{Name: "api_key", Type: "string", Required: false, Sensitive: true},
},
IndexFields: []VectorStoreFieldInfo{
{Name: "collection_prefix", Type: "string", Required: false, Default: "WeKnora"},
},
},
{
Type: "sqlite",
DisplayName: "SQLite (Keywords + Vector)",
ConnectionFields: []VectorStoreFieldInfo{},
},
}
}
// ---------------------------------------------------------------------------
// BuildEnvVectorStores — virtual stores from RETRIEVE_DRIVER env var
// ---------------------------------------------------------------------------
// BuildEnvVectorStores builds virtual VectorStore entries from RETRIEVE_DRIVER.
// Returns []VectorStore (not VectorStoreResponse) so that business logic (e.g.,
// duplicate checking) can use them directly. API responses should wrap them
// via NewVectorStoreResponse.
//
// Pure function — does not call os.Getenv directly.
//
// Usage:
//
// types.BuildEnvVectorStores(os.Getenv("RETRIEVE_DRIVER"), os.Getenv)
func BuildEnvVectorStores(retrieveDriver string, envLookup EnvLookupFunc) []VectorStore {
if retrieveDriver == "" {
return nil
}
drivers := strings.Split(retrieveDriver, ",")
var stores []VectorStore
for _, driver := range drivers {
driver = strings.TrimSpace(driver)
if driver == "" {
continue
}
store := buildEnvStoreForDriver(driver, envLookup)
if store != nil {
stores = append(stores, *store)
}
}
return stores
}
// FindEnvVectorStore finds a specific env store by its virtual ID.
func FindEnvVectorStore(retrieveDriver string, envLookup EnvLookupFunc, id string) *VectorStore {
for _, s := range BuildEnvVectorStores(retrieveDriver, envLookup) {
if s.ID == id {
return &s
}
}
return nil
}
func buildEnvStoreForDriver(driver string, envLookup EnvLookupFunc) *VectorStore {
switch driver {
case "postgres":
return &VectorStore{
ID: "__env_postgres__",
Name: "postgres (env)",
EngineType: PostgresRetrieverEngineType,
ConnectionConfig: ConnectionConfig{
UseDefaultConnection: true,
},
}
case "sqlite":
return &VectorStore{
ID: "__env_sqlite__",
Name: "sqlite (env)",
EngineType: SQLiteRetrieverEngineType,
}
case "elasticsearch_v8":
return &VectorStore{
ID: "__env_elasticsearch_v8__",
Name: "elasticsearch v8 (env)",
EngineType: ElasticsearchRetrieverEngineType,
ConnectionConfig: ConnectionConfig{
Addr: envLookup("ELASTICSEARCH_ADDR"),
Username: envLookup("ELASTICSEARCH_USERNAME"),
Password: envLookup("ELASTICSEARCH_PASSWORD"),
},
IndexConfig: IndexConfig{
IndexName: envLookup("ELASTICSEARCH_INDEX"),
},
}
case "elasticsearch_v7":
return &VectorStore{
ID: "__env_elasticsearch_v7__",
Name: "elasticsearch v7 (env)",
EngineType: ElasticsearchRetrieverEngineType,
ConnectionConfig: ConnectionConfig{
Addr: envLookup("ELASTICSEARCH_ADDR"),
Username: envLookup("ELASTICSEARCH_USERNAME"),
Password: envLookup("ELASTICSEARCH_PASSWORD"),
},
IndexConfig: IndexConfig{
IndexName: envLookup("ELASTICSEARCH_INDEX"),
},
}
case "qdrant":
return &VectorStore{
ID: "__env_qdrant__",
Name: "qdrant (env)",
EngineType: QdrantRetrieverEngineType,
ConnectionConfig: ConnectionConfig{
Host: envLookup("QDRANT_HOST"),
APIKey: envLookup("QDRANT_API_KEY"),
},
}
case "milvus":
return &VectorStore{
ID: "__env_milvus__",
Name: "milvus (env)",
EngineType: MilvusRetrieverEngineType,
ConnectionConfig: ConnectionConfig{
Addr: envLookup("MILVUS_ADDRESS"),
Username: envLookup("MILVUS_USERNAME"),
Password: envLookup("MILVUS_PASSWORD"),
},
}
case "weaviate":
return &VectorStore{
ID: "__env_weaviate__",
Name: "weaviate (env)",
EngineType: WeaviateRetrieverEngineType,
ConnectionConfig: ConnectionConfig{
Host: envLookup("WEAVIATE_HOST"),
GrpcAddress: envLookup("WEAVIATE_GRPC_ADDRESS"),
Scheme: envLookup("WEAVIATE_SCHEME"),
APIKey: envLookup("WEAVIATE_API_KEY"),
},
}
default:
return nil
}
}
+239
View File
@@ -10,6 +10,245 @@ import (
"gorm.io/gorm"
)
// ---------------------------------------------------------------------------
// PR2 additions: env store builder, response DTO, types metadata
// ---------------------------------------------------------------------------
// mockEnvLookup creates a simple env lookup function from a map.
func mockEnvLookup(env map[string]string) EnvLookupFunc {
return func(key string) string {
return env[key]
}
}
func TestIsEnvStoreID(t *testing.T) {
tests := []struct {
name string
id string
expected bool
}{
{"env postgres ID", "__env_postgres__", true},
{"env elasticsearch ID", "__env_elasticsearch_v8__", true},
{"env prefix only", "__env_", true},
{"UUID ID", "550e8400-e29b-41d4-a716-446655440000", false},
{"empty string", "", false},
{"similar but not prefix", "_env_postgres__", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.expected, IsEnvStoreID(tt.id))
})
}
}
func TestBuildEnvVectorStores(t *testing.T) {
envMap := map[string]string{
"ELASTICSEARCH_ADDR": "http://es:9200",
"ELASTICSEARCH_USERNAME": "elastic",
"ELASTICSEARCH_PASSWORD": "secret",
"ELASTICSEARCH_INDEX": "my_index",
"QDRANT_HOST": "qdrant-host",
"QDRANT_API_KEY": "qd-key",
"MILVUS_ADDRESS": "milvus:19530",
"WEAVIATE_HOST": "weaviate:8080",
}
lookup := mockEnvLookup(envMap)
t.Run("empty RETRIEVE_DRIVER returns nil", func(t *testing.T) {
stores := BuildEnvVectorStores("", lookup)
assert.Nil(t, stores)
})
t.Run("single driver postgres", func(t *testing.T) {
stores := BuildEnvVectorStores("postgres", lookup)
require.Len(t, stores, 1)
assert.Equal(t, "__env_postgres__", stores[0].ID)
assert.Equal(t, "postgres (env)", stores[0].Name)
assert.Equal(t, PostgresRetrieverEngineType, stores[0].EngineType)
assert.True(t, stores[0].ConnectionConfig.UseDefaultConnection)
})
t.Run("multiple drivers", func(t *testing.T) {
stores := BuildEnvVectorStores("postgres,elasticsearch_v8", lookup)
require.Len(t, stores, 2)
assert.Equal(t, "__env_postgres__", stores[0].ID)
assert.Equal(t, "__env_elasticsearch_v8__", stores[1].ID)
assert.Equal(t, "http://es:9200", stores[1].ConnectionConfig.Addr)
assert.Equal(t, "elastic", stores[1].ConnectionConfig.Username)
assert.Equal(t, "secret", stores[1].ConnectionConfig.Password) // unmasked
assert.Equal(t, "my_index", stores[1].IndexConfig.IndexName)
})
t.Run("env store retains raw password (not masked)", func(t *testing.T) {
stores := BuildEnvVectorStores("elasticsearch_v8", lookup)
require.Len(t, stores, 1)
assert.Equal(t, "secret", stores[0].ConnectionConfig.Password)
})
t.Run("unknown driver is skipped", func(t *testing.T) {
stores := BuildEnvVectorStores("postgres,unknown_db", lookup)
require.Len(t, stores, 1)
assert.Equal(t, "__env_postgres__", stores[0].ID)
})
t.Run("whitespace trimmed", func(t *testing.T) {
stores := BuildEnvVectorStores(" postgres , elasticsearch_v8 ", lookup)
require.Len(t, stores, 2)
})
t.Run("all supported drivers", func(t *testing.T) {
stores := BuildEnvVectorStores("postgres,sqlite,elasticsearch_v8,elasticsearch_v7,qdrant,milvus,weaviate", lookup)
require.Len(t, stores, 7)
ids := make([]string, len(stores))
for i, s := range stores {
ids[i] = s.ID
}
assert.Contains(t, ids, "__env_postgres__")
assert.Contains(t, ids, "__env_sqlite__")
assert.Contains(t, ids, "__env_elasticsearch_v8__")
assert.Contains(t, ids, "__env_elasticsearch_v7__")
assert.Contains(t, ids, "__env_qdrant__")
assert.Contains(t, ids, "__env_milvus__")
assert.Contains(t, ids, "__env_weaviate__")
})
t.Run("qdrant env store", func(t *testing.T) {
stores := BuildEnvVectorStores("qdrant", lookup)
require.Len(t, stores, 1)
assert.Equal(t, "qdrant-host", stores[0].ConnectionConfig.Host)
assert.Equal(t, "qd-key", stores[0].ConnectionConfig.APIKey)
})
t.Run("milvus env store", func(t *testing.T) {
stores := BuildEnvVectorStores("milvus", lookup)
require.Len(t, stores, 1)
assert.Equal(t, "milvus:19530", stores[0].ConnectionConfig.Addr)
})
t.Run("weaviate env store", func(t *testing.T) {
stores := BuildEnvVectorStores("weaviate", lookup)
require.Len(t, stores, 1)
assert.Equal(t, "weaviate:8080", stores[0].ConnectionConfig.Host)
})
}
func TestFindEnvVectorStore(t *testing.T) {
lookup := mockEnvLookup(map[string]string{})
t.Run("found", func(t *testing.T) {
store := FindEnvVectorStore("postgres", lookup, "__env_postgres__")
require.NotNil(t, store)
assert.Equal(t, "__env_postgres__", store.ID)
})
t.Run("not found", func(t *testing.T) {
store := FindEnvVectorStore("postgres", lookup, "__env_unknown__")
assert.Nil(t, store)
})
t.Run("empty driver returns nil", func(t *testing.T) {
store := FindEnvVectorStore("", lookup, "__env_postgres__")
assert.Nil(t, store)
})
}
func TestNewVectorStoreResponse(t *testing.T) {
store := &VectorStore{
ID: "test-id",
Name: "test-store",
EngineType: ElasticsearchRetrieverEngineType,
ConnectionConfig: ConnectionConfig{
Addr: "http://es:9200",
Password: "secret",
APIKey: "my-api-key",
},
}
t.Run("masks sensitive fields", func(t *testing.T) {
resp := NewVectorStoreResponse(store, "user", false)
assert.Equal(t, "***", resp.ConnectionConfig.Password)
assert.Equal(t, "***", resp.ConnectionConfig.APIKey)
assert.Equal(t, "http://es:9200", resp.ConnectionConfig.Addr) // non-sensitive preserved
})
t.Run("preserves source and readonly", func(t *testing.T) {
resp := NewVectorStoreResponse(store, "env", true)
assert.Equal(t, "env", resp.Source)
assert.True(t, resp.ReadOnly)
})
t.Run("does not mutate original store", func(t *testing.T) {
_ = NewVectorStoreResponse(store, "user", false)
assert.Equal(t, "secret", store.ConnectionConfig.Password)
assert.Equal(t, "my-api-key", store.ConnectionConfig.APIKey)
})
t.Run("empty sensitive fields not masked to ***", func(t *testing.T) {
noSecret := &VectorStore{
ID: "test-id",
ConnectionConfig: ConnectionConfig{Addr: "http://es:9200"},
}
resp := NewVectorStoreResponse(noSecret, "user", false)
assert.Equal(t, "", resp.ConnectionConfig.Password)
assert.Equal(t, "", resp.ConnectionConfig.APIKey)
})
}
func TestGetVectorStoreTypes(t *testing.T) {
types := GetVectorStoreTypes()
t.Run("returns 6 engine types", func(t *testing.T) {
assert.Len(t, types, 6)
})
t.Run("type names match engine constants", func(t *testing.T) {
typeNames := make([]string, len(types))
for i, typ := range types {
typeNames[i] = typ.Type
}
assert.Contains(t, typeNames, "elasticsearch")
assert.Contains(t, typeNames, "postgres")
assert.Contains(t, typeNames, "qdrant")
assert.Contains(t, typeNames, "milvus")
assert.Contains(t, typeNames, "weaviate")
assert.Contains(t, typeNames, "sqlite")
})
t.Run("elasticsearch has connection and index fields", func(t *testing.T) {
var esType VectorStoreTypeInfo
for _, typ := range types {
if typ.Type == "elasticsearch" {
esType = typ
break
}
}
assert.NotEmpty(t, esType.ConnectionFields)
assert.NotEmpty(t, esType.IndexFields)
// Check sensitive field marking
var passwordField VectorStoreFieldInfo
for _, f := range esType.ConnectionFields {
if f.Name == "password" {
passwordField = f
break
}
}
assert.True(t, passwordField.Sensitive)
})
t.Run("sqlite has no connection fields", func(t *testing.T) {
var sqliteType VectorStoreTypeInfo
for _, typ := range types {
if typ.Type == "sqlite" {
sqliteType = typ
break
}
}
assert.Empty(t, sqliteType.ConnectionFields)
})
}
// testAESKey is a 32-byte key for testing AES-GCM encryption.
const testAESKey = "01234567890123456789012345678901"