feat: add registry dual-map and engine factory for VectorStore
- Extend RetrieveEngineRegistry with byStoreID map for DB store engines - Add StoreRegistry interface and EngineFactory function type - Create engine factory with per-driver SDK client construction - Load VectorStore records from DB into registry at startup - Wire Create/Delete to dynamically register/unregister engines - Add comprehensive unit tests for registry and service
This commit is contained in:
@@ -191,6 +191,16 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
must(container.Provide(repository.NewVectorStoreRepository))
|
||||
must(container.Provide(service.NewWebSearchService))
|
||||
must(container.Provide(service.NewWebSearchProviderService))
|
||||
must(container.Provide(NewEngineFactory))
|
||||
// StoreRegistry: same instance as RetrieveEngineRegistry, exposed as StoreRegistry interface.
|
||||
// NewRetrieveEngineRegistry always returns *retriever.RetrieveEngineRegistry which implements both.
|
||||
must(container.Provide(func(r interfaces.RetrieveEngineRegistry) (interfaces.StoreRegistry, error) {
|
||||
sr, ok := r.(*retriever.RetrieveEngineRegistry)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("registry does not implement StoreRegistry")
|
||||
}
|
||||
return sr, nil
|
||||
}))
|
||||
must(container.Provide(service.NewVectorStoreService))
|
||||
|
||||
// Agent service layer (requires event bus, web search service)
|
||||
@@ -923,9 +933,43 @@ func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.Ret
|
||||
}
|
||||
}
|
||||
}
|
||||
// ─── DB store registration (byStoreID) ───
|
||||
if storeReg, ok := registry.(*retriever.RetrieveEngineRegistry); ok {
|
||||
loadDBStoresIntoRegistry(storeReg, db, cfg)
|
||||
}
|
||||
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
// loadDBStoresIntoRegistry loads VectorStore records from DB and registers them
|
||||
// in the registry's byStoreID map. Failures are logged and skipped (non-fatal).
|
||||
func loadDBStoresIntoRegistry(storeRegistry interfaces.StoreRegistry, db *gorm.DB, cfg *config.Config) {
|
||||
ctx := context.Background()
|
||||
log := logger.GetLogger(ctx)
|
||||
|
||||
var stores []types.VectorStore
|
||||
// GORM soft delete automatically adds "deleted_at IS NULL" condition
|
||||
if err := db.Find(&stores).Error; err != nil {
|
||||
log.Warnf("Failed to load vector stores from DB: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(stores) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
log.Infof("Loading %d vector store(s) from database", len(stores))
|
||||
for _, store := range stores {
|
||||
svc, err := createEngineServiceFromStore(ctx, store, db, cfg)
|
||||
if err != nil {
|
||||
log.Errorf("Failed to create engine for store %s (%s): %v", store.ID, store.Name, err)
|
||||
continue
|
||||
}
|
||||
storeRegistry.RegisterWithStoreID(store.ID, svc)
|
||||
log.Infof("Registered DB vector store: id=%s, name=%s, engine=%s", store.ID, store.Name, store.EngineType)
|
||||
}
|
||||
}
|
||||
|
||||
// initAntsPool initializes the goroutine pool
|
||||
// Creates a managed goroutine pool for concurrent task execution
|
||||
// Parameters:
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
esv7 "github.com/elastic/go-elasticsearch/v7"
|
||||
"github.com/elastic/go-elasticsearch/v8"
|
||||
"github.com/milvus-io/milvus/client/v2/milvusclient"
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
"github.com/weaviate/weaviate-go-client/v5/weaviate"
|
||||
"github.com/weaviate/weaviate-go-client/v5/weaviate/auth"
|
||||
wgrpc "github.com/weaviate/weaviate-go-client/v5/weaviate/grpc"
|
||||
"google.golang.org/grpc"
|
||||
"gorm.io/gorm"
|
||||
|
||||
elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7"
|
||||
elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8"
|
||||
milvusRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/milvus"
|
||||
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
|
||||
qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant"
|
||||
sqliteRetrieverRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/sqlite"
|
||||
weaviateRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/weaviate"
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// NewEngineFactory returns an EngineFactory function closed over db and cfg.
|
||||
// Registered in dig and injected into VectorStoreService for dynamic registry updates.
|
||||
func NewEngineFactory(db *gorm.DB, cfg *config.Config) interfaces.EngineFactory {
|
||||
return func(ctx context.Context, store types.VectorStore) (interfaces.RetrieveEngineService, error) {
|
||||
return createEngineServiceFromStore(ctx, store, db, cfg)
|
||||
}
|
||||
}
|
||||
|
||||
// createEngineServiceFromStore creates a RetrieveEngineService from a VectorStore's config.
|
||||
// This is the DB store counterpart of the env-based initialization in initRetrieveEngineRegistry.
|
||||
func createEngineServiceFromStore(
|
||||
ctx context.Context,
|
||||
store types.VectorStore,
|
||||
db *gorm.DB,
|
||||
cfg *config.Config,
|
||||
) (interfaces.RetrieveEngineService, error) {
|
||||
switch store.EngineType {
|
||||
case types.PostgresRetrieverEngineType:
|
||||
return createPostgresEngine(store, db)
|
||||
case types.ElasticsearchRetrieverEngineType:
|
||||
return createElasticsearchEngine(store, cfg)
|
||||
case types.QdrantRetrieverEngineType:
|
||||
return createQdrantEngine(store)
|
||||
case types.MilvusRetrieverEngineType:
|
||||
return createMilvusEngine(ctx, store)
|
||||
case types.WeaviateRetrieverEngineType:
|
||||
return createWeaviateEngine(store)
|
||||
case types.SQLiteRetrieverEngineType:
|
||||
return createSQLiteEngine(store, db)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported engine type: %s", store.EngineType)
|
||||
}
|
||||
}
|
||||
|
||||
func createPostgresEngine(store types.VectorStore, db *gorm.DB) (interfaces.RetrieveEngineService, error) {
|
||||
if store.ConnectionConfig.UseDefaultConnection {
|
||||
repo := postgresRepo.NewPostgresRetrieveEngineRepository(db)
|
||||
return retriever.NewKVHybridRetrieveEngine(repo, types.PostgresRetrieverEngineType), nil
|
||||
}
|
||||
// Phase 1: only UseDefaultConnection is supported.
|
||||
// Custom connections require connection pool management and migration handling.
|
||||
return nil, fmt.Errorf("custom postgres connections not yet supported; use use_default_connection=true")
|
||||
}
|
||||
|
||||
func createSQLiteEngine(_ types.VectorStore, db *gorm.DB) (interfaces.RetrieveEngineService, error) {
|
||||
repo := sqliteRetrieverRepo.NewSQLiteRetrieveEngineRepository(db)
|
||||
return retriever.NewKVHybridRetrieveEngine(repo, types.SQLiteRetrieverEngineType), nil
|
||||
}
|
||||
|
||||
func createElasticsearchEngine(store types.VectorStore, cfg *config.Config) (interfaces.RetrieveEngineService, error) {
|
||||
cc := store.ConnectionConfig
|
||||
// Version-based v7/v8 SDK selection.
|
||||
// Version is auto-detected by PR2's TestConnection and saved to connection_config.
|
||||
// Empty version defaults to v8 (latest SDK).
|
||||
if isESv7(cc.Version) {
|
||||
return createElasticsearchV7Engine(cc, cfg)
|
||||
}
|
||||
return createElasticsearchV8Engine(cc, cfg)
|
||||
}
|
||||
|
||||
// isESv7 checks if the detected ES version is 7.x.
|
||||
func isESv7(version string) bool {
|
||||
return strings.HasPrefix(version, "7.")
|
||||
}
|
||||
|
||||
func createElasticsearchV8Engine(cc types.ConnectionConfig, cfg *config.Config) (interfaces.RetrieveEngineService, error) {
|
||||
client, err := elasticsearch.NewTypedClient(elasticsearch.Config{
|
||||
Addresses: []string{cc.Addr},
|
||||
Username: cc.Username,
|
||||
Password: cc.Password,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create elasticsearch v8 client: %w", err)
|
||||
}
|
||||
repo := elasticsearchRepoV8.NewElasticsearchEngineRepository(client, cfg)
|
||||
return retriever.NewKVHybridRetrieveEngine(repo, types.ElasticsearchRetrieverEngineType), nil
|
||||
}
|
||||
|
||||
func createElasticsearchV7Engine(cc types.ConnectionConfig, cfg *config.Config) (interfaces.RetrieveEngineService, error) {
|
||||
client, err := esv7.NewClient(esv7.Config{
|
||||
Addresses: []string{cc.Addr},
|
||||
Username: cc.Username,
|
||||
Password: cc.Password,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create elasticsearch v7 client: %w", err)
|
||||
}
|
||||
repo := elasticsearchRepoV7.NewElasticsearchEngineRepository(client, cfg)
|
||||
return retriever.NewKVHybridRetrieveEngine(repo, types.ElasticsearchRetrieverEngineType), nil
|
||||
}
|
||||
|
||||
func createQdrantEngine(store types.VectorStore) (interfaces.RetrieveEngineService, error) {
|
||||
cc := store.ConnectionConfig
|
||||
port := cc.Port
|
||||
if port == 0 {
|
||||
port = 6334
|
||||
}
|
||||
|
||||
client, err := qdrant.NewClient(&qdrant.Config{
|
||||
Host: cc.Host,
|
||||
Port: port,
|
||||
APIKey: cc.APIKey,
|
||||
UseTLS: cc.UseTLS,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create qdrant client: %w", err)
|
||||
}
|
||||
repo := qdrantRepo.NewQdrantRetrieveEngineRepository(client)
|
||||
return retriever.NewKVHybridRetrieveEngine(repo, types.QdrantRetrieverEngineType), nil
|
||||
}
|
||||
|
||||
func createMilvusEngine(ctx context.Context, store types.VectorStore) (interfaces.RetrieveEngineService, error) {
|
||||
cc := store.ConnectionConfig
|
||||
addr := cc.Addr
|
||||
if addr == "" {
|
||||
addr = "localhost:19530"
|
||||
}
|
||||
|
||||
milvusCfg := milvusclient.ClientConfig{
|
||||
Address: addr,
|
||||
DialOptions: []grpc.DialOption{grpc.WithTimeout(5 * time.Second)},
|
||||
}
|
||||
if cc.Username != "" {
|
||||
milvusCfg.Username = cc.Username
|
||||
}
|
||||
if cc.Password != "" {
|
||||
milvusCfg.Password = cc.Password
|
||||
}
|
||||
// NOTE: Milvus DBName is not yet in ConnectionConfig.
|
||||
// Phase 1 limitation — only the default database is used.
|
||||
|
||||
client, err := milvusclient.New(ctx, &milvusCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create milvus client: %w", err)
|
||||
}
|
||||
repo := milvusRepo.NewMilvusRetrieveEngineRepository(client)
|
||||
return retriever.NewKVHybridRetrieveEngine(repo, types.MilvusRetrieverEngineType), nil
|
||||
}
|
||||
|
||||
func createWeaviateEngine(store types.VectorStore) (interfaces.RetrieveEngineService, error) {
|
||||
cc := store.ConnectionConfig
|
||||
host := cc.Host
|
||||
if host == "" {
|
||||
host = "weaviate:8080"
|
||||
}
|
||||
grpcAddress := cc.GrpcAddress
|
||||
if grpcAddress == "" {
|
||||
grpcAddress = "weaviate:50051"
|
||||
}
|
||||
scheme := cc.Scheme
|
||||
if scheme == "" {
|
||||
scheme = "http"
|
||||
}
|
||||
|
||||
weaviateCfg := weaviate.Config{
|
||||
Host: host,
|
||||
GrpcConfig: &wgrpc.Config{
|
||||
Host: grpcAddress,
|
||||
},
|
||||
Scheme: scheme,
|
||||
}
|
||||
// Unlike the env path (which checks WEAVIATE_AUTH_ENABLED), the factory uses
|
||||
// APIKey directly — if a user provides it, they intend to use it.
|
||||
if cc.APIKey != "" {
|
||||
weaviateCfg.AuthConfig = auth.ApiKey{Value: cc.APIKey}
|
||||
}
|
||||
|
||||
client, err := weaviate.NewClient(weaviateCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create weaviate client: %w", err)
|
||||
}
|
||||
repo := weaviateRepo.NewWeaviateRetrieveEngineRepository(client)
|
||||
return retriever.NewKVHybridRetrieveEngine(repo, types.WeaviateRetrieverEngineType), nil
|
||||
}
|
||||
Reference in New Issue
Block a user