4158773849
- Introduced a new API for organization management, including creation, updating, and deletion of organizations. - Implemented organization sharing functionality for knowledge bases, allowing users to share knowledge bases with specific organizations and set permissions. - Added UI components for organization selection and sharing dialogs. - Enhanced existing components to support organization-related features, including shared knowledge base handling and organization role management. This update enhances collaboration capabilities within the application by allowing users to manage and share knowledge bases across different organizations.
679 lines
25 KiB
Go
679 lines
25 KiB
Go
// Package container implements dependency injection container setup
|
|
// Provides centralized configuration for services, repositories, and handlers
|
|
// This package is responsible for wiring up all dependencies and ensuring proper lifecycle management
|
|
package container
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"net/url"
|
|
"os"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
_ "github.com/duckdb/duckdb-go/v2"
|
|
esv7 "github.com/elastic/go-elasticsearch/v7"
|
|
"github.com/elastic/go-elasticsearch/v8"
|
|
"github.com/neo4j/neo4j-go-driver/v6/neo4j"
|
|
"github.com/panjf2000/ants/v2"
|
|
"github.com/qdrant/go-client/qdrant"
|
|
"github.com/redis/go-redis/v9"
|
|
"go.uber.org/dig"
|
|
"gorm.io/driver/postgres"
|
|
"gorm.io/gorm"
|
|
|
|
"github.com/Tencent/WeKnora/docreader/client"
|
|
"github.com/Tencent/WeKnora/internal/application/repository"
|
|
elasticsearchRepoV7 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v7"
|
|
elasticsearchRepoV8 "github.com/Tencent/WeKnora/internal/application/repository/retriever/elasticsearch/v8"
|
|
neo4jRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/neo4j"
|
|
postgresRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/postgres"
|
|
qdrantRepo "github.com/Tencent/WeKnora/internal/application/repository/retriever/qdrant"
|
|
"github.com/Tencent/WeKnora/internal/application/service"
|
|
chatpipline "github.com/Tencent/WeKnora/internal/application/service/chat_pipline"
|
|
"github.com/Tencent/WeKnora/internal/application/service/file"
|
|
"github.com/Tencent/WeKnora/internal/application/service/llmcontext"
|
|
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
|
"github.com/Tencent/WeKnora/internal/application/service/web_search"
|
|
"github.com/Tencent/WeKnora/internal/config"
|
|
"github.com/Tencent/WeKnora/internal/database"
|
|
"github.com/Tencent/WeKnora/internal/event"
|
|
"github.com/Tencent/WeKnora/internal/handler"
|
|
"github.com/Tencent/WeKnora/internal/handler/session"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/mcp"
|
|
"github.com/Tencent/WeKnora/internal/models/embedding"
|
|
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
|
"github.com/Tencent/WeKnora/internal/router"
|
|
"github.com/Tencent/WeKnora/internal/stream"
|
|
"github.com/Tencent/WeKnora/internal/tracing"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
)
|
|
|
|
// BuildContainer constructs the dependency injection container
|
|
// Registers all components, services, repositories and handlers needed by the application
|
|
// Creates a fully configured application container with proper dependency resolution
|
|
// Parameters:
|
|
// - container: Base dig container to add dependencies to
|
|
//
|
|
// Returns:
|
|
// - Configured container with all application dependencies registered
|
|
func BuildContainer(container *dig.Container) *dig.Container {
|
|
ctx := context.Background()
|
|
logger.Debugf(ctx, "[Container] Starting container initialization...")
|
|
|
|
// Register resource cleaner for proper cleanup of resources
|
|
must(container.Provide(NewResourceCleaner, dig.As(new(interfaces.ResourceCleaner))))
|
|
|
|
// Core infrastructure configuration
|
|
logger.Debugf(ctx, "[Container] Registering core infrastructure...")
|
|
must(container.Provide(config.LoadConfig))
|
|
must(container.Provide(initTracer))
|
|
must(container.Provide(initDatabase))
|
|
must(container.Provide(initFileService))
|
|
must(container.Provide(initRedisClient))
|
|
must(container.Provide(initAntsPool))
|
|
must(container.Provide(initContextStorage))
|
|
|
|
// Register goroutine pool cleanup handler
|
|
must(container.Invoke(registerPoolCleanup))
|
|
|
|
// Initialize retrieval engine registry for search capabilities
|
|
logger.Debugf(ctx, "[Container] Registering retrieval engine registry...")
|
|
must(container.Provide(initRetrieveEngineRegistry))
|
|
|
|
// External service clients
|
|
logger.Debugf(ctx, "[Container] Registering external service clients...")
|
|
must(container.Provide(initDocReaderClient))
|
|
must(container.Provide(initOllamaService))
|
|
must(container.Provide(initNeo4jClient))
|
|
must(container.Provide(stream.NewStreamManager))
|
|
logger.Debugf(ctx, "[Container] Initializing DuckDB...")
|
|
must(container.Provide(NewDuckDB))
|
|
logger.Debugf(ctx, "[Container] DuckDB registered")
|
|
|
|
// Data repositories layer
|
|
logger.Debugf(ctx, "[Container] Registering repositories...")
|
|
must(container.Provide(repository.NewTenantRepository))
|
|
must(container.Provide(repository.NewKnowledgeBaseRepository))
|
|
must(container.Provide(repository.NewKnowledgeRepository))
|
|
must(container.Provide(repository.NewChunkRepository))
|
|
must(container.Provide(repository.NewKnowledgeTagRepository))
|
|
must(container.Provide(repository.NewSessionRepository))
|
|
must(container.Provide(repository.NewMessageRepository))
|
|
must(container.Provide(repository.NewModelRepository))
|
|
must(container.Provide(repository.NewUserRepository))
|
|
must(container.Provide(repository.NewAuthTokenRepository))
|
|
must(container.Provide(neo4jRepo.NewNeo4jRepository))
|
|
must(container.Provide(repository.NewMCPServiceRepository))
|
|
must(container.Provide(repository.NewCustomAgentRepository))
|
|
must(container.Provide(repository.NewOrganizationRepository))
|
|
must(container.Provide(repository.NewKBShareRepository))
|
|
must(container.Provide(service.NewWebSearchStateService))
|
|
|
|
// MCP manager for managing MCP client connections
|
|
logger.Debugf(ctx, "[Container] Registering MCP manager...")
|
|
must(container.Provide(mcp.NewMCPManager))
|
|
|
|
// Business service layer
|
|
logger.Debugf(ctx, "[Container] Registering business services...")
|
|
must(container.Provide(service.NewTenantService))
|
|
must(container.Provide(service.NewKnowledgeBaseService))
|
|
must(container.Provide(service.NewOrganizationService))
|
|
must(container.Provide(service.NewKBShareService)) // KBShareService must be registered before KnowledgeService and KnowledgeTagService
|
|
must(container.Provide(service.NewKnowledgeService))
|
|
must(container.Provide(service.NewChunkService))
|
|
must(container.Provide(service.NewKnowledgeTagService))
|
|
must(container.Provide(embedding.NewBatchEmbedder))
|
|
must(container.Provide(service.NewModelService))
|
|
must(container.Provide(service.NewDatasetService))
|
|
must(container.Provide(service.NewEvaluationService))
|
|
must(container.Provide(service.NewUserService))
|
|
|
|
// Extract services - register individual extracters with names
|
|
must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtractor")))
|
|
must(container.Provide(service.NewDataTableSummaryService, dig.Name("dataTableSummary")))
|
|
|
|
must(container.Provide(service.NewMessageService))
|
|
must(container.Provide(service.NewMCPServiceService))
|
|
must(container.Provide(service.NewCustomAgentService))
|
|
|
|
// Web search service (needed by AgentService)
|
|
logger.Debugf(ctx, "[Container] Registering web search registry and providers...")
|
|
must(container.Provide(web_search.NewRegistry))
|
|
must(container.Invoke(registerWebSearchProviders))
|
|
must(container.Provide(service.NewWebSearchService))
|
|
|
|
// Agent service layer (requires event bus, web search service)
|
|
// SessionService is passed as parameter to CreateAgentEngine method when creating AgentService
|
|
logger.Debugf(ctx, "[Container] Registering event bus and agent service...")
|
|
must(container.Provide(event.NewEventBus))
|
|
must(container.Provide(service.NewAgentService))
|
|
|
|
// Session service (depends on agent service)
|
|
// SessionService is created after AgentService and passes itself to AgentService.CreateAgentEngine when needed
|
|
logger.Debugf(ctx, "[Container] Registering session service...")
|
|
must(container.Provide(service.NewSessionService))
|
|
|
|
logger.Debugf(ctx, "[Container] Registering asynq client and server...")
|
|
must(container.Provide(router.NewAsyncqClient))
|
|
must(container.Provide(router.NewAsynqServer))
|
|
|
|
// Chat pipeline components for processing chat requests
|
|
logger.Debugf(ctx, "[Container] Registering chat pipeline plugins...")
|
|
must(container.Provide(chatpipline.NewEventManager))
|
|
must(container.Invoke(chatpipline.NewPluginTracing))
|
|
must(container.Invoke(chatpipline.NewPluginSearch))
|
|
must(container.Invoke(chatpipline.NewPluginRerank))
|
|
must(container.Invoke(chatpipline.NewPluginMerge))
|
|
must(container.Invoke(chatpipline.NewPluginDataAnalysis))
|
|
must(container.Invoke(chatpipline.NewPluginIntoChatMessage))
|
|
must(container.Invoke(chatpipline.NewPluginChatCompletion))
|
|
must(container.Invoke(chatpipline.NewPluginChatCompletionStream))
|
|
must(container.Invoke(chatpipline.NewPluginStreamFilter))
|
|
must(container.Invoke(chatpipline.NewPluginFilterTopK))
|
|
must(container.Invoke(chatpipline.NewPluginRewrite))
|
|
must(container.Invoke(chatpipline.NewPluginLoadHistory))
|
|
must(container.Invoke(chatpipline.NewPluginExtractEntity))
|
|
must(container.Invoke(chatpipline.NewPluginSearchEntity))
|
|
must(container.Invoke(chatpipline.NewPluginSearchParallel))
|
|
logger.Debugf(ctx, "[Container] Chat pipeline plugins registered")
|
|
|
|
// HTTP handlers layer
|
|
logger.Debugf(ctx, "[Container] Registering HTTP handlers...")
|
|
must(container.Provide(handler.NewTenantHandler))
|
|
must(container.Provide(handler.NewKnowledgeBaseHandler))
|
|
must(container.Provide(handler.NewKnowledgeHandler))
|
|
must(container.Provide(handler.NewChunkHandler))
|
|
must(container.Provide(handler.NewFAQHandler))
|
|
must(container.Provide(handler.NewTagHandler))
|
|
must(container.Provide(session.NewHandler))
|
|
must(container.Provide(handler.NewMessageHandler))
|
|
must(container.Provide(handler.NewModelHandler))
|
|
must(container.Provide(handler.NewEvaluationHandler))
|
|
must(container.Provide(handler.NewInitializationHandler))
|
|
must(container.Provide(handler.NewAuthHandler))
|
|
must(container.Provide(handler.NewSystemHandler))
|
|
must(container.Provide(handler.NewMCPServiceHandler))
|
|
must(container.Provide(handler.NewWebSearchHandler))
|
|
must(container.Provide(handler.NewCustomAgentHandler))
|
|
must(container.Provide(service.NewSkillService))
|
|
must(container.Provide(handler.NewSkillHandler))
|
|
must(container.Provide(handler.NewOrganizationHandler))
|
|
logger.Debugf(ctx, "[Container] HTTP handlers registered")
|
|
|
|
// Router configuration
|
|
logger.Debugf(ctx, "[Container] Registering router and starting asynq server...")
|
|
must(container.Provide(router.NewRouter))
|
|
must(container.Invoke(router.RunAsynqServer))
|
|
|
|
logger.Infof(ctx, "[Container] Container initialization completed successfully")
|
|
return container
|
|
}
|
|
|
|
// must is a helper function for error handling
|
|
// Panics if the error is not nil, useful for configuration steps that must succeed
|
|
// Parameters:
|
|
// - err: Error to check
|
|
func must(err error) {
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// initTracer initializes OpenTelemetry tracer
|
|
// Sets up distributed tracing for observability across the application
|
|
// Parameters:
|
|
// - None
|
|
//
|
|
// Returns:
|
|
// - Configured tracer instance
|
|
// - Error if initialization fails
|
|
func initTracer() (*tracing.Tracer, error) {
|
|
return tracing.InitTracer()
|
|
}
|
|
|
|
func initRedisClient() (*redis.Client, error) {
|
|
db, err := strconv.Atoi(os.Getenv("REDIS_DB"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
client := redis.NewClient(&redis.Options{
|
|
Addr: os.Getenv("REDIS_ADDR"),
|
|
Username: os.Getenv("REDIS_USERNAME"),
|
|
Password: os.Getenv("REDIS_PASSWORD"),
|
|
DB: db,
|
|
})
|
|
|
|
// 验证连接
|
|
_, err = client.Ping(context.Background()).Result()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("连接Redis失败: %w", err)
|
|
}
|
|
|
|
return client, nil
|
|
}
|
|
|
|
func initContextStorage(redisClient *redis.Client) (llmcontext.ContextStorage, error) {
|
|
storage, err := llmcontext.NewRedisStorage(redisClient, 24*time.Hour, "context:")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return storage, nil
|
|
}
|
|
|
|
// initDatabase initializes database connection
|
|
// Creates and configures database connection based on environment configuration
|
|
// Supports multiple database backends (PostgreSQL)
|
|
// Parameters:
|
|
// - cfg: Application configuration
|
|
//
|
|
// Returns:
|
|
// - Configured database connection
|
|
// - Error if connection fails
|
|
func initDatabase(cfg *config.Config) (*gorm.DB, error) {
|
|
var dialector gorm.Dialector
|
|
var migrateDSN string
|
|
switch os.Getenv("DB_DRIVER") {
|
|
case "postgres":
|
|
// DSN for GORM (key-value format)
|
|
gormDSN := fmt.Sprintf(
|
|
"host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
|
|
os.Getenv("DB_HOST"),
|
|
os.Getenv("DB_PORT"),
|
|
os.Getenv("DB_USER"),
|
|
os.Getenv("DB_PASSWORD"),
|
|
os.Getenv("DB_NAME"),
|
|
"disable",
|
|
)
|
|
dialector = postgres.Open(gormDSN)
|
|
|
|
// DSN for golang-migrate (URL format)
|
|
// URL-encode password to handle special characters like !@#
|
|
dbPassword := os.Getenv("DB_PASSWORD")
|
|
encodedPassword := url.QueryEscape(dbPassword)
|
|
|
|
// Check if postgres is in RETRIEVE_DRIVER to determine skip_embedding
|
|
retrieveDriver := strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",")
|
|
skipEmbedding := "true"
|
|
if slices.Contains(retrieveDriver, "postgres") {
|
|
skipEmbedding = "false"
|
|
}
|
|
logger.Infof(context.Background(), "Skip embedding: %s", skipEmbedding)
|
|
|
|
migrateDSN = fmt.Sprintf(
|
|
"postgres://%s:%s@%s:%s/%s?sslmode=disable&options=-c%%20app.skip_embedding=%s",
|
|
os.Getenv("DB_USER"),
|
|
encodedPassword, // Use encoded password
|
|
os.Getenv("DB_HOST"),
|
|
os.Getenv("DB_PORT"),
|
|
os.Getenv("DB_NAME"),
|
|
skipEmbedding,
|
|
)
|
|
|
|
// Debug log (don't log password)
|
|
logger.Infof(context.Background(), "DB Config: user=%s host=%s port=%s dbname=%s",
|
|
os.Getenv("DB_USER"),
|
|
os.Getenv("DB_HOST"),
|
|
os.Getenv("DB_PORT"),
|
|
os.Getenv("DB_NAME"),
|
|
)
|
|
default:
|
|
return nil, fmt.Errorf("unsupported database driver: %s", os.Getenv("DB_DRIVER"))
|
|
}
|
|
db, err := gorm.Open(dialector, &gorm.Config{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Run database migrations automatically (optional, can be disabled via env var)
|
|
// To disable auto-migration, set AUTO_MIGRATE=false
|
|
// To enable auto-recovery from dirty state, set AUTO_RECOVER_DIRTY=true
|
|
if os.Getenv("AUTO_MIGRATE") != "false" {
|
|
logger.Infof(context.Background(), "Running database migrations...")
|
|
|
|
autoRecover := os.Getenv("AUTO_RECOVER_DIRTY") != "false"
|
|
migrationOpts := database.MigrationOptions{
|
|
AutoRecoverDirty: autoRecover,
|
|
}
|
|
|
|
// Run base migrations (all versioned migrations including embeddings)
|
|
// The embeddings migration will be conditionally executed based on skip_embedding parameter in DSN
|
|
if err := database.RunMigrationsWithOptions(migrateDSN, migrationOpts); err != nil {
|
|
// Log warning but don't fail startup - migrations might be handled externally
|
|
logger.Warnf(context.Background(), "Database migration failed: %v", err)
|
|
logger.Warnf(
|
|
context.Background(),
|
|
"Continuing with application startup. Please run migrations manually if needed.",
|
|
)
|
|
}
|
|
} else {
|
|
logger.Infof(context.Background(), "Auto-migration is disabled (AUTO_MIGRATE=false)")
|
|
}
|
|
|
|
// Get underlying SQL DB object
|
|
sqlDB, err := db.DB()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Configure connection pool parameters
|
|
sqlDB.SetMaxIdleConns(10)
|
|
sqlDB.SetConnMaxLifetime(time.Duration(10) * time.Minute)
|
|
|
|
return db, nil
|
|
}
|
|
|
|
// initFileService initializes file storage service
|
|
// Creates the appropriate file storage service based on configuration
|
|
// Supports multiple storage backends (MinIO, COS, local filesystem)
|
|
// Parameters:
|
|
// - cfg: Application configuration
|
|
//
|
|
// Returns:
|
|
// - Configured file service implementation
|
|
// - Error if initialization fails
|
|
func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
|
switch os.Getenv("STORAGE_TYPE") {
|
|
case "minio":
|
|
if os.Getenv("MINIO_ENDPOINT") == "" ||
|
|
os.Getenv("MINIO_ACCESS_KEY_ID") == "" ||
|
|
os.Getenv("MINIO_SECRET_ACCESS_KEY") == "" ||
|
|
os.Getenv("MINIO_BUCKET_NAME") == "" {
|
|
return nil, fmt.Errorf("missing MinIO configuration")
|
|
}
|
|
return file.NewMinioFileService(
|
|
os.Getenv("MINIO_ENDPOINT"),
|
|
os.Getenv("MINIO_ACCESS_KEY_ID"),
|
|
os.Getenv("MINIO_SECRET_ACCESS_KEY"),
|
|
os.Getenv("MINIO_BUCKET_NAME"),
|
|
strings.EqualFold(os.Getenv("MINIO_USE_SSL"), "true"),
|
|
)
|
|
case "cos":
|
|
if os.Getenv("COS_BUCKET_NAME") == "" ||
|
|
os.Getenv("COS_REGION") == "" ||
|
|
os.Getenv("COS_SECRET_ID") == "" ||
|
|
os.Getenv("COS_SECRET_KEY") == "" ||
|
|
os.Getenv("COS_PATH_PREFIX") == "" {
|
|
return nil, fmt.Errorf("missing COS configuration")
|
|
}
|
|
return file.NewCosFileServiceWithTempBucket(
|
|
os.Getenv("COS_BUCKET_NAME"),
|
|
os.Getenv("COS_REGION"),
|
|
os.Getenv("COS_SECRET_ID"),
|
|
os.Getenv("COS_SECRET_KEY"),
|
|
os.Getenv("COS_PATH_PREFIX"),
|
|
os.Getenv("COS_TEMP_BUCKET_NAME"), // 可选:临时桶名称(桶需配置生命周期规则自动过期)
|
|
os.Getenv("COS_TEMP_REGION"), // 可选:临时桶 region,默认与主桶相同
|
|
)
|
|
case "local":
|
|
return file.NewLocalFileService(os.Getenv("LOCAL_STORAGE_BASE_DIR")), nil
|
|
case "dummy":
|
|
return file.NewDummyFileService(), nil
|
|
default:
|
|
return nil, fmt.Errorf("unsupported storage type: %s", os.Getenv("STORAGE_TYPE"))
|
|
}
|
|
}
|
|
|
|
// initRetrieveEngineRegistry initializes the retrieval engine registry
|
|
// Sets up and configures various search engine backends based on configuration
|
|
// Supports multiple retrieval engines (PostgreSQL, ElasticsearchV7, ElasticsearchV8)
|
|
// Parameters:
|
|
// - db: Database connection
|
|
// - cfg: Application configuration
|
|
//
|
|
// Returns:
|
|
// - Configured retrieval engine registry
|
|
// - Error if initialization fails
|
|
func initRetrieveEngineRegistry(db *gorm.DB, cfg *config.Config) (interfaces.RetrieveEngineRegistry, error) {
|
|
registry := retriever.NewRetrieveEngineRegistry()
|
|
retrieveDriver := strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",")
|
|
log := logger.GetLogger(context.Background())
|
|
|
|
if slices.Contains(retrieveDriver, "postgres") {
|
|
postgresRepo := postgresRepo.NewPostgresRetrieveEngineRepository(db)
|
|
if err := registry.Register(
|
|
retriever.NewKVHybridRetrieveEngine(postgresRepo, types.PostgresRetrieverEngineType),
|
|
); err != nil {
|
|
log.Errorf("Register postgres retrieve engine failed: %v", err)
|
|
} else {
|
|
log.Infof("Register postgres retrieve engine success")
|
|
}
|
|
}
|
|
if slices.Contains(retrieveDriver, "elasticsearch_v8") {
|
|
client, err := elasticsearch.NewTypedClient(elasticsearch.Config{
|
|
Addresses: []string{os.Getenv("ELASTICSEARCH_ADDR")},
|
|
Username: os.Getenv("ELASTICSEARCH_USERNAME"),
|
|
Password: os.Getenv("ELASTICSEARCH_PASSWORD"),
|
|
})
|
|
if err != nil {
|
|
log.Errorf("Create elasticsearch_v8 client failed: %v", err)
|
|
} else {
|
|
elasticsearchRepo := elasticsearchRepoV8.NewElasticsearchEngineRepository(client, cfg)
|
|
if err := registry.Register(
|
|
retriever.NewKVHybridRetrieveEngine(
|
|
elasticsearchRepo, types.ElasticsearchRetrieverEngineType,
|
|
),
|
|
); err != nil {
|
|
log.Errorf("Register elasticsearch_v8 retrieve engine failed: %v", err)
|
|
} else {
|
|
log.Infof("Register elasticsearch_v8 retrieve engine success")
|
|
}
|
|
}
|
|
}
|
|
|
|
if slices.Contains(retrieveDriver, "elasticsearch_v7") {
|
|
client, err := esv7.NewClient(esv7.Config{
|
|
Addresses: []string{os.Getenv("ELASTICSEARCH_ADDR")},
|
|
Username: os.Getenv("ELASTICSEARCH_USERNAME"),
|
|
Password: os.Getenv("ELASTICSEARCH_PASSWORD"),
|
|
})
|
|
if err != nil {
|
|
log.Errorf("Create elasticsearch_v7 client failed: %v", err)
|
|
} else {
|
|
elasticsearchRepo := elasticsearchRepoV7.NewElasticsearchEngineRepository(client, cfg)
|
|
if err := registry.Register(
|
|
retriever.NewKVHybridRetrieveEngine(
|
|
elasticsearchRepo, types.ElasticsearchRetrieverEngineType,
|
|
),
|
|
); err != nil {
|
|
log.Errorf("Register elasticsearch_v7 retrieve engine failed: %v", err)
|
|
} else {
|
|
log.Infof("Register elasticsearch_v7 retrieve engine success")
|
|
}
|
|
}
|
|
}
|
|
|
|
if slices.Contains(retrieveDriver, "qdrant") {
|
|
qdrantHost := os.Getenv("QDRANT_HOST")
|
|
if qdrantHost == "" {
|
|
qdrantHost = "localhost"
|
|
}
|
|
|
|
qdrantPort := 6334 // Default port
|
|
if portStr := os.Getenv("QDRANT_PORT"); portStr != "" {
|
|
if port, err := strconv.Atoi(portStr); err == nil {
|
|
qdrantPort = port
|
|
}
|
|
}
|
|
|
|
// API key for authentication (optional)
|
|
qdrantAPIKey := os.Getenv("QDRANT_API_KEY")
|
|
|
|
// TLS configuration (optional, defaults to false)
|
|
// Enable TLS unless explicitly set to "false" or "0" (case insensitive)
|
|
qdrantUseTLS := false
|
|
if useTLSStr := os.Getenv("QDRANT_USE_TLS"); useTLSStr != "" {
|
|
useTLSLower := strings.ToLower(strings.TrimSpace(useTLSStr))
|
|
qdrantUseTLS = useTLSLower != "false" && useTLSLower != "0"
|
|
}
|
|
|
|
log.Infof("Connecting to Qdrant at %s:%d (TLS: %v)", qdrantHost, qdrantPort, qdrantUseTLS)
|
|
|
|
client, err := qdrant.NewClient(&qdrant.Config{
|
|
Host: qdrantHost,
|
|
Port: qdrantPort,
|
|
APIKey: qdrantAPIKey,
|
|
UseTLS: qdrantUseTLS,
|
|
})
|
|
if err != nil {
|
|
log.Errorf("Create qdrant client failed: %v", err)
|
|
} else {
|
|
qdrantRepository := qdrantRepo.NewQdrantRetrieveEngineRepository(client)
|
|
if err := registry.Register(
|
|
retriever.NewKVHybridRetrieveEngine(
|
|
qdrantRepository, types.QdrantRetrieverEngineType,
|
|
),
|
|
); err != nil {
|
|
log.Errorf("Register qdrant retrieve engine failed: %v", err)
|
|
} else {
|
|
log.Infof("Register qdrant retrieve engine success")
|
|
}
|
|
}
|
|
}
|
|
return registry, nil
|
|
}
|
|
|
|
// initAntsPool initializes the goroutine pool
|
|
// Creates a managed goroutine pool for concurrent task execution
|
|
// Parameters:
|
|
// - cfg: Application configuration
|
|
//
|
|
// Returns:
|
|
// - Configured goroutine pool
|
|
// - Error if initialization fails
|
|
func initAntsPool(cfg *config.Config) (*ants.Pool, error) {
|
|
// Default to 5 if not specified in config
|
|
poolSize := os.Getenv("CONCURRENCY_POOL_SIZE")
|
|
if poolSize == "" {
|
|
poolSize = "5"
|
|
}
|
|
poolSizeInt, err := strconv.Atoi(poolSize)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Set up the pool with pre-allocation for better performance
|
|
return ants.NewPool(poolSizeInt, ants.WithPreAlloc(true))
|
|
}
|
|
|
|
// registerPoolCleanup registers the goroutine pool for cleanup
|
|
// Ensures proper cleanup of the goroutine pool when application shuts down
|
|
// Parameters:
|
|
// - pool: Goroutine pool
|
|
// - cleaner: Resource cleaner
|
|
func registerPoolCleanup(pool *ants.Pool, cleaner interfaces.ResourceCleaner) {
|
|
cleaner.RegisterWithName("AntsPool", func() error {
|
|
pool.Release()
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// initDocReaderClient initializes the document reader client
|
|
// Creates a client for interacting with the document reader service
|
|
// Parameters:
|
|
// - cfg: Application configuration
|
|
//
|
|
// Returns:
|
|
// - Configured document reader client
|
|
// - Error if initialization fails
|
|
func initDocReaderClient(cfg *config.Config) (*client.Client, error) {
|
|
// Use the DocReader URL from environment or config
|
|
docReaderURL := os.Getenv("DOCREADER_ADDR")
|
|
if docReaderURL == "" && cfg.DocReader != nil {
|
|
docReaderURL = cfg.DocReader.Addr
|
|
}
|
|
return client.NewClient(docReaderURL)
|
|
}
|
|
|
|
// initOllamaService initializes the Ollama service client
|
|
// Creates a client for interacting with Ollama API for model inference
|
|
// Parameters:
|
|
// - None
|
|
//
|
|
// Returns:
|
|
// - Configured Ollama service client
|
|
// - Error if initialization fails
|
|
func initOllamaService() (*ollama.OllamaService, error) {
|
|
// Get Ollama service from existing factory function
|
|
return ollama.GetOllamaService()
|
|
}
|
|
|
|
func initNeo4jClient() (neo4j.Driver, error) {
|
|
ctx := context.Background()
|
|
if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" {
|
|
logger.Debugf(ctx, "NOT SUPPORT RETRIEVE GRAPH")
|
|
return nil, nil
|
|
}
|
|
uri := os.Getenv("NEO4J_URI")
|
|
username := os.Getenv("NEO4J_USERNAME")
|
|
password := os.Getenv("NEO4J_PASSWORD")
|
|
|
|
// Retry configuration
|
|
maxRetries := 30 // Max retry attempts
|
|
retryInterval := 2 * time.Second // Wait between retries
|
|
|
|
var driver neo4j.Driver
|
|
var err error
|
|
|
|
for attempt := 1; attempt <= maxRetries; attempt++ {
|
|
driver, err = neo4j.NewDriver(uri, neo4j.BasicAuth(username, password, ""))
|
|
if err != nil {
|
|
logger.Warnf(ctx, "Failed to create Neo4j driver (attempt %d/%d): %v", attempt, maxRetries, err)
|
|
time.Sleep(retryInterval)
|
|
continue
|
|
}
|
|
|
|
err = driver.VerifyAuthentication(ctx, nil)
|
|
if err == nil {
|
|
if attempt > 1 {
|
|
logger.Infof(ctx, "Successfully connected to Neo4j after %d attempts", attempt)
|
|
}
|
|
return driver, nil
|
|
}
|
|
|
|
logger.Warnf(ctx, "Failed to verify Neo4j authentication (attempt %d/%d): %v", attempt, maxRetries, err)
|
|
driver.Close(ctx)
|
|
time.Sleep(retryInterval)
|
|
}
|
|
|
|
return nil, fmt.Errorf("failed to connect to Neo4j after %d attempts: %w", maxRetries, err)
|
|
}
|
|
|
|
func NewDuckDB() (*sql.DB, error) {
|
|
sqlDB, err := sql.Open("duckdb", ":memory:")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open duckdb: %w", err)
|
|
}
|
|
|
|
// Try to load spatial extension
|
|
loadSQL := "LOAD spatial;"
|
|
if _, err := sqlDB.ExecContext(context.Background(), loadSQL); err != nil {
|
|
logger.Warnf(context.Background(), "[DuckDB] Failed to load spatial extension: %v", err)
|
|
}
|
|
|
|
return sqlDB, nil
|
|
}
|
|
|
|
// registerWebSearchProviders registers all web search providers to the registry
|
|
func registerWebSearchProviders(registry *web_search.Registry) {
|
|
// Register DuckDuckGo provider
|
|
registry.Register(web_search.DuckDuckGoProviderInfo(), func() (interfaces.WebSearchProvider, error) {
|
|
return web_search.NewDuckDuckGoProvider()
|
|
})
|
|
|
|
// Register Google provider
|
|
registry.Register(web_search.GoogleProviderInfo(), func() (interfaces.WebSearchProvider, error) {
|
|
return web_search.NewGoogleProvider()
|
|
})
|
|
|
|
// Register Bing provider
|
|
registry.Register(web_search.BingProviderInfo(), func() (interfaces.WebSearchProvider, error) {
|
|
return web_search.NewBingProvider()
|
|
})
|
|
}
|