From 7ca6fdbdb62509ef4e63ba5b9de3e7982dc06e09 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Fri, 5 Dec 2025 21:39:23 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20=E7=AE=80=E5=8C=96=E5=9B=BE=E7=89=87?= =?UTF-8?q?URL=E9=AA=8C=E8=AF=81=E9=80=BB=E8=BE=91=EF=BC=8C=E7=9B=B4?= =?UTF-8?q?=E6=8E=A5=E8=BF=94=E5=9B=9Etrue=E4=BB=A5=E6=8F=90=E9=AB=98?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/utils/security.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frontend/src/utils/security.ts b/frontend/src/utils/security.ts index b1eab2f0..6123b877 100644 --- a/frontend/src/utils/security.ts +++ b/frontend/src/utils/security.ts @@ -182,9 +182,7 @@ export function isValidImageURL(url: string): boolean { return false; } - // 检查是否为图片文件 - const imageExtensions = /\.(jpg|jpeg|png|gif|webp|svg|bmp|ico)(\?.*)?$/i; - return imageExtensions.test(url); + return true; } /** From ceed7087c28bfe43afb62464ef388c23ddb6947b Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 8 Dec 2025 17:40:10 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=E8=BF=81=E7=A7=BB=E8=84=8F=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=81=A2=E5=A4=8D=E5=92=8CNeo4j=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E9=87=8D=E8=AF=95=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 3 + docker-compose.dev.yml | 2 - docker-compose.yml | 3 +- internal/container/container.go | 45 +- internal/database/migration.go | 164 ++++- migrations/versioned/000000_init.down.sql | 62 ++ migrations/versioned/000000_init.up.sql | 281 +++++++++ migrations/versioned/000001_agent.up.sql | 736 +++++++++------------- scripts/migrate.sh | 7 +- 9 files changed, 829 insertions(+), 474 deletions(-) create mode 100644 migrations/versioned/000000_init.down.sql create mode 100644 migrations/versioned/000000_init.up.sql diff --git a/.env.example b/.env.example index d542d928..4dea80d9 100644 --- a/.env.example +++ b/.env.example @@ -54,6 +54,9 @@ REDIS_PREFIX=stream: # 当使用本地存储时,文件保存的基础目录路径 LOCAL_STORAGE_BASE_DIR=./data/files +# 是否自动恢复脏数据 +AUTO_RECOVER_DIRTY=true + TENANT_AES_KEY=weknorarag-api-key-secret-secret # 是否开启知识图谱构建和检索(构建阶段需调用大模型,耗时较长) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5dd3761e..0a7410f6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -12,8 +12,6 @@ services: - POSTGRES_DB=${DB_NAME} volumes: - postgres-data-dev:/var/lib/postgresql/data - - ./migrations/paradedb/00-init-db.sql:/docker-entrypoint-initdb.d/00-init-db.sql - - ./migrations/paradedb/01-migrate-to-paradedb.sql:/docker-entrypoint-initdb.d/01-migrate-to-paradedb.sql networks: - WeKnora-network-dev healthcheck: diff --git a/docker-compose.yml b/docker-compose.yml index 69e0a1d8..930a4240 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,6 +59,7 @@ services: - DOCREADER_ADDR=docreader:50051 - STORAGE_TYPE=${STORAGE_TYPE:-} - LOCAL_STORAGE_BASE_DIR=${LOCAL_STORAGE_BASE_DIR:-} + - AUTO_RECOVER_DIRTY=${AUTO_RECOVER_DIRTY:-true} - MINIO_ENDPOINT=minio:9000 - MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin} - MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin} @@ -151,8 +152,6 @@ services: - POSTGRES_DB=${DB_NAME} volumes: - postgres-data:/var/lib/postgresql/data - - ./migrations/paradedb/00-init-db.sql:/docker-entrypoint-initdb.d/00-init-db.sql - - ./migrations/paradedb/01-migrate-to-paradedb.sql:/docker-entrypoint-initdb.d/01-migrate-to-paradedb.sql networks: - WeKnora-network healthcheck: diff --git a/internal/container/container.go b/internal/container/container.go index add41207..6f5fa41c 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -279,9 +279,13 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { // Run database migrations automatically (optional, can be disabled via env var) // To disable auto-migration, set AUTO_MIGRATE=false + // To enable auto-recovery from dirty state, set AUTO_RECOVER_DIRTY=true if os.Getenv("AUTO_MIGRATE") != "false" { logger.Infof(context.Background(), "Running database migrations...") - if err := database.RunMigrations(migrateDSN); err != nil { + autoRecover := os.Getenv("AUTO_RECOVER_DIRTY") == "true" + if err := database.RunMigrationsWithOptions(migrateDSN, database.MigrationOptions{ + AutoRecoverDirty: autoRecover, + }); err != nil { // Log warning but don't fail startup - migrations might be handled externally logger.Warnf(context.Background(), "Database migration failed: %v", err) logger.Warnf( @@ -539,21 +543,42 @@ func initOllamaService() (*ollama.OllamaService, error) { } func initNeo4jClient() (neo4j.Driver, error) { + ctx := context.Background() if strings.ToLower(os.Getenv("NEO4J_ENABLE")) != "true" { - logger.Debugf(context.Background(), "NOT SUPPORT RETRIEVE GRAPH") + logger.Debugf(ctx, "NOT SUPPORT RETRIEVE GRAPH") return nil, nil } uri := os.Getenv("NEO4J_URI") username := os.Getenv("NEO4J_USERNAME") password := os.Getenv("NEO4J_PASSWORD") - driver, err := neo4j.NewDriver(uri, neo4j.BasicAuth(username, password, "")) - if err != nil { - return nil, err + // Retry configuration + maxRetries := 30 // Max retry attempts + retryInterval := 2 * time.Second // Wait between retries + + var driver neo4j.Driver + var err error + + for attempt := 1; attempt <= maxRetries; attempt++ { + driver, err = neo4j.NewDriver(uri, neo4j.BasicAuth(username, password, "")) + if err != nil { + logger.Warnf(ctx, "Failed to create Neo4j driver (attempt %d/%d): %v", attempt, maxRetries, err) + time.Sleep(retryInterval) + continue + } + + err = driver.VerifyAuthentication(ctx, nil) + if err == nil { + if attempt > 1 { + logger.Infof(ctx, "Successfully connected to Neo4j after %d attempts", attempt) + } + return driver, nil + } + + logger.Warnf(ctx, "Failed to verify Neo4j authentication (attempt %d/%d): %v", attempt, maxRetries, err) + driver.Close(ctx) + time.Sleep(retryInterval) } - err = driver.VerifyAuthentication(context.Background(), nil) - if err != nil { - return nil, err - } - return driver, nil + + return nil, fmt.Errorf("failed to connect to Neo4j after %d attempts: %w", maxRetries, err) } diff --git a/internal/database/migration.go b/internal/database/migration.go index 47ba3b9a..2cab9bae 100644 --- a/internal/database/migration.go +++ b/internal/database/migration.go @@ -14,11 +14,28 @@ import ( // RunMigrations executes all pending database migrations // This should be called during application startup func RunMigrations(dsn string) error { + return RunMigrationsWithOptions(dsn, MigrationOptions{AutoRecoverDirty: false}) +} + +// MigrationOptions configures migration behavior +type MigrationOptions struct { + // AutoRecoverDirty when true, automatically attempts to recover from dirty state + // by forcing to the previous version and retrying the migration + AutoRecoverDirty bool +} + +// RunMigrationsWithOptions executes all pending database migrations with custom options +func RunMigrationsWithOptions(dsn string, opts MigrationOptions) error { + ctx := context.Background() + + logger.Infof(ctx, "Starting database migration...") + // Use versioned migrations directory migrationsPath := "file://migrations/versioned" m, err := migrate.New(migrationsPath, dsn) if err != nil { + logger.Errorf(ctx, "Failed to create migrate instance: %v", err) return fmt.Errorf("failed to create migrate instance: %w", err) } defer m.Close() @@ -26,57 +43,93 @@ func RunMigrations(dsn string) error { // Check current version and dirty state before migration oldVersion, oldDirty, versionErr := m.Version() if versionErr != nil && versionErr != migrate.ErrNilVersion { - // If we can't get version and it's not because database is empty, return error + logger.Errorf(ctx, "Failed to get migration version: %v", versionErr) return fmt.Errorf("failed to get migration version: %w", versionErr) } - // If database is in dirty state, return detailed error with fix instructions - if oldDirty { - // Calculate the version to force to (usually the previous version) - forceVersion := int(oldVersion) - 1 - if oldVersion == 0 || forceVersion < 0 { - forceVersion = 0 - } - return fmt.Errorf( - "database is in dirty state at version %d. This usually means a migration failed partway through. "+ - "To fix this:\n"+ - "1. Check if the migration partially applied changes and manually fix if needed\n"+ - "2. Use the force command to set the version to the last successful migration (usually %d):\n"+ - " ./scripts/migrate.sh force %d\n"+ - " Or if using make: make migrate-force version=%d\n"+ - "3. After fixing, restart the application to retry the migration", - oldVersion, - forceVersion, - forceVersion, - forceVersion, - ) + if versionErr == migrate.ErrNilVersion { + logger.Infof(ctx, "Database has no migration history, will start from version 0") + } else { + logger.Infof(ctx, "Current migration version: %d, dirty: %v", oldVersion, oldDirty) } - // Run all pending migrations - if err := m.Up(); err != nil && err != migrate.ErrNoChange { - // Check if error is due to dirty state (in case it became dirty during migration) - currentVersion, currentDirty, versionCheckErr := m.Version() - if versionCheckErr == nil && currentDirty { + // If database is in dirty state, try to recover or return error + if oldDirty { + logger.Warnf(ctx, "Database is in dirty state at version %d", oldVersion) + if opts.AutoRecoverDirty { + logger.Infof(ctx, "AutoRecoverDirty is enabled, attempting recovery...") + if err := recoverFromDirtyState(ctx, m, oldVersion); err != nil { + return err + } + // Update oldVersion after recovery + oldVersion, _, _ = m.Version() + } else { // Calculate the version to force to (usually the previous version) - forceVersion := currentVersion - 1 - if currentVersion == 0 { + forceVersion := int(oldVersion) - 1 + if oldVersion == 0 || forceVersion < 0 { forceVersion = 0 } return fmt.Errorf( - "migration failed and database is now in dirty state at version %d. "+ + "database is in dirty state at version %d. This usually means a migration failed partway through. "+ "To fix this:\n"+ "1. Check if the migration partially applied changes and manually fix if needed\n"+ "2. Use the force command to set the version to the last successful migration (usually %d):\n"+ " ./scripts/migrate.sh force %d\n"+ " Or if using make: make migrate-force version=%d\n"+ - "3. After fixing, restart the application to retry the migration", - currentVersion, + "3. After fixing, restart the application to retry the migration\n"+ + "Or enable AutoRecoverDirty option to automatically retry", + oldVersion, forceVersion, forceVersion, forceVersion, ) } - return fmt.Errorf("failed to run migrations: %w", err) + } + + // Run all pending migrations + logger.Infof(ctx, "Running pending migrations...") + if err := m.Up(); err != nil && err != migrate.ErrNoChange { + logger.Errorf(ctx, "Migration failed: %v", err) + // Check if error is due to dirty state (in case it became dirty during migration) + currentVersion, currentDirty, versionCheckErr := m.Version() + if versionCheckErr == nil && currentDirty { + logger.Warnf(ctx, "Migration caused dirty state at version %d", currentVersion) + if opts.AutoRecoverDirty { + logger.Infof(ctx, "Attempting to recover from dirty state...") + // Try to recover and retry + if recoverErr := recoverFromDirtyState(ctx, m, currentVersion); recoverErr != nil { + return recoverErr + } + // Retry migration after recovery + logger.Infof(ctx, "Retrying migration after recovery...") + if retryErr := m.Up(); retryErr != nil && retryErr != migrate.ErrNoChange { + logger.Errorf(ctx, "Migration failed after recovery attempt: %v", retryErr) + return fmt.Errorf("migration failed after recovery attempt: %w", retryErr) + } + } else { + // Calculate the version to force to (usually the previous version) + forceVersion := currentVersion - 1 + if currentVersion == 0 { + forceVersion = 0 + } + return fmt.Errorf( + "migration failed and database is now in dirty state at version %d. "+ + "To fix this:\n"+ + "1. Check if the migration partially applied changes and manually fix if needed\n"+ + "2. Use the force command to set the version to the last successful migration (usually %d):\n"+ + " ./scripts/migrate.sh force %d\n"+ + " Or if using make: make migrate-force version=%d\n"+ + "3. After fixing, restart the application to retry the migration\n"+ + "Or enable AutoRecoverDirty option to automatically retry", + currentVersion, + forceVersion, + forceVersion, + forceVersion, + ) + } + } else { + return fmt.Errorf("failed to run migrations: %w", err) + } } // Get current version after migration @@ -86,18 +139,59 @@ func RunMigrations(dsn string) error { } if oldVersion != version { - logger.Infof(context.Background(), "Database migrated from version %d to %d", oldVersion, version) + logger.Infof(ctx, "Database migrated from version %d to %d", oldVersion, version) } else { - logger.Infof(context.Background(), "Database is up to date (version: %d)", version) + logger.Infof(ctx, "Database is up to date (version: %d)", version) } if dirty { - logger.Warnf(context.Background(), "Database is in dirty state! Manual intervention may be required.") + logger.Warnf(ctx, "Database is in dirty state! Manual intervention may be required.") } return nil } +// recoverFromDirtyState attempts to recover from a dirty migration state +// by forcing to the previous version and allowing the migration to be retried +func recoverFromDirtyState(ctx context.Context, m *migrate.Migrate, dirtyVersion uint) error { + // Special case: if dirty at version 0 (init migration), we cannot go back further + // The only option is to force to version 0 and retry, but this requires the migration to be idempotent + if dirtyVersion == 0 { + logger.Warnf(ctx, "Database is in dirty state at version 0 (init migration). "+ + "This is the initial migration, cannot rollback further. "+ + "Will attempt to clear dirty flag and retry. "+ + "Note: This only works if the init migration uses IF NOT EXISTS clauses.") + + // Force to version -1 (no version) to allow re-running version 0 + // This effectively tells migrate that no migrations have been applied + if err := m.Force(-1); err != nil { + return fmt.Errorf( + "failed to recover from dirty state at version 0. "+ + "Manual intervention required:\n"+ + "1. Check what was partially created in the database\n"+ + "2. Either drop all created objects and retry, or\n"+ + "3. Manually complete the migration and run: ./scripts/migrate.sh force 0\n"+ + "Error: %w", err) + } + + logger.Infof(ctx, "Cleared migration state, will retry from version 0") + return nil + } + + forceVersion := int(dirtyVersion) - 1 + + logger.Warnf(ctx, "Database is in dirty state at version %d, attempting auto-recovery by forcing to version %d", + dirtyVersion, forceVersion) + + // Force to previous version to clear dirty state + if err := m.Force(forceVersion); err != nil { + return fmt.Errorf("failed to force migration version during recovery: %w", err) + } + + logger.Infof(ctx, "Successfully forced migration to version %d, migration will be retried", forceVersion) + return nil +} + // GetMigrationVersion returns the current migration version func GetMigrationVersion() (uint, bool, error) { dbURL := fmt.Sprintf( diff --git a/migrations/versioned/000000_init.down.sql b/migrations/versioned/000000_init.down.sql new file mode 100644 index 00000000..34a8bae5 --- /dev/null +++ b/migrations/versioned/000000_init.down.sql @@ -0,0 +1,62 @@ +-- Drop indexes for embeddings +DROP INDEX IF EXISTS embeddings_unique_source; +DROP INDEX IF EXISTS embeddings_search_idx; + +-- Drop embeddings table +DROP TABLE IF EXISTS embeddings; + +-- Drop indexes for chunks +DROP INDEX IF EXISTS idx_chunks_tenant_kg; +DROP INDEX IF EXISTS idx_chunks_parent_id; +DROP INDEX IF EXISTS idx_chunks_chunk_type; + +-- Drop chunks table +DROP TABLE IF EXISTS chunks; + +-- Drop indexes for messages +DROP INDEX IF EXISTS idx_messages_session_id; + +-- Drop messages table +DROP TABLE IF EXISTS messages; + +-- Drop indexes for sessions +DROP INDEX IF EXISTS idx_sessions_tenant_id; + +-- Drop sessions table +DROP TABLE IF EXISTS sessions; + +-- Drop indexes for knowledges +DROP INDEX IF EXISTS idx_knowledges_tenant_id; +DROP INDEX IF EXISTS idx_knowledges_base_id; +DROP INDEX IF EXISTS idx_knowledges_parse_status; +DROP INDEX IF EXISTS idx_knowledges_enable_status; + +-- Drop knowledges table +DROP TABLE IF EXISTS knowledges; + +-- Drop indexes for knowledge_bases +DROP INDEX IF EXISTS idx_knowledge_bases_tenant_id; + +-- Drop knowledge_bases table +DROP TABLE IF EXISTS knowledge_bases; + +-- Drop indexes for models +DROP INDEX IF EXISTS idx_models_type; +DROP INDEX IF EXISTS idx_models_source; + +-- Drop models table +DROP TABLE IF EXISTS models; + +-- Drop indexes for tenants +DROP INDEX IF EXISTS idx_tenants_api_key; +DROP INDEX IF EXISTS idx_tenants_status; + +-- Drop tenants table +DROP TABLE IF EXISTS tenants; + +-- Note: Extensions are not dropped as they may be used by other databases/schemas +-- If you want to drop extensions, uncomment the following lines: +-- DROP EXTENSION IF EXISTS pg_search; +-- DROP EXTENSION IF EXISTS pg_trgm; +-- DROP EXTENSION IF EXISTS vector; +-- DROP EXTENSION IF EXISTS "uuid-ossp"; diff --git a/migrations/versioned/000000_init.up.sql b/migrations/versioned/000000_init.up.sql new file mode 100644 index 00000000..24f70e5b --- /dev/null +++ b/migrations/versioned/000000_init.up.sql @@ -0,0 +1,281 @@ +-- Migration: 000000_init +-- Description: Initialize database schema +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Starting initial database setup...'; END $$; + +-- Create extensions +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating extensions...'; END $$; +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +CREATE EXTENSION IF NOT EXISTS vector; +CREATE EXTENSION IF NOT EXISTS pg_trgm; +CREATE EXTENSION IF NOT EXISTS pg_search; + +-- Create tenant table +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: tenants'; END $$; +CREATE TABLE IF NOT EXISTS tenants ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description TEXT, + api_key VARCHAR(64) NOT NULL, + retriever_engines JSONB NOT NULL DEFAULT '[]', + status VARCHAR(50) DEFAULT 'active', + business VARCHAR(255) NOT NULL, + storage_quota BIGINT NOT NULL DEFAULT 10737418240, -- 默认10GB配额(Bytes) + storage_used BIGINT NOT NULL DEFAULT 0, -- 已使用的存储空间(Bytes) + agent_config JSONB DEFAULT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +COMMENT ON COLUMN tenants.agent_config IS 'Tenant-level agent configuration in JSON format'; + +-- Set the starting value for tenants id sequence (only if current value is less than 10000) +DO $$ +DECLARE + current_val BIGINT; +BEGIN + SELECT last_value INTO current_val FROM tenants_id_seq; + IF current_val < 10000 THEN + ALTER SEQUENCE tenants_id_seq RESTART WITH 10000; + RAISE NOTICE '[Migration 000000] Set tenants_id_seq to start at 10000'; + ELSE + RAISE NOTICE '[Migration 000000] tenants_id_seq already at % (>= 10000), skipping', current_val; + END IF; +EXCEPTION + WHEN undefined_table THEN + -- Sequence doesn't exist yet, will be created with table + RAISE NOTICE '[Migration 000000] tenants_id_seq not found, will be created with table'; +END $$; + +-- Add indexes +CREATE INDEX IF NOT EXISTS idx_tenants_api_key ON tenants(api_key); +CREATE INDEX IF NOT EXISTS idx_tenants_status ON tenants(status); + +-- Create model table +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: models'; END $$; +CREATE TABLE IF NOT EXISTS models ( + id VARCHAR(64) PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_id INTEGER NOT NULL, + name VARCHAR(255) NOT NULL, + type VARCHAR(50) NOT NULL, + source VARCHAR(50) NOT NULL, + description TEXT, + parameters JSONB NOT NULL, + is_default BOOLEAN NOT NULL DEFAULT false, + status VARCHAR(50) NOT NULL DEFAULT 'active', + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +-- Add indexes for models +CREATE INDEX IF NOT EXISTS idx_models_type ON models(type); +CREATE INDEX IF NOT EXISTS idx_models_source ON models(source); + +-- Create knowledge_base table +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: knowledge_bases'; END $$; +CREATE TABLE IF NOT EXISTS knowledge_bases ( + id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), + name VARCHAR(255) NOT NULL, + description TEXT, + tenant_id INTEGER NOT NULL, + chunking_config JSONB NOT NULL DEFAULT '{"chunk_size": 512, "chunk_overlap": 50, "split_markers": ["\n\n", "\n", "。"], "keep_separator": true}', + image_processing_config JSONB NOT NULL DEFAULT '{"enable_multimodal": false, "model_id": ""}', + embedding_model_id VARCHAR(64) NOT NULL, + summary_model_id VARCHAR(64) NOT NULL, + rerank_model_id VARCHAR(64) NOT NULL, + cos_config JSONB NOT NULL DEFAULT '{}', + vlm_config JSONB NOT NULL DEFAULT '{}', + extract_config JSONB NULL DEFAULT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +-- Add indexes for knowledge_bases +CREATE INDEX IF NOT EXISTS idx_knowledge_bases_tenant_id ON knowledge_bases(tenant_id); + +-- Create knowledge table +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: knowledges'; END $$; +CREATE TABLE IF NOT EXISTS knowledges ( + id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + type VARCHAR(50) NOT NULL, + title VARCHAR(255) NOT NULL, + description TEXT, + source VARCHAR(128) NOT NULL, + parse_status VARCHAR(50) NOT NULL DEFAULT 'unprocessed', + enable_status VARCHAR(50) NOT NULL DEFAULT 'enabled', + embedding_model_id VARCHAR(64), + file_name VARCHAR(255), + file_type VARCHAR(50), + file_size BIGINT, + file_path TEXT, + file_hash VARCHAR(64), + storage_size BIGINT NOT NULL DEFAULT 0, -- 存储大小(Byte) + metadata JSONB, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + processed_at TIMESTAMP WITH TIME ZONE, + error_message TEXT, + deleted_at TIMESTAMP WITH TIME ZONE +); + +-- Add indexes for knowledge +CREATE INDEX IF NOT EXISTS idx_knowledges_tenant_id ON knowledges(tenant_id); +CREATE INDEX IF NOT EXISTS idx_knowledges_base_id ON knowledges(knowledge_base_id); +CREATE INDEX IF NOT EXISTS idx_knowledges_parse_status ON knowledges(parse_status); +CREATE INDEX IF NOT EXISTS idx_knowledges_enable_status ON knowledges(enable_status); + +-- Create session table +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: sessions'; END $$; +CREATE TABLE IF NOT EXISTS sessions ( + id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_id INTEGER NOT NULL, + title VARCHAR(255), + description TEXT, + knowledge_base_id VARCHAR(36), + max_rounds INTEGER NOT NULL DEFAULT 5, + enable_rewrite BOOLEAN NOT NULL DEFAULT true, + fallback_strategy VARCHAR(255) NOT NULL DEFAULT 'fixed', + fallback_response TEXT NOT NULL DEFAULT '很抱歉,我暂时无法回答这个问题。', + keyword_threshold FLOAT NOT NULL DEFAULT 0.5, + vector_threshold FLOAT NOT NULL DEFAULT 0.5, + rerank_model_id VARCHAR(64), + embedding_top_k INTEGER NOT NULL DEFAULT 10, + rerank_top_k INTEGER NOT NULL DEFAULT 10, + rerank_threshold FLOAT NOT NULL DEFAULT 0.65, + summary_model_id VARCHAR(64), + summary_parameters JSONB NOT NULL DEFAULT '{}', + agent_config JSONB DEFAULT NULL, + context_config JSONB DEFAULT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +COMMENT ON COLUMN sessions.agent_config IS 'Session-level agent configuration in JSON format'; +COMMENT ON COLUMN sessions.context_config IS 'LLM context management configuration (separate from message storage)'; + +-- Create Index for sessions +CREATE INDEX IF NOT EXISTS idx_sessions_tenant_id ON sessions(tenant_id); + + +-- Create message table +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: messages'; END $$; +CREATE TABLE IF NOT EXISTS messages ( + id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), + request_id VARCHAR(36) NOT NULL, + session_id VARCHAR(36) NOT NULL, + role VARCHAR(50) NOT NULL, + content TEXT NOT NULL, + knowledge_references JSONB NOT NULL DEFAULT '[]', + agent_steps JSONB DEFAULT NULL, + is_completed BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +COMMENT ON COLUMN messages.agent_steps IS 'Agent execution steps (reasoning process and tool calls)'; + +-- Create Index for messages +CREATE INDEX IF NOT EXISTS idx_messages_session_id ON messages(session_id); + +-- Create chunks table +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: chunks'; END $$; +CREATE TABLE IF NOT EXISTS chunks ( + id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_id INTEGER NOT NULL, + knowledge_base_id VARCHAR(36) NOT NULL, + knowledge_id VARCHAR(36) NOT NULL, + content TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + is_enabled BOOLEAN NOT NULL DEFAULT true, + start_at INTEGER NOT NULL, + end_at INTEGER NOT NULL, + pre_chunk_id VARCHAR(36), + next_chunk_id VARCHAR(36), + chunk_type VARCHAR(20) NOT NULL DEFAULT 'text', + parent_chunk_id VARCHAR(36), + image_info TEXT, + relation_chunks JSONB, + indirect_relation_chunks JSONB, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP WITH TIME ZONE +); + +CREATE INDEX IF NOT EXISTS idx_chunks_tenant_kg ON chunks(tenant_id, knowledge_id); +CREATE INDEX IF NOT EXISTS idx_chunks_parent_id ON chunks(parent_chunk_id); +CREATE INDEX IF NOT EXISTS idx_chunks_chunk_type ON chunks(chunk_type); + +-- Create embeddings table +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating table: embeddings'; END $$; +CREATE TABLE IF NOT EXISTS embeddings ( + id SERIAL PRIMARY KEY, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + + source_id VARCHAR(64) NOT NULL, + source_type INTEGER NOT NULL, + chunk_id VARCHAR(64), + knowledge_id VARCHAR(64), + knowledge_base_id VARCHAR(64), + content TEXT, + dimension INTEGER NOT NULL, + embedding halfvec +); + +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Creating indexes for embeddings (this may take a while)...'; END $$; +CREATE UNIQUE INDEX IF NOT EXISTS embeddings_unique_source ON embeddings(source_id, source_type); + +-- Create BM25 search index (check if exists first) +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'embeddings_search_idx') THEN + CREATE INDEX embeddings_search_idx ON embeddings + USING bm25 (id, knowledge_base_id, content, knowledge_id, chunk_id) + WITH ( + key_field = 'id', + text_fields = '{ + "content": { + "tokenizer": {"type": "chinese_lindera"} + } + }' + ); + RAISE NOTICE '[Migration 000000] Created BM25 index embeddings_search_idx'; + ELSE + RAISE NOTICE '[Migration 000000] BM25 index embeddings_search_idx already exists'; + END IF; +END $$; + +-- Create HNSW indexes for vector search (check if exists first) +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'embeddings_embedding_idx' OR indexname LIKE 'embeddings_embedding%3584%') THEN + CREATE INDEX embeddings_embedding_idx_3584 ON embeddings + USING hnsw ((embedding::halfvec(3584)) halfvec_cosine_ops) + WITH (m = 16, ef_construction = 64) + WHERE (dimension = 3584); + RAISE NOTICE '[Migration 000000] Created HNSW index for dimension 3584'; + ELSE + RAISE NOTICE '[Migration 000000] HNSW index for dimension 3584 already exists'; + END IF; +END $$; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'embeddings_embedding_idx_798' OR indexname LIKE 'embeddings_embedding%798%') THEN + CREATE INDEX embeddings_embedding_idx_798 ON embeddings + USING hnsw ((embedding::halfvec(798)) halfvec_cosine_ops) + WITH (m = 16, ef_construction = 64) + WHERE (dimension = 798); + RAISE NOTICE '[Migration 000000] Created HNSW index for dimension 798'; + ELSE + RAISE NOTICE '[Migration 000000] HNSW index for dimension 798 already exists'; + END IF; +END $$; + +DO $$ BEGIN RAISE NOTICE '[Migration 000000] Initial database setup completed successfully!'; END $$; diff --git a/migrations/versioned/000001_agent.up.sql b/migrations/versioned/000001_agent.up.sql index 4cbea2ef..46d7382d 100644 --- a/migrations/versioned/000001_agent.up.sql +++ b/migrations/versioned/000001_agent.up.sql @@ -1,10 +1,17 @@ -BEGIN; +-- Migration: 000001_agent +-- Description: Add user authentication, agent config, MCP services and other enhancements --- Create users table +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Starting agent and authentication migration...'; END $$; + +-- ============================================================================ +-- Section 1: User Authentication Tables +-- ============================================================================ + +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Creating table: users'; END $$; CREATE TABLE IF NOT EXISTS users ( id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), - username VARCHAR(100) NOT NULL UNIQUE, - email VARCHAR(255) NOT NULL UNIQUE, + username VARCHAR(100) NOT NULL, + email VARCHAR(255) NOT NULL, password_hash VARCHAR(255) NOT NULL, avatar VARCHAR(500), tenant_id INTEGER, @@ -14,6 +21,19 @@ CREATE TABLE IF NOT EXISTS users ( deleted_at TIMESTAMP WITH TIME ZONE ); +-- Add unique constraints if not exists +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'users_username_key') THEN + ALTER TABLE users ADD CONSTRAINT users_username_key UNIQUE (username); + RAISE NOTICE '[Migration 000001] Added unique constraint on users.username'; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'users_email_key') THEN + ALTER TABLE users ADD CONSTRAINT users_email_key UNIQUE (email); + RAISE NOTICE '[Migration 000001] Added unique constraint on users.email'; + END IF; +END $$; + COMMENT ON TABLE users IS 'User accounts in the system'; COMMENT ON COLUMN users.id IS 'Unique identifier of the user'; COMMENT ON COLUMN users.username IS 'Username of the user'; @@ -29,23 +49,20 @@ CREATE INDEX IF NOT EXISTS idx_users_email ON users(email); CREATE INDEX IF NOT EXISTS idx_users_tenant_id ON users(tenant_id); CREATE INDEX IF NOT EXISTS idx_users_deleted_at ON users(deleted_at); --- Add foreign key constraint for tenant_id (only if not exists) +-- Add foreign key constraint for tenant_id DO $$ BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_constraint - WHERE conname = 'fk_users_tenant' - ) THEN - ALTER TABLE users - ADD CONSTRAINT fk_users_tenant + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_users_tenant') THEN + ALTER TABLE users ADD CONSTRAINT fk_users_tenant FOREIGN KEY (tenant_id) REFERENCES tenants(id) ON DELETE SET NULL; - RAISE NOTICE 'Added foreign key constraint fk_users_tenant'; - ELSE - RAISE NOTICE 'Foreign key constraint fk_users_tenant already exists'; + RAISE NOTICE '[Migration 000001] Added foreign key constraint fk_users_tenant'; END IF; END $$; --- Create auth_tokens table +-- Add can_access_all_tenants column to users +ALTER TABLE users ADD COLUMN IF NOT EXISTS can_access_all_tenants BOOLEAN NOT NULL DEFAULT FALSE; + +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Creating table: auth_tokens'; END $$; CREATE TABLE IF NOT EXISTS auth_tokens ( id VARCHAR(36) PRIMARY KEY DEFAULT uuid_generate_v4(), user_id VARCHAR(36) NOT NULL, @@ -71,335 +88,174 @@ CREATE INDEX IF NOT EXISTS idx_auth_tokens_token ON auth_tokens(token); CREATE INDEX IF NOT EXISTS idx_auth_tokens_token_type ON auth_tokens(token_type); CREATE INDEX IF NOT EXISTS idx_auth_tokens_expires_at ON auth_tokens(expires_at); --- Add foreign key constraint (only if not exists) +-- Add foreign key constraint for auth_tokens DO $$ BEGIN - IF NOT EXISTS ( - SELECT 1 FROM pg_constraint - WHERE conname = 'fk_auth_tokens_user' - ) THEN - ALTER TABLE auth_tokens - ADD CONSTRAINT fk_auth_tokens_user + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_auth_tokens_user') THEN + ALTER TABLE auth_tokens ADD CONSTRAINT fk_auth_tokens_user FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; - RAISE NOTICE 'Added foreign key constraint fk_auth_tokens_user'; - ELSE - RAISE NOTICE 'Foreign key constraint fk_auth_tokens_user already exists'; + RAISE NOTICE '[Migration 000001] Added foreign key constraint fk_auth_tokens_user'; END IF; END $$; --- Add is_temporary column to knowledge_bases -ALTER TABLE knowledge_bases - ADD COLUMN IF NOT EXISTS is_temporary BOOLEAN NOT NULL DEFAULT false; +-- ============================================================================ +-- Section 2: Tenant Configuration Enhancements +-- ============================================================================ -COMMENT ON COLUMN knowledge_bases.is_temporary IS 'Whether this knowledge base is temporary (ephemeral) and should be hidden from UI'; +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding tenant configuration columns...'; END $$; -- Add context_config column to tenants -ALTER TABLE tenants - ADD COLUMN IF NOT EXISTS context_config JSONB; - +ALTER TABLE tenants ADD COLUMN IF NOT EXISTS context_config JSONB; COMMENT ON COLUMN tenants.context_config IS 'Global Context configuration for this tenant (default for all sessions)'; -- Add conversation_config column to tenants -ALTER TABLE tenants - ADD COLUMN IF NOT EXISTS conversation_config JSONB; - +ALTER TABLE tenants ADD COLUMN IF NOT EXISTS conversation_config JSONB; COMMENT ON COLUMN tenants.conversation_config IS 'Global Conversation configuration for this tenant (default for normal mode sessions)'; --- For tenants table: add agent_config +-- Add web_search_config column to tenants +ALTER TABLE tenants ADD COLUMN IF NOT EXISTS web_search_config JSONB DEFAULT NULL; +COMMENT ON COLUMN tenants.web_search_config IS 'Web search configuration for the tenant'; + +-- Ensure agent_config exists and is JSONB type DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name = 'tenants' - AND column_name = 'agent_config' - ) THEN - ALTER TABLE tenants - ADD COLUMN agent_config JSONB DEFAULT NULL; - - COMMENT ON COLUMN tenants.agent_config IS 'Tenant-level agent configuration in JSON format'; - - RAISE NOTICE 'Added agent_config column to tenants table'; - ELSE - -- If field exists but type is JSON, convert to JSONB - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name = 'tenants' - AND column_name = 'agent_config' - AND data_type = 'json' - ) THEN - ALTER TABLE tenants - ALTER COLUMN agent_config TYPE JSONB USING agent_config::jsonb; - - RAISE NOTICE 'Converted tenants.agent_config from JSON to JSONB'; - ELSE - RAISE NOTICE 'agent_config column already exists in tenants table'; - END IF; - END IF; -END $$; - --- For sessions table: add agent_config -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name = 'sessions' - AND column_name = 'agent_config' - ) THEN - ALTER TABLE sessions - ADD COLUMN agent_config JSONB DEFAULT NULL; - - COMMENT ON COLUMN sessions.agent_config IS 'Session-level agent configuration in JSON format'; - - RAISE NOTICE 'Added agent_config column to sessions table'; - ELSE - RAISE NOTICE 'agent_config column already exists in sessions table'; - END IF; -END $$; - --- For sessions table: add context_config -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name = 'sessions' - AND column_name = 'context_config' - ) THEN - ALTER TABLE sessions - ADD COLUMN context_config JSONB DEFAULT NULL; - - COMMENT ON COLUMN sessions.context_config IS 'LLM context management configuration (separate from message storage)'; - - RAISE NOTICE 'Added context_config column to sessions table'; - ELSE - RAISE NOTICE 'context_config column already exists in sessions table'; - END IF; -END $$; - --- For messages table: add agent_steps -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name = 'messages' - AND column_name = 'agent_steps' - ) THEN - ALTER TABLE messages - ADD COLUMN agent_steps JSONB DEFAULT NULL; - - COMMENT ON COLUMN messages.agent_steps IS 'Agent execution steps (reasoning process and tool calls)'; - - RAISE NOTICE 'Added agent_steps column to messages table'; - ELSE - RAISE NOTICE 'agent_steps column already exists in messages table'; - END IF; -END $$; - --- Add GIN indexes for JSON fields -DO $$ -BEGIN - -- For tenants.agent_config - IF NOT EXISTS ( - SELECT 1 FROM pg_indexes - WHERE tablename = 'tenants' - AND indexname = 'idx_tenants_agent_config' - ) THEN - IF EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name = 'tenants' - AND column_name = 'agent_config' - AND data_type = 'jsonb' - ) THEN - CREATE INDEX idx_tenants_agent_config ON tenants USING GIN (agent_config); - RAISE NOTICE 'Created index idx_tenants_agent_config'; - ELSE - RAISE NOTICE 'Skipped index creation for tenants.agent_config (not JSONB type)'; - END IF; - ELSE - RAISE NOTICE 'Index idx_tenants_agent_config already exists'; - END IF; - - -- For sessions.agent_config - IF NOT EXISTS ( - SELECT 1 FROM pg_indexes - WHERE tablename = 'sessions' - AND indexname = 'idx_sessions_agent_config' - ) THEN - CREATE INDEX idx_sessions_agent_config ON sessions USING GIN (agent_config); - RAISE NOTICE 'Created index idx_sessions_agent_config'; - ELSE - RAISE NOTICE 'Index idx_sessions_agent_config already exists'; - END IF; - - -- For sessions.context_config - IF NOT EXISTS ( - SELECT 1 FROM pg_indexes - WHERE tablename = 'sessions' - AND indexname = 'idx_sessions_context_config' - ) THEN - CREATE INDEX idx_sessions_context_config ON sessions USING GIN (context_config); - RAISE NOTICE 'Created index idx_sessions_context_config'; - ELSE - RAISE NOTICE 'Index idx_sessions_context_config already exists'; - END IF; - - -- For messages.agent_steps - IF NOT EXISTS ( - SELECT 1 FROM pg_indexes - WHERE tablename = 'messages' - AND indexname = 'idx_messages_agent_steps' - ) THEN - CREATE INDEX idx_messages_agent_steps ON messages USING GIN (agent_steps); - RAISE NOTICE 'Created index idx_messages_agent_steps'; - ELSE - RAISE NOTICE 'Index idx_messages_agent_steps already exists'; - END IF; -END $$; - -COMMIT; - -BEGIN; - -WITH referenced_models AS ( - SELECT embedding_model_id AS model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != '' - UNION - SELECT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != '' - UNION - SELECT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != '' - UNION - SELECT vlm_config ->> 'model_id' - FROM knowledge_bases - WHERE deleted_at IS NULL - AND vlm_config ->> 'model_id' IS NOT NULL - AND vlm_config ->> 'model_id' != '' - UNION - SELECT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != '' -) -UPDATE models m -SET deleted_at = CURRENT_TIMESTAMP -WHERE m.deleted_at IS NULL - AND m.is_default = FALSE - AND m.tenant_id != 0 - AND m.id NOT IN (SELECT model_id FROM referenced_models); - -COMMIT; - -BEGIN; - --- Create mcp_services table -CREATE TABLE IF NOT EXISTS mcp_services ( - id VARCHAR(36) PRIMARY KEY, - tenant_id INTEGER NOT NULL, - name VARCHAR(255) NOT NULL, - description TEXT, - enabled BOOLEAN DEFAULT true, - transport_type VARCHAR(50) NOT NULL, - url VARCHAR(512) NOT NULL, - headers JSONB, - auth_config JSONB, - advanced_config JSONB, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - deleted_at TIMESTAMP -); - --- Create indexes -CREATE INDEX IF NOT EXISTS idx_mcp_services_tenant_id ON mcp_services(tenant_id); -CREATE INDEX IF NOT EXISTS idx_mcp_services_enabled ON mcp_services(enabled); -CREATE INDEX IF NOT EXISTS idx_mcp_services_deleted_at ON mcp_services(deleted_at); - --- Add comment to table -COMMENT ON TABLE mcp_services IS 'MCP service configurations'; - --- Create trigger for updated_at -CREATE OR REPLACE FUNCTION update_mcp_services_updated_at() -RETURNS TRIGGER AS $$ -BEGIN - NEW.updated_at = CURRENT_TIMESTAMP; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER trigger_mcp_services_updated_at - BEFORE UPDATE ON mcp_services - FOR EACH ROW - EXECUTE FUNCTION update_mcp_services_updated_at(); - -COMMIT; - -BEGIN; - --- Add web_search_config column to tenants table -DO $$ BEGIN IF NOT EXISTS ( SELECT 1 FROM information_schema.columns - WHERE table_name = 'tenants' AND column_name = 'web_search_config' + WHERE table_name = 'tenants' AND column_name = 'agent_config' ) THEN - ALTER TABLE tenants - ADD COLUMN web_search_config JSONB DEFAULT NULL; + ALTER TABLE tenants ADD COLUMN agent_config JSONB DEFAULT NULL; + RAISE NOTICE '[Migration 000001] Added agent_config column to tenants table'; + ELSE + -- If field exists but type is JSON, convert to JSONB + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'tenants' AND column_name = 'agent_config' AND data_type = 'json' + ) THEN + ALTER TABLE tenants ALTER COLUMN agent_config TYPE JSONB USING agent_config::jsonb; + RAISE NOTICE '[Migration 000001] Converted tenants.agent_config from JSON to JSONB'; + END IF; + END IF; +END $$; +COMMENT ON COLUMN tenants.agent_config IS 'Tenant-level agent configuration in JSON format'; + +-- ============================================================================ +-- Section 3: Session Configuration Enhancements +-- ============================================================================ + +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding session configuration columns...'; END $$; + +-- Add agent_config column to sessions +ALTER TABLE sessions ADD COLUMN IF NOT EXISTS agent_config JSONB DEFAULT NULL; +COMMENT ON COLUMN sessions.agent_config IS 'Session-level agent configuration in JSON format'; + +-- Add context_config column to sessions +ALTER TABLE sessions ADD COLUMN IF NOT EXISTS context_config JSONB DEFAULT NULL; +COMMENT ON COLUMN sessions.context_config IS 'LLM context management configuration (separate from message storage)'; + +-- ============================================================================ +-- Section 4: Messages Enhancements +-- ============================================================================ + +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding messages enhancements...'; END $$; + +-- Add agent_steps column to messages +ALTER TABLE messages ADD COLUMN IF NOT EXISTS agent_steps JSONB DEFAULT NULL; +COMMENT ON COLUMN messages.agent_steps IS 'Agent execution steps (reasoning process and tool calls)'; + +-- ============================================================================ +-- Section 5: Knowledge Base Enhancements +-- ============================================================================ + +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding knowledge base enhancements...'; END $$; + +-- Add is_temporary column to knowledge_bases +ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS is_temporary BOOLEAN NOT NULL DEFAULT false; +COMMENT ON COLUMN knowledge_bases.is_temporary IS 'Whether this knowledge base is temporary (ephemeral) and should be hidden from UI'; + +-- Add type and faq_config columns +ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS type VARCHAR(32) NOT NULL DEFAULT 'document'; +ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS faq_config JSONB; + +-- Add question_generation_config column +ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS question_generation_config JSONB NULL; + +-- Update existing rows with default type +UPDATE knowledge_bases SET type = 'document' WHERE type IS NULL OR type = ''; + +-- Drop rerank_model_id column if exists (moved to session level) +ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS rerank_model_id; + +-- ============================================================================ +-- Section 6: Knowledges Enhancements +-- ============================================================================ + +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding knowledges enhancements...'; END $$; + +-- Add tag_id column +ALTER TABLE knowledges ADD COLUMN IF NOT EXISTS tag_id VARCHAR(36); +CREATE INDEX IF NOT EXISTS idx_knowledges_tag ON knowledges(tag_id); + +-- Add summary_status column +ALTER TABLE knowledges ADD COLUMN IF NOT EXISTS summary_status VARCHAR(32) DEFAULT 'none'; +CREATE INDEX IF NOT EXISTS idx_knowledges_summary_status ON knowledges(summary_status); + +-- ============================================================================ +-- Section 7: Chunks Enhancements +-- ============================================================================ + +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding chunks enhancements...'; END $$; + +-- Add metadata column +ALTER TABLE chunks ADD COLUMN IF NOT EXISTS metadata JSONB; + +-- Add tag_id column +ALTER TABLE chunks ADD COLUMN IF NOT EXISTS tag_id VARCHAR(36); +CREATE INDEX IF NOT EXISTS idx_chunks_tag ON chunks(tag_id); + +-- Add status field to track chunk processing state +ALTER TABLE chunks ADD COLUMN IF NOT EXISTS status INT NOT NULL DEFAULT 0; + +-- Add content_hash field for quick content matching +ALTER TABLE chunks ADD COLUMN IF NOT EXISTS content_hash VARCHAR(64); +CREATE INDEX IF NOT EXISTS idx_chunks_content_hash ON chunks(content_hash); + +-- ============================================================================ +-- Section 8: Embeddings Enhancements +-- ============================================================================ + +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding embeddings enhancements...'; END $$; + +-- Add is_enabled column +ALTER TABLE embeddings ADD COLUMN IF NOT EXISTS is_enabled BOOLEAN DEFAULT TRUE; +CREATE INDEX IF NOT EXISTS idx_embeddings_is_enabled ON embeddings(is_enabled); + +-- Add index for knowledge_base_id +CREATE INDEX IF NOT EXISTS idx_embeddings_knowledge_base_id ON embeddings(knowledge_base_id); + +-- Reindex BM25 search index (idempotent - will rebuild if exists) +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'embeddings_search_idx') THEN + REINDEX INDEX embeddings_search_idx; + RAISE NOTICE '[Migration 000001] Reindexed embeddings_search_idx'; END IF; END $$; --- Add or update comment -COMMENT ON COLUMN tenants.web_search_config IS 'Web search configuration for the tenant'; +-- ============================================================================ +-- Section 9: Models Enhancements +-- ============================================================================ -COMMIT; +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Adding models enhancements...'; END $$; -BEGIN; - --- Add stdio_config and env_vars columns -ALTER TABLE mcp_services -ADD COLUMN IF NOT EXISTS stdio_config JSONB, -ADD COLUMN IF NOT EXISTS env_vars JSONB; - --- Make url column optional -ALTER TABLE mcp_services -ALTER COLUMN url DROP NOT NULL; - --- Add check constraint -ALTER TABLE mcp_services -ADD CONSTRAINT chk_mcp_transport_config CHECK ( - (transport_type = 'stdio' AND stdio_config IS NOT NULL) OR - (transport_type != 'stdio' AND url IS NOT NULL) -); - -COMMIT; - -BEGIN; - --- Add is_builtin column to models table -ALTER TABLE models -ADD COLUMN IF NOT EXISTS is_builtin BOOLEAN NOT NULL DEFAULT false; - --- Add index for is_builtin field +-- Add is_builtin column +ALTER TABLE models ADD COLUMN IF NOT EXISTS is_builtin BOOLEAN NOT NULL DEFAULT false; CREATE INDEX IF NOT EXISTS idx_models_is_builtin ON models(is_builtin); -COMMIT; +-- ============================================================================ +-- Section 10: Knowledge Tags Table +-- ============================================================================ -BEGIN; - -ALTER TABLE knowledge_bases - ADD COLUMN IF NOT EXISTS type VARCHAR(32) NOT NULL DEFAULT 'document', - ADD COLUMN IF NOT EXISTS faq_config JSONB; - -UPDATE knowledge_bases -SET type = 'document' -WHERE type IS NULL OR type = ''; - -ALTER TABLE chunks - ADD COLUMN IF NOT EXISTS metadata JSONB; - -COMMIT; - -BEGIN; - --- Tag table (per knowledge base) +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Creating table: knowledge_tags'; END $$; CREATE TABLE IF NOT EXISTS knowledge_tags ( id VARCHAR(36) PRIMARY KEY, tenant_id INTEGER NOT NULL, @@ -412,123 +268,157 @@ CREATE TABLE IF NOT EXISTS knowledge_tags ( deleted_at TIMESTAMPTZ ); -CREATE UNIQUE INDEX IF NOT EXISTS idx_knowledge_tags_kb_name - ON knowledge_tags(tenant_id, knowledge_base_id, name); +CREATE UNIQUE INDEX IF NOT EXISTS idx_knowledge_tags_kb_name ON knowledge_tags(tenant_id, knowledge_base_id, name); +CREATE INDEX IF NOT EXISTS idx_knowledge_tags_kb ON knowledge_tags(tenant_id, knowledge_base_id); -CREATE INDEX IF NOT EXISTS idx_knowledge_tags_kb - ON knowledge_tags(tenant_id, knowledge_base_id); +-- ============================================================================ +-- Section 11: MCP Services Table +-- ============================================================================ --- Tag reference on knowledges -ALTER TABLE knowledges - ADD COLUMN IF NOT EXISTS tag_id VARCHAR(36); +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Creating table: mcp_services'; END $$; +CREATE TABLE IF NOT EXISTS mcp_services ( + id VARCHAR(36) PRIMARY KEY, + tenant_id INTEGER NOT NULL, + name VARCHAR(255) NOT NULL, + description TEXT, + enabled BOOLEAN DEFAULT true, + transport_type VARCHAR(50) NOT NULL, + url VARCHAR(512), + headers JSONB, + auth_config JSONB, + advanced_config JSONB, + stdio_config JSONB, + env_vars JSONB, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + deleted_at TIMESTAMP +); -CREATE INDEX IF NOT EXISTS idx_knowledges_tag - ON knowledges(tag_id); +CREATE INDEX IF NOT EXISTS idx_mcp_services_tenant_id ON mcp_services(tenant_id); +CREATE INDEX IF NOT EXISTS idx_mcp_services_enabled ON mcp_services(enabled); +CREATE INDEX IF NOT EXISTS idx_mcp_services_deleted_at ON mcp_services(deleted_at); --- Tag reference on chunks -ALTER TABLE chunks - ADD COLUMN IF NOT EXISTS tag_id VARCHAR(36); +COMMENT ON TABLE mcp_services IS 'MCP service configurations'; -CREATE INDEX IF NOT EXISTS idx_chunks_tag - ON chunks(tag_id); +-- Create or replace trigger function for updated_at +CREATE OR REPLACE FUNCTION update_mcp_services_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; -COMMIT; +-- Create trigger if not exists +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgname = 'trigger_mcp_services_updated_at') THEN + CREATE TRIGGER trigger_mcp_services_updated_at + BEFORE UPDATE ON mcp_services + FOR EACH ROW + EXECUTE FUNCTION update_mcp_services_updated_at(); + RAISE NOTICE '[Migration 000001] Created trigger trigger_mcp_services_updated_at'; + END IF; +END $$; -BEGIN; +-- ============================================================================ +-- Section 12: GIN Indexes for JSONB Fields +-- ============================================================================ --- Add is_enabled column to embeddings table -ALTER TABLE embeddings - ADD COLUMN IF NOT EXISTS is_enabled BOOLEAN DEFAULT TRUE; +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Creating GIN indexes for JSONB fields...'; END $$; --- Create index for is_enabled column -CREATE INDEX IF NOT EXISTS idx_embeddings_is_enabled - ON embeddings(is_enabled); +DO $$ +BEGIN + -- For tenants.agent_config + IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'idx_tenants_agent_config') THEN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'tenants' AND column_name = 'agent_config' AND data_type = 'jsonb' + ) THEN + CREATE INDEX idx_tenants_agent_config ON tenants USING GIN (agent_config); + RAISE NOTICE '[Migration 000001] Created index idx_tenants_agent_config'; + END IF; + END IF; + + -- For sessions.agent_config + IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'idx_sessions_agent_config') THEN + CREATE INDEX idx_sessions_agent_config ON sessions USING GIN (agent_config); + RAISE NOTICE '[Migration 000001] Created index idx_sessions_agent_config'; + END IF; + + -- For sessions.context_config + IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'idx_sessions_context_config') THEN + CREATE INDEX idx_sessions_context_config ON sessions USING GIN (context_config); + RAISE NOTICE '[Migration 000001] Created index idx_sessions_context_config'; + END IF; + + -- For messages.agent_steps + IF NOT EXISTS (SELECT 1 FROM pg_indexes WHERE indexname = 'idx_messages_agent_steps') THEN + CREATE INDEX idx_messages_agent_steps ON messages USING GIN (agent_steps); + RAISE NOTICE '[Migration 000001] Created index idx_messages_agent_steps'; + END IF; +END $$; -COMMIT; +-- ============================================================================ +-- Section 13: Data Migrations +-- ============================================================================ -BEGIN; +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Running data migrations...'; END $$; --- Create index for knowledge_base_id -CREATE INDEX IF NOT EXISTS idx_embeddings_knowledge_base_id - ON embeddings(knowledge_base_id); +-- Clean up unreferenced models (soft delete) +DO $$ +DECLARE + affected_rows INTEGER; +BEGIN + WITH referenced_models AS ( + SELECT embedding_model_id AS model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != '' + UNION + SELECT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != '' + UNION + SELECT vlm_config ->> 'model_id' + FROM knowledge_bases + WHERE deleted_at IS NULL + AND vlm_config ->> 'model_id' IS NOT NULL + AND vlm_config ->> 'model_id' != '' + UNION + SELECT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != '' + ) + UPDATE models m + SET deleted_at = CURRENT_TIMESTAMP + WHERE m.deleted_at IS NULL + AND m.is_default = FALSE + AND m.tenant_id != 0 + AND m.id NOT IN (SELECT model_id FROM referenced_models WHERE model_id IS NOT NULL); + + GET DIAGNOSTICS affected_rows = ROW_COUNT; + IF affected_rows > 0 THEN + RAISE NOTICE '[Migration 000001] Soft deleted % unreferenced models', affected_rows; + END IF; +END $$; -COMMIT; +-- Update models tenant_id from knowledge_bases references +DO $$ +DECLARE + affected_rows INTEGER; +BEGIN + WITH tenant_source AS ( + SELECT kb.embedding_model_id AS model_id, kb.tenant_id + FROM knowledge_bases kb + WHERE kb.tenant_id IS NOT NULL AND kb.embedding_model_id IS NOT NULL AND kb.embedding_model_id <> '' + UNION + SELECT kb.summary_model_id AS model_id, kb.tenant_id + FROM knowledge_bases kb + WHERE kb.tenant_id IS NOT NULL AND kb.summary_model_id IS NOT NULL AND kb.summary_model_id <> '' + ) + UPDATE models m + SET tenant_id = ts.tenant_id + FROM tenant_source ts + WHERE m.id = ts.model_id AND m.tenant_id = 0; + + GET DIAGNOSTICS affected_rows = ROW_COUNT; + IF affected_rows > 0 THEN + RAISE NOTICE '[Migration 000001] Updated tenant_id for % models', affected_rows; + END IF; +END $$; -REINDEX INDEX embeddings_search_idx; - -BEGIN; - --- Add status field to track chunk processing state -ALTER TABLE chunks - ADD COLUMN IF NOT EXISTS status INT NOT NULL DEFAULT 0; - --- Add content_hash field for quick content matching -ALTER TABLE chunks - ADD COLUMN IF NOT EXISTS content_hash VARCHAR(64); - --- Create index on content_hash -CREATE INDEX IF NOT EXISTS idx_chunks_content_hash - ON chunks(content_hash); - -COMMIT; - -BEGIN; - -WITH tenant_source AS ( - SELECT - kb.embedding_model_id AS model_id, - kb.tenant_id - FROM knowledge_bases kb - WHERE kb.tenant_id IS NOT NULL - AND kb.embedding_model_id IS NOT NULL - AND kb.embedding_model_id <> '' - - UNION - - SELECT - kb.summary_model_id AS model_id, - kb.tenant_id - FROM knowledge_bases kb - WHERE kb.tenant_id IS NOT NULL - AND kb.summary_model_id IS NOT NULL - AND kb.summary_model_id <> '' - - UNION - - SELECT - kb.rerank_model_id AS model_id, - kb.tenant_id - FROM knowledge_bases kb - WHERE kb.tenant_id IS NOT NULL - AND kb.rerank_model_id IS NOT NULL - AND kb.rerank_model_id <> '' -) -UPDATE models m -SET tenant_id = ts.tenant_id -FROM tenant_source ts -WHERE m.id = ts.model_id - AND m.tenant_id = 0; - -COMMIT; - -BEGIN; - -ALTER TABLE knowledge_bases - DROP COLUMN IF EXISTS rerank_model_id; - -COMMIT; - -BEGIN; - -ALTER TABLE users - ADD COLUMN IF NOT EXISTS can_access_all_tenants BOOLEAN NOT NULL DEFAULT FALSE; - -COMMIT; - -ALTER TABLE knowledge_bases -ADD COLUMN IF NOT EXISTS question_generation_config JSONB NULL ; - -ALTER TABLE knowledges ADD COLUMN IF NOT EXISTS summary_status VARCHAR(32) DEFAULT 'none'; - --- Add index for efficient querying -CREATE INDEX IF NOT EXISTS idx_knowledges_summary_status ON knowledges(summary_status); +DO $$ BEGIN RAISE NOTICE '[Migration 000001] Migration completed successfully!'; END $$; diff --git a/scripts/migrate.sh b/scripts/migrate.sh index e4b182ab..f11514f8 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -96,10 +96,13 @@ case "$1" in if [ -z "$2" ]; then echo "Error: Version number is required" echo "Usage: $0 force " + echo "Note: Use -1 to reset to no version (allows re-running all migrations)" exit 1 fi - echo "Forcing migration version to $2..." - migrate -path ${MIGRATIONS_DIR} -database ${DB_URL} force $2 + VERSION="$2" + echo "Forcing migration version to $VERSION..." + # Use env to pass the command, avoiding shell flag parsing issues with negative numbers + env migrate -path "${MIGRATIONS_DIR}" -database "${DB_URL}" force -- "$VERSION" ;; goto) if [ -z "$2" ]; then From c093dadedc5cfaea3cb66153ec18615a57c8a826 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 8 Dec 2025 19:32:46 +0800 Subject: [PATCH 3/3] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81Qdrant=E5=A4=9A?= =?UTF-8?q?=E7=BB=B4=E5=BA=A6=E5=90=91=E9=87=8F=E5=AD=98=E5=82=A8=E5=92=8C?= =?UTF-8?q?=E5=85=A8=E6=96=87=E6=A3=80=E7=B4=A2=E4=BC=98=E5=8C=96=EF=BC=8C?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0Docker=20Compose=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.dev.yml | 25 + docker-compose.yml | 21 + .../repository/retriever/qdrant/repository.go | 454 +++++++++++++----- .../repository/retriever/qdrant/structs.go | 12 +- internal/application/service/chunk.go | 2 +- internal/application/service/knowledge.go | 28 +- internal/application/service/knowledgebase.go | 4 +- internal/application/service/user.go | 58 +-- internal/types/tenant.go | 49 ++ scripts/dev.sh | 83 +++- 10 files changed, 534 insertions(+), 202 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 0a7410f6..61e9b3c3 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -54,6 +54,24 @@ services: retries: 3 networks: - WeKnora-network-dev + profiles: + - minio + - full + + qdrant: + image: qdrant/qdrant:v1.16.2 + container_name: WeKnora-qdrant-dev + ports: + - "${QDRANT_REST_PORT:-6333}:6333" + - "${QDRANT_PORT:-6334}:6334" + volumes: + - qdrant_data_dev:/qdrant/storage + networks: + - WeKnora-network-dev + restart: unless-stopped + profiles: + - qdrant + - full neo4j: image: neo4j:latest @@ -72,6 +90,9 @@ services: restart: always networks: - WeKnora-network-dev + profiles: + - neo4j + - full docreader: image: wechatopenai/weknora-docreader:main @@ -130,6 +151,9 @@ services: networks: - WeKnora-network-dev restart: unless-stopped + profiles: + - jaeger + - full networks: WeKnora-network-dev: @@ -141,4 +165,5 @@ volumes: minio_data_dev: neo4j-data-dev: jaeger_data_dev: + qdrant_data_dev: diff --git a/docker-compose.yml b/docker-compose.yml index 930a4240..0d95dee4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,6 +56,11 @@ services: - ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME:-} - ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD:-} - ELASTICSEARCH_INDEX=${ELASTICSEARCH_INDEX:-} + - QDRANT_HOST=qdrant + - QDRANT_PORT=${QDRANT_PORT:-6334} + - QDRANT_COLLECTION=${QDRANT_COLLECTION:-weknora_embeddings} + - QDRANT_API_KEY=${QDRANT_API_KEY:-} + - QDRANT_USE_TLS=${QDRANT_USE_TLS:-false} - DOCREADER_ADDR=docreader:50051 - STORAGE_TYPE=${STORAGE_TYPE:-} - LOCAL_STORAGE_BASE_DIR=${LOCAL_STORAGE_BASE_DIR:-} @@ -240,6 +245,21 @@ services: - neo4j - full + qdrant: + image: qdrant/qdrant:v1.16.2 + container_name: WeKnora-qdrant + ports: + - "${QDRANT_REST_PORT:-6333}:6333" + - "${QDRANT_PORT:-6334}:6334" + volumes: + - qdrant_data:/qdrant/storage + networks: + - WeKnora-network + restart: unless-stopped + profiles: + - qdrant + - full + networks: WeKnora-network: driver: bridge @@ -250,3 +270,4 @@ volumes: jaeger_data: minio_data: neo4j-data: + qdrant_data: diff --git a/internal/application/repository/retriever/qdrant/repository.go b/internal/application/repository/retriever/qdrant/repository.go index 9095e061..e615f7fa 100644 --- a/internal/application/repository/retriever/qdrant/repository.go +++ b/internal/application/repository/retriever/qdrant/repository.go @@ -6,10 +6,11 @@ import ( "maps" "os" "slices" + "strings" + "unicode/utf8" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" - typesLocal "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" "github.com/google/uuid" "github.com/qdrant/go-client/qdrant" @@ -33,32 +34,121 @@ func NewQdrantRetrieveEngineRepository(client *qdrant.Client) interfaces.Retriev log := logger.GetLogger(context.Background()) log.Info("[Qdrant] Initializing Qdrant retriever engine repository") - collectionName := os.Getenv(envQdrantCollection) - if collectionName == "" { + collectionBaseName := os.Getenv(envQdrantCollection) + if collectionBaseName == "" { log.Warn("[Qdrant] QDRANT_COLLECTION environment variable not set, using default collection name") - collectionName = defaultCollectionName + collectionBaseName = defaultCollectionName } res := &qdrantRepository{ - client: client, - collectionName: collectionName, + client: client, + collectionBaseName: collectionBaseName, } log.Info("[Qdrant] Successfully initialized repository") return res } -func (q *qdrantRepository) EngineType() typesLocal.RetrieverEngineType { - return typesLocal.QdrantRetrieverEngineType +// getCollectionName returns the collection name for a specific dimension +func (q *qdrantRepository) getCollectionName(dimension int) string { + return fmt.Sprintf("%s_%d", q.collectionBaseName, dimension) } -func (q *qdrantRepository) Support() []typesLocal.RetrieverType { - return []typesLocal.RetrieverType{typesLocal.KeywordsRetrieverType, typesLocal.VectorRetrieverType} +// ensureCollection ensures the collection exists for the given dimension +func (q *qdrantRepository) ensureCollection(ctx context.Context, dimension int) error { + collectionName := q.getCollectionName(dimension) + + // Check cache first + if _, ok := q.initializedCollections.Load(dimension); ok { + return nil + } + + log := logger.GetLogger(ctx) + + // Check if collection exists + exists, err := q.client.CollectionExists(ctx, collectionName) + if err != nil { + log.Errorf("[Qdrant] Failed to check collection existence: %v", err) + return fmt.Errorf("failed to check collection existence: %w", err) + } + + if !exists { + log.Infof("[Qdrant] Creating collection %s with dimension %d", collectionName, dimension) + + err = q.client.CreateCollection(ctx, &qdrant.CreateCollection{ + CollectionName: collectionName, + VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{ + Size: uint64(dimension), + Distance: qdrant.Distance_Cosine, + }), + }) + if err != nil { + log.Errorf("[Qdrant] Failed to create collection: %v", err) + return fmt.Errorf("failed to create collection: %w", err) + } + + // Create payload indexes for filtering + indexFields := []string{fieldChunkID, fieldKnowledgeID, fieldKnowledgeBaseID, fieldSourceID} + for _, field := range indexFields { + _, err = q.client.CreateFieldIndex(ctx, &qdrant.CreateFieldIndexCollection{ + CollectionName: collectionName, + FieldName: field, + FieldType: qdrant.FieldType_FieldTypeKeyword.Enum(), + }) + if err != nil { + log.Warnf("[Qdrant] Failed to create index for field %s: %v", field, err) + } + } + + // Create bool index for is_enabled + _, err = q.client.CreateFieldIndex(ctx, &qdrant.CreateFieldIndexCollection{ + CollectionName: collectionName, + FieldName: fieldIsEnabled, + FieldType: qdrant.FieldType_FieldTypeBool.Enum(), + }) + if err != nil { + log.Warnf("[Qdrant] Failed to create index for field %s: %v", fieldIsEnabled, err) + } + + // Create text index for content (for keyword search) with multilingual tokenizer + // This supports Chinese, Japanese, Korean and other languages + lowercase := true + _, err = q.client.CreateFieldIndex(ctx, &qdrant.CreateFieldIndexCollection{ + CollectionName: collectionName, + FieldName: fieldContent, + FieldType: qdrant.FieldType_FieldTypeText.Enum(), + FieldIndexParams: &qdrant.PayloadIndexParams{ + IndexParams: &qdrant.PayloadIndexParams_TextIndexParams{ + TextIndexParams: &qdrant.TextIndexParams{ + Tokenizer: qdrant.TokenizerType_Multilingual, + Lowercase: &lowercase, + }, + }, + }, + }) + if err != nil { + log.Warnf("[Qdrant] Failed to create text index for content: %v", err) + } + + log.Infof("[Qdrant] Successfully created collection %s", collectionName) + } + + // Mark as initialized + q.initializedCollections.Store(dimension, true) + return nil +} + +func (q *qdrantRepository) EngineType() types.RetrieverEngineType { + return types.QdrantRetrieverEngineType +} + +func (q *qdrantRepository) Support() []types.RetrieverType { + return []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType} } // EstimateStorageSize calculates the estimated storage size for a list of indices func (q *qdrantRepository) EstimateStorageSize(ctx context.Context, - indexInfoList []*typesLocal.IndexInfo, params map[string]any, + indexInfoList []*types.IndexInfo, params map[string]any, ) int64 { var totalStorageSize int64 for _, embedding := range indexInfoList { @@ -73,7 +163,7 @@ func (q *qdrantRepository) EstimateStorageSize(ctx context.Context, // Save stores a single point in Qdrant func (q *qdrantRepository) Save(ctx context.Context, - embedding *typesLocal.IndexInfo, + embedding *types.IndexInfo, additionalParams map[string]any, ) error { log := logger.GetLogger(ctx) @@ -86,6 +176,12 @@ func (q *qdrantRepository) Save(ctx context.Context, return err } + dimension := len(embeddingDB.Embedding) + if err := q.ensureCollection(ctx, dimension); err != nil { + return err + } + + collectionName := q.getCollectionName(dimension) pointID := uuid.New().String() point := &qdrant.PointStruct{ Id: qdrant.NewID(pointID), @@ -94,7 +190,7 @@ func (q *qdrantRepository) Save(ctx context.Context, } _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Points: []*qdrant.PointStruct{point}, }) if err != nil { @@ -108,7 +204,7 @@ func (q *qdrantRepository) Save(ctx context.Context, // BatchSave stores multiple points in Qdrant using batch upsert func (q *qdrantRepository) BatchSave(ctx context.Context, - embeddingList []*typesLocal.IndexInfo, additionalParams map[string]any, + embeddingList []*types.IndexInfo, additionalParams map[string]any, ) error { log := logger.GetLogger(ctx) if len(embeddingList) == 0 { @@ -118,7 +214,9 @@ func (q *qdrantRepository) BatchSave(ctx context.Context, log.Infof("[Qdrant] Batch saving %d indices", len(embeddingList)) - points := make([]*qdrant.PointStruct, 0, len(embeddingList)) + // Group points by dimension + pointsByDimension := make(map[int][]*qdrant.PointStruct) + for _, embedding := range embeddingList { embeddingDB := toQdrantVectorEmbedding(embedding, additionalParams) if len(embeddingDB.Embedding) == 0 { @@ -126,30 +224,42 @@ func (q *qdrantRepository) BatchSave(ctx context.Context, continue } + dimension := len(embeddingDB.Embedding) point := &qdrant.PointStruct{ Id: qdrant.NewID(uuid.New().String()), Vectors: qdrant.NewVectors(embeddingDB.Embedding...), Payload: createPayload(embeddingDB), } - points = append(points, point) - log.Debugf("[Qdrant] Added chunk ID %s to batch request", embedding.ChunkID) + pointsByDimension[dimension] = append(pointsByDimension[dimension], point) + log.Debugf("[Qdrant] Added chunk ID %s to batch request (dimension: %d)", embedding.ChunkID, dimension) } - if len(points) == 0 { + if len(pointsByDimension) == 0 { log.Warn("[Qdrant] No valid points to save after filtering") return nil } - _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ - CollectionName: q.collectionName, - Points: points, - }) - if err != nil { - log.Errorf("[Qdrant] Failed to execute batch operation: %v", err) - return fmt.Errorf("failed to batch save: %w", err) + // Save points to each dimension-specific collection + totalSaved := 0 + for dimension, points := range pointsByDimension { + if err := q.ensureCollection(ctx, dimension); err != nil { + return err + } + + collectionName := q.getCollectionName(dimension) + _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ + CollectionName: collectionName, + Points: points, + }) + if err != nil { + log.Errorf("[Qdrant] Failed to execute batch operation for dimension %d: %v", dimension, err) + return fmt.Errorf("failed to batch save (dimension %d): %w", dimension, err) + } + totalSaved += len(points) + log.Infof("[Qdrant] Saved %d points to collection %s", len(points), collectionName) } - log.Infof("[Qdrant] Successfully batch saved %d indices", len(points)) + log.Infof("[Qdrant] Successfully batch saved %d indices", totalSaved) return nil } @@ -161,10 +271,11 @@ func (q *qdrantRepository) DeleteByChunkIDList(ctx context.Context, chunkIDList return nil } - log.Infof("[Qdrant] Deleting indices by chunk IDs, count: %d", len(chunkIDList)) + collectionName := q.getCollectionName(dimension) + log.Infof("[Qdrant] Deleting indices by chunk IDs from %s, count: %d", collectionName, len(chunkIDList)) _, err := q.client.Delete(ctx, &qdrant.DeletePoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ Must: []*qdrant.Condition{ qdrant.NewMatchKeywords(fieldChunkID, chunkIDList...), @@ -190,10 +301,11 @@ func (q *qdrantRepository) DeleteByKnowledgeIDList(ctx context.Context, return nil } - log.Infof("[Qdrant] Deleting indices by knowledge IDs, count: %d", len(knowledgeIDList)) + collectionName := q.getCollectionName(dimension) + log.Infof("[Qdrant] Deleting indices by knowledge IDs from %s, count: %d", collectionName, len(knowledgeIDList)) _, err := q.client.Delete(ctx, &qdrant.DeletePoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ Must: []*qdrant.Condition{ qdrant.NewMatchKeywords(fieldKnowledgeID, knowledgeIDList...), @@ -219,10 +331,11 @@ func (q *qdrantRepository) DeleteBySourceIDList(ctx context.Context, return nil } - log.Infof("[Qdrant] Deleting indices by source IDs, count: %d", len(sourceIDList)) + collectionName := q.getCollectionName(dimension) + log.Infof("[Qdrant] Deleting indices by source IDs from %s, count: %d", collectionName, len(sourceIDList)) _, err := q.client.Delete(ctx, &qdrant.DeletePoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Points: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ Must: []*qdrant.Condition{ qdrant.NewMatchKeywords(fieldSourceID, sourceIDList...), @@ -239,6 +352,7 @@ func (q *qdrantRepository) DeleteBySourceIDList(ctx context.Context, } // BatchUpdateChunkEnabledStatus updates the enabled status of chunks in batch +// This method operates on all collections since dimension is not provided func (q *qdrantRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chunkStatusMap map[string]bool) error { log := logger.GetLogger(ctx) if len(chunkStatusMap) == 0 { @@ -248,6 +362,13 @@ func (q *qdrantRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, ch log.Infof("[Qdrant] Batch updating chunk enabled status, count: %d", len(chunkStatusMap)) + // Get all collections that match our base name pattern + collections, err := q.client.ListCollections(ctx) + if err != nil { + log.Errorf("[Qdrant] Failed to list collections: %v", err) + return fmt.Errorf("failed to list collections: %w", err) + } + // Group chunks by enabled status for batch updates enabledChunkIDs := make([]string, 0) disabledChunkIDs := make([]string, 0) @@ -260,47 +381,52 @@ func (q *qdrantRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, ch } } - // Update enabled chunks - if len(enabledChunkIDs) > 0 { - _, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{ - CollectionName: q.collectionName, - Payload: qdrant.NewValueMap(map[string]any{fieldIsEnabled: true}), - PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ - Must: []*qdrant.Condition{ - qdrant.NewMatchKeywords(fieldChunkID, enabledChunkIDs...), - }, - }), - }) - if err != nil { - log.Errorf("[Qdrant] Failed to update enabled chunks: %v", err) - return fmt.Errorf("failed to update enabled chunks: %w", err) + // Update in all matching collections + for _, collectionName := range collections { + // Only process collections that start with our base name + if len(collectionName) <= len(q.collectionBaseName) || + collectionName[:len(q.collectionBaseName)] != q.collectionBaseName { + continue } - log.Infof("[Qdrant] Successfully enabled %d chunks", len(enabledChunkIDs)) - } - // Update disabled chunks - if len(disabledChunkIDs) > 0 { - _, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{ - CollectionName: q.collectionName, - Payload: qdrant.NewValueMap(map[string]any{fieldIsEnabled: false}), - PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ - Must: []*qdrant.Condition{ - qdrant.NewMatchKeywords(fieldChunkID, disabledChunkIDs...), - }, - }), - }) - if err != nil { - log.Errorf("[Qdrant] Failed to update disabled chunks: %v", err) - return fmt.Errorf("failed to update disabled chunks: %w", err) + // Update enabled chunks + if len(enabledChunkIDs) > 0 { + _, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{ + CollectionName: collectionName, + Payload: qdrant.NewValueMap(map[string]any{fieldIsEnabled: true}), + PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ + Must: []*qdrant.Condition{ + qdrant.NewMatchKeywords(fieldChunkID, enabledChunkIDs...), + }, + }), + }) + if err != nil { + log.Warnf("[Qdrant] Failed to update enabled chunks in %s: %v", collectionName, err) + } + } + + // Update disabled chunks + if len(disabledChunkIDs) > 0 { + _, err := q.client.SetPayload(ctx, &qdrant.SetPayloadPoints{ + CollectionName: collectionName, + Payload: qdrant.NewValueMap(map[string]any{fieldIsEnabled: false}), + PointsSelector: qdrant.NewPointsSelectorFilter(&qdrant.Filter{ + Must: []*qdrant.Condition{ + qdrant.NewMatchKeywords(fieldChunkID, disabledChunkIDs...), + }, + }), + }) + if err != nil { + log.Warnf("[Qdrant] Failed to update disabled chunks in %s: %v", collectionName, err) + } } - log.Infof("[Qdrant] Successfully disabled %d chunks", len(disabledChunkIDs)) } log.Infof("[Qdrant] Batch update chunk enabled status completed") return nil } -func (q *qdrantRepository) getBaseFilter(params typesLocal.RetrieveParams) *qdrant.Filter { +func (q *qdrantRepository) getBaseFilter(params types.RetrieveParams) *qdrant.Filter { must := make([]*qdrant.Condition, 0) mustNot := make([]*qdrant.Condition, 0) @@ -327,15 +453,15 @@ func (q *qdrantRepository) getBaseFilter(params typesLocal.RetrieveParams) *qdra // Retrieve dispatches the retrieval operation to the appropriate method based on retriever type func (q *qdrantRepository) Retrieve(ctx context.Context, - params typesLocal.RetrieveParams, -) ([]*typesLocal.RetrieveResult, error) { + params types.RetrieveParams, +) ([]*types.RetrieveResult, error) { log := logger.GetLogger(ctx) log.Debugf("[Qdrant] Processing retrieval request of type: %s", params.RetrieverType) switch params.RetrieverType { - case typesLocal.VectorRetrieverType: + case types.VectorRetrieverType: return q.VectorRetrieve(ctx, params) - case typesLocal.KeywordsRetrieverType: + case types.KeywordsRetrieverType: return q.KeywordsRetrieve(ctx, params) } @@ -346,11 +472,26 @@ func (q *qdrantRepository) Retrieve(ctx context.Context, // VectorRetrieve performs vector similarity search func (q *qdrantRepository) VectorRetrieve(ctx context.Context, - params typesLocal.RetrieveParams, -) ([]*typesLocal.RetrieveResult, error) { + params types.RetrieveParams, +) ([]*types.RetrieveResult, error) { log := logger.GetLogger(ctx) + dimension := len(params.Embedding) log.Infof("[Qdrant] Vector retrieval: dim=%d, topK=%d, threshold=%.4f", - len(params.Embedding), params.TopK, params.Threshold) + dimension, params.TopK, params.Threshold) + + // Get collection name based on embedding dimension + collectionName := q.getCollectionName(dimension) + + // Check if collection exists + exists, err := q.client.CollectionExists(ctx, collectionName) + if err != nil { + log.Errorf("[Qdrant] Failed to check collection existence: %v", err) + return nil, fmt.Errorf("failed to check collection: %w", err) + } + if !exists { + log.Warnf("[Qdrant] Collection %s does not exist, returning empty results", collectionName) + return buildRetrieveResult(nil, types.VectorRetrieverType), nil + } filter := q.getBaseFilter(params) @@ -358,7 +499,7 @@ func (q *qdrantRepository) VectorRetrieve(ctx context.Context, scoreThreshold := float32(params.Threshold) searchResult, err := q.client.Query(ctx, &qdrant.QueryPoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Query: qdrant.NewQuery(params.Embedding...), Filter: filter, Limit: &limit, @@ -367,10 +508,10 @@ func (q *qdrantRepository) VectorRetrieve(ctx context.Context, }) if err != nil { log.Errorf("[Qdrant] Vector search failed: %v", err) - return nil, err + return nil, fmt.Errorf("%s: %w", collectionName, err) } - var results []*typesLocal.IndexWithScore + var results []*types.IndexWithScore for _, point := range searchResult { payload := point.Payload embedding := &QdrantVectorEmbeddingWithScore{ @@ -386,7 +527,7 @@ func (q *qdrantRepository) VectorRetrieve(ctx context.Context, } pointID := point.Id.GetUuid() - results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeEmbedding)) + results = append(results, fromQdrantVectorEmbedding(pointID, embedding, types.MatchTypeEmbedding)) } if len(results) == 0 { @@ -396,58 +537,104 @@ func (q *qdrantRepository) VectorRetrieve(ctx context.Context, log.Debugf("[Qdrant] Top result score: %.4f", results[0].Score) } - return buildRetrieveResult(results, typesLocal.VectorRetrieverType), nil + return buildRetrieveResult(results, types.VectorRetrieverType), nil } // KeywordsRetrieve performs keyword-based search in document content +// This searches across all collections since keyword search doesn't depend on dimension func (q *qdrantRepository) KeywordsRetrieve(ctx context.Context, - params typesLocal.RetrieveParams, -) ([]*typesLocal.RetrieveResult, error) { + params types.RetrieveParams, +) ([]*types.RetrieveResult, error) { log := logger.GetLogger(ctx) log.Infof("[Qdrant] Performing keywords retrieval with query: %s, topK: %d", params.Query, params.TopK) - filter := q.getBaseFilter(params) - - filter.Must = append(filter.Must, qdrant.NewMatchText(fieldContent, params.Query)) - - limit := uint32(params.TopK) - scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{ - CollectionName: q.collectionName, - Filter: filter, - Limit: &limit, - WithPayload: qdrant.NewWithPayload(true), - }) + // Get all collections that match our base name pattern + collections, err := q.client.ListCollections(ctx) if err != nil { - log.Errorf("[Qdrant] Keywords search failed: %v", err) - return nil, err + log.Errorf("[Qdrant] Failed to list collections: %v", err) + return nil, fmt.Errorf("failed to list collections: %w", err) } - var results []*typesLocal.IndexWithScore - for _, point := range scrollResult { - payload := point.Payload - embedding := &QdrantVectorEmbeddingWithScore{ - QdrantVectorEmbedding: QdrantVectorEmbedding{ - Content: payload[fieldContent].GetStringValue(), - SourceID: payload[fieldSourceID].GetStringValue(), - SourceType: int(payload[fieldSourceType].GetIntegerValue()), - ChunkID: payload[fieldChunkID].GetStringValue(), - KnowledgeID: payload[fieldKnowledgeID].GetStringValue(), - KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(), - }, - Score: 1.0, + var allResults []*types.IndexWithScore + limit := uint32(params.TopK) + + log.Debugf("[Qdrant] Found %d collections, base name: %s", len(collections), q.collectionBaseName) + + // Tokenize query for OR-based search (better for Chinese and multi-word queries) + queryTokens := tokenizeQuery(params.Query) + log.Debugf("[Qdrant] Tokenized query into %d tokens: %v", len(queryTokens), queryTokens) + + // Search in all matching collections + for _, collectionName := range collections { + log.Debugf("[Qdrant] Checking collection: %s", collectionName) + // Only process collections that start with our base name + if len(collectionName) <= len(q.collectionBaseName) || + collectionName[:len(q.collectionBaseName)] != q.collectionBaseName { + log.Debugf("[Qdrant] Skipping collection %s (doesn't match base name %s)", collectionName, q.collectionBaseName) + continue } - pointID := point.Id.GetUuid() - results = append(results, fromQdrantVectorEmbedding(pointID, embedding, typesLocal.MatchTypeKeywords)) + filter := q.getBaseFilter(params) + + // Build should conditions for each token (OR logic) + // This allows matching documents that contain any of the query tokens + if len(queryTokens) > 0 { + shouldConditions := make([]*qdrant.Condition, 0, len(queryTokens)) + for _, token := range queryTokens { + shouldConditions = append(shouldConditions, qdrant.NewMatchText(fieldContent, token)) + } + filter.Should = shouldConditions + } else { + // Fallback to original query if tokenization fails + filter.Must = append(filter.Must, qdrant.NewMatchText(fieldContent, params.Query)) + } + + log.Debugf("[Qdrant] Searching in collection %s with %d should conditions", collectionName, len(filter.Should)) + + scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{ + CollectionName: collectionName, + Filter: filter, + Limit: &limit, + WithPayload: qdrant.NewWithPayload(true), + }) + if err != nil { + log.Warnf("[Qdrant] Keywords search failed in %s: %v", collectionName, err) + continue + } + + log.Debugf("[Qdrant] Found %d results in collection %s", len(scrollResult), collectionName) + + for _, point := range scrollResult { + payload := point.Payload + embedding := &QdrantVectorEmbeddingWithScore{ + QdrantVectorEmbedding: QdrantVectorEmbedding{ + Content: payload[fieldContent].GetStringValue(), + SourceID: payload[fieldSourceID].GetStringValue(), + SourceType: int(payload[fieldSourceType].GetIntegerValue()), + ChunkID: payload[fieldChunkID].GetStringValue(), + KnowledgeID: payload[fieldKnowledgeID].GetStringValue(), + KnowledgeBaseID: payload[fieldKnowledgeBaseID].GetStringValue(), + }, + Score: 1.0, + } + + pointID := point.Id.GetUuid() + allResults = append(allResults, fromQdrantVectorEmbedding(pointID, embedding, types.MatchTypeKeywords)) + } } - if len(results) == 0 { + // Limit results to topK + if len(allResults) > params.TopK { + allResults = allResults[:params.TopK] + } + + if len(allResults) == 0 { log.Warnf("[Qdrant] No keyword matches found for query: %s", params.Query) } else { - log.Infof("[Qdrant] Keywords retrieval found %d results", len(results)) + log.Infof("[Qdrant] Keywords retrieval found %d results", len(allResults)) } - return buildRetrieveResult(results, typesLocal.KeywordsRetrieverType), nil + return buildRetrieveResult(allResults, types.KeywordsRetrieverType), nil } // CopyIndices copies index data from source knowledge base to target knowledge base @@ -460,8 +647,8 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, ) error { log := logger.GetLogger(ctx) log.Infof( - "[Qdrant] Copying indices from source knowledge base %s to target knowledge base %s, count: %d", - sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap), + "[Qdrant] Copying indices from source knowledge base %s to target knowledge base %s, count: %d, dimension: %d", + sourceKnowledgeBaseID, targetKnowledgeBaseID, len(sourceToTargetChunkIDMap), dimension, ) if len(sourceToTargetChunkIDMap) == 0 { @@ -469,13 +656,20 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, return nil } + collectionName := q.getCollectionName(dimension) + + // Ensure target collection exists + if err := q.ensureCollection(ctx, dimension); err != nil { + return err + } + batchSize := uint32(64) var offset *qdrant.PointId = nil totalCopied := 0 for { scrollResult, err := q.client.Scroll(ctx, &qdrant.ScrollPoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Filter: &qdrant.Filter{ Must: []*qdrant.Condition{ qdrant.NewMatch(fieldKnowledgeBaseID, sourceKnowledgeBaseID), @@ -524,6 +718,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, fieldChunkID: targetChunkID, fieldKnowledgeID: targetKnowledgeID, fieldKnowledgeBaseID: targetKnowledgeBaseID, + fieldIsEnabled: true, }) var vectors *qdrant.Vectors @@ -549,7 +744,7 @@ func (q *qdrantRepository) CopyIndices(ctx context.Context, if len(targetPoints) > 0 { _, err := q.client.Upsert(ctx, &qdrant.UpsertPoints{ - CollectionName: q.collectionName, + CollectionName: collectionName, Points: targetPoints, }) if err != nil { @@ -588,11 +783,11 @@ func createPayload(embedding *QdrantVectorEmbedding) map[string]*qdrant.Value { return qdrant.NewValueMap(payload) } -func buildRetrieveResult(results []*typesLocal.IndexWithScore, retrieverType typesLocal.RetrieverType) []*typesLocal.RetrieveResult { - return []*typesLocal.RetrieveResult{ +func buildRetrieveResult(results []*types.IndexWithScore, retrieverType types.RetrieverType) []*types.RetrieveResult { + return []*types.RetrieveResult{ { Results: results, - RetrieverEngineType: typesLocal.QdrantRetrieverEngineType, + RetrieverEngineType: types.QdrantRetrieverEngineType, RetrieverType: retrieverType, Error: nil, }, @@ -667,3 +862,30 @@ func fromQdrantVectorEmbedding(id string, MatchType: matchType, } } + +// tokenizeQuery splits a query string into tokens for OR-based full-text search. +// It uses jieba for professional Chinese word segmentation. +func tokenizeQuery(query string) []string { + query = strings.TrimSpace(query) + if query == "" { + return nil + } + + // Use jieba for segmentation (search mode for better recall) + words := types.Jieba.CutForSearch(query, true) + + // Filter and deduplicate + seen := make(map[string]bool) + result := make([]string, 0, len(words)) + for _, word := range words { + word = strings.TrimSpace(strings.ToLower(word)) + // Skip empty, single-char, and already seen words + if utf8.RuneCountInString(word) < 2 || seen[word] { + continue + } + seen[word] = true + result = append(result, word) + } + + return result +} diff --git a/internal/application/repository/retriever/qdrant/structs.go b/internal/application/repository/retriever/qdrant/structs.go index 38ece9a2..9bae0d57 100644 --- a/internal/application/repository/retriever/qdrant/structs.go +++ b/internal/application/repository/retriever/qdrant/structs.go @@ -1,10 +1,16 @@ package qdrant -import "github.com/qdrant/go-client/qdrant" +import ( + "sync" + + "github.com/qdrant/go-client/qdrant" +) type qdrantRepository struct { - client *qdrant.Client - collectionName string + client *qdrant.Client + collectionBaseName string + // Cache for initialized collections (dimension -> true) + initializedCollections sync.Map } type QdrantVectorEmbedding struct { diff --git a/internal/application/service/chunk.go b/internal/application/service/chunk.go index 1720d3f4..dd0a9751 100644 --- a/internal/application/service/chunk.go +++ b/internal/application/service/chunk.go @@ -389,7 +389,7 @@ func (s *chunkService) DeleteGeneratedQuestion(ctx context.Context, chunkID stri sourceID := fmt.Sprintf("%s-%s", chunkID, questionID) tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ "chunk_id": chunkID, diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 2e49b1ec..f8349054 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -737,7 +737,7 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") @@ -826,7 +826,7 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed") @@ -1047,7 +1047,7 @@ func (s *knowledgeService) processChunks(ctx context.Context, // 删除旧的索引数据 tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err == nil { if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions()); err != nil { logger.Warnf(ctx, "Failed to delete existing index data (may not exist): %v", err) @@ -1663,7 +1663,7 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn } ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { logger.Errorf(ctx, "Failed to init retrieve engine: %v", err) return fmt.Errorf("failed to init retrieve engine: %w", err) @@ -1772,7 +1772,7 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy } ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { logger.Errorf(ctx, "Failed to init retrieve engine: %v", err) return fmt.Errorf("failed to init retrieve engine: %w", err) @@ -2268,7 +2268,7 @@ func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, c } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { return err } @@ -2539,7 +2539,7 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { return err } @@ -3257,7 +3257,7 @@ func (s *knowledgeService) UpdateFAQEntry(ctx context.Context, tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { return err @@ -3306,7 +3306,7 @@ func (s *knowledgeService) UpdateFAQEntryStatus(ctx context.Context, // Sync update to retriever engines chunkStatusMap := map[string]bool{chunk.ID: isEnabled} tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { return err } @@ -3366,7 +3366,7 @@ func (s *knowledgeService) UpdateFAQEntryStatusBatch(ctx context.Context, tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { return err @@ -4105,7 +4105,7 @@ func (s *knowledgeService) indexFAQChunks(ctx context.Context, logger.Debugf(ctx, "indexFAQChunks: starting to index %d chunks", len(chunks)) tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { return err } @@ -4212,7 +4212,7 @@ func (s *knowledgeService) deleteFAQChunkVectors(ctx context.Context, return err } tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { return err } @@ -4358,7 +4358,7 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle if knowledge.EmbeddingModelID != "" { retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { logger.GetLogger(ctx).WithField("error", err).Error("Failed to init retrieve engine during cleanup") @@ -4749,7 +4749,7 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task) if err == nil { retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err == nil { chunkIDs := make([]string, 0, len(chunksDeleted)) diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index e8897354..1ea1d11d 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -249,7 +249,7 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) retrieveEngine, err := retriever.NewCompositeRetrieveEngine( s.retrieveEngine, - tenantInfo.RetrieverEngines.Engines, + tenantInfo.GetEffectiveEngines(), ) if err != nil { logger.Warnf(ctx, "Failed to create retrieve engine: %v", err) @@ -438,7 +438,7 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context, tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) // Create a composite retrieval engine with tenant's configured retrievers - retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.RetrieverEngines.Engines) + retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines()) if err != nil { logger.Errorf(ctx, "Failed to create retrieval engine: %v", err) return nil, err diff --git a/internal/application/service/user.go b/internal/application/service/user.go index c33c215d..d5786b7e 100644 --- a/internal/application/service/user.go +++ b/internal/application/service/user.go @@ -64,35 +64,6 @@ func NewUserService( } } -var engine = map[string][]types.RetrieverEngineParams{ - "postgres": { - { - RetrieverType: types.KeywordsRetrieverType, - RetrieverEngineType: types.PostgresRetrieverEngineType, - }, - { - RetrieverType: types.VectorRetrieverType, - RetrieverEngineType: types.PostgresRetrieverEngineType, - }, - }, - "elasticsearch_v7": { - { - RetrieverType: types.KeywordsRetrieverType, - RetrieverEngineType: types.ElasticsearchRetrieverEngineType, - }, - }, - "elasticsearch_v8": { - { - RetrieverType: types.KeywordsRetrieverType, - RetrieverEngineType: types.ElasticsearchRetrieverEngineType, - }, - { - RetrieverType: types.VectorRetrieverType, - RetrieverEngineType: types.ElasticsearchRetrieverEngineType, - }, - }, -} - // Register creates a new user account func (s *userService) Register(ctx context.Context, req *types.RegisterRequest) (*types.User, error) { logger.Info(ctx, "Start user registration") @@ -120,21 +91,12 @@ func (s *userService) Register(ctx context.Context, req *types.RegisterRequest) return nil, errors.New("failed to process password") } - egs := []types.RetrieverEngineParams{} - for _, driver := range strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") { - if val, ok := engine[driver]; ok { - egs = append(egs, val...) - } - } - egs = uniqueRetrieverEngine(egs) - logger.Debug(ctx, "user register retriever engines") - // Create default tenant for the user + // Note: RetrieverEngines is left empty - system will use defaults from RETRIEVE_DRIVER env tenant := &types.Tenant{ - Name: fmt.Sprintf("%s's Workspace", secutils.SanitizeForLog(req.Username)), - Description: "Default workspace", - Status: "active", - RetrieverEngines: types.RetrieverEngines{Engines: egs}, + Name: fmt.Sprintf("%s's Workspace", secutils.SanitizeForLog(req.Username)), + Description: "Default workspace", + Status: "active", } createdTenant, err := s.tenantService.CreateTenant(ctx, tenant) @@ -464,15 +426,3 @@ func (s *userService) GetCurrentUser(ctx context.Context) (*types.User, error) { return user, nil } - -func uniqueRetrieverEngine(engine []types.RetrieverEngineParams) []types.RetrieverEngineParams { - seen := make(map[types.RetrieverEngineParams]bool) - var result []types.RetrieverEngineParams - for _, v := range engine { - if !seen[v] { - seen[v] = true - result = append(result, v) - } - } - return result -} diff --git a/internal/types/tenant.go b/internal/types/tenant.go index 0a597dc7..3430294e 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -3,11 +3,52 @@ package types import ( "database/sql/driver" "encoding/json" + "os" + "strings" "time" "gorm.io/gorm" ) +// retrieverEngineMapping maps RETRIEVE_DRIVER values to retriever engine configurations +var retrieverEngineMapping = map[string][]RetrieverEngineParams{ + "postgres": { + {RetrieverType: KeywordsRetrieverType, RetrieverEngineType: PostgresRetrieverEngineType}, + {RetrieverType: VectorRetrieverType, RetrieverEngineType: PostgresRetrieverEngineType}, + }, + "elasticsearch_v7": { + {RetrieverType: KeywordsRetrieverType, RetrieverEngineType: ElasticsearchRetrieverEngineType}, + }, + "elasticsearch_v8": { + {RetrieverType: KeywordsRetrieverType, RetrieverEngineType: ElasticsearchRetrieverEngineType}, + {RetrieverType: VectorRetrieverType, RetrieverEngineType: ElasticsearchRetrieverEngineType}, + }, + "qdrant": { + {RetrieverType: KeywordsRetrieverType, RetrieverEngineType: QdrantRetrieverEngineType}, + {RetrieverType: VectorRetrieverType, RetrieverEngineType: QdrantRetrieverEngineType}, + }, +} + +// GetDefaultRetrieverEngines returns the default retriever engines based on RETRIEVE_DRIVER env +func GetDefaultRetrieverEngines() []RetrieverEngineParams { + result := []RetrieverEngineParams{} + seen := make(map[string]bool) + + for _, driver := range strings.Split(os.Getenv("RETRIEVE_DRIVER"), ",") { + driver = strings.TrimSpace(driver) + if params, ok := retrieverEngineMapping[driver]; ok { + for _, p := range params { + key := string(p.RetrieverType) + ":" + string(p.RetrieverEngineType) + if !seen[key] { + seen[key] = true + result = append(result, p) + } + } + } + } + return result +} + // Tenant represents the tenant type Tenant struct { // ID @@ -49,6 +90,14 @@ type RetrieverEngines struct { Engines []RetrieverEngineParams `yaml:"engines" json:"engines" gorm:"type:json"` } +// GetEffectiveEngines returns the tenant's engines if configured, otherwise returns system defaults +func (t *Tenant) GetEffectiveEngines() []RetrieverEngineParams { + if len(t.RetrieverEngines.Engines) > 0 { + return t.RetrieverEngines.Engines + } + return GetDefaultRetrieverEngines() +} + // BeforeCreate is a hook function that is called before creating a tenant func (t *Tenant) BeforeCreate(tx *gorm.DB) error { if t.RetrieverEngines.Engines == nil { diff --git a/scripts/dev.sh b/scripts/dev.sh index 169e9c0f..13f5e7fe 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -52,10 +52,10 @@ detect_compose_cmd() { # 显示帮助信息 show_help() { printf "%b\n" "${GREEN}WeKnora 开发环境脚本${NC}" - echo "用法: $0 [命令]" + echo "用法: $0 [命令] [选项]" echo "" echo "命令:" - echo " start 启动基础设施服务(postgres, redis, minio, neo4j, docreader, jaeger)" + echo " start 启动基础设施服务(postgres, redis, docreader)" echo " stop 停止所有服务" echo " restart 重启所有服务" echo " logs 查看服务日志" @@ -64,10 +64,20 @@ show_help() { echo " frontend 启动前端开发服务器(本地运行)" echo " help 显示此帮助信息" echo "" + echo "可选 Profile(用于 start 命令):" + echo " --minio 启动 MinIO 对象存储" + echo " --qdrant 启动 Qdrant 向量数据库" + echo " --neo4j 启动 Neo4j 图数据库" + echo " --jaeger 启动 Jaeger 链路追踪" + echo " --full 启动所有可选服务" + echo "" echo "示例:" - echo " $0 start # 启动所有基础设施" - echo " $0 app # 在另一个终端启动后端" - echo " $0 frontend # 在另一个终端启动前端" + echo " $0 start # 启动基础服务" + echo " $0 start --qdrant # 启动基础服务 + Qdrant" + echo " $0 start --qdrant --jaeger # 启动基础服务 + Qdrant + Jaeger" + echo " $0 start --full # 启动所有服务" + echo " $0 app # 在另一个终端启动后端" + echo " $0 frontend # 在另一个终端启动前端" } # 检查 Docker @@ -107,8 +117,43 @@ start_services() { return 1 fi + # 解析 profile 参数 + shift # 移除 "start" 命令本身 + PROFILES="--profile full" + ENABLED_SERVICES="" + + while [ $# -gt 0 ]; do + case "$1" in + --minio) + PROFILES="$PROFILES --profile minio" + ENABLED_SERVICES="$ENABLED_SERVICES minio" + ;; + --qdrant) + PROFILES="$PROFILES --profile qdrant" + ENABLED_SERVICES="$ENABLED_SERVICES qdrant" + ;; + --neo4j) + PROFILES="$PROFILES --profile neo4j" + ENABLED_SERVICES="$ENABLED_SERVICES neo4j" + ;; + --jaeger) + PROFILES="$PROFILES --profile jaeger" + ENABLED_SERVICES="$ENABLED_SERVICES jaeger" + ;; + --full) + PROFILES="--profile full" + ENABLED_SERVICES="minio qdrant neo4j jaeger" + break + ;; + *) + log_warning "未知参数: $1" + ;; + esac + shift + done + # 启动服务 - "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml up -d + "$DOCKER_COMPOSE_BIN" $DOCKER_COMPOSE_SUBCMD -f docker-compose.dev.yml $PROFILES up -d if [ $? -eq 0 ]; then log_success "基础设施服务已启动" @@ -116,10 +161,22 @@ start_services() { log_info "服务访问地址:" echo " - PostgreSQL: localhost:5432" echo " - Redis: localhost:6379" - echo " - MinIO: localhost:9000 (Console: localhost:9001)" - echo " - Neo4j: localhost:7474 (Bolt: localhost:7687)" echo " - DocReader: localhost:50051" - echo " - Jaeger: localhost:16686" + + # 根据启用的 profile 显示额外服务 + if [[ "$ENABLED_SERVICES" == *"minio"* ]]; then + echo " - MinIO: localhost:9000 (Console: localhost:9001)" + fi + if [[ "$ENABLED_SERVICES" == *"qdrant"* ]]; then + echo " - Qdrant: localhost:6333 (gRPC: localhost:6334)" + fi + if [[ "$ENABLED_SERVICES" == *"neo4j"* ]]; then + echo " - Neo4j: localhost:7474 (Bolt: localhost:7687)" + fi + if [[ "$ENABLED_SERVICES" == *"jaeger"* ]]; then + echo " - Jaeger: localhost:16686" + fi + echo "" log_info "接下来的步骤:" printf "%b\n" "${YELLOW}1. 在新终端运行后端:${NC} make dev-app" @@ -201,6 +258,7 @@ start_app() { export REDIS_ADDR=localhost:6379 export OTEL_EXPORTER_OTLP_ENDPOINT=localhost:4317 export NEO4J_URI=bolt://localhost:7687 + export QDRANT_HOST=localhost # 确保必要的环境变量已设置 if [ -z "$DB_DRIVER" ]; then @@ -251,9 +309,10 @@ start_frontend() { } # 解析命令 -case "${1:-help}" in +CMD="${1:-help}" +case "$CMD" in start) - start_services + start_services "$@" ;; stop) stop_services @@ -277,7 +336,7 @@ case "${1:-help}" in show_help ;; *) - log_error "未知命令: $1" + log_error "未知命令: $CMD" show_help exit 1 ;;