diff --git a/migrations/versioned/000002_add_agent_config_if_missing.down.sql b/migrations/versioned/000002_add_agent_config_if_missing.down.sql new file mode 100644 index 00000000..c64ed519 --- /dev/null +++ b/migrations/versioned/000002_add_agent_config_if_missing.down.sql @@ -0,0 +1,28 @@ +-- 000016_add_agent_config_if_missing.down.sql +-- Rollback for agent_config / context_config / agent_steps columns and indexes + +BEGIN; + +-- Drop JSONB indexes if they exist +DROP INDEX IF EXISTS idx_messages_agent_steps; +DROP INDEX IF EXISTS idx_sessions_context_config; +DROP INDEX IF EXISTS idx_sessions_agent_config; +DROP INDEX IF EXISTS idx_tenants_agent_config; + +-- Drop columns if they exist +ALTER TABLE messages + DROP COLUMN IF EXISTS agent_steps; + +ALTER TABLE sessions + DROP COLUMN IF EXISTS context_config; + +ALTER TABLE sessions + DROP COLUMN IF EXISTS agent_config; + +ALTER TABLE tenants + DROP COLUMN IF EXISTS agent_config; + +COMMIT; + + + diff --git a/migrations/versioned/000002_add_agent_config_if_missing.up.sql b/migrations/versioned/000002_add_agent_config_if_missing.up.sql new file mode 100644 index 00000000..a6fe6621 --- /dev/null +++ b/migrations/versioned/000002_add_agent_config_if_missing.up.sql @@ -0,0 +1,182 @@ +-- 000016_add_agent_config_if_missing.up.sql +-- Incremental migration: add agent_config / context_config / agent_steps columns and indexes if missing +-- Ported from migrations/paradedb/02-add-agent-config-if-missing.sql + +BEGIN; + +-- ============================================ +-- 1. 为 tenants 表添加 agent_config +-- ============================================ + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'tenants' + AND column_name = 'agent_config' + ) THEN + ALTER TABLE tenants + ADD COLUMN agent_config JSONB DEFAULT NULL; + + COMMENT ON COLUMN tenants.agent_config IS 'Tenant-level agent configuration in JSON format'; + + RAISE NOTICE 'Added agent_config column to tenants table'; + ELSE + -- 如果字段已存在但类型是 JSON,转换为 JSONB + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'tenants' + AND column_name = 'agent_config' + AND data_type = 'json' + ) THEN + ALTER TABLE tenants + ALTER COLUMN agent_config TYPE JSONB USING agent_config::jsonb; + + RAISE NOTICE 'Converted tenants.agent_config from JSON to JSONB'; + ELSE + RAISE NOTICE 'agent_config column already exists in tenants table'; + END IF; + END IF; +END $$; + +-- ============================================ +-- 2. 为 sessions 表添加 agent_config +-- ============================================ + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'sessions' + AND column_name = 'agent_config' + ) THEN + ALTER TABLE sessions + ADD COLUMN agent_config JSONB DEFAULT NULL; + + COMMENT ON COLUMN sessions.agent_config IS 'Session-level agent configuration in JSON format'; + + RAISE NOTICE 'Added agent_config column to sessions table'; + ELSE + RAISE NOTICE 'agent_config column already exists in sessions table'; + END IF; +END $$; + +-- ============================================ +-- 3. 为 sessions 表添加 context_config +-- ============================================ + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'sessions' + AND column_name = 'context_config' + ) THEN + ALTER TABLE sessions + ADD COLUMN context_config JSONB DEFAULT NULL; + + COMMENT ON COLUMN sessions.context_config IS 'LLM context management configuration (separate from message storage)'; + + RAISE NOTICE 'Added context_config column to sessions table'; + ELSE + RAISE NOTICE 'context_config column already exists in sessions table'; + END IF; +END $$; + +-- ============================================ +-- 4. 为 messages 表添加 agent_steps +-- ============================================ + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'messages' + AND column_name = 'agent_steps' + ) THEN + ALTER TABLE messages + ADD COLUMN agent_steps JSONB DEFAULT NULL; + + COMMENT ON COLUMN messages.agent_steps IS 'Agent execution steps (reasoning process and tool calls)'; + + RAISE NOTICE 'Added agent_steps column to messages table'; + ELSE + RAISE NOTICE 'agent_steps column already exists in messages table'; + END IF; +END $$; + +-- ============================================ +-- 5. 为 JSON 字段添加 GIN 索引(提高查询性能) +-- ============================================ + +DO $$ +BEGIN + -- 为 tenants.agent_config 添加索引(仅当字段类型为 JSONB 时) + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'tenants' + AND indexname = 'idx_tenants_agent_config' + ) THEN + -- 检查字段类型是否为 JSONB + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'tenants' + AND column_name = 'agent_config' + AND data_type = 'jsonb' + ) THEN + CREATE INDEX idx_tenants_agent_config ON tenants USING GIN (agent_config); + RAISE NOTICE 'Created index idx_tenants_agent_config'; + ELSE + RAISE NOTICE 'Skipped index creation for tenants.agent_config (not JSONB type)'; + END IF; + ELSE + RAISE NOTICE 'Index idx_tenants_agent_config already exists'; + END IF; + + -- 为 sessions.agent_config 添加索引 + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'sessions' + AND indexname = 'idx_sessions_agent_config' + ) THEN + CREATE INDEX idx_sessions_agent_config ON sessions USING GIN (agent_config); + RAISE NOTICE 'Created index idx_sessions_agent_config'; + ELSE + RAISE NOTICE 'Index idx_sessions_agent_config already exists'; + END IF; + + -- 为 sessions.context_config 添加索引 + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'sessions' + AND indexname = 'idx_sessions_context_config' + ) THEN + CREATE INDEX idx_sessions_context_config ON sessions USING GIN (context_config); + RAISE NOTICE 'Created index idx_sessions_context_config'; + ELSE + RAISE NOTICE 'Index idx_sessions_context_config already exists'; + END IF; + + -- 为 messages.agent_steps 添加索引 + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'messages' + AND indexname = 'idx_messages_agent_steps' + ) THEN + CREATE INDEX idx_messages_agent_steps ON messages USING GIN (agent_steps); + RAISE NOTICE 'Created index idx_messages_agent_steps'; + ELSE + RAISE NOTICE 'Index idx_messages_agent_steps already exists'; + END IF; +END $$; + +COMMIT; + + + diff --git a/migrations/versioned/000003_cleanup_unreferenced_models.down.sql b/migrations/versioned/000003_cleanup_unreferenced_models.down.sql new file mode 100644 index 00000000..0034b4c9 --- /dev/null +++ b/migrations/versioned/000003_cleanup_unreferenced_models.down.sql @@ -0,0 +1,8 @@ +-- 000017_cleanup_unreferenced_models.down.sql +-- This migration performs a data cleanup (soft delete) which is not safely reversible. +-- Down migration is a no-op. + +SELECT 1; + + + diff --git a/migrations/versioned/000003_cleanup_unreferenced_models.up.sql b/migrations/versioned/000003_cleanup_unreferenced_models.up.sql new file mode 100644 index 00000000..d985d8ad --- /dev/null +++ b/migrations/versioned/000003_cleanup_unreferenced_models.up.sql @@ -0,0 +1,32 @@ +-- 000017_cleanup_unreferenced_models.up.sql +-- Cleanup unreferenced models by soft-deleting them (set deleted_at) +-- Ported from migrations/paradedb/03-cleanup-unreferenced-models.sql + +BEGIN; + +WITH referenced_models AS ( + SELECT embedding_model_id AS model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != '' + UNION + SELECT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != '' + UNION + SELECT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != '' + UNION + SELECT vlm_config ->> 'model_id' + FROM knowledge_bases + WHERE deleted_at IS NULL + AND vlm_config ->> 'model_id' IS NOT NULL + AND vlm_config ->> 'model_id' != '' + UNION + SELECT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != '' +) +UPDATE models m +SET deleted_at = CURRENT_TIMESTAMP +WHERE m.deleted_at IS NULL + AND m.is_default = FALSE + AND m.tenant_id != 0 + AND m.id NOT IN (SELECT model_id FROM referenced_models); + +COMMIT; + + + diff --git a/migrations/versioned/000014_backfill_models_tenant_id.up.sql b/migrations/versioned/000014_backfill_models_tenant_id.up.sql index 2f9a1d83..7fab94e4 100644 --- a/migrations/versioned/000014_backfill_models_tenant_id.up.sql +++ b/migrations/versioned/000014_backfill_models_tenant_id.up.sql @@ -4,16 +4,52 @@ BEGIN; WITH tenant_source AS ( + -- 从 knowledge_bases 中取出各模型 ID(embedding / summary)及其 tenant_id SELECT - k.id AS model_id, - k.tenant_id - FROM knowledges k - WHERE k.tenant_id IS NOT NULL + kb.embedding_model_id AS model_id, + kb.tenant_id + FROM knowledge_bases kb + WHERE kb.tenant_id IS NOT NULL + AND kb.embedding_model_id IS NOT NULL + AND kb.embedding_model_id <> '' + + UNION + + SELECT + kb.summary_model_id AS model_id, + kb.tenant_id + FROM knowledge_bases kb + WHERE kb.tenant_id IS NOT NULL + AND kb.summary_model_id IS NOT NULL + AND kb.summary_model_id <> '' + + UNION + + -- rerank 模型 + SELECT + kb.rerank_model_id AS model_id, + kb.tenant_id + FROM knowledge_bases kb + WHERE kb.tenant_id IS NOT NULL + AND kb.rerank_model_id IS NOT NULL + AND kb.rerank_model_id <> '' + + UNION + + -- VLM 模型(存储在 vlm_config JSON 的 model_id 字段中) + SELECT + (kb.vlm_config ->> 'model_id') AS model_id, + kb.tenant_id + FROM knowledge_bases kb + WHERE kb.tenant_id IS NOT NULL + AND kb.vlm_config ->> 'model_id' IS NOT NULL + AND kb.vlm_config ->> 'model_id' <> '' ) UPDATE models m -JOIN tenant_source ts ON m.id = ts.model_id -SET m.tenant_id = ts.tenant_id -WHERE m.tenant_id = 0; +SET tenant_id = ts.tenant_id +FROM tenant_source ts +WHERE m.id = ts.model_id + AND m.tenant_id = 0; COMMIT; diff --git a/migrations/versioned/000011_remove_kb_rerank_model.down.sql b/migrations/versioned/000015_remove_kb_rerank_model.down.sql similarity index 79% rename from migrations/versioned/000011_remove_kb_rerank_model.down.sql rename to migrations/versioned/000015_remove_kb_rerank_model.down.sql index d354e231..0678faf5 100644 --- a/migrations/versioned/000011_remove_kb_rerank_model.down.sql +++ b/migrations/versioned/000015_remove_kb_rerank_model.down.sql @@ -1,4 +1,4 @@ --- 000011_remove_kb_rerank_model.down.sql +-- 000015_remove_kb_rerank_model.down.sql -- Reintroduce rerank_model_id column to knowledge_bases table BEGIN; @@ -9,3 +9,5 @@ ALTER TABLE knowledge_bases COMMIT; + + diff --git a/migrations/versioned/000011_remove_kb_rerank_model.up.sql b/migrations/versioned/000015_remove_kb_rerank_model.up.sql similarity index 78% rename from migrations/versioned/000011_remove_kb_rerank_model.up.sql rename to migrations/versioned/000015_remove_kb_rerank_model.up.sql index a22db423..1e290f4b 100644 --- a/migrations/versioned/000011_remove_kb_rerank_model.up.sql +++ b/migrations/versioned/000015_remove_kb_rerank_model.up.sql @@ -1,4 +1,4 @@ --- 000011_remove_kb_rerank_model.up.sql +-- 000015_remove_kb_rerank_model.up.sql -- Remove rerank_model_id column from knowledge_bases table BEGIN; @@ -9,3 +9,5 @@ ALTER TABLE knowledge_bases COMMIT; + + diff --git a/migrations/versioned/000016_add_agent_config_if_missing.down.sql b/migrations/versioned/000016_add_agent_config_if_missing.down.sql new file mode 100644 index 00000000..ef7ce62c --- /dev/null +++ b/migrations/versioned/000016_add_agent_config_if_missing.down.sql @@ -0,0 +1,21 @@ +-- 000016_add_agent_config_if_missing.down.sql +-- Rollback for agent_config / context_config / agent_steps columns and indexes +BEGIN; +-- Drop JSONB indexes if they exist +DROP INDEX IF EXISTS idx_messages_agent_steps; +DROP INDEX IF EXISTS idx_sessions_context_config; +DROP INDEX IF EXISTS idx_sessions_agent_config; +DROP INDEX IF EXISTS idx_tenants_agent_config; +-- Drop columns if they exist +ALTER TABLE messages + DROP COLUMN IF EXISTS agent_steps; +ALTER TABLE sessions + DROP COLUMN IF EXISTS context_config; +ALTER TABLE sessions + DROP COLUMN IF EXISTS agent_config; +ALTER TABLE tenants + DROP COLUMN IF EXISTS agent_config; +COMMIT; + + + diff --git a/migrations/versioned/000016_add_agent_config_if_missing.up.sql b/migrations/versioned/000016_add_agent_config_if_missing.up.sql new file mode 100644 index 00000000..7c72f562 --- /dev/null +++ b/migrations/versioned/000016_add_agent_config_if_missing.up.sql @@ -0,0 +1,182 @@ +-- 000016_add_agent_config_if_missing.up.sql +-- Incremental migration: add agent_config / context_config / agent_steps columns and indexes if missing +-- Ported from migrations/paradedb/02-add-agent-config-if-missing.sql + +BEGIN; + +-- ============================================ +-- 1. 为 tenants 表添加 agent_config +-- ============================================ + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'tenants' + AND column_name = 'agent_config' + ) THEN + ALTER TABLE tenants + ADD COLUMN agent_config JSONB DEFAULT NULL; + + COMMENT ON COLUMN tenants.agent_config IS 'Tenant-level agent configuration in JSON format'; + + RAISE NOTICE 'Added agent_config column to tenants table'; + ELSE + -- 如果字段已存在但类型是 JSON,转换为 JSONB + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'tenants' + AND column_name = 'agent_config' + AND data_type = 'json' + ) THEN + ALTER TABLE tenants + ALTER COLUMN agent_config TYPE JSONB USING agent_config::jsonb; + + RAISE NOTICE 'Converted tenants.agent_config from JSON to JSONB'; + ELSE + RAISE NOTICE 'agent_config column already exists in tenants table'; + END IF; + END IF; +END $$; + +-- ============================================ +-- 2. 为 sessions 表添加 agent_config +-- ============================================ + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'sessions' + AND column_name = 'agent_config' + ) THEN + ALTER TABLE sessions + ADD COLUMN agent_config JSONB DEFAULT NULL; + + COMMENT ON COLUMN sessions.agent_config IS 'Session-level agent configuration in JSON format'; + + RAISE NOTICE 'Added agent_config column to sessions table'; + ELSE + RAISE NOTICE 'agent_config column already exists in sessions table'; + END IF; +END $$; + +-- ============================================ +-- 3. 为 sessions 表添加 context_config +-- ============================================ + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'sessions' + AND column_name = 'context_config' + ) THEN + ALTER TABLE sessions + ADD COLUMN context_config JSONB DEFAULT NULL; + + COMMENT ON COLUMN sessions.context_config IS 'LLM context management configuration (separate from message storage)'; + + RAISE NOTICE 'Added context_config column to sessions table'; + ELSE + RAISE NOTICE 'context_config column already exists in sessions table'; + END IF; +END $$; ++ +-- ============================================ +-- 4. 为 messages 表添加 agent_steps +-- ============================================ + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'messages' + AND column_name = 'agent_steps' + ) THEN + ALTER TABLE messages + ADD COLUMN agent_steps JSONB DEFAULT NULL; + + COMMENT ON COLUMN messages.agent_steps IS 'Agent execution steps (reasoning process and tool calls)'; + + RAISE NOTICE 'Added agent_steps column to messages table'; + ELSE + RAISE NOTICE 'agent_steps column already exists in messages table'; + END IF; +END $$; + +-- ============================================ +-- 5. 为 JSON 字段添加 GIN 索引(提高查询性能) +-- ============================================ + +DO $$ +BEGIN + -- 为 tenants.agent_config 添加索引(仅当字段类型为 JSONB 时) + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'tenants' + AND indexname = 'idx_tenants_agent_config' + ) THEN + -- 检查字段类型是否为 JSONB + IF EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'tenants' + AND column_name = 'agent_config' + AND data_type = 'jsonb' + ) THEN + CREATE INDEX idx_tenants_agent_config ON tenants USING GIN (agent_config); + RAISE NOTICE 'Created index idx_tenants_agent_config'; + ELSE + RAISE NOTICE 'Skipped index creation for tenants.agent_config (not JSONB type)'; + END IF; + ELSE + RAISE NOTICE 'Index idx_tenants_agent_config already exists'; + END IF; + + -- 为 sessions.agent_config 添加索引 + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'sessions' + AND indexname = 'idx_sessions_agent_config' + ) THEN + CREATE INDEX idx_sessions_agent_config ON sessions USING GIN (agent_config); + RAISE NOTICE 'Created index idx_sessions_agent_config'; + ELSE + RAISE NOTICE 'Index idx_sessions_agent_config already exists'; + END IF; + + -- 为 sessions.context_config 添加索引 + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'sessions' + AND indexname = 'idx_sessions_context_config' + ) THEN + CREATE INDEX idx_sessions_context_config ON sessions USING GIN (context_config); + RAISE NOTICE 'Created index idx_sessions_context_config'; + ELSE + RAISE NOTICE 'Index idx_sessions_context_config already exists'; + END IF; + + -- 为 messages.agent_steps 添加索引 + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'messages' + AND indexname = 'idx_messages_agent_steps' + ) THEN + CREATE INDEX idx_messages_agent_steps ON messages USING GIN (agent_steps); + RAISE NOTICE 'Created index idx_messages_agent_steps'; + ELSE + RAISE NOTICE 'Index idx_messages_agent_steps already exists'; + END IF; +END $$; + +COMMIT; + + + diff --git a/migrations/versioned/000017_cleanup_unreferenced_models.down.sql b/migrations/versioned/000017_cleanup_unreferenced_models.down.sql new file mode 100644 index 00000000..007eb28f --- /dev/null +++ b/migrations/versioned/000017_cleanup_unreferenced_models.down.sql @@ -0,0 +1,7 @@ +-- 000017_cleanup_unreferenced_models.down.sql +-- This migration performs a data cleanup (soft delete) which is not safely reversible. +-- Down migration is a no-op. +SELECT 1; + + + diff --git a/migrations/versioned/000017_cleanup_unreferenced_models.up.sql b/migrations/versioned/000017_cleanup_unreferenced_models.up.sql new file mode 100644 index 00000000..1d2fbb1c --- /dev/null +++ b/migrations/versioned/000017_cleanup_unreferenced_models.up.sql @@ -0,0 +1,29 @@ +-- 000017_cleanup_unreferenced_models.up.sql +-- Cleanup unreferenced models by soft-deleting them (set deleted_at) +-- Ported from migrations/paradedb/03-cleanup-unreferenced-models.sql +BEGIN; +WITH referenced_models AS ( + SELECT embedding_model_id AS model_id FROM knowledge_bases WHERE deleted_at IS NULL AND embedding_model_id != '' + UNION + SELECT summary_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND summary_model_id != '' + UNION + SELECT rerank_model_id FROM knowledge_bases WHERE deleted_at IS NULL AND rerank_model_id != '' + UNION + SELECT vlm_config ->> 'model_id' + FROM knowledge_bases + WHERE deleted_at IS NULL + AND vlm_config ->> 'model_id' IS NOT NULL + AND vlm_config ->> 'model_id' != '' + UNION + SELECT embedding_model_id FROM knowledges WHERE deleted_at IS NULL AND embedding_model_id IS NOT NULL AND embedding_model_id != '' +) +UPDATE models m +SET deleted_at = CURRENT_TIMESTAMP +WHERE m.deleted_at IS NULL + AND m.is_default = FALSE + AND m.tenant_id != 0 + AND m.id NOT IN (SELECT model_id FROM referenced_models); +COMMIT; + + +