From dbab7b04051f812ae36346847ca5379b249b370c Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 2 Mar 2026 11:02:19 +0800 Subject: [PATCH] feat: introduce storage provider configuration and enhance backward compatibility - Added a new `StorageProviderConfig` struct to the `KnowledgeBase` for improved storage provider management, separating provider selection from storage credentials. - Updated the `UnmarshalJSON` method to auto-populate `StorageProviderConfig` from the legacy `StorageConfig` for backward compatibility. - Refactored various service methods to utilize the new storage provider configuration, ensuring a smooth transition from the legacy system. - Implemented database migrations to add the `storage_provider_config` column and migrate existing data, maintaining compatibility with historical knowledge bases. This update enhances the clarity and flexibility of storage configurations while ensuring compatibility with existing data structures. --- client/knowledgebase.go | 15 ++- internal/application/service/knowledge.go | 65 +++++++++---- internal/application/service/knowledgebase.go | 1 + internal/container/container.go | 26 +++++ internal/handler/initialization.go | 27 ++++-- internal/types/knowledgebase.go | 95 +++++++++++++++---- migrations/sqlite/000000_init.up.sql | 1 + .../000014_storage_provider_config.down.sql | 16 ++++ .../000014_storage_provider_config.up.sql | 31 ++++++ 9 files changed, 231 insertions(+), 46 deletions(-) create mode 100644 migrations/versioned/000014_storage_provider_config.down.sql create mode 100644 migrations/versioned/000014_storage_provider_config.up.sql diff --git a/client/knowledgebase.go b/client/knowledgebase.go index 3d16caeb..9224b71a 100644 --- a/client/knowledgebase.go +++ b/client/knowledgebase.go @@ -25,9 +25,10 @@ type KnowledgeBase struct { FAQConfig *FAQConfig `json:"faq_config"` EmbeddingModelID string `json:"embedding_model_id"` SummaryModelID string `json:"summary_model_id"` - VLMConfig VLMConfig `json:"vlm_config"` - StorageConfig StorageConfig `json:"storage_config"` - ExtractConfig *ExtractConfig `json:"extract_config"` + VLMConfig VLMConfig `json:"vlm_config"` + StorageProviderConfig *StorageProviderConfig `json:"storage_provider_config"` + StorageConfig StorageConfig `json:"storage_config"` + ExtractConfig *ExtractConfig `json:"extract_config"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` // Computed fields (not stored in database) @@ -68,7 +69,13 @@ type VLMConfig struct { ModelID string `json:"model_id"` } -// StorageConfig represents the storage configuration +// StorageProviderConfig stores the KB-level storage provider selection. +type StorageProviderConfig struct { + Provider string `json:"provider"` +} + +// StorageConfig represents the legacy storage configuration (cos_config). +// Deprecated: use StorageProviderConfig for provider selection. type StorageConfig struct { SecretID string `json:"secret_id"` SecretKey string `json:"secret_key"` diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 8e5f14b6..c3c67816 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -190,8 +190,8 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, if !IsImageType(getFileType(fileName)) { logger.Info(ctx, "Non-image file with multimodal enabled, skipping COS/VLM validation") } else { - // 解析有效 provider：优先 KB 级别，其次租户默认 - provider := strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider)) + // 解析有效 provider：优先 KB 级别（新字段 > 旧字段），其次租户默认 + provider := kb.GetStorageProvider() tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil { provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider)) @@ -2286,9 +2286,9 @@ func (s *knowledgeService) GetKnowledgeFile(ctx context.Context, id string) (io. return nil, "", err } - // Resolve KB-level file service + // Resolve KB-level file service with FilePath fallback protection kb, _ := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID) - file, err := s.resolveFileService(ctx, kb).GetFile(ctx, knowledge.FilePath) + file, err := s.resolveFileServiceForPath(ctx, kb, knowledge.FilePath).GetFile(ctx, knowledge.FilePath) if err != nil { return nil, "", err } @@ -6737,13 +6737,13 @@ func (s *knowledgeService) getVLMConfig(ctx context.Context, kb *types.Knowledge } func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.KnowledgeBase) *types.DocParserStorageConfig { - sc := &kb.StorageConfig - provider := strings.ToLower(strings.TrimSpace(sc.Provider)) + provider := kb.GetStorageProvider() if provider == "" { provider = "local" } - // Backward compatibility: if KB has full params for the chosen provider, use them. + // Backward compatibility: if legacy cos_config has full params for the chosen provider, use them. + sc := &kb.StorageConfig hasKBFull := false switch provider { case "cos": @@ -6751,12 +6751,11 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno case "minio": hasKBFull = sc.BucketName != "" case "local": - // Local has no credentials; always merge from tenant if available hasKBFull = false } if hasKBFull { - logger.Infof(ctx, "[storage] buildStorageConfig use kb config: kb=%s provider=%s bucket=%s path_prefix=%s", + logger.Infof(ctx, "[storage] buildStorageConfig use legacy kb config: kb=%s provider=%s bucket=%s path_prefix=%s", kb.ID, provider, sc.BucketName, sc.PathPrefix) return &types.DocParserStorageConfig{ Provider: strings.ToUpper(provider), @@ -6769,7 +6768,7 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno } } - // Otherwise merge from tenant's StorageEngineConfig. + // Merge from tenant's StorageEngineConfig. var out types.DocParserStorageConfig out.Provider = strings.ToUpper(provider) @@ -6817,7 +6816,7 @@ func (s *knowledgeService) buildStorageConfig(ctx context.Context, kb *types.Kno } // resolveFileService returns the FileService for the given knowledge base, -// based on the KB's StorageConfig.Provider and the tenant's StorageEngineConfig. +// based on the KB's StorageProviderConfig (or legacy StorageConfig.Provider) and the tenant's StorageEngineConfig. // Falls back to the global fileSvc when no tenant-level storage config is found. func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.KnowledgeBase) interfaces.FileService { if kb == nil { @@ -6825,7 +6824,7 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno return s.fileSvc } - provider := strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider)) + provider := kb.GetStorageProvider() tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil { @@ -6915,6 +6914,40 @@ func (s *knowledgeService) resolveFileService(ctx context.Context, kb *types.Kno return svc } +// resolveFileServiceForPath is like resolveFileService but adds a safety check: +// if the resolved provider doesn't match what the filePath implies, fall back to +// the provider inferred from the file path. This protects historical data when +// tenant/KB config changes but files were stored under the old provider. +func (s *knowledgeService) resolveFileServiceForPath(ctx context.Context, kb *types.KnowledgeBase, filePath string) interfaces.FileService { + svc := s.resolveFileService(ctx, kb) + if filePath == "" { + return svc + } + + inferred := types.InferStorageFromFilePath(filePath) + if inferred == "" { + return svc + } + + configured := kb.GetStorageProvider() + if configured == "" { + tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) + if tenant != nil && tenant.StorageEngineConfig != nil { + configured = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider)) + } + } + if configured == "" { + configured = strings.ToLower(strings.TrimSpace(os.Getenv("STORAGE_TYPE"))) + } + + if configured != "" && configured != inferred { + logger.Warnf(ctx, "[storage] FilePath format mismatch: configured=%s inferred=%s filePath=%s, using global fallback", + configured, inferred, filePath) + return s.fileSvc + } + return svc +} + func IsImageType(fileType string) bool { switch fileType { case "jpg", "jpeg", "png", "gif", "webp", "bmp", "svg", "tiff": @@ -7304,7 +7337,7 @@ func (s *knowledgeService) convert( imageStorage := s.buildImageStorageConfig(ctx, kb) if imageStorage == nil { logger.Infof(ctx, "[storage] chunk image storage: kb=%s provider=%s image_storage=nil(use local temp/files)", - kb.ID, strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider))) + kb.ID, kb.GetStorageProvider()) } else { logger.Infof(ctx, "[storage] chunk image storage: kb=%s provider=%s bucket=%s endpoint=%s path_prefix=%s", kb.ID, @@ -7325,7 +7358,7 @@ func (s *knowledgeService) convert( } if !isURL { - fileReader, err := s.resolveFileService(ctx, kb).GetFile(ctx, payload.FilePath) + fileReader, err := s.resolveFileServiceForPath(ctx, kb, payload.FilePath).GetFile(ctx, payload.FilePath) if err != nil { return s.failKnowledge(ctx, knowledge, isLastRetry, "failed to get file: %v", err) } @@ -7402,7 +7435,7 @@ func (s *knowledgeService) reuploadImagesToStorage( images []docparser.StoredImage, markdown string, ) (string, []docparser.StoredImage) { - provider := strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider)) + provider := kb.GetStorageProvider() tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil { provider = strings.ToLower(strings.TrimSpace(tenant.StorageEngineConfig.DefaultProvider)) @@ -7456,7 +7489,7 @@ func (s *knowledgeService) reuploadImagesToStorage( func (s *knowledgeService) buildImageStorageConfig(ctx context.Context, kb *types.KnowledgeBase) map[string]string { provider := "" if kb != nil { - provider = strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider)) + provider = kb.GetStorageProvider() } tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant) if provider == "" && tenant != nil && tenant.StorageEngineConfig != nil { diff --git a/internal/application/service/knowledgebase.go b/internal/application/service/knowledgebase.go index 15f4eaac..ca7ed209 100644 --- a/internal/application/service/knowledgebase.go +++ b/internal/application/service/knowledgebase.go @@ -575,6 +575,7 @@ func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context, EmbeddingModelID: sourceKB.EmbeddingModelID, SummaryModelID: sourceKB.SummaryModelID, VLMConfig: sourceKB.VLMConfig, + StorageProviderConfig: sourceKB.StorageProviderConfig, StorageConfig: sourceKB.StorageConfig, FAQConfig: faqConfig, } diff --git a/internal/container/container.go b/internal/container/container.go index 57a02ede..32f8d51c 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -413,6 +413,11 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { "Continuing with application startup. Please run migrations manually if needed.", ) } + + // Post-migration: resolve __pending_env__ storage provider markers for historical KBs. + // The SQL migration marks KBs that have documents but no provider with "__pending_env__"; + // we replace that with the actual STORAGE_TYPE from the environment. + resolveStorageProviderPending(db) } else { logger.Infof(context.Background(), "Auto-migration is disabled (AUTO_MIGRATE=false)") } @@ -437,6 +442,27 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { return db, nil } +// resolveStorageProviderPending replaces the "__pending_env__" sentinel in +// knowledge_bases.storage_provider_config with the actual STORAGE_TYPE from the environment. +// This runs once after SQL migrations to bind historical KBs to their real storage provider. +func resolveStorageProviderPending(db *gorm.DB) { + storageType := strings.TrimSpace(os.Getenv("STORAGE_TYPE")) + if storageType == "" { + storageType = "local" + } + storageType = strings.ToLower(storageType) + + result := db.Exec( + `UPDATE knowledge_bases SET storage_provider_config = ? WHERE storage_provider_config IS NOT NULL AND storage_provider_config->>'provider' = '__pending_env__'`, + fmt.Sprintf(`{"provider":"%s"}`, storageType), + ) + if result.Error != nil { + logger.Warnf(context.Background(), "Failed to resolve __pending_env__ storage providers: %v", result.Error) + } else if result.RowsAffected > 0 { + logger.Infof(context.Background(), "Resolved %d knowledge bases with __pending_env__ storage provider → %s", result.RowsAffected, storageType) + } +} + // initFileService initializes file storage service // Creates the appropriate file storage service based on configuration // Supports multiple storage backends (MinIO, COS, local filesystem) diff --git a/internal/handler/initialization.go b/internal/handler/initialization.go index 094d6fca..2c29b356 100644 --- a/internal/handler/initialization.go +++ b/internal/handler/initialization.go @@ -297,12 +297,12 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) { kb.VLMConfig.ModelID = "" } - // 存储引擎：仅写入 provider，参数从租户全局 StorageEngineConfig 读取 + // 存储引擎：仅写入 provider 到新字段，参数从租户全局 StorageEngineConfig 读取 provider := strings.ToLower(strings.TrimSpace(req.StorageProvider)) if provider == "" { provider = "local" } - oldProvider := strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider)) + oldProvider := kb.GetStorageProvider() if oldProvider == "" { oldProvider = "local" } @@ -315,7 +315,7 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) { return } } - kb.StorageConfig = types.StorageConfig{Provider: provider} + kb.SetStorageProvider(provider) // 更新知识图谱配置 if req.NodeExtract.Enabled { @@ -711,6 +711,8 @@ func (h *InitializationHandler) applyKnowledgeBaseInitialization( switch req.Multimodal.StorageType { case "cos": if req.Multimodal.COS != nil { + kb.SetStorageProvider("cos") + // Legacy: also write to cos_config for backward compat with old code paths kb.StorageConfig = types.StorageConfig{ Provider: req.Multimodal.StorageType, BucketName: req.Multimodal.COS.BucketName, @@ -723,6 +725,8 @@ func (h *InitializationHandler) applyKnowledgeBaseInitialization( } case "minio": if req.Multimodal.Minio != nil { + kb.SetStorageProvider("minio") + // Legacy: also write to cos_config for backward compat with old code paths kb.StorageConfig = types.StorageConfig{ Provider: req.Multimodal.StorageType, BucketName: req.Multimodal.Minio.BucketName, @@ -734,6 +738,7 @@ func (h *InitializationHandler) applyKnowledgeBaseInitialization( } } else { kb.VLMConfig = types.VLMConfig{} + kb.SetStorageProvider("") kb.StorageConfig = types.StorageConfig{} } @@ -1333,15 +1338,16 @@ func (h *InitializationHandler) buildConfigResponse(ctx context.Context, models } } - // 判断多模态是否启用：有VLM模型ID或有存储配置 + // 判断多模态是否启用：有VLM模型ID或有存储配置（兼容新旧字段） + storageProvider := kb.GetStorageProvider() hasMultimodal := (kb.VLMConfig.IsEnabled() || - kb.StorageConfig.SecretID != "" || kb.StorageConfig.BucketName != "") + kb.StorageConfig.SecretID != "" || kb.StorageConfig.BucketName != "" || + (storageProvider != "" && storageProvider != "local")) if config["multimodal"] == nil { config["multimodal"] = map[string]interface{}{ "enabled": hasMultimodal, } } else { - // 如果已经设置过 multimodal，更新 enabled 状态 config["multimodal"].(map[string]interface{})["enabled"] = hasMultimodal } @@ -1363,16 +1369,17 @@ func (h *InitializationHandler) buildConfigResponse(ctx context.Context, models "separators": kb.ChunkingConfig.Separators, } - // 添加多模态的COS配置信息 - if kb.StorageConfig.SecretID != "" { + // 添加多模态的存储配置信息（优先读新字段，兼容旧 cos_config） + effectiveProvider := kb.GetStorageProvider() + if kb.StorageConfig.SecretID != "" || (effectiveProvider != "" && effectiveProvider != "local") { if config["multimodal"] == nil { config["multimodal"] = map[string]interface{}{ "enabled": true, } } multimodal := config["multimodal"].(map[string]interface{}) - multimodal["storageType"] = kb.StorageConfig.Provider - switch kb.StorageConfig.Provider { + multimodal["storageType"] = effectiveProvider + switch effectiveProvider { case "cos": multimodal["cos"] = map[string]interface{}{ "secretId": kb.StorageConfig.SecretID, diff --git a/internal/types/knowledgebase.go b/internal/types/knowledgebase.go index d26212bc..436d0a99 100644 --- a/internal/types/knowledgebase.go +++ b/internal/types/knowledgebase.go @@ -3,6 +3,7 @@ package types import ( "database/sql/driver" "encoding/json" + "strings" "time" "gorm.io/gorm" @@ -59,8 +60,10 @@ type KnowledgeBase struct { SummaryModelID string `yaml:"summary_model_id" json:"summary_model_id"` // VLM config VLMConfig VLMConfig `yaml:"vlm_config" json:"vlm_config" gorm:"type:json"` - // Storage config - StorageConfig StorageConfig `yaml:"storage_config" json:"storage_config" gorm:"column:cos_config;type:json"` + // Storage provider config (new): only stores provider selection; credentials from tenant StorageEngineConfig + StorageProviderConfig *StorageProviderConfig `yaml:"storage_provider_config" json:"storage_provider_config" gorm:"column:storage_provider_config;type:jsonb"` + // Deprecated: legacy COS config column. Kept for backward compatibility with old data. + StorageConfig StorageConfig `yaml:"-" json:"storage_config" gorm:"column:cos_config;type:json"` // Extract config ExtractConfig *ExtractConfig `yaml:"extract_config" json:"extract_config" gorm:"column:extract_config;type:json"` // FAQConfig stores FAQ specific configuration such as indexing strategy @@ -130,22 +133,37 @@ func (c ChunkingConfig) ResolveParserEngine(fileType string) string { return "" } -// COSConfig represents the COS configuration +// StorageProviderConfig stores the KB-level storage provider selection. +// Credentials are managed at the tenant level (StorageEngineConfig). +type StorageProviderConfig struct { + Provider string `yaml:"provider" json:"provider"` // "local", "minio", "cos", "tos" +} + +func (c StorageProviderConfig) Value() (driver.Value, error) { + return json.Marshal(c) +} + +func (c *StorageProviderConfig) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} + +// Deprecated: StorageConfig is the legacy COS configuration stored in the cos_config column. +// New code should use StorageProviderConfig. Kept for backward compatibility with old data. type StorageConfig struct { - // Secret ID - SecretID string `yaml:"secret_id" json:"secret_id"` - // Secret Key - SecretKey string `yaml:"secret_key" json:"secret_key"` - // Region - Region string `yaml:"region" json:"region"` - // Bucket Name + SecretID string `yaml:"secret_id" json:"secret_id"` + SecretKey string `yaml:"secret_key" json:"secret_key"` + Region string `yaml:"region" json:"region"` BucketName string `yaml:"bucket_name" json:"bucket_name"` - // App ID - AppID string `yaml:"app_id" json:"app_id"` - // Path Prefix + AppID string `yaml:"app_id" json:"app_id"` PathPrefix string `yaml:"path_prefix" json:"path_prefix"` - // Provider - Provider string `yaml:"provider" json:"provider"` + Provider string `yaml:"provider" json:"provider"` } func (c StorageConfig) Value() (driver.Value, error) { @@ -164,7 +182,7 @@ func (c *StorageConfig) Scan(value interface{}) error { } // UnmarshalJSON keeps backward compatibility for legacy clients that still send -// `cos_config`, while standardizing the API field as `storage_config`. +// `cos_config` or `storage_config`, while migrating to `storage_provider_config`. func (kb *KnowledgeBase) UnmarshalJSON(data []byte) error { type alias KnowledgeBase aux := struct { @@ -176,12 +194,57 @@ func (kb *KnowledgeBase) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(data, &aux); err != nil { return err } + // Backward compat: populate legacy StorageConfig from cos_config if aux.LegacyStorageConfig != nil && kb.StorageConfig == (StorageConfig{}) { kb.StorageConfig = *aux.LegacyStorageConfig } + // Auto-populate StorageProviderConfig from legacy StorageConfig if not set + if kb.StorageProviderConfig == nil && kb.StorageConfig.Provider != "" { + kb.StorageProviderConfig = &StorageProviderConfig{Provider: kb.StorageConfig.Provider} + } return nil } +// GetStorageProvider returns the effective storage provider for this KB. +// Priority: StorageProviderConfig (new) > StorageConfig.Provider (legacy cos_config). +func (kb *KnowledgeBase) GetStorageProvider() string { + if kb == nil { + return "" + } + if kb.StorageProviderConfig != nil { + p := strings.ToLower(strings.TrimSpace(kb.StorageProviderConfig.Provider)) + if p != "" && p != "__pending_env__" { + return p + } + } + return strings.ToLower(strings.TrimSpace(kb.StorageConfig.Provider)) +} + +// SetStorageProvider writes the provider to the new StorageProviderConfig field. +func (kb *KnowledgeBase) SetStorageProvider(provider string) { + if kb == nil { + return + } + kb.StorageProviderConfig = &StorageProviderConfig{Provider: provider} +} + +// InferStorageFromFilePath deduces the storage provider from a file path format. +// Used as a safety fallback when the KB's configured provider doesn't match the data. +func InferStorageFromFilePath(filePath string) string { + switch { + case strings.HasPrefix(filePath, "minio://"): + return "minio" + case strings.HasPrefix(filePath, "tos://"): + return "tos" + case strings.HasPrefix(filePath, "https://") && strings.Contains(filePath, ".cos."): + return "cos" + case strings.HasPrefix(filePath, "/"): + return "local" + default: + return "" + } +} + // ImageProcessingConfig represents the image processing configuration type ImageProcessingConfig struct { // Model ID diff --git a/migrations/sqlite/000000_init.up.sql b/migrations/sqlite/000000_init.up.sql index e9c7fced..101f7506 100644 --- a/migrations/sqlite/000000_init.up.sql +++ b/migrations/sqlite/000000_init.up.sql @@ -55,6 +55,7 @@ CREATE TABLE IF NOT EXISTS knowledge_bases ( embedding_model_id VARCHAR(64) NOT NULL, summary_model_id VARCHAR(64) NOT NULL, cos_config TEXT NOT NULL DEFAULT '{}', + storage_provider_config TEXT DEFAULT NULL, vlm_config TEXT NOT NULL DEFAULT '{}', extract_config TEXT NULL DEFAULT NULL, faq_config TEXT, diff --git a/migrations/versioned/000014_storage_provider_config.down.sql b/migrations/versioned/000014_storage_provider_config.down.sql new file mode 100644 index 00000000..2705e335 --- /dev/null +++ b/migrations/versioned/000014_storage_provider_config.down.sql @@ -0,0 +1,16 @@ +-- Rollback: copy provider back to cos_config if needed, then drop new column +UPDATE knowledge_bases +SET cos_config = jsonb_set( + COALESCE(cos_config, '{}'::jsonb), + '{provider}', + to_jsonb(storage_provider_config->>'provider') +) +WHERE storage_provider_config IS NOT NULL + AND storage_provider_config->>'provider' IS NOT NULL + AND storage_provider_config->>'provider' != '' + AND storage_provider_config->>'provider' != '__pending_env__' + AND (cos_config IS NULL + OR cos_config->>'provider' IS NULL + OR cos_config->>'provider' = ''); + +ALTER TABLE knowledge_bases DROP COLUMN IF EXISTS storage_provider_config; diff --git a/migrations/versioned/000014_storage_provider_config.up.sql b/migrations/versioned/000014_storage_provider_config.up.sql new file mode 100644 index 00000000..2bb3b3b2 --- /dev/null +++ b/migrations/versioned/000014_storage_provider_config.up.sql @@ -0,0 +1,31 @@ +-- Description: Add storage_provider_config column to knowledge_bases, migrating from legacy cos_config. +-- This separates the storage provider selection (KB-level) from storage credentials (tenant-level). +DO $$ BEGIN RAISE NOTICE '[Migration 000014] Adding storage_provider_config to knowledge_bases'; END $$; + +-- Step 1: Add new column +ALTER TABLE knowledge_bases ADD COLUMN IF NOT EXISTS storage_provider_config JSONB DEFAULT NULL; +COMMENT ON COLUMN knowledge_bases.storage_provider_config IS 'Storage provider config for this KB. Only stores provider name; credentials come from tenant StorageEngineConfig.'; + +-- Step 2: Migrate existing provider from legacy cos_config +UPDATE knowledge_bases +SET storage_provider_config = jsonb_build_object('provider', cos_config->>'provider') +WHERE cos_config IS NOT NULL + AND cos_config->>'provider' IS NOT NULL + AND cos_config->>'provider' != '' + AND (storage_provider_config IS NULL + OR storage_provider_config->>'provider' IS NULL + OR storage_provider_config->>'provider' = ''); + +-- Step 3: For KBs that have documents but no provider set, mark them with the +-- sentinel value so the application can fill in the actual STORAGE_TYPE on startup. +-- We use 'pending_migration' as a marker; the app replaces it with the real env value. +UPDATE knowledge_bases kb +SET storage_provider_config = '{"provider": "__pending_env__"}' +WHERE (kb.storage_provider_config IS NULL + OR kb.storage_provider_config->>'provider' IS NULL + OR kb.storage_provider_config->>'provider' = '') + AND EXISTS ( + SELECT 1 FROM knowledges k + WHERE k.knowledge_base_id = kb.id + AND k.deleted_at IS NULL + );