From 2c0c1b8bab8c66a4df019cdd5f9e735e148c1807 Mon Sep 17 00:00:00 2001 From: begoniezhao Date: Mon, 13 Apr 2026 17:23:16 +0800 Subject: [PATCH 1/4] feat: integrate ChatBot provider and docreader --- docker-compose.yml | 5 + frontend/src/api/model/index.ts | 42 ++ frontend/src/views/settings/ModelSettings.vue | 78 +++- internal/application/service/knowledge.go | 21 +- internal/application/service/model.go | 61 ++- internal/application/service/tenant.go | 45 +- internal/application/service/weknoracloud.go | 406 ++++++++++++++++++ internal/container/container.go | 93 +++- internal/handler/system.go | 76 +++- internal/handler/weknoracloud.go | 51 +++ internal/infrastructure/crypto/config.go | 224 ++++++++++ internal/infrastructure/crypto/crypto.go | 163 +++++++ .../docparser/engine_registry.go | 27 ++ .../docparser/weknoracloud_http_reader.go | 245 +++++++++++ internal/models/chat/chat.go | 22 +- internal/models/chat/chat_provider_spec.go | 69 +++ internal/models/chat/remote_api.go | 51 ++- internal/models/embedding/embedder.go | 15 +- internal/models/embedding/weknoracloud.go | 139 ++++++ internal/models/provider/provider.go | 1 + internal/models/provider/weknoracloud.go | 41 ++ internal/models/rerank/reranker.go | 17 +- internal/models/rerank/weknoracloud.go | 130 ++++++ internal/models/utils/signer.go | 93 ++++ internal/router/router.go | 8 + internal/types/interfaces/tenant.go | 2 + internal/types/interfaces/weknoracloud.go | 15 + internal/types/model.go | 3 + internal/types/tenant.go | 9 +- internal/types/weknoracloud.go | 25 ++ 30 files changed, 2112 insertions(+), 65 deletions(-) create mode 100644 internal/application/service/weknoracloud.go create mode 100644 internal/handler/weknoracloud.go create mode 100644 internal/infrastructure/crypto/config.go create mode 100644 internal/infrastructure/crypto/crypto.go create mode 100644 internal/infrastructure/docparser/weknoracloud_http_reader.go create mode 100644 internal/models/embedding/weknoracloud.go create mode 100644 internal/models/provider/weknoracloud.go create mode 100644 internal/models/rerank/weknoracloud.go create mode 100644 internal/models/utils/signer.go create mode 100644 internal/types/interfaces/weknoracloud.go create mode 100644 internal/types/weknoracloud.go diff --git a/docker-compose.yml b/docker-compose.yml index f10f339b..54549cbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,6 +116,11 @@ services: - SSRF_WHITELIST=${SSRF_WHITELIST:-} - CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5} - JWT_SECRET=${JWT_SECRET:-} + # Crypto: 主密钥和盐值,用于 AppSecret 等敏感字段的 AES-256 加密 + # 若不设置则自动生成并持久化到 data-files volume(/data/files/.crypto_state.json) + # 重启时自动从文件恢复,保证已加密数据可继续解密 + - CRYPTO_MASTER_KEY=${CRYPTO_MASTER_KEY:-} + - CRYPTO_SALT=${CRYPTO_SALT:-} # File size limit (in MB) - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} # Agent Skills Sandbox diff --git a/frontend/src/api/model/index.ts b/frontend/src/api/model/index.ts index d886a344..84313346 100644 --- a/frontend/src/api/model/index.ts +++ b/frontend/src/api/model/index.ts @@ -126,3 +126,45 @@ export function deleteModel(id: string): Promise { }); } +export interface InitializeWeKnoraCloudRequest { + app_id: string + app_secret: string +} + +export interface InitializeWeKnoraCloudResult { + models: Array<{ + name: string + type: string + action: 'created' | 'updated' + }> +} + +export function initializeWeKnoraCloud(data: InitializeWeKnoraCloudRequest): Promise { + return post('/api/v1/models/weknoracloud/initialize', data) as Promise +} + +export interface WeKnoraCloudStatusResult { + has_models: boolean + needs_reinit: boolean + reason?: string +} + +export function getWeKnoraCloudStatus(): Promise { + return new Promise((resolve, reject) => { + get('/api/v1/models/weknoracloud/status') + .then((response: any) => { + // status 接口直接返回对象,不包在 success/data 中 + if (response && typeof response.has_models === 'boolean') { + resolve(response) + } else if (response?.success && response?.data) { + resolve(response.data) + } else { + resolve({ has_models: false, needs_reinit: false }) + } + }) + .catch(() => { + resolve({ has_models: false, needs_reinit: false }) + }) + }) +} + diff --git a/frontend/src/views/settings/ModelSettings.vue b/frontend/src/views/settings/ModelSettings.vue index 94ae37d4..e735bb27 100644 --- a/frontend/src/views/settings/ModelSettings.vue +++ b/frontend/src/views/settings/ModelSettings.vue @@ -18,6 +18,46 @@ + +
+
+

WeKnoraCloud 厂商配置

+
+ +
+ +
+ WeKnoraCloud 凭证已失效
+ {{ weKnoraCloudReinitReason || '服务重启后加密密钥已变更,已保存的凭证无法解密。' }}请重新填写 APPID 和 APPSECRET 并点击"保存并初始化"以恢复服务。 +
+
+
+
+ + + + + + + + 保存并初始化 + +
+

+ 提交后将自动创建或更新三个模型(Chat / Embedding / Rerank)并设为各类型默认 +

+
+
+
@@ -295,7 +335,7 @@ import { MessagePlugin } from 'tdesign-vue-next' import { AddIcon } from 'tdesign-icons-vue-next' import { useI18n } from 'vue-i18n' import ModelEditorDialog from '@/components/ModelEditorDialog.vue' -import { listModels, createModel, updateModel as updateModelAPI, deleteModel as deleteModelAPI, type ModelConfig } from '@/api/model' +import { listModels, createModel, updateModel as updateModelAPI, deleteModel as deleteModelAPI, initializeWeKnoraCloud, getWeKnoraCloudStatus, type ModelConfig } from '@/api/model' const { t } = useI18n() @@ -304,6 +344,38 @@ const currentModelType = ref<'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr'>(' const editingModel = ref(null) const loading = ref(true) +// WeKnoraCloud 配置 +const weKnoraCloudForm = ref({ appId: '', appSecret: '' }) +const weKnoraCloudLoading = ref(false) +const weKnoraCloudNeedsReinit = ref(false) +const weKnoraCloudReinitReason = ref('') + +const handleWeKnoraCloudInit = async () => { + if (!weKnoraCloudForm.value.appId || !weKnoraCloudForm.value.appSecret) { + MessagePlugin.warning('请填写 APPID 和 APPSECRET') + return + } + weKnoraCloudLoading.value = true + try { + const result = await initializeWeKnoraCloud({ + app_id: weKnoraCloudForm.value.appId, + app_secret: weKnoraCloudForm.value.appSecret, + }) + const updatedCount = result.models.filter((m: any) => m.action === 'updated').length + const createdCount = result.models.filter((m: any) => m.action === 'created').length + MessagePlugin.success(`初始化成功:新建 ${createdCount} 个,更新 ${updatedCount} 个模型`) + weKnoraCloudForm.value.appId = '' + weKnoraCloudForm.value.appSecret = '' + weKnoraCloudNeedsReinit.value = false + weKnoraCloudReinitReason.value = '' + await loadModels() + } catch (err: any) { + MessagePlugin.error(err?.message || 'WeKnoraCloud 初始化失败') + } finally { + weKnoraCloudLoading.value = false + } +} + // 模型列表数据 const allModels = ref([]) @@ -361,6 +433,10 @@ const loadModels = async () => { // 直接获取所有模型,不分类型 const models = await listModels() allModels.value = models + // 检查 WeKnoraCloud 凭证状态 + const status = await getWeKnoraCloudStatus() + weKnoraCloudNeedsReinit.value = status.needs_reinit + weKnoraCloudReinitReason.value = status.reason || '' } catch (error: any) { console.error('加载模型列表失败:', error) MessagePlugin.error(error.message) diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 2eef0660..7851275c 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -65,6 +65,7 @@ type knowledgeService struct { repo interfaces.KnowledgeRepository kbService interfaces.KnowledgeBaseService tenantRepo interfaces.TenantRepository + tenantService interfaces.TenantService documentReader interfaces.DocumentReader chunkService interfaces.ChunkService chunkRepo interfaces.ChunkRepository @@ -92,6 +93,7 @@ func NewKnowledgeService( documentReader interfaces.DocumentReader, kbService interfaces.KnowledgeBaseService, tenantRepo interfaces.TenantRepository, + tenantService interfaces.TenantService, chunkService interfaces.ChunkService, chunkRepo interfaces.ChunkRepository, tagRepo interfaces.KnowledgeTagRepository, @@ -110,6 +112,7 @@ func NewKnowledgeService( repo: repo, kbService: kbService, tenantRepo: tenantRepo, + tenantService: tenantService, documentReader: documentReader, chunkService: chunkService, chunkRepo: chunkRepo, @@ -7994,7 +7997,7 @@ func (s *knowledgeService) convert( logger.Infof(ctx, "[convert] kb=%s fileType=%s isURL=%v engine=%q rules=%+v", kb.ID, fileType, isURL, parserEngine, kb.ChunkingConfig.ParserEngineRules) - var reader interfaces.DocReader = s.resolveDocReader(parserEngine, fileType, isURL, overrides) + var reader interfaces.DocReader = s.resolveDocReader(ctx, parserEngine, fileType, isURL, overrides) if reader == nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = "Document parsing service is not configured. Please use text/paragraph import or set DOCREADER_ADDR." @@ -8042,10 +8045,24 @@ func (s *knowledgeService) convert( // resolveDocReader returns the appropriate DocReader for the given engine. // Returns nil when the required service is unavailable. -func (s *knowledgeService) resolveDocReader(engine, fileType string, isURL bool, overrides map[string]string) interfaces.DocReader { +func (s *knowledgeService) resolveDocReader(ctx context.Context, engine, fileType string, isURL bool, overrides map[string]string) interfaces.DocReader { switch engine { case docparser.SimpleEngineName: return &docparser.SimpleFormatReader{} + case docparser.WeKnoraCloudEngineName: + addr := strings.TrimSpace(overrides["docreader_addr"]) + if !IsWeKnoraCloudDocReaderAddr(addr) { + return nil + } + creds := s.tenantService.GetDocreaderCredentials(ctx) + if creds == nil { + return nil + } + reader, err := docparser.NewWeKnoraCloudSignedDocumentReader(addr, creds.AppID, creds.APIKey) + if err != nil { + return nil + } + return reader case "mineru": return docparser.NewMinerUReader(overrides) case "mineru_cloud": diff --git a/internal/application/service/model.go b/internal/application/service/model.go index 80bdc2d2..86db67cf 100644 --- a/internal/application/service/model.go +++ b/internal/application/service/model.go @@ -4,11 +4,12 @@ import ( "context" "errors" + "github.com/Tencent/WeKnora/internal/infrastructure/crypto" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/asr" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/embedding" "github.com/Tencent/WeKnora/internal/models/rerank" - "github.com/Tencent/WeKnora/internal/models/asr" "github.com/Tencent/WeKnora/internal/models/utils/ollama" "github.com/Tencent/WeKnora/internal/models/vlm" "github.com/Tencent/WeKnora/internal/types" @@ -23,17 +24,35 @@ type modelService struct { repo interfaces.ModelRepository ollamaService *ollama.OllamaService pooler embedding.EmbedderPooler + cryptoSvc *crypto.CryptoService } // NewModelService creates a new model service instance -func NewModelService(repo interfaces.ModelRepository, ollamaService *ollama.OllamaService, pooler embedding.EmbedderPooler) interfaces.ModelService { +func NewModelService(repo interfaces.ModelRepository, + ollamaService *ollama.OllamaService, + pooler embedding.EmbedderPooler, + cryptoSvc *crypto.CryptoService, +) interfaces.ModelService { return &modelService{ repo: repo, ollamaService: ollamaService, pooler: pooler, + cryptoSvc: cryptoSvc, } } +// decryptAppSecret 解密 AppSecret(如果为空或 cryptoSvc 为空则原样返回) +func (s *modelService) decryptAppSecret(encrypted string) string { + if encrypted == "" { + return encrypted + } + plain, err := s.cryptoSvc.DecryptString(encrypted) + if err != nil { + return "" + } + return plain +} + // CreateModel creates a new model in the repository // For local models, it initiates an asynchronous download process // Remote models are immediately set to active status @@ -256,7 +275,9 @@ func (s *modelService) GetEmbeddingModel(ctx context.Context, modelId string) (e Dimensions: model.Parameters.EmbeddingParameters.Dimension, TruncatePromptTokens: model.Parameters.EmbeddingParameters.TruncatePromptTokens, Provider: model.Parameters.Provider, - Extra: stringMapToAnyMap(model.Parameters.ExtraConfig), + ExtraConfig: model.Parameters.ExtraConfig, + AppID: model.Parameters.AppID, + AppSecret: s.decryptAppSecret(model.Parameters.AppSecret), }, s.pooler, s.ollamaService) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ @@ -312,7 +333,9 @@ func (s *modelService) GetEmbeddingModelForTenant(ctx context.Context, modelId s Dimensions: model.Parameters.EmbeddingParameters.Dimension, TruncatePromptTokens: model.Parameters.EmbeddingParameters.TruncatePromptTokens, Provider: model.Parameters.Provider, - Extra: stringMapToAnyMap(model.Parameters.ExtraConfig), + ExtraConfig: model.Parameters.ExtraConfig, + AppID: model.Parameters.AppID, + AppSecret: s.decryptAppSecret(model.Parameters.AppSecret), }, s.pooler, s.ollamaService) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ @@ -343,11 +366,15 @@ func (s *modelService) GetRerankModel(ctx context.Context, modelId string) (rera // Initialize the reranker with model configuration reranker, err := rerank.NewReranker(&rerank.RerankerConfig{ - ModelID: model.ID, - APIKey: model.Parameters.APIKey, - BaseURL: model.Parameters.BaseURL, - ModelName: model.Name, - Source: model.Source, + ModelID: model.ID, + APIKey: model.Parameters.APIKey, + BaseURL: model.Parameters.BaseURL, + ModelName: model.Name, + Source: model.Source, + Provider: model.Parameters.Provider, + ExtraConfig: model.Parameters.ExtraConfig, + AppID: model.Parameters.AppID, + AppSecret: s.decryptAppSecret(model.Parameters.AppSecret), }) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ @@ -391,13 +418,15 @@ func (s *modelService) GetChatModel(ctx context.Context, modelId string) (chat.C // Initialize the chat model with model configuration chatModel, err := chat.NewChat(&chat.ChatConfig{ - ModelID: model.ID, - APIKey: model.Parameters.APIKey, - BaseURL: model.Parameters.BaseURL, - ModelName: model.Name, - Source: model.Source, - Provider: model.Parameters.Provider, - Extra: stringMapToAnyMap(model.Parameters.ExtraConfig), + ModelID: model.ID, + APIKey: model.Parameters.APIKey, + BaseURL: model.Parameters.BaseURL, + ModelName: model.Name, + Source: model.Source, + Provider: model.Parameters.Provider, + ExtraConfig: model.Parameters.ExtraConfig, + AppID: model.Parameters.AppID, + AppSecret: s.decryptAppSecret(model.Parameters.AppSecret), }, s.ollamaService) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ diff --git a/internal/application/service/tenant.go b/internal/application/service/tenant.go index f3ceeafe..faed4314 100644 --- a/internal/application/service/tenant.go +++ b/internal/application/service/tenant.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/Tencent/WeKnora/internal/infrastructure/crypto" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -33,12 +34,13 @@ type ListTenantsParams struct { // tenantService implements the TenantService interface type tenantService struct { - repo interfaces.TenantRepository // Repository for tenant data operations + repo interfaces.TenantRepository // Repository for tenant data operations + cryptoSvc *crypto.CryptoService } // NewTenantService creates a new tenant service instance -func NewTenantService(repo interfaces.TenantRepository) interfaces.TenantService { - return &tenantService{repo: repo} +func NewTenantService(repo interfaces.TenantRepository, cryptoSvc *crypto.CryptoService) interfaces.TenantService { + return &tenantService{repo: repo, cryptoSvc: cryptoSvc} } // CreateTenant creates a new tenant @@ -342,3 +344,40 @@ func (s *tenantService) GetTenantByIDForUser(ctx context.Context, tenantID uint6 return tenant, nil } + +func (s *tenantService) GetDocreaderCredentials(ctx context.Context) *types.DocreaderCredentials { + // Try to get tenant info directly first + if tenant, ok := types.TenantInfoFromContext(ctx); ok { + if tenant.ParserEngineConfig != nil { + appID := strings.TrimSpace(tenant.ParserEngineConfig.DocreaderAppID) + if appID == "" || tenant.ParserEngineConfig.DocreaderAPIKey == "" { + return nil + } + apiKey, err := s.cryptoSvc.DecryptString(tenant.ParserEngineConfig.DocreaderAPIKey) + if err != nil || apiKey == "" { + return nil + } + return &types.DocreaderCredentials{AppID: appID, APIKey: apiKey} + } + } + + // If no tenant info in context, try to get tenant ID and load tenant + tenantID, ok := types.TenantIDFromContext(ctx) + if !ok { + return nil + } + + // Load tenant from repo if we only have tenantID + tenant, err := s.repo.GetTenantByID(ctx, tenantID) + if err == nil && tenant != nil && tenant.ParserEngineConfig != nil { + appID := strings.TrimSpace(tenant.ParserEngineConfig.DocreaderAppID) + if appID != "" && tenant.ParserEngineConfig.DocreaderAPIKey != "" { + apiKey, err := s.cryptoSvc.DecryptString(tenant.ParserEngineConfig.DocreaderAPIKey) + if err == nil && apiKey != "" { + return &types.DocreaderCredentials{AppID: appID, APIKey: apiKey} + } + } + } + + return nil +} diff --git a/internal/application/service/weknoracloud.go b/internal/application/service/weknoracloud.go new file mode 100644 index 00000000..f63fe45f --- /dev/null +++ b/internal/application/service/weknoracloud.go @@ -0,0 +1,406 @@ +package service + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/infrastructure/crypto" + "github.com/Tencent/WeKnora/internal/models/provider" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" +) + +type weKnoraCloudService struct { + repo interfaces.ModelRepository + tenantRepo interfaces.TenantRepository + cryptoSvc *crypto.CryptoService +} + +// NewWeKnoraCloudService 构造 WeKnoraCloudService +func NewWeKnoraCloudService( + repo interfaces.ModelRepository, + tenantRepo interfaces.TenantRepository, + cryptoSvc *crypto.CryptoService, +) interfaces.WeKnoraCloudService { + return &weKnoraCloudService{ + repo: repo, + tenantRepo: tenantRepo, + cryptoSvc: cryptoSvc, + } +} + +func IsWeKnoraCloudDocReaderAddr(addr string) bool { + return strings.TrimSuffix(strings.TrimSpace(addr), "/") == strings.TrimRight(provider.WeKnoraCloudBaseURL, "/")+"/api/v1/doc/reader" +} + +// weKnoraCloudModelDefs 三个内置模型的静态定义 +var weKnoraCloudModelDefs = []struct { + name string + modelType types.ModelType + remoteModelName string +}{ + {name: "weknoracloud-chat", modelType: types.ModelTypeKnowledgeQA, remoteModelName: "chat"}, + {name: "weknoracloud-embedding", modelType: types.ModelTypeEmbedding, remoteModelName: "embedding"}, + {name: "weknoracloud-rerank", modelType: types.ModelTypeRerank, remoteModelName: "rerank"}, +} + +func (s *weKnoraCloudService) Initialize(ctx context.Context, appID, appSecret string) (*types.InitializeResult, error) { + if err := s.validateInitializeInput(appID, appSecret); err != nil { + return nil, err + } + + prepared, err := s.prepareInitialize(ctx, appID, appSecret) + if err != nil { + return nil, err + } + + result := &types.InitializeResult{} + + actions, err := s.installBuiltinModels(ctx, prepared.tenantID, appID, prepared.encryptedSecret) + if err != nil { + return nil, s.failInitializeAndRollback(ctx, prepared.tenantID, prepared.snapshot, err) + } + result.Models = actions + + if err := s.updateTenantDocReaderAddr(ctx, prepared.tenantID, appID, prepared.encryptedSecret); err != nil { + return nil, s.failInitializeAndRollback(ctx, prepared.tenantID, prepared.snapshot, err) + } + + return result, nil +} + +// validateInitializeInput validates that appID and appSecret are not empty. +func (s *weKnoraCloudService) validateInitializeInput(appID, appSecret string) error { + if appID == "" { + return fmt.Errorf("app_id is required") + } + if appSecret == "" { + return fmt.Errorf("app_secret is required") + } + return nil +} + +// weKnoraCloudInitPrepared holds the prepared state for initialize operation. +type weKnoraCloudInitPrepared struct { + tenantID uint64 + encryptedSecret string + snapshot *weKnoraCloudInitSnapshot +} + +// prepareInitialize performs pre-initialization checks: ping WeKnoraCloud, encrypt secret, and snapshot current state. +func (s *weKnoraCloudService) prepareInitialize(ctx context.Context, appID, appSecret string) (*weKnoraCloudInitPrepared, error) { + // Step 1: Check WeKnoraCloud service reachability + if err := s.pingWeKnoraCloud(ctx, appID, appSecret); err != nil { + return nil, fmt.Errorf("WeKnoraCloud service reachability check failed: %w", err) + } + + // Step 2: Encrypt appSecret + encryptedSecret, err := s.cryptoSvc.EncryptString(appSecret) + if err != nil { + return nil, fmt.Errorf("encrypt AppSecret failed: %w", err) + } + + // Step 3: Get tenantID and take snapshot + tenantID := types.MustTenantIDFromContext(ctx) + snapshot, err := s.snapshotInitializeState(ctx, tenantID) + if err != nil { + return nil, fmt.Errorf("snapshot initialize state failed: %w", err) + } + + return &weKnoraCloudInitPrepared{ + tenantID: tenantID, + encryptedSecret: encryptedSecret, + snapshot: snapshot, + }, nil +} + +// installBuiltinModels installs or updates the three builtin WeKnoraCloud models (chat, embedding, rerank). +// Returns a slice of ModelAction describing what was done. +func (s *weKnoraCloudService) installBuiltinModels( + ctx context.Context, + tenantID uint64, + appID, encryptedSecret string, +) ([]types.ModelAction, error) { + result := make([]types.ModelAction, 0, len(weKnoraCloudModelDefs)) + + for _, def := range weKnoraCloudModelDefs { + action, err := s.upsertModel(ctx, tenantID, def.name, def.modelType, def.remoteModelName, appID, encryptedSecret) + if err != nil { + return nil, fmt.Errorf("upsert model %s failed: %w", def.name, err) + } + result = append(result, types.ModelAction{ + Name: def.name, + Type: string(def.modelType), + Action: action, + }) + } + + return result, nil +} + +// weKnoraCloudInitSnapshot holds a snapshot of the state before initialization. +type weKnoraCloudInitSnapshot struct { + modelsByType map[types.ModelType][]*types.Model + tenant *types.Tenant +} + +func (s *weKnoraCloudService) snapshotInitializeState(ctx context.Context, tenantID uint64) (*weKnoraCloudInitSnapshot, error) { + snapshot := &weKnoraCloudInitSnapshot{modelsByType: make(map[types.ModelType][]*types.Model, len(weKnoraCloudModelDefs))} + for _, def := range weKnoraCloudModelDefs { + models, err := s.repo.List(ctx, tenantID, def.modelType, types.ModelSourceRemote) + if err != nil { + return nil, err + } + copies := make([]*types.Model, 0, len(models)) + for _, model := range models { + copies = append(copies, cloneWeKnoraCloudModel(model)) + } + snapshot.modelsByType[def.modelType] = copies + } + if s.tenantRepo != nil { + tenant, err := s.tenantRepo.GetTenantByID(ctx, tenantID) + if err != nil { + return nil, err + } + snapshot.tenant = cloneWeKnoraCloudTenant(tenant) + } + return snapshot, nil +} + +func (s *weKnoraCloudService) failInitializeAndRollback(ctx context.Context, tenantID uint64, snapshot *weKnoraCloudInitSnapshot, err error) error { + if rollbackErr := s.rollbackInitializeState(ctx, tenantID, snapshot); rollbackErr != nil { + return fmt.Errorf("%w;回滚失败:%v", err, rollbackErr) + } + return err +} + +func (s *weKnoraCloudService) rollbackInitializeState(ctx context.Context, tenantID uint64, snapshot *weKnoraCloudInitSnapshot) error { + if snapshot == nil { + return nil + } + var rollbackErrs []string + for _, def := range weKnoraCloudModelDefs { + currentModels, err := s.repo.List(ctx, tenantID, def.modelType, types.ModelSourceRemote) + if err != nil { + rollbackErrs = append(rollbackErrs, err.Error()) + continue + } + snapshotByID := map[string]*types.Model{} + for _, model := range snapshot.modelsByType[def.modelType] { + snapshotByID[model.ID] = cloneWeKnoraCloudModel(model) + } + for _, current := range currentModels { + if current == nil { + continue + } + if _, ok := snapshotByID[current.ID]; ok { + continue + } + if current.Parameters.Provider == string(provider.ProviderWeKnoraCloud) && isWeKnoraCloudBuiltinModelName(current.Name) { + if err := s.repo.Delete(ctx, tenantID, current.ID); err != nil { + rollbackErrs = append(rollbackErrs, err.Error()) + } + } + } + for _, model := range snapshot.modelsByType[def.modelType] { + if err := s.repo.Update(ctx, cloneWeKnoraCloudModel(model)); err != nil { + rollbackErrs = append(rollbackErrs, err.Error()) + } + } + } + if snapshot.tenant != nil && s.tenantRepo != nil { + if err := s.tenantRepo.UpdateTenant(ctx, cloneWeKnoraCloudTenant(snapshot.tenant)); err != nil { + rollbackErrs = append(rollbackErrs, err.Error()) + } + } + if len(rollbackErrs) > 0 { + return fmt.Errorf("%s", strings.Join(rollbackErrs, "; ")) + } + return nil +} + +func isWeKnoraCloudBuiltinModelName(name string) bool { + for _, def := range weKnoraCloudModelDefs { + if def.name == name { + return true + } + } + return false +} + +func cloneWeKnoraCloudModel(model *types.Model) *types.Model { + if model == nil { + return nil + } + clone := *model + if model.Parameters.ExtraConfig != nil { + clone.Parameters.ExtraConfig = make(map[string]string, len(model.Parameters.ExtraConfig)) + for k, v := range model.Parameters.ExtraConfig { + clone.Parameters.ExtraConfig[k] = v + } + } + return &clone +} + +func cloneWeKnoraCloudTenant(tenant *types.Tenant) *types.Tenant { + if tenant == nil { + return nil + } + clone := *tenant + if tenant.ParserEngineConfig != nil { + cfg := *tenant.ParserEngineConfig + clone.ParserEngineConfig = &cfg + } + return &clone +} + +// pingWeKnoraCloud 调用 WeKnoraCloud 服务的 GET /health 接口验证可达性。 +// 注意:这里仅检查服务可达,不校验 APPID / API Key 是否有效。 +func (s *weKnoraCloudService) pingWeKnoraCloud(ctx context.Context, appID, appSecret string) error { + baseURL := strings.TrimRight(provider.WeKnoraCloudBaseURL, "/") + healthURL := baseURL + "/api/v1/health" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthURL, nil) + if err != nil { + return fmt.Errorf("创建健康检查请求失败: %w", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("WeKnoraCloud 服务不可达 (%s): %w", healthURL, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("WeKnoraCloud 健康检查返回非 200 状态码: %d", resp.StatusCode) + } + return nil +} + +// CheckStatus 检查 WeKnoraCloud 凭证是否可正常解密 +func (s *weKnoraCloudService) CheckStatus(ctx context.Context) (*types.WeKnoraCloudStatusResult, error) { + tenantID := types.MustTenantIDFromContext(ctx) + + tenant, err := s.tenantRepo.GetTenantByID(ctx, tenantID) + if err != nil || tenant == nil { + return &types.WeKnoraCloudStatusResult{HasModels: false, NeedsReinit: false}, nil + } + + // Check if tenant has WeKnoraCloud credentials in parser config + if tenant.ParserEngineConfig == nil || tenant.ParserEngineConfig.DocreaderAppID == "" || tenant.ParserEngineConfig.DocreaderAPIKey == "" { + return &types.WeKnoraCloudStatusResult{ + HasModels: true, + NeedsReinit: true, + Reason: "WeKnoraCloud 凭证为空,请重新填写 APPID 和 API Key", + }, nil + } + + // Try to decrypt the API key + _, decErr := s.cryptoSvc.DecryptString(tenant.ParserEngineConfig.DocreaderAPIKey) + if decErr != nil { + return &types.WeKnoraCloudStatusResult{ + HasModels: true, + NeedsReinit: true, + Reason: "WeKnoraCloud 凭证解密失败(服务重启后加密密钥已变更),请重新填写 APPID 和 API Key", + }, nil + } + + return &types.WeKnoraCloudStatusResult{HasModels: true, NeedsReinit: false}, nil +} + +// updateTenantDocReaderAddr updates tenant parser config with WeKnoraCloud credentials and DocReaderAddr +func (s *weKnoraCloudService) updateTenantDocReaderAddr(ctx context.Context, tenantID uint64, appID, encryptedSecret string) error { + if s.tenantRepo == nil { + return fmt.Errorf("tenant repository is required") + } + + tenant, err := s.tenantRepo.GetTenantByID(ctx, tenantID) + if err != nil { + return err + } + if tenant.ParserEngineConfig == nil { + tenant.ParserEngineConfig = &types.ParserEngineConfig{} + } + tenant.ParserEngineConfig.DocReaderAddr = strings.TrimRight(provider.WeKnoraCloudBaseURL, "/") + "/api/v1/doc/reader" + tenant.ParserEngineConfig.DocreaderAppID = appID + tenant.ParserEngineConfig.DocreaderAPIKey = encryptedSecret + return s.tenantRepo.UpdateTenant(ctx, tenant) +} + +func (s *weKnoraCloudService) upsertModel( + ctx context.Context, + tenantID uint64, + name string, + modelType types.ModelType, + remoteModelName string, + appID, encryptedSecret string, +) (string, error) { + // 列出该类型的远程模型,过滤出 weknoracloud provider + models, err := s.repo.List(ctx, tenantID, modelType, types.ModelSourceRemote) + if err != nil { + return "", err + } + + var existing *types.Model + for _, m := range models { + if m.Name == name && m.Parameters.Provider == string(provider.ProviderWeKnoraCloud) { + existing = m + break + } + } + + // 先清除同类型其他模型的默认标记 + excludeID := "" + if existing != nil { + excludeID = existing.ID + } + if err := s.repo.ClearDefaultByType(ctx, uint(tenantID), modelType, excludeID); err != nil { + return "", err + } + + if existing != nil { + // 更新凭证和默认标记 + existing.Parameters.BaseURL = provider.WeKnoraCloudBaseURL + existing.Parameters.AppID = appID + existing.Parameters.AppSecret = encryptedSecret + if existing.Parameters.ExtraConfig == nil { + existing.Parameters.ExtraConfig = map[string]string{} + } + existing.Parameters.ExtraConfig["remote_model_name"] = remoteModelName + existing.IsDefault = true + if err := s.repo.Update(ctx, existing); err != nil { + return "", err + } + return "updated", nil + } + + // 创建新模型 + newModel := &types.Model{ + ID: uuid.New().String(), + TenantID: tenantID, + Name: name, + Type: modelType, + Source: types.ModelSourceRemote, + IsBuiltin: true, + IsDefault: true, + Status: types.ModelStatusActive, + Parameters: types.ModelParameters{ + Provider: string(provider.ProviderWeKnoraCloud), + BaseURL: provider.WeKnoraCloudBaseURL, + AppID: appID, + AppSecret: encryptedSecret, + ExtraConfig: map[string]string{ + "remote_model_name": remoteModelName, + }, + }, + } + if err := s.repo.Create(ctx, newModel); err != nil { + return "", err + } + return "created", nil +} diff --git a/internal/container/container.go b/internal/container/container.go index 56ba1059..1edac5a2 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -6,6 +6,7 @@ package container import ( "context" "database/sql" + "encoding/base64" "encoding/json" "fmt" "net/url" @@ -63,6 +64,7 @@ import ( "github.com/Tencent/WeKnora/internal/im/telegram" "github.com/Tencent/WeKnora/internal/im/wechat" "github.com/Tencent/WeKnora/internal/im/wecom" + "github.com/Tencent/WeKnora/internal/infrastructure/crypto" "github.com/Tencent/WeKnora/internal/infrastructure/docparser" infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search" "github.com/Tencent/WeKnora/internal/logger" @@ -98,6 +100,7 @@ func BuildContainer(container *dig.Container) *dig.Container { // Core infrastructure configuration logger.Debugf(ctx, "[Container] Registering core infrastructure...") must(container.Provide(config.LoadConfig)) + must(container.Provide(initCryptoService)) must(container.Provide(initTracer)) must(container.Provide(initDatabase)) must(container.Provide(initFileService)) @@ -169,6 +172,7 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(service.NewDatasetService)) must(container.Provide(service.NewEvaluationService)) must(container.Provide(service.NewUserService)) + must(container.Provide(service.NewWeKnoraCloudService)) // Extract services - register individual extracters with names must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtractor"))) @@ -269,6 +273,7 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(imPkg.NewService)) must(container.Invoke(registerIMAdapterFactories)) must(container.Provide(handler.NewIMHandler)) + must(container.Provide(handler.NewWeKnoraCloudHandler)) logger.Debugf(ctx, "[Container] HTTP handlers registered") // Router configuration @@ -306,6 +311,92 @@ func initTracer() (*tracing.Tracer, error) { return tracing.InitTracer() } +// cryptoStateFile 是持久化 crypto 状态的文件路径,存储在 data-files volume 中 +// 保证重启后能恢复相同的 masterKey 和 salt,避免已加密数据无法解密 +const cryptoStateFile = "/data/files/.crypto_state.json" + +// cryptoState 持久化存储的 crypto 配置 +type cryptoState struct { + MasterKey string `json:"master_key"` + Salt string `json:"salt"` // base64 编码 +} + +func initCryptoService() (*crypto.CryptoService, error) { + cfg := crypto.LoadConfigFromEnv() + + // 如果环境变量同时提供了 MasterKey 和 Salt,直接使用(优先级最高) + if cfg.MasterKey != "" && cfg.Salt != "" { + return crypto.NewCryptoServiceFromConfig(cfg) + } + + // 尝试从持久化文件恢复状态 + if state, err := loadCryptoState(cryptoStateFile); err == nil { + // 文件读取成功,环境变量中未覆盖的字段从文件补充 + if cfg.MasterKey == "" { + cfg.MasterKey = state.MasterKey + } + if cfg.Salt == "" { + cfg.Salt = state.Salt + } + return crypto.NewCryptoServiceFromConfig(cfg) + } + + // 文件不存在或读取失败,使用/生成配置后写入文件 + if cfg.MasterKey == "" { + cfg.MasterKey = "weknora-default-key" + } + + // 调用 NewCryptoServiceFromConfig 会在 cfg.Salt 为空时自动生成随机 salt + svc, err := crypto.NewCryptoServiceFromConfig(cfg) + if err != nil { + return nil, err + } + + // 将实际使用的 masterKey 和 salt 持久化,供重启恢复 + saltB64 := base64.StdEncoding.EncodeToString(svc.GetSalt()) + state := &cryptoState{ + MasterKey: cfg.MasterKey, + Salt: saltB64, + } + if saveErr := saveCryptoState(cryptoStateFile, state); saveErr != nil { + // 持久化失败仅记录警告,不阻止服务启动 + logger.Warnf(context.Background(), "[CryptoService] 无法持久化 crypto 状态到 %s: %v (重启后已加密数据将无法解密)", cryptoStateFile, saveErr) + } else { + logger.Infof(context.Background(), "[CryptoService] crypto 状态已持久化到 %s", cryptoStateFile) + } + + return svc, nil +} + +// loadCryptoState 从文件加载 crypto 状态 +func loadCryptoState(path string) (*cryptoState, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var state cryptoState + if err := json.Unmarshal(data, &state); err != nil { + return nil, fmt.Errorf("parse crypto state: %w", err) + } + if state.MasterKey == "" || state.Salt == "" { + return nil, fmt.Errorf("invalid crypto state: missing fields") + } + return &state, nil +} + +// saveCryptoState 将 crypto 状态写入文件 +func saveCryptoState(path string, state *cryptoState) error { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("mkdir: %w", err) + } + data, err := json.Marshal(state) + if err != nil { + return fmt.Errorf("marshal: %w", err) + } + // 权限 0o600:仅 owner 可读写 + return os.WriteFile(path, data, 0o600) +} + func initRedisClient() (*redis.Client, error) { redisAddr := os.Getenv("REDIS_ADDR") if redisAddr == "" { @@ -407,7 +498,7 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { dbPath = "./data/weknora.db" } if dir := filepath.Dir(dbPath); dir != "." && dir != "" { - if err := os.MkdirAll(dir, 0755); err != nil { + if err := os.MkdirAll(dir, 0o755); err != nil { return nil, fmt.Errorf("failed to create SQLite data directory %s: %w", dir, err) } } diff --git a/internal/handler/system.go b/internal/handler/system.go index bd7d5540..e1dbd323 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" + "github.com/Tencent/WeKnora/internal/application/service" "github.com/Tencent/WeKnora/internal/application/service/file" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/database" @@ -28,14 +29,20 @@ type SystemHandler struct { cfg *config.Config neo4jDriver neo4j.Driver documentReader interfaces.DocumentReader + tenantSvc interfaces.TenantService } // NewSystemHandler creates a new system handler -func NewSystemHandler(cfg *config.Config, neo4jDriver neo4j.Driver, documentReader interfaces.DocumentReader) *SystemHandler { +func NewSystemHandler(cfg *config.Config, + neo4jDriver neo4j.Driver, + documentReader interfaces.DocumentReader, + tenantSvc interfaces.TenantService, +) *SystemHandler { return &SystemHandler{ cfg: cfg, neo4jDriver: neo4jDriver, documentReader: documentReader, + tenantSvc: tenantSvc, } } @@ -133,9 +140,6 @@ func (h *SystemHandler) getDocReaderConnInfo() (addr, transport string) { // @Success 200 {object} map[string]interface{} "解析引擎列表" // @Router /system/parser-engines [get] func (h *SystemHandler) ListParserEngines(c *gin.Context) { - docreaderAddr, docreaderTransport := h.getDocReaderConnInfo() - connected := h.documentReader != nil && h.documentReader.IsConnected() - var overrides map[string]string if v, exists := c.Get(types.TenantInfoContextKey.String()); exists { if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.ParserEngineConfig != nil { @@ -143,7 +147,9 @@ func (h *SystemHandler) ListParserEngines(c *gin.Context) { } } - remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides) + reader, docreaderAddr, docreaderTransport := h.resolveDocReader(c.Request.Context(), overrides) + connected := reader != nil && reader.IsConnected() + remoteEngines := h.fetchRemoteEngines(c.Request.Context(), reader, overrides) engines := docparser.ListAllEngines(connected, overrides, remoteEngines) c.JSON(200, gin.H{"code": 0, "msg": "success", "data": engines, "docreader_addr": docreaderAddr, "docreader_transport": docreaderTransport, "connected": connected}) } @@ -194,7 +200,7 @@ func (h *SystemHandler) ReconnectDocReader(c *gin.Context) { overrides = tenant.ParserEngineConfig.ToOverridesMap() } } - remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides) + remoteEngines := h.fetchRemoteEngines(c.Request.Context(), h.documentReader, overrides) engines := docparser.ListAllEngines(true, overrides, remoteEngines) _, docreaderTransport := h.getDocReaderConnInfo() @@ -211,28 +217,46 @@ func (h *SystemHandler) ReconnectDocReader(c *gin.Context) { // @Success 200 // @Router /system/parser-engines/check [post] func (h *SystemHandler) CheckParserEngines(c *gin.Context) { - docreaderAddr, docreaderTransport := h.getDocReaderConnInfo() - connected := h.documentReader != nil && h.documentReader.IsConnected() - var body types.ParserEngineConfig if err := c.ShouldBindJSON(&body); err != nil { c.JSON(400, gin.H{"code": 1, "msg": "请求体格式错误"}) return } overrides := body.ToOverridesMap() - remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides) + reader, docreaderAddr, docreaderTransport := h.resolveDocReader(c.Request.Context(), overrides) + connected := reader != nil && reader.IsConnected() + remoteEngines := h.fetchRemoteEngines(c.Request.Context(), reader, overrides) engines := docparser.ListAllEngines(connected, overrides, remoteEngines) c.JSON(200, gin.H{"code": 0, "msg": "success", "data": engines, "docreader_addr": docreaderAddr, "docreader_transport": docreaderTransport, "connected": connected}) } +func (h *SystemHandler) resolveDocReader(ctx context.Context, overrides map[string]string) (interfaces.DocumentReader, string, string) { + if len(overrides) > 0 { + if addr := strings.TrimSpace(overrides["docreader_addr"]); addr != "" && service.IsWeKnoraCloudDocReaderAddr(addr) { + reader := h.ResolveDocumentReader(ctx, addr) + return reader, addr, transportFromDocReaderAddr(addr) + } + } + + addr, transport := h.getDocReaderConnInfo() + return h.documentReader, addr, transport +} + +func transportFromDocReaderAddr(addr string) string { + if strings.HasPrefix(strings.ToLower(strings.TrimSpace(addr)), "https://") { + return "https" + } + return "http" +} + // fetchRemoteEngines queries the remote docreader for its engine list. // Returns nil on any error (e.g. not connected), letting the caller // fall back to Go's static registry only. -func (h *SystemHandler) fetchRemoteEngines(ctx context.Context, overrides map[string]string) []types.ParserEngineInfo { - if h.documentReader == nil || !h.documentReader.IsConnected() { +func (h *SystemHandler) fetchRemoteEngines(ctx context.Context, reader interfaces.DocumentReader, overrides map[string]string) []types.ParserEngineInfo { + if reader == nil || !reader.IsConnected() { return nil } - engines, err := h.documentReader.ListEngines(ctx, overrides) + engines, err := reader.ListEngines(ctx, overrides) if err != nil { logger.Warnf(ctx, "Failed to fetch remote engines from docreader: %v", err) return nil @@ -768,7 +792,7 @@ func (h *SystemHandler) checkMinio(c *gin.Context, ctx context.Context, cfg *typ if cfg.Mode == "remote" { if blocked, reason := isBlockedStorageEndpoint(endpoint); blocked { - logger.Warnf(ctx, "Storage check: MinIO endpoint blocked by SSRF protection", "endpoint", endpoint) + logger.Warnf(ctx, "Storage check: MinIO endpoint blocked by SSRF protection endpoint=%s", endpoint) c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: reason}}) return } @@ -935,3 +959,27 @@ func (h *SystemHandler) checkS3(c *gin.Context, ctx context.Context, cfg *types. } c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}}) } + +func (h *SystemHandler) ResolveDocumentReader(ctx context.Context, addr string) interfaces.DocumentReader { + if addr == "" { + return h.documentReader + } + + if service.IsWeKnoraCloudDocReaderAddr(addr) { + creds := h.tenantSvc.GetDocreaderCredentials(ctx) + if creds == nil { + return nil + } + reader, err := docparser.NewWeKnoraCloudSignedDocumentReader(addr, creds.AppID, creds.APIKey) + if err != nil { + return nil + } + return reader + } + + reader, err := docparser.NewHTTPDocumentReader(addr) + if err != nil || reader == nil { + return reader + } + return reader +} diff --git a/internal/handler/weknoracloud.go b/internal/handler/weknoracloud.go new file mode 100644 index 00000000..fdd918e5 --- /dev/null +++ b/internal/handler/weknoracloud.go @@ -0,0 +1,51 @@ +package handler + +import ( + "net/http" + + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" +) + +// WeKnoraCloudHandler 处理 WeKnoraCloud 初始化请求 +type WeKnoraCloudHandler struct { + svc interfaces.WeKnoraCloudService +} + +// NewWeKnoraCloudHandler 构造函数 +func NewWeKnoraCloudHandler(svc interfaces.WeKnoraCloudService) *WeKnoraCloudHandler { + return &WeKnoraCloudHandler{svc: svc} +} + +type initializeWeKnoraCloudRequest struct { + AppID string `json:"app_id" binding:"required"` + AppSecret string `json:"app_secret" binding:"required"` +} + +// Initialize POST /api/v1/models/weknoracloud/initialize +func (h *WeKnoraCloudHandler) Initialize(c *gin.Context) { + var req initializeWeKnoraCloudRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + result, err := h.svc.Initialize(c.Request.Context(), req.AppID, req.AppSecret) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, result) +} + +// Status GET /api/v1/models/weknoracloud/status +// 检查当前租户的 WeKnoraCloud 凭证是否完好,如需重新初始化则返回 needs_reinit=true +func (h *WeKnoraCloudHandler) Status(c *gin.Context) { + result, err := h.svc.CheckStatus(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, result) +} diff --git a/internal/infrastructure/crypto/config.go b/internal/infrastructure/crypto/config.go new file mode 100644 index 00000000..f9db8c7e --- /dev/null +++ b/internal/infrastructure/crypto/config.go @@ -0,0 +1,224 @@ +package crypto + +import ( + "encoding/base64" + "fmt" + "os" + "strconv" +) + +// Config 加密服务配置结构 +type Config struct { + // MasterKey 主密钥,用于派生加密密钥 + // 可以从环境变量或配置文件中读取 + MasterKey string `json:"master_key" yaml:"master_key" env:"CRYPTO_MASTER_KEY"` + + // Salt 盐值,用于密钥派生 + // 如果为空,将自动生成随机盐值 + Salt string `json:"salt" yaml:"salt" env:"CRYPTO_SALT"` + + // SaltLength 盐值长度(当Salt为空时使用) + SaltLength int `json:"salt_length" yaml:"salt_length" env:"CRYPTO_SALT_LENGTH" default:"16"` + + // KeyDerivationIterations PBKDF2迭代次数 + KeyDerivationIterations int `json:"key_derivation_iterations" yaml:"key_derivation_iterations" env:"CRYPTO_ITERATIONS" default:"10000"` + + // KeyLength 派生密钥长度 + KeyLength int `json:"key_length" yaml:"key_length" env:"CRYPTO_KEY_LENGTH" default:"32"` +} + +// DefaultConfig 返回默认配置 +func DefaultConfig() *Config { + return &Config{ + MasterKey: "", + Salt: "", + SaltLength: 16, + KeyDerivationIterations: 10000, + KeyLength: 32, + } +} + +// LoadConfigFromEnv 从环境变量加载配置 +func LoadConfigFromEnv() *Config { + config := DefaultConfig() + + // 从环境变量读取主密钥 + if masterKey := os.Getenv("CRYPTO_MASTER_KEY"); masterKey != "" { + config.MasterKey = masterKey + } + + // 从环境变量读取盐值 + if salt := os.Getenv("CRYPTO_SALT"); salt != "" { + config.Salt = salt + } + + // 从环境变量读取盐值长度 + if saltLengthStr := os.Getenv("CRYPTO_SALT_LENGTH"); saltLengthStr != "" { + if saltLength, err := strconv.Atoi(saltLengthStr); err == nil && saltLength >= 8 { + config.SaltLength = saltLength + } + } + + // 从环境变量读取迭代次数 + if iterationsStr := os.Getenv("CRYPTO_ITERATIONS"); iterationsStr != "" { + if iterations, err := strconv.Atoi(iterationsStr); err == nil && iterations > 0 { + config.KeyDerivationIterations = iterations + } + } + + // 从环境变量读取密钥长度 + if keyLengthStr := os.Getenv("CRYPTO_KEY_LENGTH"); keyLengthStr != "" { + if keyLength, err := strconv.Atoi(keyLengthStr); err == nil && keyLength >= 16 { + config.KeyLength = keyLength + } + } + + return config +} + +// Validate 验证配置是否有效 +func (c *Config) Validate() error { + if c.MasterKey == "" { + return fmt.Errorf("master key cannot be empty") + } + + if c.SaltLength < 8 { + return fmt.Errorf("salt length must be at least 8 bytes") + } + + if c.KeyDerivationIterations < 1000 { + return fmt.Errorf("key derivation iterations must be at least 1000") + } + + if c.KeyLength < 16 { + return fmt.Errorf("key length must be at least 16 bytes") + } + + return nil +} + +// GetSaltBytes 获取盐值的字节数组形式 +func (c *Config) GetSaltBytes() ([]byte, error) { + if c.Salt != "" { + // 如果配置了盐值,解码Base64格式 + saltBytes, err := base64.StdEncoding.DecodeString(c.Salt) + if err != nil { + return nil, fmt.Errorf("failed to decode salt from base64: %w", err) + } + return saltBytes, nil + } + + // 如果没有配置盐值,生成随机盐值 + salt, err := GenerateRandomSalt(c.SaltLength) + if err != nil { + return nil, fmt.Errorf("failed to generate random salt: %w", err) + } + + return salt, nil +} + +// NewCryptoServiceFromConfig 从配置创建加密服务 +func NewCryptoServiceFromConfig(config *Config) (*CryptoService, error) { + if err := config.Validate(); err != nil { + return nil, fmt.Errorf("invalid config: %w", err) + } + + saltBytes, err := config.GetSaltBytes() + if err != nil { + return nil, fmt.Errorf("failed to get salt bytes: %w", err) + } + + return NewCryptoService(config.MasterKey, saltBytes) +} + +// ExampleConfigUsage 展示配置使用示例 +func ExampleConfigUsage() { + fmt.Println("=== 配置使用示例 ===") + + // 方法1: 手动创建配置 + manualConfig := &Config{ + MasterKey: "my-secret-master-key", + Salt: "", // 自动生成盐值 + SaltLength: 16, + KeyDerivationIterations: 10000, + KeyLength: 32, + } + + cryptoService1, err := NewCryptoServiceFromConfig(manualConfig) + if err != nil { + fmt.Printf("手动配置创建失败: %v\n", err) + } else { + fmt.Println("手动配置创建成功") + _ = cryptoService1 + } + + // 方法2: 从环境变量加载配置 + // 首先设置环境变量 + os.Setenv("CRYPTO_MASTER_KEY", "env-master-key") + os.Setenv("CRYPTO_SALT", base64.StdEncoding.EncodeToString([]byte("env-salt-123456"))) + os.Setenv("CRYPTO_SALT_LENGTH", "16") + os.Setenv("CRYPTO_ITERATIONS", "10000") + os.Setenv("CRYPTO_KEY_LENGTH", "32") + + envConfig := LoadConfigFromEnv() + cryptoService2, err := NewCryptoServiceFromConfig(envConfig) + if err != nil { + fmt.Printf("环境变量配置创建失败: %v\n", err) + } else { + fmt.Println("环境变量配置创建成功") + _ = cryptoService2 + } + + // 清理环境变量 + os.Unsetenv("CRYPTO_MASTER_KEY") + os.Unsetenv("CRYPTO_SALT") + os.Unsetenv("CRYPTO_SALT_LENGTH") + os.Unsetenv("CRYPTO_ITERATIONS") + os.Unsetenv("CRYPTO_KEY_LENGTH") +} + +// GenerateConfigExample 生成配置示例 +func GenerateConfigExample() { + fmt.Println("\n=== 配置生成示例 ===") + + // 生成随机主密钥 + masterKey, err := GenerateRandomKey(32) + if err != nil { + fmt.Printf("生成主密钥失败: %v\n", err) + return + } + + // 生成随机盐值 + salt, err := GenerateRandomSalt(16) + if err != nil { + fmt.Printf("生成盐值失败: %v\n", err) + return + } + + saltBase64 := base64.StdEncoding.EncodeToString(salt) + + fmt.Println("生成的配置示例:") + fmt.Printf("CRYPTO_MASTER_KEY=%s\n", masterKey) + fmt.Printf("CRYPTO_SALT=%s\n", saltBase64) + fmt.Printf("CRYPTO_SALT_LENGTH=16\n") + fmt.Printf("CRYPTO_ITERATIONS=10000\n") + fmt.Printf("CRYPTO_KEY_LENGTH=32\n") + + fmt.Println("\nYAML格式配置示例:") + fmt.Printf(`master_key: %s +salt: %s +salt_length: 16 +key_derivation_iterations: 10000 +key_length: 32 +`, masterKey, saltBase64) + + fmt.Println("\nJSON格式配置示例:") + fmt.Printf(`{ + "master_key": "%s", + "salt": "%s", + "salt_length": 16, + "key_derivation_iterations": 10000, + "key_length": 32 +} +`, masterKey, saltBase64) +} diff --git a/internal/infrastructure/crypto/crypto.go b/internal/infrastructure/crypto/crypto.go new file mode 100644 index 00000000..d2a47465 --- /dev/null +++ b/internal/infrastructure/crypto/crypto.go @@ -0,0 +1,163 @@ +package crypto + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + + "golang.org/x/crypto/pbkdf2" +) + +// CryptoService 提供密钥加密解密服务 +type CryptoService struct { + masterKey []byte + salt []byte +} + +// NewCryptoService 创建新的加密服务实例 +// masterKey: 主密钥,用于派生加密密钥 +// salt: 盐值,用于密钥派生,如果为空则生成随机盐 +func NewCryptoService(masterKey string, salt []byte) (*CryptoService, error) { + if masterKey == "" { + return nil, errors.New("master key cannot be empty") + } + + // 如果未提供盐值,生成随机盐 + if salt == nil || len(salt) == 0 { + salt = make([]byte, 16) + if _, err := rand.Read(salt); err != nil { + return nil, fmt.Errorf("failed to generate salt: %w", err) + } + } + + return &CryptoService{ + masterKey: []byte(masterKey), + salt: salt, + }, nil +} + +// deriveKey 使用PBKDF2派生加密密钥 +func (cs *CryptoService) deriveKey() []byte { + return pbkdf2.Key(cs.masterKey, cs.salt, 10000, 32, sha256.New) +} + +// Encrypt 加密数据 +func (cs *CryptoService) Encrypt(plaintext []byte) (string, error) { + if len(plaintext) == 0 { + return "", errors.New("plaintext cannot be empty") + } + + key := cs.deriveKey() + + // 创建AES块密码 + block, err := aes.NewCipher(key) + if err != nil { + return "", fmt.Errorf("failed to create cipher: %w", err) + } + + // 生成随机IV + iv := make([]byte, aes.BlockSize) + if _, err := rand.Read(iv); err != nil { + return "", fmt.Errorf("failed to generate IV: %w", err) + } + + // 使用CTR模式加密 + stream := cipher.NewCTR(block, iv) + ciphertext := make([]byte, len(plaintext)) + stream.XORKeyStream(ciphertext, plaintext) + + // 组合IV和密文 + result := make([]byte, len(iv)+len(ciphertext)) + copy(result[:aes.BlockSize], iv) + copy(result[aes.BlockSize:], ciphertext) + + // 返回Base64编码的结果 + return base64.StdEncoding.EncodeToString(result), nil +} + +// Decrypt 解密数据 +func (cs *CryptoService) Decrypt(encryptedData string) ([]byte, error) { + if encryptedData == "" { + return nil, errors.New("encrypted data cannot be empty") + } + + // 解码Base64数据 + data, err := base64.StdEncoding.DecodeString(encryptedData) + if err != nil { + return nil, fmt.Errorf("failed to decode base64: %w", err) + } + + if len(data) < aes.BlockSize { + return nil, errors.New("encrypted data too short") + } + + key := cs.deriveKey() + + // 创建AES块密码 + block, err := aes.NewCipher(key) + if err != nil { + return nil, fmt.Errorf("failed to create cipher: %w", err) + } + + // 提取IV和密文 + iv := data[:aes.BlockSize] + ciphertext := data[aes.BlockSize:] + + // 使用CTR模式解密 + stream := cipher.NewCTR(block, iv) + plaintext := make([]byte, len(ciphertext)) + stream.XORKeyStream(plaintext, ciphertext) + + return plaintext, nil +} + +// EncryptString 加密字符串 +func (cs *CryptoService) EncryptString(plaintext string) (string, error) { + return cs.Encrypt([]byte(plaintext)) +} + +// DecryptString 解密字符串 +func (cs *CryptoService) DecryptString(encryptedData string) (string, error) { + plaintext, err := cs.Decrypt(encryptedData) + if err != nil { + return "", err + } + return string(plaintext), nil +} + +// GetSalt 获取当前使用的盐值 +func (cs *CryptoService) GetSalt() []byte { + return cs.salt +} + +// GenerateRandomKey 生成随机密钥 +func GenerateRandomKey(length int) (string, error) { + if length < 16 { + return "", errors.New("key length must be at least 16 bytes") + } + + key := make([]byte, length) + if _, err := rand.Read(key); err != nil { + return "", fmt.Errorf("failed to generate random key: %w", err) + } + + return base64.StdEncoding.EncodeToString(key), nil +} + +// GenerateRandomSalt 生成随机盐值 +func GenerateRandomSalt(length int) ([]byte, error) { + if length < 8 { + return nil, errors.New("salt length must be at least 8 bytes") + } + + salt := make([]byte, length) + if _, err := rand.Read(salt); err != nil { + return nil, fmt.Errorf("failed to generate random salt: %w", err) + } + + return salt, nil +} diff --git a/internal/infrastructure/docparser/engine_registry.go b/internal/infrastructure/docparser/engine_registry.go index 294ccdd6..09af532d 100644 --- a/internal/infrastructure/docparser/engine_registry.go +++ b/internal/infrastructure/docparser/engine_registry.go @@ -3,6 +3,7 @@ package docparser import ( "strings" + "github.com/Tencent/WeKnora/internal/models/provider" "github.com/Tencent/WeKnora/internal/types" ) @@ -27,6 +28,7 @@ func RegisterEngine(e EngineRegistration) { func init() { RegisterEngine(&builtinEngine{}) RegisterEngine(&simpleEngine{}) + RegisterEngine(&weKnoraCloudEngine{}) RegisterEngine(&mineruEngine{}) RegisterEngine(&mineruCloudEngine{}) } @@ -54,6 +56,9 @@ func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]str // SimpleEngineName is the engine name for Go-native simple format handling. const SimpleEngineName = "simple" +// WeKnoraCloudEngineName is the engine name for WeKnoraCloud-backed document parsing. +const WeKnoraCloudEngineName = "weknoracloud" + // --------------------------------------------------------------------------- // simple — Go handles md/txt/csv natively, no external service needed. // Distinct from docreader's "builtin" which uses Python libraries for @@ -73,6 +78,24 @@ func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string return true, "" } +// --------------------------------------------------------------------------- +// weknoracloud — Tenant-scoped WeKnoraCloud docreader with signed requests. +// --------------------------------------------------------------------------- + +type weKnoraCloudEngine struct{} + +func (e *weKnoraCloudEngine) Name() string { return WeKnoraCloudEngineName } +func (e *weKnoraCloudEngine) Description() string { return "WeKnoraCloud document reader" } +func (e *weKnoraCloudEngine) FileTypes(_ bool) []string { + return weKnoraCloudSupportedFileTypes() +} +func (e *weKnoraCloudEngine) CheckAvailable(docreaderConnected bool, overrides map[string]string) (bool, string) { + if docreaderConnected && isWeKnoraCloudDocReaderAddr(overrides["docreader_addr"]) { + return true, "" + } + return false, "WeKnoraCloud document reader is not initialized" +} + // --------------------------------------------------------------------------- // mineru — Go-native, calls self-hosted MinerU API directly // --------------------------------------------------------------------------- @@ -169,3 +192,7 @@ func ListAllEngines(docreaderConnected bool, overrides map[string]string, remote return result } + +func isWeKnoraCloudDocReaderAddr(addr string) bool { + return strings.TrimSuffix(strings.TrimSpace(addr), "/") == strings.TrimRight(provider.WeKnoraCloudBaseURL, "/")+"/api/v1/doc/reader" +} diff --git a/internal/infrastructure/docparser/weknoracloud_http_reader.go b/internal/infrastructure/docparser/weknoracloud_http_reader.go new file mode 100644 index 00000000..ce936f5c --- /dev/null +++ b/internal/infrastructure/docparser/weknoracloud_http_reader.go @@ -0,0 +1,245 @@ +package docparser + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "sort" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" +) + +// WeKnoraCloudSignedDocumentReader implements the docreader HTTP protocol with WeKnoraCloud signing. +type WeKnoraCloudSignedDocumentReader struct { + baseURL string + appID string + apiKey string + client *http.Client + initialPollInterval time.Duration + maxPollInterval time.Duration + pollTimeout time.Duration +} + +func NewWeKnoraCloudSignedDocumentReader(baseURL, appID, apiKey string) (*WeKnoraCloudSignedDocumentReader, error) { + baseURL = strings.TrimSuffix(strings.TrimSpace(baseURL), "/") + if baseURL == "" { + return nil, fmt.Errorf("docreader base URL is required") + } + if appID == "" { + return nil, fmt.Errorf("WeKnoraCloud appID is required") + } + if apiKey == "" { + return nil, fmt.Errorf("WeKnoraCloud apiKey is required") + } + return &WeKnoraCloudSignedDocumentReader{ + baseURL: baseURL, + appID: appID, + apiKey: apiKey, + initialPollInterval: 500 * time.Millisecond, + maxPollInterval: 10 * time.Second, + pollTimeout: 20 * time.Minute, + client: &http.Client{ + Timeout: 500 * time.Minute, + Transport: &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 90 * time.Second, + MaxIdleConnsPerHost: 5, + }, + }, + }, nil +} + +func (p *WeKnoraCloudSignedDocumentReader) Reconnect(addr string) error { + p.baseURL = strings.TrimSuffix(strings.TrimSpace(addr), "/") + return nil +} + +func (p *WeKnoraCloudSignedDocumentReader) IsConnected() bool { return p.baseURL != "" } + +func (p *WeKnoraCloudSignedDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) { + if !p.IsConnected() { + return nil, errNotConnected + } + return []types.ParserEngineInfo{{ + Name: WeKnoraCloudEngineName, + Description: "WeKnoraCloud signed docreader", + FileTypes: weKnoraCloudSupportedFileTypes(), + Available: true, + }}, nil +} + +func (p *WeKnoraCloudSignedDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) { + if !p.IsConnected() { + return nil, errNotConnected + } + body := httpReadRequest{ + FileName: req.FileName, + FileType: req.FileType, + URL: req.URL, + Title: req.Title, + RequestID: req.RequestID, + Config: &httpReadConfig{ + ParserEngine: req.ParserEngine, + ParserEngineOverrides: req.ParserEngineOverrides, + }, + } + if len(req.FileContent) > 0 { + body.FileContent = base64.StdEncoding.EncodeToString(req.FileContent) + } + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("http marshal read request: %w", err) + } + httpReq, err := p.newSignedRequest(ctx, http.MethodPost, p.baseURL, jsonBody) + if err != nil { + return nil, err + } + resp, err := p.client.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("http read failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("http read status %d: %s", resp.StatusCode, string(bodyBytes)) + } + var submit weKnoraCloudAsyncSubmitResponse + if err := json.NewDecoder(resp.Body).Decode(&submit); err != nil { + return nil, fmt.Errorf("http decode read submit response: %w", err) + } + if strings.TrimSpace(submit.TaskID) == "" { + return nil, fmt.Errorf("weknoracloud docreader submit response missing task_id") + } + return p.pollTaskResult(ctx, submit.TaskID) +} + +type weKnoraCloudAsyncSubmitResponse struct { + TaskID string `json:"task_id"` + Status string `json:"status"` + Message string `json:"message"` + CreatedAt int64 `json:"created_at"` +} + +type weKnoraCloudAsyncTaskResponse struct { + TaskID string `json:"task_id"` + Status string `json:"status"` + Message string `json:"message"` + Progress float64 `json:"progress"` + Result *httpReadResponse `json:"result"` + Error string `json:"error"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + +func weKnoraCloudSupportedFileTypes() []string { + seen := map[string]struct{}{} + var fileTypes []string + for _, engine := range localEngines { + if engine.Name() == WeKnoraCloudEngineName { + continue + } + for _, fileType := range engine.FileTypes(true) { + if _, ok := seen[fileType]; ok { + continue + } + seen[fileType] = struct{}{} + fileTypes = append(fileTypes, fileType) + } + } + sort.Strings(fileTypes) + return fileTypes +} + +func (p *WeKnoraCloudSignedDocumentReader) pollTaskResult(ctx context.Context, taskID string) (*types.ReadResult, error) { + pollCtx := ctx + if _, ok := ctx.Deadline(); !ok && p.pollTimeout > 0 { + var cancel context.CancelFunc + pollCtx, cancel = context.WithTimeout(ctx, p.pollTimeout) + defer cancel() + } + statusURL := strings.TrimSuffix(p.baseURL, "/reader") + "/" + taskID + currentInterval := p.initialPollInterval + for { + httpReq, err := p.newSignedRequest(pollCtx, http.MethodGet, statusURL, nil) + if err != nil { + return nil, err + } + resp, err := p.client.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("http poll task failed: %w", err) + } + var taskResp weKnoraCloudAsyncTaskResponse + func() { + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + err = fmt.Errorf("http poll task status %d: %s", resp.StatusCode, string(bodyBytes)) + return + } + if decodeErr := json.NewDecoder(resp.Body).Decode(&taskResp); decodeErr != nil { + err = fmt.Errorf("http decode task response: %w", decodeErr) + } + }() + if err != nil { + return nil, err + } + switch taskResp.Status { + case "completed": + if taskResp.Result == nil { + return &types.ReadResult{}, nil + } + return fromHTTPReadResponse(taskResp.Result), nil + case "failed": + if taskResp.Error != "" { + return nil, fmt.Errorf("weknoracloud docreader task failed: %s", taskResp.Error) + } + return nil, fmt.Errorf("weknoracloud docreader task failed: %s", taskResp.Message) + case "cancelled": + if taskResp.Error != "" { + return nil, fmt.Errorf("weknoracloud docreader task cancelled: %s", taskResp.Error) + } + return nil, fmt.Errorf("weknoracloud docreader task cancelled") + } + if err := pollCtx.Err(); err != nil { + return nil, err + } + + // Exponential backoff: multiply by 1.5 each time, cap at maxPollInterval + select { + case <-pollCtx.Done(): + return nil, pollCtx.Err() + case <-time.After(currentInterval): + // Update interval for next iteration + nextInterval := time.Duration(float64(currentInterval) * 1.5) + if nextInterval > p.maxPollInterval { + nextInterval = p.maxPollInterval + } + currentInterval = nextInterval + } + } +} + +func (p *WeKnoraCloudSignedDocumentReader) newSignedRequest(ctx context.Context, method, url string, body []byte) (*http.Request, error) { + requestID := uuid.New().String() + if len(body) == 0 { + body = []byte("{}") + } + httpReq, err := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("http new request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.ContentLength = int64(len(body)) + for k, v := range utils.Sign(p.appID, p.apiKey, requestID, string(body)) { + httpReq.Header.Set(k, v) + } + return httpReq, nil +} diff --git a/internal/models/chat/chat.go b/internal/models/chat/chat.go index 5360880b..f9a2b99c 100644 --- a/internal/models/chat/chat.go +++ b/internal/models/chat/chat.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "net/http" "strings" "github.com/Tencent/WeKnora/internal/models/provider" @@ -93,13 +94,15 @@ type Chat interface { } type ChatConfig struct { - Source types.ModelSource - BaseURL string - ModelName string - APIKey string - ModelID string - Provider string - Extra map[string]any + Source types.ModelSource + BaseURL string + ModelName string + APIKey string + ModelID string + Provider string + ExtraConfig map[string]string + AppID string + AppSecret string // 加密值,由工厂函数调用方传入,在 NewWeKnoraCloudChat 中使用前已解密 } // NewChat 创建聊天实例 @@ -134,6 +137,11 @@ func NewRemoteChat(config *ChatConfig) (Chat, error) { if spec.EndpointCustomizer != nil { remoteChat.SetEndpointCustomizer(spec.EndpointCustomizer) } + if spec.HeaderCustomizer != nil { + remoteChat.SetHeaderCustomizer(func(req *http.Request, body []byte) error { + return spec.HeaderCustomizer(remoteChat, req, body) + }) + } } return remoteChat, nil diff --git a/internal/models/chat/chat_provider_spec.go b/internal/models/chat/chat_provider_spec.go index cd0ded9e..b35ebe44 100644 --- a/internal/models/chat/chat_provider_spec.go +++ b/internal/models/chat/chat_provider_spec.go @@ -2,10 +2,13 @@ package chat import ( "context" + "net/http" "strings" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/provider" + modelutils "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/google/uuid" "github.com/sashabaranov/go-openai" ) @@ -20,11 +23,22 @@ type ProviderSpec struct { RequestCustomizer func(req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool) (any, bool) // EndpointCustomizer: provider-specific endpoint URL override. EndpointCustomizer func(baseURL string, modelID string, isStream bool) string + // HeaderCustomizer: provider-specific raw HTTP header customization. + HeaderCustomizer func(chat *RemoteAPIChat, req *http.Request, body []byte) error } // chatProviderSpecs is the ordered list of provider specs. // Order matters: more specific specs (with ModelMatcher) should come before generic ones. var chatProviderSpecs = []ProviderSpec{ + // WeKnoraCloud + { + Provider: provider.ProviderWeKnoraCloud, + RequestCustomizer: weKnoraCloudRequestCustomizer, + EndpointCustomizer: func(baseURL string, _ string, _ bool) string { + return strings.TrimRight(baseURL, "/") + "/api/v1/chat/completions" + }, + HeaderCustomizer: weKnoraCloudHeaderCustomizer, + }, // Aliyun Qwen Thinking Models (must be before generic Aliyun) { Provider: provider.ProviderAliyun, @@ -95,6 +109,61 @@ type ThinkingChatCompletionRequest struct { // --- Customizer functions --- +type weKnoraCloudChatRequest struct { + Model string `json:"model"` + Messages []weKnoraCloudMessage `json:"messages"` + Stream bool `json:"stream"` + MaxTokens int `json:"max_tokens,omitempty"` + Temperature float64 `json:"temperature,omitempty"` + TopP float64 `json:"top_p,omitempty"` +} + +type weKnoraCloudMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +func weKnoraCloudRequestCustomizer(req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool) (any, bool) { + weKnoraCloudReq := weKnoraCloudChatRequest{ + Model: req.Model, + Messages: convertToWeKnoraCloudMessagesFromOpenAI(req.Messages), + Stream: isStream, + } + if opts != nil { + weKnoraCloudReq.Temperature = opts.Temperature + weKnoraCloudReq.TopP = opts.TopP + weKnoraCloudReq.MaxTokens = opts.MaxTokens + } + return weKnoraCloudReq, true +} + +func weKnoraCloudHeaderCustomizer(chat *RemoteAPIChat, req *http.Request, body []byte) error { + requestID := uuid.NewString() + headers := modelutils.Sign(chat.appID, chat.appSecret, requestID, string(body)) + for k, v := range headers { + req.Header.Set(k, v) + } + return nil +} + +func convertToWeKnoraCloudMessagesFromOpenAI(messages []openai.ChatCompletionMessage) []weKnoraCloudMessage { + result := make([]weKnoraCloudMessage, 0, len(messages)) + for _, m := range messages { + content := m.Content + if content == "" && len(m.MultiContent) > 0 { + var textParts []string + for _, part := range m.MultiContent { + if part.Type == openai.ChatMessagePartTypeText && part.Text != "" { + textParts = append(textParts, part.Text) + } + } + content = strings.Join(textParts, "\n") + } + result = append(result, weKnoraCloudMessage{Role: m.Role, Content: content}) + } + return result +} + // qwenThinkingRequestCustomizer 自定义 Qwen 系列(阿里云)模型的思考请求 func qwenThinkingRequestCustomizer( req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool, diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go index 39e3ae53..25846811 100644 --- a/internal/models/chat/remote_api.go +++ b/internal/models/chat/remote_api.go @@ -39,6 +39,8 @@ type RemoteAPIChat struct { baseURL string apiKey string provider provider.ProviderName + appID string + appSecret string // requestCustomizer 允许子类自定义请求 // 返回自定义请求体(如果为 nil 则使用标准请求)和是否需要使用原始 HTTP 请求 @@ -47,6 +49,9 @@ type RemoteAPIChat struct { // endpointCustomizer 允许子类自定义请求的 endpoint // 返回是否使用自定义请求地址, 返回空则使用默认OpenAI格式地址 endpointCustomizer func(baseURL string, modelID string, isStream bool) (endpoint string) + + // headerCustomizer 允许子类自定义原始 HTTP 请求头(例如签名认证) + headerCustomizer func(req *http.Request, body []byte) error } // NewRemoteAPIChat 创建远程 API 聊天实例 @@ -69,11 +74,9 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) { config.AzureModelMapperFunc = func(model string) string { return model } - if chatConfig.Extra != nil { - if v, ok := chatConfig.Extra["api_version"]; ok { - if vs, ok := v.(string); ok && vs != "" { - config.APIVersion = vs - } + if chatConfig.ExtraConfig != nil { + if v, ok := chatConfig.ExtraConfig["api_version"]; ok { + config.APIVersion = v } } } else { @@ -83,13 +86,30 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) { } } + modelName := chatConfig.ModelName + if chatConfig.ExtraConfig != nil { + if override := strings.TrimSpace(chatConfig.ExtraConfig["remote_model_name"]); override != "" { + modelName = override + } + } + if providerName == provider.ProviderWeKnoraCloud { + if chatConfig.AppID == "" { + return nil, fmt.Errorf("WeKnoraCloud provider: AppID is required") + } + if chatConfig.AppSecret == "" { + return nil, fmt.Errorf("WeKnoraCloud provider: AppSecret is required") + } + } + return &RemoteAPIChat{ - modelName: chatConfig.ModelName, + modelName: modelName, client: openai.NewClientWithConfig(config), modelID: chatConfig.ModelID, baseURL: chatConfig.BaseURL, apiKey: apiKey, provider: providerName, + appID: chatConfig.AppID, + appSecret: chatConfig.AppSecret, }, nil } @@ -103,6 +123,11 @@ func (c *RemoteAPIChat) SetEndpointCustomizer(customizer func(baseURL string, mo c.endpointCustomizer = customizer } +// SetHeaderCustomizer 设置原始 HTTP 请求头自定义器 +func (c *RemoteAPIChat) SetHeaderCustomizer(customizer func(req *http.Request, body []byte) error) { + c.headerCustomizer = customizer +} + // ConvertMessages 转换消息格式为 OpenAI 格式(导出供子类使用) func (c *RemoteAPIChat) ConvertMessages(messages []Message) []openai.ChatCompletionMessage { openaiMessages := make([]openai.ChatCompletionMessage, 0, len(messages)) @@ -330,7 +355,12 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu } httpReq.Header.Set("Content-Type", "application/json") - if c.provider == provider.ProviderAzureOpenAI { + + if c.headerCustomizer != nil { + if err := c.headerCustomizer(httpReq, jsonData); err != nil { + return nil, fmt.Errorf("customize headers: %w", err) + } + } else if c.provider == provider.ProviderAzureOpenAI { httpReq.Header.Set("api-key", c.apiKey) } else { httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) @@ -486,7 +516,12 @@ func (c *RemoteAPIChat) chatStreamWithRawHTTP(ctx context.Context, endpoint stri } httpReq.Header.Set("Content-Type", "application/json") - if c.provider == provider.ProviderAzureOpenAI { + + if c.headerCustomizer != nil { + if err := c.headerCustomizer(httpReq, jsonData); err != nil { + return nil, fmt.Errorf("customize headers: %w", err) + } + } else if c.provider == provider.ProviderAzureOpenAI { httpReq.Header.Set("api-key", c.apiKey) } else { httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) diff --git a/internal/models/embedding/embedder.go b/internal/models/embedding/embedder.go index d62abb51..a73d3208 100644 --- a/internal/models/embedding/embedder.go +++ b/internal/models/embedding/embedder.go @@ -47,7 +47,9 @@ type Config struct { Dimensions int `json:"dimensions"` ModelID string `json:"model_id"` Provider string `json:"provider"` - Extra map[string]any `json:"extra,omitempty"` + ExtraConfig map[string]string `json:"extra_config"` + AppID string + AppSecret string // 加密值,工厂函数调用方传入,使用前已解密 } // NewEmbedder creates an embedder based on the configuration @@ -129,11 +131,9 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll return embedder, err case provider.ProviderAzureOpenAI: apiVersion := "2024-10-21" - if config.Extra != nil { - if v, ok := config.Extra["api_version"]; ok { - if vs, ok := v.(string); ok && vs != "" { - apiVersion = vs - } + if config.ExtraConfig != nil { + if v, ok := config.ExtraConfig["api_version"]; ok { + apiVersion = v } } embedder, err = NewAzureOpenAIEmbedder(config.APIKey, @@ -153,6 +153,9 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll config.ModelID, pooler) return embedder, err + case provider.ProviderWeKnoraCloud: + embedder, err = NewWeKnoraCloudEmbedder(config) + return embedder, err default: // Use OpenAI-compatible embedder for other providers embedder, err = NewOpenAIEmbedder(config.APIKey, diff --git a/internal/models/embedding/weknoracloud.go b/internal/models/embedding/weknoracloud.go new file mode 100644 index 00000000..9dec9682 --- /dev/null +++ b/internal/models/embedding/weknoracloud.go @@ -0,0 +1,139 @@ +package embedding + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/google/uuid" +) + +const weKnoraCloudEmbedPath = "/api/v1/embeddings" + +// WeKnoraCloudEmbedder 实现 embedding.Embedder 接口,对接 WeKnoraCloud /api/v1/embeddings +type WeKnoraCloudEmbedder struct { + modelName string + remoteModelName string + modelID string + appID string + apiKey string + baseURL string + dimensions int + client *http.Client + EmbedderPooler +} + +// NewWeKnoraCloudEmbedder 构造 WeKnoraCloudEmbedder +func NewWeKnoraCloudEmbedder(config Config) (*WeKnoraCloudEmbedder, error) { + if config.AppID == "" { + return nil, fmt.Errorf("WeKnoraCloud embedder: AppID is required") + } + if config.AppSecret == "" { + return nil, fmt.Errorf("WeKnoraCloud embedder: AppSecret is required") + } + remoteModelName := "" + if config.ExtraConfig != nil { + remoteModelName = strings.TrimSpace(config.ExtraConfig["remote_model_name"]) + } + return &WeKnoraCloudEmbedder{ + modelName: config.ModelName, + remoteModelName: remoteModelName, + modelID: config.ModelID, + appID: config.AppID, + apiKey: config.AppSecret, + baseURL: strings.TrimRight(config.BaseURL, "/"), + dimensions: config.Dimensions, + client: &http.Client{Timeout: 60 * time.Second}, + }, nil +} + +type weKnoraCloudEmbedRequest struct { + Model string `json:"model"` + Input []string `json:"input"` +} + +type weKnoraCloudEmbedResponse struct { + Data []struct { + Index int `json:"index"` + Embedding []float32 `json:"embedding"` + } `json:"data"` +} + +func (e *WeKnoraCloudEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { + results, err := e.BatchEmbed(ctx, []string{text}) + if err != nil { + return nil, err + } + if len(results) == 0 { + return nil, fmt.Errorf("weknoracloud embedder: empty response") + } + return results[0], nil +} + +func (e *WeKnoraCloudEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) { + reqBody := weKnoraCloudEmbedRequest{Model: e.effectiveModelName(), Input: texts} + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("weknoracloud embedder: marshal: %w", err) + } + + requestID := uuid.New().String() + headers := utils.Sign(e.appID, e.apiKey, requestID, string(bodyBytes)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.baseURL+weKnoraCloudEmbedPath, bytes.NewReader(bodyBytes)) + if err != nil { + return nil, fmt.Errorf("weknoracloud embedder: create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := e.client.Do(req) + if err != nil { + return nil, fmt.Errorf("weknoracloud embedder: do request: %w", err) + } + defer resp.Body.Close() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("weknoracloud embedder: read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("weknoracloud embedder: status %d: %s", resp.StatusCode, string(respBytes)) + } + + var embedResp weKnoraCloudEmbedResponse + if err := json.Unmarshal(respBytes, &embedResp); err != nil { + return nil, fmt.Errorf("weknoracloud embedder: unmarshal: %w", err) + } + + result := make([][]float32, len(texts)) + for _, item := range embedResp.Data { + if item.Index < len(result) { + result[item.Index] = item.Embedding + } + } + return result, nil +} + +func (e *WeKnoraCloudEmbedder) BatchEmbedWithPool(ctx context.Context, model Embedder, texts []string) ([][]float32, error) { + return e.BatchEmbed(ctx, texts) +} + +func (e *WeKnoraCloudEmbedder) effectiveModelName() string { + if e.remoteModelName != "" { + return e.remoteModelName + } + return e.modelName +} + +func (e *WeKnoraCloudEmbedder) GetModelName() string { return e.modelName } +func (e *WeKnoraCloudEmbedder) GetModelID() string { return e.modelID } +func (e *WeKnoraCloudEmbedder) GetDimensions() int { return e.dimensions } diff --git a/internal/models/provider/provider.go b/internal/models/provider/provider.go index 801963c1..6a2da7b7 100644 --- a/internal/models/provider/provider.go +++ b/internal/models/provider/provider.go @@ -87,6 +87,7 @@ func AllProviders() []ProviderName { ProviderNvidia, ProviderNovita, ProviderAzureOpenAI, + ProviderWeKnoraCloud, } } diff --git a/internal/models/provider/weknoracloud.go b/internal/models/provider/weknoracloud.go new file mode 100644 index 00000000..39e44d79 --- /dev/null +++ b/internal/models/provider/weknoracloud.go @@ -0,0 +1,41 @@ +package provider + +import "github.com/Tencent/WeKnora/internal/types" + +const ( + ProviderWeKnoraCloud ProviderName = "weknoracloud" + + // WeKnoraCloudBaseURL WeKnoraCloud 服务硬编码 Base URL(统一入口,路径由各实现拼接) + WeKnoraCloudBaseURL = "https://weknora.weixin.qq.com" +) + +type WeKnoraCloudProvider struct{} + +func init() { + Register(&WeKnoraCloudProvider{}) +} + +func (p *WeKnoraCloudProvider) Info() ProviderInfo { + return ProviderInfo{ + Name: ProviderWeKnoraCloud, + DisplayName: "WeKnoraCloud", + Description: "WeKnoraCloud 平台,提供 Chat / Embedding / Rerank 模型,使用 APPID + API Key 签名认证", + DefaultURLs: map[types.ModelType]string{ + types.ModelTypeKnowledgeQA: WeKnoraCloudBaseURL, + types.ModelTypeEmbedding: WeKnoraCloudBaseURL, + types.ModelTypeRerank: WeKnoraCloudBaseURL, + }, + ModelTypes: []types.ModelType{ + types.ModelTypeKnowledgeQA, + types.ModelTypeEmbedding, + types.ModelTypeRerank, + }, + RequiresAuth: true, + } +} + +func (p *WeKnoraCloudProvider) ValidateConfig(config *Config) error { + // AppID/AppSecret 通过专用初始化接口写入,此处仅做结构校验。 + // 其中 AppSecret 字段当前实际承载上游 API Key。 + return nil +} diff --git a/internal/models/rerank/reranker.go b/internal/models/rerank/reranker.go index 5ed45dba..2c5a1ba2 100644 --- a/internal/models/rerank/reranker.go +++ b/internal/models/rerank/reranker.go @@ -78,12 +78,15 @@ func (d *DocumentInfo) UnmarshalJSON(data []byte) error { } type RerankerConfig struct { - APIKey string - BaseURL string - ModelName string - Source types.ModelSource - ModelID string - Provider string // Provider identifier: openai, aliyun, zhipu, siliconflow, jina, generic + APIKey string + BaseURL string + ModelName string + Source types.ModelSource + ModelID string + Provider string // Provider identifier: openai, aliyun, zhipu, siliconflow, jina, generic + ExtraConfig map[string]string + AppID string + AppSecret string // 加密值,工厂函数调用方传入,使用前已解密 } // NewReranker creates a reranker based on the configuration @@ -103,6 +106,8 @@ func NewReranker(config *RerankerConfig) (Reranker, error) { return NewJinaReranker(config) case provider.ProviderNvidia: return NewNvidiaReranker(config) + case provider.ProviderWeKnoraCloud: + return NewWeKnoraCloudReranker(config) default: return NewOpenAIReranker(config) } diff --git a/internal/models/rerank/weknoracloud.go b/internal/models/rerank/weknoracloud.go new file mode 100644 index 00000000..ea985d75 --- /dev/null +++ b/internal/models/rerank/weknoracloud.go @@ -0,0 +1,130 @@ +package rerank + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/google/uuid" +) + +const weKnoraCloudRerankPath = "/api/v1/rerank" + +// WeKnoraCloudReranker 实现 rerank.Reranker 接口,对接 WeKnoraCloud /api/v1/rerank +type WeKnoraCloudReranker struct { + modelName string + remoteModelName string + modelID string + appID string + apiKey string + baseURL string + client *http.Client +} + +// NewWeKnoraCloudReranker 构造 WeKnoraCloudReranker +func NewWeKnoraCloudReranker(config *RerankerConfig) (*WeKnoraCloudReranker, error) { + if config.AppID == "" { + return nil, fmt.Errorf("WeKnoraCloud reranker: AppID is required") + } + if config.AppSecret == "" { + return nil, fmt.Errorf("WeKnoraCloud reranker: AppSecret is required") + } + remoteModelName := "" + if config.ExtraConfig != nil { + remoteModelName = strings.TrimSpace(config.ExtraConfig["remote_model_name"]) + } + return &WeKnoraCloudReranker{ + modelName: config.ModelName, + remoteModelName: remoteModelName, + modelID: config.ModelID, + appID: config.AppID, + apiKey: config.AppSecret, + baseURL: strings.TrimRight(config.BaseURL, "/"), + client: &http.Client{Timeout: 60 * time.Second}, + }, nil +} + +type weKnoraCloudRerankRequest struct { + Model string `json:"model"` + Query string `json:"query"` + Documents []string `json:"documents"` +} + +type weKnoraCloudRerankResponse struct { + Results []struct { + Index int `json:"index"` + RelevanceScore float64 `json:"relevance_score"` + Document struct { + Text string `json:"text"` + } `json:"document"` + } `json:"results"` +} + +func (r *WeKnoraCloudReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { + reqBody := weKnoraCloudRerankRequest{ + Model: r.effectiveModelName(), + Query: query, + Documents: documents, + } + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("weknoracloud reranker: marshal: %w", err) + } + + requestID := uuid.New().String() + headers := utils.Sign(r.appID, r.apiKey, requestID, string(bodyBytes)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.baseURL+weKnoraCloudRerankPath, bytes.NewReader(bodyBytes)) + if err != nil { + return nil, fmt.Errorf("weknoracloud reranker: create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := r.client.Do(req) + if err != nil { + return nil, fmt.Errorf("weknoracloud reranker: do request: %w", err) + } + defer resp.Body.Close() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("weknoracloud reranker: read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("weknoracloud reranker: status %d: %s", resp.StatusCode, string(respBytes)) + } + + var rerankResp weKnoraCloudRerankResponse + if err := json.Unmarshal(respBytes, &rerankResp); err != nil { + return nil, fmt.Errorf("weknoracloud reranker: unmarshal: %w", err) + } + + results := make([]RankResult, 0, len(rerankResp.Results)) + for _, item := range rerankResp.Results { + results = append(results, RankResult{ + Index: item.Index, + RelevanceScore: item.RelevanceScore, + Document: DocumentInfo{Text: item.Document.Text}, + }) + } + return results, nil +} + +func (r *WeKnoraCloudReranker) effectiveModelName() string { + if r.remoteModelName != "" { + return r.remoteModelName + } + return r.modelName +} + +func (r *WeKnoraCloudReranker) GetModelName() string { return r.modelName } +func (r *WeKnoraCloudReranker) GetModelID() string { return r.modelID } diff --git a/internal/models/utils/signer.go b/internal/models/utils/signer.go new file mode 100644 index 00000000..dfbecbb6 --- /dev/null +++ b/internal/models/utils/signer.go @@ -0,0 +1,93 @@ +package utils + +import ( + "bytes" + "crypto/md5" + "fmt" + "math/rand" + "sort" + "strconv" + "strings" + "time" +) + +const ( + nonceChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + nonceLength = 16 +) + +// Sign 按 WeKnoraCloud 参考实现生成请求头。 +// appID: 上游 APPID +// apiKey: 上游 API Key(当前沿用 AppSecret 字段承载) +// requestID: 每次请求唯一的 UUID 字符串 +// bodyJSON: 请求体 JSON 字符串,空请求体传 "{}" +func Sign(appID, apiKey, requestID, bodyJSON string) map[string]string { + timestamp := strconv.FormatInt(time.Now().Unix(), 10) + nonce := generateNonce(nonceLength) + + bodyForHash := bodyJSON + if bodyForHash == "" { + bodyForHash = "{}" + } + bodyMD5 := md5Hex(bodyForHash) + + params := map[string]string{ + "x-appid": appID, + "x-api-key": apiKey, + "x-request-id": requestID, + "x-timestamp": timestamp, + "x-nonce": nonce, + "body": bodyMD5, + } + + keys := make([]string, 0, len(params)) + for k := range params { + keys = append(keys, k) + } + sort.Strings(keys) + + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, rfc3986Encode(k)+"="+rfc3986Encode(params[k])) + } + signature := md5Hex(strings.Join(parts, "&")) + + return map[string]string{ + "X-APPID": appID, + "X-API-Key": apiKey, + "X-Request-ID": requestID, + "X-Timestamp": timestamp, + "X-Nonce": nonce, + "X-Signature": signature, + } +} + +func md5Hex(s string) string { + h := md5.New() + h.Write([]byte(s)) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func generateNonce(length int) string { + b := make([]byte, length) + for i := range b { + b[i] = nonceChars[rand.Intn(len(nonceChars))] + } + return string(b) +} + +// rfc3986Encode 对字符串做 RFC3986 编码 +// 保留字符:A-Z a-z 0-9 - _ . ~ +func rfc3986Encode(s string) string { + var buf bytes.Buffer + for _, r := range s { + if (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || + (r >= '0' && r <= '9') || r == '-' || r == '_' || + r == '.' || r == '~' { + buf.WriteRune(r) + } else { + fmt.Fprintf(&buf, "%%%02X", r) + } + } + return buf.String() +} diff --git a/internal/router/router.go b/internal/router/router.go index 5b5bdaf0..9da4a23e 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -62,6 +62,7 @@ type RouterParams struct { OrganizationHandler *handler.OrganizationHandler IMHandler *handler.IMHandler DataSourceHandler *handler.DataSourceHandler + WeKnoraCloudHandler *handler.WeKnoraCloudHandler } // NewRouter 创建新的路由 @@ -144,6 +145,7 @@ func NewRouter(params RouterParams) *gin.Engine { RegisterOrganizationRoutes(v1, params.OrganizationHandler) RegisterIMChannelRoutes(v1, params.IMHandler) RegisterDataSourceRoutes(v1, params.DataSourceHandler) + RegisterWeKnoraCloudRoutes(v1, params.WeKnoraCloudHandler) } return r @@ -805,3 +807,9 @@ func RegisterDataSourceRoutes(r *gin.RouterGroup, handler *handler.DataSourceHan ds.GET("/logs/:log_id", handler.GetSyncLog) } } + +// RegisterWeKnoraCloudRoutes 注册 WeKnoraCloud 初始化路由 +func RegisterWeKnoraCloudRoutes(r *gin.RouterGroup, handler *handler.WeKnoraCloudHandler) { + r.POST("/models/weknoracloud/initialize", handler.Initialize) + r.GET("/models/weknoracloud/status", handler.Status) +} diff --git a/internal/types/interfaces/tenant.go b/internal/types/interfaces/tenant.go index c5c9c367..5942c7e6 100644 --- a/internal/types/interfaces/tenant.go +++ b/internal/types/interfaces/tenant.go @@ -28,6 +28,8 @@ type TenantService interface { SearchTenants(ctx context.Context, keyword string, tenantID uint64, page, pageSize int) ([]*types.Tenant, int64, error) // GetTenantByIDForUser gets a tenant by ID with permission check GetTenantByIDForUser(ctx context.Context, tenantID uint64, userID string) (*types.Tenant, error) + // GetTenantByAPIKey gets a tenant by API key + GetDocreaderCredentials(ctx context.Context) *types.DocreaderCredentials } // TenantRepository defines the tenant repository interface diff --git a/internal/types/interfaces/weknoracloud.go b/internal/types/interfaces/weknoracloud.go new file mode 100644 index 00000000..4b7ff1fa --- /dev/null +++ b/internal/types/interfaces/weknoracloud.go @@ -0,0 +1,15 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// WeKnoraCloudService 处理 WeKnoraCloud 厂商的初始化逻辑 +type WeKnoraCloudService interface { + Initialize(ctx context.Context, appID, appSecret string) (*types.InitializeResult, error) + // CheckStatus 检查当前租户的 WeKnoraCloud 凭证是否可正常解密 + // needsReinit=true 表示加密状态已损坏(salt 变更等),需要用户重新填写凭证 + CheckStatus(ctx context.Context) (*types.WeKnoraCloudStatusResult, error) +} diff --git a/internal/types/model.go b/internal/types/model.go index 9ee1e92d..6f1ffa4f 100644 --- a/internal/types/model.go +++ b/internal/types/model.go @@ -68,6 +68,9 @@ type ModelParameters struct { Provider string `yaml:"provider" json:"provider"` // Provider identifier: openai, aliyun, zhipu, generic ExtraConfig map[string]string `yaml:"extra_config" json:"extra_config"` // Provider-specific configuration SupportsVision bool `yaml:"supports_vision" json:"supports_vision"` // Whether the model accepts image/multimodal input + // WeKnoraCloud 厂商专用凭证 + AppID string `yaml:"app_id,omitempty" json:"app_id,omitempty"` + AppSecret string `yaml:"app_secret,omitempty" json:"app_secret,omitempty"` // AES-256 加密存储,实际承载上游 API Key } // Model represents the AI model diff --git a/internal/types/tenant.go b/internal/types/tenant.go index 0ee8b105..b6081842 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -233,7 +233,11 @@ func (c *ConversationConfig) Scan(value interface{}) error { // ParserEngineConfig holds tenant-level overrides for document parser engines (e.g. MinerU endpoint, API key). // These values take precedence over environment variables when parsing documents. type ParserEngineConfig struct { - DocReaderAddr string `json:"docreader_addr"` // 文档解析服务地址 + DocReaderAddr string `json:"docreader_addr"` // 文档解析服务地址 + // docreader 凭证 + DocreaderAppID string `json:"docreader_app_id,omitempty"` + DocreaderAPIKey string `json:"docreader_api_key,omitempty"` + MinerUEndpoint string `json:"mineru_endpoint"` // MinerU 自建服务端点 MinerUAPIKey string `json:"mineru_api_key"` // MinerU 云 API Key @@ -259,6 +263,9 @@ func (c *ParserEngineConfig) ToOverridesMap() map[string]string { return nil } m := make(map[string]string) + if c.DocReaderAddr != "" { + m["docreader_addr"] = c.DocReaderAddr + } if c.MinerUEndpoint != "" { m["mineru_endpoint"] = c.MinerUEndpoint } diff --git a/internal/types/weknoracloud.go b/internal/types/weknoracloud.go new file mode 100644 index 00000000..7b931c0e --- /dev/null +++ b/internal/types/weknoracloud.go @@ -0,0 +1,25 @@ +package types + +// WeKnoraCloudStatusResult 状态检查结果 +type WeKnoraCloudStatusResult struct { + HasModels bool `json:"has_models"` // 是否已配置 WeKnoraCloud 模型 + NeedsReinit bool `json:"needs_reinit"` // 是否需要重新初始化(凭证损坏) + Reason string `json:"reason,omitempty"` // 需要重新初始化的原因 +} + +// InitializeResult 返回每个模型的操作结果 +type InitializeResult struct { + Models []ModelAction `json:"models"` +} + +// ModelAction 单个模型的操作结果 +type ModelAction struct { + Name string `json:"name"` + Type string `json:"type"` + Action string `json:"action"` // "created" | "updated" +} + +type DocreaderCredentials struct { + AppID string + APIKey string +} From d4edd374dfd14bb475eb430f0ae4d8f2bcb8a2aa Mon Sep 17 00:00:00 2001 From: begoniezhao Date: Mon, 13 Apr 2026 20:43:59 +0800 Subject: [PATCH 2/4] refactor: replace CryptoService with utils AES helpers and simplify encryption - Remove infrastructure/crypto package (CryptoService, config, PBKDF2-based encryption) - Replace with lightweight utils.GetAESKey/EncryptAESGCM/DecryptAESGCM helpers - Remove cryptoSvc dependency from modelService, tenantService, weKnoraCloudService - Remove crypto state persistence logic from container initialization - Add TruncatePromptTokens field to WeKnoraCloud embed request - Update frontend hint text for WeKnoraCloud credential errors Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend/src/views/settings/ModelSettings.vue | 2 +- internal/application/service/model.go | 14 +- internal/application/service/tenant.go | 25 +- internal/application/service/weknoracloud.go | 32 +-- internal/container/container.go | 89 ------- internal/infrastructure/crypto/config.go | 224 ------------------ internal/infrastructure/crypto/crypto.go | 163 ------------- internal/models/embedding/weknoracloud.go | 7 +- 8 files changed, 41 insertions(+), 515 deletions(-) delete mode 100644 internal/infrastructure/crypto/config.go delete mode 100644 internal/infrastructure/crypto/crypto.go diff --git a/frontend/src/views/settings/ModelSettings.vue b/frontend/src/views/settings/ModelSettings.vue index e735bb27..e22b293b 100644 --- a/frontend/src/views/settings/ModelSettings.vue +++ b/frontend/src/views/settings/ModelSettings.vue @@ -28,7 +28,7 @@
WeKnoraCloud 凭证已失效
- {{ weKnoraCloudReinitReason || '服务重启后加密密钥已变更,已保存的凭证无法解密。' }}请重新填写 APPID 和 APPSECRET 并点击"保存并初始化"以恢复服务。 + {{ weKnoraCloudReinitReason || '服务重启后加密密钥已变更,已保存的凭证无法解密。' }}
diff --git a/internal/application/service/model.go b/internal/application/service/model.go index 86db67cf..65ec1756 100644 --- a/internal/application/service/model.go +++ b/internal/application/service/model.go @@ -4,7 +4,6 @@ import ( "context" "errors" - "github.com/Tencent/WeKnora/internal/infrastructure/crypto" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/asr" "github.com/Tencent/WeKnora/internal/models/chat" @@ -14,6 +13,7 @@ import ( "github.com/Tencent/WeKnora/internal/models/vlm" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" ) // ErrModelNotFound is returned when a model cannot be found in the repository @@ -24,20 +24,17 @@ type modelService struct { repo interfaces.ModelRepository ollamaService *ollama.OllamaService pooler embedding.EmbedderPooler - cryptoSvc *crypto.CryptoService } // NewModelService creates a new model service instance func NewModelService(repo interfaces.ModelRepository, ollamaService *ollama.OllamaService, pooler embedding.EmbedderPooler, - cryptoSvc *crypto.CryptoService, ) interfaces.ModelService { return &modelService{ repo: repo, ollamaService: ollamaService, pooler: pooler, - cryptoSvc: cryptoSvc, } } @@ -46,11 +43,12 @@ func (s *modelService) decryptAppSecret(encrypted string) string { if encrypted == "" { return encrypted } - plain, err := s.cryptoSvc.DecryptString(encrypted) - if err != nil { - return "" + if key := utils.GetAESKey(); key != nil { + if encrypted, err := utils.DecryptAESGCM(encrypted, key); err == nil { + return encrypted + } } - return plain + return encrypted } // CreateModel creates a new model in the repository diff --git a/internal/application/service/tenant.go b/internal/application/service/tenant.go index faed4314..0f26c141 100644 --- a/internal/application/service/tenant.go +++ b/internal/application/service/tenant.go @@ -13,7 +13,6 @@ import ( "strings" "time" - "github.com/Tencent/WeKnora/internal/infrastructure/crypto" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -34,13 +33,12 @@ type ListTenantsParams struct { // tenantService implements the TenantService interface type tenantService struct { - repo interfaces.TenantRepository // Repository for tenant data operations - cryptoSvc *crypto.CryptoService + repo interfaces.TenantRepository // Repository for tenant data operations } // NewTenantService creates a new tenant service instance -func NewTenantService(repo interfaces.TenantRepository, cryptoSvc *crypto.CryptoService) interfaces.TenantService { - return &tenantService{repo: repo, cryptoSvc: cryptoSvc} +func NewTenantService(repo interfaces.TenantRepository) interfaces.TenantService { + return &tenantService{repo: repo} } // CreateTenant creates a new tenant @@ -353,11 +351,12 @@ func (s *tenantService) GetDocreaderCredentials(ctx context.Context) *types.Docr if appID == "" || tenant.ParserEngineConfig.DocreaderAPIKey == "" { return nil } - apiKey, err := s.cryptoSvc.DecryptString(tenant.ParserEngineConfig.DocreaderAPIKey) - if err != nil || apiKey == "" { - return nil + if key := utils.GetAESKey(); key != nil { + if encrypted, err := utils.DecryptAESGCM(tenant.ParserEngineConfig.DocreaderAPIKey, key); err == nil { + return &types.DocreaderCredentials{AppID: appID, APIKey: encrypted} + } } - return &types.DocreaderCredentials{AppID: appID, APIKey: apiKey} + return &types.DocreaderCredentials{AppID: appID, APIKey: tenant.ParserEngineConfig.DocreaderAPIKey} } } @@ -372,10 +371,12 @@ func (s *tenantService) GetDocreaderCredentials(ctx context.Context) *types.Docr if err == nil && tenant != nil && tenant.ParserEngineConfig != nil { appID := strings.TrimSpace(tenant.ParserEngineConfig.DocreaderAppID) if appID != "" && tenant.ParserEngineConfig.DocreaderAPIKey != "" { - apiKey, err := s.cryptoSvc.DecryptString(tenant.ParserEngineConfig.DocreaderAPIKey) - if err == nil && apiKey != "" { - return &types.DocreaderCredentials{AppID: appID, APIKey: apiKey} + if key := utils.GetAESKey(); key != nil { + if encrypted, err := utils.DecryptAESGCM(tenant.ParserEngineConfig.DocreaderAPIKey, key); err == nil { + return &types.DocreaderCredentials{AppID: appID, APIKey: encrypted} + } } + return &types.DocreaderCredentials{AppID: appID, APIKey: tenant.ParserEngineConfig.DocreaderAPIKey} } } diff --git a/internal/application/service/weknoracloud.go b/internal/application/service/weknoracloud.go index f63fe45f..ae7fc394 100644 --- a/internal/application/service/weknoracloud.go +++ b/internal/application/service/weknoracloud.go @@ -7,29 +7,28 @@ import ( "strings" "time" - "github.com/Tencent/WeKnora/internal/infrastructure/crypto" "github.com/Tencent/WeKnora/internal/models/provider" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/Tencent/WeKnora/internal/utils" "github.com/google/uuid" ) +const WeKnoraCloudAPI = "https://weknora.woa.com/platform/openapi" + type weKnoraCloudService struct { repo interfaces.ModelRepository tenantRepo interfaces.TenantRepository - cryptoSvc *crypto.CryptoService } // NewWeKnoraCloudService 构造 WeKnoraCloudService func NewWeKnoraCloudService( repo interfaces.ModelRepository, tenantRepo interfaces.TenantRepository, - cryptoSvc *crypto.CryptoService, ) interfaces.WeKnoraCloudService { return &weKnoraCloudService{ repo: repo, tenantRepo: tenantRepo, - cryptoSvc: cryptoSvc, } } @@ -99,9 +98,11 @@ func (s *weKnoraCloudService) prepareInitialize(ctx context.Context, appID, appS } // Step 2: Encrypt appSecret - encryptedSecret, err := s.cryptoSvc.EncryptString(appSecret) - if err != nil { - return nil, fmt.Errorf("encrypt AppSecret failed: %w", err) + encryptedSecret := appSecret + if key := utils.GetAESKey(); key != nil { + if encrypted, err := utils.EncryptAESGCM(appSecret, key); err == nil { + encryptedSecret = encrypted + } } // Step 3: Get tenantID and take snapshot @@ -296,18 +297,19 @@ func (s *weKnoraCloudService) CheckStatus(ctx context.Context) (*types.WeKnoraCl return &types.WeKnoraCloudStatusResult{ HasModels: true, NeedsReinit: true, - Reason: "WeKnoraCloud 凭证为空,请重新填写 APPID 和 API Key", + Reason: fmt.Sprintf("WeKnoraCloud 凭证为空,请重新填写 APPID 和 API Key, 请前往:%s", WeKnoraCloudAPI), }, nil } // Try to decrypt the API key - _, decErr := s.cryptoSvc.DecryptString(tenant.ParserEngineConfig.DocreaderAPIKey) - if decErr != nil { - return &types.WeKnoraCloudStatusResult{ - HasModels: true, - NeedsReinit: true, - Reason: "WeKnoraCloud 凭证解密失败(服务重启后加密密钥已变更),请重新填写 APPID 和 API Key", - }, nil + if key := utils.GetAESKey(); key != nil { + if _, err := utils.DecryptAESGCM(tenant.ParserEngineConfig.DocreaderAPIKey, key); err != nil { + return &types.WeKnoraCloudStatusResult{ + HasModels: true, + NeedsReinit: true, + Reason: "WeKnoraCloud 凭证解密失败(服务重启后加密密钥已变更),请重新填写 APPID 和 API Key", + }, nil + } } return &types.WeKnoraCloudStatusResult{HasModels: true, NeedsReinit: false}, nil diff --git a/internal/container/container.go b/internal/container/container.go index 1edac5a2..85880059 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -6,7 +6,6 @@ package container import ( "context" "database/sql" - "encoding/base64" "encoding/json" "fmt" "net/url" @@ -64,7 +63,6 @@ import ( "github.com/Tencent/WeKnora/internal/im/telegram" "github.com/Tencent/WeKnora/internal/im/wechat" "github.com/Tencent/WeKnora/internal/im/wecom" - "github.com/Tencent/WeKnora/internal/infrastructure/crypto" "github.com/Tencent/WeKnora/internal/infrastructure/docparser" infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search" "github.com/Tencent/WeKnora/internal/logger" @@ -100,7 +98,6 @@ func BuildContainer(container *dig.Container) *dig.Container { // Core infrastructure configuration logger.Debugf(ctx, "[Container] Registering core infrastructure...") must(container.Provide(config.LoadConfig)) - must(container.Provide(initCryptoService)) must(container.Provide(initTracer)) must(container.Provide(initDatabase)) must(container.Provide(initFileService)) @@ -311,92 +308,6 @@ func initTracer() (*tracing.Tracer, error) { return tracing.InitTracer() } -// cryptoStateFile 是持久化 crypto 状态的文件路径,存储在 data-files volume 中 -// 保证重启后能恢复相同的 masterKey 和 salt,避免已加密数据无法解密 -const cryptoStateFile = "/data/files/.crypto_state.json" - -// cryptoState 持久化存储的 crypto 配置 -type cryptoState struct { - MasterKey string `json:"master_key"` - Salt string `json:"salt"` // base64 编码 -} - -func initCryptoService() (*crypto.CryptoService, error) { - cfg := crypto.LoadConfigFromEnv() - - // 如果环境变量同时提供了 MasterKey 和 Salt,直接使用(优先级最高) - if cfg.MasterKey != "" && cfg.Salt != "" { - return crypto.NewCryptoServiceFromConfig(cfg) - } - - // 尝试从持久化文件恢复状态 - if state, err := loadCryptoState(cryptoStateFile); err == nil { - // 文件读取成功,环境变量中未覆盖的字段从文件补充 - if cfg.MasterKey == "" { - cfg.MasterKey = state.MasterKey - } - if cfg.Salt == "" { - cfg.Salt = state.Salt - } - return crypto.NewCryptoServiceFromConfig(cfg) - } - - // 文件不存在或读取失败,使用/生成配置后写入文件 - if cfg.MasterKey == "" { - cfg.MasterKey = "weknora-default-key" - } - - // 调用 NewCryptoServiceFromConfig 会在 cfg.Salt 为空时自动生成随机 salt - svc, err := crypto.NewCryptoServiceFromConfig(cfg) - if err != nil { - return nil, err - } - - // 将实际使用的 masterKey 和 salt 持久化,供重启恢复 - saltB64 := base64.StdEncoding.EncodeToString(svc.GetSalt()) - state := &cryptoState{ - MasterKey: cfg.MasterKey, - Salt: saltB64, - } - if saveErr := saveCryptoState(cryptoStateFile, state); saveErr != nil { - // 持久化失败仅记录警告,不阻止服务启动 - logger.Warnf(context.Background(), "[CryptoService] 无法持久化 crypto 状态到 %s: %v (重启后已加密数据将无法解密)", cryptoStateFile, saveErr) - } else { - logger.Infof(context.Background(), "[CryptoService] crypto 状态已持久化到 %s", cryptoStateFile) - } - - return svc, nil -} - -// loadCryptoState 从文件加载 crypto 状态 -func loadCryptoState(path string) (*cryptoState, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var state cryptoState - if err := json.Unmarshal(data, &state); err != nil { - return nil, fmt.Errorf("parse crypto state: %w", err) - } - if state.MasterKey == "" || state.Salt == "" { - return nil, fmt.Errorf("invalid crypto state: missing fields") - } - return &state, nil -} - -// saveCryptoState 将 crypto 状态写入文件 -func saveCryptoState(path string, state *cryptoState) error { - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return fmt.Errorf("mkdir: %w", err) - } - data, err := json.Marshal(state) - if err != nil { - return fmt.Errorf("marshal: %w", err) - } - // 权限 0o600:仅 owner 可读写 - return os.WriteFile(path, data, 0o600) -} - func initRedisClient() (*redis.Client, error) { redisAddr := os.Getenv("REDIS_ADDR") if redisAddr == "" { diff --git a/internal/infrastructure/crypto/config.go b/internal/infrastructure/crypto/config.go deleted file mode 100644 index f9db8c7e..00000000 --- a/internal/infrastructure/crypto/config.go +++ /dev/null @@ -1,224 +0,0 @@ -package crypto - -import ( - "encoding/base64" - "fmt" - "os" - "strconv" -) - -// Config 加密服务配置结构 -type Config struct { - // MasterKey 主密钥,用于派生加密密钥 - // 可以从环境变量或配置文件中读取 - MasterKey string `json:"master_key" yaml:"master_key" env:"CRYPTO_MASTER_KEY"` - - // Salt 盐值,用于密钥派生 - // 如果为空,将自动生成随机盐值 - Salt string `json:"salt" yaml:"salt" env:"CRYPTO_SALT"` - - // SaltLength 盐值长度(当Salt为空时使用) - SaltLength int `json:"salt_length" yaml:"salt_length" env:"CRYPTO_SALT_LENGTH" default:"16"` - - // KeyDerivationIterations PBKDF2迭代次数 - KeyDerivationIterations int `json:"key_derivation_iterations" yaml:"key_derivation_iterations" env:"CRYPTO_ITERATIONS" default:"10000"` - - // KeyLength 派生密钥长度 - KeyLength int `json:"key_length" yaml:"key_length" env:"CRYPTO_KEY_LENGTH" default:"32"` -} - -// DefaultConfig 返回默认配置 -func DefaultConfig() *Config { - return &Config{ - MasterKey: "", - Salt: "", - SaltLength: 16, - KeyDerivationIterations: 10000, - KeyLength: 32, - } -} - -// LoadConfigFromEnv 从环境变量加载配置 -func LoadConfigFromEnv() *Config { - config := DefaultConfig() - - // 从环境变量读取主密钥 - if masterKey := os.Getenv("CRYPTO_MASTER_KEY"); masterKey != "" { - config.MasterKey = masterKey - } - - // 从环境变量读取盐值 - if salt := os.Getenv("CRYPTO_SALT"); salt != "" { - config.Salt = salt - } - - // 从环境变量读取盐值长度 - if saltLengthStr := os.Getenv("CRYPTO_SALT_LENGTH"); saltLengthStr != "" { - if saltLength, err := strconv.Atoi(saltLengthStr); err == nil && saltLength >= 8 { - config.SaltLength = saltLength - } - } - - // 从环境变量读取迭代次数 - if iterationsStr := os.Getenv("CRYPTO_ITERATIONS"); iterationsStr != "" { - if iterations, err := strconv.Atoi(iterationsStr); err == nil && iterations > 0 { - config.KeyDerivationIterations = iterations - } - } - - // 从环境变量读取密钥长度 - if keyLengthStr := os.Getenv("CRYPTO_KEY_LENGTH"); keyLengthStr != "" { - if keyLength, err := strconv.Atoi(keyLengthStr); err == nil && keyLength >= 16 { - config.KeyLength = keyLength - } - } - - return config -} - -// Validate 验证配置是否有效 -func (c *Config) Validate() error { - if c.MasterKey == "" { - return fmt.Errorf("master key cannot be empty") - } - - if c.SaltLength < 8 { - return fmt.Errorf("salt length must be at least 8 bytes") - } - - if c.KeyDerivationIterations < 1000 { - return fmt.Errorf("key derivation iterations must be at least 1000") - } - - if c.KeyLength < 16 { - return fmt.Errorf("key length must be at least 16 bytes") - } - - return nil -} - -// GetSaltBytes 获取盐值的字节数组形式 -func (c *Config) GetSaltBytes() ([]byte, error) { - if c.Salt != "" { - // 如果配置了盐值,解码Base64格式 - saltBytes, err := base64.StdEncoding.DecodeString(c.Salt) - if err != nil { - return nil, fmt.Errorf("failed to decode salt from base64: %w", err) - } - return saltBytes, nil - } - - // 如果没有配置盐值,生成随机盐值 - salt, err := GenerateRandomSalt(c.SaltLength) - if err != nil { - return nil, fmt.Errorf("failed to generate random salt: %w", err) - } - - return salt, nil -} - -// NewCryptoServiceFromConfig 从配置创建加密服务 -func NewCryptoServiceFromConfig(config *Config) (*CryptoService, error) { - if err := config.Validate(); err != nil { - return nil, fmt.Errorf("invalid config: %w", err) - } - - saltBytes, err := config.GetSaltBytes() - if err != nil { - return nil, fmt.Errorf("failed to get salt bytes: %w", err) - } - - return NewCryptoService(config.MasterKey, saltBytes) -} - -// ExampleConfigUsage 展示配置使用示例 -func ExampleConfigUsage() { - fmt.Println("=== 配置使用示例 ===") - - // 方法1: 手动创建配置 - manualConfig := &Config{ - MasterKey: "my-secret-master-key", - Salt: "", // 自动生成盐值 - SaltLength: 16, - KeyDerivationIterations: 10000, - KeyLength: 32, - } - - cryptoService1, err := NewCryptoServiceFromConfig(manualConfig) - if err != nil { - fmt.Printf("手动配置创建失败: %v\n", err) - } else { - fmt.Println("手动配置创建成功") - _ = cryptoService1 - } - - // 方法2: 从环境变量加载配置 - // 首先设置环境变量 - os.Setenv("CRYPTO_MASTER_KEY", "env-master-key") - os.Setenv("CRYPTO_SALT", base64.StdEncoding.EncodeToString([]byte("env-salt-123456"))) - os.Setenv("CRYPTO_SALT_LENGTH", "16") - os.Setenv("CRYPTO_ITERATIONS", "10000") - os.Setenv("CRYPTO_KEY_LENGTH", "32") - - envConfig := LoadConfigFromEnv() - cryptoService2, err := NewCryptoServiceFromConfig(envConfig) - if err != nil { - fmt.Printf("环境变量配置创建失败: %v\n", err) - } else { - fmt.Println("环境变量配置创建成功") - _ = cryptoService2 - } - - // 清理环境变量 - os.Unsetenv("CRYPTO_MASTER_KEY") - os.Unsetenv("CRYPTO_SALT") - os.Unsetenv("CRYPTO_SALT_LENGTH") - os.Unsetenv("CRYPTO_ITERATIONS") - os.Unsetenv("CRYPTO_KEY_LENGTH") -} - -// GenerateConfigExample 生成配置示例 -func GenerateConfigExample() { - fmt.Println("\n=== 配置生成示例 ===") - - // 生成随机主密钥 - masterKey, err := GenerateRandomKey(32) - if err != nil { - fmt.Printf("生成主密钥失败: %v\n", err) - return - } - - // 生成随机盐值 - salt, err := GenerateRandomSalt(16) - if err != nil { - fmt.Printf("生成盐值失败: %v\n", err) - return - } - - saltBase64 := base64.StdEncoding.EncodeToString(salt) - - fmt.Println("生成的配置示例:") - fmt.Printf("CRYPTO_MASTER_KEY=%s\n", masterKey) - fmt.Printf("CRYPTO_SALT=%s\n", saltBase64) - fmt.Printf("CRYPTO_SALT_LENGTH=16\n") - fmt.Printf("CRYPTO_ITERATIONS=10000\n") - fmt.Printf("CRYPTO_KEY_LENGTH=32\n") - - fmt.Println("\nYAML格式配置示例:") - fmt.Printf(`master_key: %s -salt: %s -salt_length: 16 -key_derivation_iterations: 10000 -key_length: 32 -`, masterKey, saltBase64) - - fmt.Println("\nJSON格式配置示例:") - fmt.Printf(`{ - "master_key": "%s", - "salt": "%s", - "salt_length": 16, - "key_derivation_iterations": 10000, - "key_length": 32 -} -`, masterKey, saltBase64) -} diff --git a/internal/infrastructure/crypto/crypto.go b/internal/infrastructure/crypto/crypto.go deleted file mode 100644 index d2a47465..00000000 --- a/internal/infrastructure/crypto/crypto.go +++ /dev/null @@ -1,163 +0,0 @@ -package crypto - -import ( - "crypto/aes" - "crypto/cipher" - "crypto/rand" - "crypto/sha256" - "encoding/base64" - "errors" - "fmt" - - "golang.org/x/crypto/pbkdf2" -) - -// CryptoService 提供密钥加密解密服务 -type CryptoService struct { - masterKey []byte - salt []byte -} - -// NewCryptoService 创建新的加密服务实例 -// masterKey: 主密钥,用于派生加密密钥 -// salt: 盐值,用于密钥派生,如果为空则生成随机盐 -func NewCryptoService(masterKey string, salt []byte) (*CryptoService, error) { - if masterKey == "" { - return nil, errors.New("master key cannot be empty") - } - - // 如果未提供盐值,生成随机盐 - if salt == nil || len(salt) == 0 { - salt = make([]byte, 16) - if _, err := rand.Read(salt); err != nil { - return nil, fmt.Errorf("failed to generate salt: %w", err) - } - } - - return &CryptoService{ - masterKey: []byte(masterKey), - salt: salt, - }, nil -} - -// deriveKey 使用PBKDF2派生加密密钥 -func (cs *CryptoService) deriveKey() []byte { - return pbkdf2.Key(cs.masterKey, cs.salt, 10000, 32, sha256.New) -} - -// Encrypt 加密数据 -func (cs *CryptoService) Encrypt(plaintext []byte) (string, error) { - if len(plaintext) == 0 { - return "", errors.New("plaintext cannot be empty") - } - - key := cs.deriveKey() - - // 创建AES块密码 - block, err := aes.NewCipher(key) - if err != nil { - return "", fmt.Errorf("failed to create cipher: %w", err) - } - - // 生成随机IV - iv := make([]byte, aes.BlockSize) - if _, err := rand.Read(iv); err != nil { - return "", fmt.Errorf("failed to generate IV: %w", err) - } - - // 使用CTR模式加密 - stream := cipher.NewCTR(block, iv) - ciphertext := make([]byte, len(plaintext)) - stream.XORKeyStream(ciphertext, plaintext) - - // 组合IV和密文 - result := make([]byte, len(iv)+len(ciphertext)) - copy(result[:aes.BlockSize], iv) - copy(result[aes.BlockSize:], ciphertext) - - // 返回Base64编码的结果 - return base64.StdEncoding.EncodeToString(result), nil -} - -// Decrypt 解密数据 -func (cs *CryptoService) Decrypt(encryptedData string) ([]byte, error) { - if encryptedData == "" { - return nil, errors.New("encrypted data cannot be empty") - } - - // 解码Base64数据 - data, err := base64.StdEncoding.DecodeString(encryptedData) - if err != nil { - return nil, fmt.Errorf("failed to decode base64: %w", err) - } - - if len(data) < aes.BlockSize { - return nil, errors.New("encrypted data too short") - } - - key := cs.deriveKey() - - // 创建AES块密码 - block, err := aes.NewCipher(key) - if err != nil { - return nil, fmt.Errorf("failed to create cipher: %w", err) - } - - // 提取IV和密文 - iv := data[:aes.BlockSize] - ciphertext := data[aes.BlockSize:] - - // 使用CTR模式解密 - stream := cipher.NewCTR(block, iv) - plaintext := make([]byte, len(ciphertext)) - stream.XORKeyStream(plaintext, ciphertext) - - return plaintext, nil -} - -// EncryptString 加密字符串 -func (cs *CryptoService) EncryptString(plaintext string) (string, error) { - return cs.Encrypt([]byte(plaintext)) -} - -// DecryptString 解密字符串 -func (cs *CryptoService) DecryptString(encryptedData string) (string, error) { - plaintext, err := cs.Decrypt(encryptedData) - if err != nil { - return "", err - } - return string(plaintext), nil -} - -// GetSalt 获取当前使用的盐值 -func (cs *CryptoService) GetSalt() []byte { - return cs.salt -} - -// GenerateRandomKey 生成随机密钥 -func GenerateRandomKey(length int) (string, error) { - if length < 16 { - return "", errors.New("key length must be at least 16 bytes") - } - - key := make([]byte, length) - if _, err := rand.Read(key); err != nil { - return "", fmt.Errorf("failed to generate random key: %w", err) - } - - return base64.StdEncoding.EncodeToString(key), nil -} - -// GenerateRandomSalt 生成随机盐值 -func GenerateRandomSalt(length int) ([]byte, error) { - if length < 8 { - return nil, errors.New("salt length must be at least 8 bytes") - } - - salt := make([]byte, length) - if _, err := rand.Read(salt); err != nil { - return nil, fmt.Errorf("failed to generate random salt: %w", err) - } - - return salt, nil -} diff --git a/internal/models/embedding/weknoracloud.go b/internal/models/embedding/weknoracloud.go index 9dec9682..158d7af6 100644 --- a/internal/models/embedding/weknoracloud.go +++ b/internal/models/embedding/weknoracloud.go @@ -54,8 +54,9 @@ func NewWeKnoraCloudEmbedder(config Config) (*WeKnoraCloudEmbedder, error) { } type weKnoraCloudEmbedRequest struct { - Model string `json:"model"` - Input []string `json:"input"` + Model string `json:"model"` + Input []string `json:"input"` + TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"` } type weKnoraCloudEmbedResponse struct { @@ -77,7 +78,7 @@ func (e *WeKnoraCloudEmbedder) Embed(ctx context.Context, text string) ([]float3 } func (e *WeKnoraCloudEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) { - reqBody := weKnoraCloudEmbedRequest{Model: e.effectiveModelName(), Input: texts} + reqBody := weKnoraCloudEmbedRequest{Model: e.effectiveModelName(), Input: texts, TruncatePromptTokens: 512} bodyBytes, err := json.Marshal(reqBody) if err != nil { return nil, fmt.Errorf("weknoracloud embedder: marshal: %w", err) From 30f349d06addd52454e4db1f1e9162fe0a3f43f4 Mon Sep 17 00:00:00 2001 From: nullkey Date: Mon, 13 Apr 2026 20:43:17 +0800 Subject: [PATCH 3/4] feat(im): support custom endpoints for private WeCom deployments Allow configuring custom WebSocket and API base URLs for WeCom via optional credentials fields (ws_endpoint, api_base_url), enabling integration with privately deployed WeCom instances. Changes: - LongConnClient and WebhookAdapter accept custom endpoint URLs - Endpoint URLs validated at construction (scheme + SSRF check) - Private deployment hostnames added to file download SSRF allowlist - Frontend forms added for both WebSocket and Webhook modes - i18n support for zh-CN, en-US, ru-RU, ko-KR Uses default public cloud endpoints when not specified. --- frontend/src/components/IMChannelPanel.vue | 10 ++ frontend/src/i18n/locales/en-US.ts | 2 + frontend/src/i18n/locales/ko-KR.ts | 2 + frontend/src/i18n/locales/ru-RU.ts | 2 + frontend/src/i18n/locales/zh-CN.ts | 2 + internal/container/container.go | 7 +- internal/im/wecom/longconn.go | 32 +++-- internal/im/wecom/mention_test.go | 5 +- internal/im/wecom/webhook_adapter.go | 133 ++++++++++++++------- internal/im/wecom/ws_adapter.go | 2 +- 10 files changed, 144 insertions(+), 53 deletions(-) diff --git a/frontend/src/components/IMChannelPanel.vue b/frontend/src/components/IMChannelPanel.vue index 24668ab6..28f25f38 100644 --- a/frontend/src/components/IMChannelPanel.vue +++ b/frontend/src/components/IMChannelPanel.vue @@ -174,6 +174,11 @@
+
+ + +

{{ $t('agentEditor.im.wecomWSEndpointHint') }}

+
diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index cf7ff963..a245f1d2 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -3349,6 +3349,8 @@ export default { 'When on, bot replies are new top-level posts in the channel. When off (default), they stay in the thread and the main view only shows “N replies”.', modeHint: 'WebSocket is recommended for easier setup', consoleTip: 'to get credentials', + wecomWSEndpointHint: 'Optional. For private WeCom deployments, enter a custom WebSocket address. Leave empty to use the default public cloud endpoint. For internal network addresses, add the hostname to the SSRF_WHITELIST env var.', + wecomAPIBaseURLHint: 'Optional. For private WeCom deployments, enter a custom API base URL. Leave empty to use the default public cloud endpoint. For internal network addresses, add the hostname to the SSRF_WHITELIST env var.', fileKnowledgeBase: 'File Storage Knowledge Base', fileKnowledgeBasePlaceholder: 'Select a knowledge base (optional)', fileKnowledgeBaseHint: 'When configured, files sent by users will be automatically saved to this knowledge base', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 57200733..1c584aca 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -3376,6 +3376,8 @@ export default { "켜면 봇 답변이 채널의 새 최상위 게시물로 올라갑니다. 끄면(기본) 스레드 답변이며 메인 화면에는 「N개 답변」만 보입니다.", modeHint: "WebSocket 방식이 설정이 더 간편하여 권장됩니다", consoleTip: "자격 증명 정보를 가져오세요", + wecomWSEndpointHint: "선택 사항. 프라이빗 WeCom 배포 시 사용자 정의 WebSocket 주소를 입력하세요. 비워두면 기본 퍼블릭 클라우드 주소를 사용합니다. 내부 네트워크 주소의 경우 SSRF_WHITELIST 환경 변수에 호스트명을 추가하세요.", + wecomAPIBaseURLHint: "선택 사항. 프라이빗 WeCom 배포 시 사용자 정의 API 기본 URL을 입력하세요. 비워두면 기본 퍼블릭 클라우드 주소를 사용합니다. 내부 네트워크 주소의 경우 SSRF_WHITELIST 환경 변수에 호스트명을 추가하세요.", fileKnowledgeBase: "파일 저장 지식 베이스", fileKnowledgeBasePlaceholder: "지식 베이스 선택 (선택 사항)", fileKnowledgeBaseHint: "설정 시 사용자가 보낸 파일이 자동으로 해당 지식 베이스에 저장됩니다", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index ab81a6c3..94dee79f 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -3014,6 +3014,8 @@ export default { 'Включено — ответы бота как новые сообщения в канале. Выключено (по умолчанию) — ответы в ветке, в основном окне только «N ответов».', modeHint: 'Рекомендуется WebSocket — проще настроить', consoleTip: 'для получения учётных данных', + wecomWSEndpointHint: 'Необязательно. Для приватного развёртывания WeCom укажите пользовательский адрес WebSocket. Оставьте пустым для публичного облака. Для адресов внутренней сети добавьте имя хоста в переменную SSRF_WHITELIST.', + wecomAPIBaseURLHint: 'Необязательно. Для приватного развёртывания WeCom укажите пользовательский базовый URL API. Оставьте пустым для публичного облака. Для адресов внутренней сети добавьте имя хоста в переменную SSRF_WHITELIST.', fileKnowledgeBase: 'База знаний для файлов', fileKnowledgeBasePlaceholder: 'Выберите базу знаний (необязательно)', fileKnowledgeBaseHint: 'При настройке файлы, отправленные пользователями, автоматически сохраняются в эту базу знаний', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 75902aec..6bf46465 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -3324,6 +3324,8 @@ export default { "开启后 Bot 回复作为频道内新帖子;关闭(默认)则作为原消息的线程回复,主窗口仅显示「N 条回复」。", modeHint: "推荐使用 WebSocket 方式接入,配置更简单", consoleTip: "前往获取凭证信息", + wecomWSEndpointHint: "可选,私有化部署时填写自定义 WebSocket 地址,留空则使用默认公有云地址。内网地址需将域名加入 SSRF_WHITELIST 环境变量", + wecomAPIBaseURLHint: "可选,私有化部署时填写自定义 API 地址,留空则使用默认公有云地址。内网地址需将域名加入 SSRF_WHITELIST 环境变量", fileKnowledgeBase: "文件保存知识库", fileKnowledgeBasePlaceholder: "选择知识库(可选)", fileKnowledgeBaseHint: "配置后,用户发送的文件将自动保存到该知识库中", diff --git a/internal/container/container.go b/internal/container/container.go index 85880059..bc56bcce 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -1059,6 +1059,7 @@ func registerIMAdapterFactories(imService *imPkg.Service) { getString(creds, "token"), getString(creds, "encoding_aes_key"), corpAgentID, + getString(creds, "api_base_url"), ) if err != nil { return nil, nil, err @@ -1066,12 +1067,16 @@ func registerIMAdapterFactories(imService *imPkg.Service) { return adapter, nil, nil case "websocket": - client := wecom.NewLongConnClient( + client, err := wecom.NewLongConnClient( getString(creds, "bot_id"), getString(creds, "bot_secret"), + getString(creds, "ws_endpoint"), getString(creds, "bot_name"), msgHandler, ) + if err != nil { + return nil, nil, err + } wsCtx, wsCancel := context.WithCancel(context.Background()) go func() { diff --git a/internal/im/wecom/longconn.go b/internal/im/wecom/longconn.go index e9ead313..fe202ae5 100644 --- a/internal/im/wecom/longconn.go +++ b/internal/im/wecom/longconn.go @@ -27,7 +27,7 @@ import ( ) const ( - wecomWSEndpoint = "wss://openws.work.weixin.qq.com" + defaultWSEndpoint = "wss://openws.work.weixin.qq.com" cmdSubscribe = "aibot_subscribe" cmdPing = "ping" @@ -122,9 +122,11 @@ type MessageHandler func(ctx context.Context, msg *im.IncomingMessage) error // LongConnClient manages a WeCom intelligent bot WebSocket long connection. type LongConnClient struct { - botID string - secret string - handler MessageHandler + botID string + secret string + endpoint string + extraAllowedHost string // hostname from custom endpoint for SSRF allowlist + handler MessageHandler conn *ws.Conn mu sync.Mutex @@ -143,17 +145,27 @@ type LongConnClient struct { } // NewLongConnClient creates a WeCom long connection client. +// wsEndpoint overrides the default WebSocket URL; empty uses the public cloud endpoint. // botName is the bot's display name for @mention stripping; empty to auto-detect. -func NewLongConnClient(botID, secret, botName string, handler MessageHandler) *LongConnClient { +func NewLongConnClient(botID, secret, wsEndpoint, botName string, handler MessageHandler) (*LongConnClient, error) { + if wsEndpoint == "" { + wsEndpoint = defaultWSEndpoint + } + wsEndpoint = strings.TrimRight(wsEndpoint, "/") + if err := validateEndpointURL(wsEndpoint, defaultWSEndpoint, "wss"); err != nil { + return nil, fmt.Errorf("invalid ws_endpoint: %w", err) + } c := &LongConnClient{ - botID: botID, - secret: secret, - handler: handler, + botID: botID, + secret: secret, + endpoint: wsEndpoint, + extraAllowedHost: extraHostFromEndpoint(wsEndpoint, defaultWSEndpoint), + handler: handler, } if botName != "" { c.botDisplayName.Store(botName) } - return c + return c, nil } // Start connects and runs the long connection loop. It reconnects automatically on failure. @@ -345,7 +357,7 @@ func (c *LongConnClient) sendStreamFrame(incoming *im.IncomingMessage, streamID, } func (c *LongConnClient) connectAndRun(ctx context.Context) error { - conn, _, err := ws.DefaultDialer.DialContext(ctx, wecomWSEndpoint, nil) + conn, _, err := ws.DefaultDialer.DialContext(ctx, c.endpoint, nil) if err != nil { return fmt.Errorf("dial: %w", err) } diff --git a/internal/im/wecom/mention_test.go b/internal/im/wecom/mention_test.go index 51887fc5..62767fd8 100644 --- a/internal/im/wecom/mention_test.go +++ b/internal/im/wecom/mention_test.go @@ -85,7 +85,10 @@ func TestLongConnClient_StripAtMention(t *testing.T) { }) t.Run("NewLongConnClient with bot name", func(t *testing.T) { - c := NewLongConnClient("id", "secret", "My Bot", nil) + c, err := NewLongConnClient("id", "secret", "", "My Bot", nil) + if err != nil { + t.Fatal(err) + } got := c.stripAtMention("@My Bot /help") if got != "/help" { diff --git a/internal/im/wecom/webhook_adapter.go b/internal/im/wecom/webhook_adapter.go index a46bb20a..1166eb85 100644 --- a/internal/im/wecom/webhook_adapter.go +++ b/internal/im/wecom/webhook_adapter.go @@ -39,15 +39,52 @@ import ( var httpClient = &http.Client{Timeout: 30 * time.Second} +const defaultAPIBaseURL = "https://qyapi.weixin.qq.com" + +// extraHostFromEndpoint returns the lowercased hostname from endpoint if it +// differs from defaultEndpoint; otherwise returns "". Used to extend the SSRF +// allowlist for private deployments. +func extraHostFromEndpoint(endpoint, defaultEndpoint string) string { + if endpoint == "" || endpoint == defaultEndpoint { + return "" + } + if u, err := url.Parse(endpoint); err == nil { + return strings.ToLower(u.Hostname()) + } + return "" +} + +// validateEndpointURL checks that a custom endpoint URL uses a secure scheme +// and does not point to a private/internal address, preventing accidental +// credential leakage (e.g. access tokens sent to a rogue server). +func validateEndpointURL(endpoint, defaultEndpoint, requiredScheme string) error { + if endpoint == "" || endpoint == defaultEndpoint { + return nil + } + u, err := url.Parse(endpoint) + if err != nil { + return fmt.Errorf("invalid endpoint URL: %w", err) + } + if u.Scheme != requiredScheme { + return fmt.Errorf("endpoint must use %s:// scheme, got %s://", requiredScheme, u.Scheme) + } + if err := secutils.ValidateURLForSSRF(endpoint); err != nil { + return fmt.Errorf("%w (for private deployments on internal networks, add the hostname to SSRF_WHITELIST)", err) + } + return nil +} + // WebhookAdapter implements im.Adapter for WeCom in webhook (self-built app callback) mode. // Messages arrive via HTTP callback; replies are sent via the WeCom REST API. type WebhookAdapter struct { - corpID string - token string - encodingAESKey string - aesKey []byte - agentSecret string - corpAgentID int + corpID string + token string + encodingAESKey string + aesKey []byte + agentSecret string + corpAgentID int + apiBaseURL string // WeCom API base URL (e.g. "https://qyapi.weixin.qq.com") + extraAllowedHost string // hostname from apiBaseURL for SSRF allowlist (empty if default) // Token cache tokenMu sync.Mutex @@ -59,20 +96,32 @@ type WebhookAdapter struct { var _ im.FileDownloader = (*WebhookAdapter)(nil) // NewWebhookAdapter creates a new WeCom webhook adapter. -func NewWebhookAdapter(corpID, agentSecret, token, encodingAESKey string, corpAgentID int) (*WebhookAdapter, error) { +// apiBaseURL overrides the default WeCom API base URL; empty uses the public cloud endpoint. +func NewWebhookAdapter(corpID, agentSecret, token, encodingAESKey string, corpAgentID int, apiBaseURL string) (*WebhookAdapter, error) { // Decode the AES key from base64 aesKey, err := base64.StdEncoding.DecodeString(encodingAESKey + "=") if err != nil { return nil, fmt.Errorf("decode encoding_aes_key: %w", err) } + if apiBaseURL == "" { + apiBaseURL = defaultAPIBaseURL + } + apiBaseURL = strings.TrimRight(apiBaseURL, "/") + + if err := validateEndpointURL(apiBaseURL, defaultAPIBaseURL, "https"); err != nil { + return nil, fmt.Errorf("invalid api_base_url: %w", err) + } + return &WebhookAdapter{ - corpID: corpID, - token: token, - encodingAESKey: encodingAESKey, - aesKey: aesKey, - agentSecret: agentSecret, - corpAgentID: corpAgentID, + corpID: corpID, + token: token, + encodingAESKey: encodingAESKey, + aesKey: aesKey, + agentSecret: agentSecret, + corpAgentID: corpAgentID, + apiBaseURL: apiBaseURL, + extraAllowedHost: extraHostFromEndpoint(apiBaseURL, defaultAPIBaseURL), }, nil } @@ -257,7 +306,7 @@ func (a *WebhookAdapter) sendToAppChat(ctx context.Context, accessToken, chatID return fmt.Errorf("marshal payload: %w", err) } - sendURL := fmt.Sprintf("https://qyapi.weixin.qq.com/cgi-bin/appchat/send?access_token=%s", accessToken) + sendURL := fmt.Sprintf("%s/cgi-bin/appchat/send?access_token=%s", a.apiBaseURL, accessToken) req, err := http.NewRequestWithContext(ctx, http.MethodPost, sendURL, bytes.NewReader(payloadBytes)) if err != nil { return fmt.Errorf("create request: %w", err) @@ -301,7 +350,7 @@ func (a *WebhookAdapter) sendToUser(ctx context.Context, accessToken, userID str return fmt.Errorf("marshal payload: %w", err) } - sendURL := fmt.Sprintf("https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=%s", accessToken) + sendURL := fmt.Sprintf("%s/cgi-bin/message/send?access_token=%s", a.apiBaseURL, accessToken) req, err := http.NewRequestWithContext(ctx, http.MethodPost, sendURL, bytes.NewReader(payloadBytes)) if err != nil { return fmt.Errorf("create request: %w", err) @@ -338,8 +387,8 @@ func (a *WebhookAdapter) getAccessToken(ctx context.Context) (string, error) { return a.tokenCache, nil } - tokenURL := fmt.Sprintf("https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid=%s&corpsecret=%s", - a.corpID, a.agentSecret) + tokenURL := fmt.Sprintf("%s/cgi-bin/gettoken?corpid=%s&corpsecret=%s", + a.apiBaseURL, a.corpID, a.agentSecret) req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL, nil) if err != nil { @@ -460,10 +509,10 @@ type wecomMessage struct { CreateTime int64 `xml:"CreateTime"` MsgType string `xml:"MsgType"` Content string `xml:"Content"` // text - PicUrl string `xml:"PicUrl"` // image: download URL - MediaId string `xml:"MediaId"` // image/voice/video: media ID for download - Format string `xml:"Format"` // voice: audio format (amr/speex) - ThumbMediaId string `xml:"ThumbMediaId"` // video: thumbnail media ID + PicUrl string `xml:"PicUrl"` // image: download URL + MediaId string `xml:"MediaId"` // image/voice/video: media ID for download + Format string `xml:"Format"` // voice: audio format (amr/speex) + ThumbMediaId string `xml:"ThumbMediaId"` // video: thumbnail media ID MsgID string `xml:"MsgId"` AgentID string `xml:"AgentID"` ChatID string `xml:"ChatId"` @@ -488,7 +537,7 @@ func (a *WebhookAdapter) DownloadFile(ctx context.Context, msg *im.IncomingMessa // If FileKey looks like a URL, download directly if strings.HasPrefix(msg.FileKey, "http://") || strings.HasPrefix(msg.FileKey, "https://") { - return downloadFromURL(ctx, msg.FileKey, fileName) + return downloadFromURL(ctx, msg.FileKey, fileName, a.extraAllowedHost) } // Otherwise treat as media_id, download via temporary media API @@ -497,9 +546,9 @@ func (a *WebhookAdapter) DownloadFile(ctx context.Context, msg *im.IncomingMessa return nil, "", fmt.Errorf("get access token: %w", err) } - apiURL := fmt.Sprintf("https://qyapi.weixin.qq.com/cgi-bin/media/get?access_token=%s&media_id=%s", - accessToken, msg.FileKey) - return downloadFromURL(ctx, apiURL, fileName) + apiURL := fmt.Sprintf("%s/cgi-bin/media/get?access_token=%s&media_id=%s", + a.apiBaseURL, accessToken, msg.FileKey) + return downloadFromURL(ctx, apiURL, fileName, a.extraAllowedHost) } // downloadFromURL performs a GET request and returns the response body. @@ -507,9 +556,9 @@ func (a *WebhookAdapter) DownloadFile(ctx context.Context, msg *im.IncomingMessa // 1. Content-Disposition: attachment; filename="xxx.pdf" // 2. Content-Type → extension mapping (fallback for platforms like WeCom that // don't provide the original filename in the callback JSON) -func downloadFromURL(ctx context.Context, rawURL, fileName string) (io.ReadCloser, string, error) { +func downloadFromURL(ctx context.Context, rawURL, fileName string, extraAllowedHost string) (io.ReadCloser, string, error) { // SSRF protection: reject internal/private URLs unless on the WeCom API allowlist. - if !isAllowedIMAPIHost(rawURL) { + if !isAllowedIMAPIHost(rawURL, extraAllowedHost) { if err := secutils.ValidateURLForSSRF(rawURL); err != nil { return nil, "", fmt.Errorf("URL rejected for security reasons: %v", err) } @@ -600,12 +649,16 @@ var allowedIMAPIHosts = []string{ } // isAllowedIMAPIHost returns true if rawURL points to a known IM platform API host. -func isAllowedIMAPIHost(rawURL string) bool { +// extraHost is an optional additional trusted hostname (e.g. from a private deployment). +func isAllowedIMAPIHost(rawURL string, extraHost string) bool { u, err := url.Parse(rawURL) if err != nil { return false } hostname := strings.ToLower(u.Hostname()) + if extraHost != "" && hostname == extraHost { + return true + } for _, allowed := range allowedIMAPIHosts { if hostname == allowed { return true @@ -623,20 +676,20 @@ func contentTypeToExt(ct string) string { ct = strings.ToLower(ct) mapping := map[string]string{ - "application/pdf": "pdf", - "application/msword": "doc", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", - "application/vnd.ms-excel": "xls", + "application/pdf": "pdf", + "application/msword": "doc", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx", + "application/vnd.ms-excel": "xls", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx", - "application/vnd.ms-powerpoint": "ppt", + "application/vnd.ms-powerpoint": "ppt", "application/vnd.openxmlformats-officedocument.presentationml.presentation": "pptx", - "text/plain": "txt", - "text/markdown": "md", - "text/csv": "csv", - "image/png": "png", - "image/jpeg": "jpg", - "image/gif": "gif", - "image/webp": "webp", + "text/plain": "txt", + "text/markdown": "md", + "text/csv": "csv", + "image/png": "png", + "image/jpeg": "jpg", + "image/gif": "gif", + "image/webp": "webp", } return mapping[ct] diff --git a/internal/im/wecom/ws_adapter.go b/internal/im/wecom/ws_adapter.go index 4cf686e9..a0dcb423 100644 --- a/internal/im/wecom/ws_adapter.go +++ b/internal/im/wecom/ws_adapter.go @@ -82,7 +82,7 @@ func (a *WSAdapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) ( } // Download the (encrypted) file content - reader, fileName, err := downloadFromURL(ctx, msg.FileKey, fileName) + reader, fileName, err := downloadFromURL(ctx, msg.FileKey, fileName, a.client.extraAllowedHost) if err != nil { return nil, "", err } From 9edf0839ed2ee2fd04e45f3309acefefac6119dc Mon Sep 17 00:00:00 2001 From: "ochan.kwon" Date: Fri, 10 Apr 2026 09:32:36 +0900 Subject: [PATCH 4/4] feat: add VectorStore service layer with types and connection testing Add VectorStoreService with CRUD validation, duplicate checking (DB + env stores), and engine-specific health checks for 6 vector database types. Include VectorStoreResponse DTO, env store builder, engine type metadata, and comprehensive unit tests. --- .../application/repository/vectorstore.go | 9 + internal/application/service/vectorstore.go | 121 ++++ .../service/vectorstore_healthcheck.go | 228 ++++++++ .../application/service/vectorstore_test.go | 525 ++++++++++++++++++ internal/types/interfaces/vectorstore.go | 18 + internal/types/vectorstore.go | 266 +++++++++ internal/types/vectorstore_test.go | 239 ++++++++ 7 files changed, 1406 insertions(+) create mode 100644 internal/application/service/vectorstore.go create mode 100644 internal/application/service/vectorstore_healthcheck.go create mode 100644 internal/application/service/vectorstore_test.go diff --git a/internal/application/repository/vectorstore.go b/internal/application/repository/vectorstore.go index 183a1ad1..09bb6230 100644 --- a/internal/application/repository/vectorstore.go +++ b/internal/application/repository/vectorstore.go @@ -59,6 +59,15 @@ func (r *vectorStoreRepository) Update(ctx context.Context, store *types.VectorS ).Select("name").Updates(store).Error } +// UpdateConnectionConfig updates only the connection_config JSONB column. +// Used for saving auto-detected metadata (e.g., server version) without +// touching user-immutable fields like engine_type or index_config. +func (r *vectorStoreRepository) UpdateConnectionConfig(ctx context.Context, store *types.VectorStore) error { + return r.db.WithContext(ctx).Model(&types.VectorStore{}).Where( + "id = ? AND tenant_id = ?", store.ID, store.TenantID, + ).Select("connection_config").Updates(store).Error +} + // Delete soft-deletes a vector store func (r *vectorStoreRepository) Delete(ctx context.Context, tenantID uint64, id string) error { return r.db.WithContext(ctx).Where( diff --git a/internal/application/service/vectorstore.go b/internal/application/service/vectorstore.go new file mode 100644 index 00000000..4bc91a0b --- /dev/null +++ b/internal/application/service/vectorstore.go @@ -0,0 +1,121 @@ +package service + +import ( + "context" + "os" + + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +// vectorStoreService implements interfaces.VectorStoreService +type vectorStoreService struct { + repo interfaces.VectorStoreRepository +} + +// NewVectorStoreService creates a new vector store service +func NewVectorStoreService( + repo interfaces.VectorStoreRepository, +) interfaces.VectorStoreService { + return &vectorStoreService{repo: repo} +} + +// CreateStore validates and creates a new vector store. +func (s *vectorStoreService) CreateStore(ctx context.Context, store *types.VectorStore) error { + // 1. Basic validation (name, engine_type, tenant_id) + if err := store.Validate(); err != nil { + return err + } + + // 2. Engine-specific connection config validation + if err := validateConnectionConfig(store.EngineType, store.ConnectionConfig); err != nil { + return err + } + + // 3. Duplicate check — DB stores + endpoint := store.ConnectionConfig.GetEndpoint() + indexName := store.IndexConfig.GetIndexNameOrDefault(store.EngineType) + + exists, err := s.repo.ExistsByEndpointAndIndex(ctx, store.TenantID, store.EngineType, endpoint, indexName) + if err != nil { + return errors.NewInternalServerError("failed to check for duplicates") + } + if exists { + return errors.NewConflictError("a vector store with the same endpoint and index already exists") + } + + // 4. Duplicate check — env stores (pure function, no os.Getenv in types) + for _, envStore := range types.BuildEnvVectorStores(os.Getenv("RETRIEVE_DRIVER"), os.Getenv) { + if envStore.EngineType == store.EngineType && + envStore.ConnectionConfig.GetEndpoint() == endpoint && + envStore.IndexConfig.GetIndexNameOrDefault(store.EngineType) == indexName { + return errors.NewConflictError( + "a vector store with the same endpoint and index is already configured via environment variables") + } + } + + // 5. Persist + logger.Infof(ctx, "Creating vector store: tenant=%d, name=%s, engine=%s", + store.TenantID, secutils.SanitizeForLog(store.Name), store.EngineType) + return s.repo.Create(ctx, store) +} + +// UpdateStore updates an existing vector store (name only). +func (s *vectorStoreService) UpdateStore(ctx context.Context, store *types.VectorStore) error { + if store.TenantID == 0 { + return errors.NewValidationError("tenant_id is required") + } + if store.Name == "" { + return errors.NewValidationError("name is required") + } + + logger.Infof(ctx, "Updating vector store: tenant=%d, id=%s", store.TenantID, store.ID) + return s.repo.Update(ctx, store) +} + +// DeleteStore deletes a vector store by tenant + id. +// Phase 2: KB binding check will be added here. +func (s *vectorStoreService) DeleteStore(ctx context.Context, tenantID uint64, id string) error { + logger.Infof(ctx, "Deleting vector store: tenant=%d, id=%s", tenantID, id) + return s.repo.Delete(ctx, tenantID, id) +} + +// SaveDetectedVersion updates the connection_config.version for a stored vector store. +// Works on a copy to avoid mutating the caller's object. +func (s *vectorStoreService) SaveDetectedVersion(ctx context.Context, store *types.VectorStore, version string) error { + updated := *store + updated.ConnectionConfig.Version = version + return s.repo.UpdateConnectionConfig(ctx, &updated) +} + +// validateConnectionConfig validates required fields per engine type. +func validateConnectionConfig(engineType types.RetrieverEngineType, config types.ConnectionConfig) error { + switch engineType { + case types.ElasticsearchRetrieverEngineType: + if config.Addr == "" { + return errors.NewValidationError("addr is required for elasticsearch") + } + case types.PostgresRetrieverEngineType: + if !config.UseDefaultConnection && config.Addr == "" { + return errors.NewValidationError("addr or use_default_connection is required for postgres") + } + case types.QdrantRetrieverEngineType: + if config.Host == "" { + return errors.NewValidationError("host is required for qdrant") + } + case types.MilvusRetrieverEngineType: + if config.Addr == "" { + return errors.NewValidationError("addr is required for milvus") + } + case types.WeaviateRetrieverEngineType: + if config.Host == "" { + return errors.NewValidationError("host is required for weaviate") + } + case types.SQLiteRetrieverEngineType: + // No connection config needed for SQLite + } + return nil +} diff --git a/internal/application/service/vectorstore_healthcheck.go b/internal/application/service/vectorstore_healthcheck.go new file mode 100644 index 00000000..3fb3a7bf --- /dev/null +++ b/internal/application/service/vectorstore_healthcheck.go @@ -0,0 +1,228 @@ +package service + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "time" + + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + _ "github.com/jackc/pgx/v5/stdlib" // pgx driver for database/sql + "github.com/qdrant/go-client/qdrant" + "github.com/weaviate/weaviate-go-client/v5/weaviate" + "github.com/weaviate/weaviate-go-client/v5/weaviate/auth" + wgrpc "github.com/weaviate/weaviate-go-client/v5/weaviate/grpc" +) + +const connectionTestTimeout = 10 * time.Second + +// TestConnection tests connectivity to a vector database. +// Returns the detected server version on success (e.g., "7.10.1"), empty string if unknown. +func (s *vectorStoreService) TestConnection( + ctx context.Context, + engineType types.RetrieverEngineType, + config types.ConnectionConfig, +) (string, error) { + switch engineType { + case types.ElasticsearchRetrieverEngineType: + return testElasticsearchConnection(ctx, config) + case types.PostgresRetrieverEngineType: + return testPostgresConnection(ctx, config) + case types.QdrantRetrieverEngineType: + return testQdrantConnection(ctx, config) + case types.MilvusRetrieverEngineType: + return testMilvusConnection(ctx, config) + case types.WeaviateRetrieverEngineType: + return testWeaviateConnection(ctx, config) + case types.SQLiteRetrieverEngineType: + // SQLite is file-based, no remote connection to test + return "", nil + default: + return "", errors.NewBadRequestError( + fmt.Sprintf("connection test not supported for engine type: %s", engineType)) + } +} + +func testElasticsearchConnection(ctx context.Context, config types.ConnectionConfig) (string, error) { + // Use plain HTTP GET to the root endpoint instead of the go-elasticsearch SDK. + // The v8 SDK's TypedClient performs a product check that rejects ES7 servers, + // so we use a raw HTTP request to support both v7 and v8. + req, err := http.NewRequestWithContext(ctx, http.MethodGet, config.Addr, nil) + if err != nil { + return "", errors.NewBadRequestError("failed to create elasticsearch request: invalid address") + } + if config.Username != "" { + req.SetBasicAuth(config.Username, config.Password) + } + + client := &http.Client{Timeout: connectionTestTimeout} + resp, err := client.Do(req) + if err != nil { + logger.Warnf(ctx, "Elasticsearch connection test failed: %v", err) + return "", errors.NewBadRequestError("failed to connect to elasticsearch: connection refused or authentication failed") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + logger.Warnf(ctx, "Elasticsearch connection test returned status %d", resp.StatusCode) + return "", errors.NewBadRequestError("failed to connect to elasticsearch: authentication failed or server error") + } + + // Parse version from response: {"version": {"number": "7.10.1"}, ...} + body, err := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if err != nil { + return "", nil // connected but version unknown + } + + var esInfo struct { + Version struct { + Number string `json:"number"` + } `json:"version"` + } + if err := json.Unmarshal(body, &esInfo); err != nil { + return "", nil // connected but version unparseable + } + + return esInfo.Version.Number, nil +} + +func testPostgresConnection(ctx context.Context, config types.ConnectionConfig) (string, error) { + testCtx, cancel := context.WithTimeout(ctx, connectionTestTimeout) + defer cancel() + + if config.UseDefaultConnection { + // Using the default app DB connection — always reachable if the app is running. + // Cannot query version without a DB handle; return empty. + return "", nil + } + + db, err := sql.Open("pgx", config.Addr) + if err != nil { + return "", errors.NewBadRequestError("failed to create postgres connection: invalid configuration") + } + defer db.Close() + + if err := db.PingContext(testCtx); err != nil { + logger.Warnf(ctx, "Postgres connection test failed: %v", err) + return "", errors.NewBadRequestError("failed to connect to postgres: connection refused or authentication failed") + } + + // Detect version + var version string + if err := db.QueryRowContext(testCtx, "SHOW server_version").Scan(&version); err != nil { + logger.Warnf(ctx, "Postgres version detection failed: %v", err) + return "", nil // connected but version unknown + } + + return version, nil +} + +func testQdrantConnection(ctx context.Context, config types.ConnectionConfig) (string, error) { + testCtx, cancel := context.WithTimeout(ctx, connectionTestTimeout) + defer cancel() + + port := config.Port + if port == 0 { + port = 6334 + } + + client, err := qdrant.NewClient(&qdrant.Config{ + Host: config.Host, + Port: port, + APIKey: config.APIKey, + UseTLS: config.UseTLS, + }) + if err != nil { + return "", errors.NewBadRequestError("failed to create qdrant client: invalid configuration") + } + defer client.Close() + + result, err := client.HealthCheck(testCtx) + if err != nil { + logger.Warnf(ctx, "Qdrant connection test failed: %v", err) + return "", errors.NewBadRequestError("failed to connect to qdrant: connection refused or authentication failed") + } + + return result.GetVersion(), nil +} + +func testMilvusConnection(ctx context.Context, config types.ConnectionConfig) (string, error) { + // Use TCP dial instead of the Milvus SDK to avoid protobuf namespace conflict + // between milvus-proto and qdrant-client (both register "common.proto"). + // A TCP dial is sufficient for connectivity verification; the Milvus SDK client + // creation in container.go (PR 3) will validate full gRPC connectivity. + // Version detection is not possible with TCP dial alone. + testCtx, cancel := context.WithTimeout(ctx, connectionTestTimeout) + defer cancel() + + addr := config.Addr + if addr == "" { + addr = "localhost:19530" + } + + conn, err := (&net.Dialer{}).DialContext(testCtx, "tcp", addr) + if err != nil { + logger.Warnf(ctx, "Milvus connection test failed: %v", err) + return "", errors.NewBadRequestError("failed to connect to milvus: connection refused or server unreachable") + } + defer conn.Close() + + return "", nil +} + +func testWeaviateConnection(ctx context.Context, config types.ConnectionConfig) (string, error) { + testCtx, cancel := context.WithTimeout(ctx, connectionTestTimeout) + defer cancel() + + host := config.Host + if host == "" { + host = "weaviate:8080" + } + grpcAddress := config.GrpcAddress + if grpcAddress == "" { + grpcAddress = "weaviate:50051" + } + scheme := config.Scheme + if scheme == "" { + scheme = "http" + } + + weaviateCfg := weaviate.Config{ + Host: host, + GrpcConfig: &wgrpc.Config{ + Host: grpcAddress, + }, + Scheme: scheme, + } + if config.APIKey != "" { + weaviateCfg.AuthConfig = auth.ApiKey{Value: config.APIKey} + } + + // Weaviate Go client v5 does not expose a Close() method; + // it uses HTTP + gRPC transports that are managed internally. + client, err := weaviate.NewClient(weaviateCfg) + if err != nil { + logger.Warnf(ctx, "Weaviate connection test failed: %v", err) + return "", errors.NewBadRequestError("failed to create weaviate client: invalid configuration") + } + + isReady, err := client.Misc().ReadyChecker().Do(testCtx) + if err != nil || !isReady { + logger.Warnf(ctx, "Weaviate connection test failed: ready=%v, err=%v", isReady, err) + return "", errors.NewBadRequestError("failed to connect to weaviate: server not ready or authentication failed") + } + + // Detect version via /v1/meta + meta, err := client.Misc().MetaGetter().Do(testCtx) + if err != nil || meta == nil { + return "", nil // connected but version unknown + } + + return meta.Version, nil +} diff --git a/internal/application/service/vectorstore_test.go b/internal/application/service/vectorstore_test.go new file mode 100644 index 00000000..00cd5c17 --- /dev/null +++ b/internal/application/service/vectorstore_test.go @@ -0,0 +1,525 @@ +package service + +import ( + "context" + "testing" + + "github.com/Tencent/WeKnora/internal/errors" + "github.com/Tencent/WeKnora/internal/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// --------------------------------------------------------------------------- +// Mock repository +// --------------------------------------------------------------------------- + +type mockVectorStoreRepo struct { + stores []*types.VectorStore + createErr error + updateErr error + deleteErr error + existsByEndpointErr error + existsByEndpoint bool +} + +func (m *mockVectorStoreRepo) Create(_ context.Context, store *types.VectorStore) error { + if m.createErr != nil { + return m.createErr + } + m.stores = append(m.stores, store) + return nil +} + +func (m *mockVectorStoreRepo) GetByID(_ context.Context, tenantID uint64, id string) (*types.VectorStore, error) { + for _, s := range m.stores { + if s.ID == id && s.TenantID == tenantID { + return s, nil + } + } + return nil, nil +} + +func (m *mockVectorStoreRepo) List(_ context.Context, tenantID uint64) ([]*types.VectorStore, error) { + var result []*types.VectorStore + for _, s := range m.stores { + if s.TenantID == tenantID { + result = append(result, s) + } + } + return result, nil +} + +func (m *mockVectorStoreRepo) Update(_ context.Context, store *types.VectorStore) error { + return m.updateErr +} + +func (m *mockVectorStoreRepo) UpdateConnectionConfig(_ context.Context, _ *types.VectorStore) error { + return m.updateErr +} + +func (m *mockVectorStoreRepo) Delete(_ context.Context, _ uint64, _ string) error { + return m.deleteErr +} + +func (m *mockVectorStoreRepo) ExistsByEndpointAndIndex( + _ context.Context, _ uint64, _ types.RetrieverEngineType, _ string, _ string, +) (bool, error) { + if m.existsByEndpointErr != nil { + return false, m.existsByEndpointErr + } + return m.existsByEndpoint, nil +} + +// --------------------------------------------------------------------------- +// CreateStore tests +// --------------------------------------------------------------------------- + +func TestCreateStore_Success(t *testing.T) { + repo := &mockVectorStoreRepo{} + svc := NewVectorStoreService(repo) + + store := &types.VectorStore{ + TenantID: 1, + Name: "test-es", + EngineType: types.ElasticsearchRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{ + Addr: "http://es:9200", + }, + } + + err := svc.CreateStore(context.Background(), store) + assert.NoError(t, err) + assert.Len(t, repo.stores, 1) +} + +func TestCreateStore_ValidationError(t *testing.T) { + repo := &mockVectorStoreRepo{} + svc := NewVectorStoreService(repo) + + tests := []struct { + name string + store *types.VectorStore + }{ + { + name: "empty name", + store: &types.VectorStore{TenantID: 1, EngineType: types.PostgresRetrieverEngineType}, + }, + { + name: "invalid engine type", + store: &types.VectorStore{TenantID: 1, Name: "test", EngineType: "unknown"}, + }, + { + name: "zero tenant ID", + store: &types.VectorStore{Name: "test", EngineType: types.PostgresRetrieverEngineType}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := svc.CreateStore(context.Background(), tt.store) + require.Error(t, err) + var appErr *errors.AppError + assert.ErrorAs(t, err, &appErr) + }) + } +} + +func TestCreateStore_ConnectionConfigValidation(t *testing.T) { + repo := &mockVectorStoreRepo{} + svc := NewVectorStoreService(repo) + + tests := []struct { + name string + store *types.VectorStore + wantError bool + }{ + { + name: "elasticsearch without addr", + store: &types.VectorStore{ + TenantID: 1, Name: "test", + EngineType: types.ElasticsearchRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{}, + }, + wantError: true, + }, + { + name: "postgres without addr or default connection", + store: &types.VectorStore{ + TenantID: 1, Name: "test", + EngineType: types.PostgresRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{}, + }, + wantError: true, + }, + { + name: "postgres with use_default_connection", + store: &types.VectorStore{ + TenantID: 1, Name: "test", + EngineType: types.PostgresRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{UseDefaultConnection: true}, + }, + wantError: false, + }, + { + name: "qdrant without host", + store: &types.VectorStore{ + TenantID: 1, Name: "test", + EngineType: types.QdrantRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{}, + }, + wantError: true, + }, + { + name: "milvus without addr", + store: &types.VectorStore{ + TenantID: 1, Name: "test", + EngineType: types.MilvusRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{}, + }, + wantError: true, + }, + { + name: "weaviate without host", + store: &types.VectorStore{ + TenantID: 1, Name: "test", + EngineType: types.WeaviateRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{}, + }, + wantError: true, + }, + { + name: "sqlite with no config (ok)", + store: &types.VectorStore{ + TenantID: 1, Name: "test", + EngineType: types.SQLiteRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{}, + }, + wantError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := svc.CreateStore(context.Background(), tt.store) + if tt.wantError { + require.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestCreateStore_DuplicateCheck_DBStore(t *testing.T) { + repo := &mockVectorStoreRepo{existsByEndpoint: true} + svc := NewVectorStoreService(repo) + + store := &types.VectorStore{ + TenantID: 1, + Name: "dup-store", + EngineType: types.ElasticsearchRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{ + Addr: "http://es:9200", + }, + } + + err := svc.CreateStore(context.Background(), store) + require.Error(t, err) + + var appErr *errors.AppError + require.ErrorAs(t, err, &appErr) + assert.Equal(t, errors.ErrConflict, appErr.Code) +} + +func TestCreateStore_DuplicateCheck_DBError(t *testing.T) { + repo := &mockVectorStoreRepo{ + existsByEndpointErr: assert.AnError, + } + svc := NewVectorStoreService(repo) + + store := &types.VectorStore{ + TenantID: 1, + Name: "test", + EngineType: types.ElasticsearchRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{ + Addr: "http://es:9200", + }, + } + + err := svc.CreateStore(context.Background(), store) + require.Error(t, err) +} + +func TestCreateStore_DuplicateCheck_EnvStore(t *testing.T) { + // Set up env to simulate an existing elasticsearch env store + t.Setenv("RETRIEVE_DRIVER", "elasticsearch_v8") + t.Setenv("ELASTICSEARCH_ADDR", "http://es:9200") + t.Setenv("ELASTICSEARCH_USERNAME", "elastic") + t.Setenv("ELASTICSEARCH_PASSWORD", "secret") + t.Setenv("ELASTICSEARCH_INDEX", "xwrag_default") + + repo := &mockVectorStoreRepo{existsByEndpoint: false} // no DB duplicate + svc := NewVectorStoreService(repo) + + store := &types.VectorStore{ + TenantID: 1, + Name: "dup-env-store", + EngineType: types.ElasticsearchRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{ + Addr: "http://es:9200", + }, + IndexConfig: types.IndexConfig{ + IndexName: "xwrag_default", + }, + } + + err := svc.CreateStore(context.Background(), store) + require.Error(t, err) + + var appErr *errors.AppError + require.ErrorAs(t, err, &appErr) + assert.Equal(t, errors.ErrConflict, appErr.Code) + assert.Contains(t, appErr.Error(), "environment variables") +} + +func TestCreateStore_DuplicateCheck_EnvStore_DifferentIndex_Allowed(t *testing.T) { + // Same endpoint as env store but different index — should be allowed + t.Setenv("RETRIEVE_DRIVER", "elasticsearch_v8") + t.Setenv("ELASTICSEARCH_ADDR", "http://es:9200") + t.Setenv("ELASTICSEARCH_INDEX", "xwrag_default") + + repo := &mockVectorStoreRepo{existsByEndpoint: false} + svc := NewVectorStoreService(repo) + + store := &types.VectorStore{ + TenantID: 1, + Name: "different-index", + EngineType: types.ElasticsearchRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{ + Addr: "http://es:9200", + }, + IndexConfig: types.IndexConfig{ + IndexName: "different_index", + }, + } + + err := svc.CreateStore(context.Background(), store) + assert.NoError(t, err) +} + +func TestCreateStore_DifferentEndpointSameIndex_Allowed(t *testing.T) { + repo := &mockVectorStoreRepo{existsByEndpoint: false} + svc := NewVectorStoreService(repo) + + store := &types.VectorStore{ + TenantID: 1, + Name: "new-store", + EngineType: types.ElasticsearchRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{ + Addr: "http://es-new:9200", + }, + IndexConfig: types.IndexConfig{ + IndexName: "shared_index", + }, + } + + err := svc.CreateStore(context.Background(), store) + assert.NoError(t, err) +} + +// --------------------------------------------------------------------------- +// UpdateStore tests +// --------------------------------------------------------------------------- + +func TestUpdateStore_Success(t *testing.T) { + repo := &mockVectorStoreRepo{} + svc := NewVectorStoreService(repo) + + store := &types.VectorStore{ + ID: "test-id", + TenantID: 1, + Name: "updated-name", + } + + err := svc.UpdateStore(context.Background(), store) + assert.NoError(t, err) +} + +func TestUpdateStore_ValidationError(t *testing.T) { + repo := &mockVectorStoreRepo{} + svc := NewVectorStoreService(repo) + + tests := []struct { + name string + store *types.VectorStore + }{ + { + name: "empty name", + store: &types.VectorStore{ID: "id", TenantID: 1, Name: ""}, + }, + { + name: "zero tenant ID", + store: &types.VectorStore{ID: "id", TenantID: 0, Name: "test"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := svc.UpdateStore(context.Background(), tt.store) + require.Error(t, err) + }) + } +} + +// --------------------------------------------------------------------------- +// DeleteStore tests +// --------------------------------------------------------------------------- + +func TestDeleteStore_Success(t *testing.T) { + repo := &mockVectorStoreRepo{} + svc := NewVectorStoreService(repo) + + err := svc.DeleteStore(context.Background(), 1, "test-id") + assert.NoError(t, err) +} + +func TestDeleteStore_RepoError(t *testing.T) { + repo := &mockVectorStoreRepo{deleteErr: assert.AnError} + svc := NewVectorStoreService(repo) + + err := svc.DeleteStore(context.Background(), 1, "test-id") + assert.Error(t, err) +} + +// --------------------------------------------------------------------------- +// TestConnection tests +// --------------------------------------------------------------------------- + +func TestTestConnection_UnsupportedEngineType(t *testing.T) { + repo := &mockVectorStoreRepo{} + svc := NewVectorStoreService(repo) + + _, err := svc.TestConnection(context.Background(), "unknown_engine", types.ConnectionConfig{}) + require.Error(t, err) + + var appErr *errors.AppError + require.ErrorAs(t, err, &appErr) + assert.Equal(t, errors.ErrBadRequest, appErr.Code) +} + +func TestTestConnection_SQLiteAlwaysSucceeds(t *testing.T) { + repo := &mockVectorStoreRepo{} + svc := NewVectorStoreService(repo) + + version, err := svc.TestConnection(context.Background(), types.SQLiteRetrieverEngineType, types.ConnectionConfig{}) + assert.NoError(t, err) + assert.Empty(t, version) +} + +func TestTestConnection_PostgresDefaultConnection(t *testing.T) { + repo := &mockVectorStoreRepo{} + svc := NewVectorStoreService(repo) + + version, err := svc.TestConnection(context.Background(), types.PostgresRetrieverEngineType, + types.ConnectionConfig{UseDefaultConnection: true}) + assert.NoError(t, err) + assert.Empty(t, version) // default connection cannot detect version without DB handle +} + +// --------------------------------------------------------------------------- +// validateConnectionConfig tests +// --------------------------------------------------------------------------- + +func TestValidateConnectionConfig(t *testing.T) { + tests := []struct { + name string + engineType types.RetrieverEngineType + config types.ConnectionConfig + wantError bool + }{ + { + name: "elasticsearch valid", + engineType: types.ElasticsearchRetrieverEngineType, + config: types.ConnectionConfig{Addr: "http://es:9200"}, + wantError: false, + }, + { + name: "elasticsearch missing addr", + engineType: types.ElasticsearchRetrieverEngineType, + config: types.ConnectionConfig{}, + wantError: true, + }, + { + name: "postgres with default connection", + engineType: types.PostgresRetrieverEngineType, + config: types.ConnectionConfig{UseDefaultConnection: true}, + wantError: false, + }, + { + name: "postgres with addr", + engineType: types.PostgresRetrieverEngineType, + config: types.ConnectionConfig{Addr: "postgres://host:5432/db"}, + wantError: false, + }, + { + name: "postgres without addr or default", + engineType: types.PostgresRetrieverEngineType, + config: types.ConnectionConfig{}, + wantError: true, + }, + { + name: "qdrant valid", + engineType: types.QdrantRetrieverEngineType, + config: types.ConnectionConfig{Host: "qdrant-host"}, + wantError: false, + }, + { + name: "qdrant missing host", + engineType: types.QdrantRetrieverEngineType, + config: types.ConnectionConfig{}, + wantError: true, + }, + { + name: "milvus valid", + engineType: types.MilvusRetrieverEngineType, + config: types.ConnectionConfig{Addr: "milvus:19530"}, + wantError: false, + }, + { + name: "milvus missing addr", + engineType: types.MilvusRetrieverEngineType, + config: types.ConnectionConfig{}, + wantError: true, + }, + { + name: "weaviate valid", + engineType: types.WeaviateRetrieverEngineType, + config: types.ConnectionConfig{Host: "weaviate:8080"}, + wantError: false, + }, + { + name: "weaviate missing host", + engineType: types.WeaviateRetrieverEngineType, + config: types.ConnectionConfig{}, + wantError: true, + }, + { + name: "sqlite always valid", + engineType: types.SQLiteRetrieverEngineType, + config: types.ConnectionConfig{}, + wantError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateConnectionConfig(tt.engineType, tt.config) + if tt.wantError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/internal/types/interfaces/vectorstore.go b/internal/types/interfaces/vectorstore.go index 99ccfeb2..5eca6990 100644 --- a/internal/types/interfaces/vectorstore.go +++ b/internal/types/interfaces/vectorstore.go @@ -6,6 +6,22 @@ import ( "github.com/Tencent/WeKnora/internal/types" ) +// VectorStoreService defines the service interface for vector store management. +// Tenant isolation is enforced by the handler layer (getOwnedStore pattern). +type VectorStoreService interface { + // CreateStore validates and creates a new vector store. + CreateStore(ctx context.Context, store *types.VectorStore) error + // UpdateStore updates an existing vector store (name only). + UpdateStore(ctx context.Context, store *types.VectorStore) error + // DeleteStore deletes a vector store by tenant + id. + DeleteStore(ctx context.Context, tenantID uint64, id string) error + // TestConnection tests connectivity to a vector database. + // Returns the detected server version on success (e.g., "7.10.1"), empty string if unknown. + TestConnection(ctx context.Context, engineType types.RetrieverEngineType, config types.ConnectionConfig) (string, error) + // SaveDetectedVersion updates the connection_config.version for a stored vector store. + SaveDetectedVersion(ctx context.Context, store *types.VectorStore, version string) error +} + // VectorStoreRepository defines the repository interface for VectorStore CRUD. type VectorStoreRepository interface { // Create creates a new vector store @@ -16,6 +32,8 @@ type VectorStoreRepository interface { List(ctx context.Context, tenantID uint64) ([]*types.VectorStore, error) // Update updates a vector store (only mutable fields: name) Update(ctx context.Context, store *types.VectorStore) error + // UpdateConnectionConfig updates only the connection_config column + UpdateConnectionConfig(ctx context.Context, store *types.VectorStore) error // Delete soft-deletes a vector store Delete(ctx context.Context, tenantID uint64, id string) error // ExistsByEndpointAndIndex checks if a store with the same endpoint and index already exists diff --git a/internal/types/vectorstore.go b/internal/types/vectorstore.go index 42c5d8d0..e75eb771 100644 --- a/internal/types/vectorstore.go +++ b/internal/types/vectorstore.go @@ -4,6 +4,7 @@ import ( "database/sql/driver" "encoding/json" "fmt" + "strings" "time" "github.com/Tencent/WeKnora/internal/errors" @@ -12,6 +13,18 @@ import ( "gorm.io/gorm" ) +// EnvStoreIDPrefix is the prefix for virtual env store IDs. +const EnvStoreIDPrefix = "__env_" + +// IsEnvStoreID checks if the given ID is an env store virtual ID. +func IsEnvStoreID(id string) bool { + return strings.HasPrefix(id, EnvStoreIDPrefix) +} + +// EnvLookupFunc is a function type for looking up environment variables. +// In production: os.Getenv, in tests: custom lookup function. +type EnvLookupFunc func(string) string + // VectorStore represents a configured vector database instance for a tenant. // Each tenant can register multiple VectorStore entries (even of the same engine type) // to support multi-store scenarios (e.g., ES-hot + ES-warm clusters). @@ -100,6 +113,9 @@ type ConnectionConfig struct { Scheme string `yaml:"scheme" json:"scheme,omitempty"` // Postgres UseDefaultConnection bool `yaml:"use_default_connection" json:"use_default_connection,omitempty"` + // Version is the detected server version (e.g., "7.10.1", "16.2", "1.12.6"). + // Auto-populated by TestConnection on successful connectivity check. + Version string `yaml:"version" json:"version,omitempty"` } // Value implements the driver.Valuer interface. @@ -237,3 +253,253 @@ func (c IndexConfig) GetIndexNameOrDefault(engineType RetrieverEngineType) strin return c.IndexName } } + +// --------------------------------------------------------------------------- +// VectorStoreResponse — API response DTO +// --------------------------------------------------------------------------- + +// VectorStoreResponse is the API response DTO for vector store. +// Wraps VectorStore with additional metadata (source, readonly). +type VectorStoreResponse struct { + VectorStore + Source string `json:"source"` // "env" or "user" + ReadOnly bool `json:"readonly"` // env stores are read-only +} + +// NewVectorStoreResponse creates a response DTO from a VectorStore +// with sensitive fields masked. +func NewVectorStoreResponse(store *VectorStore, source string, readonly bool) VectorStoreResponse { + masked := *store + masked.ConnectionConfig = store.ConnectionConfig.MaskSensitiveFields() + return VectorStoreResponse{ + VectorStore: masked, + Source: source, + ReadOnly: readonly, + } +} + +// --------------------------------------------------------------------------- +// VectorStore type metadata — for /types endpoint +// --------------------------------------------------------------------------- + +// VectorStoreTypeInfo describes a supported engine type and its configuration schema. +type VectorStoreTypeInfo struct { + Type string `json:"type"` + DisplayName string `json:"display_name"` + ConnectionFields []VectorStoreFieldInfo `json:"connection_fields"` + IndexFields []VectorStoreFieldInfo `json:"index_fields,omitempty"` +} + +// VectorStoreFieldInfo describes a single configuration field. +type VectorStoreFieldInfo struct { + Name string `json:"name"` + Type string `json:"type"` // "string", "number", "boolean" + Required bool `json:"required"` + Sensitive bool `json:"sensitive,omitempty"` + Default any `json:"default,omitempty"` + Description string `json:"description,omitempty"` +} + +// GetVectorStoreTypes returns metadata for all supported engine types. +func GetVectorStoreTypes() []VectorStoreTypeInfo { + return []VectorStoreTypeInfo{ + { + Type: "elasticsearch", + DisplayName: "Elasticsearch (Keywords + Vector)", + ConnectionFields: []VectorStoreFieldInfo{ + {Name: "addr", Type: "string", Required: true, Description: "Elasticsearch URL (e.g., http://localhost:9200)"}, + {Name: "username", Type: "string", Required: false}, + {Name: "password", Type: "string", Required: false, Sensitive: true}, + }, + IndexFields: []VectorStoreFieldInfo{ + {Name: "index_name", Type: "string", Required: false, Default: "xwrag_default"}, + {Name: "number_of_shards", Type: "number", Required: false}, + {Name: "number_of_replicas", Type: "number", Required: false}, + }, + }, + { + Type: "postgres", + DisplayName: "PostgreSQL (Keywords + Vector)", + ConnectionFields: []VectorStoreFieldInfo{ + {Name: "use_default_connection", Type: "boolean", Required: false, Default: true, + Description: "Use the application's default database connection"}, + {Name: "addr", Type: "string", Required: false, + Description: "PostgreSQL connection string (required if use_default_connection is false)"}, + {Name: "username", Type: "string", Required: false}, + {Name: "password", Type: "string", Required: false, Sensitive: true}, + }, + }, + { + Type: "qdrant", + DisplayName: "Qdrant (Keywords + Vector)", + ConnectionFields: []VectorStoreFieldInfo{ + {Name: "host", Type: "string", Required: true, Description: "Qdrant host"}, + {Name: "port", Type: "number", Required: false, Default: 6334}, + {Name: "api_key", Type: "string", Required: false, Sensitive: true}, + {Name: "use_tls", Type: "boolean", Required: false, Default: false}, + }, + IndexFields: []VectorStoreFieldInfo{ + {Name: "collection_prefix", Type: "string", Required: false, Default: "weknora_embeddings"}, + }, + }, + { + Type: "milvus", + DisplayName: "Milvus (Keywords + Vector)", + ConnectionFields: []VectorStoreFieldInfo{ + {Name: "addr", Type: "string", Required: true, Description: "Milvus address (e.g., localhost:19530)"}, + {Name: "username", Type: "string", Required: false}, + {Name: "password", Type: "string", Required: false, Sensitive: true}, + }, + IndexFields: []VectorStoreFieldInfo{ + {Name: "collection_name", Type: "string", Required: false, Default: "weknora_embeddings"}, + }, + }, + { + Type: "weaviate", + DisplayName: "Weaviate (Keywords + Vector)", + ConnectionFields: []VectorStoreFieldInfo{ + {Name: "host", Type: "string", Required: true, Description: "Weaviate host (e.g., weaviate:8080)"}, + {Name: "grpc_address", Type: "string", Required: false, Default: "weaviate:50051"}, + {Name: "scheme", Type: "string", Required: false, Default: "http"}, + {Name: "api_key", Type: "string", Required: false, Sensitive: true}, + }, + IndexFields: []VectorStoreFieldInfo{ + {Name: "collection_prefix", Type: "string", Required: false, Default: "WeKnora"}, + }, + }, + { + Type: "sqlite", + DisplayName: "SQLite (Keywords + Vector)", + ConnectionFields: []VectorStoreFieldInfo{}, + }, + } +} + +// --------------------------------------------------------------------------- +// BuildEnvVectorStores — virtual stores from RETRIEVE_DRIVER env var +// --------------------------------------------------------------------------- + +// BuildEnvVectorStores builds virtual VectorStore entries from RETRIEVE_DRIVER. +// Returns []VectorStore (not VectorStoreResponse) so that business logic (e.g., +// duplicate checking) can use them directly. API responses should wrap them +// via NewVectorStoreResponse. +// +// Pure function — does not call os.Getenv directly. +// +// Usage: +// +// types.BuildEnvVectorStores(os.Getenv("RETRIEVE_DRIVER"), os.Getenv) +func BuildEnvVectorStores(retrieveDriver string, envLookup EnvLookupFunc) []VectorStore { + if retrieveDriver == "" { + return nil + } + + drivers := strings.Split(retrieveDriver, ",") + var stores []VectorStore + + for _, driver := range drivers { + driver = strings.TrimSpace(driver) + if driver == "" { + continue + } + + store := buildEnvStoreForDriver(driver, envLookup) + if store != nil { + stores = append(stores, *store) + } + } + return stores +} + +// FindEnvVectorStore finds a specific env store by its virtual ID. +func FindEnvVectorStore(retrieveDriver string, envLookup EnvLookupFunc, id string) *VectorStore { + for _, s := range BuildEnvVectorStores(retrieveDriver, envLookup) { + if s.ID == id { + return &s + } + } + return nil +} + +func buildEnvStoreForDriver(driver string, envLookup EnvLookupFunc) *VectorStore { + switch driver { + case "postgres": + return &VectorStore{ + ID: "__env_postgres__", + Name: "postgres (env)", + EngineType: PostgresRetrieverEngineType, + ConnectionConfig: ConnectionConfig{ + UseDefaultConnection: true, + }, + } + case "sqlite": + return &VectorStore{ + ID: "__env_sqlite__", + Name: "sqlite (env)", + EngineType: SQLiteRetrieverEngineType, + } + case "elasticsearch_v8": + return &VectorStore{ + ID: "__env_elasticsearch_v8__", + Name: "elasticsearch v8 (env)", + EngineType: ElasticsearchRetrieverEngineType, + ConnectionConfig: ConnectionConfig{ + Addr: envLookup("ELASTICSEARCH_ADDR"), + Username: envLookup("ELASTICSEARCH_USERNAME"), + Password: envLookup("ELASTICSEARCH_PASSWORD"), + }, + IndexConfig: IndexConfig{ + IndexName: envLookup("ELASTICSEARCH_INDEX"), + }, + } + case "elasticsearch_v7": + return &VectorStore{ + ID: "__env_elasticsearch_v7__", + Name: "elasticsearch v7 (env)", + EngineType: ElasticsearchRetrieverEngineType, + ConnectionConfig: ConnectionConfig{ + Addr: envLookup("ELASTICSEARCH_ADDR"), + Username: envLookup("ELASTICSEARCH_USERNAME"), + Password: envLookup("ELASTICSEARCH_PASSWORD"), + }, + IndexConfig: IndexConfig{ + IndexName: envLookup("ELASTICSEARCH_INDEX"), + }, + } + case "qdrant": + return &VectorStore{ + ID: "__env_qdrant__", + Name: "qdrant (env)", + EngineType: QdrantRetrieverEngineType, + ConnectionConfig: ConnectionConfig{ + Host: envLookup("QDRANT_HOST"), + APIKey: envLookup("QDRANT_API_KEY"), + }, + } + case "milvus": + return &VectorStore{ + ID: "__env_milvus__", + Name: "milvus (env)", + EngineType: MilvusRetrieverEngineType, + ConnectionConfig: ConnectionConfig{ + Addr: envLookup("MILVUS_ADDRESS"), + Username: envLookup("MILVUS_USERNAME"), + Password: envLookup("MILVUS_PASSWORD"), + }, + } + case "weaviate": + return &VectorStore{ + ID: "__env_weaviate__", + Name: "weaviate (env)", + EngineType: WeaviateRetrieverEngineType, + ConnectionConfig: ConnectionConfig{ + Host: envLookup("WEAVIATE_HOST"), + GrpcAddress: envLookup("WEAVIATE_GRPC_ADDRESS"), + Scheme: envLookup("WEAVIATE_SCHEME"), + APIKey: envLookup("WEAVIATE_API_KEY"), + }, + } + default: + return nil + } +} diff --git a/internal/types/vectorstore_test.go b/internal/types/vectorstore_test.go index cca159ee..bf0d3bcf 100644 --- a/internal/types/vectorstore_test.go +++ b/internal/types/vectorstore_test.go @@ -10,6 +10,245 @@ import ( "gorm.io/gorm" ) +// --------------------------------------------------------------------------- +// PR2 additions: env store builder, response DTO, types metadata +// --------------------------------------------------------------------------- + +// mockEnvLookup creates a simple env lookup function from a map. +func mockEnvLookup(env map[string]string) EnvLookupFunc { + return func(key string) string { + return env[key] + } +} + +func TestIsEnvStoreID(t *testing.T) { + tests := []struct { + name string + id string + expected bool + }{ + {"env postgres ID", "__env_postgres__", true}, + {"env elasticsearch ID", "__env_elasticsearch_v8__", true}, + {"env prefix only", "__env_", true}, + {"UUID ID", "550e8400-e29b-41d4-a716-446655440000", false}, + {"empty string", "", false}, + {"similar but not prefix", "_env_postgres__", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, IsEnvStoreID(tt.id)) + }) + } +} + +func TestBuildEnvVectorStores(t *testing.T) { + envMap := map[string]string{ + "ELASTICSEARCH_ADDR": "http://es:9200", + "ELASTICSEARCH_USERNAME": "elastic", + "ELASTICSEARCH_PASSWORD": "secret", + "ELASTICSEARCH_INDEX": "my_index", + "QDRANT_HOST": "qdrant-host", + "QDRANT_API_KEY": "qd-key", + "MILVUS_ADDRESS": "milvus:19530", + "WEAVIATE_HOST": "weaviate:8080", + } + lookup := mockEnvLookup(envMap) + + t.Run("empty RETRIEVE_DRIVER returns nil", func(t *testing.T) { + stores := BuildEnvVectorStores("", lookup) + assert.Nil(t, stores) + }) + + t.Run("single driver postgres", func(t *testing.T) { + stores := BuildEnvVectorStores("postgres", lookup) + require.Len(t, stores, 1) + assert.Equal(t, "__env_postgres__", stores[0].ID) + assert.Equal(t, "postgres (env)", stores[0].Name) + assert.Equal(t, PostgresRetrieverEngineType, stores[0].EngineType) + assert.True(t, stores[0].ConnectionConfig.UseDefaultConnection) + }) + + t.Run("multiple drivers", func(t *testing.T) { + stores := BuildEnvVectorStores("postgres,elasticsearch_v8", lookup) + require.Len(t, stores, 2) + assert.Equal(t, "__env_postgres__", stores[0].ID) + assert.Equal(t, "__env_elasticsearch_v8__", stores[1].ID) + assert.Equal(t, "http://es:9200", stores[1].ConnectionConfig.Addr) + assert.Equal(t, "elastic", stores[1].ConnectionConfig.Username) + assert.Equal(t, "secret", stores[1].ConnectionConfig.Password) // unmasked + assert.Equal(t, "my_index", stores[1].IndexConfig.IndexName) + }) + + t.Run("env store retains raw password (not masked)", func(t *testing.T) { + stores := BuildEnvVectorStores("elasticsearch_v8", lookup) + require.Len(t, stores, 1) + assert.Equal(t, "secret", stores[0].ConnectionConfig.Password) + }) + + t.Run("unknown driver is skipped", func(t *testing.T) { + stores := BuildEnvVectorStores("postgres,unknown_db", lookup) + require.Len(t, stores, 1) + assert.Equal(t, "__env_postgres__", stores[0].ID) + }) + + t.Run("whitespace trimmed", func(t *testing.T) { + stores := BuildEnvVectorStores(" postgres , elasticsearch_v8 ", lookup) + require.Len(t, stores, 2) + }) + + t.Run("all supported drivers", func(t *testing.T) { + stores := BuildEnvVectorStores("postgres,sqlite,elasticsearch_v8,elasticsearch_v7,qdrant,milvus,weaviate", lookup) + require.Len(t, stores, 7) + + ids := make([]string, len(stores)) + for i, s := range stores { + ids[i] = s.ID + } + assert.Contains(t, ids, "__env_postgres__") + assert.Contains(t, ids, "__env_sqlite__") + assert.Contains(t, ids, "__env_elasticsearch_v8__") + assert.Contains(t, ids, "__env_elasticsearch_v7__") + assert.Contains(t, ids, "__env_qdrant__") + assert.Contains(t, ids, "__env_milvus__") + assert.Contains(t, ids, "__env_weaviate__") + }) + + t.Run("qdrant env store", func(t *testing.T) { + stores := BuildEnvVectorStores("qdrant", lookup) + require.Len(t, stores, 1) + assert.Equal(t, "qdrant-host", stores[0].ConnectionConfig.Host) + assert.Equal(t, "qd-key", stores[0].ConnectionConfig.APIKey) + }) + + t.Run("milvus env store", func(t *testing.T) { + stores := BuildEnvVectorStores("milvus", lookup) + require.Len(t, stores, 1) + assert.Equal(t, "milvus:19530", stores[0].ConnectionConfig.Addr) + }) + + t.Run("weaviate env store", func(t *testing.T) { + stores := BuildEnvVectorStores("weaviate", lookup) + require.Len(t, stores, 1) + assert.Equal(t, "weaviate:8080", stores[0].ConnectionConfig.Host) + }) +} + +func TestFindEnvVectorStore(t *testing.T) { + lookup := mockEnvLookup(map[string]string{}) + + t.Run("found", func(t *testing.T) { + store := FindEnvVectorStore("postgres", lookup, "__env_postgres__") + require.NotNil(t, store) + assert.Equal(t, "__env_postgres__", store.ID) + }) + + t.Run("not found", func(t *testing.T) { + store := FindEnvVectorStore("postgres", lookup, "__env_unknown__") + assert.Nil(t, store) + }) + + t.Run("empty driver returns nil", func(t *testing.T) { + store := FindEnvVectorStore("", lookup, "__env_postgres__") + assert.Nil(t, store) + }) +} + +func TestNewVectorStoreResponse(t *testing.T) { + store := &VectorStore{ + ID: "test-id", + Name: "test-store", + EngineType: ElasticsearchRetrieverEngineType, + ConnectionConfig: ConnectionConfig{ + Addr: "http://es:9200", + Password: "secret", + APIKey: "my-api-key", + }, + } + + t.Run("masks sensitive fields", func(t *testing.T) { + resp := NewVectorStoreResponse(store, "user", false) + assert.Equal(t, "***", resp.ConnectionConfig.Password) + assert.Equal(t, "***", resp.ConnectionConfig.APIKey) + assert.Equal(t, "http://es:9200", resp.ConnectionConfig.Addr) // non-sensitive preserved + }) + + t.Run("preserves source and readonly", func(t *testing.T) { + resp := NewVectorStoreResponse(store, "env", true) + assert.Equal(t, "env", resp.Source) + assert.True(t, resp.ReadOnly) + }) + + t.Run("does not mutate original store", func(t *testing.T) { + _ = NewVectorStoreResponse(store, "user", false) + assert.Equal(t, "secret", store.ConnectionConfig.Password) + assert.Equal(t, "my-api-key", store.ConnectionConfig.APIKey) + }) + + t.Run("empty sensitive fields not masked to ***", func(t *testing.T) { + noSecret := &VectorStore{ + ID: "test-id", + ConnectionConfig: ConnectionConfig{Addr: "http://es:9200"}, + } + resp := NewVectorStoreResponse(noSecret, "user", false) + assert.Equal(t, "", resp.ConnectionConfig.Password) + assert.Equal(t, "", resp.ConnectionConfig.APIKey) + }) +} + +func TestGetVectorStoreTypes(t *testing.T) { + types := GetVectorStoreTypes() + + t.Run("returns 6 engine types", func(t *testing.T) { + assert.Len(t, types, 6) + }) + + t.Run("type names match engine constants", func(t *testing.T) { + typeNames := make([]string, len(types)) + for i, typ := range types { + typeNames[i] = typ.Type + } + assert.Contains(t, typeNames, "elasticsearch") + assert.Contains(t, typeNames, "postgres") + assert.Contains(t, typeNames, "qdrant") + assert.Contains(t, typeNames, "milvus") + assert.Contains(t, typeNames, "weaviate") + assert.Contains(t, typeNames, "sqlite") + }) + + t.Run("elasticsearch has connection and index fields", func(t *testing.T) { + var esType VectorStoreTypeInfo + for _, typ := range types { + if typ.Type == "elasticsearch" { + esType = typ + break + } + } + assert.NotEmpty(t, esType.ConnectionFields) + assert.NotEmpty(t, esType.IndexFields) + + // Check sensitive field marking + var passwordField VectorStoreFieldInfo + for _, f := range esType.ConnectionFields { + if f.Name == "password" { + passwordField = f + break + } + } + assert.True(t, passwordField.Sensitive) + }) + + t.Run("sqlite has no connection fields", func(t *testing.T) { + var sqliteType VectorStoreTypeInfo + for _, typ := range types { + if typ.Type == "sqlite" { + sqliteType = typ + break + } + } + assert.Empty(t, sqliteType.ConnectionFields) + }) +} + // testAESKey is a 32-byte key for testing AES-GCM encryption. const testAESKey = "01234567890123456789012345678901"