diff --git a/docker-compose.yml b/docker-compose.yml index f10f339b..54549cbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -116,6 +116,11 @@ services: - SSRF_WHITELIST=${SSRF_WHITELIST:-} - CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5} - JWT_SECRET=${JWT_SECRET:-} + # Crypto: 主密钥和盐值,用于 AppSecret 等敏感字段的 AES-256 加密 + # 若不设置则自动生成并持久化到 data-files volume(/data/files/.crypto_state.json) + # 重启时自动从文件恢复,保证已加密数据可继续解密 + - CRYPTO_MASTER_KEY=${CRYPTO_MASTER_KEY:-} + - CRYPTO_SALT=${CRYPTO_SALT:-} # File size limit (in MB) - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} # Agent Skills Sandbox diff --git a/frontend/src/api/model/index.ts b/frontend/src/api/model/index.ts index d886a344..84313346 100644 --- a/frontend/src/api/model/index.ts +++ b/frontend/src/api/model/index.ts @@ -126,3 +126,45 @@ export function deleteModel(id: string): Promise { }); } +export interface InitializeWeKnoraCloudRequest { + app_id: string + app_secret: string +} + +export interface InitializeWeKnoraCloudResult { + models: Array<{ + name: string + type: string + action: 'created' | 'updated' + }> +} + +export function initializeWeKnoraCloud(data: InitializeWeKnoraCloudRequest): Promise { + return post('/api/v1/models/weknoracloud/initialize', data) as Promise +} + +export interface WeKnoraCloudStatusResult { + has_models: boolean + needs_reinit: boolean + reason?: string +} + +export function getWeKnoraCloudStatus(): Promise { + return new Promise((resolve, reject) => { + get('/api/v1/models/weknoracloud/status') + .then((response: any) => { + // status 接口直接返回对象,不包在 success/data 中 + if (response && typeof response.has_models === 'boolean') { + resolve(response) + } else if (response?.success && response?.data) { + resolve(response.data) + } else { + resolve({ has_models: false, needs_reinit: false }) + } + }) + .catch(() => { + resolve({ has_models: false, needs_reinit: false }) + }) + }) +} + diff --git a/frontend/src/views/settings/ModelSettings.vue b/frontend/src/views/settings/ModelSettings.vue index 94ae37d4..e735bb27 100644 --- a/frontend/src/views/settings/ModelSettings.vue +++ b/frontend/src/views/settings/ModelSettings.vue @@ -18,6 +18,46 @@ + +
+
+

WeKnoraCloud 厂商配置

+
+ +
+ +
+ WeKnoraCloud 凭证已失效
+ {{ weKnoraCloudReinitReason || '服务重启后加密密钥已变更,已保存的凭证无法解密。' }}请重新填写 APPID 和 APPSECRET 并点击"保存并初始化"以恢复服务。 +
+
+
+
+ + + + + + + + 保存并初始化 + +
+

+ 提交后将自动创建或更新三个模型(Chat / Embedding / Rerank)并设为各类型默认 +

+
+
+
@@ -295,7 +335,7 @@ import { MessagePlugin } from 'tdesign-vue-next' import { AddIcon } from 'tdesign-icons-vue-next' import { useI18n } from 'vue-i18n' import ModelEditorDialog from '@/components/ModelEditorDialog.vue' -import { listModels, createModel, updateModel as updateModelAPI, deleteModel as deleteModelAPI, type ModelConfig } from '@/api/model' +import { listModels, createModel, updateModel as updateModelAPI, deleteModel as deleteModelAPI, initializeWeKnoraCloud, getWeKnoraCloudStatus, type ModelConfig } from '@/api/model' const { t } = useI18n() @@ -304,6 +344,38 @@ const currentModelType = ref<'chat' | 'embedding' | 'rerank' | 'vllm' | 'asr'>(' const editingModel = ref(null) const loading = ref(true) +// WeKnoraCloud 配置 +const weKnoraCloudForm = ref({ appId: '', appSecret: '' }) +const weKnoraCloudLoading = ref(false) +const weKnoraCloudNeedsReinit = ref(false) +const weKnoraCloudReinitReason = ref('') + +const handleWeKnoraCloudInit = async () => { + if (!weKnoraCloudForm.value.appId || !weKnoraCloudForm.value.appSecret) { + MessagePlugin.warning('请填写 APPID 和 APPSECRET') + return + } + weKnoraCloudLoading.value = true + try { + const result = await initializeWeKnoraCloud({ + app_id: weKnoraCloudForm.value.appId, + app_secret: weKnoraCloudForm.value.appSecret, + }) + const updatedCount = result.models.filter((m: any) => m.action === 'updated').length + const createdCount = result.models.filter((m: any) => m.action === 'created').length + MessagePlugin.success(`初始化成功:新建 ${createdCount} 个,更新 ${updatedCount} 个模型`) + weKnoraCloudForm.value.appId = '' + weKnoraCloudForm.value.appSecret = '' + weKnoraCloudNeedsReinit.value = false + weKnoraCloudReinitReason.value = '' + await loadModels() + } catch (err: any) { + MessagePlugin.error(err?.message || 'WeKnoraCloud 初始化失败') + } finally { + weKnoraCloudLoading.value = false + } +} + // 模型列表数据 const allModels = ref([]) @@ -361,6 +433,10 @@ const loadModels = async () => { // 直接获取所有模型,不分类型 const models = await listModels() allModels.value = models + // 检查 WeKnoraCloud 凭证状态 + const status = await getWeKnoraCloudStatus() + weKnoraCloudNeedsReinit.value = status.needs_reinit + weKnoraCloudReinitReason.value = status.reason || '' } catch (error: any) { console.error('加载模型列表失败:', error) MessagePlugin.error(error.message) diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 2eef0660..7851275c 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -65,6 +65,7 @@ type knowledgeService struct { repo interfaces.KnowledgeRepository kbService interfaces.KnowledgeBaseService tenantRepo interfaces.TenantRepository + tenantService interfaces.TenantService documentReader interfaces.DocumentReader chunkService interfaces.ChunkService chunkRepo interfaces.ChunkRepository @@ -92,6 +93,7 @@ func NewKnowledgeService( documentReader interfaces.DocumentReader, kbService interfaces.KnowledgeBaseService, tenantRepo interfaces.TenantRepository, + tenantService interfaces.TenantService, chunkService interfaces.ChunkService, chunkRepo interfaces.ChunkRepository, tagRepo interfaces.KnowledgeTagRepository, @@ -110,6 +112,7 @@ func NewKnowledgeService( repo: repo, kbService: kbService, tenantRepo: tenantRepo, + tenantService: tenantService, documentReader: documentReader, chunkService: chunkService, chunkRepo: chunkRepo, @@ -7994,7 +7997,7 @@ func (s *knowledgeService) convert( logger.Infof(ctx, "[convert] kb=%s fileType=%s isURL=%v engine=%q rules=%+v", kb.ID, fileType, isURL, parserEngine, kb.ChunkingConfig.ParserEngineRules) - var reader interfaces.DocReader = s.resolveDocReader(parserEngine, fileType, isURL, overrides) + var reader interfaces.DocReader = s.resolveDocReader(ctx, parserEngine, fileType, isURL, overrides) if reader == nil { knowledge.ParseStatus = "failed" knowledge.ErrorMessage = "Document parsing service is not configured. Please use text/paragraph import or set DOCREADER_ADDR." @@ -8042,10 +8045,24 @@ func (s *knowledgeService) convert( // resolveDocReader returns the appropriate DocReader for the given engine. // Returns nil when the required service is unavailable. -func (s *knowledgeService) resolveDocReader(engine, fileType string, isURL bool, overrides map[string]string) interfaces.DocReader { +func (s *knowledgeService) resolveDocReader(ctx context.Context, engine, fileType string, isURL bool, overrides map[string]string) interfaces.DocReader { switch engine { case docparser.SimpleEngineName: return &docparser.SimpleFormatReader{} + case docparser.WeKnoraCloudEngineName: + addr := strings.TrimSpace(overrides["docreader_addr"]) + if !IsWeKnoraCloudDocReaderAddr(addr) { + return nil + } + creds := s.tenantService.GetDocreaderCredentials(ctx) + if creds == nil { + return nil + } + reader, err := docparser.NewWeKnoraCloudSignedDocumentReader(addr, creds.AppID, creds.APIKey) + if err != nil { + return nil + } + return reader case "mineru": return docparser.NewMinerUReader(overrides) case "mineru_cloud": diff --git a/internal/application/service/model.go b/internal/application/service/model.go index 80bdc2d2..86db67cf 100644 --- a/internal/application/service/model.go +++ b/internal/application/service/model.go @@ -4,11 +4,12 @@ import ( "context" "errors" + "github.com/Tencent/WeKnora/internal/infrastructure/crypto" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/models/asr" "github.com/Tencent/WeKnora/internal/models/chat" "github.com/Tencent/WeKnora/internal/models/embedding" "github.com/Tencent/WeKnora/internal/models/rerank" - "github.com/Tencent/WeKnora/internal/models/asr" "github.com/Tencent/WeKnora/internal/models/utils/ollama" "github.com/Tencent/WeKnora/internal/models/vlm" "github.com/Tencent/WeKnora/internal/types" @@ -23,17 +24,35 @@ type modelService struct { repo interfaces.ModelRepository ollamaService *ollama.OllamaService pooler embedding.EmbedderPooler + cryptoSvc *crypto.CryptoService } // NewModelService creates a new model service instance -func NewModelService(repo interfaces.ModelRepository, ollamaService *ollama.OllamaService, pooler embedding.EmbedderPooler) interfaces.ModelService { +func NewModelService(repo interfaces.ModelRepository, + ollamaService *ollama.OllamaService, + pooler embedding.EmbedderPooler, + cryptoSvc *crypto.CryptoService, +) interfaces.ModelService { return &modelService{ repo: repo, ollamaService: ollamaService, pooler: pooler, + cryptoSvc: cryptoSvc, } } +// decryptAppSecret 解密 AppSecret(如果为空或 cryptoSvc 为空则原样返回) +func (s *modelService) decryptAppSecret(encrypted string) string { + if encrypted == "" { + return encrypted + } + plain, err := s.cryptoSvc.DecryptString(encrypted) + if err != nil { + return "" + } + return plain +} + // CreateModel creates a new model in the repository // For local models, it initiates an asynchronous download process // Remote models are immediately set to active status @@ -256,7 +275,9 @@ func (s *modelService) GetEmbeddingModel(ctx context.Context, modelId string) (e Dimensions: model.Parameters.EmbeddingParameters.Dimension, TruncatePromptTokens: model.Parameters.EmbeddingParameters.TruncatePromptTokens, Provider: model.Parameters.Provider, - Extra: stringMapToAnyMap(model.Parameters.ExtraConfig), + ExtraConfig: model.Parameters.ExtraConfig, + AppID: model.Parameters.AppID, + AppSecret: s.decryptAppSecret(model.Parameters.AppSecret), }, s.pooler, s.ollamaService) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ @@ -312,7 +333,9 @@ func (s *modelService) GetEmbeddingModelForTenant(ctx context.Context, modelId s Dimensions: model.Parameters.EmbeddingParameters.Dimension, TruncatePromptTokens: model.Parameters.EmbeddingParameters.TruncatePromptTokens, Provider: model.Parameters.Provider, - Extra: stringMapToAnyMap(model.Parameters.ExtraConfig), + ExtraConfig: model.Parameters.ExtraConfig, + AppID: model.Parameters.AppID, + AppSecret: s.decryptAppSecret(model.Parameters.AppSecret), }, s.pooler, s.ollamaService) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ @@ -343,11 +366,15 @@ func (s *modelService) GetRerankModel(ctx context.Context, modelId string) (rera // Initialize the reranker with model configuration reranker, err := rerank.NewReranker(&rerank.RerankerConfig{ - ModelID: model.ID, - APIKey: model.Parameters.APIKey, - BaseURL: model.Parameters.BaseURL, - ModelName: model.Name, - Source: model.Source, + ModelID: model.ID, + APIKey: model.Parameters.APIKey, + BaseURL: model.Parameters.BaseURL, + ModelName: model.Name, + Source: model.Source, + Provider: model.Parameters.Provider, + ExtraConfig: model.Parameters.ExtraConfig, + AppID: model.Parameters.AppID, + AppSecret: s.decryptAppSecret(model.Parameters.AppSecret), }) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ @@ -391,13 +418,15 @@ func (s *modelService) GetChatModel(ctx context.Context, modelId string) (chat.C // Initialize the chat model with model configuration chatModel, err := chat.NewChat(&chat.ChatConfig{ - ModelID: model.ID, - APIKey: model.Parameters.APIKey, - BaseURL: model.Parameters.BaseURL, - ModelName: model.Name, - Source: model.Source, - Provider: model.Parameters.Provider, - Extra: stringMapToAnyMap(model.Parameters.ExtraConfig), + ModelID: model.ID, + APIKey: model.Parameters.APIKey, + BaseURL: model.Parameters.BaseURL, + ModelName: model.Name, + Source: model.Source, + Provider: model.Parameters.Provider, + ExtraConfig: model.Parameters.ExtraConfig, + AppID: model.Parameters.AppID, + AppSecret: s.decryptAppSecret(model.Parameters.AppSecret), }, s.ollamaService) if err != nil { logger.ErrorWithFields(ctx, err, map[string]interface{}{ diff --git a/internal/application/service/tenant.go b/internal/application/service/tenant.go index f3ceeafe..faed4314 100644 --- a/internal/application/service/tenant.go +++ b/internal/application/service/tenant.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/Tencent/WeKnora/internal/infrastructure/crypto" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -33,12 +34,13 @@ type ListTenantsParams struct { // tenantService implements the TenantService interface type tenantService struct { - repo interfaces.TenantRepository // Repository for tenant data operations + repo interfaces.TenantRepository // Repository for tenant data operations + cryptoSvc *crypto.CryptoService } // NewTenantService creates a new tenant service instance -func NewTenantService(repo interfaces.TenantRepository) interfaces.TenantService { - return &tenantService{repo: repo} +func NewTenantService(repo interfaces.TenantRepository, cryptoSvc *crypto.CryptoService) interfaces.TenantService { + return &tenantService{repo: repo, cryptoSvc: cryptoSvc} } // CreateTenant creates a new tenant @@ -342,3 +344,40 @@ func (s *tenantService) GetTenantByIDForUser(ctx context.Context, tenantID uint6 return tenant, nil } + +func (s *tenantService) GetDocreaderCredentials(ctx context.Context) *types.DocreaderCredentials { + // Try to get tenant info directly first + if tenant, ok := types.TenantInfoFromContext(ctx); ok { + if tenant.ParserEngineConfig != nil { + appID := strings.TrimSpace(tenant.ParserEngineConfig.DocreaderAppID) + if appID == "" || tenant.ParserEngineConfig.DocreaderAPIKey == "" { + return nil + } + apiKey, err := s.cryptoSvc.DecryptString(tenant.ParserEngineConfig.DocreaderAPIKey) + if err != nil || apiKey == "" { + return nil + } + return &types.DocreaderCredentials{AppID: appID, APIKey: apiKey} + } + } + + // If no tenant info in context, try to get tenant ID and load tenant + tenantID, ok := types.TenantIDFromContext(ctx) + if !ok { + return nil + } + + // Load tenant from repo if we only have tenantID + tenant, err := s.repo.GetTenantByID(ctx, tenantID) + if err == nil && tenant != nil && tenant.ParserEngineConfig != nil { + appID := strings.TrimSpace(tenant.ParserEngineConfig.DocreaderAppID) + if appID != "" && tenant.ParserEngineConfig.DocreaderAPIKey != "" { + apiKey, err := s.cryptoSvc.DecryptString(tenant.ParserEngineConfig.DocreaderAPIKey) + if err == nil && apiKey != "" { + return &types.DocreaderCredentials{AppID: appID, APIKey: apiKey} + } + } + } + + return nil +} diff --git a/internal/application/service/weknoracloud.go b/internal/application/service/weknoracloud.go new file mode 100644 index 00000000..f63fe45f --- /dev/null +++ b/internal/application/service/weknoracloud.go @@ -0,0 +1,406 @@ +package service + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/infrastructure/crypto" + "github.com/Tencent/WeKnora/internal/models/provider" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/google/uuid" +) + +type weKnoraCloudService struct { + repo interfaces.ModelRepository + tenantRepo interfaces.TenantRepository + cryptoSvc *crypto.CryptoService +} + +// NewWeKnoraCloudService 构造 WeKnoraCloudService +func NewWeKnoraCloudService( + repo interfaces.ModelRepository, + tenantRepo interfaces.TenantRepository, + cryptoSvc *crypto.CryptoService, +) interfaces.WeKnoraCloudService { + return &weKnoraCloudService{ + repo: repo, + tenantRepo: tenantRepo, + cryptoSvc: cryptoSvc, + } +} + +func IsWeKnoraCloudDocReaderAddr(addr string) bool { + return strings.TrimSuffix(strings.TrimSpace(addr), "/") == strings.TrimRight(provider.WeKnoraCloudBaseURL, "/")+"/api/v1/doc/reader" +} + +// weKnoraCloudModelDefs 三个内置模型的静态定义 +var weKnoraCloudModelDefs = []struct { + name string + modelType types.ModelType + remoteModelName string +}{ + {name: "weknoracloud-chat", modelType: types.ModelTypeKnowledgeQA, remoteModelName: "chat"}, + {name: "weknoracloud-embedding", modelType: types.ModelTypeEmbedding, remoteModelName: "embedding"}, + {name: "weknoracloud-rerank", modelType: types.ModelTypeRerank, remoteModelName: "rerank"}, +} + +func (s *weKnoraCloudService) Initialize(ctx context.Context, appID, appSecret string) (*types.InitializeResult, error) { + if err := s.validateInitializeInput(appID, appSecret); err != nil { + return nil, err + } + + prepared, err := s.prepareInitialize(ctx, appID, appSecret) + if err != nil { + return nil, err + } + + result := &types.InitializeResult{} + + actions, err := s.installBuiltinModels(ctx, prepared.tenantID, appID, prepared.encryptedSecret) + if err != nil { + return nil, s.failInitializeAndRollback(ctx, prepared.tenantID, prepared.snapshot, err) + } + result.Models = actions + + if err := s.updateTenantDocReaderAddr(ctx, prepared.tenantID, appID, prepared.encryptedSecret); err != nil { + return nil, s.failInitializeAndRollback(ctx, prepared.tenantID, prepared.snapshot, err) + } + + return result, nil +} + +// validateInitializeInput validates that appID and appSecret are not empty. +func (s *weKnoraCloudService) validateInitializeInput(appID, appSecret string) error { + if appID == "" { + return fmt.Errorf("app_id is required") + } + if appSecret == "" { + return fmt.Errorf("app_secret is required") + } + return nil +} + +// weKnoraCloudInitPrepared holds the prepared state for initialize operation. +type weKnoraCloudInitPrepared struct { + tenantID uint64 + encryptedSecret string + snapshot *weKnoraCloudInitSnapshot +} + +// prepareInitialize performs pre-initialization checks: ping WeKnoraCloud, encrypt secret, and snapshot current state. +func (s *weKnoraCloudService) prepareInitialize(ctx context.Context, appID, appSecret string) (*weKnoraCloudInitPrepared, error) { + // Step 1: Check WeKnoraCloud service reachability + if err := s.pingWeKnoraCloud(ctx, appID, appSecret); err != nil { + return nil, fmt.Errorf("WeKnoraCloud service reachability check failed: %w", err) + } + + // Step 2: Encrypt appSecret + encryptedSecret, err := s.cryptoSvc.EncryptString(appSecret) + if err != nil { + return nil, fmt.Errorf("encrypt AppSecret failed: %w", err) + } + + // Step 3: Get tenantID and take snapshot + tenantID := types.MustTenantIDFromContext(ctx) + snapshot, err := s.snapshotInitializeState(ctx, tenantID) + if err != nil { + return nil, fmt.Errorf("snapshot initialize state failed: %w", err) + } + + return &weKnoraCloudInitPrepared{ + tenantID: tenantID, + encryptedSecret: encryptedSecret, + snapshot: snapshot, + }, nil +} + +// installBuiltinModels installs or updates the three builtin WeKnoraCloud models (chat, embedding, rerank). +// Returns a slice of ModelAction describing what was done. +func (s *weKnoraCloudService) installBuiltinModels( + ctx context.Context, + tenantID uint64, + appID, encryptedSecret string, +) ([]types.ModelAction, error) { + result := make([]types.ModelAction, 0, len(weKnoraCloudModelDefs)) + + for _, def := range weKnoraCloudModelDefs { + action, err := s.upsertModel(ctx, tenantID, def.name, def.modelType, def.remoteModelName, appID, encryptedSecret) + if err != nil { + return nil, fmt.Errorf("upsert model %s failed: %w", def.name, err) + } + result = append(result, types.ModelAction{ + Name: def.name, + Type: string(def.modelType), + Action: action, + }) + } + + return result, nil +} + +// weKnoraCloudInitSnapshot holds a snapshot of the state before initialization. +type weKnoraCloudInitSnapshot struct { + modelsByType map[types.ModelType][]*types.Model + tenant *types.Tenant +} + +func (s *weKnoraCloudService) snapshotInitializeState(ctx context.Context, tenantID uint64) (*weKnoraCloudInitSnapshot, error) { + snapshot := &weKnoraCloudInitSnapshot{modelsByType: make(map[types.ModelType][]*types.Model, len(weKnoraCloudModelDefs))} + for _, def := range weKnoraCloudModelDefs { + models, err := s.repo.List(ctx, tenantID, def.modelType, types.ModelSourceRemote) + if err != nil { + return nil, err + } + copies := make([]*types.Model, 0, len(models)) + for _, model := range models { + copies = append(copies, cloneWeKnoraCloudModel(model)) + } + snapshot.modelsByType[def.modelType] = copies + } + if s.tenantRepo != nil { + tenant, err := s.tenantRepo.GetTenantByID(ctx, tenantID) + if err != nil { + return nil, err + } + snapshot.tenant = cloneWeKnoraCloudTenant(tenant) + } + return snapshot, nil +} + +func (s *weKnoraCloudService) failInitializeAndRollback(ctx context.Context, tenantID uint64, snapshot *weKnoraCloudInitSnapshot, err error) error { + if rollbackErr := s.rollbackInitializeState(ctx, tenantID, snapshot); rollbackErr != nil { + return fmt.Errorf("%w;回滚失败:%v", err, rollbackErr) + } + return err +} + +func (s *weKnoraCloudService) rollbackInitializeState(ctx context.Context, tenantID uint64, snapshot *weKnoraCloudInitSnapshot) error { + if snapshot == nil { + return nil + } + var rollbackErrs []string + for _, def := range weKnoraCloudModelDefs { + currentModels, err := s.repo.List(ctx, tenantID, def.modelType, types.ModelSourceRemote) + if err != nil { + rollbackErrs = append(rollbackErrs, err.Error()) + continue + } + snapshotByID := map[string]*types.Model{} + for _, model := range snapshot.modelsByType[def.modelType] { + snapshotByID[model.ID] = cloneWeKnoraCloudModel(model) + } + for _, current := range currentModels { + if current == nil { + continue + } + if _, ok := snapshotByID[current.ID]; ok { + continue + } + if current.Parameters.Provider == string(provider.ProviderWeKnoraCloud) && isWeKnoraCloudBuiltinModelName(current.Name) { + if err := s.repo.Delete(ctx, tenantID, current.ID); err != nil { + rollbackErrs = append(rollbackErrs, err.Error()) + } + } + } + for _, model := range snapshot.modelsByType[def.modelType] { + if err := s.repo.Update(ctx, cloneWeKnoraCloudModel(model)); err != nil { + rollbackErrs = append(rollbackErrs, err.Error()) + } + } + } + if snapshot.tenant != nil && s.tenantRepo != nil { + if err := s.tenantRepo.UpdateTenant(ctx, cloneWeKnoraCloudTenant(snapshot.tenant)); err != nil { + rollbackErrs = append(rollbackErrs, err.Error()) + } + } + if len(rollbackErrs) > 0 { + return fmt.Errorf("%s", strings.Join(rollbackErrs, "; ")) + } + return nil +} + +func isWeKnoraCloudBuiltinModelName(name string) bool { + for _, def := range weKnoraCloudModelDefs { + if def.name == name { + return true + } + } + return false +} + +func cloneWeKnoraCloudModel(model *types.Model) *types.Model { + if model == nil { + return nil + } + clone := *model + if model.Parameters.ExtraConfig != nil { + clone.Parameters.ExtraConfig = make(map[string]string, len(model.Parameters.ExtraConfig)) + for k, v := range model.Parameters.ExtraConfig { + clone.Parameters.ExtraConfig[k] = v + } + } + return &clone +} + +func cloneWeKnoraCloudTenant(tenant *types.Tenant) *types.Tenant { + if tenant == nil { + return nil + } + clone := *tenant + if tenant.ParserEngineConfig != nil { + cfg := *tenant.ParserEngineConfig + clone.ParserEngineConfig = &cfg + } + return &clone +} + +// pingWeKnoraCloud 调用 WeKnoraCloud 服务的 GET /health 接口验证可达性。 +// 注意:这里仅检查服务可达,不校验 APPID / API Key 是否有效。 +func (s *weKnoraCloudService) pingWeKnoraCloud(ctx context.Context, appID, appSecret string) error { + baseURL := strings.TrimRight(provider.WeKnoraCloudBaseURL, "/") + healthURL := baseURL + "/api/v1/health" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthURL, nil) + if err != nil { + return fmt.Errorf("创建健康检查请求失败: %w", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("WeKnoraCloud 服务不可达 (%s): %w", healthURL, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("WeKnoraCloud 健康检查返回非 200 状态码: %d", resp.StatusCode) + } + return nil +} + +// CheckStatus 检查 WeKnoraCloud 凭证是否可正常解密 +func (s *weKnoraCloudService) CheckStatus(ctx context.Context) (*types.WeKnoraCloudStatusResult, error) { + tenantID := types.MustTenantIDFromContext(ctx) + + tenant, err := s.tenantRepo.GetTenantByID(ctx, tenantID) + if err != nil || tenant == nil { + return &types.WeKnoraCloudStatusResult{HasModels: false, NeedsReinit: false}, nil + } + + // Check if tenant has WeKnoraCloud credentials in parser config + if tenant.ParserEngineConfig == nil || tenant.ParserEngineConfig.DocreaderAppID == "" || tenant.ParserEngineConfig.DocreaderAPIKey == "" { + return &types.WeKnoraCloudStatusResult{ + HasModels: true, + NeedsReinit: true, + Reason: "WeKnoraCloud 凭证为空,请重新填写 APPID 和 API Key", + }, nil + } + + // Try to decrypt the API key + _, decErr := s.cryptoSvc.DecryptString(tenant.ParserEngineConfig.DocreaderAPIKey) + if decErr != nil { + return &types.WeKnoraCloudStatusResult{ + HasModels: true, + NeedsReinit: true, + Reason: "WeKnoraCloud 凭证解密失败(服务重启后加密密钥已变更),请重新填写 APPID 和 API Key", + }, nil + } + + return &types.WeKnoraCloudStatusResult{HasModels: true, NeedsReinit: false}, nil +} + +// updateTenantDocReaderAddr updates tenant parser config with WeKnoraCloud credentials and DocReaderAddr +func (s *weKnoraCloudService) updateTenantDocReaderAddr(ctx context.Context, tenantID uint64, appID, encryptedSecret string) error { + if s.tenantRepo == nil { + return fmt.Errorf("tenant repository is required") + } + + tenant, err := s.tenantRepo.GetTenantByID(ctx, tenantID) + if err != nil { + return err + } + if tenant.ParserEngineConfig == nil { + tenant.ParserEngineConfig = &types.ParserEngineConfig{} + } + tenant.ParserEngineConfig.DocReaderAddr = strings.TrimRight(provider.WeKnoraCloudBaseURL, "/") + "/api/v1/doc/reader" + tenant.ParserEngineConfig.DocreaderAppID = appID + tenant.ParserEngineConfig.DocreaderAPIKey = encryptedSecret + return s.tenantRepo.UpdateTenant(ctx, tenant) +} + +func (s *weKnoraCloudService) upsertModel( + ctx context.Context, + tenantID uint64, + name string, + modelType types.ModelType, + remoteModelName string, + appID, encryptedSecret string, +) (string, error) { + // 列出该类型的远程模型,过滤出 weknoracloud provider + models, err := s.repo.List(ctx, tenantID, modelType, types.ModelSourceRemote) + if err != nil { + return "", err + } + + var existing *types.Model + for _, m := range models { + if m.Name == name && m.Parameters.Provider == string(provider.ProviderWeKnoraCloud) { + existing = m + break + } + } + + // 先清除同类型其他模型的默认标记 + excludeID := "" + if existing != nil { + excludeID = existing.ID + } + if err := s.repo.ClearDefaultByType(ctx, uint(tenantID), modelType, excludeID); err != nil { + return "", err + } + + if existing != nil { + // 更新凭证和默认标记 + existing.Parameters.BaseURL = provider.WeKnoraCloudBaseURL + existing.Parameters.AppID = appID + existing.Parameters.AppSecret = encryptedSecret + if existing.Parameters.ExtraConfig == nil { + existing.Parameters.ExtraConfig = map[string]string{} + } + existing.Parameters.ExtraConfig["remote_model_name"] = remoteModelName + existing.IsDefault = true + if err := s.repo.Update(ctx, existing); err != nil { + return "", err + } + return "updated", nil + } + + // 创建新模型 + newModel := &types.Model{ + ID: uuid.New().String(), + TenantID: tenantID, + Name: name, + Type: modelType, + Source: types.ModelSourceRemote, + IsBuiltin: true, + IsDefault: true, + Status: types.ModelStatusActive, + Parameters: types.ModelParameters{ + Provider: string(provider.ProviderWeKnoraCloud), + BaseURL: provider.WeKnoraCloudBaseURL, + AppID: appID, + AppSecret: encryptedSecret, + ExtraConfig: map[string]string{ + "remote_model_name": remoteModelName, + }, + }, + } + if err := s.repo.Create(ctx, newModel); err != nil { + return "", err + } + return "created", nil +} diff --git a/internal/container/container.go b/internal/container/container.go index 56ba1059..1edac5a2 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -6,6 +6,7 @@ package container import ( "context" "database/sql" + "encoding/base64" "encoding/json" "fmt" "net/url" @@ -63,6 +64,7 @@ import ( "github.com/Tencent/WeKnora/internal/im/telegram" "github.com/Tencent/WeKnora/internal/im/wechat" "github.com/Tencent/WeKnora/internal/im/wecom" + "github.com/Tencent/WeKnora/internal/infrastructure/crypto" "github.com/Tencent/WeKnora/internal/infrastructure/docparser" infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search" "github.com/Tencent/WeKnora/internal/logger" @@ -98,6 +100,7 @@ func BuildContainer(container *dig.Container) *dig.Container { // Core infrastructure configuration logger.Debugf(ctx, "[Container] Registering core infrastructure...") must(container.Provide(config.LoadConfig)) + must(container.Provide(initCryptoService)) must(container.Provide(initTracer)) must(container.Provide(initDatabase)) must(container.Provide(initFileService)) @@ -169,6 +172,7 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(service.NewDatasetService)) must(container.Provide(service.NewEvaluationService)) must(container.Provide(service.NewUserService)) + must(container.Provide(service.NewWeKnoraCloudService)) // Extract services - register individual extracters with names must(container.Provide(service.NewChunkExtractService, dig.Name("chunkExtractor"))) @@ -269,6 +273,7 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(imPkg.NewService)) must(container.Invoke(registerIMAdapterFactories)) must(container.Provide(handler.NewIMHandler)) + must(container.Provide(handler.NewWeKnoraCloudHandler)) logger.Debugf(ctx, "[Container] HTTP handlers registered") // Router configuration @@ -306,6 +311,92 @@ func initTracer() (*tracing.Tracer, error) { return tracing.InitTracer() } +// cryptoStateFile 是持久化 crypto 状态的文件路径,存储在 data-files volume 中 +// 保证重启后能恢复相同的 masterKey 和 salt,避免已加密数据无法解密 +const cryptoStateFile = "/data/files/.crypto_state.json" + +// cryptoState 持久化存储的 crypto 配置 +type cryptoState struct { + MasterKey string `json:"master_key"` + Salt string `json:"salt"` // base64 编码 +} + +func initCryptoService() (*crypto.CryptoService, error) { + cfg := crypto.LoadConfigFromEnv() + + // 如果环境变量同时提供了 MasterKey 和 Salt,直接使用(优先级最高) + if cfg.MasterKey != "" && cfg.Salt != "" { + return crypto.NewCryptoServiceFromConfig(cfg) + } + + // 尝试从持久化文件恢复状态 + if state, err := loadCryptoState(cryptoStateFile); err == nil { + // 文件读取成功,环境变量中未覆盖的字段从文件补充 + if cfg.MasterKey == "" { + cfg.MasterKey = state.MasterKey + } + if cfg.Salt == "" { + cfg.Salt = state.Salt + } + return crypto.NewCryptoServiceFromConfig(cfg) + } + + // 文件不存在或读取失败,使用/生成配置后写入文件 + if cfg.MasterKey == "" { + cfg.MasterKey = "weknora-default-key" + } + + // 调用 NewCryptoServiceFromConfig 会在 cfg.Salt 为空时自动生成随机 salt + svc, err := crypto.NewCryptoServiceFromConfig(cfg) + if err != nil { + return nil, err + } + + // 将实际使用的 masterKey 和 salt 持久化,供重启恢复 + saltB64 := base64.StdEncoding.EncodeToString(svc.GetSalt()) + state := &cryptoState{ + MasterKey: cfg.MasterKey, + Salt: saltB64, + } + if saveErr := saveCryptoState(cryptoStateFile, state); saveErr != nil { + // 持久化失败仅记录警告,不阻止服务启动 + logger.Warnf(context.Background(), "[CryptoService] 无法持久化 crypto 状态到 %s: %v (重启后已加密数据将无法解密)", cryptoStateFile, saveErr) + } else { + logger.Infof(context.Background(), "[CryptoService] crypto 状态已持久化到 %s", cryptoStateFile) + } + + return svc, nil +} + +// loadCryptoState 从文件加载 crypto 状态 +func loadCryptoState(path string) (*cryptoState, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var state cryptoState + if err := json.Unmarshal(data, &state); err != nil { + return nil, fmt.Errorf("parse crypto state: %w", err) + } + if state.MasterKey == "" || state.Salt == "" { + return nil, fmt.Errorf("invalid crypto state: missing fields") + } + return &state, nil +} + +// saveCryptoState 将 crypto 状态写入文件 +func saveCryptoState(path string, state *cryptoState) error { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("mkdir: %w", err) + } + data, err := json.Marshal(state) + if err != nil { + return fmt.Errorf("marshal: %w", err) + } + // 权限 0o600:仅 owner 可读写 + return os.WriteFile(path, data, 0o600) +} + func initRedisClient() (*redis.Client, error) { redisAddr := os.Getenv("REDIS_ADDR") if redisAddr == "" { @@ -407,7 +498,7 @@ func initDatabase(cfg *config.Config) (*gorm.DB, error) { dbPath = "./data/weknora.db" } if dir := filepath.Dir(dbPath); dir != "." && dir != "" { - if err := os.MkdirAll(dir, 0755); err != nil { + if err := os.MkdirAll(dir, 0o755); err != nil { return nil, fmt.Errorf("failed to create SQLite data directory %s: %w", dir, err) } } diff --git a/internal/handler/system.go b/internal/handler/system.go index bd7d5540..e1dbd323 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" + "github.com/Tencent/WeKnora/internal/application/service" "github.com/Tencent/WeKnora/internal/application/service/file" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/database" @@ -28,14 +29,20 @@ type SystemHandler struct { cfg *config.Config neo4jDriver neo4j.Driver documentReader interfaces.DocumentReader + tenantSvc interfaces.TenantService } // NewSystemHandler creates a new system handler -func NewSystemHandler(cfg *config.Config, neo4jDriver neo4j.Driver, documentReader interfaces.DocumentReader) *SystemHandler { +func NewSystemHandler(cfg *config.Config, + neo4jDriver neo4j.Driver, + documentReader interfaces.DocumentReader, + tenantSvc interfaces.TenantService, +) *SystemHandler { return &SystemHandler{ cfg: cfg, neo4jDriver: neo4jDriver, documentReader: documentReader, + tenantSvc: tenantSvc, } } @@ -133,9 +140,6 @@ func (h *SystemHandler) getDocReaderConnInfo() (addr, transport string) { // @Success 200 {object} map[string]interface{} "解析引擎列表" // @Router /system/parser-engines [get] func (h *SystemHandler) ListParserEngines(c *gin.Context) { - docreaderAddr, docreaderTransport := h.getDocReaderConnInfo() - connected := h.documentReader != nil && h.documentReader.IsConnected() - var overrides map[string]string if v, exists := c.Get(types.TenantInfoContextKey.String()); exists { if tenant, ok := v.(*types.Tenant); ok && tenant != nil && tenant.ParserEngineConfig != nil { @@ -143,7 +147,9 @@ func (h *SystemHandler) ListParserEngines(c *gin.Context) { } } - remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides) + reader, docreaderAddr, docreaderTransport := h.resolveDocReader(c.Request.Context(), overrides) + connected := reader != nil && reader.IsConnected() + remoteEngines := h.fetchRemoteEngines(c.Request.Context(), reader, overrides) engines := docparser.ListAllEngines(connected, overrides, remoteEngines) c.JSON(200, gin.H{"code": 0, "msg": "success", "data": engines, "docreader_addr": docreaderAddr, "docreader_transport": docreaderTransport, "connected": connected}) } @@ -194,7 +200,7 @@ func (h *SystemHandler) ReconnectDocReader(c *gin.Context) { overrides = tenant.ParserEngineConfig.ToOverridesMap() } } - remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides) + remoteEngines := h.fetchRemoteEngines(c.Request.Context(), h.documentReader, overrides) engines := docparser.ListAllEngines(true, overrides, remoteEngines) _, docreaderTransport := h.getDocReaderConnInfo() @@ -211,28 +217,46 @@ func (h *SystemHandler) ReconnectDocReader(c *gin.Context) { // @Success 200 // @Router /system/parser-engines/check [post] func (h *SystemHandler) CheckParserEngines(c *gin.Context) { - docreaderAddr, docreaderTransport := h.getDocReaderConnInfo() - connected := h.documentReader != nil && h.documentReader.IsConnected() - var body types.ParserEngineConfig if err := c.ShouldBindJSON(&body); err != nil { c.JSON(400, gin.H{"code": 1, "msg": "请求体格式错误"}) return } overrides := body.ToOverridesMap() - remoteEngines := h.fetchRemoteEngines(c.Request.Context(), overrides) + reader, docreaderAddr, docreaderTransport := h.resolveDocReader(c.Request.Context(), overrides) + connected := reader != nil && reader.IsConnected() + remoteEngines := h.fetchRemoteEngines(c.Request.Context(), reader, overrides) engines := docparser.ListAllEngines(connected, overrides, remoteEngines) c.JSON(200, gin.H{"code": 0, "msg": "success", "data": engines, "docreader_addr": docreaderAddr, "docreader_transport": docreaderTransport, "connected": connected}) } +func (h *SystemHandler) resolveDocReader(ctx context.Context, overrides map[string]string) (interfaces.DocumentReader, string, string) { + if len(overrides) > 0 { + if addr := strings.TrimSpace(overrides["docreader_addr"]); addr != "" && service.IsWeKnoraCloudDocReaderAddr(addr) { + reader := h.ResolveDocumentReader(ctx, addr) + return reader, addr, transportFromDocReaderAddr(addr) + } + } + + addr, transport := h.getDocReaderConnInfo() + return h.documentReader, addr, transport +} + +func transportFromDocReaderAddr(addr string) string { + if strings.HasPrefix(strings.ToLower(strings.TrimSpace(addr)), "https://") { + return "https" + } + return "http" +} + // fetchRemoteEngines queries the remote docreader for its engine list. // Returns nil on any error (e.g. not connected), letting the caller // fall back to Go's static registry only. -func (h *SystemHandler) fetchRemoteEngines(ctx context.Context, overrides map[string]string) []types.ParserEngineInfo { - if h.documentReader == nil || !h.documentReader.IsConnected() { +func (h *SystemHandler) fetchRemoteEngines(ctx context.Context, reader interfaces.DocumentReader, overrides map[string]string) []types.ParserEngineInfo { + if reader == nil || !reader.IsConnected() { return nil } - engines, err := h.documentReader.ListEngines(ctx, overrides) + engines, err := reader.ListEngines(ctx, overrides) if err != nil { logger.Warnf(ctx, "Failed to fetch remote engines from docreader: %v", err) return nil @@ -768,7 +792,7 @@ func (h *SystemHandler) checkMinio(c *gin.Context, ctx context.Context, cfg *typ if cfg.Mode == "remote" { if blocked, reason := isBlockedStorageEndpoint(endpoint); blocked { - logger.Warnf(ctx, "Storage check: MinIO endpoint blocked by SSRF protection", "endpoint", endpoint) + logger.Warnf(ctx, "Storage check: MinIO endpoint blocked by SSRF protection endpoint=%s", endpoint) c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: false, Message: reason}}) return } @@ -935,3 +959,27 @@ func (h *SystemHandler) checkS3(c *gin.Context, ctx context.Context, cfg *types. } c.JSON(200, gin.H{"code": 0, "data": StorageCheckResponse{OK: true, Message: fmt.Sprintf("连接成功,Bucket「%s」已确认存在", cfg.BucketName)}}) } + +func (h *SystemHandler) ResolveDocumentReader(ctx context.Context, addr string) interfaces.DocumentReader { + if addr == "" { + return h.documentReader + } + + if service.IsWeKnoraCloudDocReaderAddr(addr) { + creds := h.tenantSvc.GetDocreaderCredentials(ctx) + if creds == nil { + return nil + } + reader, err := docparser.NewWeKnoraCloudSignedDocumentReader(addr, creds.AppID, creds.APIKey) + if err != nil { + return nil + } + return reader + } + + reader, err := docparser.NewHTTPDocumentReader(addr) + if err != nil || reader == nil { + return reader + } + return reader +} diff --git a/internal/handler/weknoracloud.go b/internal/handler/weknoracloud.go new file mode 100644 index 00000000..fdd918e5 --- /dev/null +++ b/internal/handler/weknoracloud.go @@ -0,0 +1,51 @@ +package handler + +import ( + "net/http" + + "github.com/Tencent/WeKnora/internal/types/interfaces" + "github.com/gin-gonic/gin" +) + +// WeKnoraCloudHandler 处理 WeKnoraCloud 初始化请求 +type WeKnoraCloudHandler struct { + svc interfaces.WeKnoraCloudService +} + +// NewWeKnoraCloudHandler 构造函数 +func NewWeKnoraCloudHandler(svc interfaces.WeKnoraCloudService) *WeKnoraCloudHandler { + return &WeKnoraCloudHandler{svc: svc} +} + +type initializeWeKnoraCloudRequest struct { + AppID string `json:"app_id" binding:"required"` + AppSecret string `json:"app_secret" binding:"required"` +} + +// Initialize POST /api/v1/models/weknoracloud/initialize +func (h *WeKnoraCloudHandler) Initialize(c *gin.Context) { + var req initializeWeKnoraCloudRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + result, err := h.svc.Initialize(c.Request.Context(), req.AppID, req.AppSecret) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + c.JSON(http.StatusOK, result) +} + +// Status GET /api/v1/models/weknoracloud/status +// 检查当前租户的 WeKnoraCloud 凭证是否完好,如需重新初始化则返回 needs_reinit=true +func (h *WeKnoraCloudHandler) Status(c *gin.Context) { + result, err := h.svc.CheckStatus(c.Request.Context()) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + c.JSON(http.StatusOK, result) +} diff --git a/internal/infrastructure/crypto/config.go b/internal/infrastructure/crypto/config.go new file mode 100644 index 00000000..f9db8c7e --- /dev/null +++ b/internal/infrastructure/crypto/config.go @@ -0,0 +1,224 @@ +package crypto + +import ( + "encoding/base64" + "fmt" + "os" + "strconv" +) + +// Config 加密服务配置结构 +type Config struct { + // MasterKey 主密钥,用于派生加密密钥 + // 可以从环境变量或配置文件中读取 + MasterKey string `json:"master_key" yaml:"master_key" env:"CRYPTO_MASTER_KEY"` + + // Salt 盐值,用于密钥派生 + // 如果为空,将自动生成随机盐值 + Salt string `json:"salt" yaml:"salt" env:"CRYPTO_SALT"` + + // SaltLength 盐值长度(当Salt为空时使用) + SaltLength int `json:"salt_length" yaml:"salt_length" env:"CRYPTO_SALT_LENGTH" default:"16"` + + // KeyDerivationIterations PBKDF2迭代次数 + KeyDerivationIterations int `json:"key_derivation_iterations" yaml:"key_derivation_iterations" env:"CRYPTO_ITERATIONS" default:"10000"` + + // KeyLength 派生密钥长度 + KeyLength int `json:"key_length" yaml:"key_length" env:"CRYPTO_KEY_LENGTH" default:"32"` +} + +// DefaultConfig 返回默认配置 +func DefaultConfig() *Config { + return &Config{ + MasterKey: "", + Salt: "", + SaltLength: 16, + KeyDerivationIterations: 10000, + KeyLength: 32, + } +} + +// LoadConfigFromEnv 从环境变量加载配置 +func LoadConfigFromEnv() *Config { + config := DefaultConfig() + + // 从环境变量读取主密钥 + if masterKey := os.Getenv("CRYPTO_MASTER_KEY"); masterKey != "" { + config.MasterKey = masterKey + } + + // 从环境变量读取盐值 + if salt := os.Getenv("CRYPTO_SALT"); salt != "" { + config.Salt = salt + } + + // 从环境变量读取盐值长度 + if saltLengthStr := os.Getenv("CRYPTO_SALT_LENGTH"); saltLengthStr != "" { + if saltLength, err := strconv.Atoi(saltLengthStr); err == nil && saltLength >= 8 { + config.SaltLength = saltLength + } + } + + // 从环境变量读取迭代次数 + if iterationsStr := os.Getenv("CRYPTO_ITERATIONS"); iterationsStr != "" { + if iterations, err := strconv.Atoi(iterationsStr); err == nil && iterations > 0 { + config.KeyDerivationIterations = iterations + } + } + + // 从环境变量读取密钥长度 + if keyLengthStr := os.Getenv("CRYPTO_KEY_LENGTH"); keyLengthStr != "" { + if keyLength, err := strconv.Atoi(keyLengthStr); err == nil && keyLength >= 16 { + config.KeyLength = keyLength + } + } + + return config +} + +// Validate 验证配置是否有效 +func (c *Config) Validate() error { + if c.MasterKey == "" { + return fmt.Errorf("master key cannot be empty") + } + + if c.SaltLength < 8 { + return fmt.Errorf("salt length must be at least 8 bytes") + } + + if c.KeyDerivationIterations < 1000 { + return fmt.Errorf("key derivation iterations must be at least 1000") + } + + if c.KeyLength < 16 { + return fmt.Errorf("key length must be at least 16 bytes") + } + + return nil +} + +// GetSaltBytes 获取盐值的字节数组形式 +func (c *Config) GetSaltBytes() ([]byte, error) { + if c.Salt != "" { + // 如果配置了盐值,解码Base64格式 + saltBytes, err := base64.StdEncoding.DecodeString(c.Salt) + if err != nil { + return nil, fmt.Errorf("failed to decode salt from base64: %w", err) + } + return saltBytes, nil + } + + // 如果没有配置盐值,生成随机盐值 + salt, err := GenerateRandomSalt(c.SaltLength) + if err != nil { + return nil, fmt.Errorf("failed to generate random salt: %w", err) + } + + return salt, nil +} + +// NewCryptoServiceFromConfig 从配置创建加密服务 +func NewCryptoServiceFromConfig(config *Config) (*CryptoService, error) { + if err := config.Validate(); err != nil { + return nil, fmt.Errorf("invalid config: %w", err) + } + + saltBytes, err := config.GetSaltBytes() + if err != nil { + return nil, fmt.Errorf("failed to get salt bytes: %w", err) + } + + return NewCryptoService(config.MasterKey, saltBytes) +} + +// ExampleConfigUsage 展示配置使用示例 +func ExampleConfigUsage() { + fmt.Println("=== 配置使用示例 ===") + + // 方法1: 手动创建配置 + manualConfig := &Config{ + MasterKey: "my-secret-master-key", + Salt: "", // 自动生成盐值 + SaltLength: 16, + KeyDerivationIterations: 10000, + KeyLength: 32, + } + + cryptoService1, err := NewCryptoServiceFromConfig(manualConfig) + if err != nil { + fmt.Printf("手动配置创建失败: %v\n", err) + } else { + fmt.Println("手动配置创建成功") + _ = cryptoService1 + } + + // 方法2: 从环境变量加载配置 + // 首先设置环境变量 + os.Setenv("CRYPTO_MASTER_KEY", "env-master-key") + os.Setenv("CRYPTO_SALT", base64.StdEncoding.EncodeToString([]byte("env-salt-123456"))) + os.Setenv("CRYPTO_SALT_LENGTH", "16") + os.Setenv("CRYPTO_ITERATIONS", "10000") + os.Setenv("CRYPTO_KEY_LENGTH", "32") + + envConfig := LoadConfigFromEnv() + cryptoService2, err := NewCryptoServiceFromConfig(envConfig) + if err != nil { + fmt.Printf("环境变量配置创建失败: %v\n", err) + } else { + fmt.Println("环境变量配置创建成功") + _ = cryptoService2 + } + + // 清理环境变量 + os.Unsetenv("CRYPTO_MASTER_KEY") + os.Unsetenv("CRYPTO_SALT") + os.Unsetenv("CRYPTO_SALT_LENGTH") + os.Unsetenv("CRYPTO_ITERATIONS") + os.Unsetenv("CRYPTO_KEY_LENGTH") +} + +// GenerateConfigExample 生成配置示例 +func GenerateConfigExample() { + fmt.Println("\n=== 配置生成示例 ===") + + // 生成随机主密钥 + masterKey, err := GenerateRandomKey(32) + if err != nil { + fmt.Printf("生成主密钥失败: %v\n", err) + return + } + + // 生成随机盐值 + salt, err := GenerateRandomSalt(16) + if err != nil { + fmt.Printf("生成盐值失败: %v\n", err) + return + } + + saltBase64 := base64.StdEncoding.EncodeToString(salt) + + fmt.Println("生成的配置示例:") + fmt.Printf("CRYPTO_MASTER_KEY=%s\n", masterKey) + fmt.Printf("CRYPTO_SALT=%s\n", saltBase64) + fmt.Printf("CRYPTO_SALT_LENGTH=16\n") + fmt.Printf("CRYPTO_ITERATIONS=10000\n") + fmt.Printf("CRYPTO_KEY_LENGTH=32\n") + + fmt.Println("\nYAML格式配置示例:") + fmt.Printf(`master_key: %s +salt: %s +salt_length: 16 +key_derivation_iterations: 10000 +key_length: 32 +`, masterKey, saltBase64) + + fmt.Println("\nJSON格式配置示例:") + fmt.Printf(`{ + "master_key": "%s", + "salt": "%s", + "salt_length": 16, + "key_derivation_iterations": 10000, + "key_length": 32 +} +`, masterKey, saltBase64) +} diff --git a/internal/infrastructure/crypto/crypto.go b/internal/infrastructure/crypto/crypto.go new file mode 100644 index 00000000..d2a47465 --- /dev/null +++ b/internal/infrastructure/crypto/crypto.go @@ -0,0 +1,163 @@ +package crypto + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + + "golang.org/x/crypto/pbkdf2" +) + +// CryptoService 提供密钥加密解密服务 +type CryptoService struct { + masterKey []byte + salt []byte +} + +// NewCryptoService 创建新的加密服务实例 +// masterKey: 主密钥,用于派生加密密钥 +// salt: 盐值,用于密钥派生,如果为空则生成随机盐 +func NewCryptoService(masterKey string, salt []byte) (*CryptoService, error) { + if masterKey == "" { + return nil, errors.New("master key cannot be empty") + } + + // 如果未提供盐值,生成随机盐 + if salt == nil || len(salt) == 0 { + salt = make([]byte, 16) + if _, err := rand.Read(salt); err != nil { + return nil, fmt.Errorf("failed to generate salt: %w", err) + } + } + + return &CryptoService{ + masterKey: []byte(masterKey), + salt: salt, + }, nil +} + +// deriveKey 使用PBKDF2派生加密密钥 +func (cs *CryptoService) deriveKey() []byte { + return pbkdf2.Key(cs.masterKey, cs.salt, 10000, 32, sha256.New) +} + +// Encrypt 加密数据 +func (cs *CryptoService) Encrypt(plaintext []byte) (string, error) { + if len(plaintext) == 0 { + return "", errors.New("plaintext cannot be empty") + } + + key := cs.deriveKey() + + // 创建AES块密码 + block, err := aes.NewCipher(key) + if err != nil { + return "", fmt.Errorf("failed to create cipher: %w", err) + } + + // 生成随机IV + iv := make([]byte, aes.BlockSize) + if _, err := rand.Read(iv); err != nil { + return "", fmt.Errorf("failed to generate IV: %w", err) + } + + // 使用CTR模式加密 + stream := cipher.NewCTR(block, iv) + ciphertext := make([]byte, len(plaintext)) + stream.XORKeyStream(ciphertext, plaintext) + + // 组合IV和密文 + result := make([]byte, len(iv)+len(ciphertext)) + copy(result[:aes.BlockSize], iv) + copy(result[aes.BlockSize:], ciphertext) + + // 返回Base64编码的结果 + return base64.StdEncoding.EncodeToString(result), nil +} + +// Decrypt 解密数据 +func (cs *CryptoService) Decrypt(encryptedData string) ([]byte, error) { + if encryptedData == "" { + return nil, errors.New("encrypted data cannot be empty") + } + + // 解码Base64数据 + data, err := base64.StdEncoding.DecodeString(encryptedData) + if err != nil { + return nil, fmt.Errorf("failed to decode base64: %w", err) + } + + if len(data) < aes.BlockSize { + return nil, errors.New("encrypted data too short") + } + + key := cs.deriveKey() + + // 创建AES块密码 + block, err := aes.NewCipher(key) + if err != nil { + return nil, fmt.Errorf("failed to create cipher: %w", err) + } + + // 提取IV和密文 + iv := data[:aes.BlockSize] + ciphertext := data[aes.BlockSize:] + + // 使用CTR模式解密 + stream := cipher.NewCTR(block, iv) + plaintext := make([]byte, len(ciphertext)) + stream.XORKeyStream(plaintext, ciphertext) + + return plaintext, nil +} + +// EncryptString 加密字符串 +func (cs *CryptoService) EncryptString(plaintext string) (string, error) { + return cs.Encrypt([]byte(plaintext)) +} + +// DecryptString 解密字符串 +func (cs *CryptoService) DecryptString(encryptedData string) (string, error) { + plaintext, err := cs.Decrypt(encryptedData) + if err != nil { + return "", err + } + return string(plaintext), nil +} + +// GetSalt 获取当前使用的盐值 +func (cs *CryptoService) GetSalt() []byte { + return cs.salt +} + +// GenerateRandomKey 生成随机密钥 +func GenerateRandomKey(length int) (string, error) { + if length < 16 { + return "", errors.New("key length must be at least 16 bytes") + } + + key := make([]byte, length) + if _, err := rand.Read(key); err != nil { + return "", fmt.Errorf("failed to generate random key: %w", err) + } + + return base64.StdEncoding.EncodeToString(key), nil +} + +// GenerateRandomSalt 生成随机盐值 +func GenerateRandomSalt(length int) ([]byte, error) { + if length < 8 { + return nil, errors.New("salt length must be at least 8 bytes") + } + + salt := make([]byte, length) + if _, err := rand.Read(salt); err != nil { + return nil, fmt.Errorf("failed to generate random salt: %w", err) + } + + return salt, nil +} diff --git a/internal/infrastructure/docparser/engine_registry.go b/internal/infrastructure/docparser/engine_registry.go index 294ccdd6..09af532d 100644 --- a/internal/infrastructure/docparser/engine_registry.go +++ b/internal/infrastructure/docparser/engine_registry.go @@ -3,6 +3,7 @@ package docparser import ( "strings" + "github.com/Tencent/WeKnora/internal/models/provider" "github.com/Tencent/WeKnora/internal/types" ) @@ -27,6 +28,7 @@ func RegisterEngine(e EngineRegistration) { func init() { RegisterEngine(&builtinEngine{}) RegisterEngine(&simpleEngine{}) + RegisterEngine(&weKnoraCloudEngine{}) RegisterEngine(&mineruEngine{}) RegisterEngine(&mineruCloudEngine{}) } @@ -54,6 +56,9 @@ func (e *builtinEngine) CheckAvailable(docreaderConnected bool, _ map[string]str // SimpleEngineName is the engine name for Go-native simple format handling. const SimpleEngineName = "simple" +// WeKnoraCloudEngineName is the engine name for WeKnoraCloud-backed document parsing. +const WeKnoraCloudEngineName = "weknoracloud" + // --------------------------------------------------------------------------- // simple — Go handles md/txt/csv natively, no external service needed. // Distinct from docreader's "builtin" which uses Python libraries for @@ -73,6 +78,24 @@ func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string return true, "" } +// --------------------------------------------------------------------------- +// weknoracloud — Tenant-scoped WeKnoraCloud docreader with signed requests. +// --------------------------------------------------------------------------- + +type weKnoraCloudEngine struct{} + +func (e *weKnoraCloudEngine) Name() string { return WeKnoraCloudEngineName } +func (e *weKnoraCloudEngine) Description() string { return "WeKnoraCloud document reader" } +func (e *weKnoraCloudEngine) FileTypes(_ bool) []string { + return weKnoraCloudSupportedFileTypes() +} +func (e *weKnoraCloudEngine) CheckAvailable(docreaderConnected bool, overrides map[string]string) (bool, string) { + if docreaderConnected && isWeKnoraCloudDocReaderAddr(overrides["docreader_addr"]) { + return true, "" + } + return false, "WeKnoraCloud document reader is not initialized" +} + // --------------------------------------------------------------------------- // mineru — Go-native, calls self-hosted MinerU API directly // --------------------------------------------------------------------------- @@ -169,3 +192,7 @@ func ListAllEngines(docreaderConnected bool, overrides map[string]string, remote return result } + +func isWeKnoraCloudDocReaderAddr(addr string) bool { + return strings.TrimSuffix(strings.TrimSpace(addr), "/") == strings.TrimRight(provider.WeKnoraCloudBaseURL, "/")+"/api/v1/doc/reader" +} diff --git a/internal/infrastructure/docparser/weknoracloud_http_reader.go b/internal/infrastructure/docparser/weknoracloud_http_reader.go new file mode 100644 index 00000000..ce936f5c --- /dev/null +++ b/internal/infrastructure/docparser/weknoracloud_http_reader.go @@ -0,0 +1,245 @@ +package docparser + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "sort" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/Tencent/WeKnora/internal/types" + "github.com/google/uuid" +) + +// WeKnoraCloudSignedDocumentReader implements the docreader HTTP protocol with WeKnoraCloud signing. +type WeKnoraCloudSignedDocumentReader struct { + baseURL string + appID string + apiKey string + client *http.Client + initialPollInterval time.Duration + maxPollInterval time.Duration + pollTimeout time.Duration +} + +func NewWeKnoraCloudSignedDocumentReader(baseURL, appID, apiKey string) (*WeKnoraCloudSignedDocumentReader, error) { + baseURL = strings.TrimSuffix(strings.TrimSpace(baseURL), "/") + if baseURL == "" { + return nil, fmt.Errorf("docreader base URL is required") + } + if appID == "" { + return nil, fmt.Errorf("WeKnoraCloud appID is required") + } + if apiKey == "" { + return nil, fmt.Errorf("WeKnoraCloud apiKey is required") + } + return &WeKnoraCloudSignedDocumentReader{ + baseURL: baseURL, + appID: appID, + apiKey: apiKey, + initialPollInterval: 500 * time.Millisecond, + maxPollInterval: 10 * time.Second, + pollTimeout: 20 * time.Minute, + client: &http.Client{ + Timeout: 500 * time.Minute, + Transport: &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 90 * time.Second, + MaxIdleConnsPerHost: 5, + }, + }, + }, nil +} + +func (p *WeKnoraCloudSignedDocumentReader) Reconnect(addr string) error { + p.baseURL = strings.TrimSuffix(strings.TrimSpace(addr), "/") + return nil +} + +func (p *WeKnoraCloudSignedDocumentReader) IsConnected() bool { return p.baseURL != "" } + +func (p *WeKnoraCloudSignedDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) { + if !p.IsConnected() { + return nil, errNotConnected + } + return []types.ParserEngineInfo{{ + Name: WeKnoraCloudEngineName, + Description: "WeKnoraCloud signed docreader", + FileTypes: weKnoraCloudSupportedFileTypes(), + Available: true, + }}, nil +} + +func (p *WeKnoraCloudSignedDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) { + if !p.IsConnected() { + return nil, errNotConnected + } + body := httpReadRequest{ + FileName: req.FileName, + FileType: req.FileType, + URL: req.URL, + Title: req.Title, + RequestID: req.RequestID, + Config: &httpReadConfig{ + ParserEngine: req.ParserEngine, + ParserEngineOverrides: req.ParserEngineOverrides, + }, + } + if len(req.FileContent) > 0 { + body.FileContent = base64.StdEncoding.EncodeToString(req.FileContent) + } + jsonBody, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("http marshal read request: %w", err) + } + httpReq, err := p.newSignedRequest(ctx, http.MethodPost, p.baseURL, jsonBody) + if err != nil { + return nil, err + } + resp, err := p.client.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("http read failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusAccepted && resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("http read status %d: %s", resp.StatusCode, string(bodyBytes)) + } + var submit weKnoraCloudAsyncSubmitResponse + if err := json.NewDecoder(resp.Body).Decode(&submit); err != nil { + return nil, fmt.Errorf("http decode read submit response: %w", err) + } + if strings.TrimSpace(submit.TaskID) == "" { + return nil, fmt.Errorf("weknoracloud docreader submit response missing task_id") + } + return p.pollTaskResult(ctx, submit.TaskID) +} + +type weKnoraCloudAsyncSubmitResponse struct { + TaskID string `json:"task_id"` + Status string `json:"status"` + Message string `json:"message"` + CreatedAt int64 `json:"created_at"` +} + +type weKnoraCloudAsyncTaskResponse struct { + TaskID string `json:"task_id"` + Status string `json:"status"` + Message string `json:"message"` + Progress float64 `json:"progress"` + Result *httpReadResponse `json:"result"` + Error string `json:"error"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + +func weKnoraCloudSupportedFileTypes() []string { + seen := map[string]struct{}{} + var fileTypes []string + for _, engine := range localEngines { + if engine.Name() == WeKnoraCloudEngineName { + continue + } + for _, fileType := range engine.FileTypes(true) { + if _, ok := seen[fileType]; ok { + continue + } + seen[fileType] = struct{}{} + fileTypes = append(fileTypes, fileType) + } + } + sort.Strings(fileTypes) + return fileTypes +} + +func (p *WeKnoraCloudSignedDocumentReader) pollTaskResult(ctx context.Context, taskID string) (*types.ReadResult, error) { + pollCtx := ctx + if _, ok := ctx.Deadline(); !ok && p.pollTimeout > 0 { + var cancel context.CancelFunc + pollCtx, cancel = context.WithTimeout(ctx, p.pollTimeout) + defer cancel() + } + statusURL := strings.TrimSuffix(p.baseURL, "/reader") + "/" + taskID + currentInterval := p.initialPollInterval + for { + httpReq, err := p.newSignedRequest(pollCtx, http.MethodGet, statusURL, nil) + if err != nil { + return nil, err + } + resp, err := p.client.Do(httpReq) + if err != nil { + return nil, fmt.Errorf("http poll task failed: %w", err) + } + var taskResp weKnoraCloudAsyncTaskResponse + func() { + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + err = fmt.Errorf("http poll task status %d: %s", resp.StatusCode, string(bodyBytes)) + return + } + if decodeErr := json.NewDecoder(resp.Body).Decode(&taskResp); decodeErr != nil { + err = fmt.Errorf("http decode task response: %w", decodeErr) + } + }() + if err != nil { + return nil, err + } + switch taskResp.Status { + case "completed": + if taskResp.Result == nil { + return &types.ReadResult{}, nil + } + return fromHTTPReadResponse(taskResp.Result), nil + case "failed": + if taskResp.Error != "" { + return nil, fmt.Errorf("weknoracloud docreader task failed: %s", taskResp.Error) + } + return nil, fmt.Errorf("weknoracloud docreader task failed: %s", taskResp.Message) + case "cancelled": + if taskResp.Error != "" { + return nil, fmt.Errorf("weknoracloud docreader task cancelled: %s", taskResp.Error) + } + return nil, fmt.Errorf("weknoracloud docreader task cancelled") + } + if err := pollCtx.Err(); err != nil { + return nil, err + } + + // Exponential backoff: multiply by 1.5 each time, cap at maxPollInterval + select { + case <-pollCtx.Done(): + return nil, pollCtx.Err() + case <-time.After(currentInterval): + // Update interval for next iteration + nextInterval := time.Duration(float64(currentInterval) * 1.5) + if nextInterval > p.maxPollInterval { + nextInterval = p.maxPollInterval + } + currentInterval = nextInterval + } + } +} + +func (p *WeKnoraCloudSignedDocumentReader) newSignedRequest(ctx context.Context, method, url string, body []byte) (*http.Request, error) { + requestID := uuid.New().String() + if len(body) == 0 { + body = []byte("{}") + } + httpReq, err := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("http new request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.ContentLength = int64(len(body)) + for k, v := range utils.Sign(p.appID, p.apiKey, requestID, string(body)) { + httpReq.Header.Set(k, v) + } + return httpReq, nil +} diff --git a/internal/models/chat/chat.go b/internal/models/chat/chat.go index 5360880b..f9a2b99c 100644 --- a/internal/models/chat/chat.go +++ b/internal/models/chat/chat.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "net/http" "strings" "github.com/Tencent/WeKnora/internal/models/provider" @@ -93,13 +94,15 @@ type Chat interface { } type ChatConfig struct { - Source types.ModelSource - BaseURL string - ModelName string - APIKey string - ModelID string - Provider string - Extra map[string]any + Source types.ModelSource + BaseURL string + ModelName string + APIKey string + ModelID string + Provider string + ExtraConfig map[string]string + AppID string + AppSecret string // 加密值,由工厂函数调用方传入,在 NewWeKnoraCloudChat 中使用前已解密 } // NewChat 创建聊天实例 @@ -134,6 +137,11 @@ func NewRemoteChat(config *ChatConfig) (Chat, error) { if spec.EndpointCustomizer != nil { remoteChat.SetEndpointCustomizer(spec.EndpointCustomizer) } + if spec.HeaderCustomizer != nil { + remoteChat.SetHeaderCustomizer(func(req *http.Request, body []byte) error { + return spec.HeaderCustomizer(remoteChat, req, body) + }) + } } return remoteChat, nil diff --git a/internal/models/chat/chat_provider_spec.go b/internal/models/chat/chat_provider_spec.go index cd0ded9e..b35ebe44 100644 --- a/internal/models/chat/chat_provider_spec.go +++ b/internal/models/chat/chat_provider_spec.go @@ -2,10 +2,13 @@ package chat import ( "context" + "net/http" "strings" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/models/provider" + modelutils "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/google/uuid" "github.com/sashabaranov/go-openai" ) @@ -20,11 +23,22 @@ type ProviderSpec struct { RequestCustomizer func(req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool) (any, bool) // EndpointCustomizer: provider-specific endpoint URL override. EndpointCustomizer func(baseURL string, modelID string, isStream bool) string + // HeaderCustomizer: provider-specific raw HTTP header customization. + HeaderCustomizer func(chat *RemoteAPIChat, req *http.Request, body []byte) error } // chatProviderSpecs is the ordered list of provider specs. // Order matters: more specific specs (with ModelMatcher) should come before generic ones. var chatProviderSpecs = []ProviderSpec{ + // WeKnoraCloud + { + Provider: provider.ProviderWeKnoraCloud, + RequestCustomizer: weKnoraCloudRequestCustomizer, + EndpointCustomizer: func(baseURL string, _ string, _ bool) string { + return strings.TrimRight(baseURL, "/") + "/api/v1/chat/completions" + }, + HeaderCustomizer: weKnoraCloudHeaderCustomizer, + }, // Aliyun Qwen Thinking Models (must be before generic Aliyun) { Provider: provider.ProviderAliyun, @@ -95,6 +109,61 @@ type ThinkingChatCompletionRequest struct { // --- Customizer functions --- +type weKnoraCloudChatRequest struct { + Model string `json:"model"` + Messages []weKnoraCloudMessage `json:"messages"` + Stream bool `json:"stream"` + MaxTokens int `json:"max_tokens,omitempty"` + Temperature float64 `json:"temperature,omitempty"` + TopP float64 `json:"top_p,omitempty"` +} + +type weKnoraCloudMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +func weKnoraCloudRequestCustomizer(req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool) (any, bool) { + weKnoraCloudReq := weKnoraCloudChatRequest{ + Model: req.Model, + Messages: convertToWeKnoraCloudMessagesFromOpenAI(req.Messages), + Stream: isStream, + } + if opts != nil { + weKnoraCloudReq.Temperature = opts.Temperature + weKnoraCloudReq.TopP = opts.TopP + weKnoraCloudReq.MaxTokens = opts.MaxTokens + } + return weKnoraCloudReq, true +} + +func weKnoraCloudHeaderCustomizer(chat *RemoteAPIChat, req *http.Request, body []byte) error { + requestID := uuid.NewString() + headers := modelutils.Sign(chat.appID, chat.appSecret, requestID, string(body)) + for k, v := range headers { + req.Header.Set(k, v) + } + return nil +} + +func convertToWeKnoraCloudMessagesFromOpenAI(messages []openai.ChatCompletionMessage) []weKnoraCloudMessage { + result := make([]weKnoraCloudMessage, 0, len(messages)) + for _, m := range messages { + content := m.Content + if content == "" && len(m.MultiContent) > 0 { + var textParts []string + for _, part := range m.MultiContent { + if part.Type == openai.ChatMessagePartTypeText && part.Text != "" { + textParts = append(textParts, part.Text) + } + } + content = strings.Join(textParts, "\n") + } + result = append(result, weKnoraCloudMessage{Role: m.Role, Content: content}) + } + return result +} + // qwenThinkingRequestCustomizer 自定义 Qwen 系列(阿里云)模型的思考请求 func qwenThinkingRequestCustomizer( req *openai.ChatCompletionRequest, opts *ChatOptions, isStream bool, diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go index 39e3ae53..25846811 100644 --- a/internal/models/chat/remote_api.go +++ b/internal/models/chat/remote_api.go @@ -39,6 +39,8 @@ type RemoteAPIChat struct { baseURL string apiKey string provider provider.ProviderName + appID string + appSecret string // requestCustomizer 允许子类自定义请求 // 返回自定义请求体(如果为 nil 则使用标准请求)和是否需要使用原始 HTTP 请求 @@ -47,6 +49,9 @@ type RemoteAPIChat struct { // endpointCustomizer 允许子类自定义请求的 endpoint // 返回是否使用自定义请求地址, 返回空则使用默认OpenAI格式地址 endpointCustomizer func(baseURL string, modelID string, isStream bool) (endpoint string) + + // headerCustomizer 允许子类自定义原始 HTTP 请求头(例如签名认证) + headerCustomizer func(req *http.Request, body []byte) error } // NewRemoteAPIChat 创建远程 API 聊天实例 @@ -69,11 +74,9 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) { config.AzureModelMapperFunc = func(model string) string { return model } - if chatConfig.Extra != nil { - if v, ok := chatConfig.Extra["api_version"]; ok { - if vs, ok := v.(string); ok && vs != "" { - config.APIVersion = vs - } + if chatConfig.ExtraConfig != nil { + if v, ok := chatConfig.ExtraConfig["api_version"]; ok { + config.APIVersion = v } } } else { @@ -83,13 +86,30 @@ func NewRemoteAPIChat(chatConfig *ChatConfig) (*RemoteAPIChat, error) { } } + modelName := chatConfig.ModelName + if chatConfig.ExtraConfig != nil { + if override := strings.TrimSpace(chatConfig.ExtraConfig["remote_model_name"]); override != "" { + modelName = override + } + } + if providerName == provider.ProviderWeKnoraCloud { + if chatConfig.AppID == "" { + return nil, fmt.Errorf("WeKnoraCloud provider: AppID is required") + } + if chatConfig.AppSecret == "" { + return nil, fmt.Errorf("WeKnoraCloud provider: AppSecret is required") + } + } + return &RemoteAPIChat{ - modelName: chatConfig.ModelName, + modelName: modelName, client: openai.NewClientWithConfig(config), modelID: chatConfig.ModelID, baseURL: chatConfig.BaseURL, apiKey: apiKey, provider: providerName, + appID: chatConfig.AppID, + appSecret: chatConfig.AppSecret, }, nil } @@ -103,6 +123,11 @@ func (c *RemoteAPIChat) SetEndpointCustomizer(customizer func(baseURL string, mo c.endpointCustomizer = customizer } +// SetHeaderCustomizer 设置原始 HTTP 请求头自定义器 +func (c *RemoteAPIChat) SetHeaderCustomizer(customizer func(req *http.Request, body []byte) error) { + c.headerCustomizer = customizer +} + // ConvertMessages 转换消息格式为 OpenAI 格式(导出供子类使用) func (c *RemoteAPIChat) ConvertMessages(messages []Message) []openai.ChatCompletionMessage { openaiMessages := make([]openai.ChatCompletionMessage, 0, len(messages)) @@ -330,7 +355,12 @@ func (c *RemoteAPIChat) chatWithRawHTTP(ctx context.Context, endpoint string, cu } httpReq.Header.Set("Content-Type", "application/json") - if c.provider == provider.ProviderAzureOpenAI { + + if c.headerCustomizer != nil { + if err := c.headerCustomizer(httpReq, jsonData); err != nil { + return nil, fmt.Errorf("customize headers: %w", err) + } + } else if c.provider == provider.ProviderAzureOpenAI { httpReq.Header.Set("api-key", c.apiKey) } else { httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) @@ -486,7 +516,12 @@ func (c *RemoteAPIChat) chatStreamWithRawHTTP(ctx context.Context, endpoint stri } httpReq.Header.Set("Content-Type", "application/json") - if c.provider == provider.ProviderAzureOpenAI { + + if c.headerCustomizer != nil { + if err := c.headerCustomizer(httpReq, jsonData); err != nil { + return nil, fmt.Errorf("customize headers: %w", err) + } + } else if c.provider == provider.ProviderAzureOpenAI { httpReq.Header.Set("api-key", c.apiKey) } else { httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) diff --git a/internal/models/embedding/embedder.go b/internal/models/embedding/embedder.go index d62abb51..a73d3208 100644 --- a/internal/models/embedding/embedder.go +++ b/internal/models/embedding/embedder.go @@ -47,7 +47,9 @@ type Config struct { Dimensions int `json:"dimensions"` ModelID string `json:"model_id"` Provider string `json:"provider"` - Extra map[string]any `json:"extra,omitempty"` + ExtraConfig map[string]string `json:"extra_config"` + AppID string + AppSecret string // 加密值,工厂函数调用方传入,使用前已解密 } // NewEmbedder creates an embedder based on the configuration @@ -129,11 +131,9 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll return embedder, err case provider.ProviderAzureOpenAI: apiVersion := "2024-10-21" - if config.Extra != nil { - if v, ok := config.Extra["api_version"]; ok { - if vs, ok := v.(string); ok && vs != "" { - apiVersion = vs - } + if config.ExtraConfig != nil { + if v, ok := config.ExtraConfig["api_version"]; ok { + apiVersion = v } } embedder, err = NewAzureOpenAIEmbedder(config.APIKey, @@ -153,6 +153,9 @@ func NewEmbedder(config Config, pooler EmbedderPooler, ollamaService *ollama.Oll config.ModelID, pooler) return embedder, err + case provider.ProviderWeKnoraCloud: + embedder, err = NewWeKnoraCloudEmbedder(config) + return embedder, err default: // Use OpenAI-compatible embedder for other providers embedder, err = NewOpenAIEmbedder(config.APIKey, diff --git a/internal/models/embedding/weknoracloud.go b/internal/models/embedding/weknoracloud.go new file mode 100644 index 00000000..9dec9682 --- /dev/null +++ b/internal/models/embedding/weknoracloud.go @@ -0,0 +1,139 @@ +package embedding + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/google/uuid" +) + +const weKnoraCloudEmbedPath = "/api/v1/embeddings" + +// WeKnoraCloudEmbedder 实现 embedding.Embedder 接口,对接 WeKnoraCloud /api/v1/embeddings +type WeKnoraCloudEmbedder struct { + modelName string + remoteModelName string + modelID string + appID string + apiKey string + baseURL string + dimensions int + client *http.Client + EmbedderPooler +} + +// NewWeKnoraCloudEmbedder 构造 WeKnoraCloudEmbedder +func NewWeKnoraCloudEmbedder(config Config) (*WeKnoraCloudEmbedder, error) { + if config.AppID == "" { + return nil, fmt.Errorf("WeKnoraCloud embedder: AppID is required") + } + if config.AppSecret == "" { + return nil, fmt.Errorf("WeKnoraCloud embedder: AppSecret is required") + } + remoteModelName := "" + if config.ExtraConfig != nil { + remoteModelName = strings.TrimSpace(config.ExtraConfig["remote_model_name"]) + } + return &WeKnoraCloudEmbedder{ + modelName: config.ModelName, + remoteModelName: remoteModelName, + modelID: config.ModelID, + appID: config.AppID, + apiKey: config.AppSecret, + baseURL: strings.TrimRight(config.BaseURL, "/"), + dimensions: config.Dimensions, + client: &http.Client{Timeout: 60 * time.Second}, + }, nil +} + +type weKnoraCloudEmbedRequest struct { + Model string `json:"model"` + Input []string `json:"input"` +} + +type weKnoraCloudEmbedResponse struct { + Data []struct { + Index int `json:"index"` + Embedding []float32 `json:"embedding"` + } `json:"data"` +} + +func (e *WeKnoraCloudEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { + results, err := e.BatchEmbed(ctx, []string{text}) + if err != nil { + return nil, err + } + if len(results) == 0 { + return nil, fmt.Errorf("weknoracloud embedder: empty response") + } + return results[0], nil +} + +func (e *WeKnoraCloudEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]float32, error) { + reqBody := weKnoraCloudEmbedRequest{Model: e.effectiveModelName(), Input: texts} + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("weknoracloud embedder: marshal: %w", err) + } + + requestID := uuid.New().String() + headers := utils.Sign(e.appID, e.apiKey, requestID, string(bodyBytes)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.baseURL+weKnoraCloudEmbedPath, bytes.NewReader(bodyBytes)) + if err != nil { + return nil, fmt.Errorf("weknoracloud embedder: create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := e.client.Do(req) + if err != nil { + return nil, fmt.Errorf("weknoracloud embedder: do request: %w", err) + } + defer resp.Body.Close() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("weknoracloud embedder: read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("weknoracloud embedder: status %d: %s", resp.StatusCode, string(respBytes)) + } + + var embedResp weKnoraCloudEmbedResponse + if err := json.Unmarshal(respBytes, &embedResp); err != nil { + return nil, fmt.Errorf("weknoracloud embedder: unmarshal: %w", err) + } + + result := make([][]float32, len(texts)) + for _, item := range embedResp.Data { + if item.Index < len(result) { + result[item.Index] = item.Embedding + } + } + return result, nil +} + +func (e *WeKnoraCloudEmbedder) BatchEmbedWithPool(ctx context.Context, model Embedder, texts []string) ([][]float32, error) { + return e.BatchEmbed(ctx, texts) +} + +func (e *WeKnoraCloudEmbedder) effectiveModelName() string { + if e.remoteModelName != "" { + return e.remoteModelName + } + return e.modelName +} + +func (e *WeKnoraCloudEmbedder) GetModelName() string { return e.modelName } +func (e *WeKnoraCloudEmbedder) GetModelID() string { return e.modelID } +func (e *WeKnoraCloudEmbedder) GetDimensions() int { return e.dimensions } diff --git a/internal/models/provider/provider.go b/internal/models/provider/provider.go index 801963c1..6a2da7b7 100644 --- a/internal/models/provider/provider.go +++ b/internal/models/provider/provider.go @@ -87,6 +87,7 @@ func AllProviders() []ProviderName { ProviderNvidia, ProviderNovita, ProviderAzureOpenAI, + ProviderWeKnoraCloud, } } diff --git a/internal/models/provider/weknoracloud.go b/internal/models/provider/weknoracloud.go new file mode 100644 index 00000000..39e44d79 --- /dev/null +++ b/internal/models/provider/weknoracloud.go @@ -0,0 +1,41 @@ +package provider + +import "github.com/Tencent/WeKnora/internal/types" + +const ( + ProviderWeKnoraCloud ProviderName = "weknoracloud" + + // WeKnoraCloudBaseURL WeKnoraCloud 服务硬编码 Base URL(统一入口,路径由各实现拼接) + WeKnoraCloudBaseURL = "https://weknora.weixin.qq.com" +) + +type WeKnoraCloudProvider struct{} + +func init() { + Register(&WeKnoraCloudProvider{}) +} + +func (p *WeKnoraCloudProvider) Info() ProviderInfo { + return ProviderInfo{ + Name: ProviderWeKnoraCloud, + DisplayName: "WeKnoraCloud", + Description: "WeKnoraCloud 平台,提供 Chat / Embedding / Rerank 模型,使用 APPID + API Key 签名认证", + DefaultURLs: map[types.ModelType]string{ + types.ModelTypeKnowledgeQA: WeKnoraCloudBaseURL, + types.ModelTypeEmbedding: WeKnoraCloudBaseURL, + types.ModelTypeRerank: WeKnoraCloudBaseURL, + }, + ModelTypes: []types.ModelType{ + types.ModelTypeKnowledgeQA, + types.ModelTypeEmbedding, + types.ModelTypeRerank, + }, + RequiresAuth: true, + } +} + +func (p *WeKnoraCloudProvider) ValidateConfig(config *Config) error { + // AppID/AppSecret 通过专用初始化接口写入,此处仅做结构校验。 + // 其中 AppSecret 字段当前实际承载上游 API Key。 + return nil +} diff --git a/internal/models/rerank/reranker.go b/internal/models/rerank/reranker.go index 5ed45dba..2c5a1ba2 100644 --- a/internal/models/rerank/reranker.go +++ b/internal/models/rerank/reranker.go @@ -78,12 +78,15 @@ func (d *DocumentInfo) UnmarshalJSON(data []byte) error { } type RerankerConfig struct { - APIKey string - BaseURL string - ModelName string - Source types.ModelSource - ModelID string - Provider string // Provider identifier: openai, aliyun, zhipu, siliconflow, jina, generic + APIKey string + BaseURL string + ModelName string + Source types.ModelSource + ModelID string + Provider string // Provider identifier: openai, aliyun, zhipu, siliconflow, jina, generic + ExtraConfig map[string]string + AppID string + AppSecret string // 加密值,工厂函数调用方传入,使用前已解密 } // NewReranker creates a reranker based on the configuration @@ -103,6 +106,8 @@ func NewReranker(config *RerankerConfig) (Reranker, error) { return NewJinaReranker(config) case provider.ProviderNvidia: return NewNvidiaReranker(config) + case provider.ProviderWeKnoraCloud: + return NewWeKnoraCloudReranker(config) default: return NewOpenAIReranker(config) } diff --git a/internal/models/rerank/weknoracloud.go b/internal/models/rerank/weknoracloud.go new file mode 100644 index 00000000..ea985d75 --- /dev/null +++ b/internal/models/rerank/weknoracloud.go @@ -0,0 +1,130 @@ +package rerank + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/models/utils" + "github.com/google/uuid" +) + +const weKnoraCloudRerankPath = "/api/v1/rerank" + +// WeKnoraCloudReranker 实现 rerank.Reranker 接口,对接 WeKnoraCloud /api/v1/rerank +type WeKnoraCloudReranker struct { + modelName string + remoteModelName string + modelID string + appID string + apiKey string + baseURL string + client *http.Client +} + +// NewWeKnoraCloudReranker 构造 WeKnoraCloudReranker +func NewWeKnoraCloudReranker(config *RerankerConfig) (*WeKnoraCloudReranker, error) { + if config.AppID == "" { + return nil, fmt.Errorf("WeKnoraCloud reranker: AppID is required") + } + if config.AppSecret == "" { + return nil, fmt.Errorf("WeKnoraCloud reranker: AppSecret is required") + } + remoteModelName := "" + if config.ExtraConfig != nil { + remoteModelName = strings.TrimSpace(config.ExtraConfig["remote_model_name"]) + } + return &WeKnoraCloudReranker{ + modelName: config.ModelName, + remoteModelName: remoteModelName, + modelID: config.ModelID, + appID: config.AppID, + apiKey: config.AppSecret, + baseURL: strings.TrimRight(config.BaseURL, "/"), + client: &http.Client{Timeout: 60 * time.Second}, + }, nil +} + +type weKnoraCloudRerankRequest struct { + Model string `json:"model"` + Query string `json:"query"` + Documents []string `json:"documents"` +} + +type weKnoraCloudRerankResponse struct { + Results []struct { + Index int `json:"index"` + RelevanceScore float64 `json:"relevance_score"` + Document struct { + Text string `json:"text"` + } `json:"document"` + } `json:"results"` +} + +func (r *WeKnoraCloudReranker) Rerank(ctx context.Context, query string, documents []string) ([]RankResult, error) { + reqBody := weKnoraCloudRerankRequest{ + Model: r.effectiveModelName(), + Query: query, + Documents: documents, + } + bodyBytes, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("weknoracloud reranker: marshal: %w", err) + } + + requestID := uuid.New().String() + headers := utils.Sign(r.appID, r.apiKey, requestID, string(bodyBytes)) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.baseURL+weKnoraCloudRerankPath, bytes.NewReader(bodyBytes)) + if err != nil { + return nil, fmt.Errorf("weknoracloud reranker: create request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := r.client.Do(req) + if err != nil { + return nil, fmt.Errorf("weknoracloud reranker: do request: %w", err) + } + defer resp.Body.Close() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("weknoracloud reranker: read response: %w", err) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("weknoracloud reranker: status %d: %s", resp.StatusCode, string(respBytes)) + } + + var rerankResp weKnoraCloudRerankResponse + if err := json.Unmarshal(respBytes, &rerankResp); err != nil { + return nil, fmt.Errorf("weknoracloud reranker: unmarshal: %w", err) + } + + results := make([]RankResult, 0, len(rerankResp.Results)) + for _, item := range rerankResp.Results { + results = append(results, RankResult{ + Index: item.Index, + RelevanceScore: item.RelevanceScore, + Document: DocumentInfo{Text: item.Document.Text}, + }) + } + return results, nil +} + +func (r *WeKnoraCloudReranker) effectiveModelName() string { + if r.remoteModelName != "" { + return r.remoteModelName + } + return r.modelName +} + +func (r *WeKnoraCloudReranker) GetModelName() string { return r.modelName } +func (r *WeKnoraCloudReranker) GetModelID() string { return r.modelID } diff --git a/internal/models/utils/signer.go b/internal/models/utils/signer.go new file mode 100644 index 00000000..dfbecbb6 --- /dev/null +++ b/internal/models/utils/signer.go @@ -0,0 +1,93 @@ +package utils + +import ( + "bytes" + "crypto/md5" + "fmt" + "math/rand" + "sort" + "strconv" + "strings" + "time" +) + +const ( + nonceChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + nonceLength = 16 +) + +// Sign 按 WeKnoraCloud 参考实现生成请求头。 +// appID: 上游 APPID +// apiKey: 上游 API Key(当前沿用 AppSecret 字段承载) +// requestID: 每次请求唯一的 UUID 字符串 +// bodyJSON: 请求体 JSON 字符串,空请求体传 "{}" +func Sign(appID, apiKey, requestID, bodyJSON string) map[string]string { + timestamp := strconv.FormatInt(time.Now().Unix(), 10) + nonce := generateNonce(nonceLength) + + bodyForHash := bodyJSON + if bodyForHash == "" { + bodyForHash = "{}" + } + bodyMD5 := md5Hex(bodyForHash) + + params := map[string]string{ + "x-appid": appID, + "x-api-key": apiKey, + "x-request-id": requestID, + "x-timestamp": timestamp, + "x-nonce": nonce, + "body": bodyMD5, + } + + keys := make([]string, 0, len(params)) + for k := range params { + keys = append(keys, k) + } + sort.Strings(keys) + + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, rfc3986Encode(k)+"="+rfc3986Encode(params[k])) + } + signature := md5Hex(strings.Join(parts, "&")) + + return map[string]string{ + "X-APPID": appID, + "X-API-Key": apiKey, + "X-Request-ID": requestID, + "X-Timestamp": timestamp, + "X-Nonce": nonce, + "X-Signature": signature, + } +} + +func md5Hex(s string) string { + h := md5.New() + h.Write([]byte(s)) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +func generateNonce(length int) string { + b := make([]byte, length) + for i := range b { + b[i] = nonceChars[rand.Intn(len(nonceChars))] + } + return string(b) +} + +// rfc3986Encode 对字符串做 RFC3986 编码 +// 保留字符:A-Z a-z 0-9 - _ . ~ +func rfc3986Encode(s string) string { + var buf bytes.Buffer + for _, r := range s { + if (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || + (r >= '0' && r <= '9') || r == '-' || r == '_' || + r == '.' || r == '~' { + buf.WriteRune(r) + } else { + fmt.Fprintf(&buf, "%%%02X", r) + } + } + return buf.String() +} diff --git a/internal/router/router.go b/internal/router/router.go index 5b5bdaf0..9da4a23e 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -62,6 +62,7 @@ type RouterParams struct { OrganizationHandler *handler.OrganizationHandler IMHandler *handler.IMHandler DataSourceHandler *handler.DataSourceHandler + WeKnoraCloudHandler *handler.WeKnoraCloudHandler } // NewRouter 创建新的路由 @@ -144,6 +145,7 @@ func NewRouter(params RouterParams) *gin.Engine { RegisterOrganizationRoutes(v1, params.OrganizationHandler) RegisterIMChannelRoutes(v1, params.IMHandler) RegisterDataSourceRoutes(v1, params.DataSourceHandler) + RegisterWeKnoraCloudRoutes(v1, params.WeKnoraCloudHandler) } return r @@ -805,3 +807,9 @@ func RegisterDataSourceRoutes(r *gin.RouterGroup, handler *handler.DataSourceHan ds.GET("/logs/:log_id", handler.GetSyncLog) } } + +// RegisterWeKnoraCloudRoutes 注册 WeKnoraCloud 初始化路由 +func RegisterWeKnoraCloudRoutes(r *gin.RouterGroup, handler *handler.WeKnoraCloudHandler) { + r.POST("/models/weknoracloud/initialize", handler.Initialize) + r.GET("/models/weknoracloud/status", handler.Status) +} diff --git a/internal/types/interfaces/tenant.go b/internal/types/interfaces/tenant.go index c5c9c367..5942c7e6 100644 --- a/internal/types/interfaces/tenant.go +++ b/internal/types/interfaces/tenant.go @@ -28,6 +28,8 @@ type TenantService interface { SearchTenants(ctx context.Context, keyword string, tenantID uint64, page, pageSize int) ([]*types.Tenant, int64, error) // GetTenantByIDForUser gets a tenant by ID with permission check GetTenantByIDForUser(ctx context.Context, tenantID uint64, userID string) (*types.Tenant, error) + // GetTenantByAPIKey gets a tenant by API key + GetDocreaderCredentials(ctx context.Context) *types.DocreaderCredentials } // TenantRepository defines the tenant repository interface diff --git a/internal/types/interfaces/weknoracloud.go b/internal/types/interfaces/weknoracloud.go new file mode 100644 index 00000000..4b7ff1fa --- /dev/null +++ b/internal/types/interfaces/weknoracloud.go @@ -0,0 +1,15 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// WeKnoraCloudService 处理 WeKnoraCloud 厂商的初始化逻辑 +type WeKnoraCloudService interface { + Initialize(ctx context.Context, appID, appSecret string) (*types.InitializeResult, error) + // CheckStatus 检查当前租户的 WeKnoraCloud 凭证是否可正常解密 + // needsReinit=true 表示加密状态已损坏(salt 变更等),需要用户重新填写凭证 + CheckStatus(ctx context.Context) (*types.WeKnoraCloudStatusResult, error) +} diff --git a/internal/types/model.go b/internal/types/model.go index 9ee1e92d..6f1ffa4f 100644 --- a/internal/types/model.go +++ b/internal/types/model.go @@ -68,6 +68,9 @@ type ModelParameters struct { Provider string `yaml:"provider" json:"provider"` // Provider identifier: openai, aliyun, zhipu, generic ExtraConfig map[string]string `yaml:"extra_config" json:"extra_config"` // Provider-specific configuration SupportsVision bool `yaml:"supports_vision" json:"supports_vision"` // Whether the model accepts image/multimodal input + // WeKnoraCloud 厂商专用凭证 + AppID string `yaml:"app_id,omitempty" json:"app_id,omitempty"` + AppSecret string `yaml:"app_secret,omitempty" json:"app_secret,omitempty"` // AES-256 加密存储,实际承载上游 API Key } // Model represents the AI model diff --git a/internal/types/tenant.go b/internal/types/tenant.go index 0ee8b105..b6081842 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -233,7 +233,11 @@ func (c *ConversationConfig) Scan(value interface{}) error { // ParserEngineConfig holds tenant-level overrides for document parser engines (e.g. MinerU endpoint, API key). // These values take precedence over environment variables when parsing documents. type ParserEngineConfig struct { - DocReaderAddr string `json:"docreader_addr"` // 文档解析服务地址 + DocReaderAddr string `json:"docreader_addr"` // 文档解析服务地址 + // docreader 凭证 + DocreaderAppID string `json:"docreader_app_id,omitempty"` + DocreaderAPIKey string `json:"docreader_api_key,omitempty"` + MinerUEndpoint string `json:"mineru_endpoint"` // MinerU 自建服务端点 MinerUAPIKey string `json:"mineru_api_key"` // MinerU 云 API Key @@ -259,6 +263,9 @@ func (c *ParserEngineConfig) ToOverridesMap() map[string]string { return nil } m := make(map[string]string) + if c.DocReaderAddr != "" { + m["docreader_addr"] = c.DocReaderAddr + } if c.MinerUEndpoint != "" { m["mineru_endpoint"] = c.MinerUEndpoint } diff --git a/internal/types/weknoracloud.go b/internal/types/weknoracloud.go new file mode 100644 index 00000000..7b931c0e --- /dev/null +++ b/internal/types/weknoracloud.go @@ -0,0 +1,25 @@ +package types + +// WeKnoraCloudStatusResult 状态检查结果 +type WeKnoraCloudStatusResult struct { + HasModels bool `json:"has_models"` // 是否已配置 WeKnoraCloud 模型 + NeedsReinit bool `json:"needs_reinit"` // 是否需要重新初始化(凭证损坏) + Reason string `json:"reason,omitempty"` // 需要重新初始化的原因 +} + +// InitializeResult 返回每个模型的操作结果 +type InitializeResult struct { + Models []ModelAction `json:"models"` +} + +// ModelAction 单个模型的操作结果 +type ModelAction struct { + Name string `json:"name"` + Type string `json:"type"` + Action string `json:"action"` // "created" | "updated" +} + +type DocreaderCredentials struct { + AppID string + APIKey string +}