feat: enhance localization and configuration support

- Added timezone and language settings to the environment configuration.
- Introduced built-in agent configurations with multilingual support for various agents.
- Updated Docker Compose to utilize new environment variables for timezone and language.
- Created new prompt templates for question generation, summary generation, and keywords extraction.
This commit is contained in:
wizardchen
2026-03-17 15:33:19 +08:00
committed by lyingbug
parent 09d9fe5ddb
commit 139a9c40ff
54 changed files with 2524 additions and 1482 deletions
+55
View File
@@ -471,6 +471,30 @@ func (h *InitializationHandler) getKnowledgeBaseForInitialization(ctx context.Co
}
func (h *InitializationHandler) validateInitializationConfigs(ctx context.Context, req *InitializationRequest) error {
// SSRF validation for all user-supplied BaseURLs
urlsToCheck := []struct {
label string
url string
}{
{"LLM BaseURL", req.LLM.BaseURL},
{"Embedding BaseURL", req.Embedding.BaseURL},
{"Rerank BaseURL", req.Rerank.BaseURL},
}
if req.Multimodal.VLM != nil {
urlsToCheck = append(urlsToCheck, struct {
label string
url string
}{"VLM BaseURL", req.Multimodal.VLM.BaseURL})
}
for _, u := range urlsToCheck {
if u.url != "" {
if err := utils.ValidateURLForSSRF(u.url); err != nil {
logger.Warnf(ctx, "SSRF validation failed for %s: %v", u.label, err)
return errors.NewBadRequestError(fmt.Sprintf("%s 未通过安全校验: %v", u.label, err))
}
}
}
if err := h.validateMultimodalConfig(ctx, req); err != nil {
return err
}
@@ -1461,6 +1485,13 @@ func (h *InitializationHandler) CheckRemoteModel(c *gin.Context) {
return
}
// SSRF validation
if err := utils.ValidateURLForSSRF(req.BaseURL); err != nil {
logger.Warnf(ctx, "SSRF validation failed for remote model BaseURL: %v", err)
c.Error(errors.NewBadRequestError(fmt.Sprintf("Base URL 未通过安全校验: %v", err)))
return
}
// 创建模型配置进行测试
modelConfig := &types.Model{
Name: req.ModelName,
@@ -1518,6 +1549,15 @@ func (h *InitializationHandler) TestEmbeddingModel(c *gin.Context) {
return
}
// SSRF validation for embedding BaseURL
if req.BaseURL != "" {
if err := utils.ValidateURLForSSRF(req.BaseURL); err != nil {
logger.Warnf(ctx, "SSRF validation failed for embedding BaseURL: %v", err)
c.Error(errors.NewBadRequestError(fmt.Sprintf("Base URL 未通过安全校验: %v", err)))
return
}
}
// 检查是否是阿里云多模态 embedding 模型(暂不支持)
if strings.ToLower(req.Provider) == "aliyun" {
modelNameLower := strings.ToLower(req.ModelName)
@@ -1705,6 +1745,13 @@ func (h *InitializationHandler) CheckRerankModel(c *gin.Context) {
return
}
// SSRF validation
if err := utils.ValidateURLForSSRF(req.BaseURL); err != nil {
logger.Warnf(ctx, "SSRF validation failed for rerank BaseURL: %v", err)
c.Error(errors.NewBadRequestError(fmt.Sprintf("Base URL 未通过安全校验: %v", err)))
return
}
// 检查Rerank模型连接和功能
available, message := h.checkRerankModelConnection(
ctx, req.ModelName, req.BaseURL, req.APIKey,
@@ -1788,6 +1835,14 @@ func (h *InitializationHandler) TestMultimodalFunction(c *gin.Context) {
c.Error(errors.NewBadRequestError("VLM模型名称和Base URL不能为空"))
return
}
// SSRF validation for VLM BaseURL
if err := utils.ValidateURLForSSRF(req.VLMBaseURL); err != nil {
logger.Warnf(ctx, "SSRF validation failed for VLM BaseURL: %v", err)
c.Error(errors.NewBadRequestError(fmt.Sprintf("VLM Base URL 未通过安全校验: %v", err)))
return
}
switch req.StorageType {
case "cos":
// 必填:SecretID/SecretKey/Region/BucketName/AppID;PathPrefix 可选
+8
View File
@@ -360,6 +360,14 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
secutils.SanitizeForLog(req.FileName),
secutils.SanitizeForLog(req.FileType),
)
// SSRF validation for user-supplied URL
if err := secutils.ValidateURLForSSRF(req.URL); err != nil {
logger.Warnf(ctx, "SSRF validation failed for knowledge URL: %v", err)
c.Error(errors.NewBadRequestError(fmt.Sprintf("URL 未通过安全校验: %v", err)))
return
}
logger.Infof(ctx,
"Creating knowledge from URL, knowledge base ID: %s, URL: %s",
secutils.SanitizeForLog(kbID),
+20
View File
@@ -1,6 +1,7 @@
package handler
import (
"fmt"
"net/http"
"github.com/Tencent/WeKnora/internal/errors"
@@ -53,6 +54,15 @@ func (h *MCPServiceHandler) CreateMCPService(c *gin.Context) {
}
service.TenantID = tenantID
// SSRF validation for MCP service URL
if service.URL != nil && *service.URL != "" {
if err := secutils.ValidateURLForSSRF(*service.URL); err != nil {
logger.Warnf(ctx, "SSRF validation failed for MCP service URL: %v", err)
c.Error(errors.NewBadRequestError(fmt.Sprintf("MCP service URL 未通过安全校验: %v", err)))
return
}
}
if err := h.mcpServiceService.CreateMCPService(ctx, &service); err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_name": secutils.SanitizeForLog(service.Name)})
c.Error(errors.NewInternalServerError("Failed to create MCP service: " + err.Error()))
@@ -207,6 +217,16 @@ func (h *MCPServiceHandler) UpdateMCPService(c *gin.Context) {
// Explicitly set to nil if provided as null/empty
service.URL = nil
}
// SSRF validation for updated MCP service URL
if service.URL != nil && *service.URL != "" {
if err := secutils.ValidateURLForSSRF(*service.URL); err != nil {
logger.Warnf(ctx, "SSRF validation failed for MCP service URL: %v", err)
c.Error(errors.NewBadRequestError(fmt.Sprintf("MCP service URL 未通过安全校验: %v", err)))
return
}
}
if stdioConfig, ok := updateData["stdio_config"].(map[string]interface{}); ok {
config := &types.MCPStdioConfig{}
if command, ok := stdioConfig["command"].(string); ok {
+18
View File
@@ -1,6 +1,7 @@
package handler
import (
"fmt"
"net/http"
"github.com/Tencent/WeKnora/internal/application/service"
@@ -102,6 +103,15 @@ func (h *ModelHandler) CreateModel(c *gin.Context) {
logger.Infof(ctx, "Creating model, Tenant ID: %d, Model name: %s, Model type: %s",
tenantID, secutils.SanitizeForLog(req.Name), secutils.SanitizeForLog(string(req.Type)))
// SSRF validation for model BaseURL
if req.Parameters.BaseURL != "" {
if err := secutils.ValidateURLForSSRF(req.Parameters.BaseURL); err != nil {
logger.Warnf(ctx, "SSRF validation failed for model BaseURL: %v", err)
c.Error(errors.NewBadRequestError(fmt.Sprintf("Base URL 未通过安全校验: %v", err)))
return
}
}
model := &types.Model{
TenantID: tenantID,
Name: secutils.SanitizeForLog(req.Name),
@@ -293,6 +303,14 @@ func (h *ModelHandler) UpdateModel(c *gin.Context) {
model.Description = req.Description
// Check if any Parameters field is set (can't use struct comparison due to map field)
if req.Parameters.BaseURL != "" || req.Parameters.APIKey != "" || req.Parameters.Provider != "" {
// SSRF validation for updated model BaseURL
if req.Parameters.BaseURL != "" {
if err := secutils.ValidateURLForSSRF(req.Parameters.BaseURL); err != nil {
logger.Warnf(ctx, "SSRF validation failed for model BaseURL: %v", err)
c.Error(errors.NewBadRequestError(fmt.Sprintf("Base URL 未通过安全校验: %v", err)))
return
}
}
model.Parameters = req.Parameters
}
model.Source = req.Source
+15
View File
@@ -16,6 +16,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@@ -169,6 +170,13 @@ func (h *SystemHandler) ReconnectDocReader(c *gin.Context) {
return
}
// SSRF validation for docreader address
if err := secutils.ValidateURLForSSRF(addr); err != nil {
logger.Warnf(c.Request.Context(), "SSRF validation failed for docreader addr: %v", err)
c.JSON(400, gin.H{"code": 1, "msg": fmt.Sprintf("地址未通过安全校验: %v", err)})
return
}
if h.documentReader == nil {
c.JSON(500, gin.H{"code": 1, "msg": "document converter not initialized"})
return
@@ -637,11 +645,18 @@ func sanitizeStorageCheckError(err error) string {
// isBlockedStorageEndpoint checks whether a storage endpoint resolves to a dangerous
// address (cloud metadata, loopback, link-local). Unlike the stricter IsSSRFSafeURL,
// this allows private IPs since MinIO is commonly deployed on internal networks.
// It also respects the SSRF_WHITELIST environment variable for whitelisted hosts.
func isBlockedStorageEndpoint(endpoint string) (bool, string) {
host, _, err := net.SplitHostPort(endpoint)
if err != nil {
host = endpoint
}
// Check SSRF whitelist first – whitelisted hosts bypass the block check.
if secutils.IsSSRFWhitelisted(host) {
return false, ""
}
hostLower := strings.ToLower(host)
blockedHosts := []string{
+18 -3
View File
@@ -504,7 +504,7 @@ func (h *TenantHandler) GetTenantAgentConfig(c *gin.Context) {
"reflection_enabled": agent.DefaultAgentReflectionEnabled,
"allowed_tools": agenttools.DefaultAllowedTools(),
"temperature": agent.DefaultAgentTemperature,
"system_prompt": agent.ProgressiveRAGSystemPrompt,
"system_prompt": agent.GetProgressiveRAGSystemPrompt(h.config),
"use_custom_system_prompt": false,
"available_tools": availableTools,
"available_placeholders": availablePlaceholders,
@@ -516,7 +516,7 @@ func (h *TenantHandler) GetTenantAgentConfig(c *gin.Context) {
// Get system prompt, use default if empty
systemPrompt := tenant.AgentConfig.ResolveSystemPrompt(true) // webSearchEnabled doesn't matter for unified prompt
if systemPrompt == "" {
systemPrompt = agent.ProgressiveRAGSystemPrompt
systemPrompt = agent.GetProgressiveRAGSystemPrompt(h.config)
}
logger.Infof(ctx, "Retrieved tenant agent config successfully, Tenant ID: %d", tenant.ID)
@@ -1023,9 +1023,24 @@ func (h *TenantHandler) GetPromptTemplates(c *gin.Context) {
templates = &config.PromptTemplatesConfig{}
}
// Determine user language from context (set by Language middleware)
lang, _ := types.LanguageFromContext(c.Request.Context())
// Build a localized copy so the original config is never mutated
localized := &config.PromptTemplatesConfig{
SystemPrompt: config.LocalizeTemplates(templates.SystemPrompt, lang),
ContextTemplate: config.LocalizeTemplates(templates.ContextTemplate, lang),
Rewrite: config.LocalizeTemplates(templates.Rewrite, lang),
Fallback: config.LocalizeTemplates(templates.Fallback, lang),
GenerateSessionTitle: templates.GenerateSessionTitle,
GenerateSummary: templates.GenerateSummary,
KeywordsExtraction: templates.KeywordsExtraction,
AgentSystemPrompt: config.LocalizeTemplates(templates.AgentSystemPrompt, lang),
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": templates,
"data": localized,
})
}