refactor: Sanitize log inputs to prevent injection attacks

This commit is contained in:
wizardchen
2025-11-25 22:19:41 +08:00
parent 2f45bf7d86
commit bf35a2861c
10 changed files with 82 additions and 41 deletions
+1 -1
View File
@@ -42,7 +42,7 @@ func (h *ChunkHandler) GetChunkByIDOnly(c *gin.Context) {
return
}
logger.Infof(ctx, "Retrieving chunk by ID, chunk ID: %s, tenant ID: %d", chunkID, tenantID)
logger.Infof(ctx, "Retrieving chunk by ID, chunk ID: %s, tenant ID: %d", secutils.SanitizeForLog(chunkID), tenantID)
// Get chunk by ID
chunk, err := h.service.GetChunkByID(ctx, chunkID)
+2 -1
View File
@@ -9,6 +9,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// FAQHandler handles FAQ knowledge base operations.
@@ -173,7 +174,7 @@ func (h *FAQHandler) SearchFAQ(c *gin.Context) {
return
}
entries, err := h.knowledgeService.SearchFAQEntries(ctx, c.Param("id"), &req)
entries, err := h.knowledgeService.SearchFAQEntries(ctx, secutils.SanitizeForLog(c.Param("id")), &req)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(err)
+7 -6
View File
@@ -26,6 +26,7 @@ import (
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/ollama/ollama/api"
@@ -209,7 +210,7 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
// 获取知识库信息
kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr)
if err != nil || kb == nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": kbIdStr})
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)})
c.Error(errors.NewNotFoundError("知识库不存在"))
return
}
@@ -351,12 +352,12 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
c.Error(errors.NewBadRequestError(err.Error()))
return
}
logger.Infof(ctx, "Starting knowledge base configuration update, kbId: %s, request: %s", kbIdStr, utils.ToJSON(req))
logger.Infof(ctx, "Starting knowledge base configuration update, kbId: %s, request: %s", secutils.SanitizeForLog(kbIdStr), secutils.SanitizeForLog(utils.ToJSON(req)))
// 获取指定知识库信息
kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbIdStr)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": kbIdStr})
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)})
c.Error(errors.NewInternalServerError("获取知识库信息失败: " + err.Error()))
return
}
@@ -669,7 +670,7 @@ func (h *InitializationHandler) InitializeByKB(c *gin.Context) {
err = h.kbRepository.UpdateKnowledgeBase(ctx, kb)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": kbIdStr})
logger.ErrorWithFields(ctx, err, map[string]interface{}{"kbId": secutils.SanitizeForLog(kbIdStr)})
c.Error(errors.NewInternalServerError("更新知识库配置失败: " + err.Error()))
return
}
@@ -768,7 +769,7 @@ func (h *InitializationHandler) CheckOllamaModels(c *gin.Context) {
modelStatus[modelName] = available
}
logger.Infof(ctx, "Model %s availability: %v", modelName, modelStatus[modelName])
logger.Infof(ctx, "Model %s availability: %v", secutils.SanitizeForLog(modelName), modelStatus[modelName])
}
c.JSON(http.StatusOK, gin.H{
@@ -1332,7 +1333,7 @@ func (h *InitializationHandler) TestEmbeddingModel(c *gin.Context) {
emb, err := embedding.NewEmbedder(cfg)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"model": req.ModelName})
logger.ErrorWithFields(ctx, err, map[string]interface{}{"model": secutils.SanitizeForLog(req.ModelName)})
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": gin.H{`available`: false, `message`: fmt.Sprintf("创建Embedder失败: %v", err), `dimension`: 0},
+3 -2
View File
@@ -11,6 +11,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
)
@@ -182,8 +183,8 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
return
}
logger.Infof(ctx, "Received URL request: %s", req.URL)
logger.Infof(ctx, "Creating knowledge from URL, knowledge base ID: %s, URL: %s", kbID, req.URL)
logger.Infof(ctx, "Received URL request: %s", secutils.SanitizeForLog(req.URL))
logger.Infof(ctx, "Creating knowledge from URL, knowledge base ID: %s, URL: %s", secutils.SanitizeForLog(kbID), secutils.SanitizeForLog(req.URL))
// Create knowledge entry from the URL
knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.EnableMultimodel)
+11 -10
View File
@@ -7,6 +7,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
)
@@ -94,7 +95,7 @@ func (h *MCPServiceHandler) GetMCPService(c *gin.Context) {
service, err := h.mcpServiceService.GetMCPServiceByID(ctx, tenantID, serviceID)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": serviceID})
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": secutils.SanitizeForLog(serviceID)})
c.Error(errors.NewNotFoundError("MCP service not found"))
return
}
@@ -210,12 +211,12 @@ func (h *MCPServiceHandler) UpdateMCPService(c *gin.Context) {
}
if err := h.mcpServiceService.UpdateMCPService(ctx, &service); err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": serviceID})
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": secutils.SanitizeForLog(serviceID)})
c.Error(errors.NewInternalServerError("Failed to update MCP service: " + err.Error()))
return
}
logger.Infof(ctx, "MCP service updated successfully: %s", serviceID)
logger.Infof(ctx, "MCP service updated successfully: %s", secutils.SanitizeForLog(serviceID))
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": service,
@@ -236,12 +237,12 @@ func (h *MCPServiceHandler) DeleteMCPService(c *gin.Context) {
}
if err := h.mcpServiceService.DeleteMCPService(ctx, tenantID, serviceID); err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": serviceID})
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": secutils.SanitizeForLog(serviceID)})
c.Error(errors.NewInternalServerError("Failed to delete MCP service: " + err.Error()))
return
}
logger.Infof(ctx, "MCP service deleted successfully: %s", serviceID)
logger.Infof(ctx, "MCP service deleted successfully: %s", secutils.SanitizeForLog(serviceID))
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "MCP service deleted successfully",
@@ -261,11 +262,11 @@ func (h *MCPServiceHandler) TestMCPService(c *gin.Context) {
return
}
logger.Infof(ctx, "Testing MCP service: %s", serviceID)
logger.Infof(ctx, "Testing MCP service: %s", secutils.SanitizeForLog(serviceID))
result, err := h.mcpServiceService.TestMCPService(ctx, tenantID, serviceID)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": serviceID})
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": secutils.SanitizeForLog(serviceID)})
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": types.MCPTestResult{
@@ -276,7 +277,7 @@ func (h *MCPServiceHandler) TestMCPService(c *gin.Context) {
return
}
logger.Infof(ctx, "MCP service test completed: %s, success: %v", serviceID, result.Success)
logger.Infof(ctx, "MCP service test completed: %s, success: %v", secutils.SanitizeForLog(serviceID), result.Success)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": result,
@@ -298,7 +299,7 @@ func (h *MCPServiceHandler) GetMCPServiceTools(c *gin.Context) {
tools, err := h.mcpServiceService.GetMCPServiceTools(ctx, tenantID, serviceID)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": serviceID})
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": secutils.SanitizeForLog(serviceID)})
c.Error(errors.NewInternalServerError("Failed to get MCP service tools: " + err.Error()))
return
}
@@ -324,7 +325,7 @@ func (h *MCPServiceHandler) GetMCPServiceResources(c *gin.Context) {
resources, err := h.mcpServiceService.GetMCPServiceResources(ctx, tenantID, serviceID)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": serviceID})
logger.ErrorWithFields(ctx, err, map[string]interface{}{"service_id": secutils.SanitizeForLog(serviceID)})
c.Error(errors.NewInternalServerError("Failed to get MCP service resources: " + err.Error()))
return
}
+2 -1
View File
@@ -8,6 +8,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
)
@@ -92,7 +93,7 @@ func (h *ModelHandler) CreateModel(c *gin.Context) {
}
logger.Infof(ctx, "Creating model, Tenant ID: %d, Model name: %s, Model type: %s",
tenantID, req.Name, req.Type)
tenantID, secutils.SanitizeForLog(req.Name), secutils.SanitizeForLog(string(req.Type)))
model := &types.Model{
TenantID: tenantID,
+7 -6
View File
@@ -12,6 +12,7 @@ import (
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
)
@@ -103,12 +104,12 @@ func (h *Handler) KnowledgeQA(c *gin.Context) {
return
}
logger.Infof(ctx, "Knowledge QA request, session ID: %s, query: %s", sessionID, request.Query)
logger.Infof(ctx, "Knowledge QA request, session ID: %s, query: %s", secutils.SanitizeForLog(sessionID), secutils.SanitizeForLog(request.Query))
// Get session to prepare knowledge base IDs
session, err := h.sessionService.GetSession(ctx, sessionID)
if err != nil {
logger.Errorf(ctx, "Failed to get session, session ID: %s, error: %v", sessionID, err)
logger.Errorf(ctx, "Failed to get session, session ID: %s, error: %v", secutils.SanitizeForLog(sessionID), err)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
@@ -117,7 +118,7 @@ func (h *Handler) KnowledgeQA(c *gin.Context) {
knowledgeBaseIDs := request.KnowledgeBaseIDs
if len(knowledgeBaseIDs) == 0 && session.KnowledgeBaseID != "" {
knowledgeBaseIDs = []string{session.KnowledgeBaseID}
logger.Infof(ctx, "No knowledge base IDs in request, using session default: %s", session.KnowledgeBaseID)
logger.Infof(ctx, "No knowledge base IDs in request, using session default: %s", secutils.SanitizeForLog(session.KnowledgeBaseID))
}
// Use shared function to handle KnowledgeQA request
@@ -158,13 +159,13 @@ func (h *Handler) AgentQA(c *gin.Context) {
// Get session information first
session, err := h.sessionService.GetSession(ctx, sessionID)
if err != nil {
logger.Errorf(ctx, "Failed to get session, session ID: %s, error: %v", sessionID, err)
logger.Errorf(ctx, "Failed to get session, session ID: %s, error: %v", secutils.SanitizeForLog(sessionID), err)
c.Error(errors.NewNotFoundError("Session not found"))
return
}
sessionJSON, err := json.Marshal(session)
if err != nil {
logger.Errorf(ctx, "Failed to marshal session, session ID: %s, error: %v", sessionID, err)
logger.Errorf(ctx, "Failed to marshal session, session ID: %s, error: %v", secutils.SanitizeForLog(sessionID), err)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
@@ -274,7 +275,7 @@ func (h *Handler) AgentQA(c *gin.Context) {
// If still empty, use session default knowledge base
if len(knowledgeBaseIDs) == 0 && session.KnowledgeBaseID != "" {
knowledgeBaseIDs = []string{session.KnowledgeBaseID}
logger.Infof(ctx, "Using session default knowledge base: %s", session.KnowledgeBaseID)
logger.Infof(ctx, "Using session default knowledge base: %s", secutils.SanitizeForLog(session.KnowledgeBaseID))
}
// Validate at least one knowledge base is available
+12 -11
View File
@@ -10,6 +10,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
)
@@ -188,17 +189,17 @@ func Logger() gin.HandlerFunc {
}
}
// 构建日志消息
logMsg := logger.GetLogger(c)
logMsg = logMsg.WithFields(map[string]interface{}{
"request_id": requestID,
"method": method,
"path": path,
"status_code": statusCode,
"size": c.Writer.Size(),
"latency": latency.String(),
"client_ip": clientIP,
})
// 构建日志消息
logMsg := logger.GetLogger(c)
logMsg = logMsg.WithFields(map[string]interface{}{
"request_id": requestID,
"method": method,
"path": secutils.SanitizeForLog(path),
"status_code": statusCode,
"size": c.Writer.Size(),
"latency": latency.String(),
"client_ip": secutils.SanitizeForLog(clientIP),
})
// 添加请求体(如果有)
if requestBody != "" {
+3 -3
View File
@@ -82,10 +82,10 @@ func (r *OpenAIReranker) Rerank(ctx context.Context, query string, documents []s
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", r.apiKey))
// Log the curl equivalent for debugging
// Log the curl equivalent for debugging (API key masked for security)
logger.GetLogger(ctx).Infof(
"curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer %s\" -d '%s'",
r.baseURL, r.apiKey, string(jsonData),
"curl -X POST %s/rerank -H \"Content-Type: application/json\" -H \"Authorization: Bearer ***\" -d '%s'",
r.baseURL, string(jsonData),
)
resp, err := r.client.Do(req)
+34
View File
@@ -169,3 +169,37 @@ func SanitizeForDisplay(input string) string {
return escaped
}
// SanitizeForLog 清理日志输入,防止日志注入攻击
// 日志注入攻击是指攻击者通过在输入中插入换行符和其他控制字符,
// 伪造日志条目,可能导致日志分析工具误判或隐藏恶意活动
func SanitizeForLog(input string) string {
if input == "" {
return ""
}
// 替换换行符(LF, CR, CRLF)为空格,防止日志注入
sanitized := strings.ReplaceAll(input, "\n", " ")
sanitized = strings.ReplaceAll(sanitized, "\r", " ")
// 替换制表符为空格
sanitized = strings.ReplaceAll(sanitized, "\t", " ")
// 移除其他控制字符(ASCII 0-31,除了空格已处理的)
var builder strings.Builder
for _, r := range sanitized {
// 保留可打印字符和常用Unicode字符
if r >= 32 || r == ' ' {
builder.WriteRune(r)
}
}
sanitized = builder.String()
// 限制长度,防止日志溢出
if len(sanitized) > 1000 {
sanitized = sanitized[:1000] + "...[truncated]"
}
return sanitized
}