Merge branch 'main' into login_page

This commit is contained in:
lyingbug
2025-09-16 10:24:05 +08:00
committed by GitHub
18 changed files with 526 additions and 46 deletions
@@ -12,6 +12,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
// PluginIntoChatMessage handles the transformation of search results into chat messages
@@ -50,9 +51,16 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
weekdayName := []string{"星期日", "星期一", "星期二", "星期三", "星期四", "星期五", "星期六"}
var userContent bytes.Buffer
// 验证用户查询的安全性
safeQuery, isValid := secutils.ValidateInput(chatManage.Query)
if !isValid {
logger.Errorf(ctx, "Invalid user query: %s", chatManage.Query)
return ErrTemplateExecute.WithError(fmt.Errorf("用户查询包含非法内容"))
}
// Execute template with context data
err = tmpl.Execute(&userContent, map[string]interface{}{
"Query": chatManage.Query, // User's original query
"Query": safeQuery, // User's original query
"Contexts": passages, // Extracted passages from search results
"CurrentTime": time.Now().Format("2006-01-02 15:04:05"), // Formatted current time
"CurrentWeek": weekdayName[time.Now().Weekday()], // Current weekday in Chinese
+3 -3
View File
@@ -23,8 +23,8 @@ type cosFileService struct {
}
// NewCosFileService creates a new COS file service instance
func NewCosFileService(appId, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com", appId, region)
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
u, err := url.Parse(bucketURL)
if err != nil {
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
@@ -59,7 +59,7 @@ func (s *cosFileService) SaveFile(ctx context.Context,
if err != nil {
return "", fmt.Errorf("failed to upload file to COS: %w", err)
}
return fmt.Sprintf("https://%s/%s", s.bucketURL, objectName), nil
return fmt.Sprintf("%s%s", s.bucketURL, objectName), nil
}
// GetFile retrieves a file from COS storage by its path URL
+26 -7
View File
@@ -25,6 +25,7 @@ import (
"github.com/Tencent/WeKnora/internal/tracing"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/Tencent/WeKnora/services/docreader/src/client"
"github.com/Tencent/WeKnora/services/docreader/src/proto"
"github.com/google/uuid"
@@ -191,15 +192,22 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
metadataJSON = types.JSON(metadataBytes)
}
// 验证文件名安全性
safeFilename, isValid := secutils.ValidateInput(file.Filename)
if !isValid {
logger.Errorf(ctx, "Invalid filename: %s", file.Filename)
return nil, werrors.NewValidationError("文件名包含非法字符")
}
// Create knowledge record
logger.Info(ctx, "Creating knowledge record")
knowledge := &types.Knowledge{
TenantID: tenantID,
KnowledgeBaseID: kbID,
Type: "file",
Title: file.Filename,
FileName: file.Filename,
FileType: getFileType(file.Filename),
Title: safeFilename,
FileName: safeFilename,
FileType: getFileType(safeFilename),
FileSize: file.Size,
FileHash: hash,
ParseStatus: "pending",
@@ -258,10 +266,10 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
return nil, err
}
// Validate URL format
// Validate URL format and security
logger.Info(ctx, "Validating URL")
if !isValidURL(url) {
logger.Error(ctx, "Invalid URL format")
if !isValidURL(url) || !secutils.IsValidURL(url) {
logger.Error(ctx, "Invalid or unsafe URL format")
return nil, ErrInvalidURL
}
@@ -339,6 +347,17 @@ func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context,
logger.Info(ctx, "Start creating knowledge from passage")
logger.Infof(ctx, "Knowledge base ID: %s, passage count: %d", kbID, len(passage))
// 验证段落内容安全性
safePassages := make([]string, 0, len(passage))
for i, p := range passage {
safePassage, isValid := secutils.ValidateInput(p)
if !isValid {
logger.Errorf(ctx, "Invalid passage content at index %d", i)
return nil, werrors.NewValidationError(fmt.Sprintf("段落 %d 包含非法内容", i+1))
}
safePassages = append(safePassages, safePassage)
}
// Get knowledge base configuration
logger.Info(ctx, "Getting knowledge base configuration")
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID)
@@ -370,7 +389,7 @@ func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context,
// Process passages asynchronously
logger.Info(ctx, "Starting asynchronous passage processing")
go s.processDocumentFromPassage(ctx, kb, knowledge, passage)
go s.processDocumentFromPassage(ctx, kb, knowledge, safePassages)
logger.Infof(ctx, "Knowledge from passage created successfully, ID: %s", knowledge.ID)
return knowledge, nil
+2 -2
View File
@@ -229,7 +229,7 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
false,
)
case "cos":
if os.Getenv("COS_APP_ID") == "" ||
if os.Getenv("COS_BUCKET_NAME") == "" ||
os.Getenv("COS_REGION") == "" ||
os.Getenv("COS_SECRET_ID") == "" ||
os.Getenv("COS_SECRET_KEY") == "" ||
@@ -237,7 +237,7 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
return nil, fmt.Errorf("missing COS configuration")
}
return file.NewCosFileService(
os.Getenv("COS_APP_ID"),
os.Getenv("COS_BUCKET_NAME"),
os.Getenv("COS_REGION"),
os.Getenv("COS_SECRET_ID"),
os.Getenv("COS_SECRET_KEY"),
+8
View File
@@ -8,6 +8,7 @@ import (
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
secutils "github.com/Tencent/WeKnora/internal/utils"
"github.com/gin-gonic/gin"
)
@@ -52,6 +53,13 @@ func (h *ChunkHandler) ListKnowledgeChunks(c *gin.Context) {
return
}
// 对 chunk 内容进行安全清理
for _, chunk := range result.Data.([]*types.Chunk) {
if chunk.Content != "" {
chunk.Content = secutils.SanitizeForDisplay(chunk.Content)
}
}
logger.Infof(
ctx, "Successfully retrieved knowledge chunks list, knowledge ID: %s, total: %d",
knowledgeID, result.Total,
+4
View File
@@ -180,6 +180,10 @@ func (h *InitializationHandler) CheckStatus(c *gin.Context) {
})
return
}
// ignore api key in response for security
for _, model := range models {
model.Parameters.APIKey = ""
}
logger.Info(ctx, "System is already initialized")
c.JSON(http.StatusOK, gin.H{
+1 -1
View File
@@ -87,7 +87,7 @@ type CreateSessionRequest struct {
func (h *SessionHandler) CreateSession(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start creating session")
logger.Infof(ctx, "Start creating session, config: %+v", h.config.Conversation)
// Parse and validate the request body
var request CreateSessionRequest
+170
View File
@@ -0,0 +1,170 @@
package utils
import (
"html"
"regexp"
"strings"
"unicode/utf8"
)
// XSS 防护相关正则表达式
var (
// 匹配潜在的 XSS 攻击模式
xssPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)<script[^>]*>.*?</script>`),
regexp.MustCompile(`(?i)<iframe[^>]*>.*?</iframe>`),
regexp.MustCompile(`(?i)<object[^>]*>.*?</object>`),
regexp.MustCompile(`(?i)<embed[^>]*>.*?</embed>`),
regexp.MustCompile(`(?i)<form[^>]*>.*?</form>`),
regexp.MustCompile(`(?i)<input[^>]*>`),
regexp.MustCompile(`(?i)<button[^>]*>.*?</button>`),
regexp.MustCompile(`(?i)javascript:`),
regexp.MustCompile(`(?i)vbscript:`),
regexp.MustCompile(`(?i)onload\s*=`),
regexp.MustCompile(`(?i)onerror\s*=`),
regexp.MustCompile(`(?i)onclick\s*=`),
regexp.MustCompile(`(?i)onmouseover\s*=`),
regexp.MustCompile(`(?i)onfocus\s*=`),
regexp.MustCompile(`(?i)onblur\s*=`),
}
)
// SanitizeHTML 清理 HTML 内容,防止 XSS 攻击
func SanitizeHTML(input string) string {
if input == "" {
return ""
}
// 检查输入长度
if len(input) > 10000 {
input = input[:10000]
}
// 检查是否包含潜在的 XSS 攻击
for _, pattern := range xssPatterns {
if pattern.MatchString(input) {
// 如果包含恶意内容,进行 HTML 转义
return html.EscapeString(input)
}
}
// 如果内容相对安全,返回原内容
return input
}
// EscapeHTML 转义 HTML 特殊字符
func EscapeHTML(input string) string {
if input == "" {
return ""
}
return html.EscapeString(input)
}
// ValidateInput 验证用户输入
func ValidateInput(input string) (string, bool) {
if input == "" {
return "", true
}
// 检查长度
if len(input) > 10000 {
return "", false
}
// 检查是否包含控制字符
for _, r := range input {
if r < 32 && r != 9 && r != 10 && r != 13 {
return "", false
}
}
// 检查 UTF-8 有效性
if !utf8.ValidString(input) {
return "", false
}
// 检查是否包含潜在的 XSS 攻击
for _, pattern := range xssPatterns {
if pattern.MatchString(input) {
return "", false
}
}
return strings.TrimSpace(input), true
}
// IsValidURL 验证 URL 是否安全
func IsValidURL(url string) bool {
if url == "" {
return false
}
// 检查长度
if len(url) > 2048 {
return false
}
// 检查协议
if !strings.HasPrefix(strings.ToLower(url), "http://") &&
!strings.HasPrefix(strings.ToLower(url), "https://") {
return false
}
// 检查是否包含恶意内容
for _, pattern := range xssPatterns {
if pattern.MatchString(url) {
return false
}
}
return true
}
// IsValidImageURL 验证图片 URL 是否安全
func IsValidImageURL(url string) bool {
if !IsValidURL(url) {
return false
}
// 检查是否为图片文件
imageExtensions := []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".ico"}
lowerURL := strings.ToLower(url)
for _, ext := range imageExtensions {
if strings.Contains(lowerURL, ext) {
return true
}
}
return false
}
// CleanMarkdown 清理 Markdown 内容
func CleanMarkdown(input string) string {
if input == "" {
return ""
}
// 移除潜在的恶意脚本
cleaned := input
for _, pattern := range xssPatterns {
cleaned = pattern.ReplaceAllString(cleaned, "")
}
return strings.TrimSpace(cleaned)
}
// SanitizeForDisplay 为显示清理内容
func SanitizeForDisplay(input string) string {
if input == "" {
return ""
}
// 首先清理 Markdown
cleaned := CleanMarkdown(input)
// 然后进行 HTML 转义
escaped := html.EscapeString(cleaned)
return escaped
}