Merge branch 'main' into login_page
This commit is contained in:
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// PluginIntoChatMessage handles the transformation of search results into chat messages
|
||||
@@ -50,9 +51,16 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
weekdayName := []string{"星期日", "星期一", "星期二", "星期三", "星期四", "星期五", "星期六"}
|
||||
var userContent bytes.Buffer
|
||||
|
||||
// 验证用户查询的安全性
|
||||
safeQuery, isValid := secutils.ValidateInput(chatManage.Query)
|
||||
if !isValid {
|
||||
logger.Errorf(ctx, "Invalid user query: %s", chatManage.Query)
|
||||
return ErrTemplateExecute.WithError(fmt.Errorf("用户查询包含非法内容"))
|
||||
}
|
||||
|
||||
// Execute template with context data
|
||||
err = tmpl.Execute(&userContent, map[string]interface{}{
|
||||
"Query": chatManage.Query, // User's original query
|
||||
"Query": safeQuery, // User's original query
|
||||
"Contexts": passages, // Extracted passages from search results
|
||||
"CurrentTime": time.Now().Format("2006-01-02 15:04:05"), // Formatted current time
|
||||
"CurrentWeek": weekdayName[time.Now().Weekday()], // Current weekday in Chinese
|
||||
|
||||
@@ -23,8 +23,8 @@ type cosFileService struct {
|
||||
}
|
||||
|
||||
// NewCosFileService creates a new COS file service instance
|
||||
func NewCosFileService(appId, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com", appId, region)
|
||||
func NewCosFileService(bucketName, region, secretId, secretKey, cosPathPrefix string) (interfaces.FileService, error) {
|
||||
bucketURL := fmt.Sprintf("https://%s.cos.%s.myqcloud.com/", bucketName, region)
|
||||
u, err := url.Parse(bucketURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse bucketURL: %w", err)
|
||||
@@ -59,7 +59,7 @@ func (s *cosFileService) SaveFile(ctx context.Context,
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file to COS: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("https://%s/%s", s.bucketURL, objectName), nil
|
||||
return fmt.Sprintf("%s%s", s.bucketURL, objectName), nil
|
||||
}
|
||||
|
||||
// GetFile retrieves a file from COS storage by its path URL
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/tracing"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/Tencent/WeKnora/services/docreader/src/client"
|
||||
"github.com/Tencent/WeKnora/services/docreader/src/proto"
|
||||
"github.com/google/uuid"
|
||||
@@ -191,15 +192,22 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
|
||||
metadataJSON = types.JSON(metadataBytes)
|
||||
}
|
||||
|
||||
// 验证文件名安全性
|
||||
safeFilename, isValid := secutils.ValidateInput(file.Filename)
|
||||
if !isValid {
|
||||
logger.Errorf(ctx, "Invalid filename: %s", file.Filename)
|
||||
return nil, werrors.NewValidationError("文件名包含非法字符")
|
||||
}
|
||||
|
||||
// Create knowledge record
|
||||
logger.Info(ctx, "Creating knowledge record")
|
||||
knowledge := &types.Knowledge{
|
||||
TenantID: tenantID,
|
||||
KnowledgeBaseID: kbID,
|
||||
Type: "file",
|
||||
Title: file.Filename,
|
||||
FileName: file.Filename,
|
||||
FileType: getFileType(file.Filename),
|
||||
Title: safeFilename,
|
||||
FileName: safeFilename,
|
||||
FileType: getFileType(safeFilename),
|
||||
FileSize: file.Size,
|
||||
FileHash: hash,
|
||||
ParseStatus: "pending",
|
||||
@@ -258,10 +266,10 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Validate URL format
|
||||
// Validate URL format and security
|
||||
logger.Info(ctx, "Validating URL")
|
||||
if !isValidURL(url) {
|
||||
logger.Error(ctx, "Invalid URL format")
|
||||
if !isValidURL(url) || !secutils.IsValidURL(url) {
|
||||
logger.Error(ctx, "Invalid or unsafe URL format")
|
||||
return nil, ErrInvalidURL
|
||||
}
|
||||
|
||||
@@ -339,6 +347,17 @@ func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context,
|
||||
logger.Info(ctx, "Start creating knowledge from passage")
|
||||
logger.Infof(ctx, "Knowledge base ID: %s, passage count: %d", kbID, len(passage))
|
||||
|
||||
// 验证段落内容安全性
|
||||
safePassages := make([]string, 0, len(passage))
|
||||
for i, p := range passage {
|
||||
safePassage, isValid := secutils.ValidateInput(p)
|
||||
if !isValid {
|
||||
logger.Errorf(ctx, "Invalid passage content at index %d", i)
|
||||
return nil, werrors.NewValidationError(fmt.Sprintf("段落 %d 包含非法内容", i+1))
|
||||
}
|
||||
safePassages = append(safePassages, safePassage)
|
||||
}
|
||||
|
||||
// Get knowledge base configuration
|
||||
logger.Info(ctx, "Getting knowledge base configuration")
|
||||
kb, err := s.kbService.GetKnowledgeBaseByID(ctx, kbID)
|
||||
@@ -370,7 +389,7 @@ func (s *knowledgeService) CreateKnowledgeFromPassage(ctx context.Context,
|
||||
|
||||
// Process passages asynchronously
|
||||
logger.Info(ctx, "Starting asynchronous passage processing")
|
||||
go s.processDocumentFromPassage(ctx, kb, knowledge, passage)
|
||||
go s.processDocumentFromPassage(ctx, kb, knowledge, safePassages)
|
||||
|
||||
logger.Infof(ctx, "Knowledge from passage created successfully, ID: %s", knowledge.ID)
|
||||
return knowledge, nil
|
||||
|
||||
@@ -229,7 +229,7 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
||||
false,
|
||||
)
|
||||
case "cos":
|
||||
if os.Getenv("COS_APP_ID") == "" ||
|
||||
if os.Getenv("COS_BUCKET_NAME") == "" ||
|
||||
os.Getenv("COS_REGION") == "" ||
|
||||
os.Getenv("COS_SECRET_ID") == "" ||
|
||||
os.Getenv("COS_SECRET_KEY") == "" ||
|
||||
@@ -237,7 +237,7 @@ func initFileService(cfg *config.Config) (interfaces.FileService, error) {
|
||||
return nil, fmt.Errorf("missing COS configuration")
|
||||
}
|
||||
return file.NewCosFileService(
|
||||
os.Getenv("COS_APP_ID"),
|
||||
os.Getenv("COS_BUCKET_NAME"),
|
||||
os.Getenv("COS_REGION"),
|
||||
os.Getenv("COS_SECRET_ID"),
|
||||
os.Getenv("COS_SECRET_KEY"),
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
@@ -52,6 +53,13 @@ func (h *ChunkHandler) ListKnowledgeChunks(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// 对 chunk 内容进行安全清理
|
||||
for _, chunk := range result.Data.([]*types.Chunk) {
|
||||
if chunk.Content != "" {
|
||||
chunk.Content = secutils.SanitizeForDisplay(chunk.Content)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(
|
||||
ctx, "Successfully retrieved knowledge chunks list, knowledge ID: %s, total: %d",
|
||||
knowledgeID, result.Total,
|
||||
|
||||
@@ -180,6 +180,10 @@ func (h *InitializationHandler) CheckStatus(c *gin.Context) {
|
||||
})
|
||||
return
|
||||
}
|
||||
// ignore api key in response for security
|
||||
for _, model := range models {
|
||||
model.Parameters.APIKey = ""
|
||||
}
|
||||
|
||||
logger.Info(ctx, "System is already initialized")
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
|
||||
@@ -87,7 +87,7 @@ type CreateSessionRequest struct {
|
||||
func (h *SessionHandler) CreateSession(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
|
||||
logger.Info(ctx, "Start creating session")
|
||||
logger.Infof(ctx, "Start creating session, config: %+v", h.config.Conversation)
|
||||
|
||||
// Parse and validate the request body
|
||||
var request CreateSessionRequest
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"html"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// XSS 防护相关正则表达式
|
||||
var (
|
||||
// 匹配潜在的 XSS 攻击模式
|
||||
xssPatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)<script[^>]*>.*?</script>`),
|
||||
regexp.MustCompile(`(?i)<iframe[^>]*>.*?</iframe>`),
|
||||
regexp.MustCompile(`(?i)<object[^>]*>.*?</object>`),
|
||||
regexp.MustCompile(`(?i)<embed[^>]*>.*?</embed>`),
|
||||
regexp.MustCompile(`(?i)<form[^>]*>.*?</form>`),
|
||||
regexp.MustCompile(`(?i)<input[^>]*>`),
|
||||
regexp.MustCompile(`(?i)<button[^>]*>.*?</button>`),
|
||||
regexp.MustCompile(`(?i)javascript:`),
|
||||
regexp.MustCompile(`(?i)vbscript:`),
|
||||
regexp.MustCompile(`(?i)onload\s*=`),
|
||||
regexp.MustCompile(`(?i)onerror\s*=`),
|
||||
regexp.MustCompile(`(?i)onclick\s*=`),
|
||||
regexp.MustCompile(`(?i)onmouseover\s*=`),
|
||||
regexp.MustCompile(`(?i)onfocus\s*=`),
|
||||
regexp.MustCompile(`(?i)onblur\s*=`),
|
||||
}
|
||||
)
|
||||
|
||||
// SanitizeHTML 清理 HTML 内容,防止 XSS 攻击
|
||||
func SanitizeHTML(input string) string {
|
||||
if input == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 检查输入长度
|
||||
if len(input) > 10000 {
|
||||
input = input[:10000]
|
||||
}
|
||||
|
||||
// 检查是否包含潜在的 XSS 攻击
|
||||
for _, pattern := range xssPatterns {
|
||||
if pattern.MatchString(input) {
|
||||
// 如果包含恶意内容,进行 HTML 转义
|
||||
return html.EscapeString(input)
|
||||
}
|
||||
}
|
||||
|
||||
// 如果内容相对安全,返回原内容
|
||||
return input
|
||||
}
|
||||
|
||||
// EscapeHTML 转义 HTML 特殊字符
|
||||
func EscapeHTML(input string) string {
|
||||
if input == "" {
|
||||
return ""
|
||||
}
|
||||
return html.EscapeString(input)
|
||||
}
|
||||
|
||||
// ValidateInput 验证用户输入
|
||||
func ValidateInput(input string) (string, bool) {
|
||||
if input == "" {
|
||||
return "", true
|
||||
}
|
||||
|
||||
// 检查长度
|
||||
if len(input) > 10000 {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// 检查是否包含控制字符
|
||||
for _, r := range input {
|
||||
if r < 32 && r != 9 && r != 10 && r != 13 {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// 检查 UTF-8 有效性
|
||||
if !utf8.ValidString(input) {
|
||||
return "", false
|
||||
}
|
||||
|
||||
// 检查是否包含潜在的 XSS 攻击
|
||||
for _, pattern := range xssPatterns {
|
||||
if pattern.MatchString(input) {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(input), true
|
||||
}
|
||||
|
||||
// IsValidURL 验证 URL 是否安全
|
||||
func IsValidURL(url string) bool {
|
||||
if url == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查长度
|
||||
if len(url) > 2048 {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查协议
|
||||
if !strings.HasPrefix(strings.ToLower(url), "http://") &&
|
||||
!strings.HasPrefix(strings.ToLower(url), "https://") {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查是否包含恶意内容
|
||||
for _, pattern := range xssPatterns {
|
||||
if pattern.MatchString(url) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// IsValidImageURL 验证图片 URL 是否安全
|
||||
func IsValidImageURL(url string) bool {
|
||||
if !IsValidURL(url) {
|
||||
return false
|
||||
}
|
||||
|
||||
// 检查是否为图片文件
|
||||
imageExtensions := []string{".jpg", ".jpeg", ".png", ".gif", ".webp", ".svg", ".bmp", ".ico"}
|
||||
lowerURL := strings.ToLower(url)
|
||||
|
||||
for _, ext := range imageExtensions {
|
||||
if strings.Contains(lowerURL, ext) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// CleanMarkdown 清理 Markdown 内容
|
||||
func CleanMarkdown(input string) string {
|
||||
if input == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 移除潜在的恶意脚本
|
||||
cleaned := input
|
||||
for _, pattern := range xssPatterns {
|
||||
cleaned = pattern.ReplaceAllString(cleaned, "")
|
||||
}
|
||||
|
||||
return strings.TrimSpace(cleaned)
|
||||
}
|
||||
|
||||
// SanitizeForDisplay 为显示清理内容
|
||||
func SanitizeForDisplay(input string) string {
|
||||
if input == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
// 首先清理 Markdown
|
||||
cleaned := CleanMarkdown(input)
|
||||
|
||||
// 然后进行 HTML 转义
|
||||
escaped := html.EscapeString(cleaned)
|
||||
|
||||
return escaped
|
||||
}
|
||||
Reference in New Issue
Block a user