diff --git a/.env.example b/.env.example index 20ccc0a7..28489480 100644 --- a/.env.example +++ b/.env.example @@ -152,4 +152,9 @@ COS_ENABLE_OLD_DOMAIN=true # NEO4J_USERNAME=neo4j # Neo4j的密码 -# NEO4J_PASSWORD=password \ No newline at end of file +# NEO4J_PASSWORD=password + +# ========== 文件上传大小限制 ========== +# 统一的文件大小限制(MB),默认为50MB +# 影响:单文件上传、gRPC消息大小、Nginx请求体大小 +# MAX_FILE_SIZE_MB=50 \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index f741aca6..a4dc7902 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -118,6 +118,8 @@ services: - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME:-} - MINIO_USE_SSL=${MINIO_USE_SSL:-} - WEB_PROXY=${WEB_PROXY:-} + # File size limit (in MB) + - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} healthcheck: test: ["CMD", "grpc_health_probe", "-addr=:50051"] interval: 30s diff --git a/docker-compose.yml b/docker-compose.yml index dfc0c84e..e72a2582 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,8 @@ services: container_name: WeKnora-frontend ports: - "${FRONTEND_PORT:-80}:80" + environment: + - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} depends_on: app: condition: service_healthy @@ -96,6 +98,8 @@ services: - INIT_RERANK_MODEL_NAME=${INIT_RERANK_MODEL_NAME:-} - INIT_RERANK_MODEL_BASE_URL=${INIT_RERANK_MODEL_BASE_URL:-} - INIT_RERANK_MODEL_API_KEY=${INIT_RERANK_MODEL_API_KEY:-} + # File size limit (in MB) + - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} depends_on: redis: condition: service_started @@ -137,6 +141,8 @@ services: - MINIO_USE_SSL=${MINIO_USE_SSL:-} - WEB_PROXY=${WEB_PROXY:-} - MINERU_ENDPOINT=${MINERU_ENDPOINT:-} + # File size limit (in MB) + - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} healthcheck: test: ["CMD", "grpc_health_probe", "-addr=:50051"] interval: 30s diff --git a/docreader/client/client.go b/docreader/client/client.go index bba854a9..2a0f9126 100644 --- a/docreader/client/client.go +++ b/docreader/client/client.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "os" + "strconv" "time" "github.com/Tencent/WeKnora/docreader/proto" @@ -12,9 +13,16 @@ import ( "google.golang.org/grpc/resolver" ) -const ( - maxMessageSize = 50 * 1024 * 1024 // 50MB -) +// getMaxMessageSize returns the maximum gRPC message size in bytes. +// Default is 50MB, can be configured via MAX_FILE_SIZE_MB environment variable. +func getMaxMessageSize() int { + if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" { + if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 { + return size * 1024 * 1024 + } + } + return 50 * 1024 * 1024 // default 50MB +} // Logger is the default logger used by the client var Logger = log.New(os.Stdout, "[DocReader] ", log.LstdFlags|log.Lmicroseconds) @@ -40,13 +48,14 @@ type Client struct { func NewClient(addr string) (*Client, error) { Logger.Printf("INFO: Creating new DocReader client connecting to %s", addr) - // 设置消息大小限制 + // 设置消息大小限制 (configurable via GRPC_MAX_MESSAGE_SIZE_MB) + maxMsgSize := getMaxMessageSize() opts := []grpc.DialOption{ grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`), grpc.WithDefaultCallOptions( - grpc.MaxCallRecvMsgSize(maxMessageSize), - grpc.MaxCallSendMsgSize(maxMessageSize), + grpc.MaxCallRecvMsgSize(maxMsgSize), + grpc.MaxCallSendMsgSize(maxMsgSize), ), } resolver.SetDefaultScheme("dns") diff --git a/docreader/main.py b/docreader/main.py index 3cef075a..28c57cc7 100644 --- a/docreader/main.py +++ b/docreader/main.py @@ -57,8 +57,22 @@ logger.info("Initializing server logging") # Initialize request ID logging init_logging_request_id() -# Set max message size to 50MB -MAX_MESSAGE_LENGTH = 50 * 1024 * 1024 + +def get_max_message_length() -> int: + """Get max gRPC message length from environment variable. + Default is 50MB, can be configured via MAX_FILE_SIZE_MB. + """ + try: + size_mb = int(os.environ.get("MAX_FILE_SIZE_MB", "50")) + if size_mb > 0: + return size_mb * 1024 * 1024 + except ValueError: + pass + return 50 * 1024 * 1024 # default 50MB + + +# Set max message size (default 50MB, configurable via MAX_FILE_SIZE_MB) +MAX_MESSAGE_LENGTH = get_max_message_length() parser = Parser() diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 045e8ebe..6ecb5d36 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -27,11 +27,14 @@ FROM nginx:stable-alpine as production-stage # 复制构建产物到nginx服务目录 COPY --from=build-stage /app/dist /usr/share/nginx/html -# 复制nginx配置文件 -COPY nginx.conf /etc/nginx/conf.d/default.conf +# 复制nginx配置模板文件 +COPY nginx.conf /etc/nginx/templates/default.conf.template + +# 设置默认环境变量(MB) +ENV MAX_FILE_SIZE_MB=50 # 暴露端口 EXPOSE 80 -# 启动nginx -CMD ["nginx", "-g", "daemon off;"] \ No newline at end of file +# 启动时将 MAX_FILE_SIZE_MB 转换为带单位的 MAX_FILE_SIZE,然后替换到 nginx 配置 +CMD ["/bin/sh", "-c", "export MAX_FILE_SIZE=${MAX_FILE_SIZE_MB}M && envsubst '${MAX_FILE_SIZE}' < /etc/nginx/templates/default.conf.template > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"] \ No newline at end of file diff --git a/frontend/nginx.conf b/frontend/nginx.conf index e855553e..aa4b0ca1 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -1,7 +1,8 @@ server { listen 80; server_name localhost; - client_max_body_size 50M; + # Default 50M, configured via MAX_FILE_SIZE_MB env var + client_max_body_size ${MAX_FILE_SIZE}; # 安全头配置 add_header X-Frame-Options "SAMEORIGIN" always; diff --git a/internal/handler/initialization.go b/internal/handler/initialization.go index 468c0f58..2bd8a87e 100644 --- a/internal/handler/initialization.go +++ b/internal/handler/initialization.go @@ -1823,10 +1823,11 @@ func (h *InitializationHandler) TestMultimodalFunction(c *gin.Context) { return } - // 验证文件大小 (10MB) - if header.Size > 10*1024*1024 { + // 验证文件大小 (default 50MB, configurable via MAX_FILE_SIZE_MB) + maxSize := utils.GetMaxFileSize() + if header.Size > maxSize { logger.Error(ctx, "File size too large") - c.Error(errors.NewBadRequestError("图片文件大小不能超过10MB")) + c.Error(errors.NewBadRequestError(fmt.Sprintf("图片文件大小不能超过%dMB", utils.GetMaxFileSizeMB()))) return } logger.Infof(ctx, "Processing image: %s", utils.SanitizeForLog(header.Filename)) diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index 9347c25b..2c31080e 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -118,6 +118,14 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) { return } + // Validate file size (configurable via MAX_FILE_SIZE_MB) + maxSize := secutils.GetMaxFileSize() + if file.Size > maxSize { + logger.Error(ctx, "File size too large") + c.Error(errors.NewBadRequestError(fmt.Sprintf("文件大小不能超过%dMB", secutils.GetMaxFileSizeMB()))) + return + } + // Get custom filename if provided (for folder uploads with path) customFileName := c.PostForm("fileName") customFileName = secutils.SanitizeForLog(customFileName) diff --git a/internal/utils/filesize.go b/internal/utils/filesize.go new file mode 100644 index 00000000..c41b0494 --- /dev/null +++ b/internal/utils/filesize.go @@ -0,0 +1,27 @@ +package utils + +import ( + "os" + "strconv" +) + +// GetMaxFileSize returns the maximum file upload size in bytes. +// Default is 50MB, can be configured via MAX_FILE_SIZE_MB environment variable. +func GetMaxFileSize() int64 { + if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" { + if size, err := strconv.ParseInt(sizeStr, 10, 64); err == nil && size > 0 { + return size * 1024 * 1024 + } + } + return 50 * 1024 * 1024 // default 50MB +} + +// GetMaxFileSizeMB returns the maximum file upload size in MB. +func GetMaxFileSizeMB() int64 { + if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" { + if size, err := strconv.ParseInt(sizeStr, 10, 64); err == nil && size > 0 { + return size + } + } + return 50 // default 50MB +}