diff --git a/docreader/parser/base_parser.py b/docreader/parser/base_parser.py index e3a1b0c0..4a7fb19a 100644 --- a/docreader/parser/base_parser.py +++ b/docreader/parser/base_parser.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- import asyncio import io +import ipaddress import logging import os import re import time from abc import ABC, abstractmethod from typing import Dict, List, Optional, Tuple +from urllib.parse import urlparse import requests from PIL import Image @@ -30,6 +32,60 @@ class BaseParser(ABC): _ocr_engine = None _ocr_engine_failed = False + @staticmethod + def _is_safe_url(url: str) -> bool: + """Validate URL to prevent SSRF attacks + + Args: + url: URL to validate + + Returns: + True if URL is safe to fetch, False otherwise + """ + try: + parsed_url = urlparse(url) + + # Only allow http and https schemes + if parsed_url.scheme not in ['http', 'https']: + logger.warning(f"Rejected URL with invalid scheme: {parsed_url.scheme}") + return False + + # Extract hostname + hostname = parsed_url.hostname + if not hostname: + logger.warning("No hostname found in URL") + return False + + # Try to parse as IP address + try: + ip = ipaddress.ip_address(hostname) + # Reject private, loopback, link-local, multicast addresses + if (ip.is_private or ip.is_loopback or + ip.is_link_local or ip.is_multicast or + ip.is_reserved): + logger.warning(f"Rejected URL with restricted IP: {ip}") + return False + except ValueError: + # Not an IP address, validate hostname + # Reject localhost and common internal hostnames + hostname_lower = hostname.lower() + restricted_hostnames = [ + 'localhost', + '127.0.0.1', + '::1', + 'metadata.google.internal', + 'metadata.tencentyun.com', + '169.254.169.254', # AWS metadata endpoint + ] + if hostname_lower in restricted_hostnames or hostname_lower.endswith('.local'): + logger.warning(f"Rejected URL with restricted hostname: {hostname}") + return False + + return True + except Exception as e: + logger.warning(f"Error validating URL: {str(e)}") + return False + @classmethod def get_ocr_engine(cls, backend_type="paddle", **kwargs): """Get OCR engine instance @@ -681,6 +737,11 @@ class BaseParser(ABC): if is_storage_url: logger.info(f"Image already on COS: {img_url}, no need to re-upload") try: + # Validate URL to prevent SSRF attacks + if not self._is_safe_url(img_url): + logger.error(f"URL failed validation check: {img_url}") + return img_url, img_url, None + # Still need to get image object for OCR processing # Get proxy settings from environment variables http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") @@ -727,6 +788,11 @@ class BaseParser(ABC): # Normal remote URL download handling else: + # Validate URL to prevent SSRF attacks + if not self._is_safe_url(img_url): + logger.error(f"URL failed validation check: {img_url}") + return img_url, img_url, None + # Get proxy settings from environment variables http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY") diff --git a/internal/mcp/client.go b/internal/mcp/client.go index 70315c7f..f971412c 100644 --- a/internal/mcp/client.go +++ b/internal/mcp/client.go @@ -4,7 +4,10 @@ import ( "context" "encoding/json" "fmt" + "net" "net/http" + "net/url" + "strings" "time" "github.com/Tencent/WeKnora/internal/logger" @@ -58,6 +61,31 @@ type mcpGoClient struct { initialized bool } +// validateURLScheme validates that authentication endpoints use HTTPS +func validateURLScheme(serviceURL string, hasBasicAuth bool, hasTokenAuth bool) error { + if !hasBasicAuth && !hasTokenAuth { + // No authentication, no HTTPS requirement + return nil + } + + if serviceURL == "" { + return fmt.Errorf("URL is required") + } + + // Parse the URL + parsedURL, err := url.Parse(serviceURL) + if err != nil { + return fmt.Errorf("invalid URL: %w", err) + } + + // Enforce HTTPS for authenticated endpoints + if parsedURL.Scheme != "https" { + return fmt.Errorf("HTTPS is required for authenticated connections, got %s scheme", parsedURL.Scheme) + } + + return nil +} + // NewMCPClient creates a new MCP client based on the transport type func NewMCPClient(config *ClientConfig) (MCPClient, error) { // Create HTTP client with timeout