From 1d322c525a6197d24f07555b4b18ad33313b4328 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 2 Jul 2026 12:11:40 +0800 Subject: [PATCH] fix(security): block SSRF via redirects in knowledge URL import Use the SSRF-safe HTTP client for file_url downloads and add redirect-aware guards to the docreader Playwright path, closing GHSA-44fp, GHSA-rv6v, and GHSA-c7m6. --- docreader/parser/web_parser.py | 22 ++ docreader/tests/test_ssrf.py | 46 ++++ docreader/tests/test_web_parser.py | 25 ++ docreader/utils/ssrf.py | 250 ++++++++++++++++++ .../application/service/knowledge_util.go | 6 +- .../service/knowledge_util_ssrf_test.go | 39 +++ 6 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 docreader/tests/test_ssrf.py create mode 100644 docreader/utils/ssrf.py create mode 100644 internal/application/service/knowledge_util_ssrf_test.go diff --git a/docreader/parser/web_parser.py b/docreader/parser/web_parser.py index ba135dc2..40a25279 100644 --- a/docreader/parser/web_parser.py +++ b/docreader/parser/web_parser.py @@ -14,6 +14,7 @@ from docreader.parser.base_parser import BaseParser from docreader.parser.chain_parser import PipelineParser from docreader.parser.markdown_parser import MarkdownParser from docreader.utils import endecode +from docreader.utils.ssrf import is_ssrf_safe_url logger = logging.getLogger(__name__) @@ -122,6 +123,22 @@ async def read_visible_text(page: Page) -> str: ) +async def install_ssrf_route_guard(page: Page) -> None: + """Block navigation/subresource requests to SSRF-restricted targets (incl. redirects).""" + + async def handle_route(route) -> None: + safe, reason = is_ssrf_safe_url(route.request.url) + if not safe: + logger.warning( + "SSRF guard blocked request to %s: %s", route.request.url, reason + ) + await route.abort("blockedbyclient") + return + await route.continue_() + + await page.route("**/*", handle_route) + + class StdWebParser(BaseParser): """Standard web page parser using Playwright and Trafilatura. @@ -154,6 +171,10 @@ class StdWebParser(BaseParser): """ logger.info(f"Starting web page scraping for URL: {url}") empty = _ScrapeResult(html="", visible_text="", page_title="") + safe, reason = is_ssrf_safe_url(url) + if not safe: + logger.error("URL blocked by SSRF guard before navigation: %s", reason) + return empty try: async with async_playwright() as p: kwargs = {} @@ -163,6 +184,7 @@ class StdWebParser(BaseParser): logger.info("Launching WebKit browser") browser = await p.webkit.launch(**kwargs) page = await browser.new_page() + await install_ssrf_route_guard(page) logger.info(f"Navigating to URL: {url}") try: diff --git a/docreader/tests/test_ssrf.py b/docreader/tests/test_ssrf.py new file mode 100644 index 00000000..d42b6468 --- /dev/null +++ b/docreader/tests/test_ssrf.py @@ -0,0 +1,46 @@ +import os +import unittest +from unittest.mock import patch + +from docreader.utils.ssrf import is_ssrf_safe_url, reset_ssrf_whitelist_cache_for_test + + +class TestSSRFValidation(unittest.TestCase): + def setUp(self) -> None: + self._env_patch = patch.dict( + os.environ, + {"SSRF_WHITELIST": "", "SSRF_WHITELIST_EXTRA": ""}, + clear=False, + ) + self._env_patch.start() + reset_ssrf_whitelist_cache_for_test() + + def tearDown(self) -> None: + self._env_patch.stop() + reset_ssrf_whitelist_cache_for_test() + + def test_blocks_loopback_ip(self): + safe, reason = is_ssrf_safe_url("http://127.0.0.1:8080/page") + self.assertFalse(safe) + self.assertTrue(reason) + + def test_blocks_restricted_hostname(self): + safe, reason = is_ssrf_safe_url("http://host.docker.internal/secret") + self.assertFalse(safe) + self.assertIn("restricted", reason) + + def test_blocks_metadata_host(self): + safe, reason = is_ssrf_safe_url( + "http://169.254.169.254/latest/meta-data/iam/security-credentials/" + ) + self.assertFalse(safe) + self.assertTrue(reason) + + def test_allows_public_https(self): + safe, reason = is_ssrf_safe_url("https://example.com/article") + self.assertTrue(safe, reason) + self.assertEqual(reason, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/docreader/tests/test_web_parser.py b/docreader/tests/test_web_parser.py index 867cc9f4..0d24c87f 100644 --- a/docreader/tests/test_web_parser.py +++ b/docreader/tests/test_web_parser.py @@ -1,12 +1,29 @@ +import os import unittest +from unittest.mock import patch from docreader.parser.web_parser import ( build_visible_text_fallback, extract_markdown_from_html, + install_ssrf_route_guard, ) +from docreader.utils.ssrf import is_ssrf_safe_url, reset_ssrf_whitelist_cache_for_test class TestWebParserHelpers(unittest.TestCase): + def setUp(self) -> None: + self._env_patch = patch.dict( + os.environ, + {"SSRF_WHITELIST": "", "SSRF_WHITELIST_EXTRA": ""}, + clear=False, + ) + self._env_patch.start() + reset_ssrf_whitelist_cache_for_test() + + def tearDown(self) -> None: + self._env_patch.stop() + reset_ssrf_whitelist_cache_for_test() + def test_extract_markdown_empty_html(self): self.assertIsNone(extract_markdown_from_html("")) self.assertIsNone(extract_markdown_from_html(" ")) @@ -37,6 +54,14 @@ class TestWebParserHelpers(unittest.TestCase): md = build_visible_text_fallback(text, page_title="") self.assertEqual(md, text) + def test_install_ssrf_route_guard_is_importable(self): + self.assertTrue(callable(install_ssrf_route_guard)) + + def test_redirect_target_blocked_before_navigation(self): + safe, reason = is_ssrf_safe_url("http://127.0.0.1:39127/audit.txt") + self.assertFalse(safe) + self.assertTrue(reason) + if __name__ == "__main__": unittest.main() diff --git a/docreader/utils/ssrf.py b/docreader/utils/ssrf.py new file mode 100644 index 00000000..c13f6a41 --- /dev/null +++ b/docreader/utils/ssrf.py @@ -0,0 +1,250 @@ +"""SSRF URL validation for docreader outbound HTTP requests. + +Mirrors the core policy in internal/utils/security.go so redirect targets +during Playwright navigation are blocked the same way as Go-side imports. +""" + +from __future__ import annotations + +import ipaddress +import os +import re +import socket +from functools import lru_cache +from typing import FrozenSet, Optional, Tuple, Union +from urllib.parse import urlparse + +RESTRICTED_HOSTNAMES: FrozenSet[str] = frozenset( + { + "localhost", + "127.0.0.1", + "::1", + "0.0.0.0", + "metadata.google.internal", + "metadata.tencentyun.com", + "metadata.aws.internal", + "host.docker.internal", + "gateway.docker.internal", + "kubernetes.docker.internal", + "kubernetes", + "kubernetes.default", + "kubernetes.default.svc", + "kubernetes.default.svc.cluster.local", + } +) + +RESTRICTED_SUFFIXES: Tuple[str, ...] = ( + ".local", + ".localhost", + ".internal", + ".corp", + ".lan", + ".home", + ".localdomain", + ".svc.cluster.local", + ".pod.cluster.local", +) + +EXTRA_RESTRICTED_CIDRS: Tuple[Union[ipaddress.IPv4Network, ipaddress.IPv6Network], ...] = tuple( + ipaddress.ip_network(cidr) + for cidr in ( + "100.64.0.0/10", + "198.18.0.0/15", + "198.51.100.0/24", + "203.0.113.0/24", + "192.0.0.0/24", + "192.0.2.0/24", + "0.0.0.0/8", + "240.0.0.0/4", + "255.255.255.255/32", + "172.17.0.0/16", + "172.18.0.0/16", + "172.19.0.0/16", + "172.20.0.0/16", + ) +) + +BLOCKED_PORTS: FrozenSet[str] = frozenset( + { + "22", + "23", + "25", + "445", + "3389", + "5432", + "3306", + "6379", + "27017", + "9200", + "2379", + "2380", + "8500", + "4001", + } +) + +_IP_LIKE_PATTERNS = ( + re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"), + re.compile(r"^\d{8,10}$"), + re.compile(r"^0[0-7]+\."), + re.compile(r"(?i)^0x[0-9a-f]+\."), + re.compile(r"(?i)^0x[0-9a-f]{6,8}$"), + re.compile(r"(?i)^[0-9a-f:]+::[0-9a-f:]*$"), + re.compile(r"(?i)^[0-9a-f]{1,4}(:[0-9a-f]{1,4}){7}$"), +) + + +def _normalize_url(raw_url: str) -> str: + if "://" not in raw_url: + return f"https://{raw_url}" + return raw_url + + +@lru_cache(maxsize=1) +def _load_whitelist() -> Tuple[FrozenSet[str], Tuple[str, ...], Tuple[Union[ipaddress.IPv4Network, ipaddress.IPv6Network], ...]]: + entries: list[str] = [] + for env_key in ("SSRF_WHITELIST", "SSRF_WHITELIST_EXTRA"): + raw = os.environ.get(env_key, "") + if raw.strip(): + entries.extend(part.strip() for part in raw.split(",") if part.strip()) + + exact_hosts: set[str] = set() + suffix_hosts: list[str] = [] + cidr_nets: list[Union[ipaddress.IPv4Network, ipaddress.IPv6Network]] = [] + for entry in entries: + lowered = entry.lower() + if lowered.startswith("*."): + suffix_hosts.append(lowered[1:]) + continue + if "/" in lowered: + try: + cidr_nets.append(ipaddress.ip_network(lowered, strict=False)) + except ValueError: + continue + continue + exact_hosts.add(lowered) + return frozenset(exact_hosts), tuple(suffix_hosts), tuple(cidr_nets) + + +def _is_whitelisted(hostname: str) -> bool: + lowered = hostname.lower() + exact_hosts, suffix_hosts, cidr_nets = _load_whitelist() + if lowered in exact_hosts: + return True + for suffix in suffix_hosts: + if lowered.endswith(suffix) or lowered == suffix.lstrip("."): + return True + try: + ip = ipaddress.ip_address(lowered) + except ValueError: + return False + return any(ip in net for net in cidr_nets) + + +def _is_ip_like_hostname(hostname: str) -> bool: + return any(pattern.search(hostname) for pattern in _IP_LIKE_PATTERNS) + + +def _is_restricted_ip(ip: Union[ipaddress.IPv4Address, ipaddress.IPv6Address]) -> Optional[str]: + if ip.is_private: + return "private IP address" + if ip.is_loopback: + return "loopback address" + if ip.is_link_local: + return "link-local address" + if ip.is_multicast: + return "multicast address" + if ip.is_unspecified: + return "unspecified address" + if isinstance(ip, ipaddress.IPv4Address): + for net in EXTRA_RESTRICTED_CIDRS: + if isinstance(net, ipaddress.IPv4Network) and ip in net: + return f"restricted range {net}" + if isinstance(ip, ipaddress.IPv6Address): + # Site-local (fec0::/10) + if (ip.packed[0] == 0xFE) and (ip.packed[1] & 0xC0) == 0xC0: + return "site-local IPv6 address" + return None + + +def _resolve_host_ips(hostname: str) -> Tuple[Tuple[Union[ipaddress.IPv4Address, ipaddress.IPv6Address], ...], Optional[str]]: + try: + infos = socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM) + except socket.gaierror as exc: + return (), f"DNS resolution failed for hostname {hostname}: {exc}" + ips: list[Union[ipaddress.IPv4Address, ipaddress.IPv6Address]] = [] + seen: set[str] = set() + for info in infos: + sockaddr = info[4] + if not sockaddr: + continue + ip_str = sockaddr[0] + if ip_str in seen: + continue + seen.add(ip_str) + try: + ips.append(ipaddress.ip_address(ip_str)) + except ValueError: + continue + if not ips: + return (), f"DNS resolution failed for hostname {hostname}: no addresses" + return tuple(ips), None + + +def is_ssrf_safe_url(raw_url: str) -> Tuple[bool, str]: + """Return (safe, reason). reason is empty when safe is True.""" + if not raw_url or not raw_url.strip(): + return False, "URL is empty" + + normalized = _normalize_url(raw_url.strip()) + parsed = urlparse(normalized) + scheme = (parsed.scheme or "").lower() + if scheme not in {"http", "https"}: + return False, f"invalid scheme: {scheme or '(none)'} (only http/https allowed)" + + hostname = (parsed.hostname or "").strip() + if not hostname: + return False, "URL has no hostname" + + hostname_lower = hostname.lower() + if _is_whitelisted(hostname_lower): + return True, "" + + if hostname_lower in RESTRICTED_HOSTNAMES: + return False, f"hostname {hostname_lower} is restricted" + + for suffix in RESTRICTED_SUFFIXES: + if hostname_lower.endswith(suffix): + return False, f"hostname suffix {suffix} is restricted" + + try: + ipaddress.ip_address(hostname_lower) + return False, "direct IP address access is not allowed, use domain name or add to SSRF_WHITELIST" + except ValueError: + pass + + if _is_ip_like_hostname(hostname_lower): + return False, "IP-like hostname format is not allowed" + + resolved_ips, resolve_err = _resolve_host_ips(hostname_lower) + if resolve_err: + return False, resolve_err + + for resolved_ip in resolved_ips: + reason = _is_restricted_ip(resolved_ip) + if reason: + return ( + False, + f"hostname {hostname_lower} resolves to restricted IP {resolved_ip}: {reason}", + ) + + port = parsed.port + if port is not None and str(port) in BLOCKED_PORTS: + return False, f"port {port} is blocked for security reasons" + + return True, "" + + +def reset_ssrf_whitelist_cache_for_test() -> None: + """Clear cached whitelist entries (for unit tests only).""" + _load_whitelist.cache_clear() diff --git a/internal/application/service/knowledge_util.go b/internal/application/service/knowledge_util.go index 7add9178..ef11e78b 100644 --- a/internal/application/service/knowledge_util.go +++ b/internal/application/service/knowledge_util.go @@ -16,6 +16,7 @@ import ( "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" + secutils "github.com/Tencent/WeKnora/internal/utils" ) // isValidFileType checks if a file type is supported @@ -336,7 +337,10 @@ func IsVideoType(fileType string) bool { // the function resolves the value from Content-Disposition / URL path and writes it back. // It does NOT perform SSRF validation — callers are responsible for that. func downloadFileFromURL(ctx context.Context, fileURL string, payloadFileName, payloadFileType *string) ([]byte, error) { - httpClient := &http.Client{Timeout: 60 * time.Second} + httpClient := secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{ + Timeout: 60 * time.Second, + MaxRedirects: 10, + }) req, err := http.NewRequestWithContext(ctx, http.MethodGet, fileURL, nil) if err != nil { return nil, fmt.Errorf("failed to create request for file URL: %w", err) diff --git a/internal/application/service/knowledge_util_ssrf_test.go b/internal/application/service/knowledge_util_ssrf_test.go new file mode 100644 index 00000000..c3e8d0a9 --- /dev/null +++ b/internal/application/service/knowledge_util_ssrf_test.go @@ -0,0 +1,39 @@ +package service + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +func TestDownloadFileFromURLBlocksRedirectToLoopback(t *testing.T) { + internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("INTERNAL_SECRET=metadata-token-AKIAEXAMPLE")) + })) + defer internal.Close() + + if err := secutils.ValidateURLForSSRF(internal.URL); err == nil { + t.Fatalf("precondition: direct internal URL should be blocked") + } + + attacker := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, internal.URL, http.StatusFound) + })) + defer attacker.Close() + + attackerURL := attacker.URL + "/malicious.pdf" + fileName := "" + fileType := "pdf" + _, err := downloadFileFromURL(context.Background(), attackerURL, &fileName, &fileType) + if err == nil { + t.Fatal("expected redirect to loopback to be blocked") + } + if !strings.Contains(err.Error(), "redirect blocked") && !strings.Contains(err.Error(), "connection blocked") { + t.Fatalf("unexpected error: %v", err) + } +}