fix(security): support IPv6 in SSRF validation via whitelist mechanism

- Keep strict mode blocking all direct IPs (IPv4 and IPv6 uniformly)
- Unify all SSRF call sites to use ValidateURLForSSRF (whitelist-aware)
- Add Teredo (2001:0000::/32) and 6to4 (2002::/16) tunnel detection
- Make redirect handler and DNS pinning respect SSRF_WHITELIST
- Unexport isSSRFSafeURL to prevent future callers bypassing whitelist
- Add scheme validation for whitelisted redirect targets
- Document IPv6 whitelist syntax in .env.example
- Add comprehensive IPv6 test coverage
This commit is contained in:
nullkey
2026-03-31 18:52:25 +08:00
committed by lyingbug
parent a5ca9a2784
commit 0b64ef9bd1
9 changed files with 231 additions and 52 deletions
+3 -2
View File
@@ -105,8 +105,9 @@ TENANT_AES_KEY=weknorarag-api-key-secret-secret
SYSTEM_AES_KEY=weknora-system-aes-key-32bytes!!
# SSRF 校验白名单(可选)。逗号分隔;每条可为:精确域名(api.internal)、通配域名(*.example.com)、
# IP(203.0.113.5)或 CIDR(10.0.0.0/8)。列入者会在 URL 校验等地方绕过常规 SSRF 规则,生产环境请谨慎配置。
# SSRF_WHITELIST=internal.service,*.corp.example,172.16.0.0/12
# IPv4(203.0.113.5)、IPv6(2001:db8::1,不带方括号)或 CIDR(10.0.0.0/8, 2001:db8::/32)。
# 列入者会在 URL 校验等地方绕过常规 SSRF 规则,生产环境请谨慎配置。
# SSRF_WHITELIST=internal.service,*.corp.example,172.16.0.0/12,2001:db8::1,fd00::/8
# 是否开启知识图谱构建和检索(构建阶段需调用大模型,耗时较长)
ENABLE_GRAPH_RAG=false
+7 -5
View File
@@ -262,9 +262,9 @@ func (t *WebFetchTool) validateAndResolve(p webFetchParams) (*validatedParams, e
return nil, fmt.Errorf("invalid URL format")
}
// SSRF protection: validate URL is safe (scheme, hostname, and that resolved IPs are not restricted)
if safe, reason := utils.IsSSRFSafeURL(p.URL); !safe {
return nil, fmt.Errorf("URL rejected for security reasons: %s", reason)
// SSRF protection: validate URL is safe (uses centralised entry-point with whitelist support)
if err := utils.ValidateURLForSSRF(p.URL); err != nil {
return nil, fmt.Errorf("URL rejected for security reasons: %v", err)
}
u, err := url.Parse(p.URL)
@@ -281,14 +281,16 @@ func (t *WebFetchTool) validateAndResolve(p webFetchParams) (*validatedParams, e
}
}
// Resolve and pin to the first public IP (same resolver as IsSSRFSafeURL; we pin so chromedp cannot re-resolve)
// Resolve and pin to the first safe IP (same resolver as isSSRFSafeURL; we pin so chromedp cannot re-resolve).
// Whitelisted hosts may resolve to private/restricted IPs, so we allow any IP for them.
ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip", hostname)
if err != nil || len(ips) == 0 {
return nil, fmt.Errorf("DNS lookup failed for %s: %w", hostname, err)
}
isWhitelisted := utils.IsSSRFWhitelisted(hostname)
var pinnedIP net.IP
for _, ip := range ips {
if utils.IsPublicIP(ip) {
if isWhitelisted || utils.IsPublicIP(ip) {
pinnedIP = ip
break
}
+9 -9
View File
@@ -473,9 +473,9 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
return nil, ErrInvalidURL
}
// SSRF protection: validate URL is safe to fetch
if safe, reason := secutils.IsSSRFSafeURL(url); !safe {
logger.Errorf(ctx, "URL rejected for SSRF protection: %s, reason: %s", url, reason)
// SSRF protection: validate URL is safe to fetch (uses centralised entry-point with whitelist support)
if err := secutils.ValidateURLForSSRF(url); err != nil {
logger.Errorf(ctx, "URL rejected for SSRF protection: %s, err: %v", url, err)
return nil, ErrInvalidURL
}
@@ -659,8 +659,8 @@ func (s *knowledgeService) createKnowledgeFromFileURL(
logger.Error(ctx, "Invalid or unsafe file URL format")
return nil, ErrInvalidURL
}
if safe, reason := secutils.IsSSRFSafeURL(fileURL); !safe {
logger.Errorf(ctx, "File URL rejected for SSRF protection: %s, reason: %s", fileURL, reason)
if err := secutils.ValidateURLForSSRF(fileURL); err != nil {
logger.Errorf(ctx, "File URL rejected for SSRF protection: %s, err: %v", fileURL, err)
return nil, ErrInvalidURL
}
@@ -7458,8 +7458,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
if payload.FileURL != "" {
// file_url import: SSRF re-check (防 DNS 重绑定), download, persist, then delegate to convert()
if safe, reason := secutils.IsSSRFSafeURL(payload.FileURL); !safe {
logger.Errorf(ctx, "File URL rejected for SSRF protection in ProcessDocument: %s, reason: %s", payload.FileURL, reason)
if err := secutils.ValidateURLForSSRF(payload.FileURL); err != nil {
logger.Errorf(ctx, "File URL rejected for SSRF protection in ProcessDocument: %s, err: %v", payload.FileURL, err)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "File URL is not allowed for security reasons"
knowledge.UpdatedAt = time.Now()
@@ -7668,8 +7668,8 @@ func (s *knowledgeService) convert(
overrides := s.getParserEngineOverridesFromContext(ctx)
if isURL {
if safe, reason := secutils.IsSSRFSafeURL(payload.URL); !safe {
logger.Errorf(ctx, "URL rejected for SSRF protection: %s, reason: %s", payload.URL, reason)
if err := secutils.ValidateURLForSSRF(payload.URL); err != nil {
logger.Errorf(ctx, "URL rejected for SSRF protection: %s, err: %v", payload.URL, err)
knowledge.ParseStatus = "failed"
knowledge.ErrorMessage = "URL is not allowed for security reasons"
knowledge.UpdatedAt = time.Now()
+1 -1
View File
@@ -643,7 +643,7 @@ func sanitizeStorageCheckError(err error) string {
}
// isBlockedStorageEndpoint checks whether a storage endpoint resolves to a dangerous
// address (cloud metadata, loopback, link-local). Unlike the stricter IsSSRFSafeURL,
// address (cloud metadata, loopback, link-local). Unlike the stricter isSSRFSafeURL,
// this allows private IPs since MinIO is commonly deployed on internal networks.
// It also respects the SSRF_WHITELIST environment variable for whitelisted hosts.
func isBlockedStorageEndpoint(endpoint string) (bool, string) {
+3 -3
View File
@@ -503,8 +503,8 @@ func (a *WebhookAdapter) DownloadFile(ctx context.Context, msg *im.IncomingMessa
func downloadFromURL(ctx context.Context, rawURL, fileName string) (io.ReadCloser, string, error) {
// SSRF protection: reject internal/private URLs unless on the WeCom API allowlist.
if !isAllowedIMAPIHost(rawURL) {
if safe, reason := secutils.IsSSRFSafeURL(rawURL); !safe {
return nil, "", fmt.Errorf("URL rejected for security reasons: %s", reason)
if err := secutils.ValidateURLForSSRF(rawURL); err != nil {
return nil, "", fmt.Errorf("URL rejected for security reasons: %v", err)
}
}
@@ -582,7 +582,7 @@ func downloadFromURL(ctx context.Context, rawURL, fileName string) (io.ReadClose
}
// allowedIMAPIHosts lists IM platform API hosts that are trusted for file downloads.
// URLs pointing to these hosts bypass IsSSRFSafeURL checks because the WeCom API
// URLs pointing to these hosts bypass isSSRFSafeURL checks because the WeCom API
// itself returns these URLs in callback payloads (e.g. temporary media links).
var allowedIMAPIHosts = []string{
"qyapi.weixin.qq.com",
@@ -667,9 +667,9 @@ func (r *ImageResolver) ResolveRemoteImages(
continue
}
// --- SSRF check ---
if safe, reason := secutils.IsSSRFSafeURL(imgURL); !safe {
log.Printf("WARN: remote image blocked by SSRF check (%s): %s", reason, imgURL)
// --- SSRF check (centralised entry-point with whitelist support) ---
if err := secutils.ValidateURLForSSRF(imgURL); err != nil {
log.Printf("WARN: remote image blocked by SSRF check (%v): %s", err, imgURL)
continue
}
@@ -345,8 +345,8 @@ func (c *MinerUCloudReader) extractDoneResult(_ context.Context, item *extractRe
var imgRefPattern = regexp.MustCompile(`!\[[^\]]*\]\(([^)]+)\)`)
func downloadAndExtractZip(zipURL string) (string, []types.ImageRef, error) {
if safe, reason := utils.IsSSRFSafeURL(zipURL); !safe {
return "", nil, fmt.Errorf("zip URL blocked by SSRF check: %s", reason)
if err := utils.ValidateURLForSSRF(zipURL); err != nil {
return "", nil, fmt.Errorf("zip URL blocked by SSRF check: %v", err)
}
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{Timeout: 120 * time.Second, MaxRedirects: 5})
resp, err := client.Get(zipURL)
+43 -20
View File
@@ -302,6 +302,19 @@ func isRestrictedIP(ip net.IP) (bool, string) {
return true, fmt.Sprintf("IPv4-mapped %s", reason)
}
}
// Teredo tunneling addresses: 2001:0000::/32
// Embed arbitrary IPv4 in the payload; can reach internal hosts via relay.
if ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x00 && ip[3] == 0x00 {
return true, "Teredo tunneling address"
}
// 6to4 addresses: 2002::/16
// Bits 16-47 carry an IPv4 address; block when embedded IPv4 is restricted.
if ip[0] == 0x20 && ip[1] == 0x02 {
embeddedIP := net.IP(ip[2:6])
if restricted, reason := isRestrictedIP(embeddedIP); restricted {
return true, fmt.Sprintf("6to4 embedded %s", reason)
}
}
}
return false, ""
@@ -357,7 +370,7 @@ func isIPLikeHostname(hostname string) bool {
return false
}
// IsSSRFSafeURL validates a URL to prevent SSRF attacks
// isSSRFSafeURL validates a URL to prevent SSRF attacks
// It checks for:
// - Valid http/https protocol
// - Private IP addresses (10.x.x.x, 172.16-31.x.x, 192.168.x.x)
@@ -365,7 +378,7 @@ func isIPLikeHostname(hostname string) bool {
// - Link-local addresses (169.254.x.x, fe80::)
// - Cloud metadata endpoints
// - Reserved hostnames (localhost, *.local, etc.)
func IsSSRFSafeURL(rawURL string) (bool, string) {
func isSSRFSafeURL(rawURL string) (bool, string) {
if rawURL == "" {
return false, "URL is empty"
}
@@ -408,11 +421,14 @@ func IsSSRFSafeURL(rawURL string) (bool, string) {
}
}
// STRICT MODE: Completely block IP addresses in URLs
// This prevents all IP-based SSRF attacks including edge cases and bypasses
// STRICT MODE: Block all direct IP addresses in URLs (both IPv4 and IPv6).
// This prevents IP-based SSRF attacks including obfuscation, tunneling, and
// transition mechanism bypasses. Legitimate IPs should be whitelisted via
// SSRF_WHITELIST env var; the whitelist is checked by ValidateURLForSSRF
// before this function is called.
ip := net.ParseIP(hostname)
if ip != nil {
return false, "direct IP address access is not allowed, use domain name instead"
return false, "direct IP address access is not allowed, use domain name or add to SSRF_WHITELIST"
}
// Also check for IP addresses in various formats that ParseIP might not catch
@@ -425,11 +441,6 @@ func IsSSRFSafeURL(rawURL string) (bool, string) {
// This prevents DNS rebinding attacks where a domain resolves to internal IPs
ips, err := net.LookupIP(hostname)
if err != nil {
// DNS resolution failed - reject the URL for security
// This prevents attacks where:
// 1. The domain is only resolvable within internal network (intranet domains)
// 2. Different DNS servers between validation and actual request
// 3. Attacker-controlled DNS that selectively responds
return false, fmt.Sprintf("DNS resolution failed for hostname %s: cannot verify if it resolves to safe IP", hostname)
}
@@ -760,9 +771,18 @@ func NewSSRFSafeHTTPClient(config SSRFSafeHTTPClientConfig) *http.Client {
return fmt.Errorf("stopped after %d redirects", config.MaxRedirects)
}
// Validate the redirect target URL for SSRF
// Validate the redirect target URL for SSRF (whitelist-aware).
// Even whitelisted hosts must use http/https to prevent scheme-based attacks.
redirectScheme := strings.ToLower(req.URL.Scheme)
if redirectScheme != "http" && redirectScheme != "https" {
return fmt.Errorf("%w: invalid scheme %s", ErrSSRFRedirectBlocked, redirectScheme)
}
redirectHost := req.URL.Hostname()
if redirectHost != "" && IsSSRFWhitelisted(redirectHost) {
return nil
}
redirectURL := req.URL.String()
if safe, reason := IsSSRFSafeURL(redirectURL); !safe {
if safe, reason := isSSRFSafeURL(redirectURL); !safe {
return fmt.Errorf("%w: %s", ErrSSRFRedirectBlocked, reason)
}
@@ -782,7 +802,9 @@ func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, e
}
// Whitelisted hosts bypass all dial-time SSRF checks, consistent with
// ValidateURLForSSRF which skips IsSSRFSafeURL for whitelisted hosts.
// ValidateURLForSSRF which skips isSSRFSafeURL for whitelisted hosts.
// NOTE: This intentionally relaxes DNS-rebinding protection for whitelisted
// hosts. Admins must ensure whitelisted domains are under their control.
if IsSystemProxy(addr) || IsSSRFWhitelisted(host) {
dialer := &net.Dialer{
Timeout: 30 * time.Second,
@@ -834,10 +856,11 @@ func SSRFSafeDialContext(ctx context.Context, network, addr string) (net.Conn, e
// allowed host patterns. Each entry can be:
// - An exact domain: "example.com"
// - A wildcard domain: "*.example.com" (matches all subdomains)
// - An IP address: "203.0.113.5"
// - A CIDR range: "10.0.0.0/8"
// - An IPv4 address: "203.0.113.5"
// - An IPv6 address: "2001:db8::1"
// - A CIDR range (v4 or v6): "10.0.0.0/8", "2001:db8::/32"
//
// Whitelisted entries bypass the normal SSRF checks performed by IsSSRFSafeURL.
// Whitelisted entries bypass the normal SSRF checks performed by isSSRFSafeURL.
var (
ssrfWhitelistOnce sync.Once
@@ -932,16 +955,16 @@ func IsSSRFWhitelisted(hostname string) bool {
return false
}
// ResetSSRFWhitelistForTest resets the whitelist singleton so tests can
// resetSSRFWhitelistForTest resets the whitelist singleton so tests can
// re-read the environment variable. NOT for production use.
func ResetSSRFWhitelistForTest() {
func resetSSRFWhitelistForTest() {
ssrfWhitelistOnce = sync.Once{}
ssrfWhitelist = nil
}
// ValidateURLForSSRF is the centralised entry-point that all handlers should
// call to validate a user-supplied URL. It first checks the SSRF_WHITELIST;
// whitelisted hosts skip the full IsSSRFSafeURL check.
// whitelisted hosts skip the full isSSRFSafeURL check.
//
// rawURL may be a full URL ("https://example.com/v1") or a bare host/host:port
// (for cases like ReconnectDocReader). If a scheme is missing the function
@@ -975,7 +998,7 @@ func ValidateURLForSSRF(rawURL string) error {
}
// Delegate to the full SSRF validation (uses the normalised URL).
if safe, reason := IsSSRFSafeURL(normalized); !safe {
if safe, reason := isSSRFSafeURL(normalized); !safe {
return fmt.Errorf("SSRF validation failed: %s", reason)
}
return nil
+160 -7
View File
@@ -1,11 +1,13 @@
package utils
import (
"net"
"os"
"strings"
"testing"
)
func TestIsSSRFSafeURL(t *testing.T) {
func TestSSRFSafeURL(t *testing.T) {
t.Parallel()
tests := []struct {
@@ -44,6 +46,7 @@ func TestIsSSRFSafeURL(t *testing.T) {
wantOK: false,
wantReasonSub: "hostname suffix .internal is restricted",
},
// --- All direct IPs are blocked by isSSRFSafeURL (strict mode) ---
{
name: "direct IPv4 blocked",
rawURL: "https://8.8.8.8/dns-query",
@@ -51,11 +54,36 @@ func TestIsSSRFSafeURL(t *testing.T) {
wantReasonSub: "direct IP address access is not allowed",
},
{
name: "direct IPv6 blocked",
name: "direct public IPv6 blocked",
rawURL: "https://[2001:4860:4860::8888]/dns-query",
wantOK: false,
wantReasonSub: "direct IP address access is not allowed",
},
{
name: "loopback IPv6 blocked",
rawURL: "https://[::1]/admin",
wantOK: false,
wantReasonSub: "is restricted",
},
{
name: "link-local IPv6 blocked",
rawURL: "https://[fe80::1]/admin",
wantOK: false,
wantReasonSub: "direct IP address access is not allowed",
},
{
name: "ULA IPv6 blocked",
rawURL: "https://[fd12:3456:789a::1]/admin",
wantOK: false,
wantReasonSub: "direct IP address access is not allowed",
},
{
name: "IPv4-mapped IPv6 blocked",
rawURL: "https://[::ffff:127.0.0.1]/admin",
wantOK: false,
wantReasonSub: "direct IP address access is not allowed",
},
// --- IP obfuscation ---
{
name: "IP-like decimal hostname blocked",
rawURL: "https://2130706433/",
@@ -68,6 +96,7 @@ func TestIsSSRFSafeURL(t *testing.T) {
wantOK: false,
wantReasonSub: "IP-like hostname format is not allowed",
},
// --- Port blocking ---
{
name: "blocked internal service port",
rawURL: "https://example.com:3306/db",
@@ -81,21 +110,21 @@ func TestIsSSRFSafeURL(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
ok, reason := IsSSRFSafeURL(tt.rawURL)
ok, reason := isSSRFSafeURL(tt.rawURL)
if ok != tt.wantOK {
t.Fatalf("IsSSRFSafeURL(%q) ok = %v, want %v, reason = %q", tt.rawURL, ok, tt.wantOK, reason)
t.Fatalf("isSSRFSafeURL(%q) ok = %v, want %v, reason = %q", tt.rawURL, ok, tt.wantOK, reason)
}
if tt.wantReasonSub != "" && !strings.Contains(reason, tt.wantReasonSub) {
t.Fatalf("IsSSRFSafeURL(%q) reason = %q, want contains %q", tt.rawURL, reason, tt.wantReasonSub)
t.Fatalf("isSSRFSafeURL(%q) reason = %q, want contains %q", tt.rawURL, reason, tt.wantReasonSub)
}
})
}
}
func TestIsSSRFSafeURL_AllowPublicDomain(t *testing.T) {
func TestSSRFSafeURL_AllowPublicDomain(t *testing.T) {
t.Parallel()
ok, reason := IsSSRFSafeURL("https://example.com/path")
ok, reason := isSSRFSafeURL("https://example.com/path")
if !ok {
// This path depends on runtime DNS/network. If DNS is unavailable, skip to keep CI stable.
if strings.Contains(reason, "DNS resolution failed") {
@@ -104,3 +133,127 @@ func TestIsSSRFSafeURL_AllowPublicDomain(t *testing.T) {
t.Fatalf("expected public domain to be allowed, got ok=%v reason=%q", ok, reason)
}
}
// TestValidateURLForSSRF_IPv6Whitelist verifies that whitelisted IPv6 addresses
// bypass the strict IP block in isSSRFSafeURL.
func TestValidateURLForSSRF_IPv6Whitelist(t *testing.T) {
tests := []struct {
name string
whitelist string
rawURL string
wantErr bool
}{
{
name: "exact IPv6 whitelisted",
whitelist: "2001:4860:4860::8888",
rawURL: "https://[2001:4860:4860::8888]/dns-query",
wantErr: false,
},
{
name: "IPv6 CIDR whitelisted",
whitelist: "2001:db8::/32",
rawURL: "https://[2001:db8::1]/page",
wantErr: false,
},
{
name: "IPv6 not in whitelist still blocked",
whitelist: "2001:db8::/32",
rawURL: "https://[2001:4860:4860::8888]/dns-query",
wantErr: true,
},
{
name: "IPv4 whitelisted",
whitelist: "8.8.8.8",
rawURL: "https://8.8.8.8/dns-query",
wantErr: false,
},
{
name: "wildcard domain whitelisted",
whitelist: "*.example.com",
rawURL: "https://api.example.com/v1",
wantErr: false,
},
{
name: "wildcard domain root whitelisted",
whitelist: "*.example.com",
rawURL: "https://example.com/v1",
wantErr: false,
},
{
name: "bare host without scheme normalised",
whitelist: "internal.service",
rawURL: "internal.service:8080/path",
wantErr: false,
},
{
name: "empty whitelist blocks direct IP",
whitelist: "",
rawURL: "https://8.8.8.8/dns-query",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Reset whitelist singleton so we can set a new value
resetSSRFWhitelistForTest()
os.Setenv("SSRF_WHITELIST", tt.whitelist)
defer func() {
os.Unsetenv("SSRF_WHITELIST")
resetSSRFWhitelistForTest()
}()
err := ValidateURLForSSRF(tt.rawURL)
if (err != nil) != tt.wantErr {
t.Fatalf("ValidateURLForSSRF(%q) with whitelist=%q: err = %v, wantErr = %v",
tt.rawURL, tt.whitelist, err, tt.wantErr)
}
})
}
}
// TestIsRestrictedIP_IPv6 tests IPv6-specific restricted range detection.
func TestIsRestrictedIP_IPv6(t *testing.T) {
t.Parallel()
tests := []struct {
name string
ip string
wantBlock bool
wantReason string
}{
{"loopback", "::1", true, "loopback"},
{"unspecified", "::", true, "unspecified"},
{"link-local", "fe80::1", true, "link-local"},
{"ULA", "fd12:3456:789a::1", true, ""},
{"site-local", "fec0::1", true, "site-local"},
{"Teredo", "2001:0000:4136:e378:8000:63bf:3fff:fdd2", true, "Teredo"},
{"6to4 private", "2002:c0a8:0101::1", true, "6to4"},
{"6to4 public", "2002:0808:0808::1", false, ""},
{"public IPv6", "2001:4860:4860::8888", false, ""},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
ip := parseIPForTest(t, tt.ip)
blocked, reason := isRestrictedIP(ip)
if blocked != tt.wantBlock {
t.Fatalf("isRestrictedIP(%s) = %v, want %v (reason: %s)", tt.ip, blocked, tt.wantBlock, reason)
}
if tt.wantReason != "" && !strings.Contains(strings.ToLower(reason), strings.ToLower(tt.wantReason)) {
t.Fatalf("isRestrictedIP(%s) reason = %q, want contains %q", tt.ip, reason, tt.wantReason)
}
})
}
}
func parseIPForTest(t *testing.T, s string) net.IP {
t.Helper()
ip := net.ParseIP(s)
if ip == nil {
t.Fatalf("invalid test IP: %s", s)
}
return ip
}