fix(security): harden web_fetch and datasource connector SSRF paths

Use SSRF-safe HTTP clients in chat pipeline web_fetch and Feishu/Yuque/Notion
connectors, validating tenant-supplied base_url values before outbound requests.
This commit is contained in:
wizardchen
2026-07-02 12:21:39 +08:00
committed by lyingbug
parent 1d322c525a
commit 8325dac738
13 changed files with 179 additions and 66 deletions
+7 -48
View File
@@ -4,10 +4,8 @@ package web_fetch
import (
"context"
"crypto/tls"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
@@ -24,13 +22,12 @@ const (
)
// FetchURLContent fetches a URL and returns its text content (HTML converted to clean text).
// Includes SSRF validation, DNS pinning, browser-like headers, and content size limits.
// Includes SSRF validation, redirect re-validation, dial-time guards, and content size limits.
func FetchURLContent(ctx context.Context, rawURL string) (string, error) {
if rawURL == "" {
return "", fmt.Errorf("url is empty")
}
// SSRF validation
if err := utils.ValidateURLForSSRF(rawURL); err != nil {
return "", fmt.Errorf("URL rejected: %w", err)
}
@@ -40,53 +37,22 @@ func FetchURLContent(ctx context.Context, rawURL string) (string, error) {
return "", fmt.Errorf("invalid URL: %w", err)
}
hostname := u.Hostname()
port := u.Port()
if port == "" {
if u.Scheme == "https" {
port = "443"
} else {
port = "80"
}
}
// DNS pinning: resolve once, use pinned IP
ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip", hostname)
if err != nil || len(ips) == 0 {
return "", fmt.Errorf("DNS lookup failed for %s: %w", hostname, err)
}
var pinnedIP net.IP
for _, ip := range ips {
if utils.IsPublicIP(ip) {
pinnedIP = ip
break
}
}
if pinnedIP == nil {
return "", fmt.Errorf("no public IP for host %s", hostname)
}
// Build request with pinned IP
hostPort := net.JoinHostPort(pinnedIP.String(), port)
fetchURL := *u
fetchURL.Host = hostPort
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fetchURL.String(), nil)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return "", err
}
req.Host = hostname
// Browser-like headers to reduce 403 rejections.
// These match a real Chrome browser fingerprint.
req.Header.Set("User-Agent",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
req.Header.Set("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
req.Header.Set("Accept-Encoding", "identity") // no gzip to simplify reading
req.Header.Set("Accept-Encoding", "identity")
req.Header.Set("Cache-Control", "no-cache")
req.Header.Set("Pragma", "no-cache")
req.Header.Set("Sec-Ch-Ua", `"Chromium";v="131", "Not_A Brand";v="24"`)
@@ -99,16 +65,10 @@ func FetchURLContent(ctx context.Context, rawURL string) (string, error) {
req.Header.Set("Upgrade-Insecure-Requests", "1")
req.Header.Set("Referer", u.Scheme+"://"+hostname+"/")
// Custom transport: TLS ServerName for certificate validation with pinned IP.
client := &http.Client{
Timeout: fetchTimeout,
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
ServerName: hostname,
},
},
// Follow redirects (default behavior), up to 10 hops
}
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
Timeout: fetchTimeout,
MaxRedirects: 10,
})
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("fetch failed: %w", err)
@@ -143,7 +103,6 @@ func htmlToText(html string) string {
})
text := sb.String()
// Normalize whitespace: collapse blank lines
lines := strings.Split(text, "\n")
var cleaned []string
for _, line := range lines {