From 2b3f76e418e428ac9cab6d63451d25f3052c156d Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 2 Feb 2026 11:03:30 +0800 Subject: [PATCH] feat(web_fetch): enhance web fetch tool with DNS pinning and validation improvements - Introduced a new validatedParams struct to hold validated input along with DNS-pinned host/IP for SSRF protection. - Updated the validateParams function to validate and resolve the host to a single public IP, ensuring safe outbound fetches. - Modified executeFetch and fetchHTMLContent methods to utilize the new validatedParams for improved security and clarity. - Enhanced logging to reflect the display URL and provide better context during fetch operations. This update strengthens the web fetch tool's security against SSRF attacks and improves the overall robustness of URL handling. --- internal/agent/tools/web_fetch.go | 140 +++++++++++++++++++++--------- internal/utils/security.go | 15 +++- 2 files changed, 112 insertions(+), 43 deletions(-) diff --git a/internal/agent/tools/web_fetch.go b/internal/agent/tools/web_fetch.go index 8b747a70..f001782c 100644 --- a/internal/agent/tools/web_fetch.go +++ b/internal/agent/tools/web_fetch.go @@ -5,7 +5,9 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" + "net/url" "regexp" "strings" "sync" @@ -58,6 +60,16 @@ type webFetchParams struct { Prompt string } +// validatedParams holds validated input plus DNS-pinned host/IP for SSRF protection. +// PinnedIP is the single IP we resolved at validation time; chromedp and HTTP both use it. +type validatedParams struct { + URL string + Prompt string + Host string + Port string + PinnedIP net.IP +} + // webFetchItemResult is the result for a web fetch item type webFetchItemResult struct { output string @@ -124,7 +136,10 @@ func (t *WebFetchTool) Execute(ctx context.Context, args json.RawMessage) (*type go func(index int, p webFetchParams) { defer wg.Done() - if err := t.validateParams(p); err != nil { + // Normalize URL before validation so we pin the host we actually fetch (e.g. raw.githubusercontent.com) + finalURL := t.normalizeGitHubURL(p.URL) + vp, err := t.validateAndResolve(webFetchParams{URL: finalURL, Prompt: p.Prompt}) + if err != nil { results[index] = &webFetchItemResult{ err: err, data: map[string]interface{}{ @@ -137,7 +152,7 @@ func (t *WebFetchTool) Execute(ctx context.Context, args json.RawMessage) (*type return } - output, data, err := t.executeFetch(ctx, p) + output, data, err := t.executeFetch(ctx, vp, p.URL) results[index] = &webFetchItemResult{ output: output, data: data, @@ -234,42 +249,78 @@ func (t *WebFetchTool) parseParams(item interface{}) webFetchParams { return params } -// validateParams validates the parameters for a web fetch item -func (t *WebFetchTool) validateParams(p webFetchParams) error { +// validateAndResolve validates parameters and resolves the host to a single public IP (DNS pinning). +// The returned PinnedIP is used for both chromedp (host-resolver-rules) and HTTP to prevent DNS rebinding. +func (t *WebFetchTool) validateAndResolve(p webFetchParams) (*validatedParams, error) { if p.URL == "" { - return fmt.Errorf("url is required") + return nil, fmt.Errorf("url is required") } if p.Prompt == "" { - return fmt.Errorf("prompt is required") + return nil, fmt.Errorf("prompt is required") } if !strings.HasPrefix(p.URL, "http://") && !strings.HasPrefix(p.URL, "https://") { - return fmt.Errorf("invalid URL format") + return nil, fmt.Errorf("invalid URL format") } - // SSRF protection: validate URL is safe to fetch + // SSRF protection: validate URL is safe (scheme, hostname, and that resolved IPs are not restricted) if safe, reason := utils.IsSSRFSafeURL(p.URL); !safe { - return fmt.Errorf("URL rejected for security reasons: %s", reason) + return nil, fmt.Errorf("URL rejected for security reasons: %s", reason) } - return nil + u, err := url.Parse(p.URL) + if err != nil { + return nil, fmt.Errorf("invalid URL: %w", err) + } + hostname := u.Hostname() + port := u.Port() + if port == "" { + if u.Scheme == "https" { + port = "443" + } else { + port = "80" + } + } + + // Resolve and pin to the first public IP (same resolver as IsSSRFSafeURL; we pin so chromedp cannot re-resolve) + ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip", hostname) + if err != nil || len(ips) == 0 { + return nil, fmt.Errorf("DNS lookup failed for %s: %w", hostname, err) + } + var pinnedIP net.IP + for _, ip := range ips { + if utils.IsPublicIP(ip) { + pinnedIP = ip + break + } + } + if pinnedIP == nil { + return nil, fmt.Errorf("no public IP available for host %s", hostname) + } + + return &validatedParams{ + URL: p.URL, + Prompt: p.Prompt, + Host: hostname, + Port: port, + PinnedIP: pinnedIP, + }, nil } -// executeFetch executes a web fetch item +// executeFetch executes a web fetch item. displayURL is the URL shown to the user (e.g. original); vp.URL is the normalized URL we fetch. func (t *WebFetchTool) executeFetch( ctx context.Context, - params webFetchParams, + vp *validatedParams, + displayURL string, ) (string, map[string]interface{}, error) { - logger.Infof(ctx, "[Tool][WebFetch] Fetching URL: %s", params.URL) + logger.Infof(ctx, "[Tool][WebFetch] Fetching URL: %s", displayURL) - finalURL := t.normalizeGitHubURL(params.URL) - - htmlContent, method, err := t.fetchHTMLContent(ctx, finalURL) + htmlContent, method, err := t.fetchHTMLContent(ctx, vp) if err != nil { - logger.Errorf(ctx, "[Tool][WebFetch] 获取页面失败 url=%s err=%v", finalURL, err) - return fmt.Sprintf("URL: %s\n错误: %v\n", params.URL, err), + logger.Errorf(ctx, "[Tool][WebFetch] 获取页面失败 url=%s err=%v", vp.URL, err) + return fmt.Sprintf("URL: %s\n错误: %v\n", displayURL, err), map[string]interface{}{ - "url": params.URL, - "prompt": params.Prompt, + "url": displayURL, + "prompt": vp.Prompt, "error": err.Error(), }, err } @@ -277,17 +328,18 @@ func (t *WebFetchTool) executeFetch( textContent := t.convertHTMLToText(htmlContent) resultData := map[string]interface{}{ - "url": params.URL, - "prompt": params.Prompt, + "url": displayURL, + "prompt": vp.Prompt, "raw_content": textContent, "content_length": len(textContent), "method": method, } + params := webFetchParams{URL: displayURL, Prompt: vp.Prompt} var summary string var summaryErr error summary, summaryErr = t.processWithLLM(ctx, params, textContent) if summaryErr != nil { - logger.Warnf(ctx, "[Tool][WebFetch] LLM 处理失败 url=%s err=%v", params.URL, summaryErr) + logger.Warnf(ctx, "[Tool][WebFetch] LLM 处理失败 url=%s err=%v", displayURL, summaryErr) } else if summary != "" { resultData["summary"] = summary } @@ -360,18 +412,18 @@ func (t *WebFetchTool) buildOutputText(params webFetchParams, content string, su return builder.String() } -// fetchHTMLContent fetches the HTML content for a web fetch item -func (t *WebFetchTool) fetchHTMLContent(ctx context.Context, targetURL string) (string, string, error) { - html, err := t.fetchWithChromedp(ctx, targetURL) +// fetchHTMLContent fetches the HTML content for a web fetch item using pinned IP (DNS pinning). +func (t *WebFetchTool) fetchHTMLContent(ctx context.Context, vp *validatedParams) (string, string, error) { + html, err := t.fetchWithChromedp(ctx, vp) if err == nil && strings.TrimSpace(html) != "" { return html, "chromedp", nil } if err != nil { - logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取失败 url=%s err=%v,尝试直接请求", targetURL, err) + logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取失败 url=%s err=%v,尝试直接请求", vp.URL, err) } - html, httpErr := t.fetchWithHTTP(ctx, targetURL) + html, httpErr := t.fetchWithHTTP(ctx, vp) if httpErr != nil { if err != nil { return "", "", fmt.Errorf("chromedp error: %v; http error: %w", err, httpErr) @@ -382,12 +434,15 @@ func (t *WebFetchTool) fetchHTMLContent(ctx context.Context, targetURL string) ( return html, "http", nil } -// fetchWithChromedp fetches the HTML content with Chromedp -func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, targetURL string) (string, error) { - logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取开始 url=%s", targetURL) +// fetchWithChromedp fetches the HTML content with Chromedp. Uses host-resolver-rules to pin host to vp.PinnedIP (DNS rebinding protection). +func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, vp *validatedParams) (string, error) { + logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取开始 url=%s", vp.URL) + // DNS pinning: force Chrome to use the IP we resolved at validation time, not a second resolution. + hostRule := fmt.Sprintf("MAP %s %s", vp.Host, vp.PinnedIP.String()) opts := append( chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("host-resolver-rules", hostRule), chromedp.Flag("headless", true), chromedp.Flag("disable-setuid-sandbox", true), chromedp.Flag("disable-dev-shm-usage", true), @@ -410,7 +465,7 @@ func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, targetURL string) var html string err := chromedp.Run(ctx, - chromedp.Navigate(targetURL), + chromedp.Navigate(vp.URL), chromedp.WaitReady("body", chromedp.ByQuery), chromedp.OuterHTML("html", &html), ) @@ -418,13 +473,13 @@ func (t *WebFetchTool) fetchWithChromedp(ctx context.Context, targetURL string) return "", fmt.Errorf("chromedp run failed: %w", err) } - logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取成功 url=%s", targetURL) + logger.Debugf(ctx, "[Tool][WebFetch] Chromedp 抓取成功 url=%s", vp.URL) return html, nil } -// fetchWithHTTP fetches the HTML content with HTTP -func (t *WebFetchTool) fetchWithHTTP(ctx context.Context, targetURL string) (string, error) { - resp, err := t.fetchWithTimeout(ctx, targetURL) +// fetchWithHTTP fetches the HTML content with HTTP using pinned IP (same as chromedp path). +func (t *WebFetchTool) fetchWithHTTP(ctx context.Context, vp *validatedParams) (string, error) { + resp, err := t.fetchWithTimeout(ctx, vp) if err != nil { return "", err } @@ -443,12 +498,19 @@ func (t *WebFetchTool) fetchWithHTTP(ctx context.Context, targetURL string) (str return string(htmlBytes), nil } -// fetchWithTimeout fetches the HTML content with a timeout -func (t *WebFetchTool) fetchWithTimeout(ctx context.Context, targetURL string) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) +// fetchWithTimeout fetches the HTML content with a timeout. Uses pinned IP and original Host header (DNS pinning). +func (t *WebFetchTool) fetchWithTimeout(ctx context.Context, vp *validatedParams) (*http.Response, error) { + // Connect to pinned IP so we do not re-resolve; set Host so the server gets the right virtual host. + hostPort := net.JoinHostPort(vp.PinnedIP.String(), vp.Port) + rawURL := vp.URL + u, _ := url.Parse(rawURL) + u.Host = hostPort + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } + // Preserve original host for TLS SNI and Host header (required for virtual hosting). + req.Host = net.JoinHostPort(vp.Host, vp.Port) req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; WebFetchTool/1.0)") req.Header.Set( diff --git a/internal/utils/security.go b/internal/utils/security.go index 3a200a9c..15924c8c 100644 --- a/internal/utils/security.go +++ b/internal/utils/security.go @@ -244,6 +244,13 @@ func isRestrictedIP(ip net.IP) (bool, string) { return false, "" } +// IsPublicIP returns true if the IP is safe for outbound fetch (not private, loopback, link-local, etc.). +// Used for DNS pinning: after resolving a hostname we pick the first public IP and pin all requests to it. +func IsPublicIP(ip net.IP) bool { + restricted, _ := isRestrictedIP(ip) + return !restricted +} + // isZeros checks if a byte slice is all zeros func isZeros(b []byte) bool { for _, v := range b { @@ -652,10 +659,10 @@ func ValidateStdioConfig(command string, args []string, envVars map[string]strin // SSRFSafeHTTPClientConfig contains configuration for the SSRF-safe HTTP client type SSRFSafeHTTPClientConfig struct { - Timeout time.Duration - MaxRedirects int - DisableKeepAlives bool - DisableCompression bool + Timeout time.Duration + MaxRedirects int + DisableKeepAlives bool + DisableCompression bool } // DefaultSSRFSafeHTTPClientConfig returns the default configuration