Files
Enterprise-Knowledge-Base/internal/infrastructure/web_fetch/fetcher.go
T
wizardchen 8325dac738 fix(security): harden web_fetch and datasource connector SSRF paths
Use SSRF-safe HTTP clients in chat pipeline web_fetch and Feishu/Yuque/Notion
connectors, validating tenant-supplied base_url values before outbound requests.
2026-07-02 12:27:48 +08:00

131 lines
3.7 KiB
Go

// Package web_fetch provides a public URL content fetcher with SSRF protection.
// It extracts core logic from the agent WebFetchTool so it can be used by the chat pipeline.
package web_fetch
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/utils"
)
const (
fetchTimeout = 15 * time.Second
maxBodySize = 100 * 1024 // 100KB
)
// FetchURLContent fetches a URL and returns its text content (HTML converted to clean text).
// Includes SSRF validation, redirect re-validation, dial-time guards, and content size limits.
func FetchURLContent(ctx context.Context, rawURL string) (string, error) {
if rawURL == "" {
return "", fmt.Errorf("url is empty")
}
if err := utils.ValidateURLForSSRF(rawURL); err != nil {
return "", fmt.Errorf("URL rejected: %w", err)
}
u, err := url.Parse(rawURL)
if err != nil {
return "", fmt.Errorf("invalid URL: %w", err)
}
hostname := u.Hostname()
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return "", err
}
// Browser-like headers to reduce 403 rejections.
req.Header.Set("User-Agent",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
req.Header.Set("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
req.Header.Set("Accept-Encoding", "identity")
req.Header.Set("Cache-Control", "no-cache")
req.Header.Set("Pragma", "no-cache")
req.Header.Set("Sec-Ch-Ua", `"Chromium";v="131", "Not_A Brand";v="24"`)
req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
req.Header.Set("Sec-Ch-Ua-Platform", `"macOS"`)
req.Header.Set("Sec-Fetch-Dest", "document")
req.Header.Set("Sec-Fetch-Mode", "navigate")
req.Header.Set("Sec-Fetch-Site", "none")
req.Header.Set("Sec-Fetch-User", "?1")
req.Header.Set("Upgrade-Insecure-Requests", "1")
req.Header.Set("Referer", u.Scheme+"://"+hostname+"/")
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
Timeout: fetchTimeout,
MaxRedirects: 10,
})
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("fetch failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodySize))
if err != nil {
return "", fmt.Errorf("read failed: %w", err)
}
text := htmlToText(string(body))
logger.Infof(ctx, "[WebFetch] fetched %s → %d chars", rawURL, len(text))
return text, nil
}
// htmlToText extracts clean text from HTML, removing scripts/styles/nav.
func htmlToText(html string) string {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
if err != nil {
return stripTags(html)
}
doc.Find("script, style, nav, footer, header, iframe, noscript, svg, img").Remove()
var sb strings.Builder
doc.Find("body").Each(func(i int, s *goquery.Selection) {
sb.WriteString(s.Text())
})
text := sb.String()
lines := strings.Split(text, "\n")
var cleaned []string
for _, line := range lines {
line = strings.TrimSpace(line)
if line != "" {
cleaned = append(cleaned, line)
}
}
return strings.Join(cleaned, "\n")
}
func stripTags(s string) string {
var sb strings.Builder
inTag := false
for _, r := range s {
if r == '<' {
inTag = true
} else if r == '>' {
inTag = false
} else if !inTag {
sb.WriteRune(r)
}
}
return sb.String()
}