feat(web-search): refactor web search provider
- Updated relevant files to include provider registration, implementation, and metadata. - Enhanced frontend components to support provider management and configuration. - Added localization for new provider settings and messages. - Implemented backend repository methods for CRUD operations on web search providers.
This commit is contained in:
@@ -0,0 +1,171 @@
|
||||
// Package web_fetch provides a public URL content fetcher with SSRF protection.
|
||||
// It extracts core logic from the agent WebFetchTool so it can be used by the chat pipeline.
|
||||
package web_fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
fetchTimeout = 15 * time.Second
|
||||
maxBodySize = 100 * 1024 // 100KB
|
||||
)
|
||||
|
||||
// FetchURLContent fetches a URL and returns its text content (HTML converted to clean text).
|
||||
// Includes SSRF validation, DNS pinning, browser-like headers, and content size limits.
|
||||
func FetchURLContent(ctx context.Context, rawURL string) (string, error) {
|
||||
if rawURL == "" {
|
||||
return "", fmt.Errorf("url is empty")
|
||||
}
|
||||
|
||||
// SSRF validation
|
||||
if safe, reason := utils.IsSSRFSafeURL(rawURL); !safe {
|
||||
return "", fmt.Errorf("URL rejected: %s", reason)
|
||||
}
|
||||
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
hostname := u.Hostname()
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
if u.Scheme == "https" {
|
||||
port = "443"
|
||||
} else {
|
||||
port = "80"
|
||||
}
|
||||
}
|
||||
|
||||
// DNS pinning: resolve once, use pinned IP
|
||||
ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip", hostname)
|
||||
if err != nil || len(ips) == 0 {
|
||||
return "", fmt.Errorf("DNS lookup failed for %s: %w", hostname, err)
|
||||
}
|
||||
var pinnedIP net.IP
|
||||
for _, ip := range ips {
|
||||
if utils.IsPublicIP(ip) {
|
||||
pinnedIP = ip
|
||||
break
|
||||
}
|
||||
}
|
||||
if pinnedIP == nil {
|
||||
return "", fmt.Errorf("no public IP for host %s", hostname)
|
||||
}
|
||||
|
||||
// Build request with pinned IP
|
||||
hostPort := net.JoinHostPort(pinnedIP.String(), port)
|
||||
fetchURL := *u
|
||||
fetchURL.Host = hostPort
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fetchURL.String(), nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Host = hostname
|
||||
|
||||
// Browser-like headers to reduce 403 rejections.
|
||||
// These match a real Chrome browser fingerprint.
|
||||
req.Header.Set("User-Agent",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept",
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
|
||||
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
|
||||
req.Header.Set("Accept-Encoding", "identity") // no gzip to simplify reading
|
||||
req.Header.Set("Cache-Control", "no-cache")
|
||||
req.Header.Set("Pragma", "no-cache")
|
||||
req.Header.Set("Sec-Ch-Ua", `"Chromium";v="131", "Not_A Brand";v="24"`)
|
||||
req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
|
||||
req.Header.Set("Sec-Ch-Ua-Platform", `"macOS"`)
|
||||
req.Header.Set("Sec-Fetch-Dest", "document")
|
||||
req.Header.Set("Sec-Fetch-Mode", "navigate")
|
||||
req.Header.Set("Sec-Fetch-Site", "none")
|
||||
req.Header.Set("Sec-Fetch-User", "?1")
|
||||
req.Header.Set("Upgrade-Insecure-Requests", "1")
|
||||
req.Header.Set("Referer", u.Scheme+"://"+hostname+"/")
|
||||
|
||||
// Custom transport: TLS ServerName for certificate validation with pinned IP.
|
||||
client := &http.Client{
|
||||
Timeout: fetchTimeout,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
ServerName: hostname,
|
||||
},
|
||||
},
|
||||
// Follow redirects (default behavior), up to 10 hops
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetch failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP %d %s", resp.StatusCode, resp.Status)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxBodySize))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read failed: %w", err)
|
||||
}
|
||||
|
||||
text := htmlToText(string(body))
|
||||
logger.Infof(ctx, "[WebFetch] fetched %s → %d chars", rawURL, len(text))
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// htmlToText extracts clean text from HTML, removing scripts/styles/nav.
|
||||
func htmlToText(html string) string {
|
||||
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
|
||||
if err != nil {
|
||||
return stripTags(html)
|
||||
}
|
||||
doc.Find("script, style, nav, footer, header, iframe, noscript, svg, img").Remove()
|
||||
|
||||
var sb strings.Builder
|
||||
doc.Find("body").Each(func(i int, s *goquery.Selection) {
|
||||
sb.WriteString(s.Text())
|
||||
})
|
||||
text := sb.String()
|
||||
|
||||
// Normalize whitespace: collapse blank lines
|
||||
lines := strings.Split(text, "\n")
|
||||
var cleaned []string
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" {
|
||||
cleaned = append(cleaned, line)
|
||||
}
|
||||
}
|
||||
return strings.Join(cleaned, "\n")
|
||||
}
|
||||
|
||||
func stripTags(s string) string {
|
||||
var sb strings.Builder
|
||||
inTag := false
|
||||
for _, r := range s {
|
||||
if r == '<' {
|
||||
inTag = true
|
||||
} else if r == '>' {
|
||||
inTag = false
|
||||
} else if !inTag {
|
||||
sb.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
Reference in New Issue
Block a user