fix(security): harden web_fetch and datasource connector SSRF paths
Use SSRF-safe HTTP clients in chat pipeline web_fetch and Feishu/Yuque/Notion connectors, validating tenant-supplied base_url values before outbound requests.
This commit is contained in:
@@ -12,6 +12,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
)
|
||||
|
||||
@@ -55,7 +56,7 @@ func NewClient(config *Config) *Client {
|
||||
baseURL: config.GetBaseURL(),
|
||||
appID: config.AppID,
|
||||
appSecret: config.AppSecret,
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
httpClient: datasource.NewConnectorHTTPClient(30 * time.Second),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/datasource"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
@@ -499,6 +500,10 @@ func parseFeishuConfig(config *types.DataSourceConfig) (*Config, error) {
|
||||
return nil, fmt.Errorf("feishu app_id and app_secret are required")
|
||||
}
|
||||
|
||||
if err := datasource.ValidateConnectorBaseURL(feishuConfig.GetBaseURL()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &feishuConfig, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -6,14 +6,22 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost")
|
||||
secutils.ResetSSRFWhitelistForTest()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
// Fake Feishu API server
|
||||
// ──────────────────────────────────────────────────────────────────────
|
||||
@@ -374,13 +382,13 @@ func TestParseFeishuConfig(t *testing.T) {
|
||||
Credentials: map[string]interface{}{
|
||||
"app_id": "id1",
|
||||
"app_secret": "sec1",
|
||||
"base_url": "https://custom.example.com",
|
||||
"base_url": "https://open.feishu.cn",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if cfg.AppID != "id1" || cfg.AppSecret != "sec1" || cfg.BaseURL != "https://custom.example.com" {
|
||||
if cfg.AppID != "id1" || cfg.AppSecret != "sec1" || cfg.BaseURL != "https://open.feishu.cn" {
|
||||
t.Errorf("unexpected config: %+v", cfg)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -26,16 +26,19 @@ type notionClient struct {
|
||||
}
|
||||
|
||||
// newClient creates a new Notion API client.
|
||||
func newClient(token, baseURL string) *notionClient {
|
||||
func newClient(token, baseURL string) (*notionClient, error) {
|
||||
if baseURL == "" {
|
||||
baseURL = DefaultBaseURL
|
||||
}
|
||||
if err := datasource.ValidateConnectorBaseURL(baseURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ¬ionClient{
|
||||
token: token,
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
limiter: rate.NewLimiter(rate.Limit(3), 3), // 3 req/s with burst of 3 (matches Notion's actual behavior)
|
||||
httpClient: datasource.NewConnectorHTTPClient(30 * time.Second),
|
||||
limiter: rate.NewLimiter(rate.Limit(3), 3),
|
||||
baseURL: baseURL,
|
||||
}
|
||||
}, nil
|
||||
}
|
||||
|
||||
const maxRetries = 3
|
||||
|
||||
@@ -7,8 +7,22 @@ import (
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
func mustTestClient(t *testing.T, token, baseURL string) *notionClient {
|
||||
t.Helper()
|
||||
t.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost")
|
||||
secutils.ResetSSRFWhitelistForTest()
|
||||
|
||||
client, err := newClient(token, baseURL)
|
||||
if err != nil {
|
||||
t.Fatalf("newClient: %v", err)
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
// fakeNotion builds an httptest.Server that emulates relevant Notion API endpoints.
|
||||
func fakeNotion() (*httptest.Server, *Config) {
|
||||
mux := http.NewServeMux()
|
||||
@@ -233,7 +247,7 @@ func TestClientPing(t *testing.T) {
|
||||
ts, cfg := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient(cfg.APIKey, ts.URL)
|
||||
client := mustTestClient(t, cfg.APIKey, ts.URL)
|
||||
if err := client.Ping(context.Background()); err != nil {
|
||||
t.Fatalf("Ping() error: %v", err)
|
||||
}
|
||||
@@ -243,7 +257,7 @@ func TestClientPing_InvalidToken(t *testing.T) {
|
||||
ts, _ := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient("wrong-token", ts.URL)
|
||||
client := mustTestClient(t, "wrong-token", ts.URL)
|
||||
err := client.Ping(context.Background())
|
||||
if err == nil {
|
||||
t.Fatal("expected error for invalid token")
|
||||
@@ -254,7 +268,7 @@ func TestClientSearchPages(t *testing.T) {
|
||||
ts, cfg := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient(cfg.APIKey, ts.URL)
|
||||
client := mustTestClient(t, cfg.APIKey, ts.URL)
|
||||
pages, err := client.SearchPages(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("SearchPages() error: %v", err)
|
||||
@@ -277,7 +291,7 @@ func TestClientGetBlockChildrenAll(t *testing.T) {
|
||||
ts, cfg := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient(cfg.APIKey, ts.URL)
|
||||
client := mustTestClient(t, cfg.APIKey, ts.URL)
|
||||
blocks, err := client.GetBlockChildrenAll(context.Background(), "page-1")
|
||||
if err != nil {
|
||||
t.Fatalf("GetBlockChildrenAll() error: %v", err)
|
||||
@@ -301,7 +315,7 @@ func TestClientGetPage(t *testing.T) {
|
||||
ts, cfg := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient(cfg.APIKey, ts.URL)
|
||||
client := mustTestClient(t, cfg.APIKey, ts.URL)
|
||||
page, err := client.GetPage(context.Background(), "page-1")
|
||||
if err != nil {
|
||||
t.Fatalf("GetPage() error: %v", err)
|
||||
@@ -318,7 +332,7 @@ func TestClientQueryDatabaseAll(t *testing.T) {
|
||||
ts, cfg := fakeNotion()
|
||||
defer ts.Close()
|
||||
|
||||
client := newClient(cfg.APIKey, ts.URL)
|
||||
client := mustTestClient(t, cfg.APIKey, ts.URL)
|
||||
records, err := client.QueryDatabaseAll(context.Background(), "db-1")
|
||||
if err != nil {
|
||||
t.Fatalf("QueryDatabaseAll() error: %v", err)
|
||||
|
||||
@@ -40,7 +40,10 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig
|
||||
return err
|
||||
}
|
||||
|
||||
client := newClient(notionCfg.APIKey, extractBaseURL(config))
|
||||
client, err := newClient(notionCfg.APIKey, extractBaseURL(config))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return client.Ping(ctx)
|
||||
}
|
||||
|
||||
@@ -71,7 +74,10 @@ func (c *Connector) ListResources(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := newClient(notionCfg.APIKey, extractBaseURL(config))
|
||||
client, err := newClient(notionCfg.APIKey, extractBaseURL(config))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pages, err := client.SearchPages(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search notion pages: %w", err)
|
||||
@@ -131,7 +137,10 @@ func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := newClient(notionCfg.APIKey, extractBaseURL(config))
|
||||
client, err := newClient(notionCfg.APIKey, extractBaseURL(config))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
visited := c.excludedSetFromListResources(ctx, config, resourceIDs)
|
||||
var allItems []types.FetchedItem
|
||||
|
||||
@@ -167,7 +176,10 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour
|
||||
return nil, nil, fmt.Errorf("no resource IDs configured")
|
||||
}
|
||||
|
||||
client := newClient(notionCfg.APIKey, extractBaseURL(config))
|
||||
client, err := newClient(notionCfg.APIKey, extractBaseURL(config))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Parse previous cursor
|
||||
var prevCursor notionCursor
|
||||
|
||||
@@ -36,7 +36,7 @@ func newClient(cfg *Config) *client {
|
||||
return &client{
|
||||
baseURL: cfg.GetBaseURL(),
|
||||
token: cfg.APIToken,
|
||||
httpClient: &http.Client{Timeout: defaultTimeout},
|
||||
httpClient: datasource.NewConnectorHTTPClient(defaultTimeout),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,11 +4,19 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost")
|
||||
secutils.ResetSSRFWhitelistForTest()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func makeDSConfig(f *fakeYuque, resourceIDs []string) *types.DataSourceConfig {
|
||||
return &types.DataSourceConfig{
|
||||
Type: types.ConnectorTypeYuque,
|
||||
|
||||
@@ -77,6 +77,9 @@ func parseYuqueConfig(config *types.DataSourceConfig) (*Config, error) {
|
||||
if strings.TrimSpace(cfg.APIToken) == "" {
|
||||
return nil, fmt.Errorf("%w: api_token is required", datasource.ErrInvalidCredentials)
|
||||
}
|
||||
if err := datasource.ValidateConnectorBaseURL(cfg.GetBaseURL()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package datasource
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// ValidateConnectorBaseURL checks a connector API base URL against the SSRF policy.
|
||||
// Empty rawURL is allowed; callers apply their own default before issuing requests.
|
||||
func ValidateConnectorBaseURL(rawURL string) error {
|
||||
url := strings.TrimSpace(rawURL)
|
||||
if url == "" {
|
||||
return nil
|
||||
}
|
||||
if !strings.Contains(url, "://") {
|
||||
url = "https://" + url
|
||||
}
|
||||
if err := utils.ValidateURLForSSRF(url); err != nil {
|
||||
return fmt.Errorf("base_url SSRF validation failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewConnectorHTTPClient returns an HTTP client with redirect and dial-time SSRF guards.
|
||||
func NewConnectorHTTPClient(timeout time.Duration) *http.Client {
|
||||
cfg := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
cfg.Timeout = timeout
|
||||
return utils.NewSSRFSafeHTTPClient(cfg)
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package datasource
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
func TestValidateConnectorBaseURLBlocksLoopback(t *testing.T) {
|
||||
secutils.ResetSSRFWhitelistForTest()
|
||||
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
|
||||
|
||||
err := ValidateConnectorBaseURL("http://127.0.0.1:8000")
|
||||
if err == nil {
|
||||
t.Fatal("expected loopback base_url to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateConnectorBaseURLAllowsPublicHTTPS(t *testing.T) {
|
||||
secutils.ResetSSRFWhitelistForTest()
|
||||
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
|
||||
|
||||
if err := ValidateConnectorBaseURL("https://open.feishu.cn"); err != nil {
|
||||
t.Fatalf("expected public base_url to pass: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -4,10 +4,8 @@ package web_fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
@@ -24,13 +22,12 @@ const (
|
||||
)
|
||||
|
||||
// FetchURLContent fetches a URL and returns its text content (HTML converted to clean text).
|
||||
// Includes SSRF validation, DNS pinning, browser-like headers, and content size limits.
|
||||
// Includes SSRF validation, redirect re-validation, dial-time guards, and content size limits.
|
||||
func FetchURLContent(ctx context.Context, rawURL string) (string, error) {
|
||||
if rawURL == "" {
|
||||
return "", fmt.Errorf("url is empty")
|
||||
}
|
||||
|
||||
// SSRF validation
|
||||
if err := utils.ValidateURLForSSRF(rawURL); err != nil {
|
||||
return "", fmt.Errorf("URL rejected: %w", err)
|
||||
}
|
||||
@@ -40,53 +37,22 @@ func FetchURLContent(ctx context.Context, rawURL string) (string, error) {
|
||||
return "", fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
hostname := u.Hostname()
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
if u.Scheme == "https" {
|
||||
port = "443"
|
||||
} else {
|
||||
port = "80"
|
||||
}
|
||||
}
|
||||
|
||||
// DNS pinning: resolve once, use pinned IP
|
||||
ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip", hostname)
|
||||
if err != nil || len(ips) == 0 {
|
||||
return "", fmt.Errorf("DNS lookup failed for %s: %w", hostname, err)
|
||||
}
|
||||
var pinnedIP net.IP
|
||||
for _, ip := range ips {
|
||||
if utils.IsPublicIP(ip) {
|
||||
pinnedIP = ip
|
||||
break
|
||||
}
|
||||
}
|
||||
if pinnedIP == nil {
|
||||
return "", fmt.Errorf("no public IP for host %s", hostname)
|
||||
}
|
||||
|
||||
// Build request with pinned IP
|
||||
hostPort := net.JoinHostPort(pinnedIP.String(), port)
|
||||
fetchURL := *u
|
||||
fetchURL.Host = hostPort
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, fetchTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fetchURL.String(), nil)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Host = hostname
|
||||
|
||||
// Browser-like headers to reduce 403 rejections.
|
||||
// These match a real Chrome browser fingerprint.
|
||||
req.Header.Set("User-Agent",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept",
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
|
||||
req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
|
||||
req.Header.Set("Accept-Encoding", "identity") // no gzip to simplify reading
|
||||
req.Header.Set("Accept-Encoding", "identity")
|
||||
req.Header.Set("Cache-Control", "no-cache")
|
||||
req.Header.Set("Pragma", "no-cache")
|
||||
req.Header.Set("Sec-Ch-Ua", `"Chromium";v="131", "Not_A Brand";v="24"`)
|
||||
@@ -99,16 +65,10 @@ func FetchURLContent(ctx context.Context, rawURL string) (string, error) {
|
||||
req.Header.Set("Upgrade-Insecure-Requests", "1")
|
||||
req.Header.Set("Referer", u.Scheme+"://"+hostname+"/")
|
||||
|
||||
// Custom transport: TLS ServerName for certificate validation with pinned IP.
|
||||
client := &http.Client{
|
||||
Timeout: fetchTimeout,
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
ServerName: hostname,
|
||||
},
|
||||
},
|
||||
// Follow redirects (default behavior), up to 10 hops
|
||||
}
|
||||
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
|
||||
Timeout: fetchTimeout,
|
||||
MaxRedirects: 10,
|
||||
})
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fetch failed: %w", err)
|
||||
@@ -143,7 +103,6 @@ func htmlToText(html string) string {
|
||||
})
|
||||
text := sb.String()
|
||||
|
||||
// Normalize whitespace: collapse blank lines
|
||||
lines := strings.Split(text, "\n")
|
||||
var cleaned []string
|
||||
for _, line := range lines {
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
package web_fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchURLContentBlocksRedirectToLoopback(t *testing.T) {
|
||||
internalHit := make(chan struct{}, 1)
|
||||
internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
internalHit <- struct{}{}
|
||||
fmt.Fprint(w, "<html><body>WEKNORA_INTERNAL_CANARY_178193</body></html>")
|
||||
}))
|
||||
defer internal.Close()
|
||||
|
||||
if _, err := FetchURLContent(context.Background(), internal.URL); err == nil {
|
||||
t.Fatal("direct loopback URL was not rejected")
|
||||
}
|
||||
|
||||
attacker := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, internal.URL, http.StatusFound)
|
||||
}))
|
||||
defer attacker.Close()
|
||||
|
||||
got, err := FetchURLContent(context.Background(), attacker.URL+"/entry")
|
||||
if err == nil {
|
||||
if strings.Contains(got, "WEKNORA_INTERNAL_CANARY_178193") {
|
||||
t.Fatalf("redirect followed to loopback, got %q", got)
|
||||
}
|
||||
t.Fatal("expected redirect to loopback to be blocked")
|
||||
}
|
||||
select {
|
||||
case <-internalHit:
|
||||
t.Fatal("internal loopback server was reached via redirect")
|
||||
default:
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user