From 8325dac738b65d7aa7d181708ad7a9320b9fd24e Mon Sep 17 00:00:00 2001 From: wizardchen Date: Thu, 2 Jul 2026 12:21:39 +0800 Subject: [PATCH] fix(security): harden web_fetch and datasource connector SSRF paths Use SSRF-safe HTTP clients in chat pipeline web_fetch and Feishu/Yuque/Notion connectors, validating tenant-supplied base_url values before outbound requests. --- .../datasource/connector/feishu/client.go | 3 +- .../datasource/connector/feishu/connector.go | 5 ++ .../connector/feishu/connector_test.go | 12 +++- .../datasource/connector/notion/client.go | 11 ++-- .../connector/notion/client_test.go | 26 +++++++-- .../datasource/connector/notion/connector.go | 20 +++++-- internal/datasource/connector/yuque/client.go | 2 +- .../connector/yuque/connector_test.go | 8 +++ internal/datasource/connector/yuque/types.go | 3 + internal/datasource/httpclient.go | 33 +++++++++++ internal/datasource/httpclient_test.go | 26 +++++++++ internal/infrastructure/web_fetch/fetcher.go | 55 +++---------------- .../web_fetch/fetcher_ssrf_test.go | 41 ++++++++++++++ 13 files changed, 179 insertions(+), 66 deletions(-) create mode 100644 internal/datasource/httpclient.go create mode 100644 internal/datasource/httpclient_test.go create mode 100644 internal/infrastructure/web_fetch/fetcher_ssrf_test.go diff --git a/internal/datasource/connector/feishu/client.go b/internal/datasource/connector/feishu/client.go index ad3a29d3..91f70d44 100644 --- a/internal/datasource/connector/feishu/client.go +++ b/internal/datasource/connector/feishu/client.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/Tencent/WeKnora/internal/datasource" "github.com/Tencent/WeKnora/internal/logger" ) @@ -55,7 +56,7 @@ func NewClient(config *Config) *Client { baseURL: config.GetBaseURL(), appID: config.AppID, appSecret: config.AppSecret, - httpClient: &http.Client{Timeout: 30 * time.Second}, + httpClient: datasource.NewConnectorHTTPClient(30 * time.Second), } } diff --git a/internal/datasource/connector/feishu/connector.go b/internal/datasource/connector/feishu/connector.go index 7e5817c2..d810f3d2 100644 --- a/internal/datasource/connector/feishu/connector.go +++ b/internal/datasource/connector/feishu/connector.go @@ -10,6 +10,7 @@ import ( "time" "unicode/utf8" + "github.com/Tencent/WeKnora/internal/datasource" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" ) @@ -499,6 +500,10 @@ func parseFeishuConfig(config *types.DataSourceConfig) (*Config, error) { return nil, fmt.Errorf("feishu app_id and app_secret are required") } + if err := datasource.ValidateConnectorBaseURL(feishuConfig.GetBaseURL()); err != nil { + return nil, err + } + return &feishuConfig, nil } diff --git a/internal/datasource/connector/feishu/connector_test.go b/internal/datasource/connector/feishu/connector_test.go index 0768d77c..e3fa3b8f 100644 --- a/internal/datasource/connector/feishu/connector_test.go +++ b/internal/datasource/connector/feishu/connector_test.go @@ -6,14 +6,22 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" "strings" "testing" "time" "unicode/utf8" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/Tencent/WeKnora/internal/types" ) +func TestMain(m *testing.M) { + os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost") + secutils.ResetSSRFWhitelistForTest() + os.Exit(m.Run()) +} + // ────────────────────────────────────────────────────────────────────── // Fake Feishu API server // ────────────────────────────────────────────────────────────────────── @@ -374,13 +382,13 @@ func TestParseFeishuConfig(t *testing.T) { Credentials: map[string]interface{}{ "app_id": "id1", "app_secret": "sec1", - "base_url": "https://custom.example.com", + "base_url": "https://open.feishu.cn", }, }) if err != nil { t.Fatalf("unexpected error: %v", err) } - if cfg.AppID != "id1" || cfg.AppSecret != "sec1" || cfg.BaseURL != "https://custom.example.com" { + if cfg.AppID != "id1" || cfg.AppSecret != "sec1" || cfg.BaseURL != "https://open.feishu.cn" { t.Errorf("unexpected config: %+v", cfg) } }) diff --git a/internal/datasource/connector/notion/client.go b/internal/datasource/connector/notion/client.go index a91e355e..40646fb0 100644 --- a/internal/datasource/connector/notion/client.go +++ b/internal/datasource/connector/notion/client.go @@ -26,16 +26,19 @@ type notionClient struct { } // newClient creates a new Notion API client. -func newClient(token, baseURL string) *notionClient { +func newClient(token, baseURL string) (*notionClient, error) { if baseURL == "" { baseURL = DefaultBaseURL } + if err := datasource.ValidateConnectorBaseURL(baseURL); err != nil { + return nil, err + } return ¬ionClient{ token: token, - httpClient: &http.Client{Timeout: 30 * time.Second}, - limiter: rate.NewLimiter(rate.Limit(3), 3), // 3 req/s with burst of 3 (matches Notion's actual behavior) + httpClient: datasource.NewConnectorHTTPClient(30 * time.Second), + limiter: rate.NewLimiter(rate.Limit(3), 3), baseURL: baseURL, - } + }, nil } const maxRetries = 3 diff --git a/internal/datasource/connector/notion/client_test.go b/internal/datasource/connector/notion/client_test.go index 1eeec22e..943942af 100644 --- a/internal/datasource/connector/notion/client_test.go +++ b/internal/datasource/connector/notion/client_test.go @@ -7,8 +7,22 @@ import ( "net/http/httptest" "testing" "time" + + secutils "github.com/Tencent/WeKnora/internal/utils" ) +func mustTestClient(t *testing.T, token, baseURL string) *notionClient { + t.Helper() + t.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost") + secutils.ResetSSRFWhitelistForTest() + + client, err := newClient(token, baseURL) + if err != nil { + t.Fatalf("newClient: %v", err) + } + return client +} + // fakeNotion builds an httptest.Server that emulates relevant Notion API endpoints. func fakeNotion() (*httptest.Server, *Config) { mux := http.NewServeMux() @@ -233,7 +247,7 @@ func TestClientPing(t *testing.T) { ts, cfg := fakeNotion() defer ts.Close() - client := newClient(cfg.APIKey, ts.URL) + client := mustTestClient(t, cfg.APIKey, ts.URL) if err := client.Ping(context.Background()); err != nil { t.Fatalf("Ping() error: %v", err) } @@ -243,7 +257,7 @@ func TestClientPing_InvalidToken(t *testing.T) { ts, _ := fakeNotion() defer ts.Close() - client := newClient("wrong-token", ts.URL) + client := mustTestClient(t, "wrong-token", ts.URL) err := client.Ping(context.Background()) if err == nil { t.Fatal("expected error for invalid token") @@ -254,7 +268,7 @@ func TestClientSearchPages(t *testing.T) { ts, cfg := fakeNotion() defer ts.Close() - client := newClient(cfg.APIKey, ts.URL) + client := mustTestClient(t, cfg.APIKey, ts.URL) pages, err := client.SearchPages(context.Background()) if err != nil { t.Fatalf("SearchPages() error: %v", err) @@ -277,7 +291,7 @@ func TestClientGetBlockChildrenAll(t *testing.T) { ts, cfg := fakeNotion() defer ts.Close() - client := newClient(cfg.APIKey, ts.URL) + client := mustTestClient(t, cfg.APIKey, ts.URL) blocks, err := client.GetBlockChildrenAll(context.Background(), "page-1") if err != nil { t.Fatalf("GetBlockChildrenAll() error: %v", err) @@ -301,7 +315,7 @@ func TestClientGetPage(t *testing.T) { ts, cfg := fakeNotion() defer ts.Close() - client := newClient(cfg.APIKey, ts.URL) + client := mustTestClient(t, cfg.APIKey, ts.URL) page, err := client.GetPage(context.Background(), "page-1") if err != nil { t.Fatalf("GetPage() error: %v", err) @@ -318,7 +332,7 @@ func TestClientQueryDatabaseAll(t *testing.T) { ts, cfg := fakeNotion() defer ts.Close() - client := newClient(cfg.APIKey, ts.URL) + client := mustTestClient(t, cfg.APIKey, ts.URL) records, err := client.QueryDatabaseAll(context.Background(), "db-1") if err != nil { t.Fatalf("QueryDatabaseAll() error: %v", err) diff --git a/internal/datasource/connector/notion/connector.go b/internal/datasource/connector/notion/connector.go index 4d91fb27..05d107ca 100644 --- a/internal/datasource/connector/notion/connector.go +++ b/internal/datasource/connector/notion/connector.go @@ -40,7 +40,10 @@ func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig return err } - client := newClient(notionCfg.APIKey, extractBaseURL(config)) + client, err := newClient(notionCfg.APIKey, extractBaseURL(config)) + if err != nil { + return err + } return client.Ping(ctx) } @@ -71,7 +74,10 @@ func (c *Connector) ListResources( return nil, err } - client := newClient(notionCfg.APIKey, extractBaseURL(config)) + client, err := newClient(notionCfg.APIKey, extractBaseURL(config)) + if err != nil { + return nil, err + } pages, err := client.SearchPages(ctx) if err != nil { return nil, fmt.Errorf("search notion pages: %w", err) @@ -131,7 +137,10 @@ func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig return nil, err } - client := newClient(notionCfg.APIKey, extractBaseURL(config)) + client, err := newClient(notionCfg.APIKey, extractBaseURL(config)) + if err != nil { + return nil, err + } visited := c.excludedSetFromListResources(ctx, config, resourceIDs) var allItems []types.FetchedItem @@ -167,7 +176,10 @@ func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSour return nil, nil, fmt.Errorf("no resource IDs configured") } - client := newClient(notionCfg.APIKey, extractBaseURL(config)) + client, err := newClient(notionCfg.APIKey, extractBaseURL(config)) + if err != nil { + return nil, nil, err + } // Parse previous cursor var prevCursor notionCursor diff --git a/internal/datasource/connector/yuque/client.go b/internal/datasource/connector/yuque/client.go index b403c10a..7de251ef 100644 --- a/internal/datasource/connector/yuque/client.go +++ b/internal/datasource/connector/yuque/client.go @@ -36,7 +36,7 @@ func newClient(cfg *Config) *client { return &client{ baseURL: cfg.GetBaseURL(), token: cfg.APIToken, - httpClient: &http.Client{Timeout: defaultTimeout}, + httpClient: datasource.NewConnectorHTTPClient(defaultTimeout), } } diff --git a/internal/datasource/connector/yuque/connector_test.go b/internal/datasource/connector/yuque/connector_test.go index e38dcb05..670d0c88 100644 --- a/internal/datasource/connector/yuque/connector_test.go +++ b/internal/datasource/connector/yuque/connector_test.go @@ -4,11 +4,19 @@ import ( "context" "net/http" "net/http/httptest" + "os" "testing" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/Tencent/WeKnora/internal/types" ) +func TestMain(m *testing.M) { + os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost") + secutils.ResetSSRFWhitelistForTest() + os.Exit(m.Run()) +} + func makeDSConfig(f *fakeYuque, resourceIDs []string) *types.DataSourceConfig { return &types.DataSourceConfig{ Type: types.ConnectorTypeYuque, diff --git a/internal/datasource/connector/yuque/types.go b/internal/datasource/connector/yuque/types.go index f24ec571..63d878d5 100644 --- a/internal/datasource/connector/yuque/types.go +++ b/internal/datasource/connector/yuque/types.go @@ -77,6 +77,9 @@ func parseYuqueConfig(config *types.DataSourceConfig) (*Config, error) { if strings.TrimSpace(cfg.APIToken) == "" { return nil, fmt.Errorf("%w: api_token is required", datasource.ErrInvalidCredentials) } + if err := datasource.ValidateConnectorBaseURL(cfg.GetBaseURL()); err != nil { + return nil, err + } return &cfg, nil } diff --git a/internal/datasource/httpclient.go b/internal/datasource/httpclient.go new file mode 100644 index 00000000..e7727e17 --- /dev/null +++ b/internal/datasource/httpclient.go @@ -0,0 +1,33 @@ +package datasource + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/utils" +) + +// ValidateConnectorBaseURL checks a connector API base URL against the SSRF policy. +// Empty rawURL is allowed; callers apply their own default before issuing requests. +func ValidateConnectorBaseURL(rawURL string) error { + url := strings.TrimSpace(rawURL) + if url == "" { + return nil + } + if !strings.Contains(url, "://") { + url = "https://" + url + } + if err := utils.ValidateURLForSSRF(url); err != nil { + return fmt.Errorf("base_url SSRF validation failed: %w", err) + } + return nil +} + +// NewConnectorHTTPClient returns an HTTP client with redirect and dial-time SSRF guards. +func NewConnectorHTTPClient(timeout time.Duration) *http.Client { + cfg := utils.DefaultSSRFSafeHTTPClientConfig() + cfg.Timeout = timeout + return utils.NewSSRFSafeHTTPClient(cfg) +} diff --git a/internal/datasource/httpclient_test.go b/internal/datasource/httpclient_test.go new file mode 100644 index 00000000..811aecae --- /dev/null +++ b/internal/datasource/httpclient_test.go @@ -0,0 +1,26 @@ +package datasource + +import ( + "testing" + + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +func TestValidateConnectorBaseURLBlocksLoopback(t *testing.T) { + secutils.ResetSSRFWhitelistForTest() + t.Cleanup(secutils.ResetSSRFWhitelistForTest) + + err := ValidateConnectorBaseURL("http://127.0.0.1:8000") + if err == nil { + t.Fatal("expected loopback base_url to be rejected") + } +} + +func TestValidateConnectorBaseURLAllowsPublicHTTPS(t *testing.T) { + secutils.ResetSSRFWhitelistForTest() + t.Cleanup(secutils.ResetSSRFWhitelistForTest) + + if err := ValidateConnectorBaseURL("https://open.feishu.cn"); err != nil { + t.Fatalf("expected public base_url to pass: %v", err) + } +} diff --git a/internal/infrastructure/web_fetch/fetcher.go b/internal/infrastructure/web_fetch/fetcher.go index 96e5764d..dd45de90 100644 --- a/internal/infrastructure/web_fetch/fetcher.go +++ b/internal/infrastructure/web_fetch/fetcher.go @@ -4,10 +4,8 @@ package web_fetch import ( "context" - "crypto/tls" "fmt" "io" - "net" "net/http" "net/url" "strings" @@ -24,13 +22,12 @@ const ( ) // FetchURLContent fetches a URL and returns its text content (HTML converted to clean text). -// Includes SSRF validation, DNS pinning, browser-like headers, and content size limits. +// Includes SSRF validation, redirect re-validation, dial-time guards, and content size limits. func FetchURLContent(ctx context.Context, rawURL string) (string, error) { if rawURL == "" { return "", fmt.Errorf("url is empty") } - // SSRF validation if err := utils.ValidateURLForSSRF(rawURL); err != nil { return "", fmt.Errorf("URL rejected: %w", err) } @@ -40,53 +37,22 @@ func FetchURLContent(ctx context.Context, rawURL string) (string, error) { return "", fmt.Errorf("invalid URL: %w", err) } hostname := u.Hostname() - port := u.Port() - if port == "" { - if u.Scheme == "https" { - port = "443" - } else { - port = "80" - } - } - - // DNS pinning: resolve once, use pinned IP - ips, err := net.DefaultResolver.LookupIP(context.Background(), "ip", hostname) - if err != nil || len(ips) == 0 { - return "", fmt.Errorf("DNS lookup failed for %s: %w", hostname, err) - } - var pinnedIP net.IP - for _, ip := range ips { - if utils.IsPublicIP(ip) { - pinnedIP = ip - break - } - } - if pinnedIP == nil { - return "", fmt.Errorf("no public IP for host %s", hostname) - } - - // Build request with pinned IP - hostPort := net.JoinHostPort(pinnedIP.String(), port) - fetchURL := *u - fetchURL.Host = hostPort ctx, cancel := context.WithTimeout(ctx, fetchTimeout) defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, fetchURL.String(), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) if err != nil { return "", err } - req.Host = hostname // Browser-like headers to reduce 403 rejections. - // These match a real Chrome browser fingerprint. req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6") - req.Header.Set("Accept-Encoding", "identity") // no gzip to simplify reading + req.Header.Set("Accept-Encoding", "identity") req.Header.Set("Cache-Control", "no-cache") req.Header.Set("Pragma", "no-cache") req.Header.Set("Sec-Ch-Ua", `"Chromium";v="131", "Not_A Brand";v="24"`) @@ -99,16 +65,10 @@ func FetchURLContent(ctx context.Context, rawURL string) (string, error) { req.Header.Set("Upgrade-Insecure-Requests", "1") req.Header.Set("Referer", u.Scheme+"://"+hostname+"/") - // Custom transport: TLS ServerName for certificate validation with pinned IP. - client := &http.Client{ - Timeout: fetchTimeout, - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - ServerName: hostname, - }, - }, - // Follow redirects (default behavior), up to 10 hops - } + client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{ + Timeout: fetchTimeout, + MaxRedirects: 10, + }) resp, err := client.Do(req) if err != nil { return "", fmt.Errorf("fetch failed: %w", err) @@ -143,7 +103,6 @@ func htmlToText(html string) string { }) text := sb.String() - // Normalize whitespace: collapse blank lines lines := strings.Split(text, "\n") var cleaned []string for _, line := range lines { diff --git a/internal/infrastructure/web_fetch/fetcher_ssrf_test.go b/internal/infrastructure/web_fetch/fetcher_ssrf_test.go new file mode 100644 index 00000000..0289682b --- /dev/null +++ b/internal/infrastructure/web_fetch/fetcher_ssrf_test.go @@ -0,0 +1,41 @@ +package web_fetch + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestFetchURLContentBlocksRedirectToLoopback(t *testing.T) { + internalHit := make(chan struct{}, 1) + internal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + internalHit <- struct{}{} + fmt.Fprint(w, "WEKNORA_INTERNAL_CANARY_178193") + })) + defer internal.Close() + + if _, err := FetchURLContent(context.Background(), internal.URL); err == nil { + t.Fatal("direct loopback URL was not rejected") + } + + attacker := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, internal.URL, http.StatusFound) + })) + defer attacker.Close() + + got, err := FetchURLContent(context.Background(), attacker.URL+"/entry") + if err == nil { + if strings.Contains(got, "WEKNORA_INTERNAL_CANARY_178193") { + t.Fatalf("redirect followed to loopback, got %q", got) + } + t.Fatal("expected redirect to loopback to be blocked") + } + select { + case <-internalHit: + t.Fatal("internal loopback server was reached via redirect") + default: + } +}