diff --git a/.env.example b/.env.example index c3346f1c..c9fedbaf 100644 --- a/.env.example +++ b/.env.example @@ -685,10 +685,13 @@ LANGFUSE_HOST=http://langfuse-web:3000 # # ===== 向量库地址的 SSRF 校验 ===== # 通过 API 创建/测试向量库连接时,地址会经过 SSRF 校验。docker-compose 内置服务 -# (qdrant, milvus, weaviate, doris-fe, searxng)已通过 SSRF_WHITELIST_EXTRA 默认放行。 +# (qdrant, milvus, weaviate, doris-fe, doris-be, searxng)已通过 SSRF_WHITELIST_EXTRA 默认放行。 # 如使用外部私网地址或被拦截端口(5432, 9200 等),请将主机加入 SSRF_WHITELIST_EXTRA # (注意:自定义该变量会覆盖 compose 默认值,需重新包含内置服务名),例如: -# SSRF_WHITELIST_EXTRA=searxng,qdrant,milvus,weaviate,doris-fe,my-opensearch.internal +# SSRF_WHITELIST_EXTRA=searxng,qdrant,milvus,weaviate,doris-fe,doris-be,my-opensearch.internal +# +# Doris Stream Load:FE 会将请求 307 重定向到 BE;docker compose 默认放行 +# doris-fe 与 doris-be(见 SSRF_WHITELIST_EXTRA)。 # # 本地开发(go run + docker-compose.dev.yml --profile opensearch):后端跑在宿主机, # OpenSearch 经 published 端口 http://localhost:9200 访问;通过 API 创建该向量库前, diff --git a/docker-compose.yml b/docker-compose.yml index ecd34e39..ad3a2db5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -291,7 +291,9 @@ services: # overrides don't clobber it. (postgres is intentionally NOT listed: it # is not a user-registerable vector store, and whitelisting the app's own # DB host would skip the 5432 port block.) - - SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng,qdrant,milvus,weaviate,doris-fe} + # doris-be: Stream Load follows FE→BE redirects that must receive Basic + # credentials on whitelisted BE hosts. + - SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng,qdrant,milvus,weaviate,doris-fe,doris-be} - CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5} - JWT_SECRET=${JWT_SECRET:-} # File size limit (in MB) diff --git a/internal/application/repository/retriever/doris/repository.go b/internal/application/repository/retriever/doris/repository.go index 72ccd78a..41fcb4de 100644 --- a/internal/application/repository/retriever/doris/repository.go +++ b/internal/application/repository/retriever/doris/repository.go @@ -4,7 +4,6 @@ import ( "context" "database/sql" "fmt" - "net/http" "os" "strings" @@ -42,15 +41,15 @@ func NewDorisRetrieveEngineRepository( } repo := &dorisRepository{ - db: db, - httpClient: &http.Client{}, - feHTTPBase: strings.TrimRight(feHTTPBase, "/"), - username: username, - password: password, - database: database, - tableBaseName: tableBaseName, - bucketsNum: indexCfg.GetBucketsNum(0), - replicationNum: indexCfg.GetReplicationNum(0), + db: db, + httpClient: newDorisStreamLoadHTTPClient(), + feHTTPBase: strings.TrimRight(feHTTPBase, "/"), + username: username, + password: password, + database: database, + tableBaseName: tableBaseName, + bucketsNum: indexCfg.GetBucketsNum(0), + replicationNum: indexCfg.GetReplicationNum(0), compatModeRequested: compatMode, } log.Infof("[Doris] Repository initialized: db=%s, base=%s, fe_http=%s, compat_mode=%s", diff --git a/internal/application/repository/retriever/doris/repository_test.go b/internal/application/repository/retriever/doris/repository_test.go index f6dd70b2..03d27a73 100644 --- a/internal/application/repository/retriever/doris/repository_test.go +++ b/internal/application/repository/retriever/doris/repository_test.go @@ -15,6 +15,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -25,6 +26,12 @@ import ( // 返回的 cleanup 用 defer 调用即可。 func newTestRepo(t *testing.T) (*dorisRepository, sqlmock.Sqlmock, *httptest.Server, func()) { t.Helper() + // httptest binds to loopback, which production SSRF policy correctly blocks. + // Make that one test host explicit and restore the process-global policy when + // the test completes. + secutils.SetSSRFWhitelistFromRaw("127.0.0.1") + t.Cleanup(secutils.ResetSSRFWhitelistForTest) + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp)) require.NoError(t, err) @@ -231,6 +238,63 @@ func TestPartialUpdateRows_FailureSurfaced(t *testing.T) { assert.Contains(t, err.Error(), "stream load failed") } +func TestStreamLoadOnce_BlocksUnsafeTarget(t *testing.T) { + repo, _, _, cleanup := newTestRepo(t) + defer cleanup() + + // Override the helper's loopback allowance: link-local metadata endpoints + // must be rejected before the HTTP client is called. + secutils.SetSSRFWhitelistFromRaw("") + repo.feHTTPBase = "http://169.254.169.254" + + err := repo.streamLoadOnce(context.Background(), "t", []string{"id"}, + []map[string]any{{"id": "x"}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "blocked by SSRF validation") +} + +func TestDorisStreamLoadHTTPClient_BlocksUnsafeRedirect(t *testing.T) { + secutils.SetSSRFWhitelistFromRaw("127.0.0.1") + t.Cleanup(secutils.ResetSSRFWhitelistForTest) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://169.254.169.254/latest/meta-data", http.StatusTemporaryRedirect) + })) + defer server.Close() + + req, err := http.NewRequest(http.MethodPut, server.URL, strings.NewReader("[]")) + require.NoError(t, err) + _, err = newDorisStreamLoadHTTPClient().Do(req) + require.Error(t, err) + assert.ErrorIs(t, err, secutils.ErrSSRFRedirectBlocked) +} + +func TestDorisStreamLoadHTTPClient_ForwardsAuthorizationToTrustedRedirect(t *testing.T) { + secutils.SetSSRFWhitelistFromRaw("127.0.0.1") + t.Cleanup(secutils.ResetSSRFWhitelistForTest) + + var gotAuthorization string + be := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuthorization = r.Header.Get(headerAuthorization) + w.WriteHeader(http.StatusOK) + })) + defer be.Close() + + fe := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, be.URL, http.StatusTemporaryRedirect) + })) + defer fe.Close() + + req, err := http.NewRequest(http.MethodPut, fe.URL, strings.NewReader("[]")) + require.NoError(t, err) + req.Header.Set(headerAuthorization, "Basic trusted-doris-credential") + + resp, err := newDorisStreamLoadHTTPClient().Do(req) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, "Basic trusted-doris-credential", gotAuthorization) +} + // --------------------------------------------------------------------------- // repository.go:SQL 形态 // --------------------------------------------------------------------------- diff --git a/internal/application/repository/retriever/doris/streamload.go b/internal/application/repository/retriever/doris/streamload.go index a1ca4176..708f6b97 100644 --- a/internal/application/repository/retriever/doris/streamload.go +++ b/internal/application/repository/retriever/doris/streamload.go @@ -8,9 +8,11 @@ import ( "fmt" "io" "net/http" + "net/url" "strings" "github.com/Tencent/WeKnora/internal/logger" + secutils "github.com/Tencent/WeKnora/internal/utils" ) // Stream Load 相关常量。 @@ -52,7 +54,47 @@ type streamLoadResponse struct { // streamLoadURL 拼装某张表的 Stream Load HTTP 端点。 func (r *dorisRepository) streamLoadURL(table string) string { return fmt.Sprintf("%s/api/%s/%s/_stream_load", - r.feHTTPBase, r.database, table) + r.feHTTPBase, url.PathEscape(r.database), url.PathEscape(table)) +} + +// newDorisStreamLoadHTTPClient protects both the initial FE request and every +// FE -> BE redirect at connection time. Doris requires Basic auth to survive +// the 307 redirect, so redirect targets on another host must be explicitly +// trusted through the SSRF whitelist before credentials are forwarded. +func newDorisStreamLoadHTTPClient() *http.Client { + cfg := secutils.DefaultSSRFSafeHTTPClientConfig() + // Stream Load calls carry their own context deadline. Preserve the previous + // client behaviour instead of imposing the generic 30-second HTTP timeout. + cfg.Timeout = 0 + + client := secutils.NewSSRFSafeHTTPClient(cfg) + ssrfCheckRedirect := client.CheckRedirect + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + // Keep the shared client's redirect limit, scheme/target validation and + // sensitive-header stripping as the authoritative SSRF behaviour. + if err := ssrfCheckRedirect(req, via); err != nil { + return err + } + if len(via) == 0 { + return nil + } + + sourceHost := via[len(via)-1].URL.Hostname() + targetHost := req.URL.Hostname() + if !strings.EqualFold(sourceHost, targetHost) && !secutils.IsSSRFWhitelisted(targetHost) { + return fmt.Errorf( + "stream load redirect blocked: target host %q is not trusted to receive credentials", + targetHost, + ) + } + + // The shared client deliberately strips Authorization on cross-host + // redirects. Doris is the exceptional case where an explicitly trusted + // BE needs the same Basic credentials after the FE's 307 response. + req.Header.Set(headerAuthorization, via[0].Header.Get(headerAuthorization)) + return nil + } + return client } // partialUpdateRows 把若干行通过 Stream Load 的 partial update 模式写回目标表。 @@ -96,6 +138,12 @@ func (r *dorisRepository) streamLoadOnce(ctx context.Context, } url := r.streamLoadURL(table) + // Validate at the final outbound boundary as well as when user-supplied + // vector-store configuration is created. This covers stored/env configs and + // keeps the tainted value from reaching http.Client.Do unchecked. + if err := secutils.ValidateURLForSSRF(url); err != nil { + return fmt.Errorf("stream load URL blocked by SSRF validation: %w", err) + } req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(body)) if err != nil { return fmt.Errorf("build stream load request: %w", err)