fix(doris): harden Stream Load HTTP with SSRF-safe client
Doris Stream Load previously used a plain http.Client, leaving FE→BE redirects and stored FE URLs without outbound SSRF checks. Use the shared SSRFSafeHTTPClient, validate URLs at send time, and only forward Basic credentials to whitelisted BE hosts after FE 307 redirects.
This commit is contained in:
+5
-2
@@ -685,10 +685,13 @@ LANGFUSE_HOST=http://langfuse-web:3000
|
||||
#
|
||||
# ===== 向量库地址的 SSRF 校验 =====
|
||||
# 通过 API 创建/测试向量库连接时,地址会经过 SSRF 校验。docker-compose 内置服务
|
||||
# (qdrant, milvus, weaviate, doris-fe, searxng)已通过 SSRF_WHITELIST_EXTRA 默认放行。
|
||||
# (qdrant, milvus, weaviate, doris-fe, doris-be, searxng)已通过 SSRF_WHITELIST_EXTRA 默认放行。
|
||||
# 如使用外部私网地址或被拦截端口(5432, 9200 等),请将主机加入 SSRF_WHITELIST_EXTRA
|
||||
# (注意:自定义该变量会覆盖 compose 默认值,需重新包含内置服务名),例如:
|
||||
# SSRF_WHITELIST_EXTRA=searxng,qdrant,milvus,weaviate,doris-fe,my-opensearch.internal
|
||||
# SSRF_WHITELIST_EXTRA=searxng,qdrant,milvus,weaviate,doris-fe,doris-be,my-opensearch.internal
|
||||
#
|
||||
# Doris Stream Load:FE 会将请求 307 重定向到 BE;docker compose 默认放行
|
||||
# doris-fe 与 doris-be(见 SSRF_WHITELIST_EXTRA)。
|
||||
#
|
||||
# 本地开发(go run + docker-compose.dev.yml --profile opensearch):后端跑在宿主机,
|
||||
# OpenSearch 经 published 端口 http://localhost:9200 访问;通过 API 创建该向量库前,
|
||||
|
||||
+3
-1
@@ -291,7 +291,9 @@ services:
|
||||
# overrides don't clobber it. (postgres is intentionally NOT listed: it
|
||||
# is not a user-registerable vector store, and whitelisting the app's own
|
||||
# DB host would skip the 5432 port block.)
|
||||
- SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng,qdrant,milvus,weaviate,doris-fe}
|
||||
# doris-be: Stream Load follows FE→BE redirects that must receive Basic
|
||||
# credentials on whitelisted BE hosts.
|
||||
- SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng,qdrant,milvus,weaviate,doris-fe,doris-be}
|
||||
- CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5}
|
||||
- JWT_SECRET=${JWT_SECRET:-}
|
||||
# File size limit (in MB)
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
@@ -42,15 +41,15 @@ func NewDorisRetrieveEngineRepository(
|
||||
}
|
||||
|
||||
repo := &dorisRepository{
|
||||
db: db,
|
||||
httpClient: &http.Client{},
|
||||
feHTTPBase: strings.TrimRight(feHTTPBase, "/"),
|
||||
username: username,
|
||||
password: password,
|
||||
database: database,
|
||||
tableBaseName: tableBaseName,
|
||||
bucketsNum: indexCfg.GetBucketsNum(0),
|
||||
replicationNum: indexCfg.GetReplicationNum(0),
|
||||
db: db,
|
||||
httpClient: newDorisStreamLoadHTTPClient(),
|
||||
feHTTPBase: strings.TrimRight(feHTTPBase, "/"),
|
||||
username: username,
|
||||
password: password,
|
||||
database: database,
|
||||
tableBaseName: tableBaseName,
|
||||
bucketsNum: indexCfg.GetBucketsNum(0),
|
||||
replicationNum: indexCfg.GetReplicationNum(0),
|
||||
compatModeRequested: compatMode,
|
||||
}
|
||||
log.Infof("[Doris] Repository initialized: db=%s, base=%s, fe_http=%s, compat_mode=%s",
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -25,6 +26,12 @@ import (
|
||||
// 返回的 cleanup 用 defer 调用即可。
|
||||
func newTestRepo(t *testing.T) (*dorisRepository, sqlmock.Sqlmock, *httptest.Server, func()) {
|
||||
t.Helper()
|
||||
// httptest binds to loopback, which production SSRF policy correctly blocks.
|
||||
// Make that one test host explicit and restore the process-global policy when
|
||||
// the test completes.
|
||||
secutils.SetSSRFWhitelistFromRaw("127.0.0.1")
|
||||
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
|
||||
|
||||
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherRegexp))
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -231,6 +238,63 @@ func TestPartialUpdateRows_FailureSurfaced(t *testing.T) {
|
||||
assert.Contains(t, err.Error(), "stream load failed")
|
||||
}
|
||||
|
||||
func TestStreamLoadOnce_BlocksUnsafeTarget(t *testing.T) {
|
||||
repo, _, _, cleanup := newTestRepo(t)
|
||||
defer cleanup()
|
||||
|
||||
// Override the helper's loopback allowance: link-local metadata endpoints
|
||||
// must be rejected before the HTTP client is called.
|
||||
secutils.SetSSRFWhitelistFromRaw("")
|
||||
repo.feHTTPBase = "http://169.254.169.254"
|
||||
|
||||
err := repo.streamLoadOnce(context.Background(), "t", []string{"id"},
|
||||
[]map[string]any{{"id": "x"}})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "blocked by SSRF validation")
|
||||
}
|
||||
|
||||
func TestDorisStreamLoadHTTPClient_BlocksUnsafeRedirect(t *testing.T) {
|
||||
secutils.SetSSRFWhitelistFromRaw("127.0.0.1")
|
||||
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "http://169.254.169.254/latest/meta-data", http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
req, err := http.NewRequest(http.MethodPut, server.URL, strings.NewReader("[]"))
|
||||
require.NoError(t, err)
|
||||
_, err = newDorisStreamLoadHTTPClient().Do(req)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, secutils.ErrSSRFRedirectBlocked)
|
||||
}
|
||||
|
||||
func TestDorisStreamLoadHTTPClient_ForwardsAuthorizationToTrustedRedirect(t *testing.T) {
|
||||
secutils.SetSSRFWhitelistFromRaw("127.0.0.1")
|
||||
t.Cleanup(secutils.ResetSSRFWhitelistForTest)
|
||||
|
||||
var gotAuthorization string
|
||||
be := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuthorization = r.Header.Get(headerAuthorization)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer be.Close()
|
||||
|
||||
fe := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, be.URL, http.StatusTemporaryRedirect)
|
||||
}))
|
||||
defer fe.Close()
|
||||
|
||||
req, err := http.NewRequest(http.MethodPut, fe.URL, strings.NewReader("[]"))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set(headerAuthorization, "Basic trusted-doris-credential")
|
||||
|
||||
resp, err := newDorisStreamLoadHTTPClient().Do(req)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, "Basic trusted-doris-credential", gotAuthorization)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// repository.go:SQL 形态
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -8,9 +8,11 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// Stream Load 相关常量。
|
||||
@@ -52,7 +54,47 @@ type streamLoadResponse struct {
|
||||
// streamLoadURL 拼装某张表的 Stream Load HTTP 端点。
|
||||
func (r *dorisRepository) streamLoadURL(table string) string {
|
||||
return fmt.Sprintf("%s/api/%s/%s/_stream_load",
|
||||
r.feHTTPBase, r.database, table)
|
||||
r.feHTTPBase, url.PathEscape(r.database), url.PathEscape(table))
|
||||
}
|
||||
|
||||
// newDorisStreamLoadHTTPClient protects both the initial FE request and every
|
||||
// FE -> BE redirect at connection time. Doris requires Basic auth to survive
|
||||
// the 307 redirect, so redirect targets on another host must be explicitly
|
||||
// trusted through the SSRF whitelist before credentials are forwarded.
|
||||
func newDorisStreamLoadHTTPClient() *http.Client {
|
||||
cfg := secutils.DefaultSSRFSafeHTTPClientConfig()
|
||||
// Stream Load calls carry their own context deadline. Preserve the previous
|
||||
// client behaviour instead of imposing the generic 30-second HTTP timeout.
|
||||
cfg.Timeout = 0
|
||||
|
||||
client := secutils.NewSSRFSafeHTTPClient(cfg)
|
||||
ssrfCheckRedirect := client.CheckRedirect
|
||||
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
||||
// Keep the shared client's redirect limit, scheme/target validation and
|
||||
// sensitive-header stripping as the authoritative SSRF behaviour.
|
||||
if err := ssrfCheckRedirect(req, via); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(via) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
sourceHost := via[len(via)-1].URL.Hostname()
|
||||
targetHost := req.URL.Hostname()
|
||||
if !strings.EqualFold(sourceHost, targetHost) && !secutils.IsSSRFWhitelisted(targetHost) {
|
||||
return fmt.Errorf(
|
||||
"stream load redirect blocked: target host %q is not trusted to receive credentials",
|
||||
targetHost,
|
||||
)
|
||||
}
|
||||
|
||||
// The shared client deliberately strips Authorization on cross-host
|
||||
// redirects. Doris is the exceptional case where an explicitly trusted
|
||||
// BE needs the same Basic credentials after the FE's 307 response.
|
||||
req.Header.Set(headerAuthorization, via[0].Header.Get(headerAuthorization))
|
||||
return nil
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
// partialUpdateRows 把若干行通过 Stream Load 的 partial update 模式写回目标表。
|
||||
@@ -96,6 +138,12 @@ func (r *dorisRepository) streamLoadOnce(ctx context.Context,
|
||||
}
|
||||
|
||||
url := r.streamLoadURL(table)
|
||||
// Validate at the final outbound boundary as well as when user-supplied
|
||||
// vector-store configuration is created. This covers stored/env configs and
|
||||
// keeps the tainted value from reaching http.Client.Do unchecked.
|
||||
if err := secutils.ValidateURLForSSRF(url); err != nil {
|
||||
return fmt.Errorf("stream load URL blocked by SSRF validation: %w", err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return fmt.Errorf("build stream load request: %w", err)
|
||||
|
||||
Reference in New Issue
Block a user