diff --git a/.env.example b/.env.example index 0886b704..a1cd105b 100644 --- a/.env.example +++ b/.env.example @@ -236,6 +236,18 @@ SYSTEM_AES_KEY=weknora-system-aes-key-32bytes!! # IPv4（203.0.113.5）、IPv6（2001:db8::1，不带方括号）或 CIDR（10.0.0.0/8, 2001:db8::/32）。 # 列入者会在 URL 校验等地方绕过常规 SSRF 规则，生产环境请谨慎配置。 # SSRF_WHITELIST=internal.service,*.corp.example,172.16.0.0/12,2001:db8::1,fd00::/8 +# +# ===== 向量库地址的 SSRF 校验 ===== +# 通过 API 创建/测试向量库连接时，地址会经过 SSRF 校验。docker-compose 内置服务 +# （qdrant, milvus, weaviate, doris-fe）已通过 SSRF_WHITELIST_EXTRA 默认放行。 +# 如使用外部私网地址或被拦截端口（5432, 9200 等），请将主机加入 SSRF_WHITELIST_EXTRA +# （注意：自定义该变量会覆盖 compose 默认值，需重新包含内置服务名），例如： +# SSRF_WHITELIST_EXTRA=searxng,qdrant,milvus,weaviate,doris-fe,my-opensearch.internal +# +# 本地开发（go run + docker-compose.dev.yml --profile opensearch）：后端跑在宿主机， +# OpenSearch 经 published 端口 http://localhost:9200 访问；通过 API 创建该向量库前， +# 需把 localhost 加入 SSRF_WHITELIST（9200 是被拦截端口，localhost 是受限主机名）： +# SSRF_WHITELIST=localhost # ===== SearXNG（可选自建网络搜索）===== # 启用：`docker compose --profile searxng up -d`。 diff --git a/docker-compose.yml b/docker-compose.yml index 3fbe4f63..7b3c0e15 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -168,9 +168,15 @@ services: - SSRF_WHITELIST=${SSRF_WHITELIST:-} # 保留原始 URL 的图片域名白名单（逗号分隔，不替换为 provider://） - IMAGE_HOST_KEEP_URL=${IMAGE_HOST_KEEP_URL:-} - # Always allow the optional searxng sidecar (compose service hostname); - # merged on top of SSRF_WHITELIST so user overrides don't clobber it. - - SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng} + # Always allow the optional searxng sidecar and the bundled vector-store + # service hostnames (compose service names). Vector-store create/test now + # runs SSRF validation on user-supplied addresses, which would otherwise + # reject these private docker-network hosts; whitelisting them keeps the + # out-of-box experience. Merged on top of SSRF_WHITELIST so user + # overrides don't clobber it. (postgres is intentionally NOT listed: it + # is not a user-registerable vector store, and whitelisting the app's own + # DB host would skip the 5432 port block.) + - SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng,qdrant,milvus,weaviate,doris-fe} - CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5} - JWT_SECRET=${JWT_SECRET:-} # Crypto: 主密钥和盐值，用于 AppSecret 等敏感字段的 AES-256 加密 diff --git a/docs/dev/opensearch-integration-test.md b/docs/dev/opensearch-integration-test.md index 0edd5f1c..8b31ac81 100644 --- a/docs/dev/opensearch-integration-test.md +++ b/docs/dev/opensearch-integration-test.md @@ -44,6 +44,20 @@ curl -s 'localhost:9200/_cat/plugins?format=json' | jq -r '.[].component' | grep ### Option A — DB store (UI / API) +> **SSRF whitelist (dev).** `CreateStore` and the raw connection test validate +> the user-supplied `addr` against the SSRF policy. `http://localhost:9200` +> is rejected by default — `localhost` is a restricted hostname and `9200` is +> a blocked port. When the backend runs on the host (`go run`), add `localhost` +> to the whitelist in your `.env` before registering: +> +> ```bash +> SSRF_WHITELIST=localhost +> ``` +> +> The containerised compose deployment whitelists the bundled vector-store +> service names automatically (`SSRF_WHITELIST_EXTRA`), so this step is +> dev-only. The env-store path (Option B) is not affected. + `POST /api/v1/vector-stores`: ```json diff --git a/internal/application/service/vectorstore.go b/internal/application/service/vectorstore.go index a15c18b3..6ec25587 100644 --- a/internal/application/service/vectorstore.go +++ b/internal/application/service/vectorstore.go @@ -62,6 +62,13 @@ func (s *vectorStoreService) CreateStore(ctx context.Context, store *types.Vecto return err } + // 2.1. SSRF validation on user-supplied addresses (whitelist-first). + // Placed before any network I/O (step 5 TestConnection, step 7 registry + // factory) so a blocked address never triggers an outbound connection. + if err := validateConnectionAddrSSRF(store.EngineType, store.ConnectionConfig); err != nil { + return err + } + // 2.5. Index config validation (bounds, name characters) if err := types.ValidateIndexConfig(store.IndexConfig); err != nil { return err @@ -470,6 +477,102 @@ func validateConnectionConfig(engineType types.RetrieverEngineType, config types return nil } +// validateConnectionAddrSSRF validates every user-supplied address field of a +// connection config against the SSRF policy (whitelist first, then the strict +// IP / port / DNS checks inside secutils.ValidateURLForSSRF). It is applied +// ONLY at user-input boundaries — CreateStore and TestRawConnection. Env +// stores and already-stored configs are trusted and intentionally skip it. +// +// Unknown engine types are REJECTED (fail-closed): a newly added engine must +// not be able to reach a dial path without an explicit address mapping here. +// Empty fields are skipped — required-field presence is the responsibility of +// validateConnectionConfig, which runs first on every guarded path. +func validateConnectionAddrSSRF(engineType types.RetrieverEngineType, config types.ConnectionConfig) error { + // check validates a single address field. Empty fields are no-ops so this + // helper is independent of required-field enforcement. + check := func(addr string) error { + if addr == "" { + return nil + } + if err := secutils.ValidateURLForSSRF(addr); err != nil { + return errors.NewValidationError( + secutils.FormatSSRFError("vector store address", addr, err)) + } + return nil + } + + switch engineType { + case types.ElasticsearchRetrieverEngineType, + types.OpenSearchRetrieverEngineType, + types.MilvusRetrieverEngineType, + types.TencentVectorDBRetrieverEngineType, + types.DorisRetrieverEngineType: + // Single address field: a URL (es/opensearch) or bare host:port + // (milvus/tencent/doris). ValidateURLForSSRF normalises both. + return check(config.Addr) + case types.QdrantRetrieverEngineType: + // Host (+ optional Port) — combine so the port blocklist applies to + // the actual dial target rather than just the bare host. + addr := config.Host + if addr != "" && config.Port != 0 { + addr = fmt.Sprintf("%s:%d", config.Host, config.Port) + } + return check(addr) + case types.WeaviateRetrieverEngineType: + // Both the HTTP host and the gRPC address are dialed by the driver, + // so both must be validated (validating Host alone leaves GrpcAddress + // as an open SSRF vector). + if err := check(config.Host); err != nil { + return err + } + return check(config.GrpcAddress) + case types.SQLiteRetrieverEngineType: + // File-based engine; no remote address to validate. + return nil + default: + // Fail closed. Engines without a DB-store address mapping (postgres, + // infinity, elasticfaiss, and any future engine) must not silently + // bypass SSRF validation. The guarded callers (CreateStore, + // TestRawConnection) already restrict to validEngineTypes, so this is + // defence-in-depth rather than a user-facing path. + return errors.NewValidationError( + fmt.Sprintf("SSRF validation is not configured for engine type: %s", engineType)) + } +} + +// TestRawConnection validates raw (unpersisted) user-supplied connection config +// — engine-type allowlist, required fields, then the SSRF policy — before +// delegating to TestConnection. Handlers MUST use this for raw user input +// (e.g. POST /vector-stores/test). +// +// TestConnection itself stays validation-free for trusted callers (env stores +// and stored configs already validated at create time, which legitimately use +// internal hosts such as localhost). Do NOT consolidate the two methods. +func (s *vectorStoreService) TestRawConnection( + ctx context.Context, + engineType types.RetrieverEngineType, + config types.ConnectionConfig, +) (string, error) { + // 1. Engine-type allowlist. Only DB-registerable engines may be probed + // with raw credentials; this blocks e.g. a raw postgres probe against + // the application's own database host (a credential oracle). + if !types.IsValidEngineType(engineType) { + return "", errors.NewValidationError( + fmt.Sprintf("connection test is not supported for engine type: %s", engineType)) + } + // 2. Required fields. Prevents an empty field from falling through to a + // driver's internal default (e.g. milvus empty addr -> localhost:19530), + // which would otherwise dial an internal host unchecked. + if err := validateConnectionConfig(engineType, config); err != nil { + return "", err + } + // 3. SSRF policy on every user-supplied address field. + if err := validateConnectionAddrSSRF(engineType, config); err != nil { + return "", err + } + return s.TestConnection(ctx, engineType, config) +} + // openSearch HNSW bound constants. Shards / replicas are NOT validated here — // the flat types.ValidateIndexConfig already enforces those caps for every // engine. These caps mirror the GetVectorStoreTypes Min/Max so the UI and diff --git a/internal/application/service/vectorstore_healthcheck.go b/internal/application/service/vectorstore_healthcheck.go index fca65e6b..36eadee9 100644 --- a/internal/application/service/vectorstore_healthcheck.go +++ b/internal/application/service/vectorstore_healthcheck.go @@ -71,7 +71,15 @@ func testElasticsearchConnection(ctx context.Context, config types.ConnectionCon req.SetBasicAuth(config.Username, config.Password) } - client := &http.Client{Timeout: connectionTestTimeout} + client := &http.Client{ + Timeout: connectionTestTimeout, + // A health probe must not follow redirects: a malicious or compromised + // endpoint could return a 302 to an internal address, defeating the + // SSRF check that only validated the original (user-supplied) Addr. + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } resp, err := client.Do(req) if err != nil { logger.Warnf(ctx, "Elasticsearch connection test failed: %v", err) diff --git a/internal/application/service/vectorstore_ssrf_test.go b/internal/application/service/vectorstore_ssrf_test.go new file mode 100644 index 00000000..2bb6df62 --- /dev/null +++ b/internal/application/service/vectorstore_ssrf_test.go @@ -0,0 +1,232 @@ +package service + +import ( + "context" + "os" + "testing" + + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// withSSRFWhitelist sets SSRF_WHITELIST for the duration of a test, resetting +// the cached singleton both before and after so neither this test nor a +// neighbour sees a stale whitelist (see ResetSSRFWhitelistForTest docs). +func withSSRFWhitelist(t *testing.T, whitelist string) { + t.Helper() + utils.ResetSSRFWhitelistForTest() + require.NoError(t, os.Setenv("SSRF_WHITELIST", whitelist)) + t.Cleanup(func() { + _ = os.Unsetenv("SSRF_WHITELIST") + utils.ResetSSRFWhitelistForTest() + }) +} + +func TestValidateConnectionAddrSSRF(t *testing.T) { + // Whitelist a benign host so "pass" cases have a deterministic, DNS-free + // way through. Reject cases use direct private IPs, which are blocked + // before any DNS lookup, keeping the test non-flaky. + withSSRFWhitelist(t, "vector.allowed.test") + + tests := []struct { + name string + engineType types.RetrieverEngineType + config types.ConnectionConfig + wantError bool + }{ + { + name: "elasticsearch private IP blocked", + engineType: types.ElasticsearchRetrieverEngineType, + config: types.ConnectionConfig{Addr: "http://10.0.0.5:9200"}, + wantError: true, + }, + { + name: "elasticsearch whitelisted host allowed", + engineType: types.ElasticsearchRetrieverEngineType, + config: types.ConnectionConfig{Addr: "http://vector.allowed.test:9200"}, + wantError: false, + }, + { + name: "opensearch loopback blocked", + engineType: types.OpenSearchRetrieverEngineType, + config: types.ConnectionConfig{Addr: "http://127.0.0.1:9200"}, + wantError: true, + }, + { + name: "milvus empty addr skipped (presence is validateConnectionConfig's job)", + engineType: types.MilvusRetrieverEngineType, + config: types.ConnectionConfig{}, + wantError: false, + }, + { + name: "doris private IP blocked", + engineType: types.DorisRetrieverEngineType, + config: types.ConnectionConfig{Addr: "192.168.1.10:9030"}, + wantError: true, + }, + { + name: "qdrant host+port private IP blocked", + engineType: types.QdrantRetrieverEngineType, + config: types.ConnectionConfig{Host: "10.1.2.3", Port: 6334}, + wantError: true, + }, + { + name: "qdrant whitelisted host allowed", + engineType: types.QdrantRetrieverEngineType, + config: types.ConnectionConfig{Host: "vector.allowed.test", Port: 6334}, + wantError: false, + }, + { + name: "weaviate host ok but grpc_address private IP blocked", + engineType: types.WeaviateRetrieverEngineType, + config: types.ConnectionConfig{ + Host: "vector.allowed.test", + GrpcAddress: "10.0.0.9:50051", + }, + wantError: true, + }, + { + name: "weaviate both fields whitelisted allowed", + engineType: types.WeaviateRetrieverEngineType, + config: types.ConnectionConfig{ + Host: "vector.allowed.test", + GrpcAddress: "vector.allowed.test:50051", + }, + wantError: false, + }, + { + name: "sqlite skipped (no remote address)", + engineType: types.SQLiteRetrieverEngineType, + config: types.ConnectionConfig{}, + wantError: false, + }, + { + name: "unknown engine fails closed", + engineType: types.RetrieverEngineType("some-future-engine"), + config: types.ConnectionConfig{Addr: "http://vector.allowed.test"}, + wantError: true, + }, + { + name: "infinity (legacy, unmapped) fails closed", + engineType: types.InfinityRetrieverEngineType, + config: types.ConnectionConfig{Addr: "http://vector.allowed.test"}, + wantError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateConnectionAddrSSRF(tt.engineType, tt.config) + if tt.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +// TestValidateConnectionAddrSSRF_WhitelistSkipsPortBlock pins the intentional +// behaviour that a whitelisted host bypasses the port blocklist (whitelist +// trust is host-granular). If this ever changes, the bundled-service defaults +// in docker-compose would silently break, so the decision is asserted here. +func TestValidateConnectionAddrSSRF_WhitelistSkipsPortBlock(t *testing.T) { + withSSRFWhitelist(t, "qdrant") + // 6379 (redis) is on the blocklist, but a whitelisted host skips all + // checks including the port block. + err := validateConnectionAddrSSRF(types.QdrantRetrieverEngineType, + types.ConnectionConfig{Host: "qdrant", Port: 6379}) + require.NoError(t, err) +} + +// TestValidateConnectionAddrSSRF_Completeness guards against a future engine +// being added to validEngineTypes without an SSRF address mapping. Such an +// engine would fall into the fail-closed default branch and error even with a +// whitelisted address — this test catches that at build/test time. +func TestValidateConnectionAddrSSRF_Completeness(t *testing.T) { + withSSRFWhitelist(t, "any.allowed.test") + + // Derive the engine list from the live registry (GetVectorStoreTypes is + // backed by validEngineTypes) rather than a hard-coded slice, so a newly + // added registerable engine that lacks an SSRF address mapping fails this + // test instead of passing vacuously. + storeTypes := types.GetVectorStoreTypes() + require.NotEmpty(t, storeTypes) + + for _, st := range storeTypes { + et := types.RetrieverEngineType(st.Type) + if !types.IsValidEngineType(et) { + continue // env-only / legacy engines are not user-registerable + } + // All address fields point at a whitelisted host, so the only way to + // get an error is the fail-closed default branch (= missing mapping). + err := validateConnectionAddrSSRF(et, types.ConnectionConfig{ + Addr: "any.allowed.test", + Host: "any.allowed.test", + GrpcAddress: "any.allowed.test", + }) + require.NoErrorf(t, err, + "engine %q is registerable but has no SSRF address mapping (fell into fail-closed default)", et) + } +} + +func TestTestRawConnection_Rejections(t *testing.T) { + withSSRFWhitelist(t, "vector.allowed.test") + repo := &mockVectorStoreRepo{} + svc := NewVectorStoreService(repo, nil, nil, nil, nil) + + tests := []struct { + name string + engineType types.RetrieverEngineType + config types.ConnectionConfig + }{ + { + // postgres is not a user-registerable vector store; raw-testing it + // would otherwise dial the app's own DB host (credential oracle). + name: "postgres rejected by engine allowlist", + engineType: types.PostgresRetrieverEngineType, + config: types.ConnectionConfig{Addr: "postgres://u:p@vector.allowed.test:5432/db"}, + }, + { + // empty addr must be rejected by required-field validation before + // the driver falls back to its localhost:19530 default. + name: "milvus empty addr rejected (no localhost fallback)", + engineType: types.MilvusRetrieverEngineType, + config: types.ConnectionConfig{}, + }, + { + name: "elasticsearch private IP rejected by SSRF", + engineType: types.ElasticsearchRetrieverEngineType, + config: types.ConnectionConfig{Addr: "http://10.0.0.5:9200"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := svc.TestRawConnection(context.Background(), tt.engineType, tt.config) + require.Error(t, err) + }) + } +} + +func TestCreateStore_SSRFRejected(t *testing.T) { + withSSRFWhitelist(t, "vector.allowed.test") + repo := &mockVectorStoreRepo{} + svc := NewVectorStoreService(repo, nil, nil, nil, nil) + + store := &types.VectorStore{ + TenantID: 1, + Name: "es-internal", + EngineType: types.ElasticsearchRetrieverEngineType, + ConnectionConfig: types.ConnectionConfig{ + Addr: "http://169.254.169.254:9200", // cloud metadata endpoint + }, + } + + err := svc.CreateStore(context.Background(), store) + require.Error(t, err) + // Rejected before persistence and before any connection probe. + assert.Empty(t, repo.stores) +} diff --git a/internal/application/service/vectorstore_test.go b/internal/application/service/vectorstore_test.go index 1ff5e45b..40ad1833 100644 --- a/internal/application/service/vectorstore_test.go +++ b/internal/application/service/vectorstore_test.go @@ -27,6 +27,10 @@ func newFakeESServer(t *testing.T) *httptest.Server { _, _ = w.Write([]byte(`{"version":{"number":"7.10.1"}}`)) })) t.Cleanup(srv.Close) + // CreateStore now SSRF-validates the connection addr before the probe. + // The httptest server listens on 127.0.0.1, which the SSRF policy blocks + // by default, so whitelist it for the duration of the test. + withSSRFWhitelist(t, "127.0.0.1") return srv } @@ -308,6 +312,10 @@ func TestCreateStore_ConnectionConfigValidation(t *testing.T) { } func TestCreateStore_DuplicateCheck_DBStore(t *testing.T) { + // "es" is an unresolvable host on a blocked port (9200); whitelist it so + // the test reaches the duplicate check (step 3) rather than failing the + // SSRF guard (step 2.1). + withSSRFWhitelist(t, "es") repo := &mockVectorStoreRepo{existsByEndpoint: true} svc := NewVectorStoreService(repo, nil, nil, nil, nil) @@ -329,6 +337,9 @@ func TestCreateStore_DuplicateCheck_DBStore(t *testing.T) { } func TestCreateStore_DuplicateCheck_DBError(t *testing.T) { + // Whitelist "es" so the flow reaches the duplicate-check DB error (step 3) + // instead of stopping at the SSRF guard (step 2.1). + withSSRFWhitelist(t, "es") repo := &mockVectorStoreRepo{ existsByEndpointErr: assert.AnError, } @@ -348,6 +359,9 @@ func TestCreateStore_DuplicateCheck_DBError(t *testing.T) { } func TestCreateStore_DuplicateCheck_EnvStore(t *testing.T) { + // Whitelist "es" so the flow reaches the env-store duplicate check rather + // than failing the SSRF guard (step 2.1) on the unresolvable es:9200 host. + withSSRFWhitelist(t, "es") // Set up env to simulate an existing elasticsearch env store t.Setenv("RETRIEVE_DRIVER", "elasticsearch_v8") t.Setenv("ELASTICSEARCH_ADDR", "http://es:9200") diff --git a/internal/handler/vectorstore.go b/internal/handler/vectorstore.go index 46421b3c..5b0156d1 100644 --- a/internal/handler/vectorstore.go +++ b/internal/handler/vectorstore.go @@ -444,7 +444,10 @@ func (h *VectorStoreHandler) TestStoreRaw(c *gin.Context) { return } - version, err := h.service.TestConnection(ctx, req.EngineType, req.ConnectionConfig) + // Raw user input: TestRawConnection applies the engine-type allowlist, + // required-field, and SSRF checks before any dial (unlike TestStoreByID, + // which probes trusted stored/env configs). + version, err := h.service.TestRawConnection(ctx, req.EngineType, req.ConnectionConfig) if err != nil { logger.Warnf(ctx, "Vector store connection test failed: %v", err) c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()}) diff --git a/internal/types/interfaces/vectorstore.go b/internal/types/interfaces/vectorstore.go index 9ea0cef0..c37a2afd 100644 --- a/internal/types/interfaces/vectorstore.go +++ b/internal/types/interfaces/vectorstore.go @@ -36,7 +36,15 @@ type VectorStoreService interface { DeleteStore(ctx context.Context, tenantID uint64, id string) error // TestConnection tests connectivity to a vector database. // Returns the detected server version on success (e.g., "7.10.1"), empty string if unknown. + // + // Validation-free: intended for trusted configs (env stores, stored + // configs already validated at create time). Handlers receiving raw + // user input MUST use TestRawConnection instead. TestConnection(ctx context.Context, engineType types.RetrieverEngineType, config types.ConnectionConfig) (string, error) + // TestRawConnection validates raw user-supplied connection config + // (engine-type allowlist, required fields, SSRF policy) and then delegates + // to TestConnection. This is the entry point for unpersisted user input. + TestRawConnection(ctx context.Context, engineType types.RetrieverEngineType, config types.ConnectionConfig) (string, error) // SaveDetectedVersion updates the connection_config.version for a stored vector store. SaveDetectedVersion(ctx context.Context, store *types.VectorStore, version string) error