feat: search engine adds proxy addresses and adds SSRF verification.

This commit is contained in:
hjz
2026-04-15 08:53:09 +08:00
committed by lyingbug
parent 092044b68f
commit 7bd36bf954
16 changed files with 228 additions and 14 deletions
+2
View File
@@ -10,6 +10,7 @@ export interface WebSearchProviderEntity {
parameters: {
api_key?: string
engine_id?: string
proxy_url?: string
extra_config?: Record<string, string>
}
is_default?: boolean
@@ -23,6 +24,7 @@ export interface WebSearchProviderTypeInfo {
name: string
requires_api_key: boolean
requires_engine_id?: boolean
supports_proxy?: boolean
description?: string
docs_url?: string
}
+3
View File
@@ -776,6 +776,9 @@ export default {
providerLabel: 'Search Provider',
providerDescription: 'Choose the search engine service used for web search',
providerPlaceholder: 'Select a search engine...',
proxyUrlLabel: 'HTTP proxy',
proxyUrlPlaceholder: 'e.g. http://proxy.example.com:3128 (optional; http/https only)',
proxyUrlHelp: 'Use when outbound access to the search API requires a proxy; leave empty to rely on HTTP_PROXY/HTTPS_PROXY environment variables.',
apiKeyLabel: 'API Key',
apiKeyDescription: 'Enter the API key for the selected search provider',
apiKeyPlaceholder: 'Enter API key',
+3
View File
@@ -636,6 +636,9 @@ export default {
providerLabel: "검색 엔진 프로바이더",
providerDescription: "웹 검색에 사용할 검색 엔진 서비스 선택",
providerPlaceholder: "검색 엔진 선택...",
proxyUrlLabel: "HTTP 프록시",
proxyUrlPlaceholder: "예: http://127.0.0.1:7890 (선택 사항)",
proxyUrlHelp: "검색 API 접근에 프록시가 필요한 경우 입력합니다. 비우면 HTTP_PROXY/HTTPS_PROXY를 사용합니다.",
apiKeyLabel: "API 키",
apiKeyDescription: "선택한 검색 엔진의 API 키 입력",
apiKeyPlaceholder: "API 키를 입력하세요",
+3
View File
@@ -754,6 +754,9 @@ export default {
providerLabel: 'Провайдер поиска',
providerDescription: 'Выберите поисковый сервис, используемый для веб-поиска',
providerPlaceholder: 'Выберите поисковую систему...',
proxyUrlLabel: 'HTTP-прокси',
proxyUrlPlaceholder: 'Напр. http://127.0.0.1:7890 (необязательно)',
proxyUrlHelp: 'Укажите, если доступ к API поиска нужен через прокси; иначе используются переменные HTTP_PROXY/HTTPS_PROXY.',
apiKeyLabel: 'API-ключ',
apiKeyDescription: 'Введите API-ключ выбранного провайдера поиска',
apiKeyPlaceholder: 'Введите API-ключ',
+3
View File
@@ -636,6 +636,9 @@ export default {
providerLabel: "搜索引擎提供商",
providerDescription: "选择用于网络搜索的搜索引擎服务",
providerPlaceholder: "选择搜索引擎...",
proxyUrlLabel: "HTTP 代理",
proxyUrlPlaceholder: "例如 http://proxy.example.com:3128(可选,仅支持 http/https)",
proxyUrlHelp: "若环境访问搜索 API 需代理,在此填写;留空则使用系统环境变量 HTTP(S)_PROXY。",
apiKeyLabel: "API 密钥",
apiKeyDescription: "输入所选搜索引擎的 API 密钥",
apiKeyPlaceholder: "请输入 API 密钥",
@@ -99,6 +99,16 @@
</t-form-item>
</template>
<t-form-item v-if="selectedProviderType?.supports_proxy" :label="t('webSearchSettings.proxyUrlLabel')" name="parameters.proxy_url">
<t-input
v-model="providerForm.parameters.proxy_url"
:placeholder="t('webSearchSettings.proxyUrlPlaceholder')"
/>
<template #help>
<span class="switch-help">{{ t('webSearchSettings.proxyUrlHelp') }}</span>
</template>
</t-form-item>
<div class="form-divider"></div>
<t-form-item :label="t('webSearchSettings.setAsDefault')" name="is_default">
@@ -164,7 +174,7 @@ const providerForm = ref<{
name: string
provider: string
description: string
parameters: { api_key?: string; engine_id?: string }
parameters: { api_key?: string; engine_id?: string; proxy_url?: string }
is_default: boolean
}>({
name: '',
@@ -228,6 +238,7 @@ const editProvider = (entity: WebSearchProviderEntity) => {
parameters: {
api_key: '',
engine_id: entity.parameters?.engine_id || '',
proxy_url: entity.parameters?.proxy_url || '',
},
is_default: entity.is_default || false,
}
+14 -3
View File
@@ -105,7 +105,8 @@ func (s *WebSearchService) resolveProvider(
return nil, fmt.Errorf("web search provider not found: %s", providerID)
}
provider, err := s.registry.CreateProvider(string(entity.Provider), entity.Parameters)
params := mergeProxyFromWebSearchConfig(entity.Parameters, cfg)
provider, err := s.registry.CreateProvider(string(entity.Provider), params)
if err != nil {
return nil, fmt.Errorf("failed to create provider %s (%s): %w", entity.Name, entity.Provider, err)
}
@@ -115,9 +116,9 @@ func (s *WebSearchService) resolveProvider(
// Backward compatibility: use the deprecated config.Provider field
if cfg.Provider != "" {
logger.Warnf(ctx, "Using deprecated WebSearchConfig.Provider field: %s. Please migrate to WebSearchProviderEntity.", cfg.Provider)
params := types.WebSearchProviderParameters{
params := mergeProxyFromWebSearchConfig(types.WebSearchProviderParameters{
APIKey: cfg.APIKey,
}
}, cfg)
provider, err := s.registry.CreateProvider(cfg.Provider, params)
if err != nil {
return nil, fmt.Errorf("web search provider %s is not available: %w", cfg.Provider, err)
@@ -128,6 +129,16 @@ func (s *WebSearchService) resolveProvider(
return nil, fmt.Errorf("no web search provider configured")
}
// mergeProxyFromWebSearchConfig applies cfg.ProxyURL over stored provider params when non-empty (call-time override).
func mergeProxyFromWebSearchConfig(base types.WebSearchProviderParameters, cfg *types.WebSearchConfig) types.WebSearchProviderParameters {
if cfg != nil {
if pu := strings.TrimSpace(cfg.ProxyURL); pu != "" {
base.ProxyURL = pu
}
}
return base
}
// CompressWithRAG performs RAG-based compression using a temporary, hidden knowledge base.
// The temporary knowledge base is deleted after use. The UI will not list it due to repo filtering.
func (s *WebSearchService) CompressWithRAG(
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -60,6 +61,12 @@ func (s *webSearchProviderService) UpdateProvider(ctx context.Context, provider
}
}
if provider.Provider != "" {
if err := validateProviderParameters(provider.Provider, provider.Parameters); err != nil {
return err
}
}
logger.Infof(ctx, "Updating web search provider: tenant=%d, id=%s", provider.TenantID, provider.ID)
return s.repo.Update(ctx, provider)
}
@@ -114,5 +121,12 @@ func validateProviderParameters(provider types.WebSearchProviderType, params typ
case types.WebSearchProviderTypeDuckDuckGo:
// No API key required
}
if err := validateOptionalProxyURL(params.ProxyURL); err != nil {
return err
}
return nil
}
func validateOptionalProxyURL(proxyURL string) error {
return infra_web_search.ValidateProxyURL(proxyURL)
}
+3 -2
View File
@@ -54,8 +54,9 @@ func NewBingProvider(params types.WebSearchProviderParameters) (interfaces.WebSe
if params.APIKey == "" {
return nil, fmt.Errorf("API key is required for Bing provider")
}
client := &http.Client{
Timeout: defaultBingTimeout,
client, err := NewSearchHTTPClient(defaultBingTimeout, params.ProxyURL)
if err != nil {
return nil, err
}
return &BingProvider{
client: client,
@@ -23,13 +23,13 @@ type DuckDuckGoProvider struct {
}
// NewDuckDuckGoProvider creates a new DuckDuckGo provider.
// DuckDuckGo is free and requires no API key or configuration.
// DuckDuckGo is free and requires no API key; optional ProxyURL tunnels requests via an HTTP/HTTPS proxy.
func NewDuckDuckGoProvider(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) {
return &DuckDuckGoProvider{
client: &http.Client{
Timeout: 30 * time.Second,
},
}, nil
client, err := NewSearchHTTPClient(30*time.Second, params.ProxyURL)
if err != nil {
return nil, err
}
return &DuckDuckGoProvider{client: client}, nil
}
// Name returns the provider name
@@ -3,6 +3,7 @@ package web_search
import (
"context"
"fmt"
"time"
"google.golang.org/api/customsearch/v1"
"google.golang.org/api/option"
@@ -29,8 +30,13 @@ func NewGoogleProvider(params types.WebSearchProviderParameters) (interfaces.Web
return nil, fmt.Errorf("engine ID is required for Google provider")
}
httpClient, err := NewSearchHTTPClient(30*time.Second, params.ProxyURL)
if err != nil {
return nil, err
}
clientOpts := []option.ClientOption{
option.WithAPIKey(params.APIKey),
option.WithHTTPClient(httpClient),
}
srv, err := customsearch.NewService(context.Background(), clientOpts...)
if err != nil {
@@ -0,0 +1,68 @@
package web_search
import (
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/utils"
)
// ValidateProxyURL delegates to utils.ValidateURLForSSRF (only http/https pass that check).
func ValidateProxyURL(proxyURL string) error {
proxyURL = strings.TrimSpace(proxyURL)
if proxyURL == "" {
return nil
}
return utils.ValidateURLForSSRF(proxyURL)
}
func ssrfSafeRedirect(maxRedirects int) func(*http.Request, []*http.Request) error {
return func(req *http.Request, via []*http.Request) error {
if len(via) >= maxRedirects {
return fmt.Errorf("stopped after %d redirects", maxRedirects)
}
if err := utils.ValidateURLForSSRF(req.URL.String()); err != nil {
return fmt.Errorf("%w: %v", utils.ErrSSRFRedirectBlocked, err)
}
return nil
}
}
// NewSearchHTTPClient builds an http.Client for outbound web search requests.
// It uses utils.SSRFSafeDialContext, optional explicit or environment proxy, and
// redirect validation consistent with utils.NewSSRFSafeHTTPClient.
func NewSearchHTTPClient(timeout time.Duration, proxyURL string) (*http.Client, error) {
proxyURL = strings.TrimSpace(proxyURL)
def, ok := http.DefaultTransport.(*http.Transport)
if !ok {
return nil, fmt.Errorf("default HTTP transport is not *http.Transport")
}
t := def.Clone()
t.DialContext = utils.SSRFSafeDialContext
if proxyURL != "" {
if err := ValidateProxyURL(proxyURL); err != nil {
return nil, err
}
u, err := url.Parse(proxyURL)
if err != nil {
return nil, fmt.Errorf("invalid proxy_url: %w", err)
}
if u.Scheme == "" || u.Host == "" {
return nil, fmt.Errorf("invalid proxy_url: scheme and host are required")
}
t.Proxy = http.ProxyURL(u)
} else {
t.Proxy = http.ProxyFromEnvironment
}
cfg := utils.DefaultSSRFSafeHTTPClientConfig()
return &http.Client{
Timeout: timeout,
Transport: t,
CheckRedirect: ssrfSafeRedirect(cfg.MaxRedirects),
}, nil
}
@@ -0,0 +1,78 @@
package web_search
import (
"net/http"
"net/url"
"strings"
"testing"
)
func TestValidateProxyURL(t *testing.T) {
tests := []struct {
name string
raw string
wantErr bool
}{
{"empty", "", false},
{"whitespace only", " ", false},
{"no scheme private IP bare host:port", "192.168.0.1:3128", true},
{"localhost http", "http://localhost:3128", true},
{"direct IPv4 http", "http://192.168.1.1:3128", true},
{"socks5 to IP blocked", "socks5://10.0.0.1:1080", true},
{"bad scheme", "ftp://proxy.example:21", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := ValidateProxyURL(tt.raw)
if tt.wantErr && err == nil {
t.Fatalf("expected error, got nil")
}
if !tt.wantErr && err != nil {
t.Fatalf("unexpected error: %v", err)
}
})
}
}
func TestNewSearchHTTPClient_RejectsUnsafeProxy(t *testing.T) {
_, err := NewSearchHTTPClient(0, "http://127.0.0.1:3128")
if err == nil {
t.Fatal("expected error for disallowed proxy URL (127.0.0.1)")
}
if !strings.Contains(err.Error(), "SSRF") {
t.Fatalf("expected SSRF-related error, got: %v", err)
}
}
func TestNewSearchHTTPClient_AcceptsEmptyProxy(t *testing.T) {
c, err := NewSearchHTTPClient(0, "")
if err != nil {
t.Fatal(err)
}
if c.Transport == nil {
t.Fatal("expected transport")
}
if c.CheckRedirect == nil {
t.Fatal("expected CheckRedirect for SSRF-safe redirects")
}
}
func TestSsrfSafeRedirect_BlocksNonHTTPSScheme(t *testing.T) {
h := ssrfSafeRedirect(10)
req := &http.Request{URL: mustParse(t, "ftp://evil.com/")}
// len(via)==0 so first redirect
err := h(req, nil)
if err == nil {
t.Fatal("expected error for ftp redirect")
}
}
func mustParse(t *testing.T, s string) *url.URL {
t.Helper()
u, err := url.Parse(s)
if err != nil {
t.Fatal(err)
}
return u
}
+3 -2
View File
@@ -36,8 +36,9 @@ func NewTavilyProvider(params types.WebSearchProviderParameters) (interfaces.Web
if params.APIKey == "" {
return nil, fmt.Errorf("API key is required for Tavily provider")
}
client := &http.Client{
Timeout: defaultTavilyTimeout,
client, err := NewSearchHTTPClient(defaultTavilyTimeout, params.ProxyURL)
if err != nil {
return nil, err
}
return &TavilyProvider{
client: client,
+1
View File
@@ -22,6 +22,7 @@ type WebSearchConfig struct {
EmbeddingDimension int `json:"embedding_dimension,omitempty"` // 嵌入维度(用于RAG压缩)
RerankModelID string `json:"rerank_model_id,omitempty"` // 重排模型ID(用于RAG压缩)
DocumentFragments int `json:"document_fragments,omitempty"` // 文档片段数量(用于RAG压缩)
ProxyURL string `json:"proxy_url,omitempty"` // Optional per-request proxy override; normally empty — use WebSearchProviderEntity.Parameters.proxy_url. Merged at call time when set.
}
// Value implements driver.Valuer interface for WebSearchConfig
+9
View File
@@ -70,6 +70,9 @@ type WebSearchProviderParameters struct {
APIKey string `yaml:"api_key" json:"api_key,omitempty"`
// Google Custom Search Engine ID (only for Google provider)
EngineID string `yaml:"engine_id" json:"engine_id,omitempty"`
// Optional HTTP/HTTPS proxy URL for outbound search requests (e.g. http://host:port); validated with utils.ValidateURLForSSRF.
// Does not replace the search API endpoint; only tunnels traffic to the official APIs.
ProxyURL string `yaml:"proxy_url" json:"proxy_url,omitempty"`
// Provider-specific extra configuration for future extensibility
ExtraConfig map[string]string `yaml:"extra_config" json:"extra_config,omitempty"`
}
@@ -117,6 +120,8 @@ type WebSearchProviderTypeInfo struct {
RequiresAPIKey bool `json:"requires_api_key"`
// Whether the provider requires an engine ID (e.g., Google CSE)
RequiresEngineID bool `json:"requires_engine_id"`
// Whether optional proxy_url in parameters is honored for outbound requests
SupportsProxy bool `json:"supports_proxy"`
// Description
Description string `json:"description"`
// URL to the provider's official website or documentation for obtaining credentials
@@ -130,6 +135,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo {
ID: "duckduckgo",
Name: "DuckDuckGo",
RequiresAPIKey: false,
SupportsProxy: true,
Description: "DuckDuckGo Search (free, no API key required)",
DocsURL: "https://duckduckgo.com/",
},
@@ -137,6 +143,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo {
ID: "bing",
Name: "Bing",
RequiresAPIKey: true,
SupportsProxy: true,
Description: "Bing Search API (requires API key from Azure)",
DocsURL: "https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/overview",
},
@@ -145,6 +152,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo {
Name: "Google",
RequiresAPIKey: true,
RequiresEngineID: true,
SupportsProxy: true,
Description: "Google Custom Search API (requires API key and engine ID)",
DocsURL: "https://developers.google.com/custom-search/v1/overview",
},
@@ -152,6 +160,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo {
ID: "tavily",
Name: "Tavily",
RequiresAPIKey: true,
SupportsProxy: true,
Description: "Tavily Search API (requires API key)",
DocsURL: "https://tavily.com/",
},