diff --git a/frontend/src/api/web-search-provider.ts b/frontend/src/api/web-search-provider.ts index de203fb2..e315ce78 100644 --- a/frontend/src/api/web-search-provider.ts +++ b/frontend/src/api/web-search-provider.ts @@ -10,6 +10,7 @@ export interface WebSearchProviderEntity { parameters: { api_key?: string engine_id?: string + proxy_url?: string extra_config?: Record } is_default?: boolean @@ -23,6 +24,7 @@ export interface WebSearchProviderTypeInfo { name: string requires_api_key: boolean requires_engine_id?: boolean + supports_proxy?: boolean description?: string docs_url?: string } diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 8c95f99f..4bed7b8f 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -776,6 +776,9 @@ export default { providerLabel: 'Search Provider', providerDescription: 'Choose the search engine service used for web search', providerPlaceholder: 'Select a search engine...', + proxyUrlLabel: 'HTTP proxy', + proxyUrlPlaceholder: 'e.g. http://proxy.example.com:3128 (optional; http/https only)', + proxyUrlHelp: 'Use when outbound access to the search API requires a proxy; leave empty to rely on HTTP_PROXY/HTTPS_PROXY environment variables.', apiKeyLabel: 'API Key', apiKeyDescription: 'Enter the API key for the selected search provider', apiKeyPlaceholder: 'Enter API key', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index c52cb0b2..7071150c 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -636,6 +636,9 @@ export default { providerLabel: "검색 엔진 프로바이더", providerDescription: "웹 검색에 사용할 검색 엔진 서비스 선택", providerPlaceholder: "검색 엔진 선택...", + proxyUrlLabel: "HTTP 프록시", + proxyUrlPlaceholder: "예: http://127.0.0.1:7890 (선택 사항)", + proxyUrlHelp: "검색 API 접근에 프록시가 필요한 경우 입력합니다. 비우면 HTTP_PROXY/HTTPS_PROXY를 사용합니다.", apiKeyLabel: "API 키", apiKeyDescription: "선택한 검색 엔진의 API 키 입력", apiKeyPlaceholder: "API 키를 입력하세요", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 44ae3dd0..022f76e9 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -754,6 +754,9 @@ export default { providerLabel: 'Провайдер поиска', providerDescription: 'Выберите поисковый сервис, используемый для веб-поиска', providerPlaceholder: 'Выберите поисковую систему...', + proxyUrlLabel: 'HTTP-прокси', + proxyUrlPlaceholder: 'Напр. http://127.0.0.1:7890 (необязательно)', + proxyUrlHelp: 'Укажите, если доступ к API поиска нужен через прокси; иначе используются переменные HTTP_PROXY/HTTPS_PROXY.', apiKeyLabel: 'API-ключ', apiKeyDescription: 'Введите API-ключ выбранного провайдера поиска', apiKeyPlaceholder: 'Введите API-ключ', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 32ea5810..8a3180d1 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -636,6 +636,9 @@ export default { providerLabel: "搜索引擎提供商", providerDescription: "选择用于网络搜索的搜索引擎服务", providerPlaceholder: "选择搜索引擎...", + proxyUrlLabel: "HTTP 代理", + proxyUrlPlaceholder: "例如 http://proxy.example.com:3128(可选,仅支持 http/https)", + proxyUrlHelp: "若环境访问搜索 API 需代理,在此填写;留空则使用系统环境变量 HTTP(S)_PROXY。", apiKeyLabel: "API 密钥", apiKeyDescription: "输入所选搜索引擎的 API 密钥", apiKeyPlaceholder: "请输入 API 密钥", diff --git a/frontend/src/views/settings/WebSearchSettings.vue b/frontend/src/views/settings/WebSearchSettings.vue index 5301d777..7a5f6ff6 100644 --- a/frontend/src/views/settings/WebSearchSettings.vue +++ b/frontend/src/views/settings/WebSearchSettings.vue @@ -99,6 +99,16 @@ + + + + +
@@ -164,7 +174,7 @@ const providerForm = ref<{ name: string provider: string description: string - parameters: { api_key?: string; engine_id?: string } + parameters: { api_key?: string; engine_id?: string; proxy_url?: string } is_default: boolean }>({ name: '', @@ -228,6 +238,7 @@ const editProvider = (entity: WebSearchProviderEntity) => { parameters: { api_key: '', engine_id: entity.parameters?.engine_id || '', + proxy_url: entity.parameters?.proxy_url || '', }, is_default: entity.is_default || false, } diff --git a/internal/application/service/web_search.go b/internal/application/service/web_search.go index e41cc2bc..f4164480 100644 --- a/internal/application/service/web_search.go +++ b/internal/application/service/web_search.go @@ -105,7 +105,8 @@ func (s *WebSearchService) resolveProvider( return nil, fmt.Errorf("web search provider not found: %s", providerID) } - provider, err := s.registry.CreateProvider(string(entity.Provider), entity.Parameters) + params := mergeProxyFromWebSearchConfig(entity.Parameters, cfg) + provider, err := s.registry.CreateProvider(string(entity.Provider), params) if err != nil { return nil, fmt.Errorf("failed to create provider %s (%s): %w", entity.Name, entity.Provider, err) } @@ -115,9 +116,9 @@ func (s *WebSearchService) resolveProvider( // Backward compatibility: use the deprecated config.Provider field if cfg.Provider != "" { logger.Warnf(ctx, "Using deprecated WebSearchConfig.Provider field: %s. Please migrate to WebSearchProviderEntity.", cfg.Provider) - params := types.WebSearchProviderParameters{ + params := mergeProxyFromWebSearchConfig(types.WebSearchProviderParameters{ APIKey: cfg.APIKey, - } + }, cfg) provider, err := s.registry.CreateProvider(cfg.Provider, params) if err != nil { return nil, fmt.Errorf("web search provider %s is not available: %w", cfg.Provider, err) @@ -128,6 +129,16 @@ func (s *WebSearchService) resolveProvider( return nil, fmt.Errorf("no web search provider configured") } +// mergeProxyFromWebSearchConfig applies cfg.ProxyURL over stored provider params when non-empty (call-time override). +func mergeProxyFromWebSearchConfig(base types.WebSearchProviderParameters, cfg *types.WebSearchConfig) types.WebSearchProviderParameters { + if cfg != nil { + if pu := strings.TrimSpace(cfg.ProxyURL); pu != "" { + base.ProxyURL = pu + } + } + return base +} + // CompressWithRAG performs RAG-based compression using a temporary, hidden knowledge base. // The temporary knowledge base is deleted after use. The UI will not list it due to repo filtering. func (s *WebSearchService) CompressWithRAG( diff --git a/internal/application/service/web_search_provider.go b/internal/application/service/web_search_provider.go index f61ec899..c8a8fa68 100644 --- a/internal/application/service/web_search_provider.go +++ b/internal/application/service/web_search_provider.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" @@ -60,6 +61,12 @@ func (s *webSearchProviderService) UpdateProvider(ctx context.Context, provider } } + if provider.Provider != "" { + if err := validateProviderParameters(provider.Provider, provider.Parameters); err != nil { + return err + } + } + logger.Infof(ctx, "Updating web search provider: tenant=%d, id=%s", provider.TenantID, provider.ID) return s.repo.Update(ctx, provider) } @@ -114,5 +121,12 @@ func validateProviderParameters(provider types.WebSearchProviderType, params typ case types.WebSearchProviderTypeDuckDuckGo: // No API key required } + if err := validateOptionalProxyURL(params.ProxyURL); err != nil { + return err + } return nil } + +func validateOptionalProxyURL(proxyURL string) error { + return infra_web_search.ValidateProxyURL(proxyURL) +} diff --git a/internal/infrastructure/web_search/bing.go b/internal/infrastructure/web_search/bing.go index 68eab98e..b5ba35b4 100644 --- a/internal/infrastructure/web_search/bing.go +++ b/internal/infrastructure/web_search/bing.go @@ -54,8 +54,9 @@ func NewBingProvider(params types.WebSearchProviderParameters) (interfaces.WebSe if params.APIKey == "" { return nil, fmt.Errorf("API key is required for Bing provider") } - client := &http.Client{ - Timeout: defaultBingTimeout, + client, err := NewSearchHTTPClient(defaultBingTimeout, params.ProxyURL) + if err != nil { + return nil, err } return &BingProvider{ client: client, diff --git a/internal/infrastructure/web_search/duckduckgo.go b/internal/infrastructure/web_search/duckduckgo.go index 5a405c18..0eaa8db2 100644 --- a/internal/infrastructure/web_search/duckduckgo.go +++ b/internal/infrastructure/web_search/duckduckgo.go @@ -23,13 +23,13 @@ type DuckDuckGoProvider struct { } // NewDuckDuckGoProvider creates a new DuckDuckGo provider. -// DuckDuckGo is free and requires no API key or configuration. +// DuckDuckGo is free and requires no API key; optional ProxyURL tunnels requests via an HTTP/HTTPS proxy. func NewDuckDuckGoProvider(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) { - return &DuckDuckGoProvider{ - client: &http.Client{ - Timeout: 30 * time.Second, - }, - }, nil + client, err := NewSearchHTTPClient(30*time.Second, params.ProxyURL) + if err != nil { + return nil, err + } + return &DuckDuckGoProvider{client: client}, nil } // Name returns the provider name diff --git a/internal/infrastructure/web_search/google.go b/internal/infrastructure/web_search/google.go index af355676..2da39288 100644 --- a/internal/infrastructure/web_search/google.go +++ b/internal/infrastructure/web_search/google.go @@ -3,6 +3,7 @@ package web_search import ( "context" "fmt" + "time" "google.golang.org/api/customsearch/v1" "google.golang.org/api/option" @@ -29,8 +30,13 @@ func NewGoogleProvider(params types.WebSearchProviderParameters) (interfaces.Web return nil, fmt.Errorf("engine ID is required for Google provider") } + httpClient, err := NewSearchHTTPClient(30*time.Second, params.ProxyURL) + if err != nil { + return nil, err + } clientOpts := []option.ClientOption{ option.WithAPIKey(params.APIKey), + option.WithHTTPClient(httpClient), } srv, err := customsearch.NewService(context.Background(), clientOpts...) if err != nil { diff --git a/internal/infrastructure/web_search/proxy.go b/internal/infrastructure/web_search/proxy.go new file mode 100644 index 00000000..d7bc0689 --- /dev/null +++ b/internal/infrastructure/web_search/proxy.go @@ -0,0 +1,68 @@ +package web_search + +import ( + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/utils" +) + +// ValidateProxyURL delegates to utils.ValidateURLForSSRF (only http/https pass that check). +func ValidateProxyURL(proxyURL string) error { + proxyURL = strings.TrimSpace(proxyURL) + if proxyURL == "" { + return nil + } + return utils.ValidateURLForSSRF(proxyURL) +} + +func ssrfSafeRedirect(maxRedirects int) func(*http.Request, []*http.Request) error { + return func(req *http.Request, via []*http.Request) error { + if len(via) >= maxRedirects { + return fmt.Errorf("stopped after %d redirects", maxRedirects) + } + if err := utils.ValidateURLForSSRF(req.URL.String()); err != nil { + return fmt.Errorf("%w: %v", utils.ErrSSRFRedirectBlocked, err) + } + return nil + } +} + +// NewSearchHTTPClient builds an http.Client for outbound web search requests. +// It uses utils.SSRFSafeDialContext, optional explicit or environment proxy, and +// redirect validation consistent with utils.NewSSRFSafeHTTPClient. +func NewSearchHTTPClient(timeout time.Duration, proxyURL string) (*http.Client, error) { + proxyURL = strings.TrimSpace(proxyURL) + def, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return nil, fmt.Errorf("default HTTP transport is not *http.Transport") + } + t := def.Clone() + t.DialContext = utils.SSRFSafeDialContext + + if proxyURL != "" { + if err := ValidateProxyURL(proxyURL); err != nil { + return nil, err + } + u, err := url.Parse(proxyURL) + if err != nil { + return nil, fmt.Errorf("invalid proxy_url: %w", err) + } + if u.Scheme == "" || u.Host == "" { + return nil, fmt.Errorf("invalid proxy_url: scheme and host are required") + } + t.Proxy = http.ProxyURL(u) + } else { + t.Proxy = http.ProxyFromEnvironment + } + + cfg := utils.DefaultSSRFSafeHTTPClientConfig() + return &http.Client{ + Timeout: timeout, + Transport: t, + CheckRedirect: ssrfSafeRedirect(cfg.MaxRedirects), + }, nil +} diff --git a/internal/infrastructure/web_search/proxy_test.go b/internal/infrastructure/web_search/proxy_test.go new file mode 100644 index 00000000..46f318d1 --- /dev/null +++ b/internal/infrastructure/web_search/proxy_test.go @@ -0,0 +1,78 @@ +package web_search + +import ( + "net/http" + "net/url" + "strings" + "testing" +) + +func TestValidateProxyURL(t *testing.T) { + tests := []struct { + name string + raw string + wantErr bool + }{ + {"empty", "", false}, + {"whitespace only", " ", false}, + {"no scheme private IP bare host:port", "192.168.0.1:3128", true}, + {"localhost http", "http://localhost:3128", true}, + {"direct IPv4 http", "http://192.168.1.1:3128", true}, + {"socks5 to IP blocked", "socks5://10.0.0.1:1080", true}, + {"bad scheme", "ftp://proxy.example:21", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateProxyURL(tt.raw) + if tt.wantErr && err == nil { + t.Fatalf("expected error, got nil") + } + if !tt.wantErr && err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} + +func TestNewSearchHTTPClient_RejectsUnsafeProxy(t *testing.T) { + _, err := NewSearchHTTPClient(0, "http://127.0.0.1:3128") + if err == nil { + t.Fatal("expected error for disallowed proxy URL (127.0.0.1)") + } + if !strings.Contains(err.Error(), "SSRF") { + t.Fatalf("expected SSRF-related error, got: %v", err) + } +} + +func TestNewSearchHTTPClient_AcceptsEmptyProxy(t *testing.T) { + c, err := NewSearchHTTPClient(0, "") + if err != nil { + t.Fatal(err) + } + if c.Transport == nil { + t.Fatal("expected transport") + } + if c.CheckRedirect == nil { + t.Fatal("expected CheckRedirect for SSRF-safe redirects") + } +} + +func TestSsrfSafeRedirect_BlocksNonHTTPSScheme(t *testing.T) { + h := ssrfSafeRedirect(10) + req := &http.Request{URL: mustParse(t, "ftp://evil.com/")} + // len(via)==0 so first redirect + err := h(req, nil) + if err == nil { + t.Fatal("expected error for ftp redirect") + } +} + +func mustParse(t *testing.T, s string) *url.URL { + t.Helper() + u, err := url.Parse(s) + if err != nil { + t.Fatal(err) + } + return u +} diff --git a/internal/infrastructure/web_search/tavily.go b/internal/infrastructure/web_search/tavily.go index 614adfbf..4a1e3526 100644 --- a/internal/infrastructure/web_search/tavily.go +++ b/internal/infrastructure/web_search/tavily.go @@ -36,8 +36,9 @@ func NewTavilyProvider(params types.WebSearchProviderParameters) (interfaces.Web if params.APIKey == "" { return nil, fmt.Errorf("API key is required for Tavily provider") } - client := &http.Client{ - Timeout: defaultTavilyTimeout, + client, err := NewSearchHTTPClient(defaultTavilyTimeout, params.ProxyURL) + if err != nil { + return nil, err } return &TavilyProvider{ client: client, diff --git a/internal/types/web_search.go b/internal/types/web_search.go index 5b351d0d..1cf59210 100644 --- a/internal/types/web_search.go +++ b/internal/types/web_search.go @@ -22,6 +22,7 @@ type WebSearchConfig struct { EmbeddingDimension int `json:"embedding_dimension,omitempty"` // 嵌入维度(用于RAG压缩) RerankModelID string `json:"rerank_model_id,omitempty"` // 重排模型ID(用于RAG压缩) DocumentFragments int `json:"document_fragments,omitempty"` // 文档片段数量(用于RAG压缩) + ProxyURL string `json:"proxy_url,omitempty"` // Optional per-request proxy override; normally empty — use WebSearchProviderEntity.Parameters.proxy_url. Merged at call time when set. } // Value implements driver.Valuer interface for WebSearchConfig diff --git a/internal/types/web_search_provider.go b/internal/types/web_search_provider.go index 78a82761..58c14928 100644 --- a/internal/types/web_search_provider.go +++ b/internal/types/web_search_provider.go @@ -70,6 +70,9 @@ type WebSearchProviderParameters struct { APIKey string `yaml:"api_key" json:"api_key,omitempty"` // Google Custom Search Engine ID (only for Google provider) EngineID string `yaml:"engine_id" json:"engine_id,omitempty"` + // Optional HTTP/HTTPS proxy URL for outbound search requests (e.g. http://host:port); validated with utils.ValidateURLForSSRF. + // Does not replace the search API endpoint; only tunnels traffic to the official APIs. + ProxyURL string `yaml:"proxy_url" json:"proxy_url,omitempty"` // Provider-specific extra configuration for future extensibility ExtraConfig map[string]string `yaml:"extra_config" json:"extra_config,omitempty"` } @@ -117,6 +120,8 @@ type WebSearchProviderTypeInfo struct { RequiresAPIKey bool `json:"requires_api_key"` // Whether the provider requires an engine ID (e.g., Google CSE) RequiresEngineID bool `json:"requires_engine_id"` + // Whether optional proxy_url in parameters is honored for outbound requests + SupportsProxy bool `json:"supports_proxy"` // Description Description string `json:"description"` // URL to the provider's official website or documentation for obtaining credentials @@ -130,6 +135,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo { ID: "duckduckgo", Name: "DuckDuckGo", RequiresAPIKey: false, + SupportsProxy: true, Description: "DuckDuckGo Search (free, no API key required)", DocsURL: "https://duckduckgo.com/", }, @@ -137,6 +143,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo { ID: "bing", Name: "Bing", RequiresAPIKey: true, + SupportsProxy: true, Description: "Bing Search API (requires API key from Azure)", DocsURL: "https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/overview", }, @@ -145,6 +152,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo { Name: "Google", RequiresAPIKey: true, RequiresEngineID: true, + SupportsProxy: true, Description: "Google Custom Search API (requires API key and engine ID)", DocsURL: "https://developers.google.com/custom-search/v1/overview", }, @@ -152,6 +160,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo { ID: "tavily", Name: "Tavily", RequiresAPIKey: true, + SupportsProxy: true, Description: "Tavily Search API (requires API key)", DocsURL: "https://tavily.com/", },