feat: search engine adds proxy addresses and adds SSRF verification.
This commit is contained in:
@@ -10,6 +10,7 @@ export interface WebSearchProviderEntity {
|
||||
parameters: {
|
||||
api_key?: string
|
||||
engine_id?: string
|
||||
proxy_url?: string
|
||||
extra_config?: Record<string, string>
|
||||
}
|
||||
is_default?: boolean
|
||||
@@ -23,6 +24,7 @@ export interface WebSearchProviderTypeInfo {
|
||||
name: string
|
||||
requires_api_key: boolean
|
||||
requires_engine_id?: boolean
|
||||
supports_proxy?: boolean
|
||||
description?: string
|
||||
docs_url?: string
|
||||
}
|
||||
|
||||
@@ -776,6 +776,9 @@ export default {
|
||||
providerLabel: 'Search Provider',
|
||||
providerDescription: 'Choose the search engine service used for web search',
|
||||
providerPlaceholder: 'Select a search engine...',
|
||||
proxyUrlLabel: 'HTTP proxy',
|
||||
proxyUrlPlaceholder: 'e.g. http://proxy.example.com:3128 (optional; http/https only)',
|
||||
proxyUrlHelp: 'Use when outbound access to the search API requires a proxy; leave empty to rely on HTTP_PROXY/HTTPS_PROXY environment variables.',
|
||||
apiKeyLabel: 'API Key',
|
||||
apiKeyDescription: 'Enter the API key for the selected search provider',
|
||||
apiKeyPlaceholder: 'Enter API key',
|
||||
|
||||
@@ -636,6 +636,9 @@ export default {
|
||||
providerLabel: "검색 엔진 프로바이더",
|
||||
providerDescription: "웹 검색에 사용할 검색 엔진 서비스 선택",
|
||||
providerPlaceholder: "검색 엔진 선택...",
|
||||
proxyUrlLabel: "HTTP 프록시",
|
||||
proxyUrlPlaceholder: "예: http://127.0.0.1:7890 (선택 사항)",
|
||||
proxyUrlHelp: "검색 API 접근에 프록시가 필요한 경우 입력합니다. 비우면 HTTP_PROXY/HTTPS_PROXY를 사용합니다.",
|
||||
apiKeyLabel: "API 키",
|
||||
apiKeyDescription: "선택한 검색 엔진의 API 키 입력",
|
||||
apiKeyPlaceholder: "API 키를 입력하세요",
|
||||
|
||||
@@ -754,6 +754,9 @@ export default {
|
||||
providerLabel: 'Провайдер поиска',
|
||||
providerDescription: 'Выберите поисковый сервис, используемый для веб-поиска',
|
||||
providerPlaceholder: 'Выберите поисковую систему...',
|
||||
proxyUrlLabel: 'HTTP-прокси',
|
||||
proxyUrlPlaceholder: 'Напр. http://127.0.0.1:7890 (необязательно)',
|
||||
proxyUrlHelp: 'Укажите, если доступ к API поиска нужен через прокси; иначе используются переменные HTTP_PROXY/HTTPS_PROXY.',
|
||||
apiKeyLabel: 'API-ключ',
|
||||
apiKeyDescription: 'Введите API-ключ выбранного провайдера поиска',
|
||||
apiKeyPlaceholder: 'Введите API-ключ',
|
||||
|
||||
@@ -636,6 +636,9 @@ export default {
|
||||
providerLabel: "搜索引擎提供商",
|
||||
providerDescription: "选择用于网络搜索的搜索引擎服务",
|
||||
providerPlaceholder: "选择搜索引擎...",
|
||||
proxyUrlLabel: "HTTP 代理",
|
||||
proxyUrlPlaceholder: "例如 http://proxy.example.com:3128(可选,仅支持 http/https)",
|
||||
proxyUrlHelp: "若环境访问搜索 API 需代理,在此填写;留空则使用系统环境变量 HTTP(S)_PROXY。",
|
||||
apiKeyLabel: "API 密钥",
|
||||
apiKeyDescription: "输入所选搜索引擎的 API 密钥",
|
||||
apiKeyPlaceholder: "请输入 API 密钥",
|
||||
|
||||
@@ -99,6 +99,16 @@
|
||||
</t-form-item>
|
||||
</template>
|
||||
|
||||
<t-form-item v-if="selectedProviderType?.supports_proxy" :label="t('webSearchSettings.proxyUrlLabel')" name="parameters.proxy_url">
|
||||
<t-input
|
||||
v-model="providerForm.parameters.proxy_url"
|
||||
:placeholder="t('webSearchSettings.proxyUrlPlaceholder')"
|
||||
/>
|
||||
<template #help>
|
||||
<span class="switch-help">{{ t('webSearchSettings.proxyUrlHelp') }}</span>
|
||||
</template>
|
||||
</t-form-item>
|
||||
|
||||
<div class="form-divider"></div>
|
||||
|
||||
<t-form-item :label="t('webSearchSettings.setAsDefault')" name="is_default">
|
||||
@@ -164,7 +174,7 @@ const providerForm = ref<{
|
||||
name: string
|
||||
provider: string
|
||||
description: string
|
||||
parameters: { api_key?: string; engine_id?: string }
|
||||
parameters: { api_key?: string; engine_id?: string; proxy_url?: string }
|
||||
is_default: boolean
|
||||
}>({
|
||||
name: '',
|
||||
@@ -228,6 +238,7 @@ const editProvider = (entity: WebSearchProviderEntity) => {
|
||||
parameters: {
|
||||
api_key: '',
|
||||
engine_id: entity.parameters?.engine_id || '',
|
||||
proxy_url: entity.parameters?.proxy_url || '',
|
||||
},
|
||||
is_default: entity.is_default || false,
|
||||
}
|
||||
|
||||
@@ -105,7 +105,8 @@ func (s *WebSearchService) resolveProvider(
|
||||
return nil, fmt.Errorf("web search provider not found: %s", providerID)
|
||||
}
|
||||
|
||||
provider, err := s.registry.CreateProvider(string(entity.Provider), entity.Parameters)
|
||||
params := mergeProxyFromWebSearchConfig(entity.Parameters, cfg)
|
||||
provider, err := s.registry.CreateProvider(string(entity.Provider), params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create provider %s (%s): %w", entity.Name, entity.Provider, err)
|
||||
}
|
||||
@@ -115,9 +116,9 @@ func (s *WebSearchService) resolveProvider(
|
||||
// Backward compatibility: use the deprecated config.Provider field
|
||||
if cfg.Provider != "" {
|
||||
logger.Warnf(ctx, "Using deprecated WebSearchConfig.Provider field: %s. Please migrate to WebSearchProviderEntity.", cfg.Provider)
|
||||
params := types.WebSearchProviderParameters{
|
||||
params := mergeProxyFromWebSearchConfig(types.WebSearchProviderParameters{
|
||||
APIKey: cfg.APIKey,
|
||||
}
|
||||
}, cfg)
|
||||
provider, err := s.registry.CreateProvider(cfg.Provider, params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("web search provider %s is not available: %w", cfg.Provider, err)
|
||||
@@ -128,6 +129,16 @@ func (s *WebSearchService) resolveProvider(
|
||||
return nil, fmt.Errorf("no web search provider configured")
|
||||
}
|
||||
|
||||
// mergeProxyFromWebSearchConfig applies cfg.ProxyURL over stored provider params when non-empty (call-time override).
|
||||
func mergeProxyFromWebSearchConfig(base types.WebSearchProviderParameters, cfg *types.WebSearchConfig) types.WebSearchProviderParameters {
|
||||
if cfg != nil {
|
||||
if pu := strings.TrimSpace(cfg.ProxyURL); pu != "" {
|
||||
base.ProxyURL = pu
|
||||
}
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
// CompressWithRAG performs RAG-based compression using a temporary, hidden knowledge base.
|
||||
// The temporary knowledge base is deleted after use. The UI will not list it due to repo filtering.
|
||||
func (s *WebSearchService) CompressWithRAG(
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
infra_web_search "github.com/Tencent/WeKnora/internal/infrastructure/web_search"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
@@ -60,6 +61,12 @@ func (s *webSearchProviderService) UpdateProvider(ctx context.Context, provider
|
||||
}
|
||||
}
|
||||
|
||||
if provider.Provider != "" {
|
||||
if err := validateProviderParameters(provider.Provider, provider.Parameters); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Updating web search provider: tenant=%d, id=%s", provider.TenantID, provider.ID)
|
||||
return s.repo.Update(ctx, provider)
|
||||
}
|
||||
@@ -114,5 +121,12 @@ func validateProviderParameters(provider types.WebSearchProviderType, params typ
|
||||
case types.WebSearchProviderTypeDuckDuckGo:
|
||||
// No API key required
|
||||
}
|
||||
if err := validateOptionalProxyURL(params.ProxyURL); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateOptionalProxyURL(proxyURL string) error {
|
||||
return infra_web_search.ValidateProxyURL(proxyURL)
|
||||
}
|
||||
|
||||
@@ -54,8 +54,9 @@ func NewBingProvider(params types.WebSearchProviderParameters) (interfaces.WebSe
|
||||
if params.APIKey == "" {
|
||||
return nil, fmt.Errorf("API key is required for Bing provider")
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: defaultBingTimeout,
|
||||
client, err := NewSearchHTTPClient(defaultBingTimeout, params.ProxyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &BingProvider{
|
||||
client: client,
|
||||
|
||||
@@ -23,13 +23,13 @@ type DuckDuckGoProvider struct {
|
||||
}
|
||||
|
||||
// NewDuckDuckGoProvider creates a new DuckDuckGo provider.
|
||||
// DuckDuckGo is free and requires no API key or configuration.
|
||||
// DuckDuckGo is free and requires no API key; optional ProxyURL tunnels requests via an HTTP/HTTPS proxy.
|
||||
func NewDuckDuckGoProvider(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) {
|
||||
return &DuckDuckGoProvider{
|
||||
client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}, nil
|
||||
client, err := NewSearchHTTPClient(30*time.Second, params.ProxyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &DuckDuckGoProvider{client: client}, nil
|
||||
}
|
||||
|
||||
// Name returns the provider name
|
||||
|
||||
@@ -3,6 +3,7 @@ package web_search
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"google.golang.org/api/customsearch/v1"
|
||||
"google.golang.org/api/option"
|
||||
@@ -29,8 +30,13 @@ func NewGoogleProvider(params types.WebSearchProviderParameters) (interfaces.Web
|
||||
return nil, fmt.Errorf("engine ID is required for Google provider")
|
||||
}
|
||||
|
||||
httpClient, err := NewSearchHTTPClient(30*time.Second, params.ProxyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clientOpts := []option.ClientOption{
|
||||
option.WithAPIKey(params.APIKey),
|
||||
option.WithHTTPClient(httpClient),
|
||||
}
|
||||
srv, err := customsearch.NewService(context.Background(), clientOpts...)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package web_search
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// ValidateProxyURL delegates to utils.ValidateURLForSSRF (only http/https pass that check).
|
||||
func ValidateProxyURL(proxyURL string) error {
|
||||
proxyURL = strings.TrimSpace(proxyURL)
|
||||
if proxyURL == "" {
|
||||
return nil
|
||||
}
|
||||
return utils.ValidateURLForSSRF(proxyURL)
|
||||
}
|
||||
|
||||
func ssrfSafeRedirect(maxRedirects int) func(*http.Request, []*http.Request) error {
|
||||
return func(req *http.Request, via []*http.Request) error {
|
||||
if len(via) >= maxRedirects {
|
||||
return fmt.Errorf("stopped after %d redirects", maxRedirects)
|
||||
}
|
||||
if err := utils.ValidateURLForSSRF(req.URL.String()); err != nil {
|
||||
return fmt.Errorf("%w: %v", utils.ErrSSRFRedirectBlocked, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// NewSearchHTTPClient builds an http.Client for outbound web search requests.
|
||||
// It uses utils.SSRFSafeDialContext, optional explicit or environment proxy, and
|
||||
// redirect validation consistent with utils.NewSSRFSafeHTTPClient.
|
||||
func NewSearchHTTPClient(timeout time.Duration, proxyURL string) (*http.Client, error) {
|
||||
proxyURL = strings.TrimSpace(proxyURL)
|
||||
def, ok := http.DefaultTransport.(*http.Transport)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("default HTTP transport is not *http.Transport")
|
||||
}
|
||||
t := def.Clone()
|
||||
t.DialContext = utils.SSRFSafeDialContext
|
||||
|
||||
if proxyURL != "" {
|
||||
if err := ValidateProxyURL(proxyURL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u, err := url.Parse(proxyURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid proxy_url: %w", err)
|
||||
}
|
||||
if u.Scheme == "" || u.Host == "" {
|
||||
return nil, fmt.Errorf("invalid proxy_url: scheme and host are required")
|
||||
}
|
||||
t.Proxy = http.ProxyURL(u)
|
||||
} else {
|
||||
t.Proxy = http.ProxyFromEnvironment
|
||||
}
|
||||
|
||||
cfg := utils.DefaultSSRFSafeHTTPClientConfig()
|
||||
return &http.Client{
|
||||
Timeout: timeout,
|
||||
Transport: t,
|
||||
CheckRedirect: ssrfSafeRedirect(cfg.MaxRedirects),
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package web_search
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestValidateProxyURL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
wantErr bool
|
||||
}{
|
||||
{"empty", "", false},
|
||||
{"whitespace only", " ", false},
|
||||
{"no scheme private IP bare host:port", "192.168.0.1:3128", true},
|
||||
{"localhost http", "http://localhost:3128", true},
|
||||
{"direct IPv4 http", "http://192.168.1.1:3128", true},
|
||||
{"socks5 to IP blocked", "socks5://10.0.0.1:1080", true},
|
||||
{"bad scheme", "ftp://proxy.example:21", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := ValidateProxyURL(tt.raw)
|
||||
if tt.wantErr && err == nil {
|
||||
t.Fatalf("expected error, got nil")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSearchHTTPClient_RejectsUnsafeProxy(t *testing.T) {
|
||||
_, err := NewSearchHTTPClient(0, "http://127.0.0.1:3128")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for disallowed proxy URL (127.0.0.1)")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "SSRF") {
|
||||
t.Fatalf("expected SSRF-related error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSearchHTTPClient_AcceptsEmptyProxy(t *testing.T) {
|
||||
c, err := NewSearchHTTPClient(0, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c.Transport == nil {
|
||||
t.Fatal("expected transport")
|
||||
}
|
||||
if c.CheckRedirect == nil {
|
||||
t.Fatal("expected CheckRedirect for SSRF-safe redirects")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSsrfSafeRedirect_BlocksNonHTTPSScheme(t *testing.T) {
|
||||
h := ssrfSafeRedirect(10)
|
||||
req := &http.Request{URL: mustParse(t, "ftp://evil.com/")}
|
||||
// len(via)==0 so first redirect
|
||||
err := h(req, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for ftp redirect")
|
||||
}
|
||||
}
|
||||
|
||||
func mustParse(t *testing.T, s string) *url.URL {
|
||||
t.Helper()
|
||||
u, err := url.Parse(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
@@ -36,8 +36,9 @@ func NewTavilyProvider(params types.WebSearchProviderParameters) (interfaces.Web
|
||||
if params.APIKey == "" {
|
||||
return nil, fmt.Errorf("API key is required for Tavily provider")
|
||||
}
|
||||
client := &http.Client{
|
||||
Timeout: defaultTavilyTimeout,
|
||||
client, err := NewSearchHTTPClient(defaultTavilyTimeout, params.ProxyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &TavilyProvider{
|
||||
client: client,
|
||||
|
||||
@@ -22,6 +22,7 @@ type WebSearchConfig struct {
|
||||
EmbeddingDimension int `json:"embedding_dimension,omitempty"` // 嵌入维度(用于RAG压缩)
|
||||
RerankModelID string `json:"rerank_model_id,omitempty"` // 重排模型ID(用于RAG压缩)
|
||||
DocumentFragments int `json:"document_fragments,omitempty"` // 文档片段数量(用于RAG压缩)
|
||||
ProxyURL string `json:"proxy_url,omitempty"` // Optional per-request proxy override; normally empty — use WebSearchProviderEntity.Parameters.proxy_url. Merged at call time when set.
|
||||
}
|
||||
|
||||
// Value implements driver.Valuer interface for WebSearchConfig
|
||||
|
||||
@@ -70,6 +70,9 @@ type WebSearchProviderParameters struct {
|
||||
APIKey string `yaml:"api_key" json:"api_key,omitempty"`
|
||||
// Google Custom Search Engine ID (only for Google provider)
|
||||
EngineID string `yaml:"engine_id" json:"engine_id,omitempty"`
|
||||
// Optional HTTP/HTTPS proxy URL for outbound search requests (e.g. http://host:port); validated with utils.ValidateURLForSSRF.
|
||||
// Does not replace the search API endpoint; only tunnels traffic to the official APIs.
|
||||
ProxyURL string `yaml:"proxy_url" json:"proxy_url,omitempty"`
|
||||
// Provider-specific extra configuration for future extensibility
|
||||
ExtraConfig map[string]string `yaml:"extra_config" json:"extra_config,omitempty"`
|
||||
}
|
||||
@@ -117,6 +120,8 @@ type WebSearchProviderTypeInfo struct {
|
||||
RequiresAPIKey bool `json:"requires_api_key"`
|
||||
// Whether the provider requires an engine ID (e.g., Google CSE)
|
||||
RequiresEngineID bool `json:"requires_engine_id"`
|
||||
// Whether optional proxy_url in parameters is honored for outbound requests
|
||||
SupportsProxy bool `json:"supports_proxy"`
|
||||
// Description
|
||||
Description string `json:"description"`
|
||||
// URL to the provider's official website or documentation for obtaining credentials
|
||||
@@ -130,6 +135,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo {
|
||||
ID: "duckduckgo",
|
||||
Name: "DuckDuckGo",
|
||||
RequiresAPIKey: false,
|
||||
SupportsProxy: true,
|
||||
Description: "DuckDuckGo Search (free, no API key required)",
|
||||
DocsURL: "https://duckduckgo.com/",
|
||||
},
|
||||
@@ -137,6 +143,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo {
|
||||
ID: "bing",
|
||||
Name: "Bing",
|
||||
RequiresAPIKey: true,
|
||||
SupportsProxy: true,
|
||||
Description: "Bing Search API (requires API key from Azure)",
|
||||
DocsURL: "https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/overview",
|
||||
},
|
||||
@@ -145,6 +152,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo {
|
||||
Name: "Google",
|
||||
RequiresAPIKey: true,
|
||||
RequiresEngineID: true,
|
||||
SupportsProxy: true,
|
||||
Description: "Google Custom Search API (requires API key and engine ID)",
|
||||
DocsURL: "https://developers.google.com/custom-search/v1/overview",
|
||||
},
|
||||
@@ -152,6 +160,7 @@ func GetWebSearchProviderTypes() []WebSearchProviderTypeInfo {
|
||||
ID: "tavily",
|
||||
Name: "Tavily",
|
||||
RequiresAPIKey: true,
|
||||
SupportsProxy: true,
|
||||
Description: "Tavily Search API (requires API key)",
|
||||
DocsURL: "https://tavily.com/",
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user