fix(web-search): address review fixups for SearXNG provider
- utils: 合并 SSRF_WHITELIST 与 SSRF_WHITELIST_EXTRA,避免部署侧默认值 (如 docker-compose 注入的 searxng 主机名)被用户的 SSRF_WHITELIST 自定义值覆盖。 - docker-compose.yml: 把 searxng 默认值挪到 SSRF_WHITELIST_EXTRA。 - searxng: 抽出导出函数 ValidateSearxngBaseURL,让 service 层保存校验 和 provider 构造校验完全一致;service 改为调用同一函数。 - searxng: language 由非法的 "auto" 改为 "all";移除强制 safesearch=1 让实例 settings.yml 决定。 - searxng: publishedDate 增加多格式 fallback(RFC3339Nano/无时区/ 日期-only/RFC1123 等),无法解析时 debug 日志记录。 - searxng: 解析 unresponsive_engines;结果为空时打 warn 日志,便于排查 "配置正确却搜不到结果"的情况。 - frontend: WebSearchSettings 的 Instance URL 标签/占位符走 i18n, zh-CN / en-US / ru-RU / ko-KR 四个 locale 补齐 baseUrlLabel / baseUrlPlaceholder。
This commit is contained in:
+4
-1
@@ -134,7 +134,10 @@ services:
|
||||
- NEO4J_PASSWORD=${NEO4J_PASSWORD:-password}
|
||||
- TENANT_AES_KEY=${TENANT_AES_KEY:-}
|
||||
- SYSTEM_AES_KEY=${SYSTEM_AES_KEY:-}
|
||||
- SSRF_WHITELIST=${SSRF_WHITELIST:-searxng}
|
||||
- SSRF_WHITELIST=${SSRF_WHITELIST:-}
|
||||
# Always allow the optional searxng sidecar (compose service hostname);
|
||||
# merged on top of SSRF_WHITELIST so user overrides don't clobber it.
|
||||
- SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng}
|
||||
- CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5}
|
||||
- JWT_SECRET=${JWT_SECRET:-}
|
||||
# Crypto: 主密钥和盐值,用于 AppSecret 等敏感字段的 AES-256 加密
|
||||
|
||||
@@ -824,6 +824,8 @@ export default {
|
||||
proxyUrlPlaceholder: 'e.g. http://proxy.example.com:3128 (optional; http/https only)',
|
||||
proxyUrlHelp: 'Use when outbound access to the search API requires a proxy; leave empty to rely on HTTP_PROXY/HTTPS_PROXY environment variables.',
|
||||
apiKeyLabel: 'API Key',
|
||||
baseUrlLabel: 'Instance URL',
|
||||
baseUrlPlaceholder: 'https://searxng.example.com',
|
||||
apiKeyDescription: 'Enter the API key for the selected search provider',
|
||||
apiKeyPlaceholder: 'Enter API key',
|
||||
maxResultsLabel: 'Maximum Results',
|
||||
|
||||
@@ -702,6 +702,8 @@ export default {
|
||||
proxyUrlPlaceholder: "예: http://127.0.0.1:7890 (선택 사항)",
|
||||
proxyUrlHelp: "검색 API 접근에 프록시가 필요한 경우 입력합니다. 비우면 HTTP_PROXY/HTTPS_PROXY를 사용합니다.",
|
||||
apiKeyLabel: "API 키",
|
||||
baseUrlLabel: "인스턴스 URL",
|
||||
baseUrlPlaceholder: "https://searxng.example.com",
|
||||
apiKeyDescription: "선택한 검색 엔진의 API 키 입력",
|
||||
apiKeyPlaceholder: "API 키를 입력하세요",
|
||||
maxResultsLabel: "최대 결과 수",
|
||||
|
||||
@@ -807,6 +807,8 @@ export default {
|
||||
proxyUrlPlaceholder: 'Напр. http://127.0.0.1:7890 (необязательно)',
|
||||
proxyUrlHelp: 'Укажите, если доступ к API поиска нужен через прокси; иначе используются переменные HTTP_PROXY/HTTPS_PROXY.',
|
||||
apiKeyLabel: 'API-ключ',
|
||||
baseUrlLabel: 'URL экземпляра',
|
||||
baseUrlPlaceholder: 'https://searxng.example.com',
|
||||
apiKeyDescription: 'Введите API-ключ выбранного провайдера поиска',
|
||||
apiKeyPlaceholder: 'Введите API-ключ',
|
||||
maxResultsLabel: 'Максимум результатов',
|
||||
|
||||
@@ -701,6 +701,8 @@ export default {
|
||||
proxyUrlPlaceholder: "例如 http://proxy.example.com:3128(可选,仅支持 http/https)",
|
||||
proxyUrlHelp: "若环境访问搜索 API 需代理,在此填写;留空则使用系统环境变量 HTTP(S)_PROXY。",
|
||||
apiKeyLabel: "API 密钥",
|
||||
baseUrlLabel: "实例地址",
|
||||
baseUrlPlaceholder: "https://searxng.example.com",
|
||||
apiKeyDescription: "输入所选搜索引擎的 API 密钥",
|
||||
apiKeyPlaceholder: "请输入 API 密钥",
|
||||
maxResultsLabel: "最大结果数",
|
||||
|
||||
@@ -92,10 +92,10 @@
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<t-form-item v-if="selectedProviderType?.requires_base_url" label="Instance URL" name="parameters.base_url">
|
||||
<t-form-item v-if="selectedProviderType?.requires_base_url" :label="t('webSearchSettings.baseUrlLabel')" name="parameters.base_url">
|
||||
<t-input
|
||||
v-model="providerForm.parameters.base_url"
|
||||
placeholder="https://searxng.example.com"
|
||||
:placeholder="t('webSearchSettings.baseUrlPlaceholder')"
|
||||
/>
|
||||
</t-form-item>
|
||||
<t-form-item v-if="selectedProviderType?.requires_api_key" :label="t('webSearchSettings.apiKeyLabel')" name="parameters.api_key">
|
||||
|
||||
@@ -122,11 +122,8 @@ func validateProviderParameters(provider types.WebSearchProviderType, params typ
|
||||
case types.WebSearchProviderTypeDuckDuckGo:
|
||||
// No API key required
|
||||
case types.WebSearchProviderTypeSearxng:
|
||||
if params.BaseURL == "" {
|
||||
return fmt.Errorf("base_url is required for SearXNG provider")
|
||||
}
|
||||
if err := infra_web_search.ValidateProxyURL(params.BaseURL); err != nil {
|
||||
return fmt.Errorf("invalid SearXNG base_url: %w", err)
|
||||
if err := infra_web_search.ValidateSearxngBaseURL(params.BaseURL); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := validateOptionalProxyURL(params.ProxyURL); err != nil {
|
||||
|
||||
@@ -33,21 +33,33 @@ type SearxngProvider struct {
|
||||
apiKey string
|
||||
}
|
||||
|
||||
// NewSearxngProvider builds a SearXNG provider from tenant parameters.
|
||||
func NewSearxngProvider(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) {
|
||||
base := strings.TrimSpace(params.BaseURL)
|
||||
// ValidateSearxngBaseURL validates a SearXNG instance URL: must be a non-empty,
|
||||
// absolute http(s) URL, and must pass the SSRF whitelist check. Shared between
|
||||
// the service-layer parameter validation and the provider constructor so that
|
||||
// "save" and "use" never disagree.
|
||||
func ValidateSearxngBaseURL(rawURL string) error {
|
||||
base := strings.TrimSpace(rawURL)
|
||||
if base == "" {
|
||||
return nil, fmt.Errorf("base_url is required for SearXNG provider")
|
||||
}
|
||||
if err := utils.ValidateURLForSSRF(base); err != nil {
|
||||
return nil, fmt.Errorf("invalid SearXNG base_url: %w", err)
|
||||
return fmt.Errorf("base_url is required for SearXNG provider")
|
||||
}
|
||||
parsed, err := url.Parse(base)
|
||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||
return nil, fmt.Errorf("invalid SearXNG base_url: must be an absolute http(s) URL")
|
||||
return fmt.Errorf("invalid SearXNG base_url: must be an absolute http(s) URL")
|
||||
}
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return nil, fmt.Errorf("invalid SearXNG base_url scheme: %s", parsed.Scheme)
|
||||
return fmt.Errorf("invalid SearXNG base_url scheme: %s", parsed.Scheme)
|
||||
}
|
||||
if err := utils.ValidateURLForSSRF(base); err != nil {
|
||||
return fmt.Errorf("invalid SearXNG base_url: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewSearxngProvider builds a SearXNG provider from tenant parameters.
|
||||
func NewSearxngProvider(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) {
|
||||
base := strings.TrimSpace(params.BaseURL)
|
||||
if err := ValidateSearxngBaseURL(base); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client, err := NewSearchHTTPClient(defaultSearxngTimeout, params.ProxyURL)
|
||||
@@ -82,8 +94,11 @@ func (p *SearxngProvider) Search(
|
||||
q := url.Values{}
|
||||
q.Set("q", query)
|
||||
q.Set("format", "json")
|
||||
q.Set("safesearch", "1")
|
||||
q.Set("language", "auto")
|
||||
// Use "all" (SearXNG's documented value for "no language filter") instead
|
||||
// of "auto", which is a UI-side default and not a valid /search parameter.
|
||||
// safesearch is intentionally not set here so the value configured in the
|
||||
// instance's settings.yml is honored.
|
||||
q.Set("language", "all")
|
||||
|
||||
reqURL := p.baseURL + "/search?" + q.Encode()
|
||||
logger.Infof(ctx, "[WebSearch][SearXNG] query=%q maxResults=%d url=%s", query, maxResults, p.baseURL)
|
||||
@@ -131,16 +146,46 @@ func (p *SearxngProvider) Search(
|
||||
Source: "searxng",
|
||||
}
|
||||
if includeDate && r.PublishedDate != "" {
|
||||
if t, err := time.Parse(time.RFC3339, r.PublishedDate); err == nil {
|
||||
if t, ok := parseSearxngDate(r.PublishedDate); ok {
|
||||
item.PublishedAt = &t
|
||||
} else {
|
||||
logger.Debugf(ctx, "[WebSearch][SearXNG] unparsable publishedDate=%q", r.PublishedDate)
|
||||
}
|
||||
}
|
||||
results = append(results, item)
|
||||
}
|
||||
if len(results) == 0 && len(data.UnresponsiveEngines) > 0 {
|
||||
logger.Warnf(ctx, "[WebSearch][SearXNG] empty results, unresponsive_engines=%v", data.UnresponsiveEngines)
|
||||
}
|
||||
logger.Infof(ctx, "[WebSearch][SearXNG] returned %d results", len(results))
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// searxngDateLayouts covers the formats different SearXNG engines emit for
|
||||
// publishedDate. Order matters only for performance; first match wins.
|
||||
var searxngDateLayouts = []string{
|
||||
time.RFC3339,
|
||||
time.RFC3339Nano,
|
||||
"2006-01-02T15:04:05",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02",
|
||||
time.RFC1123Z,
|
||||
time.RFC1123,
|
||||
}
|
||||
|
||||
func parseSearxngDate(s string) (time.Time, bool) {
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return time.Time{}, false
|
||||
}
|
||||
for _, layout := range searxngDateLayouts {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
type searxngResponse struct {
|
||||
Query string `json:"query"`
|
||||
Results []struct {
|
||||
@@ -149,4 +194,8 @@ type searxngResponse struct {
|
||||
Content string `json:"content"`
|
||||
PublishedDate string `json:"publishedDate,omitempty"`
|
||||
} `json:"results"`
|
||||
// UnresponsiveEngines is a list of [engine, reason] tuples returned by
|
||||
// SearXNG when one or more upstream engines fail. We surface it on empty
|
||||
// result sets to aid debugging.
|
||||
UnresponsiveEngines [][]string `json:"unresponsive_engines,omitempty"`
|
||||
}
|
||||
|
||||
@@ -880,9 +880,21 @@ func loadSSRFWhitelist() *ssrfWhitelistConfig {
|
||||
exactHosts: make(map[string]bool),
|
||||
}
|
||||
raw := os.Getenv("SSRF_WHITELIST")
|
||||
if raw == "" {
|
||||
// SSRF_WHITELIST_EXTRA is merged in addition to SSRF_WHITELIST so that
|
||||
// deployment-managed defaults (e.g. docker-compose injected sidecar host
|
||||
// names like "searxng") aren't accidentally clobbered when an operator
|
||||
// overrides SSRF_WHITELIST in their .env.
|
||||
extra := os.Getenv("SSRF_WHITELIST_EXTRA")
|
||||
if raw == "" && extra == "" {
|
||||
return
|
||||
}
|
||||
if extra != "" {
|
||||
if raw == "" {
|
||||
raw = extra
|
||||
} else {
|
||||
raw = raw + "," + extra
|
||||
}
|
||||
}
|
||||
for _, entry := range strings.Split(raw, ",") {
|
||||
entry = strings.TrimSpace(entry)
|
||||
if entry == "" {
|
||||
|
||||
Reference in New Issue
Block a user