fix(web-search): address review fixups for SearXNG provider

- utils: 合并 SSRF_WHITELIST 与 SSRF_WHITELIST_EXTRA,避免部署侧默认值
  (如 docker-compose 注入的 searxng 主机名)被用户的 SSRF_WHITELIST
  自定义值覆盖。
- docker-compose.yml: 把 searxng 默认值挪到 SSRF_WHITELIST_EXTRA。
- searxng: 抽出导出函数 ValidateSearxngBaseURL,让 service 层保存校验
  和 provider 构造校验完全一致;service 改为调用同一函数。
- searxng: language 由非法的 "auto" 改为 "all";移除强制 safesearch=1
  让实例 settings.yml 决定。
- searxng: publishedDate 增加多格式 fallback(RFC3339Nano/无时区/
  日期-only/RFC1123 等),无法解析时 debug 日志记录。
- searxng: 解析 unresponsive_engines;结果为空时打 warn 日志,便于排查
  "配置正确却搜不到结果"的情况。
- frontend: WebSearchSettings 的 Instance URL 标签/占位符走 i18n,
  zh-CN / en-US / ru-RU / ko-KR 四个 locale 补齐 baseUrlLabel /
  baseUrlPlaceholder。
This commit is contained in:
wizardchen
2026-05-10 22:21:42 +08:00
committed by lyingbug
parent 1110615300
commit d2a1006beb
9 changed files with 90 additions and 21 deletions
+4 -1
View File
@@ -134,7 +134,10 @@ services:
- NEO4J_PASSWORD=${NEO4J_PASSWORD:-password}
- TENANT_AES_KEY=${TENANT_AES_KEY:-}
- SYSTEM_AES_KEY=${SYSTEM_AES_KEY:-}
- SSRF_WHITELIST=${SSRF_WHITELIST:-searxng}
- SSRF_WHITELIST=${SSRF_WHITELIST:-}
# Always allow the optional searxng sidecar (compose service hostname);
# merged on top of SSRF_WHITELIST so user overrides don't clobber it.
- SSRF_WHITELIST_EXTRA=${SSRF_WHITELIST_EXTRA:-searxng}
- CONCURRENCY_POOL_SIZE=${CONCURRENCY_POOL_SIZE:-5}
- JWT_SECRET=${JWT_SECRET:-}
# Crypto: 主密钥和盐值,用于 AppSecret 等敏感字段的 AES-256 加密
+2
View File
@@ -824,6 +824,8 @@ export default {
proxyUrlPlaceholder: 'e.g. http://proxy.example.com:3128 (optional; http/https only)',
proxyUrlHelp: 'Use when outbound access to the search API requires a proxy; leave empty to rely on HTTP_PROXY/HTTPS_PROXY environment variables.',
apiKeyLabel: 'API Key',
baseUrlLabel: 'Instance URL',
baseUrlPlaceholder: 'https://searxng.example.com',
apiKeyDescription: 'Enter the API key for the selected search provider',
apiKeyPlaceholder: 'Enter API key',
maxResultsLabel: 'Maximum Results',
+2
View File
@@ -702,6 +702,8 @@ export default {
proxyUrlPlaceholder: "예: http://127.0.0.1:7890 (선택 사항)",
proxyUrlHelp: "검색 API 접근에 프록시가 필요한 경우 입력합니다. 비우면 HTTP_PROXY/HTTPS_PROXY를 사용합니다.",
apiKeyLabel: "API 키",
baseUrlLabel: "인스턴스 URL",
baseUrlPlaceholder: "https://searxng.example.com",
apiKeyDescription: "선택한 검색 엔진의 API 키 입력",
apiKeyPlaceholder: "API 키를 입력하세요",
maxResultsLabel: "최대 결과 수",
+2
View File
@@ -807,6 +807,8 @@ export default {
proxyUrlPlaceholder: 'Напр. http://127.0.0.1:7890 (необязательно)',
proxyUrlHelp: 'Укажите, если доступ к API поиска нужен через прокси; иначе используются переменные HTTP_PROXY/HTTPS_PROXY.',
apiKeyLabel: 'API-ключ',
baseUrlLabel: 'URL экземпляра',
baseUrlPlaceholder: 'https://searxng.example.com',
apiKeyDescription: 'Введите API-ключ выбранного провайдера поиска',
apiKeyPlaceholder: 'Введите API-ключ',
maxResultsLabel: 'Максимум результатов',
+2
View File
@@ -701,6 +701,8 @@ export default {
proxyUrlPlaceholder: "例如 http://proxy.example.com:3128(可选,仅支持 http/https)",
proxyUrlHelp: "若环境访问搜索 API 需代理,在此填写;留空则使用系统环境变量 HTTP(S)_PROXY。",
apiKeyLabel: "API 密钥",
baseUrlLabel: "实例地址",
baseUrlPlaceholder: "https://searxng.example.com",
apiKeyDescription: "输入所选搜索引擎的 API 密钥",
apiKeyPlaceholder: "请输入 API 密钥",
maxResultsLabel: "最大结果数",
@@ -92,10 +92,10 @@
</a>
</div>
<t-form-item v-if="selectedProviderType?.requires_base_url" label="Instance URL" name="parameters.base_url">
<t-form-item v-if="selectedProviderType?.requires_base_url" :label="t('webSearchSettings.baseUrlLabel')" name="parameters.base_url">
<t-input
v-model="providerForm.parameters.base_url"
placeholder="https://searxng.example.com"
:placeholder="t('webSearchSettings.baseUrlPlaceholder')"
/>
</t-form-item>
<t-form-item v-if="selectedProviderType?.requires_api_key" :label="t('webSearchSettings.apiKeyLabel')" name="parameters.api_key">
@@ -122,11 +122,8 @@ func validateProviderParameters(provider types.WebSearchProviderType, params typ
case types.WebSearchProviderTypeDuckDuckGo:
// No API key required
case types.WebSearchProviderTypeSearxng:
if params.BaseURL == "" {
return fmt.Errorf("base_url is required for SearXNG provider")
}
if err := infra_web_search.ValidateProxyURL(params.BaseURL); err != nil {
return fmt.Errorf("invalid SearXNG base_url: %w", err)
if err := infra_web_search.ValidateSearxngBaseURL(params.BaseURL); err != nil {
return err
}
}
if err := validateOptionalProxyURL(params.ProxyURL); err != nil {
+61 -12
View File
@@ -33,21 +33,33 @@ type SearxngProvider struct {
apiKey string
}
// NewSearxngProvider builds a SearXNG provider from tenant parameters.
func NewSearxngProvider(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) {
base := strings.TrimSpace(params.BaseURL)
// ValidateSearxngBaseURL validates a SearXNG instance URL: must be a non-empty,
// absolute http(s) URL, and must pass the SSRF whitelist check. Shared between
// the service-layer parameter validation and the provider constructor so that
// "save" and "use" never disagree.
func ValidateSearxngBaseURL(rawURL string) error {
base := strings.TrimSpace(rawURL)
if base == "" {
return nil, fmt.Errorf("base_url is required for SearXNG provider")
}
if err := utils.ValidateURLForSSRF(base); err != nil {
return nil, fmt.Errorf("invalid SearXNG base_url: %w", err)
return fmt.Errorf("base_url is required for SearXNG provider")
}
parsed, err := url.Parse(base)
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
return nil, fmt.Errorf("invalid SearXNG base_url: must be an absolute http(s) URL")
return fmt.Errorf("invalid SearXNG base_url: must be an absolute http(s) URL")
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return nil, fmt.Errorf("invalid SearXNG base_url scheme: %s", parsed.Scheme)
return fmt.Errorf("invalid SearXNG base_url scheme: %s", parsed.Scheme)
}
if err := utils.ValidateURLForSSRF(base); err != nil {
return fmt.Errorf("invalid SearXNG base_url: %w", err)
}
return nil
}
// NewSearxngProvider builds a SearXNG provider from tenant parameters.
func NewSearxngProvider(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) {
base := strings.TrimSpace(params.BaseURL)
if err := ValidateSearxngBaseURL(base); err != nil {
return nil, err
}
client, err := NewSearchHTTPClient(defaultSearxngTimeout, params.ProxyURL)
@@ -82,8 +94,11 @@ func (p *SearxngProvider) Search(
q := url.Values{}
q.Set("q", query)
q.Set("format", "json")
q.Set("safesearch", "1")
q.Set("language", "auto")
// Use "all" (SearXNG's documented value for "no language filter") instead
// of "auto", which is a UI-side default and not a valid /search parameter.
// safesearch is intentionally not set here so the value configured in the
// instance's settings.yml is honored.
q.Set("language", "all")
reqURL := p.baseURL + "/search?" + q.Encode()
logger.Infof(ctx, "[WebSearch][SearXNG] query=%q maxResults=%d url=%s", query, maxResults, p.baseURL)
@@ -131,16 +146,46 @@ func (p *SearxngProvider) Search(
Source: "searxng",
}
if includeDate && r.PublishedDate != "" {
if t, err := time.Parse(time.RFC3339, r.PublishedDate); err == nil {
if t, ok := parseSearxngDate(r.PublishedDate); ok {
item.PublishedAt = &t
} else {
logger.Debugf(ctx, "[WebSearch][SearXNG] unparsable publishedDate=%q", r.PublishedDate)
}
}
results = append(results, item)
}
if len(results) == 0 && len(data.UnresponsiveEngines) > 0 {
logger.Warnf(ctx, "[WebSearch][SearXNG] empty results, unresponsive_engines=%v", data.UnresponsiveEngines)
}
logger.Infof(ctx, "[WebSearch][SearXNG] returned %d results", len(results))
return results, nil
}
// searxngDateLayouts covers the formats different SearXNG engines emit for
// publishedDate. Order matters only for performance; first match wins.
var searxngDateLayouts = []string{
time.RFC3339,
time.RFC3339Nano,
"2006-01-02T15:04:05",
"2006-01-02 15:04:05",
"2006-01-02",
time.RFC1123Z,
time.RFC1123,
}
func parseSearxngDate(s string) (time.Time, bool) {
s = strings.TrimSpace(s)
if s == "" {
return time.Time{}, false
}
for _, layout := range searxngDateLayouts {
if t, err := time.Parse(layout, s); err == nil {
return t, true
}
}
return time.Time{}, false
}
type searxngResponse struct {
Query string `json:"query"`
Results []struct {
@@ -149,4 +194,8 @@ type searxngResponse struct {
Content string `json:"content"`
PublishedDate string `json:"publishedDate,omitempty"`
} `json:"results"`
// UnresponsiveEngines is a list of [engine, reason] tuples returned by
// SearXNG when one or more upstream engines fail. We surface it on empty
// result sets to aid debugging.
UnresponsiveEngines [][]string `json:"unresponsive_engines,omitempty"`
}
+13 -1
View File
@@ -880,9 +880,21 @@ func loadSSRFWhitelist() *ssrfWhitelistConfig {
exactHosts: make(map[string]bool),
}
raw := os.Getenv("SSRF_WHITELIST")
if raw == "" {
// SSRF_WHITELIST_EXTRA is merged in addition to SSRF_WHITELIST so that
// deployment-managed defaults (e.g. docker-compose injected sidecar host
// names like "searxng") aren't accidentally clobbered when an operator
// overrides SSRF_WHITELIST in their .env.
extra := os.Getenv("SSRF_WHITELIST_EXTRA")
if raw == "" && extra == "" {
return
}
if extra != "" {
if raw == "" {
raw = extra
} else {
raw = raw + "," + extra
}
}
for _, entry := range strings.Split(raw, ",") {
entry = strings.TrimSpace(entry)
if entry == "" {