feat(searxng): enhance SearXNG configuration and validation
- Updated .env.example to clarify SEARXNG_SECRET generation and added SSRF_WHITELIST_EXTRA for improved security. - Modified docker-compose files to bind SearXNG to localhost by default and introduced a one-time initialization service to set up settings.yml correctly. - Enhanced SearxngProvider with stricter URL validation, ensuring no query or fragment is present in the base URL. - Added unit tests for SearXNG validation and date parsing to ensure robustness. - Updated frontend WebSearchSettings to reflect changes in SearXNG instance URL handling. This commit improves the security and usability of the SearXNG integration, addressing potential misconfigurations and enhancing the developer experience.
This commit is contained in:
+18
-4
@@ -204,11 +204,25 @@ SYSTEM_AES_KEY=weknora-system-aes-key-32bytes!!
|
||||
# SSRF_WHITELIST=internal.service,*.corp.example,172.16.0.0/12,2001:db8::1,fd00::/8
|
||||
|
||||
# ===== SearXNG(可选自建网络搜索)=====
|
||||
# 启用:docker compose --profile searxng up -d
|
||||
# 默认对外端口 8888,容器内主机名 searxng:8080;后端通过 docker network 直接访问。
|
||||
# WeKnora 控制台中 Provider 类型选 "SearXNG",Instance URL 填 http://searxng:8080 即可。
|
||||
# 启用:先生成强随机 SEARXNG_SECRET,再 `docker compose --profile searxng up -d`。
|
||||
# openssl rand -hex 32
|
||||
# 没有 SEARXNG_SECRET 时 docker compose 会主动报错,避免使用公开默认值。
|
||||
#
|
||||
# 后端访问方式(容器化部署):控制台中 Provider 类型选 "SearXNG",
|
||||
# Instance URL 填 http://searxng:8080 ;docker compose 已默认把 `searxng` 主机名
|
||||
# 注入 SSRF_WHITELIST_EXTRA,无需额外配置。
|
||||
#
|
||||
# 本地开发(go run + docker compose -f docker-compose.dev.yml --profile searxng up):
|
||||
# 后端跑在宿主机,需走 published 端口;Instance URL 填 http://127.0.0.1:8888 ,
|
||||
# 并把 127.0.0.1 加入下方 SSRF_WHITELIST:
|
||||
# SSRF_WHITELIST=127.0.0.1
|
||||
#
|
||||
# 端口默认仅监听 127.0.0.1,避免把开了 limiter:false 的实例暴露到 LAN。
|
||||
# 如需对内网开放,显式覆盖 SEARXNG_BIND=0.0.0.0 并自行加固。
|
||||
# SEARXNG_PORT=8888
|
||||
# SEARXNG_SECRET=please-change-me-to-a-long-random-string # openssl rand -hex 32
|
||||
# SEARXNG_BIND=127.0.0.1
|
||||
# SEARXNG_SECRET=
|
||||
# SSRF_WHITELIST_EXTRA=searxng
|
||||
|
||||
# 是否开启知识图谱构建和检索(构建阶段需调用大模型,耗时较长)
|
||||
ENABLE_GRAPH_RAG=false
|
||||
|
||||
+25
-3
@@ -35,17 +35,35 @@ services:
|
||||
networks:
|
||||
- WeKnora-network-dev
|
||||
|
||||
# See docker-compose.yml searxng-init for rationale (avoid in-place rewrite of
|
||||
# the repo-tracked settings.yml by the searxng entrypoint).
|
||||
searxng-init:
|
||||
image: busybox:1.36
|
||||
container_name: WeKnora-searxng-init-dev
|
||||
command: ["sh", "-c", "cp /template/settings.yml /etc/searxng/settings.yml && chmod 0644 /etc/searxng/settings.yml"]
|
||||
volumes:
|
||||
- ./docker/searxng/settings.yml:/template/settings.yml:ro
|
||||
- searxng_config_dev:/etc/searxng
|
||||
restart: "no"
|
||||
networks:
|
||||
- WeKnora-network-dev
|
||||
profiles:
|
||||
- searxng
|
||||
- full
|
||||
|
||||
searxng:
|
||||
image: searxng/searxng:latest
|
||||
container_name: WeKnora-searxng-dev
|
||||
# Local dev: bind to loopback by default; the host-side WeKnora process
|
||||
# reaches it at http://127.0.0.1:${SEARXNG_PORT}.
|
||||
ports:
|
||||
- "${SEARXNG_PORT:-8888}:8080"
|
||||
- "${SEARXNG_BIND:-127.0.0.1}:${SEARXNG_PORT:-8888}:8080"
|
||||
volumes:
|
||||
- ./docker/searxng/settings.yml:/etc/searxng/settings.yml:rw
|
||||
- searxng_config_dev:/etc/searxng
|
||||
environment:
|
||||
- SEARXNG_BASE_URL=http://localhost:${SEARXNG_PORT:-8888}/
|
||||
- INSTANCE_NAME=weknora-searxng-dev
|
||||
- SEARXNG_SECRET=${SEARXNG_SECRET:-please-change-me-to-a-long-random-string}
|
||||
- SEARXNG_SECRET=${SEARXNG_SECRET:?SEARXNG_SECRET must be set (e.g. openssl rand -hex 32) before starting the searxng profile}
|
||||
cap_drop:
|
||||
- ALL
|
||||
cap_add:
|
||||
@@ -53,6 +71,9 @@ services:
|
||||
- SETGID
|
||||
- SETUID
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
searxng-init:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- WeKnora-network-dev
|
||||
profiles:
|
||||
@@ -445,3 +466,4 @@ volumes:
|
||||
langfuse_clickhouse_data_dev:
|
||||
langfuse_clickhouse_logs_dev:
|
||||
langfuse_minio_data_dev:
|
||||
searxng_config_dev:
|
||||
|
||||
+28
-3
@@ -241,17 +241,38 @@ services:
|
||||
networks:
|
||||
- WeKnora-network
|
||||
|
||||
# One-shot init: copy the read-only template settings.yml into a named volume
|
||||
# before searxng starts. Necessary because the searxng entrypoint sed-replaces
|
||||
# `ultrasecretkey` in /etc/searxng/settings.yml in place; bind-mounting the
|
||||
# repo file directly would write the resolved SEARXNG_SECRET back into the
|
||||
# working tree (and prevent re-templating on subsequent restarts).
|
||||
searxng-init:
|
||||
image: busybox:1.36
|
||||
container_name: WeKnora-searxng-init
|
||||
command: ["sh", "-c", "cp /template/settings.yml /etc/searxng/settings.yml && chmod 0644 /etc/searxng/settings.yml"]
|
||||
volumes:
|
||||
- ./docker/searxng/settings.yml:/template/settings.yml:ro
|
||||
- searxng_config:/etc/searxng
|
||||
restart: "no"
|
||||
networks:
|
||||
- WeKnora-network
|
||||
profiles:
|
||||
- searxng
|
||||
- full
|
||||
|
||||
searxng:
|
||||
image: searxng/searxng:latest
|
||||
container_name: WeKnora-searxng
|
||||
# Bind to loopback by default; override SEARXNG_BIND=0.0.0.0 only after
|
||||
# rotating SEARXNG_SECRET and (optionally) re-enabling limiter in settings.yml.
|
||||
ports:
|
||||
- "${SEARXNG_PORT:-8888}:8080"
|
||||
- "${SEARXNG_BIND:-127.0.0.1}:${SEARXNG_PORT:-8888}:8080"
|
||||
volumes:
|
||||
- ./docker/searxng/settings.yml:/etc/searxng/settings.yml:rw
|
||||
- searxng_config:/etc/searxng
|
||||
environment:
|
||||
- SEARXNG_BASE_URL=http://localhost:${SEARXNG_PORT:-8888}/
|
||||
- INSTANCE_NAME=weknora-searxng
|
||||
- SEARXNG_SECRET=${SEARXNG_SECRET:-please-change-me-to-a-long-random-string}
|
||||
- SEARXNG_SECRET=${SEARXNG_SECRET:?SEARXNG_SECRET must be set (e.g. openssl rand -hex 32) before starting the searxng profile}
|
||||
cap_drop:
|
||||
- ALL
|
||||
cap_add:
|
||||
@@ -259,6 +280,9 @@ services:
|
||||
- SETGID
|
||||
- SETUID
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
searxng-init:
|
||||
condition: service_completed_successfully
|
||||
networks:
|
||||
- WeKnora-network
|
||||
profiles:
|
||||
@@ -707,3 +731,4 @@ volumes:
|
||||
langfuse_clickhouse_data:
|
||||
langfuse_clickhouse_logs:
|
||||
langfuse_minio_data:
|
||||
searxng_config:
|
||||
|
||||
@@ -199,7 +199,10 @@ const selectedProviderType = computed(() => {
|
||||
})
|
||||
|
||||
const isProviderFree = (providerType: WebSearchProviderTypeInfo) => {
|
||||
return !providerType.requires_api_key && !providerType.requires_engine_id && !providerType.requires_base_url
|
||||
// "Free" here means no upstream-paid credentials are required. Self-hosted
|
||||
// providers (requires_base_url) are still free to use even though they need
|
||||
// an instance URL, so they should keep the free badge.
|
||||
return !providerType.requires_api_key && !providerType.requires_engine_id
|
||||
}
|
||||
|
||||
const isEntityFree = (entity: WebSearchProviderEntity) => {
|
||||
|
||||
@@ -16,7 +16,10 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
const defaultSearxngTimeout = 15 * time.Second
|
||||
// defaultSearxngTimeout is sized slightly above the SearXNG image's default
|
||||
// outgoing.max_request_timeout (10s in docker/searxng/settings.yml) so a slow
|
||||
// upstream engine surfaces as a SearXNG-side error instead of a client cancel.
|
||||
const defaultSearxngTimeout = 12 * time.Second
|
||||
|
||||
// SearxngProvider implements web search using a self-hosted SearXNG instance.
|
||||
//
|
||||
@@ -49,6 +52,9 @@ func ValidateSearxngBaseURL(rawURL string) error {
|
||||
if parsed.Scheme != "http" && parsed.Scheme != "https" {
|
||||
return fmt.Errorf("invalid SearXNG base_url scheme: %s", parsed.Scheme)
|
||||
}
|
||||
if parsed.RawQuery != "" || parsed.Fragment != "" {
|
||||
return fmt.Errorf("invalid SearXNG base_url: must not contain query or fragment")
|
||||
}
|
||||
if err := utils.ValidateURLForSSRF(base); err != nil {
|
||||
return fmt.Errorf("invalid SearXNG base_url: %w", err)
|
||||
}
|
||||
@@ -163,9 +169,10 @@ func (p *SearxngProvider) Search(
|
||||
|
||||
// searxngDateLayouts covers the formats different SearXNG engines emit for
|
||||
// publishedDate. Order matters only for performance; first match wins.
|
||||
// time.RFC3339 already accepts the nanosecond-precision form, so RFC3339Nano
|
||||
// is intentionally omitted.
|
||||
var searxngDateLayouts = []string{
|
||||
time.RFC3339,
|
||||
time.RFC3339Nano,
|
||||
"2006-01-02T15:04:05",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02",
|
||||
@@ -187,7 +194,6 @@ func parseSearxngDate(s string) (time.Time, bool) {
|
||||
}
|
||||
|
||||
type searxngResponse struct {
|
||||
Query string `json:"query"`
|
||||
Results []struct {
|
||||
Title string `json:"title"`
|
||||
URL string `json:"url"`
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package web_search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestValidateSearxngBaseURL(t *testing.T) {
|
||||
os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost")
|
||||
defer os.Unsetenv("SSRF_WHITELIST")
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
url string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "empty", url: "", wantErr: true},
|
||||
{name: "no scheme", url: "searxng:8080", wantErr: true},
|
||||
{name: "bad scheme", url: "ftp://searxng:8080", wantErr: true},
|
||||
{name: "with query", url: "http://127.0.0.1:8080/?x=1", wantErr: true},
|
||||
{name: "with fragment", url: "http://127.0.0.1:8080/#frag", wantErr: true},
|
||||
{name: "loopback ok via whitelist", url: "http://127.0.0.1:8888", wantErr: false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := ValidateSearxngBaseURL(tc.url)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("ValidateSearxngBaseURL(%q) err=%v wantErr=%v", tc.url, err, tc.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseSearxngDate(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
ok bool
|
||||
}{
|
||||
{"", false},
|
||||
{"2024-05-01", true},
|
||||
{"2024-05-01T12:30:45", true},
|
||||
{"2024-05-01T12:30:45Z", true},
|
||||
{"2024-05-01T12:30:45.123456789Z", true},
|
||||
{"2024-05-01 12:30:45", true},
|
||||
{"Wed, 01 May 2024 12:30:45 GMT", true},
|
||||
{"not-a-date", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.in, func(t *testing.T) {
|
||||
_, ok := parseSearxngDate(tc.in)
|
||||
if ok != tc.ok {
|
||||
t.Fatalf("parseSearxngDate(%q) ok=%v want=%v", tc.in, ok, tc.ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearxngProvider_Search(t *testing.T) {
|
||||
os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost")
|
||||
defer os.Unsetenv("SSRF_WHITELIST")
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/search" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if got := r.URL.Query().Get("format"); got != "json" {
|
||||
t.Errorf("expected format=json, got %q", got)
|
||||
}
|
||||
if got := r.URL.Query().Get("q"); got != "hello" {
|
||||
t.Errorf("expected q=hello, got %q", got)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"results": []map[string]any{
|
||||
{"title": "T1", "url": "https://e/1", "content": "c1", "publishedDate": "2024-05-01"},
|
||||
{"title": "", "url": "https://e/skip"},
|
||||
{"title": "T2", "url": "https://e/2", "content": "c2"},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
provider, err := NewSearxngProvider(types.WebSearchProviderParameters{BaseURL: srv.URL})
|
||||
if err != nil {
|
||||
t.Fatalf("NewSearxngProvider: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
results, err := provider.Search(ctx, "hello", 5, true)
|
||||
if err != nil {
|
||||
t.Fatalf("Search: %v", err)
|
||||
}
|
||||
if len(results) != 2 {
|
||||
t.Fatalf("expected 2 results (one skipped), got %d", len(results))
|
||||
}
|
||||
if results[0].PublishedAt == nil {
|
||||
t.Fatalf("expected first result PublishedAt to be set")
|
||||
}
|
||||
if got := results[0].Source; got != "searxng" {
|
||||
t.Fatalf("unexpected source: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -212,6 +212,43 @@ func TestValidateURLForSSRF_IPv6Whitelist(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSSRFWhitelistExtraMerge verifies that SSRF_WHITELIST_EXTRA is merged
|
||||
// into the effective whitelist alongside SSRF_WHITELIST, so deployment-managed
|
||||
// defaults (e.g. docker-compose injected sidecar host names) survive when an
|
||||
// operator overrides SSRF_WHITELIST in their .env.
|
||||
func TestSSRFWhitelistExtraMerge(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
main string
|
||||
extra string
|
||||
host string
|
||||
want bool
|
||||
}{
|
||||
{name: "extra only", main: "", extra: "searxng", host: "searxng", want: true},
|
||||
{name: "main only does not match extra host", main: "internal", extra: "", host: "searxng", want: false},
|
||||
{name: "both merged - main host", main: "internal", extra: "searxng", host: "internal", want: true},
|
||||
{name: "both merged - extra host", main: "internal", extra: "searxng", host: "searxng", want: true},
|
||||
{name: "neither matches unrelated host", main: "internal", extra: "searxng", host: "evil.example", want: false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resetSSRFWhitelistForTest()
|
||||
os.Setenv("SSRF_WHITELIST", tc.main)
|
||||
os.Setenv("SSRF_WHITELIST_EXTRA", tc.extra)
|
||||
defer func() {
|
||||
os.Unsetenv("SSRF_WHITELIST")
|
||||
os.Unsetenv("SSRF_WHITELIST_EXTRA")
|
||||
resetSSRFWhitelistForTest()
|
||||
}()
|
||||
if got := IsSSRFWhitelisted(tc.host); got != tc.want {
|
||||
t.Fatalf("IsSSRFWhitelisted(%q) main=%q extra=%q = %v, want %v",
|
||||
tc.host, tc.main, tc.extra, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsRestrictedIP_IPv6 tests IPv6-specific restricted range detection.
|
||||
func TestIsRestrictedIP_IPv6(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
Reference in New Issue
Block a user