From 0f5dc41f4e78fcfa752dfeb2d08de94f5f893773 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Sun, 10 May 2026 22:51:24 +0800 Subject: [PATCH] feat(searxng): enhance SearXNG configuration and validation - Updated .env.example to clarify SEARXNG_SECRET generation and added SSRF_WHITELIST_EXTRA for improved security. - Modified docker-compose files to bind SearXNG to localhost by default and introduced a one-time initialization service to set up settings.yml correctly. - Enhanced SearxngProvider with stricter URL validation, ensuring no query or fragment is present in the base URL. - Added unit tests for SearXNG validation and date parsing to ensure robustness. - Updated frontend WebSearchSettings to reflect changes in SearXNG instance URL handling. This commit improves the security and usability of the SearXNG integration, addressing potential misconfigurations and enhancing the developer experience. --- .env.example | 22 +++- docker-compose.dev.yml | 28 ++++- docker-compose.yml | 31 ++++- .../src/views/settings/WebSearchSettings.vue | 5 +- internal/infrastructure/web_search/searxng.go | 12 +- .../infrastructure/web_search/searxng_test.go | 111 ++++++++++++++++++ internal/utils/security_test.go | 37 ++++++ 7 files changed, 232 insertions(+), 14 deletions(-) create mode 100644 internal/infrastructure/web_search/searxng_test.go diff --git a/.env.example b/.env.example index 6dfeed5b..48a3d5d9 100644 --- a/.env.example +++ b/.env.example @@ -204,11 +204,25 @@ SYSTEM_AES_KEY=weknora-system-aes-key-32bytes!! # SSRF_WHITELIST=internal.service,*.corp.example,172.16.0.0/12,2001:db8::1,fd00::/8 # ===== SearXNG(可选自建网络搜索)===== -# 启用:docker compose --profile searxng up -d -# 默认对外端口 8888,容器内主机名 searxng:8080;后端通过 docker network 直接访问。 -# WeKnora 控制台中 Provider 类型选 "SearXNG",Instance URL 填 http://searxng:8080 即可。 +# 启用:先生成强随机 SEARXNG_SECRET,再 `docker compose --profile searxng up -d`。 +# openssl rand -hex 32 +# 没有 SEARXNG_SECRET 时 docker compose 会主动报错,避免使用公开默认值。 +# +# 后端访问方式(容器化部署):控制台中 Provider 类型选 "SearXNG", +# Instance URL 填 http://searxng:8080 ;docker compose 已默认把 `searxng` 主机名 +# 注入 SSRF_WHITELIST_EXTRA,无需额外配置。 +# +# 本地开发(go run + docker compose -f docker-compose.dev.yml --profile searxng up): +# 后端跑在宿主机,需走 published 端口;Instance URL 填 http://127.0.0.1:8888 , +# 并把 127.0.0.1 加入下方 SSRF_WHITELIST: +# SSRF_WHITELIST=127.0.0.1 +# +# 端口默认仅监听 127.0.0.1,避免把开了 limiter:false 的实例暴露到 LAN。 +# 如需对内网开放,显式覆盖 SEARXNG_BIND=0.0.0.0 并自行加固。 # SEARXNG_PORT=8888 -# SEARXNG_SECRET=please-change-me-to-a-long-random-string # openssl rand -hex 32 +# SEARXNG_BIND=127.0.0.1 +# SEARXNG_SECRET= +# SSRF_WHITELIST_EXTRA=searxng # 是否开启知识图谱构建和检索(构建阶段需调用大模型,耗时较长) ENABLE_GRAPH_RAG=false diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8b210d14..c1ecfdb9 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -35,17 +35,35 @@ services: networks: - WeKnora-network-dev + # See docker-compose.yml searxng-init for rationale (avoid in-place rewrite of + # the repo-tracked settings.yml by the searxng entrypoint). + searxng-init: + image: busybox:1.36 + container_name: WeKnora-searxng-init-dev + command: ["sh", "-c", "cp /template/settings.yml /etc/searxng/settings.yml && chmod 0644 /etc/searxng/settings.yml"] + volumes: + - ./docker/searxng/settings.yml:/template/settings.yml:ro + - searxng_config_dev:/etc/searxng + restart: "no" + networks: + - WeKnora-network-dev + profiles: + - searxng + - full + searxng: image: searxng/searxng:latest container_name: WeKnora-searxng-dev + # Local dev: bind to loopback by default; the host-side WeKnora process + # reaches it at http://127.0.0.1:${SEARXNG_PORT}. ports: - - "${SEARXNG_PORT:-8888}:8080" + - "${SEARXNG_BIND:-127.0.0.1}:${SEARXNG_PORT:-8888}:8080" volumes: - - ./docker/searxng/settings.yml:/etc/searxng/settings.yml:rw + - searxng_config_dev:/etc/searxng environment: - SEARXNG_BASE_URL=http://localhost:${SEARXNG_PORT:-8888}/ - INSTANCE_NAME=weknora-searxng-dev - - SEARXNG_SECRET=${SEARXNG_SECRET:-please-change-me-to-a-long-random-string} + - SEARXNG_SECRET=${SEARXNG_SECRET:?SEARXNG_SECRET must be set (e.g. openssl rand -hex 32) before starting the searxng profile} cap_drop: - ALL cap_add: @@ -53,6 +71,9 @@ services: - SETGID - SETUID restart: unless-stopped + depends_on: + searxng-init: + condition: service_completed_successfully networks: - WeKnora-network-dev profiles: @@ -445,3 +466,4 @@ volumes: langfuse_clickhouse_data_dev: langfuse_clickhouse_logs_dev: langfuse_minio_data_dev: + searxng_config_dev: diff --git a/docker-compose.yml b/docker-compose.yml index f0409292..75980639 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -241,17 +241,38 @@ services: networks: - WeKnora-network + # One-shot init: copy the read-only template settings.yml into a named volume + # before searxng starts. Necessary because the searxng entrypoint sed-replaces + # `ultrasecretkey` in /etc/searxng/settings.yml in place; bind-mounting the + # repo file directly would write the resolved SEARXNG_SECRET back into the + # working tree (and prevent re-templating on subsequent restarts). + searxng-init: + image: busybox:1.36 + container_name: WeKnora-searxng-init + command: ["sh", "-c", "cp /template/settings.yml /etc/searxng/settings.yml && chmod 0644 /etc/searxng/settings.yml"] + volumes: + - ./docker/searxng/settings.yml:/template/settings.yml:ro + - searxng_config:/etc/searxng + restart: "no" + networks: + - WeKnora-network + profiles: + - searxng + - full + searxng: image: searxng/searxng:latest container_name: WeKnora-searxng + # Bind to loopback by default; override SEARXNG_BIND=0.0.0.0 only after + # rotating SEARXNG_SECRET and (optionally) re-enabling limiter in settings.yml. ports: - - "${SEARXNG_PORT:-8888}:8080" + - "${SEARXNG_BIND:-127.0.0.1}:${SEARXNG_PORT:-8888}:8080" volumes: - - ./docker/searxng/settings.yml:/etc/searxng/settings.yml:rw + - searxng_config:/etc/searxng environment: - SEARXNG_BASE_URL=http://localhost:${SEARXNG_PORT:-8888}/ - INSTANCE_NAME=weknora-searxng - - SEARXNG_SECRET=${SEARXNG_SECRET:-please-change-me-to-a-long-random-string} + - SEARXNG_SECRET=${SEARXNG_SECRET:?SEARXNG_SECRET must be set (e.g. openssl rand -hex 32) before starting the searxng profile} cap_drop: - ALL cap_add: @@ -259,6 +280,9 @@ services: - SETGID - SETUID restart: unless-stopped + depends_on: + searxng-init: + condition: service_completed_successfully networks: - WeKnora-network profiles: @@ -707,3 +731,4 @@ volumes: langfuse_clickhouse_data: langfuse_clickhouse_logs: langfuse_minio_data: + searxng_config: diff --git a/frontend/src/views/settings/WebSearchSettings.vue b/frontend/src/views/settings/WebSearchSettings.vue index 2e42586b..e216a9cd 100644 --- a/frontend/src/views/settings/WebSearchSettings.vue +++ b/frontend/src/views/settings/WebSearchSettings.vue @@ -199,7 +199,10 @@ const selectedProviderType = computed(() => { }) const isProviderFree = (providerType: WebSearchProviderTypeInfo) => { - return !providerType.requires_api_key && !providerType.requires_engine_id && !providerType.requires_base_url + // "Free" here means no upstream-paid credentials are required. Self-hosted + // providers (requires_base_url) are still free to use even though they need + // an instance URL, so they should keep the free badge. + return !providerType.requires_api_key && !providerType.requires_engine_id } const isEntityFree = (entity: WebSearchProviderEntity) => { diff --git a/internal/infrastructure/web_search/searxng.go b/internal/infrastructure/web_search/searxng.go index 00ad1217..e240bf0b 100644 --- a/internal/infrastructure/web_search/searxng.go +++ b/internal/infrastructure/web_search/searxng.go @@ -16,7 +16,10 @@ import ( "github.com/Tencent/WeKnora/internal/utils" ) -const defaultSearxngTimeout = 15 * time.Second +// defaultSearxngTimeout is sized slightly above the SearXNG image's default +// outgoing.max_request_timeout (10s in docker/searxng/settings.yml) so a slow +// upstream engine surfaces as a SearXNG-side error instead of a client cancel. +const defaultSearxngTimeout = 12 * time.Second // SearxngProvider implements web search using a self-hosted SearXNG instance. // @@ -49,6 +52,9 @@ func ValidateSearxngBaseURL(rawURL string) error { if parsed.Scheme != "http" && parsed.Scheme != "https" { return fmt.Errorf("invalid SearXNG base_url scheme: %s", parsed.Scheme) } + if parsed.RawQuery != "" || parsed.Fragment != "" { + return fmt.Errorf("invalid SearXNG base_url: must not contain query or fragment") + } if err := utils.ValidateURLForSSRF(base); err != nil { return fmt.Errorf("invalid SearXNG base_url: %w", err) } @@ -163,9 +169,10 @@ func (p *SearxngProvider) Search( // searxngDateLayouts covers the formats different SearXNG engines emit for // publishedDate. Order matters only for performance; first match wins. +// time.RFC3339 already accepts the nanosecond-precision form, so RFC3339Nano +// is intentionally omitted. var searxngDateLayouts = []string{ time.RFC3339, - time.RFC3339Nano, "2006-01-02T15:04:05", "2006-01-02 15:04:05", "2006-01-02", @@ -187,7 +194,6 @@ func parseSearxngDate(s string) (time.Time, bool) { } type searxngResponse struct { - Query string `json:"query"` Results []struct { Title string `json:"title"` URL string `json:"url"` diff --git a/internal/infrastructure/web_search/searxng_test.go b/internal/infrastructure/web_search/searxng_test.go new file mode 100644 index 00000000..3ce18701 --- /dev/null +++ b/internal/infrastructure/web_search/searxng_test.go @@ -0,0 +1,111 @@ +package web_search + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/types" +) + +func TestValidateSearxngBaseURL(t *testing.T) { + os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost") + defer os.Unsetenv("SSRF_WHITELIST") + + cases := []struct { + name string + url string + wantErr bool + }{ + {name: "empty", url: "", wantErr: true}, + {name: "no scheme", url: "searxng:8080", wantErr: true}, + {name: "bad scheme", url: "ftp://searxng:8080", wantErr: true}, + {name: "with query", url: "http://127.0.0.1:8080/?x=1", wantErr: true}, + {name: "with fragment", url: "http://127.0.0.1:8080/#frag", wantErr: true}, + {name: "loopback ok via whitelist", url: "http://127.0.0.1:8888", wantErr: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateSearxngBaseURL(tc.url) + if (err != nil) != tc.wantErr { + t.Fatalf("ValidateSearxngBaseURL(%q) err=%v wantErr=%v", tc.url, err, tc.wantErr) + } + }) + } +} + +func TestParseSearxngDate(t *testing.T) { + cases := []struct { + in string + ok bool + }{ + {"", false}, + {"2024-05-01", true}, + {"2024-05-01T12:30:45", true}, + {"2024-05-01T12:30:45Z", true}, + {"2024-05-01T12:30:45.123456789Z", true}, + {"2024-05-01 12:30:45", true}, + {"Wed, 01 May 2024 12:30:45 GMT", true}, + {"not-a-date", false}, + } + for _, tc := range cases { + t.Run(tc.in, func(t *testing.T) { + _, ok := parseSearxngDate(tc.in) + if ok != tc.ok { + t.Fatalf("parseSearxngDate(%q) ok=%v want=%v", tc.in, ok, tc.ok) + } + }) + } +} + +func TestSearxngProvider_Search(t *testing.T) { + os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost") + defer os.Unsetenv("SSRF_WHITELIST") + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/search" { + http.NotFound(w, r) + return + } + if got := r.URL.Query().Get("format"); got != "json" { + t.Errorf("expected format=json, got %q", got) + } + if got := r.URL.Query().Get("q"); got != "hello" { + t.Errorf("expected q=hello, got %q", got) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "results": []map[string]any{ + {"title": "T1", "url": "https://e/1", "content": "c1", "publishedDate": "2024-05-01"}, + {"title": "", "url": "https://e/skip"}, + {"title": "T2", "url": "https://e/2", "content": "c2"}, + }, + }) + })) + defer srv.Close() + + provider, err := NewSearxngProvider(types.WebSearchProviderParameters{BaseURL: srv.URL}) + if err != nil { + t.Fatalf("NewSearxngProvider: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + results, err := provider.Search(ctx, "hello", 5, true) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(results) != 2 { + t.Fatalf("expected 2 results (one skipped), got %d", len(results)) + } + if results[0].PublishedAt == nil { + t.Fatalf("expected first result PublishedAt to be set") + } + if got := results[0].Source; got != "searxng" { + t.Fatalf("unexpected source: %q", got) + } +} diff --git a/internal/utils/security_test.go b/internal/utils/security_test.go index 3c017ab9..a21b70a4 100644 --- a/internal/utils/security_test.go +++ b/internal/utils/security_test.go @@ -212,6 +212,43 @@ func TestValidateURLForSSRF_IPv6Whitelist(t *testing.T) { } } +// TestSSRFWhitelistExtraMerge verifies that SSRF_WHITELIST_EXTRA is merged +// into the effective whitelist alongside SSRF_WHITELIST, so deployment-managed +// defaults (e.g. docker-compose injected sidecar host names) survive when an +// operator overrides SSRF_WHITELIST in their .env. +func TestSSRFWhitelistExtraMerge(t *testing.T) { + cases := []struct { + name string + main string + extra string + host string + want bool + }{ + {name: "extra only", main: "", extra: "searxng", host: "searxng", want: true}, + {name: "main only does not match extra host", main: "internal", extra: "", host: "searxng", want: false}, + {name: "both merged - main host", main: "internal", extra: "searxng", host: "internal", want: true}, + {name: "both merged - extra host", main: "internal", extra: "searxng", host: "searxng", want: true}, + {name: "neither matches unrelated host", main: "internal", extra: "searxng", host: "evil.example", want: false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + resetSSRFWhitelistForTest() + os.Setenv("SSRF_WHITELIST", tc.main) + os.Setenv("SSRF_WHITELIST_EXTRA", tc.extra) + defer func() { + os.Unsetenv("SSRF_WHITELIST") + os.Unsetenv("SSRF_WHITELIST_EXTRA") + resetSSRFWhitelistForTest() + }() + if got := IsSSRFWhitelisted(tc.host); got != tc.want { + t.Fatalf("IsSSRFWhitelisted(%q) main=%q extra=%q = %v, want %v", + tc.host, tc.main, tc.extra, got, tc.want) + } + }) + } +} + // TestIsRestrictedIP_IPv6 tests IPv6-specific restricted range detection. func TestIsRestrictedIP_IPv6(t *testing.T) { t.Parallel()