package web_search import ( "context" "encoding/json" "net/http" "net/http/httptest" "net/url" "strings" "testing" "time" "github.com/Tencent/WeKnora/internal/config" ) // testRoundTripper rewrites outgoing requests that target DuckDuckGo hosts // to the provided test server, preserving path and query. type testRoundTripper struct { base *url.URL next http.RoundTripper } func (t *testRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { // Only rewrite requests to duckduckgo hosts used by the provider if req.URL.Host == "html.duckduckgo.com" || req.URL.Host == "api.duckduckgo.com" { cloned := *req u := *req.URL u.Scheme = t.base.Scheme u.Host = t.base.Host // Keep original path; our test server handlers should register for the same paths. cloned.URL = &u req = &cloned } return t.next.RoundTrip(req) } func newTestClient(ts *httptest.Server) *http.Client { baseURL, _ := url.Parse(ts.URL) return &http.Client{ Timeout: 5 * time.Second, Transport: &testRoundTripper{ base: baseURL, next: http.DefaultTransport, }, } } func TestDuckDuckGoProvider_Name(t *testing.T) { p, _ := NewDuckDuckGoProvider(config.WebSearchProviderConfig{}) if p.Name() != "duckduckgo" { t.Fatalf("expected provider name duckduckgo, got %s", p.Name()) } } func TestDuckDuckGoProvider(t *testing.T) { // Minimal HTML page with two results, matching selectors used in searchHTML html := `
Example One
Snippet one
Example Two
Snippet two
` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Provider requests GET https://html.duckduckgo.com/html/?q=...&kl=... if r.URL.Path == "/html/" { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte(html)) return } t.Fatalf("unexpected request path: %s", r.URL.Path) })) defer ts.Close() // Build provider and inject our test client prov, _ := NewDuckDuckGoProvider(config.WebSearchProviderConfig{}) dp := prov.(*DuckDuckGoProvider) if dp == nil { t.Fatalf("failed to build provider") } dp.client = newTestClient(ts) ctx := context.Background() results, err := dp.Search(ctx, "weknora", 5, false) if err != nil { t.Fatalf("unexpected error: %v", err) } if len(results) != 2 { t.Fatalf("expected 2 results, got %d", len(results)) } if results[0].Title != "Example One" || !strings.HasPrefix(results[0].URL, "https://example.com/") || results[0].Snippet != "Snippet one" { t.Fatalf("unexpected first result: %+v", results[0]) } if results[1].Title != "Example Two" || !strings.HasPrefix(results[1].URL, "https://example.org/") || results[1].Snippet != "Snippet two" { t.Fatalf("unexpected second result: %+v", results[1]) } } func TestDuckDuckGoProvider_Fallback(t *testing.T) { // Simulate HTML returning non-OK to force API fallback, then a minimal API JSON apiResp := struct { AbstractText string `json:"AbstractText"` AbstractURL string `json:"AbstractURL"` Heading string `json:"Heading"` Results []struct { FirstURL string `json:"FirstURL"` Text string `json:"Text"` } `json:"Results"` }{ AbstractText: "Abstract snippet", AbstractURL: "https://example.com/abstract", Heading: "Abstract Heading", Results: []struct { FirstURL string `json:"FirstURL"` Text string `json:"Text"` }{ {FirstURL: "https://example.net/x", Text: "Title X - Detail X"}, }, } ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/html/": // Force fallback by returning 500 w.WriteHeader(http.StatusInternalServerError) default: // API endpoint path "/" w.Header().Set("Content-Type", "application/json") enc := json.NewEncoder(w) _ = enc.Encode(apiResp) } })) defer ts.Close() prov, _ := NewDuckDuckGoProvider(config.WebSearchProviderConfig{}) dp := prov.(*DuckDuckGoProvider) if dp == nil { t.Fatalf("failed to build provider") } dp.client = newTestClient(ts) ctx := context.Background() results, err := dp.Search(ctx, "weknora", 3, false) if err != nil { t.Fatalf("unexpected error: %v", err) } if len(results) == 0 { t.Fatalf("expected some results from API fallback") } if results[0].URL != "https://example.com/abstract" || results[0].Title != "Abstract Heading" { t.Fatalf("unexpected first API result: %+v", results[0]) } } // TestDuckDuckGoProvider_Search_Real tests the DuckDuckGo provider against the real DuckDuckGo service. // This is an integration test that requires network connectivity. // Run with: go test -v -run TestDuckDuckGoProvider_Search_Real ./internal/application/service/web_search func TestDuckDuckGoProvider_Search_Real(t *testing.T) { // Skip if running in CI without network access (optional check) if testing.Short() { t.Skip("Skipping real DuckDuckGo integration test in short mode") } ctx := context.Background() provider, err := NewDuckDuckGoProvider(config.WebSearchProviderConfig{}) if err != nil { t.Fatalf("Failed to create DuckDuckGo provider: %v", err) } if provider == nil { t.Fatalf("failed to build provider") } // Test with a simple, general query that should return results query := "Go programming language" maxResults := 5 results, err := provider.Search(ctx, query, maxResults, false) if err != nil { t.Fatalf("Search failed: %v", err) } // Verify we got results if len(results) == 0 { t.Fatal("Expected at least one search result, got 0") } t.Logf("Received %d results for query: %s", len(results), query) // Verify result structure for i, result := range results { if result == nil { t.Fatalf("Result[%d]: is nil", i) } if result.Title == "" { t.Errorf("Result[%d]: Title is empty", i) } if result.URL == "" { t.Errorf("Result[%d]: URL is empty", i) } if !strings.HasPrefix(result.URL, "http://") && !strings.HasPrefix(result.URL, "https://") { t.Errorf("Result[%d]: URL is not valid (should start with http:// or https://): %s", i, result.URL) } if result.Source != "duckduckgo" { t.Errorf("Result[%d]: Source should be 'duckduckgo', got '%s'", i, result.Source) } t.Logf("Result[%d]: Title=%s, URL=%s, Snippet=%s", i, result.Title, result.URL, result.Snippet) } // Verify we don't exceed maxResults if len(results) > maxResults { t.Errorf("Got %d results, expected at most %d", len(results), maxResults) } // Test with maxResults limit limitedResults, err := provider.Search(ctx, query, 2, false) if err != nil { t.Fatalf("Search with limit failed: %v", err) } if len(limitedResults) > 2 { t.Errorf("Got %d results with maxResults=2, expected at most 2", len(limitedResults)) } } // TestDuckDuckGo_SearchChinese tests the DuckDuckGo provider with Chinese query. // This verifies the Chinese language parameter (kl=cn-zh) works correctly. func TestDuckDuckGo_SearchChinese(t *testing.T) { if testing.Short() { t.Skip("Skipping real DuckDuckGo integration test in short mode") } ctx := context.Background() provider, err := NewDuckDuckGoProvider(config.WebSearchProviderConfig{}) if err != nil { t.Fatalf("Failed to create DuckDuckGo provider: %v", err) } if provider == nil { t.Fatalf("failed to build provider") } // Test with a Chinese query query := "WeKnora 企业级RAG框架 介绍 文档" maxResults := 3 results, err := provider.Search(ctx, query, maxResults, false) if err != nil { t.Fatalf("Search failed: %v", err) } if len(results) == 0 { t.Log("Warning: No results returned for Chinese query, but this might be expected") return } t.Logf("Received %d results for Chinese query: %s", len(results), query) // Verify result structure for i, result := range results { if result == nil { t.Fatalf("Result[%d]: is nil", i) } if result.Title == "" { t.Errorf("Result[%d]: Title is empty", i) } if result.URL == "" { t.Errorf("Result[%d]: URL is empty", i) } if result.Source != "duckduckgo" { t.Errorf("Result[%d]: Source should be 'duckduckgo', got '%s'", i, result.Source) } t.Logf("Result[%d]: Title=%s, URL=%s", i, result.Title, result.URL) } }