feat: add google provider for web search

Signed-off-by: joeyczheng <joeyczheng@tencent.com>
This commit is contained in:
joeyczheng
2026-01-12 18:00:31 +08:00
committed by lyingbug
parent 6393fcf87b
commit 17f9a97ce4
8 changed files with 468 additions and 68 deletions
@@ -304,6 +304,8 @@ func NewWebSearchService(cfg *config.Config) (interfaces.WebSearchService, error
switch providerConfig.ID {
case "duckduckgo":
provider, err = web_search.NewDuckDuckGoProvider(providerConfig)
case "google":
provider, err = web_search.NewGoogleProvider(providerConfig)
default:
return nil, fmt.Errorf("unknown web search provider: %s", providerConfig.ID)
}
@@ -0,0 +1,92 @@
package web_search
import (
"context"
"fmt"
"net/url"
"google.golang.org/api/customsearch/v1"
"google.golang.org/api/option"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// GoogleProvider implements web search using Google Custom Search Engine API
type GoogleProvider struct {
srv *customsearch.Service
apiKey string
engineID string
baseURL string
}
// NewGoogleProvider creates a new Google provider
func NewGoogleProvider(cfg config.WebSearchProviderConfig) (interfaces.WebSearchProvider, error) {
u, err := url.Parse(cfg.APIURL)
if err != nil {
return nil, err
}
engineID := u.Query().Get("engine_id")
if engineID == "" {
return nil, fmt.Errorf("engine_id is empty")
}
apiKey := u.Query().Get("api_key")
if apiKey == "" {
return nil, fmt.Errorf("api_key is empty")
}
clientOpts := make([]option.ClientOption, 0)
clientOpts = append(clientOpts, option.WithAPIKey(apiKey))
clientOpts = append(clientOpts, option.WithEndpoint(u.Scheme+"://"+u.Host))
srv, err := customsearch.NewService(context.Background(), clientOpts...)
if err != nil {
return nil, err
}
return &GoogleProvider{
srv: srv,
apiKey: apiKey,
engineID: engineID,
baseURL: cfg.APIURL,
}, nil
}
// Name returns the provider name
func (p *GoogleProvider) Name() string {
return "google"
}
// Search performs a web search using Google Custom Search Engine API
func (p *GoogleProvider) Search(
ctx context.Context,
query string,
maxResults int,
includeDate bool,
) ([]*types.WebSearchResult, error) {
if len(query) == 0 {
return nil, fmt.Errorf("query is empty")
}
cseCall := p.srv.Cse.List().Context(ctx).Cx(p.engineID).Q(query)
if maxResults > 0 {
cseCall = cseCall.Num(int64(maxResults))
} else {
cseCall = cseCall.Num(5)
}
cseCall = cseCall.Hl("ch-zh")
resp, err := cseCall.Do()
if err != nil {
return nil, err
}
results := make([]*types.WebSearchResult, 0)
for _, item := range resp.Items {
result := &types.WebSearchResult{
Title: item.Title,
URL: item.Link,
Snippet: item.Snippet,
Source: "google",
}
results = append(results, result)
}
return results, nil
}
@@ -0,0 +1,278 @@
package web_search
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/config"
)
func TestNewGoogleProvider(t *testing.T) {
testCases := []struct {
name string
cfg config.WebSearchProviderConfig
expected error
}{
{
name: "valid config",
cfg: config.WebSearchProviderConfig{
APIURL: "https://customsearch.googleapis.com/customsearch/v1?api_key=test&engine_id=test",
},
expected: nil,
},
{
name: "missing engine id",
cfg: config.WebSearchProviderConfig{
APIURL: "https://customsearch.googleapis.com/customsearch/v1?api_key=test",
},
expected: fmt.Errorf("engine_id is empty"),
},
{
name: "missing api key",
cfg: config.WebSearchProviderConfig{
APIURL: "https://customsearch.googleapis.com/customsearch/v1?engine_id=test",
},
expected: fmt.Errorf("api_key is empty"),
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
_, err := NewGoogleProvider(tc.cfg)
if tc.expected == nil {
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
} else {
if err == nil {
t.Fatalf("expected error %v, got nil", tc.expected)
}
if !strings.Contains(err.Error(), tc.expected.Error()) {
t.Fatalf("expected error %v, got %v", tc.expected, err)
}
}
})
}
}
func TestGoogleProvider_Name(t *testing.T) {
cfg := config.WebSearchProviderConfig{
APIURL: "https://customsearch.googleapis.com/customsearch/v1?api_key=test&engine_id=test",
}
p, err := NewGoogleProvider(cfg)
if err != nil {
t.Fatalf("failed to create Google provider: %v", err)
}
if p.Name() != "google" {
t.Fatalf("expected provider name google, got %s", p.Name())
}
}
func TestGoogleProvider_Search(t *testing.T) {
mockResponse := map[string]interface{}{
"items": []map[string]interface{}{
{
"title": "Example Search Result One",
"link": "https://example.com/page1",
"snippet": "This is the first search result snippet describing the content.",
},
{
"title": "Example Search Result Two",
"link": "https://example.org/page2",
"snippet": "This is the second search result snippet with more details.",
},
},
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/customsearch/v1" {
t.Fatalf("unexpected request path: %s", r.URL.Path)
}
query := r.URL.Query().Get("q")
if query != "weknora" {
t.Fatalf("unexpected query: %s", query)
}
cx := r.URL.Query().Get("cx")
if cx != "test-engine-id" {
t.Fatalf("unexpected engine ID: %s", cx)
}
num := r.URL.Query().Get("num")
if num != "5" {
t.Fatalf("unexpected num parameter: %s", num)
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
enc := json.NewEncoder(w)
_ = enc.Encode(mockResponse)
}))
defer ts.Close()
cfg := config.WebSearchProviderConfig{
APIURL: fmt.Sprintf("%s/customsearch/v1?api_key=test-key&engine_id=test-engine-id", ts.URL),
}
prov, err := NewGoogleProvider(cfg)
if err != nil {
t.Fatalf("failed to create Google provider: %v", err)
}
gp := prov.(*GoogleProvider)
if gp == nil {
t.Fatalf("failed to cast to GoogleProvider")
}
ctx := context.Background()
results, err := prov.Search(ctx, "weknora", 5, false)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(results) != 2 {
t.Fatalf("expected 2 results, got %d", len(results))
}
if results[0].Title != "Example Search Result One" ||
results[0].URL != "https://example.com/page1" ||
results[0].Snippet != "This is the first search result snippet describing the content." ||
results[0].Source != "google" {
t.Fatalf("unexpected first result: %+v", results[0])
}
if results[1].Title != "Example Search Result Two" ||
results[1].URL != "https://example.org/page2" ||
results[1].Snippet != "This is the second search result snippet with more details." ||
results[1].Source != "google" {
t.Fatalf("unexpected second result: %+v", results[1])
}
}
func TestGoogleProvider_Search_EmptyQuery(t *testing.T) {
cfg := config.WebSearchProviderConfig{
APIURL: "https://customsearch.googleapis.com/customsearch/v1?api_key=test&engine_id=test",
}
prov, err := NewGoogleProvider(cfg)
if err != nil {
t.Fatalf("failed to create Google provider: %v", err)
}
ctx := context.Background()
results, err := prov.Search(ctx, "", 5, false)
if err == nil {
t.Fatal("expected error for empty query, got nil")
}
if !strings.Contains(err.Error(), "query is empty") {
t.Fatalf("expected 'query is empty' error, got: %v", err)
}
if results != nil {
t.Fatalf("expected nil results for empty query, got: %v", results)
}
}
func TestGoogleProvider_Search_NoResults(t *testing.T) {
mockResponse := map[string]interface{}{
"items": []map[string]interface{}{},
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
enc := json.NewEncoder(w)
_ = enc.Encode(mockResponse)
}))
defer ts.Close()
cfg := config.WebSearchProviderConfig{
APIURL: fmt.Sprintf("%s/customsearch/v1?api_key=test-key&engine_id=test-engine-id", ts.URL),
}
prov, err := NewGoogleProvider(cfg)
if err != nil {
t.Fatalf("failed to create Google provider: %v", err)
}
ctx := context.Background()
results, err := prov.Search(ctx, "nonexistent", 5, false)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(results) != 0 {
t.Fatalf("expected 0 results, got %d", len(results))
}
}
func TestGoogleProvider_Search_ErrorResponse(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Internal Server Error"))
}))
defer ts.Close()
cfg := config.WebSearchProviderConfig{
APIURL: fmt.Sprintf("%s/customsearch/v1?api_key=test-key&engine_id=test-engine-id", ts.URL),
}
prov, err := NewGoogleProvider(cfg)
if err != nil {
t.Fatalf("failed to create Google provider: %v", err)
}
ctx := context.Background()
results, err := prov.Search(ctx, "test", 5, false)
if err == nil {
t.Fatal("expected error for server error response, got nil")
}
if results != nil {
t.Fatalf("expected nil results for error response, got: %v", results)
}
}
func TestGoogleProvider_Search_MaxResults(t *testing.T) {
mockResponse := map[string]interface{}{
"items": []map[string]interface{}{
{"title": "Result 1", "link": "https://example.com/1", "snippet": "Snippet 1"},
{"title": "Result 2", "link": "https://example.com/2", "snippet": "Snippet 2"},
{"title": "Result 3", "link": "https://example.com/3", "snippet": "Snippet 3"},
},
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
num := r.URL.Query().Get("num")
if num != "2" {
t.Fatalf("expected num=2, got %s", num)
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
enc := json.NewEncoder(w)
_ = enc.Encode(mockResponse)
}))
defer ts.Close()
cfg := config.WebSearchProviderConfig{
APIURL: fmt.Sprintf("%s/customsearch/v1?api_key=test-key&engine_id=test-engine-id", ts.URL),
}
prov, err := NewGoogleProvider(cfg)
if err != nil {
t.Fatalf("failed to create Google provider: %v", err)
}
ctx := context.Background()
results, err := prov.Search(ctx, "test", 2, false)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if len(results) != 3 {
t.Fatalf("expected 3 results, got %d", len(results))
}
if results[0].Title != "Result 1" || results[1].Title != "Result 2" || results[2].Title != "Result 3" {
t.Fatalf("unexpected results order or content")
}
}