feat(web-search): add Zhipu AI web search provider support
Integrate Zhipu's standalone Web Search API with configurable search engine and content size options, plus settings UI, docs, and tests.
This commit is contained in:
@@ -0,0 +1,271 @@
|
||||
package web_search
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultZhipuSearchURL = "https://open.bigmodel.cn/api/paas/v4/web_search"
|
||||
defaultZhipuTimeout = 15 * time.Second
|
||||
defaultZhipuResults = 10
|
||||
maxZhipuResults = 50
|
||||
maxZhipuQueryRunes = 70
|
||||
maxZhipuResponseBytes = 2 << 20
|
||||
defaultZhipuSearchEngine = "search_std"
|
||||
defaultZhipuContentSize = "medium"
|
||||
)
|
||||
|
||||
var validZhipuSearchEngines = map[string]struct{}{
|
||||
"search_std": {},
|
||||
"search_pro": {},
|
||||
"search_pro_sogou": {},
|
||||
"search_pro_quark": {},
|
||||
}
|
||||
|
||||
var validZhipuContentSizes = map[string]struct{}{
|
||||
"medium": {},
|
||||
"high": {},
|
||||
}
|
||||
|
||||
// ZhipuProvider implements web search using Zhipu AI's standalone Web Search API.
|
||||
type ZhipuProvider struct {
|
||||
client *http.Client
|
||||
baseURL string
|
||||
apiKey string
|
||||
searchEngine string
|
||||
contentSize string
|
||||
}
|
||||
|
||||
// NewZhipuProvider creates a Zhipu AI web search provider from persisted parameters.
|
||||
func NewZhipuProvider(params types.WebSearchProviderParameters) (interfaces.WebSearchProvider, error) {
|
||||
if err := ValidateZhipuParameters(params); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := NewSearchHTTPClient(defaultZhipuTimeout, params.ProxyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
searchEngine, contentSize := zhipuOptions(params.ExtraConfig)
|
||||
return &ZhipuProvider{
|
||||
client: client,
|
||||
baseURL: defaultZhipuSearchURL,
|
||||
apiKey: strings.TrimSpace(params.APIKey),
|
||||
searchEngine: searchEngine,
|
||||
contentSize: contentSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ValidateZhipuParameters validates credentials and provider-specific options.
|
||||
func ValidateZhipuParameters(params types.WebSearchProviderParameters) error {
|
||||
if strings.TrimSpace(params.APIKey) == "" {
|
||||
return fmt.Errorf("API key is required for Zhipu provider")
|
||||
}
|
||||
searchEngine, contentSize := zhipuOptions(params.ExtraConfig)
|
||||
if _, ok := validZhipuSearchEngines[searchEngine]; !ok {
|
||||
return fmt.Errorf("invalid Zhipu search engine: %s", searchEngine)
|
||||
}
|
||||
if _, ok := validZhipuContentSizes[contentSize]; !ok {
|
||||
return fmt.Errorf("invalid Zhipu content size: %s", contentSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func zhipuOptions(extraConfig map[string]string) (searchEngine, contentSize string) {
|
||||
searchEngine = defaultZhipuSearchEngine
|
||||
contentSize = defaultZhipuContentSize
|
||||
if value := strings.TrimSpace(extraConfig["search_engine"]); value != "" {
|
||||
searchEngine = value
|
||||
}
|
||||
if value := strings.TrimSpace(extraConfig["content_size"]); value != "" {
|
||||
contentSize = value
|
||||
}
|
||||
return searchEngine, contentSize
|
||||
}
|
||||
|
||||
// Name returns the provider name.
|
||||
func (p *ZhipuProvider) Name() string {
|
||||
return "zhipu"
|
||||
}
|
||||
|
||||
// Search performs a web search using Zhipu AI's standalone Web Search API.
|
||||
func (p *ZhipuProvider) Search(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
maxResults int,
|
||||
includeDate bool,
|
||||
) ([]*types.WebSearchResult, error) {
|
||||
preparedQuery := normalizeZhipuQuery(query)
|
||||
if preparedQuery == "" {
|
||||
return nil, fmt.Errorf("query is empty")
|
||||
}
|
||||
if utf8.RuneCountInString(strings.TrimSpace(query)) > maxZhipuQueryRunes {
|
||||
logger.Infof(ctx, "[WebSearch][Zhipu] truncated query to %d characters", maxZhipuQueryRunes)
|
||||
}
|
||||
if maxResults <= 0 {
|
||||
maxResults = defaultZhipuResults
|
||||
}
|
||||
if maxResults > maxZhipuResults {
|
||||
maxResults = maxZhipuResults
|
||||
}
|
||||
|
||||
requestBody := zhipuSearchRequest{
|
||||
SearchQuery: preparedQuery,
|
||||
SearchEngine: p.searchEngine,
|
||||
SearchIntent: false,
|
||||
Count: maxResults,
|
||||
ContentSize: p.contentSize,
|
||||
}
|
||||
body, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal Zhipu request: %w", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.baseURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Zhipu request: %w", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
logger.Infof(ctx, "[WebSearch][Zhipu] query=%q maxResults=%d engine=%s", preparedQuery, maxResults, p.searchEngine)
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to execute Zhipu request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := readZhipuResponseBody(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, zhipuHTTPError(resp.StatusCode, respBody)
|
||||
}
|
||||
|
||||
var response zhipuSearchResponse
|
||||
if err := json.Unmarshal(respBody, &response); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal Zhipu response: %w", err)
|
||||
}
|
||||
if response.Error.Message != "" || response.Error.Code != "" {
|
||||
return nil, fmt.Errorf("Zhipu API error (%s): %s", response.Error.Code, response.Error.Message)
|
||||
}
|
||||
|
||||
results := make([]*types.WebSearchResult, 0, len(response.SearchResult))
|
||||
for _, item := range response.SearchResult {
|
||||
if strings.TrimSpace(item.Title) == "" && strings.TrimSpace(item.Link) == "" {
|
||||
continue
|
||||
}
|
||||
result := &types.WebSearchResult{
|
||||
Title: item.Title,
|
||||
URL: item.Link,
|
||||
Snippet: item.Content,
|
||||
Source: "zhipu",
|
||||
}
|
||||
if includeDate {
|
||||
if publishedAt, ok := parseZhipuDate(item.PublishDate); ok {
|
||||
result.PublishedAt = &publishedAt
|
||||
}
|
||||
}
|
||||
results = append(results, result)
|
||||
if len(results) >= maxResults {
|
||||
break
|
||||
}
|
||||
}
|
||||
logger.Infof(ctx, "[WebSearch][Zhipu] returned %d results", len(results))
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func normalizeZhipuQuery(query string) string {
|
||||
query = strings.TrimSpace(query)
|
||||
if utf8.RuneCountInString(query) <= maxZhipuQueryRunes {
|
||||
return query
|
||||
}
|
||||
runes := []rune(query)
|
||||
return string(runes[:maxZhipuQueryRunes])
|
||||
}
|
||||
|
||||
func parseZhipuDate(value string) (time.Time, bool) {
|
||||
value = strings.TrimSpace(value)
|
||||
if value == "" {
|
||||
return time.Time{}, false
|
||||
}
|
||||
for _, layout := range []string{
|
||||
time.RFC3339Nano,
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02 15:04",
|
||||
"2006-01-02",
|
||||
} {
|
||||
if parsed, err := time.Parse(layout, value); err == nil {
|
||||
return parsed, true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
func readZhipuResponseBody(reader io.Reader) ([]byte, error) {
|
||||
body, err := io.ReadAll(io.LimitReader(reader, maxZhipuResponseBytes+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read Zhipu response: %w", err)
|
||||
}
|
||||
if len(body) > maxZhipuResponseBytes {
|
||||
return nil, fmt.Errorf("Zhipu response exceeds %d bytes", maxZhipuResponseBytes)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func zhipuHTTPError(statusCode int, body []byte) error {
|
||||
var response zhipuSearchResponse
|
||||
if err := json.Unmarshal(body, &response); err == nil && (response.Error.Code != "" || response.Error.Message != "") {
|
||||
return fmt.Errorf("Zhipu API returned status %d (%s): %s", statusCode, response.Error.Code, response.Error.Message)
|
||||
}
|
||||
detail := strings.TrimSpace(string(body))
|
||||
if len(detail) > 4096 {
|
||||
detail = detail[:4096]
|
||||
}
|
||||
if detail == "" {
|
||||
return fmt.Errorf("Zhipu API returned status %d", statusCode)
|
||||
}
|
||||
return fmt.Errorf("Zhipu API returned status %d: %s", statusCode, detail)
|
||||
}
|
||||
|
||||
type zhipuSearchRequest struct {
|
||||
SearchQuery string `json:"search_query"`
|
||||
SearchEngine string `json:"search_engine"`
|
||||
SearchIntent bool `json:"search_intent"`
|
||||
Count int `json:"count"`
|
||||
ContentSize string `json:"content_size"`
|
||||
}
|
||||
|
||||
type zhipuSearchResponse struct {
|
||||
ID string `json:"id"`
|
||||
RequestID string `json:"request_id"`
|
||||
SearchResult []zhipuSearchResult `json:"search_result"`
|
||||
Error zhipuError `json:"error"`
|
||||
}
|
||||
|
||||
type zhipuSearchResult struct {
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Link string `json:"link"`
|
||||
Media string `json:"media"`
|
||||
Icon string `json:"icon"`
|
||||
Refer string `json:"refer"`
|
||||
PublishDate string `json:"publish_date"`
|
||||
}
|
||||
|
||||
type zhipuError struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package web_search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestValidateZhipuParameters(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
params types.WebSearchProviderParameters
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "defaults", params: types.WebSearchProviderParameters{APIKey: "key"}},
|
||||
{
|
||||
name: "custom options",
|
||||
params: types.WebSearchProviderParameters{
|
||||
APIKey: "key",
|
||||
ExtraConfig: map[string]string{
|
||||
"search_engine": "search_pro_sogou",
|
||||
"content_size": "high",
|
||||
},
|
||||
},
|
||||
},
|
||||
{name: "missing key", params: types.WebSearchProviderParameters{}, wantErr: true},
|
||||
{
|
||||
name: "invalid engine",
|
||||
params: types.WebSearchProviderParameters{
|
||||
APIKey: "key",
|
||||
ExtraConfig: map[string]string{"search_engine": "unknown"},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid content size",
|
||||
params: types.WebSearchProviderParameters{
|
||||
APIKey: "key",
|
||||
ExtraConfig: map[string]string{"content_size": "large"},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := ValidateZhipuParameters(tt.params)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Fatalf("ValidateZhipuParameters() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestZhipuProviderSearch(t *testing.T) {
|
||||
query := strings.Repeat("智", maxZhipuQueryRunes+1)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
t.Errorf("method = %s, want POST", r.Method)
|
||||
}
|
||||
if got := r.Header.Get("Authorization"); got != "Bearer test-key" {
|
||||
t.Errorf("Authorization = %q, want Bearer test-key", got)
|
||||
}
|
||||
var request zhipuSearchRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
t.Errorf("decode request: %v", err)
|
||||
}
|
||||
if got := utf8.RuneCountInString(request.SearchQuery); got != maxZhipuQueryRunes {
|
||||
t.Errorf("query rune count = %d, want %d", got, maxZhipuQueryRunes)
|
||||
}
|
||||
if request.SearchEngine != "search_pro" {
|
||||
t.Errorf("search_engine = %q, want search_pro", request.SearchEngine)
|
||||
}
|
||||
if request.ContentSize != "high" {
|
||||
t.Errorf("content_size = %q, want high", request.ContentSize)
|
||||
}
|
||||
if request.SearchIntent {
|
||||
t.Error("search_intent = true, want false")
|
||||
}
|
||||
if request.Count != maxZhipuResults {
|
||||
t.Errorf("count = %d, want %d", request.Count, maxZhipuResults)
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"id": "search-id",
|
||||
"request_id": "request-id",
|
||||
"search_result": []map[string]any{
|
||||
{
|
||||
"title": "Result 1",
|
||||
"link": "https://example.com/1",
|
||||
"content": "Summary 1",
|
||||
"publish_date": "2026-07-16",
|
||||
},
|
||||
{
|
||||
"title": "Result 2",
|
||||
"link": "https://example.com/2",
|
||||
"content": "Summary 2",
|
||||
},
|
||||
},
|
||||
})
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
provider := &ZhipuProvider{
|
||||
client: srv.Client(),
|
||||
baseURL: srv.URL,
|
||||
apiKey: "test-key",
|
||||
searchEngine: "search_pro",
|
||||
contentSize: "high",
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
results, err := provider.Search(ctx, query, maxZhipuResults+1, true)
|
||||
if err != nil {
|
||||
t.Fatalf("Search() error = %v", err)
|
||||
}
|
||||
if len(results) != 2 {
|
||||
t.Fatalf("len(results) = %d, want 2", len(results))
|
||||
}
|
||||
if results[0].Snippet != "Summary 1" || results[0].Content != "" {
|
||||
t.Errorf("first result content mapping = %+v", results[0])
|
||||
}
|
||||
if results[0].Source != "zhipu" {
|
||||
t.Errorf("source = %q, want zhipu", results[0].Source)
|
||||
}
|
||||
if results[0].PublishedAt == nil || results[0].PublishedAt.Format("2006-01-02") != "2026-07-16" {
|
||||
t.Errorf("published_at = %v, want 2026-07-16", results[0].PublishedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZhipuProviderSearchDefaults(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var request zhipuSearchRequest
|
||||
_ = json.NewDecoder(r.Body).Decode(&request)
|
||||
if request.SearchEngine != defaultZhipuSearchEngine {
|
||||
t.Errorf("search_engine = %q, want %q", request.SearchEngine, defaultZhipuSearchEngine)
|
||||
}
|
||||
if request.ContentSize != defaultZhipuContentSize {
|
||||
t.Errorf("content_size = %q, want %q", request.ContentSize, defaultZhipuContentSize)
|
||||
}
|
||||
if request.Count != defaultZhipuResults {
|
||||
t.Errorf("count = %d, want %d", request.Count, defaultZhipuResults)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"search_result":[]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
provider := &ZhipuProvider{
|
||||
client: srv.Client(),
|
||||
baseURL: srv.URL,
|
||||
apiKey: "test-key",
|
||||
searchEngine: defaultZhipuSearchEngine,
|
||||
contentSize: defaultZhipuContentSize,
|
||||
}
|
||||
if _, err := provider.Search(context.Background(), "test", 0, false); err != nil {
|
||||
t.Fatalf("Search() error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZhipuProviderSearchError(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = w.Write([]byte(`{"error":{"code":"1302","message":"rate limited"}}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
provider := &ZhipuProvider{
|
||||
client: srv.Client(),
|
||||
baseURL: srv.URL,
|
||||
apiKey: "test-key",
|
||||
searchEngine: defaultZhipuSearchEngine,
|
||||
contentSize: defaultZhipuContentSize,
|
||||
}
|
||||
_, err := provider.Search(context.Background(), "test", 1, false)
|
||||
if err == nil || !strings.Contains(err.Error(), "rate limited") {
|
||||
t.Fatalf("Search() error = %v, want rate limited error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseZhipuDate(t *testing.T) {
|
||||
for _, value := range []string{
|
||||
"2026-07-16",
|
||||
"2026-07-16 12:30",
|
||||
"2026-07-16 12:30:45",
|
||||
"2026-07-16T12:30:45Z",
|
||||
} {
|
||||
if _, ok := parseZhipuDate(value); !ok {
|
||||
t.Errorf("parseZhipuDate(%q) failed", value)
|
||||
}
|
||||
}
|
||||
if _, ok := parseZhipuDate("not-a-date"); ok {
|
||||
t.Error("parseZhipuDate(not-a-date) unexpectedly succeeded")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user