07a8c1de4c
- Introduced web search configuration in config.yaml, allowing multiple providers with specific settings. - Updated InputField component to manage agent mode and web search toggles, improving user interaction. - Enhanced chat API to include web search status in requests, enabling dynamic query handling. - Implemented backend support for web search integration, merging results with knowledge base queries. - Added tenant-level web search configuration management, allowing customization per tenant.
261 lines
7.4 KiB
Go
261 lines
7.4 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/config"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
)
|
|
|
|
// WebSearchProvider defines the interface for web search providers
|
|
type WebSearchProvider interface {
|
|
Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error)
|
|
Name() string
|
|
}
|
|
|
|
// WebSearchService provides web search functionality
|
|
type WebSearchService struct {
|
|
providers map[string]WebSearchProvider
|
|
config *config.WebSearchConfig
|
|
}
|
|
|
|
// Search performs web search using the specified provider
|
|
// This method implements the interface expected by PluginSearch
|
|
func (s *WebSearchService) Search(ctx context.Context, config *types.WebSearchConfig, query string) ([]*types.WebSearchResult, error) {
|
|
if config == nil {
|
|
return nil, fmt.Errorf("web search config is required")
|
|
}
|
|
|
|
provider, ok := s.providers[config.Provider]
|
|
if !ok {
|
|
return nil, fmt.Errorf("web search provider %s is not available", config.Provider)
|
|
}
|
|
|
|
// Set API key for providers that need it
|
|
if config.APIKey != "" {
|
|
s.setProviderAPIKey(config.Provider, provider, config)
|
|
}
|
|
|
|
// Set timeout
|
|
timeout := time.Duration(s.config.Timeout) * time.Second
|
|
if timeout == 0 {
|
|
timeout = 10 * time.Second
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
|
|
// Perform search
|
|
results, err := provider.Search(ctx, query, config.MaxResults, config.IncludeDate)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("web search failed: %w", err)
|
|
}
|
|
|
|
// Apply blacklist filtering
|
|
results = s.filterBlacklist(results, config.Blacklist)
|
|
|
|
// Apply compression if needed
|
|
if config.CompressionMethod != "none" && config.CompressionMethod != "" {
|
|
// Compression will be handled later in the integration layer
|
|
// For now, we just return the results
|
|
}
|
|
|
|
return results, nil
|
|
}
|
|
|
|
// NewWebSearchService creates a new web search service
|
|
func NewWebSearchService(cfg *config.Config) (*WebSearchService, error) {
|
|
if cfg.WebSearch == nil {
|
|
return nil, fmt.Errorf("web search config is not available")
|
|
}
|
|
|
|
service := &WebSearchService{
|
|
providers: make(map[string]WebSearchProvider),
|
|
config: cfg.WebSearch,
|
|
}
|
|
|
|
// Initialize providers based on config
|
|
for _, providerConfig := range cfg.WebSearch.Providers {
|
|
var provider WebSearchProvider
|
|
var err error
|
|
|
|
switch providerConfig.ID {
|
|
case "kuaisou":
|
|
provider, err = NewKuaisouProvider(providerConfig)
|
|
case "baidu":
|
|
provider, err = NewBaiduProvider(providerConfig)
|
|
case "google":
|
|
provider, err = NewGoogleProvider(providerConfig)
|
|
case "bing":
|
|
provider, err = NewBingProvider(providerConfig)
|
|
case "bocha":
|
|
provider, err = NewBochaProvider(providerConfig)
|
|
case "zhipu":
|
|
provider, err = NewZhipuProvider(providerConfig)
|
|
case "tavily":
|
|
provider, err = NewTavilyProvider(providerConfig)
|
|
case "searxng":
|
|
provider, err = NewSearxngProvider(providerConfig)
|
|
case "exa":
|
|
provider, err = NewExaProvider(providerConfig)
|
|
default:
|
|
logger.Warnf(context.Background(), "Unknown web search provider: %s", providerConfig.ID)
|
|
continue
|
|
}
|
|
|
|
if err != nil {
|
|
logger.Warnf(context.Background(), "Failed to initialize provider %s: %v", providerConfig.ID, err)
|
|
continue
|
|
}
|
|
|
|
service.providers[providerConfig.ID] = provider
|
|
logger.Infof(context.Background(), "Initialized web search provider: %s", providerConfig.ID)
|
|
}
|
|
|
|
return service, nil
|
|
}
|
|
|
|
// setProviderAPIKey sets the API key for a provider based on its type
|
|
func (s *WebSearchService) setProviderAPIKey(providerID string, provider WebSearchProvider, config *types.WebSearchConfig) {
|
|
switch p := provider.(type) {
|
|
case *KuaisouProvider:
|
|
p.SetAPIKey(config.APIKey)
|
|
case *BochaProvider:
|
|
p.SetAPIKey(config.APIKey)
|
|
case *ZhipuProvider:
|
|
p.SetAPIKey(config.APIKey)
|
|
case *TavilyProvider:
|
|
p.SetAPIKey(config.APIKey)
|
|
case *ExaProvider:
|
|
p.SetAPIKey(config.APIKey)
|
|
case *GoogleProvider:
|
|
// Google needs both API key and search engine ID
|
|
// For now, we'll use API key as search engine ID if not provided separately
|
|
// This can be extended later to support separate fields
|
|
p.SetAPIKey(config.APIKey, config.APIKey) // TODO: Add search engine ID to config
|
|
}
|
|
}
|
|
|
|
// filterBlacklist filters results based on blacklist rules
|
|
func (s *WebSearchService) filterBlacklist(results []*types.WebSearchResult, blacklist []string) []*types.WebSearchResult {
|
|
if len(blacklist) == 0 {
|
|
return results
|
|
}
|
|
|
|
filtered := make([]*types.WebSearchResult, 0, len(results))
|
|
|
|
for _, result := range results {
|
|
shouldFilter := false
|
|
|
|
for _, rule := range blacklist {
|
|
if s.matchesBlacklistRule(result.URL, rule) {
|
|
shouldFilter = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if !shouldFilter {
|
|
filtered = append(filtered, result)
|
|
}
|
|
}
|
|
|
|
return filtered
|
|
}
|
|
|
|
// matchesBlacklistRule checks if a URL matches a blacklist rule
|
|
// Supports both pattern matching (e.g., *://*.example.com/*) and regex patterns (e.g., /example\.(net|org)/)
|
|
func (s *WebSearchService) matchesBlacklistRule(url, rule string) bool {
|
|
// Check if it's a regex pattern (starts and ends with /)
|
|
if strings.HasPrefix(rule, "/") && strings.HasSuffix(rule, "/") {
|
|
pattern := rule[1 : len(rule)-1]
|
|
matched, err := regexp.MatchString(pattern, url)
|
|
if err != nil {
|
|
logger.Warnf(context.Background(), "Invalid regex pattern in blacklist: %s, error: %v", rule, err)
|
|
return false
|
|
}
|
|
return matched
|
|
}
|
|
|
|
// Pattern matching (e.g., *://*.example.com/*)
|
|
pattern := strings.ReplaceAll(rule, "*", ".*")
|
|
pattern = "^" + pattern + "$"
|
|
matched, err := regexp.MatchString(pattern, url)
|
|
if err != nil {
|
|
logger.Warnf(context.Background(), "Invalid pattern in blacklist: %s, error: %v", rule, err)
|
|
return false
|
|
}
|
|
return matched
|
|
}
|
|
|
|
// ConvertWebSearchResults converts WebSearchResult to SearchResult
|
|
func ConvertWebSearchResults(webResults []*types.WebSearchResult) []*types.SearchResult {
|
|
results := make([]*types.SearchResult, 0, len(webResults))
|
|
|
|
for i, webResult := range webResults {
|
|
// Use URL as ChunkID for web search results
|
|
chunkID := webResult.URL
|
|
if chunkID == "" {
|
|
chunkID = fmt.Sprintf("web_search_%d", i)
|
|
}
|
|
|
|
// Combine title and snippet as content
|
|
content := webResult.Title
|
|
if webResult.Snippet != "" {
|
|
if content != "" {
|
|
content += "\n\n" + webResult.Snippet
|
|
} else {
|
|
content = webResult.Snippet
|
|
}
|
|
}
|
|
if webResult.Content != "" {
|
|
if content != "" {
|
|
content += "\n\n" + webResult.Content
|
|
} else {
|
|
content = webResult.Content
|
|
}
|
|
}
|
|
|
|
// Set a default score for web search results (0.6, indicating medium relevance)
|
|
score := 0.6
|
|
|
|
result := &types.SearchResult{
|
|
ID: chunkID,
|
|
Content: content,
|
|
KnowledgeID: "", // Web search results don't have knowledge ID
|
|
ChunkIndex: 0,
|
|
KnowledgeTitle: webResult.Title,
|
|
StartAt: 0,
|
|
EndAt: len(content),
|
|
Seq: i,
|
|
Score: score,
|
|
MatchType: types.MatchTypeWebSearch,
|
|
SubChunkID: []string{},
|
|
Metadata: map[string]string{
|
|
"url": webResult.URL,
|
|
"source": webResult.Source,
|
|
"title": webResult.Title,
|
|
"snippet": webResult.Snippet,
|
|
},
|
|
ChunkType: "web_search",
|
|
ParentChunkID: "",
|
|
ImageInfo: "",
|
|
KnowledgeFilename: "",
|
|
KnowledgeSource: "web_search",
|
|
}
|
|
|
|
// Add published date to metadata if available
|
|
if webResult.PublishedAt != nil {
|
|
result.Metadata["published_at"] = webResult.PublishedAt.Format(time.RFC3339)
|
|
}
|
|
|
|
results = append(results, result)
|
|
}
|
|
|
|
return results
|
|
}
|