Files
Enterprise-Knowledge-Base/internal/application/service/web_search.go
T
wizardchen 07a8c1de4c feat: Add web search functionality and enhance agent mode integration
- Introduced web search configuration in config.yaml, allowing multiple providers with specific settings.
- Updated InputField component to manage agent mode and web search toggles, improving user interaction.
- Enhanced chat API to include web search status in requests, enabling dynamic query handling.
- Implemented backend support for web search integration, merging results with knowledge base queries.
- Added tenant-level web search configuration management, allowing customization per tenant.
2025-11-10 03:14:12 +08:00

261 lines
7.4 KiB
Go

package service
import (
"context"
"fmt"
"regexp"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
// WebSearchProvider defines the interface for web search providers
type WebSearchProvider interface {
Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error)
Name() string
}
// WebSearchService provides web search functionality
type WebSearchService struct {
providers map[string]WebSearchProvider
config *config.WebSearchConfig
}
// Search performs web search using the specified provider
// This method implements the interface expected by PluginSearch
func (s *WebSearchService) Search(ctx context.Context, config *types.WebSearchConfig, query string) ([]*types.WebSearchResult, error) {
if config == nil {
return nil, fmt.Errorf("web search config is required")
}
provider, ok := s.providers[config.Provider]
if !ok {
return nil, fmt.Errorf("web search provider %s is not available", config.Provider)
}
// Set API key for providers that need it
if config.APIKey != "" {
s.setProviderAPIKey(config.Provider, provider, config)
}
// Set timeout
timeout := time.Duration(s.config.Timeout) * time.Second
if timeout == 0 {
timeout = 10 * time.Second
}
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
// Perform search
results, err := provider.Search(ctx, query, config.MaxResults, config.IncludeDate)
if err != nil {
return nil, fmt.Errorf("web search failed: %w", err)
}
// Apply blacklist filtering
results = s.filterBlacklist(results, config.Blacklist)
// Apply compression if needed
if config.CompressionMethod != "none" && config.CompressionMethod != "" {
// Compression will be handled later in the integration layer
// For now, we just return the results
}
return results, nil
}
// NewWebSearchService creates a new web search service
func NewWebSearchService(cfg *config.Config) (*WebSearchService, error) {
if cfg.WebSearch == nil {
return nil, fmt.Errorf("web search config is not available")
}
service := &WebSearchService{
providers: make(map[string]WebSearchProvider),
config: cfg.WebSearch,
}
// Initialize providers based on config
for _, providerConfig := range cfg.WebSearch.Providers {
var provider WebSearchProvider
var err error
switch providerConfig.ID {
case "kuaisou":
provider, err = NewKuaisouProvider(providerConfig)
case "baidu":
provider, err = NewBaiduProvider(providerConfig)
case "google":
provider, err = NewGoogleProvider(providerConfig)
case "bing":
provider, err = NewBingProvider(providerConfig)
case "bocha":
provider, err = NewBochaProvider(providerConfig)
case "zhipu":
provider, err = NewZhipuProvider(providerConfig)
case "tavily":
provider, err = NewTavilyProvider(providerConfig)
case "searxng":
provider, err = NewSearxngProvider(providerConfig)
case "exa":
provider, err = NewExaProvider(providerConfig)
default:
logger.Warnf(context.Background(), "Unknown web search provider: %s", providerConfig.ID)
continue
}
if err != nil {
logger.Warnf(context.Background(), "Failed to initialize provider %s: %v", providerConfig.ID, err)
continue
}
service.providers[providerConfig.ID] = provider
logger.Infof(context.Background(), "Initialized web search provider: %s", providerConfig.ID)
}
return service, nil
}
// setProviderAPIKey sets the API key for a provider based on its type
func (s *WebSearchService) setProviderAPIKey(providerID string, provider WebSearchProvider, config *types.WebSearchConfig) {
switch p := provider.(type) {
case *KuaisouProvider:
p.SetAPIKey(config.APIKey)
case *BochaProvider:
p.SetAPIKey(config.APIKey)
case *ZhipuProvider:
p.SetAPIKey(config.APIKey)
case *TavilyProvider:
p.SetAPIKey(config.APIKey)
case *ExaProvider:
p.SetAPIKey(config.APIKey)
case *GoogleProvider:
// Google needs both API key and search engine ID
// For now, we'll use API key as search engine ID if not provided separately
// This can be extended later to support separate fields
p.SetAPIKey(config.APIKey, config.APIKey) // TODO: Add search engine ID to config
}
}
// filterBlacklist filters results based on blacklist rules
func (s *WebSearchService) filterBlacklist(results []*types.WebSearchResult, blacklist []string) []*types.WebSearchResult {
if len(blacklist) == 0 {
return results
}
filtered := make([]*types.WebSearchResult, 0, len(results))
for _, result := range results {
shouldFilter := false
for _, rule := range blacklist {
if s.matchesBlacklistRule(result.URL, rule) {
shouldFilter = true
break
}
}
if !shouldFilter {
filtered = append(filtered, result)
}
}
return filtered
}
// matchesBlacklistRule checks if a URL matches a blacklist rule
// Supports both pattern matching (e.g., *://*.example.com/*) and regex patterns (e.g., /example\.(net|org)/)
func (s *WebSearchService) matchesBlacklistRule(url, rule string) bool {
// Check if it's a regex pattern (starts and ends with /)
if strings.HasPrefix(rule, "/") && strings.HasSuffix(rule, "/") {
pattern := rule[1 : len(rule)-1]
matched, err := regexp.MatchString(pattern, url)
if err != nil {
logger.Warnf(context.Background(), "Invalid regex pattern in blacklist: %s, error: %v", rule, err)
return false
}
return matched
}
// Pattern matching (e.g., *://*.example.com/*)
pattern := strings.ReplaceAll(rule, "*", ".*")
pattern = "^" + pattern + "$"
matched, err := regexp.MatchString(pattern, url)
if err != nil {
logger.Warnf(context.Background(), "Invalid pattern in blacklist: %s, error: %v", rule, err)
return false
}
return matched
}
// ConvertWebSearchResults converts WebSearchResult to SearchResult
func ConvertWebSearchResults(webResults []*types.WebSearchResult) []*types.SearchResult {
results := make([]*types.SearchResult, 0, len(webResults))
for i, webResult := range webResults {
// Use URL as ChunkID for web search results
chunkID := webResult.URL
if chunkID == "" {
chunkID = fmt.Sprintf("web_search_%d", i)
}
// Combine title and snippet as content
content := webResult.Title
if webResult.Snippet != "" {
if content != "" {
content += "\n\n" + webResult.Snippet
} else {
content = webResult.Snippet
}
}
if webResult.Content != "" {
if content != "" {
content += "\n\n" + webResult.Content
} else {
content = webResult.Content
}
}
// Set a default score for web search results (0.6, indicating medium relevance)
score := 0.6
result := &types.SearchResult{
ID: chunkID,
Content: content,
KnowledgeID: "", // Web search results don't have knowledge ID
ChunkIndex: 0,
KnowledgeTitle: webResult.Title,
StartAt: 0,
EndAt: len(content),
Seq: i,
Score: score,
MatchType: types.MatchTypeWebSearch,
SubChunkID: []string{},
Metadata: map[string]string{
"url": webResult.URL,
"source": webResult.Source,
"title": webResult.Title,
"snippet": webResult.Snippet,
},
ChunkType: "web_search",
ParentChunkID: "",
ImageInfo: "",
KnowledgeFilename: "",
KnowledgeSource: "web_search",
}
// Add published date to metadata if available
if webResult.PublishedAt != nil {
result.Metadata["published_at"] = webResult.PublishedAt.Format(time.RFC3339)
}
results = append(results, result)
}
return results
}