From 07a8c1de4c08402746b763f6a3028d4d30b0003e Mon Sep 17 00:00:00 2001 From: wizardchen Date: Mon, 10 Nov 2025 03:14:12 +0800 Subject: [PATCH] feat: Add web search functionality and enhance agent mode integration - Introduced web search configuration in config.yaml, allowing multiple providers with specific settings. - Updated InputField component to manage agent mode and web search toggles, improving user interaction. - Enhanced chat API to include web search status in requests, enabling dynamic query handling. - Implemented backend support for web search integration, merging results with knowledge base queries. - Added tenant-level web search configuration management, allowing customization per tenant. --- config/config.yaml | 63 +++ frontend/src/api/chat/streame.ts | 10 +- frontend/src/api/web-search.ts | 41 ++ frontend/src/components/Input-field.vue | 504 ++++++++++++++++-- frontend/src/stores/settings.ts | 15 +- frontend/src/views/chat/index.vue | 4 + frontend/src/views/settings/Settings.vue | 25 +- .../src/views/settings/WebSearchSettings.vue | 405 ++++++++++++++ .../service/chat_pipline/search.go | 103 ++++ internal/application/service/session.go | 9 +- internal/application/service/web_search.go | 260 +++++++++ .../application/service/web_search/baidu.go | 32 ++ .../application/service/web_search/bing.go | 32 ++ .../application/service/web_search/bocha.go | 38 ++ .../application/service/web_search/exa.go | 38 ++ .../application/service/web_search/google.go | 40 ++ .../application/service/web_search/kuaisou.go | 112 ++++ .../application/service/web_search/searxng.go | 32 ++ .../application/service/web_search/tavily.go | 38 ++ .../application/service/web_search/zhipu.go | 38 ++ .../application/service/web_search_baidu.go | 33 ++ .../application/service/web_search_bing.go | 33 ++ .../application/service/web_search_bocha.go | 39 ++ .../application/service/web_search_exa.go | 39 ++ .../application/service/web_search_google.go | 41 ++ .../application/service/web_search_kuaisou.go | 113 ++++ .../application/service/web_search_searxng.go | 33 ++ .../application/service/web_search_tavily.go | 39 ++ .../application/service/web_search_zhipu.go | 39 ++ internal/config/config.go | 27 + internal/container/container.go | 8 + internal/handler/session/qa.go | 7 +- internal/handler/session/types.go | 1 + internal/handler/tenant.go | 142 +++++ internal/handler/web_search.go | 63 +++ internal/router/router.go | 16 + internal/types/chat_manage.go | 4 + internal/types/embedding.go | 1 + internal/types/interfaces/session.go | 3 +- internal/types/interfaces/web_search.go | 12 + internal/types/tenant.go | 2 + internal/types/web_search.go | 59 ++ migrations/mysql/05-add-web-search-config.sql | 6 + .../paradedb/05-add-web-search-config.sql | 6 + .../000005_add_web_search_config.down.sql | 11 + .../000005_add_web_search_config.up.sql | 14 + 46 files changed, 2574 insertions(+), 56 deletions(-) create mode 100644 frontend/src/api/web-search.ts create mode 100644 frontend/src/views/settings/WebSearchSettings.vue create mode 100644 internal/application/service/web_search.go create mode 100644 internal/application/service/web_search/baidu.go create mode 100644 internal/application/service/web_search/bing.go create mode 100644 internal/application/service/web_search/bocha.go create mode 100644 internal/application/service/web_search/exa.go create mode 100644 internal/application/service/web_search/google.go create mode 100644 internal/application/service/web_search/kuaisou.go create mode 100644 internal/application/service/web_search/searxng.go create mode 100644 internal/application/service/web_search/tavily.go create mode 100644 internal/application/service/web_search/zhipu.go create mode 100644 internal/application/service/web_search_baidu.go create mode 100644 internal/application/service/web_search_bing.go create mode 100644 internal/application/service/web_search_bocha.go create mode 100644 internal/application/service/web_search_exa.go create mode 100644 internal/application/service/web_search_google.go create mode 100644 internal/application/service/web_search_kuaisou.go create mode 100644 internal/application/service/web_search_searxng.go create mode 100644 internal/application/service/web_search_tavily.go create mode 100644 internal/application/service/web_search_zhipu.go create mode 100644 internal/handler/web_search.go create mode 100644 internal/types/interfaces/web_search.go create mode 100644 internal/types/web_search.go create mode 100644 migrations/mysql/05-add-web-search-config.sql create mode 100644 migrations/paradedb/05-add-web-search-config.sql create mode 100644 migrations/versioned/000005_add_web_search_config.down.sql create mode 100644 migrations/versioned/000005_add_web_search_config.up.sql diff --git a/config/config.yaml b/config/config.yaml index 5119b9d7..29ccb79c 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -613,3 +613,66 @@ agent: - get_related_chunks - query_knowledge_graph - get_document_info + +# WebSearch 配置 +web_search: + # 可用搜索引擎列表 + providers: + - id: "kuaisou" + name: "快搜" + free: false + requires_api_key: true + description: "高性能实时网页搜索" + api_url: "https://platform.kuaisou.com/api/v1/search" + - id: "google" + name: "Google" + free: true + requires_api_key: false + description: "Google搜索(免费)" + api_url: "https://www.googleapis.com/customsearch/v1" + - id: "baidu" + name: "百度" + free: true + requires_api_key: false + description: "百度搜索(免费)" + - id: "bing" + name: "Bing" + free: true + requires_api_key: false + description: "Bing搜索(免费)" + - id: "bocha" + name: "博查" + free: false + requires_api_key: true + description: "专为AI应用设计的搜索API" + - id: "zhipu" + name: "智谱" + free: false + requires_api_key: true + description: "智谱搜索API" + - id: "tavily" + name: "Tavily" + free: false + requires_api_key: true + description: "Tavily AI搜索API" + - id: "searxng" + name: "Searxng" + free: true + requires_api_key: false + description: "Searxng开源元搜索引擎" + - id: "exa" + name: "Exa" + free: false + requires_api_key: true + description: "Exa AI搜索API" + + # 默认配置 + default: + provider: "kuaisou" + max_results: 5 + include_date: true + compression_method: "none" + blacklist: [] + + # 全局超时设置 + timeout: 10 diff --git a/frontend/src/api/chat/streame.ts b/frontend/src/api/chat/streame.ts index 54e5c37e..2d2bb2ca 100644 --- a/frontend/src/api/chat/streame.ts +++ b/frontend/src/api/chat/streame.ts @@ -28,7 +28,7 @@ export function useStream() { let renderTimer: number | null = null // 启动流式请求 - const startStream = async (params: { session_id: any; query: any; knowledge_base_ids?: string[]; agent_enabled?: boolean; summary_model_id?: string; method: string; url: string }) => { + const startStream = async (params: { session_id: any; query: any; knowledge_base_ids?: string[]; agent_enabled?: boolean; web_search_enabled?: boolean; summary_model_id?: string; mcp_service_ids?: string[]; method: string; url: string }) => { // 重置状态 output.value = ''; error.value = null; @@ -69,10 +69,18 @@ export function useStream() { if (params.knowledge_base_ids !== undefined && params.knowledge_base_ids.length > 0) { postBody.knowledge_base_ids = params.knowledge_base_ids; } + // Include web_search_enabled if provided + if (params.web_search_enabled !== undefined) { + postBody.web_search_enabled = params.web_search_enabled; + } // Include summary_model_id if provided (for non-Agent mode) if (params.summary_model_id) { postBody.summary_model_id = params.summary_model_id; } + // Include mcp_service_ids if provided (for Agent mode) + if (params.mcp_service_ids !== undefined && params.mcp_service_ids.length > 0) { + postBody.mcp_service_ids = params.mcp_service_ids; + } await fetchEventSource(url, { method: params.method, diff --git a/frontend/src/api/web-search.ts b/frontend/src/api/web-search.ts new file mode 100644 index 00000000..ec391d73 --- /dev/null +++ b/frontend/src/api/web-search.ts @@ -0,0 +1,41 @@ +import { get, put } from '@/utils/request' + +// WebSearchProviderConfig represents information about a web search provider +export interface WebSearchProviderConfig { + id: string + name: string + free: boolean + requires_api_key: boolean + description?: string + api_url?: string +} + +// WebSearchConfig represents the web search configuration for a tenant +export interface WebSearchConfig { + provider: string + api_key?: string + max_results: number + include_date: boolean + compression_method: string + blacklist: string[] + embedding_model_id?: string + embedding_dimension?: number + rerank_model_id?: string + document_fragments?: number +} + +// Get web search providers +export function getWebSearchProviders() { + return get('/api/v1/web-search/providers') +} + +// Get tenant web search config +export function getTenantWebSearchConfig() { + return get('/api/v1/tenants/web-search-config') +} + +// Update tenant web search config +export function updateTenantWebSearchConfig(config: WebSearchConfig) { + return put('/api/v1/tenants/web-search-config', config) +} + diff --git a/frontend/src/components/Input-field.vue b/frontend/src/components/Input-field.vue index 3a94e2da..b35afbea 100644 --- a/frontend/src/components/Input-field.vue +++ b/frontend/src/components/Input-field.vue @@ -17,6 +17,9 @@ const uiStore = useUIStore(); let query = ref(""); const showKbSelector = ref(false); const atButtonRef = ref(); +const showAgentModeSelector = ref(false); +const agentModeButtonRef = ref(); +const agentModeDropdownStyle = ref>({}); const props = defineProps({ isReplying: { @@ -34,6 +37,7 @@ const props = defineProps({ }); const isAgentEnabled = computed(() => settingsStore.isAgentEnabled); +const isWebSearchEnabled = computed(() => settingsStore.isWebSearchEnabled); const selectedKbIds = computed(() => settingsStore.settings.selectedKnowledgeBases || []); // 获取已选择的知识库信息 @@ -193,6 +197,11 @@ const closeModelSelector = () => { showModelSelector.value = false; }; +// 关闭 Agent 模式选择器(点击外部) +const closeAgentModeSelector = () => { + showAgentModeSelector.value = false; +}; + onMounted(() => { loadKnowledgeBases(); @@ -211,16 +220,21 @@ onMounted(() => { // 监听点击外部关闭下拉菜单 document.addEventListener('click', closeModelSelector); + document.addEventListener('click', closeAgentModeSelector); // 监听窗口大小变化,重新计算位置 window.addEventListener('resize', () => { if (showModelSelector.value) { updateModelDropdownPosition(); } + if (showAgentModeSelector.value) { + updateAgentModeDropdownPosition(); + } }); }); onUnmounted(() => { document.removeEventListener('click', closeModelSelector); + document.removeEventListener('click', closeAgentModeSelector); }); // 监听路由变化 @@ -317,6 +331,51 @@ const updateModelDropdownPosition = () => { }; }; +// 计算 Agent 模式下拉菜单位置 +const updateAgentModeDropdownPosition = () => { + const anchor = agentModeButtonRef.value; + + if (!anchor) { + agentModeDropdownStyle.value = { + position: 'fixed', + top: '50%', + left: '50%', + transform: 'translate(-50%, -50%)' + }; + return; + } + + const rect = anchor.getBoundingClientRect(); + const dropdownWidth = 200; + const offsetY = 8; + + // 计算位置:默认在按钮下方 + let top = rect.bottom + offsetY; + let left = rect.left; + + // 检查右侧边界 + if (left + dropdownWidth > window.innerWidth) { + left = window.innerWidth - dropdownWidth - 10; + } + + // 检查下方空间 + const dropdownMaxHeight = 150; + if (top + dropdownMaxHeight > window.innerHeight) { + // 如果下方空间不足,显示在按钮上方 + top = rect.top - dropdownMaxHeight - offsetY; + if (top < 10) { + top = 10; + } + } + + agentModeDropdownStyle.value = { + position: 'fixed', + top: `${Math.round(top)}px`, + left: `${Math.round(left)}px`, + width: `${dropdownWidth}px` + }; +}; + const toggleModelSelector = () => { showModelSelector.value = !showModelSelector.value; if (showModelSelector.value) { @@ -326,6 +385,36 @@ const toggleModelSelector = () => { } } +const toggleAgentModeSelector = () => { + // 如果 Agent 未就绪,显示提示 + if (!settingsStore.isAgentReady && !isAgentEnabled.value) { + toggleAgentMode(); + return; + } + + showAgentModeSelector.value = !showAgentModeSelector.value; + if (showAgentModeSelector.value) { + nextTick(() => { + updateAgentModeDropdownPosition(); + }); + } +} + +const selectAgentMode = (mode: 'normal' | 'agent') => { + if (mode === 'agent' && !settingsStore.isAgentReady) { + toggleAgentMode(); + showAgentModeSelector.value = false; + return; + } + + const shouldEnableAgent = mode === 'agent'; + if (shouldEnableAgent !== isAgentEnabled.value) { + settingsStore.toggleAgent(shouldEnableAgent); + MessagePlugin.success(shouldEnableAgent ? '已切换到 Agent 模式' : '已切换到普通模式'); + } + showAgentModeSelector.value = false; +} + const clearvalue = () => { query.value = ""; } @@ -389,6 +478,13 @@ const toggleAgentMode = () => { MessagePlugin.success(message); } +const toggleWebSearch = () => { + const currentValue = settingsStore.isWebSearchEnabled; + const newValue = !currentValue; + settingsStore.toggleWebSearch(newValue); + MessagePlugin.success(newValue ? '网络搜索已开启' : '网络搜索已关闭'); +} + const toggleKbSelector = () => { showKbSelector.value = !showKbSelector.value; } @@ -437,41 +533,142 @@ onBeforeRouteUpdate((to, from, next) => {
- - +
- -
+ - Agent + + + + + {{ isAgentEnabled ? 'Agent 模式' : '普通模式' }} + + + + +
+ + + +
+
+
+
+ 普通模式 + 基于知识库的 RAG 问答 +
+ + + +
+
+
+ Agent 模式 + ReAct 推理框架,多步思考 +
+ + + +
+ + + +
+
+ +
+
+
+ + + +
+ + + + + + +
@@ -719,30 +916,76 @@ const getImgSrc = (url: string) => { } } -.agent-btn { - width: 28px; +.agent-mode-btn { height: 28px; - padding: 0; - - &.active { - background: rgba(7, 192, 95, 0.1); + padding: 0 10px; + min-width: auto; + font-weight: 500; + border: 1.5px solid transparent; + transition: all 0.2s ease; + position: relative; + + &.active, + &.agent-active { + background: linear-gradient(135deg, rgba(7, 192, 95, 0.15) 0%, rgba(7, 192, 95, 0.1) 100%); + border-color: rgba(7, 192, 95, 0.4); + box-shadow: 0 2px 6px rgba(7, 192, 95, 0.12); + + .agent-mode-text { + color: #07C05F; + font-weight: 600; + } + + .agent-icon { + filter: brightness(0) saturate(100%) invert(58%) sepia(87%) saturate(1234%) hue-rotate(95deg) brightness(98%) contrast(89%); + } + + .dropdown-arrow { + color: #07C05F; + } &:hover { - background: rgba(7, 192, 95, 0.15); + background: linear-gradient(135deg, rgba(7, 192, 95, 0.2) 0%, rgba(7, 192, 95, 0.15) 100%); + border-color: rgba(7, 192, 95, 0.6); + box-shadow: 0 3px 10px rgba(7, 192, 95, 0.2); + transform: translateY(-1px); } } - - &.disabled { - opacity: 0.5; - cursor: not-allowed; + + &:not(.agent-active) { + background: rgba(255, 255, 255, 0.8); + border-color: #e0e0e0; + + .agent-mode-text { + color: #666; + } + + .normal-mode-icon { + color: #666; + } &:hover { - background: #f5f5f5; - transform: none; + background: rgba(255, 255, 255, 1); + border-color: #b0b0b0; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.08); } } } +.agent-icon { + width: 18px; + height: 18px; + flex-shrink: 0; +} + +.agent-mode-text { + font-size: 13px; + color: #666; + font-weight: 500; + white-space: nowrap; + margin: 0 4px; +} + .control-icon { width: 18px; height: 18px; @@ -775,6 +1018,48 @@ const getImgSrc = (url: string) => { color: #07C05F; } +.websearch-btn { + width: 28px; + height: 28px; + padding: 0; + min-width: auto; + display: flex; + align-items: center; + justify-content: center; + + &.active { + background: rgba(7, 192, 95, 0.1); + + .websearch-icon { + color: #07C05F; + } + + &:hover { + background: rgba(7, 192, 95, 0.15); + } + } + + &:not(.active) { + .websearch-icon { + color: #666; + } + + &:hover { + background: #f0f0f0; + + .websearch-icon { + color: #333; + } + } + } +} + +.websearch-icon { + width: 18px; + height: 18px; + transition: all 0.2s ease; +} + .dropdown-arrow { transition: transform 0.2s ease; width: 10px; @@ -1124,4 +1409,131 @@ const getImgSrc = (url: string) => { background: rgba(82, 196, 26, 0.1); color: #52c41a; } + +/* Agent 模式选择下拉菜单 */ +.agent-mode-selector-overlay { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + z-index: 9998; +} + +.agent-mode-selector-dropdown { + z-index: 9999; + background: white; + border-radius: 8px; + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.12), 0 2px 8px rgba(7, 192, 95, 0.08); + border: 1px solid #e5e7eb; + overflow: hidden; + padding: 4px; + min-width: 200px; +} + +.agent-mode-option { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 14px; + cursor: pointer; + transition: all 0.2s ease; + border-radius: 6px; + position: relative; + margin: 2px 0; + + &:hover:not(.disabled) { + background: rgba(7, 192, 95, 0.1); + transform: translateX(2px); + } + + &:active:not(.disabled) { + transform: translateX(1px); + } + + &.disabled { + opacity: 0.6; + cursor: not-allowed; + + &:hover { + background: transparent; + transform: none; + } + } + + // 选中状态的选项有更明显的背景 + &.selected { + background: rgba(7, 192, 95, 0.06); + + .agent-mode-option-name { + color: #07C05F; + font-weight: 700; + } + } +} + +.agent-mode-option-main { + display: flex; + flex-direction: column; + gap: 2px; + flex: 1; + min-width: 0; +} + +.agent-mode-option-name { + font-size: 14px; + font-weight: 600; + color: #333; + line-height: 1.4; + transition: color 0.2s ease; +} + +.agent-mode-option-desc { + font-size: 12px; + color: #999; + line-height: 1.3; +} + +.check-icon { + width: 16px; + height: 16px; + color: #07C05F; + flex-shrink: 0; + margin-left: 8px; +} + +.agent-mode-warning { + display: flex; + align-items: center; + margin-left: 8px; + + .warning-icon { + color: #ff9800; + font-size: 16px; + } +} + +.agent-mode-footer { + padding: 8px 12px; + border-top: 1px solid #f0f0f0; + margin-top: 4px; +} + +.agent-mode-link { + color: #07C05F; + text-decoration: none; + font-size: 12px; + font-weight: 500; + display: inline-flex; + align-items: center; + gap: 4px; + transition: all 0.2s ease; + + &:hover { + color: #00a651; + text-decoration: underline; + } +} + + diff --git a/frontend/src/stores/settings.ts b/frontend/src/stores/settings.ts index 6ca82f81..698bf957 100644 --- a/frontend/src/stores/settings.ts +++ b/frontend/src/stores/settings.ts @@ -10,6 +10,7 @@ interface Settings { selectedKnowledgeBases: string[]; // 当前选中的知识库ID列表 modelConfig: ModelConfig; // 模型配置 ollamaConfig: OllamaConfig; // Ollama配置 + webSearchEnabled: boolean; // 网络搜索是否启用 } // Agent 配置接口 @@ -71,7 +72,8 @@ const defaultSettings: Settings = { ollamaConfig: { baseUrl: "http://localhost:11434", enabled: true - } + }, + webSearchEnabled: false // 默认关闭网络搜索 }; export const useSettingsStore = defineStore("settings", { @@ -97,6 +99,9 @@ export const useSettingsStore = defineStore("settings", { // 获取模型配置 modelConfig: (state) => state.settings.modelConfig || defaultSettings.modelConfig, + + // 网络搜索是否启用 + isWebSearchEnabled: (state) => state.settings.webSearchEnabled || false, }, actions: { @@ -224,7 +229,7 @@ export const useSettingsStore = defineStore("settings", { // 移除单个知识库 removeKnowledgeBase(kbId: string) { this.settings.selectedKnowledgeBases = - this.settings.selectedKnowledgeBases.filter(id => id !== kbId); + this.settings.selectedKnowledgeBases.filter((id: string) => id !== kbId); localStorage.setItem("WeKnora_settings", JSON.stringify(this.settings)); }, @@ -238,5 +243,11 @@ export const useSettingsStore = defineStore("settings", { getSelectedKnowledgeBases(): string[] { return this.settings.selectedKnowledgeBases || []; }, + + // 启用/禁用网络搜索 + toggleWebSearch(enabled: boolean) { + this.settings.webSearchEnabled = enabled; + localStorage.setItem("WeKnora_settings", JSON.stringify(this.settings)); + }, }, }); \ No newline at end of file diff --git a/frontend/src/views/chat/index.vue b/frontend/src/views/chat/index.vue index 15f4eb72..1839a4e9 100644 --- a/frontend/src/views/chat/index.vue +++ b/frontend/src/views/chat/index.vue @@ -269,6 +269,9 @@ const sendMsg = async (value, modelId = '') => { // Get agent mode status from settings store const agentEnabled = useSettingsStoreInstance.isAgentEnabled; + // Get web search status from settings store + const webSearchEnabled = useSettingsStoreInstance.isWebSearchEnabled; + // Get knowledge_base_ids from settings store (selected by user via KnowledgeBaseSelector) const kbIds = useSettingsStoreInstance.settings.selectedKnowledgeBases || []; @@ -294,6 +297,7 @@ const sendMsg = async (value, modelId = '') => { session_id: session_id.value, knowledge_base_ids: kbIds, agent_enabled: agentEnabled, + web_search_enabled: webSearchEnabled, summary_model_id: modelId, mcp_service_ids: mcpServiceIds, query: value, diff --git a/frontend/src/views/settings/Settings.vue b/frontend/src/views/settings/Settings.vue index 2db01e26..0db9a0a5 100644 --- a/frontend/src/views/settings/Settings.vue +++ b/frontend/src/views/settings/Settings.vue @@ -26,7 +26,23 @@ }]" @click="handleNavClick(item)" > - + + + + + + + + + {{ item.label }}
+ +
+ +
+
@@ -118,6 +139,7 @@ import GeneralSettings from './GeneralSettings.vue' import ModelSettings from './ModelSettings.vue' import OllamaSettings from './OllamaSettings.vue' import McpSettings from './McpSettings.vue' +import WebSearchSettings from './WebSearchSettings.vue' const route = useRoute() const router = useRouter() @@ -142,6 +164,7 @@ const navItems = [ }, { key: 'ollama', icon: 'server', label: 'Ollama' }, { key: 'agent', icon: 'chat', label: 'Agent 配置' }, + { key: 'websearch', icon: 'search', label: '网络搜索' }, { key: 'mcp', icon: 'link', label: 'MCP 服务' }, { key: 'system', icon: 'info-circle', label: '系统信息' }, { key: 'tenant', icon: 'user-circle', label: '租户信息' }, diff --git a/frontend/src/views/settings/WebSearchSettings.vue b/frontend/src/views/settings/WebSearchSettings.vue new file mode 100644 index 00000000..ad91422e --- /dev/null +++ b/frontend/src/views/settings/WebSearchSettings.vue @@ -0,0 +1,405 @@ + + + + + + diff --git a/internal/application/service/chat_pipline/search.go b/internal/application/service/chat_pipline/search.go index f920debb..69fa0c2d 100644 --- a/internal/application/service/chat_pipline/search.go +++ b/internal/application/service/chat_pipline/search.go @@ -2,8 +2,10 @@ package chatpipline import ( "context" + "fmt" "strings" "sync" + "time" "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/logger" @@ -16,17 +18,23 @@ type PluginSearch struct { knowledgeBaseService interfaces.KnowledgeBaseService modelService interfaces.ModelService config *config.Config + webSearchService interfaces.WebSearchService + tenantService interfaces.TenantService } func NewPluginSearch(eventManager *EventManager, knowledgeBaseService interfaces.KnowledgeBaseService, modelService interfaces.ModelService, config *config.Config, + webSearchService interfaces.WebSearchService, + tenantService interfaces.TenantService, ) *PluginSearch { res := &PluginSearch{ knowledgeBaseService: knowledgeBaseService, modelService: modelService, config: config, + webSearchService: webSearchService, + tenantService: tenantService, } eventManager.Register(res) return res @@ -135,6 +143,33 @@ func (p *PluginSearch) OnEvent(ctx context.Context, chatManage.SearchResult = append(chatManage.SearchResult, processedResults...) } + // Perform web search if enabled and merge results with KB search results + if chatManage.WebSearchEnabled && p.webSearchService != nil && p.tenantService != nil && chatManage.TenantID > 0 { + // Get tenant to retrieve web search config + tenant, err := p.tenantService.GetTenantByID(ctx, chatManage.TenantID) + if err != nil { + logger.Warnf(ctx, "Failed to get tenant for web search: %v", err) + } else if tenant != nil && tenant.WebSearchConfig != nil && tenant.WebSearchConfig.Provider != "" { + // Perform web search in parallel with KB search (already completed) + logger.Infof(ctx, "Performing web search with provider: %s", tenant.WebSearchConfig.Provider) + webResults, err := p.webSearchService.Search(ctx, tenant.WebSearchConfig, chatManage.RewriteQuery) + if err != nil { + logger.Warnf(ctx, "Web search failed: %v", err) + } else { + // Convert web search results to SearchResult + webSearchResults := convertWebSearchResults(webResults) + logger.Infof(ctx, "Web search returned %d results", len(webSearchResults)) + // Merge web search results with KB search results + if len(webSearchResults) > 0 { + chatManage.SearchResult = append(chatManage.SearchResult, webSearchResults...) + logger.Infof(ctx, "Merged web search results, total results: %d", len(chatManage.SearchResult)) + } + } + } else { + logger.Warnf(ctx, "Web search enabled but no valid configuration found for tenant %d", chatManage.TenantID) + } + } + // Remove duplicate results chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult) @@ -180,3 +215,71 @@ func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult } return uniqueResults } + +// convertWebSearchResults converts WebSearchResult to SearchResult +// This is a duplicate of the function in service/web_search.go to avoid circular imports +func convertWebSearchResults(webResults []*types.WebSearchResult) []*types.SearchResult { + results := make([]*types.SearchResult, 0, len(webResults)) + + for i, webResult := range webResults { + // Use URL as ChunkID for web search results + chunkID := webResult.URL + if chunkID == "" { + chunkID = fmt.Sprintf("web_search_%d", i) + } + + // Combine title and snippet as content + content := webResult.Title + if webResult.Snippet != "" { + if content != "" { + content += "\n\n" + webResult.Snippet + } else { + content = webResult.Snippet + } + } + if webResult.Content != "" { + if content != "" { + content += "\n\n" + webResult.Content + } else { + content = webResult.Content + } + } + + // Set a default score for web search results (0.6, indicating medium relevance) + score := 0.6 + + result := &types.SearchResult{ + ID: chunkID, + Content: content, + KnowledgeID: "", // Web search results don't have knowledge ID + ChunkIndex: 0, + KnowledgeTitle: webResult.Title, + StartAt: 0, + EndAt: len(content), + Seq: i, + Score: score, + MatchType: types.MatchTypeWebSearch, + SubChunkID: []string{}, + Metadata: map[string]string{ + "url": webResult.URL, + "source": webResult.Source, + "title": webResult.Title, + "snippet": webResult.Snippet, + }, + ChunkType: "web_search", + ParentChunkID: "", + ImageInfo: "", + KnowledgeFilename: "", + KnowledgeSource: "web_search", + } + + // Add published date to metadata if available + if webResult.PublishedAt != nil { + result.Metadata["published_at"] = webResult.PublishedAt.Format(time.RFC3339) + } + + results = append(results, result) + } + + return results +} diff --git a/internal/application/service/session.go b/internal/application/service/session.go index 39a070b0..91d09a8c 100644 --- a/internal/application/service/session.go +++ b/internal/application/service/session.go @@ -400,8 +400,8 @@ func (s *sessionService) GenerateTitleAsync(ctx context.Context, session *types. // KnowledgeQA performs knowledge base question answering with LLM summarization // Events are emitted through eventBus (references, answer chunks, completion) -func (s *sessionService) KnowledgeQA(ctx context.Context, session *types.Session, query string, knowledgeBaseIDs []string, assistantMessageID string, summaryModelID string, eventBus *event.EventBus) error { - logger.Infof(ctx, "Knowledge base question answering parameters, session ID: %s, query: %s", session.ID, query) +func (s *sessionService) KnowledgeQA(ctx context.Context, session *types.Session, query string, knowledgeBaseIDs []string, assistantMessageID string, summaryModelID string, webSearchEnabled bool, eventBus *event.EventBus) error { + logger.Infof(ctx, "Knowledge base question answering parameters, session ID: %s, query: %s, webSearchEnabled: %v", session.ID, query, webSearchEnabled) // If no knowledge base IDs provided, fall back to session's default if len(knowledgeBaseIDs) == 0 { @@ -456,6 +456,11 @@ func (s *sessionService) KnowledgeQA(ctx context.Context, session *types.Session EventBus: eventBus.AsEventBusInterface(), // NEW: For pipeline to emit events directly } + // Store web search configuration in chatManage for pipeline processing + // PluginSearch will handle web search integration + chatManage.TenantID = session.TenantID + chatManage.WebSearchEnabled = webSearchEnabled + // Start knowledge QA event processing logger.Info(ctx, "Triggering knowledge base question answering event") err = s.KnowledgeQAByEvent(ctx, chatManage, types.Pipline["rag_stream"]) diff --git a/internal/application/service/web_search.go b/internal/application/service/web_search.go new file mode 100644 index 00000000..2c1b1495 --- /dev/null +++ b/internal/application/service/web_search.go @@ -0,0 +1,260 @@ +package service + +import ( + "context" + "fmt" + "regexp" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// WebSearchProvider defines the interface for web search providers +type WebSearchProvider interface { + Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) + Name() string +} + +// WebSearchService provides web search functionality +type WebSearchService struct { + providers map[string]WebSearchProvider + config *config.WebSearchConfig +} + +// Search performs web search using the specified provider +// This method implements the interface expected by PluginSearch +func (s *WebSearchService) Search(ctx context.Context, config *types.WebSearchConfig, query string) ([]*types.WebSearchResult, error) { + if config == nil { + return nil, fmt.Errorf("web search config is required") + } + + provider, ok := s.providers[config.Provider] + if !ok { + return nil, fmt.Errorf("web search provider %s is not available", config.Provider) + } + + // Set API key for providers that need it + if config.APIKey != "" { + s.setProviderAPIKey(config.Provider, provider, config) + } + + // Set timeout + timeout := time.Duration(s.config.Timeout) * time.Second + if timeout == 0 { + timeout = 10 * time.Second + } + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // Perform search + results, err := provider.Search(ctx, query, config.MaxResults, config.IncludeDate) + if err != nil { + return nil, fmt.Errorf("web search failed: %w", err) + } + + // Apply blacklist filtering + results = s.filterBlacklist(results, config.Blacklist) + + // Apply compression if needed + if config.CompressionMethod != "none" && config.CompressionMethod != "" { + // Compression will be handled later in the integration layer + // For now, we just return the results + } + + return results, nil +} + +// NewWebSearchService creates a new web search service +func NewWebSearchService(cfg *config.Config) (*WebSearchService, error) { + if cfg.WebSearch == nil { + return nil, fmt.Errorf("web search config is not available") + } + + service := &WebSearchService{ + providers: make(map[string]WebSearchProvider), + config: cfg.WebSearch, + } + + // Initialize providers based on config + for _, providerConfig := range cfg.WebSearch.Providers { + var provider WebSearchProvider + var err error + + switch providerConfig.ID { + case "kuaisou": + provider, err = NewKuaisouProvider(providerConfig) + case "baidu": + provider, err = NewBaiduProvider(providerConfig) + case "google": + provider, err = NewGoogleProvider(providerConfig) + case "bing": + provider, err = NewBingProvider(providerConfig) + case "bocha": + provider, err = NewBochaProvider(providerConfig) + case "zhipu": + provider, err = NewZhipuProvider(providerConfig) + case "tavily": + provider, err = NewTavilyProvider(providerConfig) + case "searxng": + provider, err = NewSearxngProvider(providerConfig) + case "exa": + provider, err = NewExaProvider(providerConfig) + default: + logger.Warnf(context.Background(), "Unknown web search provider: %s", providerConfig.ID) + continue + } + + if err != nil { + logger.Warnf(context.Background(), "Failed to initialize provider %s: %v", providerConfig.ID, err) + continue + } + + service.providers[providerConfig.ID] = provider + logger.Infof(context.Background(), "Initialized web search provider: %s", providerConfig.ID) + } + + return service, nil +} + +// setProviderAPIKey sets the API key for a provider based on its type +func (s *WebSearchService) setProviderAPIKey(providerID string, provider WebSearchProvider, config *types.WebSearchConfig) { + switch p := provider.(type) { + case *KuaisouProvider: + p.SetAPIKey(config.APIKey) + case *BochaProvider: + p.SetAPIKey(config.APIKey) + case *ZhipuProvider: + p.SetAPIKey(config.APIKey) + case *TavilyProvider: + p.SetAPIKey(config.APIKey) + case *ExaProvider: + p.SetAPIKey(config.APIKey) + case *GoogleProvider: + // Google needs both API key and search engine ID + // For now, we'll use API key as search engine ID if not provided separately + // This can be extended later to support separate fields + p.SetAPIKey(config.APIKey, config.APIKey) // TODO: Add search engine ID to config + } +} + +// filterBlacklist filters results based on blacklist rules +func (s *WebSearchService) filterBlacklist(results []*types.WebSearchResult, blacklist []string) []*types.WebSearchResult { + if len(blacklist) == 0 { + return results + } + + filtered := make([]*types.WebSearchResult, 0, len(results)) + + for _, result := range results { + shouldFilter := false + + for _, rule := range blacklist { + if s.matchesBlacklistRule(result.URL, rule) { + shouldFilter = true + break + } + } + + if !shouldFilter { + filtered = append(filtered, result) + } + } + + return filtered +} + +// matchesBlacklistRule checks if a URL matches a blacklist rule +// Supports both pattern matching (e.g., *://*.example.com/*) and regex patterns (e.g., /example\.(net|org)/) +func (s *WebSearchService) matchesBlacklistRule(url, rule string) bool { + // Check if it's a regex pattern (starts and ends with /) + if strings.HasPrefix(rule, "/") && strings.HasSuffix(rule, "/") { + pattern := rule[1 : len(rule)-1] + matched, err := regexp.MatchString(pattern, url) + if err != nil { + logger.Warnf(context.Background(), "Invalid regex pattern in blacklist: %s, error: %v", rule, err) + return false + } + return matched + } + + // Pattern matching (e.g., *://*.example.com/*) + pattern := strings.ReplaceAll(rule, "*", ".*") + pattern = "^" + pattern + "$" + matched, err := regexp.MatchString(pattern, url) + if err != nil { + logger.Warnf(context.Background(), "Invalid pattern in blacklist: %s, error: %v", rule, err) + return false + } + return matched +} + +// ConvertWebSearchResults converts WebSearchResult to SearchResult +func ConvertWebSearchResults(webResults []*types.WebSearchResult) []*types.SearchResult { + results := make([]*types.SearchResult, 0, len(webResults)) + + for i, webResult := range webResults { + // Use URL as ChunkID for web search results + chunkID := webResult.URL + if chunkID == "" { + chunkID = fmt.Sprintf("web_search_%d", i) + } + + // Combine title and snippet as content + content := webResult.Title + if webResult.Snippet != "" { + if content != "" { + content += "\n\n" + webResult.Snippet + } else { + content = webResult.Snippet + } + } + if webResult.Content != "" { + if content != "" { + content += "\n\n" + webResult.Content + } else { + content = webResult.Content + } + } + + // Set a default score for web search results (0.6, indicating medium relevance) + score := 0.6 + + result := &types.SearchResult{ + ID: chunkID, + Content: content, + KnowledgeID: "", // Web search results don't have knowledge ID + ChunkIndex: 0, + KnowledgeTitle: webResult.Title, + StartAt: 0, + EndAt: len(content), + Seq: i, + Score: score, + MatchType: types.MatchTypeWebSearch, + SubChunkID: []string{}, + Metadata: map[string]string{ + "url": webResult.URL, + "source": webResult.Source, + "title": webResult.Title, + "snippet": webResult.Snippet, + }, + ChunkType: "web_search", + ParentChunkID: "", + ImageInfo: "", + KnowledgeFilename: "", + KnowledgeSource: "web_search", + } + + // Add published date to metadata if available + if webResult.PublishedAt != nil { + result.Metadata["published_at"] = webResult.PublishedAt.Format(time.RFC3339) + } + + results = append(results, result) + } + + return results +} diff --git a/internal/application/service/web_search/baidu.go b/internal/application/service/web_search/baidu.go new file mode 100644 index 00000000..aa527eca --- /dev/null +++ b/internal/application/service/web_search/baidu.go @@ -0,0 +1,32 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// BaiduProvider implements web search using Baidu API +type BaiduProvider struct { + apiURL string +} + +// NewBaiduProvider creates a new Baidu provider +func NewBaiduProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &BaiduProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *BaiduProvider) Name() string { + return "baidu" +} + +// Search performs a web search using Baidu API +func (p *BaiduProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Baidu search API + return nil, fmt.Errorf("baidu search provider is not yet implemented") +} diff --git a/internal/application/service/web_search/bing.go b/internal/application/service/web_search/bing.go new file mode 100644 index 00000000..bb617841 --- /dev/null +++ b/internal/application/service/web_search/bing.go @@ -0,0 +1,32 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// BingProvider implements web search using Bing Search API +type BingProvider struct { + apiURL string +} + +// NewBingProvider creates a new Bing provider +func NewBingProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &BingProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *BingProvider) Name() string { + return "bing" +} + +// Search performs a web search using Bing Search API +func (p *BingProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Bing Search API + return nil, fmt.Errorf("bing search provider is not yet implemented") +} diff --git a/internal/application/service/web_search/bocha.go b/internal/application/service/web_search/bocha.go new file mode 100644 index 00000000..7f55b375 --- /dev/null +++ b/internal/application/service/web_search/bocha.go @@ -0,0 +1,38 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// BochaProvider implements web search using Bocha API +type BochaProvider struct { + apiKey string + apiURL string +} + +// NewBochaProvider creates a new Bocha provider +func NewBochaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &BochaProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *BochaProvider) Name() string { + return "bocha" +} + +// Search performs a web search using Bocha API +func (p *BochaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Bocha search API + return nil, fmt.Errorf("bocha search provider is not yet implemented") +} + +// SetAPIKey sets the API key for the provider +func (p *BochaProvider) SetAPIKey(apiKey string) { + p.apiKey = apiKey +} diff --git a/internal/application/service/web_search/exa.go b/internal/application/service/web_search/exa.go new file mode 100644 index 00000000..f15df037 --- /dev/null +++ b/internal/application/service/web_search/exa.go @@ -0,0 +1,38 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// ExaProvider implements web search using Exa API +type ExaProvider struct { + apiKey string + apiURL string +} + +// NewExaProvider creates a new Exa provider +func NewExaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &ExaProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *ExaProvider) Name() string { + return "exa" +} + +// Search performs a web search using Exa API +func (p *ExaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Exa search API + return nil, fmt.Errorf("exa search provider is not yet implemented") +} + +// SetAPIKey sets the API key for the provider +func (p *ExaProvider) SetAPIKey(apiKey string) { + p.apiKey = apiKey +} diff --git a/internal/application/service/web_search/google.go b/internal/application/service/web_search/google.go new file mode 100644 index 00000000..3689cb92 --- /dev/null +++ b/internal/application/service/web_search/google.go @@ -0,0 +1,40 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// GoogleProvider implements web search using Google Custom Search API +type GoogleProvider struct { + apiKey string + searchEngineID string + apiURL string +} + +// NewGoogleProvider creates a new Google provider +func NewGoogleProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &GoogleProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *GoogleProvider) Name() string { + return "google" +} + +// Search performs a web search using Google Custom Search API +func (p *GoogleProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Google Custom Search API + return nil, fmt.Errorf("google search provider is not yet implemented") +} + +// SetAPIKey sets the API key and search engine ID +func (p *GoogleProvider) SetAPIKey(apiKey, searchEngineID string) { + p.apiKey = apiKey + p.searchEngineID = searchEngineID +} diff --git a/internal/application/service/web_search/kuaisou.go b/internal/application/service/web_search/kuaisou.go new file mode 100644 index 00000000..4a0fed9a --- /dev/null +++ b/internal/application/service/web_search/kuaisou.go @@ -0,0 +1,112 @@ +package service + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// KuaisouProvider implements web search using Kuaisou API +type KuaisouProvider struct { + apiKey string + apiURL string + client *http.Client +} + +// NewKuaisouProvider creates a new Kuaisou provider +func NewKuaisouProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + if cfg.APIURL == "" { + return nil, fmt.Errorf("kuaisou API URL is required") + } + + return &KuaisouProvider{ + apiKey: "", // Will be set from tenant config + apiURL: cfg.APIURL, + client: &http.Client{ + Timeout: 10 * time.Second, + }, + }, nil +} + +// Name returns the provider name +func (p *KuaisouProvider) Name() string { + return "kuaisou" +} + +// Search performs a web search using Kuaisou API +func (p *KuaisouProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + if p.apiKey == "" { + return nil, fmt.Errorf("kuaisou API key is required") + } + + // Prepare request + reqBody := map[string]interface{}{ + "query": query, + "max_results": maxResults, + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", p.apiURL, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+p.apiKey) + + // Send request + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("kuaisou API returned status %d: %s", resp.StatusCode, string(body)) + } + + // Parse response + var response struct { + Results []struct { + Title string `json:"title"` + URL string `json:"url"` + Snippet string `json:"snippet"` + } `json:"results"` + } + + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + // Convert to WebSearchResult + results := make([]*types.WebSearchResult, 0, len(response.Results)) + for _, item := range response.Results { + results = append(results, &types.WebSearchResult{ + Title: item.Title, + URL: item.URL, + Snippet: item.Snippet, + Source: "kuaisou", + }) + } + + logger.Infof(ctx, "Kuaisou search returned %d results for query: %s", len(results), query) + return results, nil +} + +// SetAPIKey sets the API key for the provider +func (p *KuaisouProvider) SetAPIKey(apiKey string) { + p.apiKey = apiKey +} diff --git a/internal/application/service/web_search/searxng.go b/internal/application/service/web_search/searxng.go new file mode 100644 index 00000000..d2f5d283 --- /dev/null +++ b/internal/application/service/web_search/searxng.go @@ -0,0 +1,32 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// SearxngProvider implements web search using Searxng API +type SearxngProvider struct { + apiURL string +} + +// NewSearxngProvider creates a new Searxng provider +func NewSearxngProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &SearxngProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *SearxngProvider) Name() string { + return "searxng" +} + +// Search performs a web search using Searxng API +func (p *SearxngProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Searxng search API + return nil, fmt.Errorf("searxng search provider is not yet implemented") +} diff --git a/internal/application/service/web_search/tavily.go b/internal/application/service/web_search/tavily.go new file mode 100644 index 00000000..700c74a2 --- /dev/null +++ b/internal/application/service/web_search/tavily.go @@ -0,0 +1,38 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// TavilyProvider implements web search using Tavily API +type TavilyProvider struct { + apiKey string + apiURL string +} + +// NewTavilyProvider creates a new Tavily provider +func NewTavilyProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &TavilyProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *TavilyProvider) Name() string { + return "tavily" +} + +// Search performs a web search using Tavily API +func (p *TavilyProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Tavily search API + return nil, fmt.Errorf("tavily search provider is not yet implemented") +} + +// SetAPIKey sets the API key for the provider +func (p *TavilyProvider) SetAPIKey(apiKey string) { + p.apiKey = apiKey +} diff --git a/internal/application/service/web_search/zhipu.go b/internal/application/service/web_search/zhipu.go new file mode 100644 index 00000000..91b488a3 --- /dev/null +++ b/internal/application/service/web_search/zhipu.go @@ -0,0 +1,38 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// ZhipuProvider implements web search using Zhipu API +type ZhipuProvider struct { + apiKey string + apiURL string +} + +// NewZhipuProvider creates a new Zhipu provider +func NewZhipuProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &ZhipuProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *ZhipuProvider) Name() string { + return "zhipu" +} + +// Search performs a web search using Zhipu API +func (p *ZhipuProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Zhipu search API + return nil, fmt.Errorf("zhipu search provider is not yet implemented") +} + +// SetAPIKey sets the API key for the provider +func (p *ZhipuProvider) SetAPIKey(apiKey string) { + p.apiKey = apiKey +} diff --git a/internal/application/service/web_search_baidu.go b/internal/application/service/web_search_baidu.go new file mode 100644 index 00000000..43345457 --- /dev/null +++ b/internal/application/service/web_search_baidu.go @@ -0,0 +1,33 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// BaiduProvider implements web search using Baidu API +type BaiduProvider struct { + apiURL string +} + +// NewBaiduProvider creates a new Baidu provider +func NewBaiduProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &BaiduProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *BaiduProvider) Name() string { + return "baidu" +} + +// Search performs a web search using Baidu API +func (p *BaiduProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Baidu search API + return nil, fmt.Errorf("baidu search provider is not yet implemented") +} + diff --git a/internal/application/service/web_search_bing.go b/internal/application/service/web_search_bing.go new file mode 100644 index 00000000..510d443e --- /dev/null +++ b/internal/application/service/web_search_bing.go @@ -0,0 +1,33 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// BingProvider implements web search using Bing Search API +type BingProvider struct { + apiURL string +} + +// NewBingProvider creates a new Bing provider +func NewBingProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &BingProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *BingProvider) Name() string { + return "bing" +} + +// Search performs a web search using Bing Search API +func (p *BingProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Bing Search API + return nil, fmt.Errorf("bing search provider is not yet implemented") +} + diff --git a/internal/application/service/web_search_bocha.go b/internal/application/service/web_search_bocha.go new file mode 100644 index 00000000..6b5b08ac --- /dev/null +++ b/internal/application/service/web_search_bocha.go @@ -0,0 +1,39 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// BochaProvider implements web search using Bocha API +type BochaProvider struct { + apiKey string + apiURL string +} + +// NewBochaProvider creates a new Bocha provider +func NewBochaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &BochaProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *BochaProvider) Name() string { + return "bocha" +} + +// Search performs a web search using Bocha API +func (p *BochaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Bocha search API + return nil, fmt.Errorf("bocha search provider is not yet implemented") +} + +// SetAPIKey sets the API key for the provider +func (p *BochaProvider) SetAPIKey(apiKey string) { + p.apiKey = apiKey +} + diff --git a/internal/application/service/web_search_exa.go b/internal/application/service/web_search_exa.go new file mode 100644 index 00000000..cc9c7d5a --- /dev/null +++ b/internal/application/service/web_search_exa.go @@ -0,0 +1,39 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// ExaProvider implements web search using Exa API +type ExaProvider struct { + apiKey string + apiURL string +} + +// NewExaProvider creates a new Exa provider +func NewExaProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &ExaProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *ExaProvider) Name() string { + return "exa" +} + +// Search performs a web search using Exa API +func (p *ExaProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Exa search API + return nil, fmt.Errorf("exa search provider is not yet implemented") +} + +// SetAPIKey sets the API key for the provider +func (p *ExaProvider) SetAPIKey(apiKey string) { + p.apiKey = apiKey +} + diff --git a/internal/application/service/web_search_google.go b/internal/application/service/web_search_google.go new file mode 100644 index 00000000..844b48ac --- /dev/null +++ b/internal/application/service/web_search_google.go @@ -0,0 +1,41 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// GoogleProvider implements web search using Google Custom Search API +type GoogleProvider struct { + apiKey string + searchEngineID string + apiURL string +} + +// NewGoogleProvider creates a new Google provider +func NewGoogleProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &GoogleProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *GoogleProvider) Name() string { + return "google" +} + +// Search performs a web search using Google Custom Search API +func (p *GoogleProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Google Custom Search API + return nil, fmt.Errorf("google search provider is not yet implemented") +} + +// SetAPIKey sets the API key and search engine ID +func (p *GoogleProvider) SetAPIKey(apiKey, searchEngineID string) { + p.apiKey = apiKey + p.searchEngineID = searchEngineID +} + diff --git a/internal/application/service/web_search_kuaisou.go b/internal/application/service/web_search_kuaisou.go new file mode 100644 index 00000000..91ffeb58 --- /dev/null +++ b/internal/application/service/web_search_kuaisou.go @@ -0,0 +1,113 @@ +package service + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// KuaisouProvider implements web search using Kuaisou API +type KuaisouProvider struct { + apiKey string + apiURL string + client *http.Client +} + +// NewKuaisouProvider creates a new Kuaisou provider +func NewKuaisouProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + if cfg.APIURL == "" { + return nil, fmt.Errorf("kuaisou API URL is required") + } + + return &KuaisouProvider{ + apiKey: "", // Will be set from tenant config + apiURL: cfg.APIURL, + client: &http.Client{ + Timeout: 10 * time.Second, + }, + }, nil +} + +// Name returns the provider name +func (p *KuaisouProvider) Name() string { + return "kuaisou" +} + +// Search performs a web search using Kuaisou API +func (p *KuaisouProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + if p.apiKey == "" { + return nil, fmt.Errorf("kuaisou API key is required") + } + + // Prepare request + reqBody := map[string]interface{}{ + "query": query, + "max_results": maxResults, + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", p.apiURL, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+p.apiKey) + + // Send request + resp, err := p.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("kuaisou API returned status %d: %s", resp.StatusCode, string(body)) + } + + // Parse response + var response struct { + Results []struct { + Title string `json:"title"` + URL string `json:"url"` + Snippet string `json:"snippet"` + } `json:"results"` + } + + if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + // Convert to WebSearchResult + results := make([]*types.WebSearchResult, 0, len(response.Results)) + for _, item := range response.Results { + results = append(results, &types.WebSearchResult{ + Title: item.Title, + URL: item.URL, + Snippet: item.Snippet, + Source: "kuaisou", + }) + } + + logger.Infof(ctx, "Kuaisou search returned %d results for query: %s", len(results), query) + return results, nil +} + +// SetAPIKey sets the API key for the provider +func (p *KuaisouProvider) SetAPIKey(apiKey string) { + p.apiKey = apiKey +} + diff --git a/internal/application/service/web_search_searxng.go b/internal/application/service/web_search_searxng.go new file mode 100644 index 00000000..bdd820e1 --- /dev/null +++ b/internal/application/service/web_search_searxng.go @@ -0,0 +1,33 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// SearxngProvider implements web search using Searxng API +type SearxngProvider struct { + apiURL string +} + +// NewSearxngProvider creates a new Searxng provider +func NewSearxngProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &SearxngProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *SearxngProvider) Name() string { + return "searxng" +} + +// Search performs a web search using Searxng API +func (p *SearxngProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Searxng search API + return nil, fmt.Errorf("searxng search provider is not yet implemented") +} + diff --git a/internal/application/service/web_search_tavily.go b/internal/application/service/web_search_tavily.go new file mode 100644 index 00000000..365e5fed --- /dev/null +++ b/internal/application/service/web_search_tavily.go @@ -0,0 +1,39 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// TavilyProvider implements web search using Tavily API +type TavilyProvider struct { + apiKey string + apiURL string +} + +// NewTavilyProvider creates a new Tavily provider +func NewTavilyProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &TavilyProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *TavilyProvider) Name() string { + return "tavily" +} + +// Search performs a web search using Tavily API +func (p *TavilyProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Tavily search API + return nil, fmt.Errorf("tavily search provider is not yet implemented") +} + +// SetAPIKey sets the API key for the provider +func (p *TavilyProvider) SetAPIKey(apiKey string) { + p.apiKey = apiKey +} + diff --git a/internal/application/service/web_search_zhipu.go b/internal/application/service/web_search_zhipu.go new file mode 100644 index 00000000..208fe7d6 --- /dev/null +++ b/internal/application/service/web_search_zhipu.go @@ -0,0 +1,39 @@ +package service + +import ( + "context" + "fmt" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" +) + +// ZhipuProvider implements web search using Zhipu API +type ZhipuProvider struct { + apiKey string + apiURL string +} + +// NewZhipuProvider creates a new Zhipu provider +func NewZhipuProvider(cfg config.WebSearchProviderConfig) (WebSearchProvider, error) { + return &ZhipuProvider{ + apiURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *ZhipuProvider) Name() string { + return "zhipu" +} + +// Search performs a web search using Zhipu API +func (p *ZhipuProvider) Search(ctx context.Context, query string, maxResults int, includeDate bool) ([]*types.WebSearchResult, error) { + // TODO: Implement Zhipu search API + return nil, fmt.Errorf("zhipu search provider is not yet implemented") +} + +// SetAPIKey sets the API key for the provider +func (p *ZhipuProvider) SetAPIKey(apiKey string) { + p.apiKey = apiKey +} + diff --git a/internal/config/config.go b/internal/config/config.go index 46284ea6..930d0b25 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -24,6 +24,7 @@ type Config struct { StreamManager *StreamManagerConfig `yaml:"stream_manager" json:"stream_manager"` ExtractManager *ExtractManagerConfig `yaml:"extract" json:"extract"` Agent *AgentGlobalConfig `yaml:"agent" json:"agent"` + WebSearch *WebSearchConfig `yaml:"web_search" json:"web_search"` } type DocReaderConfig struct { @@ -198,3 +199,29 @@ type AgentGlobalConfig struct { ReflectionEnabled bool `yaml:"reflection_enabled" json:"reflection_enabled"` DefaultTools []string `yaml:"default_tools" json:"default_tools"` } + +// WebSearchConfig represents the web search configuration +type WebSearchConfig struct { + Providers []WebSearchProviderConfig `yaml:"providers" json:"providers"` + Default WebSearchDefaultConfig `yaml:"default" json:"default"` + Timeout int `yaml:"timeout" json:"timeout"` // 超时时间(秒) +} + +// WebSearchProviderConfig represents configuration for a web search provider +type WebSearchProviderConfig struct { + ID string `yaml:"id" json:"id"` + Name string `yaml:"name" json:"name"` + Free bool `yaml:"free" json:"free"` + RequiresAPIKey bool `yaml:"requires_api_key" json:"requires_api_key"` + Description string `yaml:"description,omitempty" json:"description,omitempty"` + APIURL string `yaml:"api_url,omitempty" json:"api_url,omitempty"` +} + +// WebSearchDefaultConfig represents the default web search configuration +type WebSearchDefaultConfig struct { + Provider string `yaml:"provider" json:"provider"` + MaxResults int `yaml:"max_results" json:"max_results"` + IncludeDate bool `yaml:"include_date" json:"include_date"` + CompressionMethod string `yaml:"compression_method" json:"compression_method"` + Blacklist []string `yaml:"blacklist" json:"blacklist"` +} diff --git a/internal/container/container.go b/internal/container/container.go index 18878047..8a28bad1 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -119,6 +119,13 @@ func BuildContainer(container *dig.Container) *dig.Container { // Session service (depends on agent service) must(container.Provide(service.NewSessionService)) + // Web search service + must(container.Provide(service.NewWebSearchService)) + // Register WebSearchService as interface for dependency injection + must(container.Provide(func(ws *service.WebSearchService) interfaces.WebSearchService { + return ws + })) + // Chat pipeline components for processing chat requests must(container.Provide(chatpipline.NewEventManager)) must(container.Invoke(chatpipline.NewPluginTracing)) @@ -148,6 +155,7 @@ func BuildContainer(container *dig.Container) *dig.Container { must(container.Provide(handler.NewAuthHandler)) must(container.Provide(handler.NewSystemHandler)) must(container.Provide(handler.NewMCPServiceHandler)) + must(container.Provide(handler.NewWebSearchHandler)) // Router configuration must(container.Provide(router.NewRouter)) diff --git a/internal/handler/session/qa.go b/internal/handler/session/qa.go index 449ed72a..1a7e3e24 100644 --- a/internal/handler/session/qa.go +++ b/internal/handler/session/qa.go @@ -118,7 +118,7 @@ func (h *Handler) KnowledgeQA(c *gin.Context) { } // Use shared function to handle KnowledgeQA request - h.handleKnowledgeQARequest(ctx, c, session, request.Query, knowledgeBaseIDs, assistantMessage, true, request.SummaryModelID) + h.handleKnowledgeQARequest(ctx, c, session, request.Query, knowledgeBaseIDs, assistantMessage, true, request.SummaryModelID, request.WebSearchEnabled) } // AgentQA handles agent-based question answering with conversation history and streaming @@ -256,7 +256,7 @@ func (h *Handler) AgentQA(c *gin.Context) { logger.Infof(ctx, "Delegating to KnowledgeQA with knowledge bases: %v", knowledgeBaseIDs) // Use shared function to handle KnowledgeQA request (no title generation for AgentQA fallback) - h.handleKnowledgeQARequest(ctx, c, session, request.Query, knowledgeBaseIDs, assistantMessage, false, request.SummaryModelID) + h.handleKnowledgeQARequest(ctx, c, session, request.Query, knowledgeBaseIDs, assistantMessage, false, request.SummaryModelID, request.WebSearchEnabled) return } @@ -353,6 +353,7 @@ func (h *Handler) handleKnowledgeQARequest( assistantMessage *types.Message, generateTitle bool, // Whether to generate title if session has no title summaryModelID string, // Optional summary model ID (overrides session default) + webSearchEnabled bool, // Whether web search is enabled ) { sessionID := session.ID requestID := getRequestID(c) @@ -421,7 +422,7 @@ func (h *Handler) handleKnowledgeQARequest( logger.ErrorWithFields(asyncCtx, errors.NewInternalServerError("Knowledge QA service panicked"), nil) } }() - err := h.sessionService.KnowledgeQA(asyncCtx, session, query, knowledgeBaseIDs, assistantMessage.ID, summaryModelID, eventBus) + err := h.sessionService.KnowledgeQA(asyncCtx, session, query, knowledgeBaseIDs, assistantMessage.ID, summaryModelID, webSearchEnabled, eventBus) if err != nil { logger.ErrorWithFields(asyncCtx, err, nil) // Emit error event to dedicated EventBus diff --git a/internal/handler/session/types.go b/internal/handler/session/types.go index af93389f..70811f7a 100644 --- a/internal/handler/session/types.go +++ b/internal/handler/session/types.go @@ -56,6 +56,7 @@ type CreateKnowledgeQARequest struct { Query string `json:"query" binding:"required"` // Query text for knowledge base search KnowledgeBaseIDs []string `json:"knowledge_base_ids"` // Selected knowledge base ID for this request AgentEnabled bool `json:"agent_enabled"` // Whether agent mode is enabled for this request + WebSearchEnabled bool `json:"web_search_enabled"` // Whether web search is enabled for this request SummaryModelID string `json:"summary_model_id"` // Optional summary model ID for this request (overrides session default) MCPServiceIDs []string `json:"mcp_service_ids"` // Optional MCP service IDs to use in agent mode } diff --git a/internal/handler/tenant.go b/internal/handler/tenant.go index fe0e9428..d32af3d8 100644 --- a/internal/handler/tenant.go +++ b/internal/handler/tenant.go @@ -410,3 +410,145 @@ func (h *TenantHandler) UpdateTenantAgentConfig(c *gin.Context) { "message": "Agent configuration updated successfully", }) } + +// GetTenantWebSearchConfig returns the web search configuration for a tenant +// Tenant ID is obtained from the authentication context +func (h *TenantHandler) GetTenantWebSearchConfig(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start getting tenant web search config") + + // Get tenant ID from authentication context + tenantID := c.GetUint(types.TenantIDContextKey.String()) + if tenantID == 0 { + logger.Error(ctx, "Tenant ID is empty") + c.Error(errors.NewBadRequestError("Tenant ID cannot be empty")) + return + } + + // Get tenant + tenant, err := h.service.GetTenantByID(ctx, tenantID) + if err != nil { + if appErr, ok := errors.IsAppError(err); ok { + logger.Error(ctx, "Failed to retrieve tenant: application error", appErr) + c.Error(appErr) + } else { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Failed to retrieve tenant").WithDetails(err.Error())) + } + return + } + + // Hide API key in response + var responseConfig *types.WebSearchConfig + if tenant.WebSearchConfig != nil { + responseConfig = &types.WebSearchConfig{ + Provider: tenant.WebSearchConfig.Provider, + APIKey: "", // Hide API key + MaxResults: tenant.WebSearchConfig.MaxResults, + IncludeDate: tenant.WebSearchConfig.IncludeDate, + CompressionMethod: tenant.WebSearchConfig.CompressionMethod, + Blacklist: tenant.WebSearchConfig.Blacklist, + EmbeddingModelID: tenant.WebSearchConfig.EmbeddingModelID, + EmbeddingDimension: tenant.WebSearchConfig.EmbeddingDimension, + RerankModelID: tenant.WebSearchConfig.RerankModelID, + DocumentFragments: tenant.WebSearchConfig.DocumentFragments, + } + // If API key exists, show a masked version + if tenant.WebSearchConfig.APIKey != "" { + responseConfig.APIKey = "***" // Masked API key + } + } + + logger.Infof(ctx, "Tenant web search config retrieved successfully, Tenant ID: %d", tenantID) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": responseConfig, + }) +} + +// UpdateTenantWebSearchConfig updates the web search configuration for a tenant +// Tenant ID is obtained from the authentication context +func (h *TenantHandler) UpdateTenantWebSearchConfig(c *gin.Context) { + ctx := c.Request.Context() + + logger.Info(ctx, "Start updating tenant web search config") + + // Get tenant ID from authentication context + tenantID := c.GetUint(types.TenantIDContextKey.String()) + if tenantID == 0 { + logger.Error(ctx, "Tenant ID is empty") + c.Error(errors.NewBadRequestError("Tenant ID cannot be empty")) + return + } + + var req types.WebSearchConfig + if err := c.ShouldBindJSON(&req); err != nil { + logger.Error(ctx, "Failed to parse request parameters", err) + c.Error(errors.NewValidationError("Invalid request data").WithDetails(err.Error())) + return + } + + // Validate configuration + if req.MaxResults < 1 || req.MaxResults > 50 { + c.Error(errors.NewBadRequestError("max_results must be between 1 and 50")) + return + } + + // Get existing tenant + tenant, err := h.service.GetTenantByID(ctx, tenantID) + if err != nil { + if appErr, ok := errors.IsAppError(err); ok { + logger.Error(ctx, "Failed to retrieve tenant: application error", appErr) + c.Error(appErr) + } else { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Failed to retrieve tenant").WithDetails(err.Error())) + } + return + } + + // Update web search configuration + // If API key is "***", keep the existing API key + if req.APIKey == "***" && tenant.WebSearchConfig != nil { + req.APIKey = tenant.WebSearchConfig.APIKey + } + + tenant.WebSearchConfig = &req + + updatedTenant, err := h.service.UpdateTenant(ctx, tenant) + if err != nil { + if appErr, ok := errors.IsAppError(err); ok { + logger.Error(ctx, "Failed to update tenant: application error", appErr) + c.Error(appErr) + } else { + logger.ErrorWithFields(ctx, err, nil) + c.Error(errors.NewInternalServerError("Failed to update tenant web search config").WithDetails(err.Error())) + } + return + } + + // Hide API key in response + responseConfig := &types.WebSearchConfig{ + Provider: updatedTenant.WebSearchConfig.Provider, + APIKey: "", // Hide API key + MaxResults: updatedTenant.WebSearchConfig.MaxResults, + IncludeDate: updatedTenant.WebSearchConfig.IncludeDate, + CompressionMethod: updatedTenant.WebSearchConfig.CompressionMethod, + Blacklist: updatedTenant.WebSearchConfig.Blacklist, + EmbeddingModelID: updatedTenant.WebSearchConfig.EmbeddingModelID, + EmbeddingDimension: updatedTenant.WebSearchConfig.EmbeddingDimension, + RerankModelID: updatedTenant.WebSearchConfig.RerankModelID, + DocumentFragments: updatedTenant.WebSearchConfig.DocumentFragments, + } + if updatedTenant.WebSearchConfig.APIKey != "" { + responseConfig.APIKey = "***" // Masked API key + } + + logger.Infof(ctx, "Tenant web search config updated successfully, Tenant ID: %d", tenantID) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": responseConfig, + "message": "Web search configuration updated successfully", + }) +} diff --git a/internal/handler/web_search.go b/internal/handler/web_search.go new file mode 100644 index 00000000..2262b879 --- /dev/null +++ b/internal/handler/web_search.go @@ -0,0 +1,63 @@ +package handler + +import ( + "net/http" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" + "github.com/gin-gonic/gin" +) + +// WebSearchHandler handles web search related requests +type WebSearchHandler struct { + cfg *config.Config +} + +// NewWebSearchHandler creates a new web search handler +func NewWebSearchHandler(cfg *config.Config) *WebSearchHandler { + return &WebSearchHandler{ + cfg: cfg, + } +} + +// GetProviders returns the list of available web search providers +// @Summary Get available web search providers +// @Description Returns the list of available web search providers from configuration +// @Tags web-search +// @Accept json +// @Produce json +// @Success 200 {object} map[string]interface{} "List of providers" +// @Router /api/v1/web-search/providers [get] +func (h *WebSearchHandler) GetProviders(c *gin.Context) { + ctx := c.Request.Context() + logger.Info(ctx, "Getting web search providers") + + if h.cfg.WebSearch == nil || len(h.cfg.WebSearch.Providers) == 0 { + logger.Warn(ctx, "No web search providers configured") + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": []types.WebSearchProviderInfo{}, + }) + return + } + + // Convert config providers to API response format + providers := make([]types.WebSearchProviderInfo, 0, len(h.cfg.WebSearch.Providers)) + for _, provider := range h.cfg.WebSearch.Providers { + providers = append(providers, types.WebSearchProviderInfo{ + ID: provider.ID, + Name: provider.Name, + Free: provider.Free, + RequiresAPIKey: provider.RequiresAPIKey, + Description: provider.Description, + APIURL: provider.APIURL, + }) + } + + logger.Infof(ctx, "Returning %d web search providers", len(providers)) + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": providers, + }) +} diff --git a/internal/router/router.go b/internal/router/router.go index 1d0575d3..d9b6d2e1 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -40,6 +40,7 @@ type RouterParams struct { InitializationHandler *handler.InitializationHandler SystemHandler *handler.SystemHandler MCPServiceHandler *handler.MCPServiceHandler + WebSearchHandler *handler.WebSearchHandler } // NewRouter 创建新的路由 @@ -87,6 +88,7 @@ func NewRouter(params RouterParams) *gin.Engine { RegisterInitializationRoutes(v1, params.InitializationHandler) RegisterSystemRoutes(v1, params.SystemHandler) RegisterMCPServiceRoutes(v1, params.MCPServiceHandler) + RegisterWebSearchRoutes(v1, params.WebSearchHandler) } return r @@ -224,6 +226,10 @@ func RegisterTenantRoutes(r *gin.RouterGroup, handler *handler.TenantHandler) { // Tenant ID is obtained from authentication context tenantRoutes.GET("/agent-config", handler.GetTenantAgentConfig) tenantRoutes.PUT("/agent-config", handler.UpdateTenantAgentConfig) + // Web search configuration management (tenant-level) + // Tenant ID is obtained from authentication context + tenantRoutes.GET("/web-search-config", handler.GetTenantWebSearchConfig) + tenantRoutes.PUT("/web-search-config", handler.UpdateTenantWebSearchConfig) } } @@ -319,3 +325,13 @@ func RegisterMCPServiceRoutes(r *gin.RouterGroup, handler *handler.MCPServiceHan mcpServices.GET("/:id/resources", handler.GetMCPServiceResources) } } + +// RegisterWebSearchRoutes registers web search routes +func RegisterWebSearchRoutes(r *gin.RouterGroup, webSearchHandler *handler.WebSearchHandler) { + // Web search providers + webSearch := r.Group("/web-search") + { + // Get available providers + webSearch.GET("/providers", webSearchHandler.GetProviders) + } +} diff --git a/internal/types/chat_manage.go b/internal/types/chat_manage.go index aa808e8e..30799354 100644 --- a/internal/types/chat_manage.go +++ b/internal/types/chat_manage.go @@ -37,6 +37,10 @@ type ChatManage struct { // Event system for streaming responses EventBus EventBusInterface `json:"-"` // EventBus for emitting streaming events MessageID string `json:"-"` // Assistant message ID for event emission + + // Web search configuration (internal use) + TenantID uint `json:"-"` // Tenant ID for retrieving web search config + WebSearchEnabled bool `json:"-"` // Whether web search is enabled for this request } // Clone creates a deep copy of the ChatManage object diff --git a/internal/types/embedding.go b/internal/types/embedding.go index 19fc8103..51b50f9e 100644 --- a/internal/types/embedding.go +++ b/internal/types/embedding.go @@ -20,6 +20,7 @@ const ( MatchTypeParentChunk // 父Chunk匹配类型 MatchTypeRelationChunk // 关系Chunk匹配类型 MatchTypeGraph + MatchTypeWebSearch // 网络搜索匹配类型 ) // IndexInfo contains information about indexed content diff --git a/internal/types/interfaces/session.go b/internal/types/interfaces/session.go index dbd04b60..c3214d34 100644 --- a/internal/types/interfaces/session.go +++ b/internal/types/interfaces/session.go @@ -29,9 +29,10 @@ type SessionService interface { // KnowledgeQA performs knowledge-based question answering // knowledgeBaseIDs: list of knowledge base IDs to search (supports multi-KB) // summaryModelID: optional summary model ID override (if empty, uses session/KB default) + // webSearchEnabled: whether to enable web search to supplement knowledge base results // Events are emitted through eventBus (references, answer chunks, completion) KnowledgeQA(ctx context.Context, - session *types.Session, query string, knowledgeBaseIDs []string, assistantMessageID string, summaryModelID string, eventBus *event.EventBus, + session *types.Session, query string, knowledgeBaseIDs []string, assistantMessageID string, summaryModelID string, webSearchEnabled bool, eventBus *event.EventBus, ) error // KnowledgeQAByEvent performs knowledge-based question answering by event KnowledgeQAByEvent(ctx context.Context, chatManage *types.ChatManage, eventList []types.EventType) error diff --git a/internal/types/interfaces/web_search.go b/internal/types/interfaces/web_search.go new file mode 100644 index 00000000..6e5ecd70 --- /dev/null +++ b/internal/types/interfaces/web_search.go @@ -0,0 +1,12 @@ +package interfaces + +import ( + "context" + + "github.com/Tencent/WeKnora/internal/types" +) + +// WebSearchService defines the interface for web search service +type WebSearchService interface { + Search(ctx context.Context, config *types.WebSearchConfig, query string) ([]*types.WebSearchResult, error) +} diff --git a/internal/types/tenant.go b/internal/types/tenant.go index edf510c4..0d5ad762 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -32,6 +32,8 @@ type Tenant struct { AgentConfig *AgentConfig `yaml:"agent_config" json:"agent_config" gorm:"type:jsonb"` // Global Context configuration for this tenant (default for all sessions) ContextConfig *ContextConfig `yaml:"context_config" json:"context_config" gorm:"type:jsonb"` + // Global WebSearch configuration for this tenant + WebSearchConfig *WebSearchConfig `yaml:"web_search_config" json:"web_search_config" gorm:"type:jsonb"` // Creation time CreatedAt time.Time `yaml:"created_at" json:"created_at"` // Last updated time diff --git a/internal/types/web_search.go b/internal/types/web_search.go new file mode 100644 index 00000000..d002cf74 --- /dev/null +++ b/internal/types/web_search.go @@ -0,0 +1,59 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" + "time" +) + +// WebSearchConfig represents the web search configuration for a tenant +type WebSearchConfig struct { + Provider string `json:"provider"` // 搜索引擎提供商ID + APIKey string `json:"api_key"` // API密钥(如果需要) + MaxResults int `json:"max_results"` // 最大搜索结果数 + IncludeDate bool `json:"include_date"` // 是否包含日期 + CompressionMethod string `json:"compression_method"` // 压缩方法:none, summary, extract, rag + Blacklist []string `json:"blacklist"` // 黑名单规则列表 + // RAG压缩相关配置 + EmbeddingModelID string `json:"embedding_model_id,omitempty"` // 嵌入模型ID(用于RAG压缩) + EmbeddingDimension int `json:"embedding_dimension,omitempty"` // 嵌入维度(用于RAG压缩) + RerankModelID string `json:"rerank_model_id,omitempty"` // 重排模型ID(用于RAG压缩) + DocumentFragments int `json:"document_fragments,omitempty"` // 文档片段数量(用于RAG压缩) +} + +// Value implements driver.Valuer interface for WebSearchConfig +func (c WebSearchConfig) Value() (driver.Value, error) { + return json.Marshal(c) +} + +// Scan implements sql.Scanner interface for WebSearchConfig +func (c *WebSearchConfig) Scan(value interface{}) error { + if value == nil { + return nil + } + b, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(b, c) +} + +// WebSearchResult represents a single web search result +type WebSearchResult struct { + Title string `json:"title"` // 搜索结果标题 + URL string `json:"url"` // 结果URL + Snippet string `json:"snippet"` // 摘要片段 + Content string `json:"content"` // 完整内容(可选,需要额外抓取) + Source string `json:"source"` // 来源(如:kuaisou, baidu等) + PublishedAt *time.Time `json:"published_at,omitempty"` // 发布时间(如果有) +} + +// WebSearchProviderInfo represents information about a web search provider +type WebSearchProviderInfo struct { + ID string `json:"id"` // 提供商ID + Name string `json:"name"` // 提供商名称 + Free bool `json:"free"` // 是否免费 + RequiresAPIKey bool `json:"requires_api_key"` // 是否需要API密钥 + Description string `json:"description"` // 描述 + APIURL string `json:"api_url,omitempty"` // API地址(可选) +} diff --git a/migrations/mysql/05-add-web-search-config.sql b/migrations/mysql/05-add-web-search-config.sql new file mode 100644 index 00000000..5b171ece --- /dev/null +++ b/migrations/mysql/05-add-web-search-config.sql @@ -0,0 +1,6 @@ +-- Add web_search_config column to tenants table +-- This migration adds support for web search configuration at tenant level + +ALTER TABLE tenants +ADD COLUMN web_search_config JSON DEFAULT NULL COMMENT 'Web search configuration for the tenant'; + diff --git a/migrations/paradedb/05-add-web-search-config.sql b/migrations/paradedb/05-add-web-search-config.sql new file mode 100644 index 00000000..25212b21 --- /dev/null +++ b/migrations/paradedb/05-add-web-search-config.sql @@ -0,0 +1,6 @@ +-- Add web_search_config column to tenants table +-- This migration adds support for web search configuration at tenant level + +ALTER TABLE tenants +ADD COLUMN web_search_config JSONB DEFAULT NULL; + diff --git a/migrations/versioned/000005_add_web_search_config.down.sql b/migrations/versioned/000005_add_web_search_config.down.sql new file mode 100644 index 00000000..30a348c1 --- /dev/null +++ b/migrations/versioned/000005_add_web_search_config.down.sql @@ -0,0 +1,11 @@ +-- 000005_add_web_search_config.down.sql +-- Remove web_search_config column from tenants table + +BEGIN; + +-- Remove web_search_config column +ALTER TABLE tenants +DROP COLUMN IF EXISTS web_search_config; + +COMMIT; + diff --git a/migrations/versioned/000005_add_web_search_config.up.sql b/migrations/versioned/000005_add_web_search_config.up.sql new file mode 100644 index 00000000..0a2ac788 --- /dev/null +++ b/migrations/versioned/000005_add_web_search_config.up.sql @@ -0,0 +1,14 @@ +-- 000005_add_web_search_config.up.sql +-- Add web_search_config column to tenants table for web search configuration + +BEGIN; + +-- Add web_search_config column to tenants table +ALTER TABLE tenants +ADD COLUMN web_search_config JSONB DEFAULT NULL; + +-- Add comment +COMMENT ON COLUMN tenants.web_search_config IS 'Web search configuration for the tenant'; + +COMMIT; +