From 91ec651ac0b1f03e8ca90c8d3d6654e6534732d1 Mon Sep 17 00:00:00 2001 From: begoniezhao Date: Thu, 8 Jan 2026 20:01:04 +0800 Subject: [PATCH 01/15] fix(doc_parser): Add secure command execution with sandbox --- docreader/parser/doc_parser.py | 113 ++++++++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 15 deletions(-) diff --git a/docreader/parser/doc_parser.py b/docreader/parser/doc_parser.py index 337ab85d..578305c8 100644 --- a/docreader/parser/doc_parser.py +++ b/docreader/parser/doc_parser.py @@ -12,9 +12,93 @@ from docreader.utils.tempfile import TempDirContext, TempFileContext logger = logging.getLogger(__name__) +class SandboxExecutor: + """Sandbox executor for running commands with proxy configuration""" + + def __init__(self, proxy: Optional[str] = None, default_timeout: int = 60): + """Initialize sandbox executor with configuration + + Args: + proxy: Proxy URL to use for network access. If None, will use WEB_PROXY environment variable + default_timeout: Default timeout in seconds for command execution + """ + # Get proxy from parameter, environment variable, or use default blocking proxy + # Use 'or None' to convert empty string to None, then apply default value + self.proxy = proxy or os.environ.get("WEB_PROXY") or "http://128.0.0.1:1" + self.default_timeout = default_timeout + + def execute_in_sandbox(self, cmd: List[str]) -> tuple: + """Execute command in sandbox with proxy configuration + + Args: + cmd: Command to execute + + Returns: + Tuple of (stdout, stderr, returncode) + """ + # Try different sandbox methods in order of preference + sandbox_methods = [ + self._execute_with_proxy, + ] + + for method in sandbox_methods: + try: + return method(cmd) + except Exception as e: + logger.warning(f"Sandbox method {method.__name__} failed: {e}") + continue + + raise RuntimeError("All sandbox methods failed") + + def _execute_with_proxy(self, cmd: List[str]) -> tuple: + """Execute command with proxy configuration + + Args: + cmd: Command to execute + + Returns: + Tuple of (stdout, stderr, returncode) + """ + # Set up environment with proxy configuration + env = os.environ.copy() + if self.proxy: + env["http_proxy"] = self.proxy + env["https_proxy"] = self.proxy + env["HTTP_PROXY"] = self.proxy + env["HTTPS_PROXY"] = self.proxy + + logger.info(f"Executing command with proxy: {' '.join(cmd)}") + if self.proxy: + logger.info(f"Using proxy: {self.proxy}") + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + ) + + try: + stdout, stderr = process.communicate(timeout=self.default_timeout) + return stdout, stderr, process.returncode + except subprocess.TimeoutExpired: + process.kill() + raise RuntimeError( + f"Command execution timeout after {self.default_timeout} seconds" + ) + + +logger = logging.getLogger(__name__) + + class DocParser(Docx2Parser): """DOC document parser""" + def __init__(self, *args, **kwargs): + """Initialize DOC parser with sandbox executor""" + super().__init__(*args, **kwargs) + self.sandbox_executor = SandboxExecutor() + def parse_into_text(self, content: bytes) -> Document: logger.info(f"Parsing DOC document, content size: {len(content)} bytes") @@ -25,7 +109,8 @@ class DocParser(Docx2Parser): # try using antiword to extract text self._parse_with_antiword, # 3. If antiword extraction fails, use textract - self._parse_with_textract, + # NOTE: _parse_with_textract is disabled due to SSRF vulnerability + # self._parse_with_textract, ] # Save byte content as a temporary file @@ -61,14 +146,13 @@ class DocParser(Docx2Parser): if not antiword_path: raise RuntimeError("antiword not found in PATH") - # Use antiword to extract text directly - process = subprocess.Popen( - [antiword_path, temp_file_path], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - stdout, stderr = process.communicate() - if process.returncode != 0: + # Use antiword to extract text directly in sandbox + cmd = [antiword_path, temp_file_path] + logger.info("Executing antiword in sandbox with proxy configuration") + + stdout, stderr, returncode = self.sandbox_executor.execute_in_sandbox(cmd) + + if returncode != 0: raise RuntimeError( f"antiword extraction failed: {stderr.decode('utf-8', errors='ignore')}" ) @@ -114,13 +198,12 @@ class DocParser(Docx2Parser): temp_dir, doc_path, ] - logger.info(f"Running command: {' '.join(cmd)}") - process = subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - stdout, stderr = process.communicate() + logger.info(f"Running command in sandbox: {' '.join(cmd)}") - if process.returncode != 0: + # Execute in sandbox with proxy configuration + stdout, stderr, returncode = self.sandbox_executor.execute_in_sandbox(cmd) + + if returncode != 0: logger.warning( f"Error converting DOC to DOCX: {stderr.decode('utf-8')}" ) From 1d10d0395a2e36e759f71616fa2bd88176b3c632 Mon Sep 17 00:00:00 2001 From: nullkey Date: Wed, 7 Jan 2026 21:41:25 +0800 Subject: [PATCH 02/15] feat:add model provider GPUStack --- frontend/src/i18n/locales/en-US.ts | 4 ++ frontend/src/i18n/locales/ru-RU.ts | 4 ++ frontend/src/i18n/locales/zh-CN.ts | 28 ++++++++++++++ internal/models/provider/gpustack.go | 57 ++++++++++++++++++++++++++++ internal/models/provider/provider.go | 5 +++ 5 files changed, 98 insertions(+) create mode 100644 internal/models/provider/gpustack.go diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 337f3f4e..be823351 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1696,6 +1696,10 @@ export default { label: 'Google Gemini', description: 'gemini-3-flash-preview, gemini-2.5-pro, etc.', }, + gpustack: { + label: 'GPUStack', + description: 'Self-hosted GPU cluster platform, OpenAI-compatible API', + }, }, } }, diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 3a8ad7ad..de7a2b4c 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -947,6 +947,10 @@ export default { label: 'Google Gemini', description: 'gemini-3-flash-preview, gemini-2.5-pro, etc.', }, + gpustack: { + label: 'GPUStack', + description: 'Приватная платформа GPU-кластеров, OpenAI-совместимый API', + }, }, } }, diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 7bd9da27..79bfab0d 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1207,6 +1207,34 @@ export default { label: "Jina AI", description: "专注于 Embedding 和 Rerank", }, + volcengine: { + label: "火山引擎", + description: "doubao-1-5-pro-32k, doubao-embedding-vision 等", + }, + deepseek: { + label: "DeepSeek", + description: "deepseek-chat, deepseek-reasoner 等", + }, + hunyuan: { + label: "腾讯混元", + description: "hunyuan-pro, hunyuan-standard, hunyuan-embedding 等", + }, + minimax: { + label: "MiniMax", + description: "MiniMax-M2.1, MiniMax-M2.1-lightning 等", + }, + mimo: { + label: "小米 MiMo", + description: "mimo-v2-flash", + }, + gemini: { + label: "Google Gemini", + description: "gemini-3-flash-preview, gemini-2.5-pro 等", + }, + gpustack: { + label: "GPUStack", + description: "私有化 GPU 集群部署平台，OpenAI 兼容 API", + }, }, }, }, diff --git a/internal/models/provider/gpustack.go b/internal/models/provider/gpustack.go new file mode 100644 index 00000000..9bbfc7e7 --- /dev/null +++ b/internal/models/provider/gpustack.go @@ -0,0 +1,57 @@ +package provider + +import ( + "fmt" + + "github.com/Tencent/WeKnora/internal/types" +) + +const ( + // GPUStackBaseURL GPUStack API BaseURL (OpenAI 兼容模式) + GPUStackBaseURL = "http://your_gpustack_server_url/v1-openai" + GPUStackEmbeddingBaseURL = "http://your_gpustack_server_url/v1-openai/embeddings" + GPUStackRerankBaseURL = "http://your_gpustack_server_url/v1/rerank" +) + +// GPUStackProvider 实现 GPUStack 的 Provider 接口 +type GPUStackProvider struct{} + +func init() { + Register(&GPUStackProvider{}) +} + +// Info 返回 GPUStack provider 的元数据 +func (p *GPUStackProvider) Info() ProviderInfo { + return ProviderInfo{ + Name: ProviderGPUStack, + DisplayName: "GPUStack", + Description: "Choose your deployed model on GPUStack", + DefaultURLs: map[types.ModelType]string{ + types.ModelTypeKnowledgeQA: GPUStackBaseURL, + types.ModelTypeEmbedding: GPUStackEmbeddingBaseURL, + types.ModelTypeRerank: GPUStackRerankBaseURL, + types.ModelTypeVLLM: GPUStackBaseURL, + }, + ModelTypes: []types.ModelType{ + types.ModelTypeKnowledgeQA, + types.ModelTypeEmbedding, + types.ModelTypeRerank, + types.ModelTypeVLLM, + }, + RequiresAuth: true, // GPUStack 需要 API Key + } +} + +// ValidateConfig 验证 GPUStack provider 配置 +func (p *GPUStackProvider) ValidateConfig(config *Config) error { + if config.BaseURL == "" { + return fmt.Errorf("base URL is required for GPUStack provider") + } + if config.APIKey == "" { + return fmt.Errorf("API key is required for GPUStack provider") + } + if config.ModelName == "" { + return fmt.Errorf("model name is required") + } + return nil +} diff --git a/internal/models/provider/provider.go b/internal/models/provider/provider.go index a8f72b30..1d35892f 100644 --- a/internal/models/provider/provider.go +++ b/internal/models/provider/provider.go @@ -39,6 +39,8 @@ const ( ProviderMiniMax ProviderName = "minimax" // 小米 Mimo ProviderMimo ProviderName = "mimo" + // GPUStack (私有化部署) + ProviderGPUStack ProviderName = "gpustack" ) // AllProviders 返回所有注册的提供者名称 @@ -57,6 +59,7 @@ func AllProviders() []ProviderName { ProviderOpenRouter, ProviderJina, ProviderMimo, + ProviderGPUStack, } } @@ -208,6 +211,8 @@ func DetectProvider(baseURL string) ProviderName { return ProviderMiniMax case containsAny(baseURL, "xiaomimimo.com"): return ProviderMimo + case containsAny(baseURL, "gpustack"): + return ProviderGPUStack default: return ProviderGeneric } From 579b57d377bc619231b74967a1900384525f72a9 Mon Sep 17 00:00:00 2001 From: nullkey Date: Thu, 8 Jan 2026 16:13:52 +0800 Subject: [PATCH 03/15] feat:add several mainstream model providers --- docs/BUILTIN_MODELS.md | 2 +- docs/api/model.md | 47 ++++++++------ frontend/src/i18n/locales/en-US.ts | 28 +++++++-- frontend/src/i18n/locales/ko-KR.ts | 87 +++++++++++++++++++++++++- frontend/src/i18n/locales/ru-RU.ts | 28 +++++++-- frontend/src/i18n/locales/zh-CN.ts | 52 ++++++++++----- internal/models/provider/generic.go | 4 +- internal/models/provider/gpustack.go | 8 +-- internal/models/provider/longcat.go | 48 ++++++++++++++ internal/models/provider/modelscope.go | 53 ++++++++++++++++ internal/models/provider/moonshot.go | 50 +++++++++++++++ internal/models/provider/provider.go | 25 ++++++++ internal/models/provider/qianfan.go | 54 ++++++++++++++++ internal/models/provider/qiniu.go | 49 +++++++++++++++ 14 files changed, 481 insertions(+), 54 deletions(-) create mode 100644 internal/models/provider/longcat.go create mode 100644 internal/models/provider/modelscope.go create mode 100644 internal/models/provider/moonshot.go create mode 100644 internal/models/provider/qianfan.go create mode 100644 internal/models/provider/qiniu.go diff --git a/docs/BUILTIN_MODELS.md b/docs/BUILTIN_MODELS.md index c1886a2b..38f92019 100644 --- a/docs/BUILTIN_MODELS.md +++ b/docs/BUILTIN_MODELS.md @@ -24,7 +24,7 @@ - 模型参数（parameters）：包括 base_url、api_key、provider 等 - 租户ID（tenant_id）：建议使用小于10000的租户ID，避免冲突 -**支持的服务商（provider）**：`generic`（自定义）、`openai`、`aliyun`、`zhipu`、`volcengine`、`hunyuan`、`deepseek`、`minimax`、`mimo`、`siliconflow`、`jina`、`openrouter`、`gemini` +**支持的服务商（provider）**：`generic`（自定义）、`openai`、`aliyun`、`zhipu`、`volcengine`、`hunyuan`、`deepseek`、`minimax`、`mimo`、`siliconflow`、`jina`、`openrouter`、`gemini`、`modelscope`、`moonshot`、`qianfan`、`qiniu`、`longcat`、`gpustack` ### 2. 执行 SQL 插入语句 diff --git a/docs/api/model.md b/docs/api/model.md index 66dbb546..b5ab8e1d 100644 --- a/docs/api/model.md +++ b/docs/api/model.md @@ -17,21 +17,27 @@ WeKnora 支持多种主流 AI 模型服务商，在创建模型时可通过 `pro ### 支持的服务商列表 -| 服务商标识 | 名称 | 支持的模型类型 | -| -------------- | ------------------ | ------------------------------- | -| `generic` | 自定义API OpenAI 兼容 | Chat, Embedding, Rerank, VLLM | -| `aliyun` | 阿里云 DashScope | Chat, Embedding, Rerank, VLLM | -| `zhipu` | 智谱 AI | Chat, Embedding, Rerank, VLLM | -| `volcengine` | 火山引擎 | Chat, Embedding, VLLM | -| `hunyuan` | 腾讯混元 | Chat, Embedding, VLLM | -| `deepseek` | DeepSeek | Chat | -| `minimax` | MiniMax | Chat, Embedding, VLLM | -| `mimo` | 小米 Mimo | Chat | -| `siliconflow` | 硅基流动 | Chat, Embedding, Rerank | -| `jina` | Jina AI | Embedding, Rerank | -| `openrouter` | OpenRouter | Chat | -| `openai` | OpenAI | Chat, Embedding, VLLM | -| `gemini` | Google Gemini | Chat, Embedding, VLLM | +| 服务商标识 | 名称 | 支持的模型类型 | +| -------------- | ---------------------------- | ------------------------------- | +| `generic` | 自定义 (OpenAI兼容接口) | Chat, Embedding, Rerank, VLLM | +| `openai` | OpenAI | Chat, Embedding, Rerank, VLLM | +| `aliyun` | 阿里云 DashScope | Chat, Embedding, Rerank, VLLM | +| `zhipu` | 智谱 BigModel | Chat, Embedding, Rerank, VLLM | +| `volcengine` | 火山引擎 Volcengine | Chat, Embedding, VLLM | +| `hunyuan` | 腾讯混元 Hunyuan | Chat, Embedding | +| `deepseek` | DeepSeek | Chat | +| `minimax` | MiniMax | Chat | +| `mimo` | 小米 MiMo | Chat | +| `siliconflow` | 硅基流动 SiliconFlow | Chat, Embedding, Rerank, VLLM | +| `jina` | Jina | Embedding, Rerank | +| `openrouter` | OpenRouter | Chat, VLLM | +| `gemini` | Google Gemini | Chat | +| `modelscope` | 魔搭 ModelScope | Chat, Embedding, VLLM | +| `moonshot` | 月之暗面 Moonshot | Chat, VLLM | +| `qianfan` | 百度千帆 Baidu Cloud | Chat, Embedding, Rerank, VLLM | +| `qiniu` | 七牛云 Qiniu | Chat | +| `longcat` | LongCat AI | Chat | +| `gpustack` | GPUStack | Chat, Embedding, Rerank, VLLM | ## GET `/models/providers` - 获取模型服务商列表 @@ -64,7 +70,7 @@ curl --location 'http://localhost:8080/api/v1/models/providers?model_type=embedd { "value": "aliyun", "label": "阿里云 DashScope", - "description": "阿里云百炼大模型服务", + "description": "qwen-plus, tongyi-embedding-vision-plus, qwen3-rerank, etc.", "defaultUrls": { "chat": "https://dashscope.aliyuncs.com/compatible-mode/v1", "embedding": "https://dashscope.aliyuncs.com/compatible-mode/v1", @@ -74,13 +80,14 @@ curl --location 'http://localhost:8080/api/v1/models/providers?model_type=embedd }, { "value": "zhipu", - "label": "智谱 AI", - "description": "智谱清言大模型", + "label": "智谱 BigModel", + "description": "glm-4.7, embedding-3, rerank, etc.", "defaultUrls": { "chat": "https://open.bigmodel.cn/api/paas/v4", - "embedding": "https://open.bigmodel.cn/api/paas/v4/embeddings" + "embedding": "https://open.bigmodel.cn/api/paas/v4/embeddings", + "rerank": "https://open.bigmodel.cn/api/paas/v4/rerank" }, - "modelTypes": ["chat", "embedding", "vllm"] + "modelTypes": ["chat", "embedding", "rerank", "vllm"] } ] } diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index be823351..9c2fbeec 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -1653,7 +1653,7 @@ export default { description: 'qwen-plus, tongyi-embedding-vision-plus, qwen3-rerank, etc.', }, zhipu: { - label: 'Zhipu AI', + label: 'Zhipu BigModel', description: 'glm-4.7, embedding-3, rerank, etc.', }, openrouter: { @@ -1661,7 +1661,7 @@ export default { description: 'openai/gpt-5.2-chat, google/gemini-3-flash-preview, etc.', }, generic: { - label: 'Custom API', + label: 'Custom (OpenAI-compatible)', description: 'Generic API endpoint', }, siliconflow: { @@ -1669,7 +1669,7 @@ export default { description: 'deepseek-ai/DeepSeek-V3.1, etc.', }, jina: { - label: 'Jina AI', + label: 'Jina', description: 'jina-clip-v1, jina-embeddings-v2-base-zh, etc.', }, volcengine: { @@ -1698,7 +1698,27 @@ export default { }, gpustack: { label: 'GPUStack', - description: 'Self-hosted GPU cluster platform, OpenAI-compatible API', + description: 'Choose your deployed model on GPUStack', + }, + modelscope: { + label: 'ModelScope', + description: 'Qwen/Qwen3-8B, Qwen/Qwen3-Embedding-8B, etc.', + }, + qiniu: { + label: 'Qiniu Cloud', + description: 'deepseek/deepseek-v3.2-251201, z-ai/glm-4.7, etc.', + }, + moonshot: { + label: 'Moonshot', + description: 'kimi-k2-turbo-preview, moonshot-v1-8k-vision-preview, etc.', + }, + qianfan: { + label: 'Baidu Qianfan', + description: 'ernie-5.0-thinking-preview, embedding-v1, bce-reranker-base, etc.', + }, + longcat: { + label: 'LongCat AI', + description: 'LongCat-Flash-Chat, LongCat-Flash-Thinking, etc.', }, }, } diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index 626d9857..56571863 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -983,6 +983,87 @@ export default { baseUrlEmpty: "Base URL은 비워둘 수 없습니다", baseUrlInvalid: "Base URL 형식이 올바르지 않습니다. 유효한 URL을 입력해주세요", }, + // Provider related translations + providerLabel: "제공업체", + providerPlaceholder: "모델 제공업체 선택", + providers: { + openai: { + label: "OpenAI", + description: "gpt-5.2, gpt-5-mini 등", + }, + aliyun: { + label: "Aliyun DashScope", + description: "qwen-plus, tongyi-embedding-vision-plus, qwen3-rerank 등", + }, + zhipu: { + label: "Zhipu BigModel", + description: "glm-4.7, embedding-3, rerank, etc.", + }, + openrouter: { + label: "OpenRouter", + description: "openai/gpt-5.2-chat, google/gemini-3-flash-preview 등", + }, + generic: { + label: "사용자 정의 (OpenAI 호환)", + description: "Generic API endpoint", + }, + siliconflow: { + label: "SiliconFlow", + description: "deepseek-ai/DeepSeek-V3.1 등", + }, + jina: { + label: "Jina", + description: "jina-clip-v1, jina-embeddings-v2-base-zh, etc.", + }, + volcengine: { + label: "Volcengine", + description: "doubao-1-5-pro-32k-250115, doubao-embedding-vision-250615 등", + }, + deepseek: { + label: "DeepSeek", + description: "deepseek-chat, deepseek-reasoner 등", + }, + hunyuan: { + label: "Hunyuan", + description: "hunyuan-pro, hunyuan-standard, hunyuan-embedding 등", + }, + minimax: { + label: "MiniMax", + description: "MiniMax-M2.1, MiniMax-M2.1-lightning 등", + }, + mimo: { + label: "MiMo", + description: "mimo-v2-flash", + }, + gemini: { + label: "Google Gemini", + description: "gemini-3-flash-preview, gemini-2.5-pro 등", + }, + gpustack: { + label: "GPUStack", + description: "Choose your deployed model on GPUStack", + }, + modelscope: { + label: "ModelScope", + description: "Qwen/Qwen3-8B, Qwen/Qwen3-Embedding-8B, etc.", + }, + qiniu: { + label: "Qiniu Cloud", + description: "deepseek/deepseek-v3.2-251201, z-ai/glm-4.7, etc.", + }, + moonshot: { + label: "Moonshot", + description: "kimi-k2-turbo-preview, moonshot-v1-8k-vision-preview, etc.", + }, + qianfan: { + label: "Baidu Qianfan", + description: "ernie-5.0-thinking-preview, embedding-v1, bce-reranker-base, etc.", + }, + longcat: { + label: "LongCat AI", + description: "LongCat-Flash-Chat, LongCat-Flash-Thinking, etc.", + }, + }, }, }, language: { @@ -1155,9 +1236,9 @@ export default { messages: { deleted: "삭제됨", deleteFailed: "삭제 실패", - file: "파일", - knowledgeBase: "지식베이스", - noResult: "결과 없음", + file: "파일", + knowledgeBase: "지식베이스", + noResult: "결과 없음", }, features: { knowledgeGraph: "지식 그래프 활성화됨", diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index de7a2b4c..6ebaa9e7 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -904,7 +904,7 @@ export default { description: 'qwen-plus, tongyi-embedding-vision-plus, qwen3-rerank, etc.', }, zhipu: { - label: 'Zhipu AI', + label: 'Zhipu BigModel', description: 'glm-4.7, embedding-3, rerank, etc.', }, openrouter: { @@ -912,7 +912,7 @@ export default { description: 'openai/gpt-5.2-chat, google/gemini-3-flash-preview, etc.', }, generic: { - label: 'Пользовательский API', + label: 'Пользовательский (OpenAI-совместимый)', description: 'Generic API endpoint', }, siliconflow: { @@ -920,7 +920,7 @@ export default { description: 'deepseek-ai/DeepSeek-V3.1, etc.', }, jina: { - label: 'Jina AI', + label: 'Jina', description: 'jina-clip-v1, jina-embeddings-v2-base-zh, etc.', }, volcengine: { @@ -949,7 +949,27 @@ export default { }, gpustack: { label: 'GPUStack', - description: 'Приватная платформа GPU-кластеров, OpenAI-совместимый API', + description: 'Choose your deployed model on GPUStack', + }, + modelscope: { + label: 'ModelScope', + description: 'Qwen/Qwen3-8B, Qwen/Qwen3-Embedding-8B, etc.', + }, + qiniu: { + label: 'Qiniu Cloud', + description: 'deepseek/deepseek-v3.2-251201, z-ai/glm-4.7, etc.', + }, + moonshot: { + label: 'Moonshot', + description: 'kimi-k2-turbo-preview, moonshot-v1-8k-vision-preview, etc.', + }, + qianfan: { + label: 'Baidu Qianfan', + description: 'ernie-5.0-thinking-preview, embedding-v1, bce-reranker-base, etc.', + }, + longcat: { + label: 'LongCat AI', + description: 'LongCat-Flash-Chat, LongCat-Flash-Thinking, etc.', }, }, } diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index 79bfab0d..59167c0d 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -1181,43 +1181,43 @@ export default { providers: { openai: { label: "OpenAI", - description: "OpenAI 官方 API", + description: "gpt-5.2, gpt-5-mini, etc.", }, aliyun: { label: "阿里云 DashScope", - description: "支持 Qwen、DeepSeek 等模型", + description: "qwen-plus, tongyi-embedding-vision-plus, qwen3-rerank, etc.", }, zhipu: { - label: "智谱 AI", - description: "支持 GLM-4 系列模型", + label: "智谱 BigModel", + description: "glm-4.7, embedding-3, rerank, etc.", }, openrouter: { label: "OpenRouter", - description: "多模型聚合平台", + description: "openai/gpt-5.2-chat, google/gemini-3-flash-preview, etc.", }, generic: { - label: "自定义 API", - description: "OpenAI 兼容的自定义端点", + label: "自定义 (OpenAI兼容接口)", + description: "Generic API endpoint (OpenAI-compatible)", }, siliconflow: { - label: "SiliconFlow", - description: "硅基流动，支持多种模型", + label: "硅基流动 SiliconFlow", + description: "deepseek-ai/DeepSeek-V3.1, etc.", }, jina: { - label: "Jina AI", - description: "专注于 Embedding 和 Rerank", + label: "Jina", + description: "jina-clip-v1, jina-embeddings-v2-base-zh, etc.", }, volcengine: { - label: "火山引擎", - description: "doubao-1-5-pro-32k, doubao-embedding-vision 等", + label: "火山引擎 Volcengine", + description: "doubao-1-5-pro-32k-250115, doubao-embedding-vision-250615, etc.", }, deepseek: { label: "DeepSeek", description: "deepseek-chat, deepseek-reasoner 等", }, hunyuan: { - label: "腾讯混元", - description: "hunyuan-pro, hunyuan-standard, hunyuan-embedding 等", + label: "腾讯混元 Hunyuan", + description: "hunyuan-pro, hunyuan-standard, hunyuan-embedding, etc.", }, minimax: { label: "MiniMax", @@ -1233,7 +1233,27 @@ export default { }, gpustack: { label: "GPUStack", - description: "私有化 GPU 集群部署平台，OpenAI 兼容 API", + description: "Choose your deployed model on GPUStack", + }, + modelscope: { + label: "魔搭 ModelScope", + description: "Qwen/Qwen3-8B, Qwen/Qwen3-Embedding-8B, etc.", + }, + qiniu: { + label: "七牛云 Qiniu", + description: "deepseek/deepseek-v3.2-251201, z-ai/glm-4.7, etc.", + }, + moonshot: { + label: "月之暗面 Moonshot", + description: "kimi-k2-turbo-preview, moonshot-v1-8k-vision-preview, etc.", + }, + qianfan: { + label: "百度千帆 Baidu Cloud", + description: "ernie-5.0-thinking-preview, embedding-v1, bce-reranker-base, etc.", + }, + longcat: { + label: "LongCat AI", + description: "LongCat-Flash-Chat, LongCat-Flash-Thinking, etc.", }, }, }, diff --git a/internal/models/provider/generic.go b/internal/models/provider/generic.go index f0ccd9a7..fe8aa862 100644 --- a/internal/models/provider/generic.go +++ b/internal/models/provider/generic.go @@ -17,8 +17,8 @@ func init() { func (p *GenericProvider) Info() ProviderInfo { return ProviderInfo{ Name: ProviderGeneric, - DisplayName: "自定义 (OpenAI格式兼容接口)", - Description: "Generic API endpoint", + DisplayName: "自定义 (OpenAI兼容接口)", + Description: "Generic API endpoint (OpenAI-compatible)", DefaultURLs: map[types.ModelType]string{}, // 需要用户自行配置填写 ModelTypes: []types.ModelType{ types.ModelTypeKnowledgeQA, diff --git a/internal/models/provider/gpustack.go b/internal/models/provider/gpustack.go index 9bbfc7e7..686b3516 100644 --- a/internal/models/provider/gpustack.go +++ b/internal/models/provider/gpustack.go @@ -8,9 +8,9 @@ import ( const ( // GPUStackBaseURL GPUStack API BaseURL (OpenAI 兼容模式) - GPUStackBaseURL = "http://your_gpustack_server_url/v1-openai" - GPUStackEmbeddingBaseURL = "http://your_gpustack_server_url/v1-openai/embeddings" - GPUStackRerankBaseURL = "http://your_gpustack_server_url/v1/rerank" + GPUStackBaseURL = "http://your_gpustack_server_url/v1-openai" + // GPUStackRerankBaseURL GPUStack Rerank API 虽然兼容OpenAI，但路径不同 (/v1/rerank 而非 /v1-openai/rerank) + GPUStackRerankBaseURL = "http://your_gpustack_server_url/v1" ) // GPUStackProvider 实现 GPUStack 的 Provider 接口 @@ -28,7 +28,7 @@ func (p *GPUStackProvider) Info() ProviderInfo { Description: "Choose your deployed model on GPUStack", DefaultURLs: map[types.ModelType]string{ types.ModelTypeKnowledgeQA: GPUStackBaseURL, - types.ModelTypeEmbedding: GPUStackEmbeddingBaseURL, + types.ModelTypeEmbedding: GPUStackBaseURL, types.ModelTypeRerank: GPUStackRerankBaseURL, types.ModelTypeVLLM: GPUStackBaseURL, }, diff --git a/internal/models/provider/longcat.go b/internal/models/provider/longcat.go new file mode 100644 index 00000000..0273463a --- /dev/null +++ b/internal/models/provider/longcat.go @@ -0,0 +1,48 @@ +package provider + +import ( + "fmt" + + "github.com/Tencent/WeKnora/internal/types" +) + +const ( + LongCatBaseURL = "https://api.longcat.chat/openai/v1" +) + +// LongCatProvider 实现 LongCat AI 的 Provider 接口 +type LongCatProvider struct{} + +func init() { + Register(&LongCatProvider{}) +} + +// Info 返回 LongCat provider 的元数据 +func (p *LongCatProvider) Info() ProviderInfo { + return ProviderInfo{ + Name: ProviderLongCat, + DisplayName: "LongCat AI", + Description: "LongCat-Flash-Chat, LongCat-Flash-Thinking, etc.", + DefaultURLs: map[types.ModelType]string{ + types.ModelTypeKnowledgeQA: LongCatBaseURL, + }, + ModelTypes: []types.ModelType{ + types.ModelTypeKnowledgeQA, + }, + RequiresAuth: true, + } +} + +// ValidateConfig 验证 LongCat provider 配置 +func (p *LongCatProvider) ValidateConfig(config *Config) error { + if config.BaseURL == "" { + return fmt.Errorf("base URL is required for LongCat provider") + } + if config.APIKey == "" { + return fmt.Errorf("API key is required for LongCat provider") + } + if config.ModelName == "" { + return fmt.Errorf("model name is required") + } + return nil +} diff --git a/internal/models/provider/modelscope.go b/internal/models/provider/modelscope.go new file mode 100644 index 00000000..aaa40709 --- /dev/null +++ b/internal/models/provider/modelscope.go @@ -0,0 +1,53 @@ +package provider + +import ( + "fmt" + + "github.com/Tencent/WeKnora/internal/types" +) + +const ( + // ModelScopeBaseURL ModelScope API BaseURL (OpenAI 兼容模式) + ModelScopeBaseURL = "https://api-inference.modelscope.cn/v1" +) + +// ModelScopeProvider 实现 ModelScope (魔搭) 的 Provider 接口 +type ModelScopeProvider struct{} + +func init() { + Register(&ModelScopeProvider{}) +} + +// Info 返回 ModelScope provider 的元数据 +func (p *ModelScopeProvider) Info() ProviderInfo { + return ProviderInfo{ + Name: ProviderModelScope, + DisplayName: "魔搭 ModelScope", + Description: "Qwen/Qwen3-8B, Qwen/Qwen3-Embedding-8B, etc.", + DefaultURLs: map[types.ModelType]string{ + types.ModelTypeKnowledgeQA: ModelScopeBaseURL, + types.ModelTypeEmbedding: ModelScopeBaseURL, + types.ModelTypeVLLM: ModelScopeBaseURL, + }, + ModelTypes: []types.ModelType{ + types.ModelTypeKnowledgeQA, + types.ModelTypeEmbedding, + types.ModelTypeVLLM, + }, + RequiresAuth: true, + } +} + +// ValidateConfig 验证 ModelScope provider 配置 +func (p *ModelScopeProvider) ValidateConfig(config *Config) error { + if config.BaseURL == "" { + return fmt.Errorf("base URL is required for ModelScope provider") + } + if config.APIKey == "" { + return fmt.Errorf("API key is required for ModelScope provider") + } + if config.ModelName == "" { + return fmt.Errorf("model name is required") + } + return nil +} diff --git a/internal/models/provider/moonshot.go b/internal/models/provider/moonshot.go new file mode 100644 index 00000000..3c6b31ee --- /dev/null +++ b/internal/models/provider/moonshot.go @@ -0,0 +1,50 @@ +package provider + +import ( + "fmt" + + "github.com/Tencent/WeKnora/internal/types" +) + +const ( + MoonshotBaseURL = "https://api.moonshot.ai/v1" +) + +// MoonshotProvider 实现 Moonshot AI (Kimi) 的 Provider 接口 +type MoonshotProvider struct{} + +func init() { + Register(&MoonshotProvider{}) +} + +// Info 返回 Moonshot provider 的元数据 +func (p *MoonshotProvider) Info() ProviderInfo { + return ProviderInfo{ + Name: ProviderMoonshot, + DisplayName: "月之暗面 Moonshot", + Description: "kimi-k2-turbo-preview, moonshot-v1-8k-vision-preview, etc.", + DefaultURLs: map[types.ModelType]string{ + types.ModelTypeKnowledgeQA: MoonshotBaseURL, + types.ModelTypeVLLM: MoonshotBaseURL, + }, + ModelTypes: []types.ModelType{ + types.ModelTypeKnowledgeQA, + types.ModelTypeVLLM, + }, + RequiresAuth: true, + } +} + +// ValidateConfig 验证 Moonshot provider 配置 +func (p *MoonshotProvider) ValidateConfig(config *Config) error { + if config.BaseURL == "" { + return fmt.Errorf("base URL is required for Moonshot provider") + } + if config.APIKey == "" { + return fmt.Errorf("API key is required for Moonshot provider") + } + if config.ModelName == "" { + return fmt.Errorf("model name is required") + } + return nil +} diff --git a/internal/models/provider/provider.go b/internal/models/provider/provider.go index 1d35892f..9742ac07 100644 --- a/internal/models/provider/provider.go +++ b/internal/models/provider/provider.go @@ -41,6 +41,16 @@ const ( ProviderMimo ProviderName = "mimo" // GPUStack (私有化部署) ProviderGPUStack ProviderName = "gpustack" + // 月之暗面 Moonshot (Kimi) + ProviderMoonshot ProviderName = "moonshot" + // 魔搭 ModelScope + ProviderModelScope ProviderName = "modelscope" + // 百度千帆 + ProviderQianfan ProviderName = "qianfan" + // 七牛云 + ProviderQiniu ProviderName = "qiniu" + // 美团 LongCat AI + ProviderLongCat ProviderName = "longcat" ) // AllProviders 返回所有注册的提供者名称 @@ -54,11 +64,16 @@ func AllProviders() []ProviderName { ProviderSiliconFlow, ProviderDeepSeek, ProviderMiniMax, + ProviderMoonshot, + ProviderModelScope, + ProviderQianfan, + ProviderQiniu, ProviderOpenAI, ProviderGemini, ProviderOpenRouter, ProviderJina, ProviderMimo, + ProviderLongCat, ProviderGPUStack, } } @@ -213,6 +228,16 @@ func DetectProvider(baseURL string) ProviderName { return ProviderMimo case containsAny(baseURL, "gpustack"): return ProviderGPUStack + case containsAny(baseURL, "modelscope.cn"): + return ProviderModelScope + case containsAny(baseURL, "qiniuapi.com", "qiniu"): + return ProviderQiniu + case containsAny(baseURL, "moonshot.ai"): + return ProviderMoonshot + case containsAny(baseURL, "qianfan.baidubce.com", "baidubce.com"): + return ProviderQianfan + case containsAny(baseURL, "longcat.chat"): + return ProviderLongCat default: return ProviderGeneric } diff --git a/internal/models/provider/qianfan.go b/internal/models/provider/qianfan.go new file mode 100644 index 00000000..aac91882 --- /dev/null +++ b/internal/models/provider/qianfan.go @@ -0,0 +1,54 @@ +package provider + +import ( + "fmt" + + "github.com/Tencent/WeKnora/internal/types" +) + +const ( + QianfanBaseURL = "https://qianfan.baidubce.com/v2" +) + +// QianfanProvider 实现百度千帆的 Provider 接口 +type QianfanProvider struct{} + +func init() { + Register(&QianfanProvider{}) +} + +// Info 返回百度千帆 provider 的元数据 +func (p *QianfanProvider) Info() ProviderInfo { + return ProviderInfo{ + Name: ProviderQianfan, + DisplayName: "百度千帆 Baidu Cloud", + Description: "ernie-5.0-thinking-preview, embedding-v1, bce-reranker-base, etc.", + DefaultURLs: map[types.ModelType]string{ + types.ModelTypeKnowledgeQA: QianfanBaseURL, + types.ModelTypeEmbedding: QianfanBaseURL, + types.ModelTypeRerank: QianfanBaseURL, + types.ModelTypeVLLM: QianfanBaseURL, + }, + ModelTypes: []types.ModelType{ + types.ModelTypeKnowledgeQA, + types.ModelTypeEmbedding, + types.ModelTypeRerank, + types.ModelTypeVLLM, + }, + RequiresAuth: true, + } +} + +// ValidateConfig 验证百度千帆 provider 配置 +func (p *QianfanProvider) ValidateConfig(config *Config) error { + if config.BaseURL == "" { + return fmt.Errorf("base URL is required for Qianfan provider") + } + if config.APIKey == "" { + return fmt.Errorf("API key is required for Qianfan provider") + } + if config.ModelName == "" { + return fmt.Errorf("model name is required") + } + return nil +} diff --git a/internal/models/provider/qiniu.go b/internal/models/provider/qiniu.go new file mode 100644 index 00000000..816e5994 --- /dev/null +++ b/internal/models/provider/qiniu.go @@ -0,0 +1,49 @@ +package provider + +import ( + "fmt" + + "github.com/Tencent/WeKnora/internal/types" +) + +const ( + // QiniuBaseURL 七牛云 API BaseURL (OpenAI 兼容模式) + QiniuBaseURL = "https://api.qnaigc.com/v1" +) + +// QiniuProvider 实现七牛云的 Provider 接口 +type QiniuProvider struct{} + +func init() { + Register(&QiniuProvider{}) +} + +// Info 返回七牛云 provider 的元数据 +func (p *QiniuProvider) Info() ProviderInfo { + return ProviderInfo{ + Name: ProviderQiniu, + DisplayName: "七牛云 Qiniu", + Description: "deepseek/deepseek-v3.2-251201, z-ai/glm-4.7, etc.", + DefaultURLs: map[types.ModelType]string{ + types.ModelTypeKnowledgeQA: QiniuBaseURL, + }, + ModelTypes: []types.ModelType{ + types.ModelTypeKnowledgeQA, + }, + RequiresAuth: true, + } +} + +// ValidateConfig 验证七牛云 provider 配置 +func (p *QiniuProvider) ValidateConfig(config *Config) error { + if config.BaseURL == "" { + return fmt.Errorf("base URL is required for Qiniu provider") + } + if config.APIKey == "" { + return fmt.Errorf("API key is required for Qiniu provider") + } + if config.ModelName == "" { + return fmt.Errorf("model name is required") + } + return nil +} From 7947305ff0a9a770d9f026158e1b2cdf5190d6b3 Mon Sep 17 00:00:00 2001 From: nullkey Date: Thu, 8 Jan 2026 21:52:06 +0800 Subject: [PATCH 04/15] =?UTF-8?q?fix:=20=E5=A2=9E=E5=8A=A0=E9=AD=94?= =?UTF-8?q?=E6=90=AD=E7=A4=BE=E5=8C=BA=E5=B5=8C=E5=85=A5=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=89=80=E9=9C=80=E7=9A=84EncodingFormat=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/models/embedding/openai.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/models/embedding/openai.go b/internal/models/embedding/openai.go index b1db6e94..3c13fdb8 100644 --- a/internal/models/embedding/openai.go +++ b/internal/models/embedding/openai.go @@ -30,7 +30,8 @@ type OpenAIEmbedder struct { type OpenAIEmbedRequest struct { Model string `json:"model"` Input []string `json:"input"` - TruncatePromptTokens int `json:"truncate_prompt_tokens"` + EncodingFormat string `json:"encoding_format,omitempty"` + TruncatePromptTokens int `json:"truncate_prompt_tokens,omitempty"` } // OpenAIEmbedResponse represents an OpenAI embedding response @@ -138,6 +139,7 @@ func (e *OpenAIEmbedder) BatchEmbed(ctx context.Context, texts []string) ([][]fl reqBody := OpenAIEmbedRequest{ Model: e.modelName, Input: texts, + EncodingFormat: "float", TruncatePromptTokens: e.truncatePromptTokens, } From bdc9171242676ed18c3b64de598d2f76e2f39eb3 Mon Sep 17 00:00:00 2001 From: begoniezhao Date: Fri, 9 Jan 2026 14:14:50 +0800 Subject: [PATCH 05/15] feat(knowledge): Omit DeletedAt on update operations --- internal/application/repository/knowledge.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go index 3d60c077..011723c0 100644 --- a/internal/application/repository/knowledge.go +++ b/internal/application/repository/knowledge.go @@ -12,6 +12,9 @@ import ( var ErrKnowledgeNotFound = errors.New("knowledge not found") +// omitFieldsOnUpdate defines fields to omit when updating knowledge +var omitFieldsOnUpdate = []string{"DeletedAt"} + // knowledgeRepository implements knowledge base and knowledge repository interface type knowledgeRepository struct { db *gorm.DB @@ -134,7 +137,7 @@ func (r *knowledgeRepository) ListPagedKnowledgeByKnowledgeBaseID( // UpdateKnowledge updates knowledge func (r *knowledgeRepository) UpdateKnowledge(ctx context.Context, knowledge *types.Knowledge) error { - err := r.db.WithContext(ctx).Save(knowledge).Error + err := r.db.WithContext(ctx).Omit(omitFieldsOnUpdate...).Save(knowledge).Error return err } @@ -143,7 +146,7 @@ func (r *knowledgeRepository) UpdateKnowledgeBatch(ctx context.Context, knowledg if len(knowledgeList) == 0 { return nil } - return r.db.Debug().WithContext(ctx).Save(knowledgeList).Error + return r.db.Debug().WithContext(ctx).Omit(omitFieldsOnUpdate...).Save(knowledgeList).Error } // DeleteKnowledge deletes knowledge From e023c26af8035e5ba80805bfebeb411a4c376170 Mon Sep 17 00:00:00 2001 From: orbisai0security Date: Thu, 8 Jan 2026 09:05:01 +0000 Subject: [PATCH 06/15] fix: resolve critical vulnerability V-001 Automatically generated security fix --- docreader/parser/base_parser.py | 66 +++++++++++++++++++++++++++++++++ internal/mcp/client.go | 28 ++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/docreader/parser/base_parser.py b/docreader/parser/base_parser.py index e3a1b0c0..4a7fb19a 100644 --- a/docreader/parser/base_parser.py +++ b/docreader/parser/base_parser.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- import asyncio import io +import ipaddress import logging import os import re import time from abc import ABC, abstractmethod from typing import Dict, List, Optional, Tuple +from urllib.parse import urlparse import requests from PIL import Image @@ -30,6 +32,60 @@ class BaseParser(ABC): _ocr_engine = None _ocr_engine_failed = False + @staticmethod + def _is_safe_url(url: str) -> bool: + """Validate URL to prevent SSRF attacks + + Args: + url: URL to validate + + Returns: + True if URL is safe to fetch, False otherwise + """ + try: + parsed_url = urlparse(url) + + # Only allow http and https schemes + if parsed_url.scheme not in ['http', 'https']: + logger.warning(f"Rejected URL with invalid scheme: {parsed_url.scheme}") + return False + + # Extract hostname + hostname = parsed_url.hostname + if not hostname: + logger.warning("No hostname found in URL") + return False + + # Try to parse as IP address + try: + ip = ipaddress.ip_address(hostname) + # Reject private, loopback, link-local, multicast addresses + if (ip.is_private or ip.is_loopback or + ip.is_link_local or ip.is_multicast or + ip.is_reserved): + logger.warning(f"Rejected URL with restricted IP: {ip}") + return False + except ValueError: + # Not an IP address, validate hostname + # Reject localhost and common internal hostnames + hostname_lower = hostname.lower() + restricted_hostnames = [ + 'localhost', + '127.0.0.1', + '::1', + 'metadata.google.internal', + 'metadata.tencentyun.com', + '169.254.169.254', # AWS metadata endpoint + ] + if hostname_lower in restricted_hostnames or hostname_lower.endswith('.local'): + logger.warning(f"Rejected URL with restricted hostname: {hostname}") + return False + + return True + except Exception as e: + logger.warning(f"Error validating URL: {str(e)}") + return False + @classmethod def get_ocr_engine(cls, backend_type="paddle", **kwargs): """Get OCR engine instance @@ -681,6 +737,11 @@ class BaseParser(ABC): if is_storage_url: logger.info(f"Image already on COS: {img_url}, no need to re-upload") try: + # Validate URL to prevent SSRF attacks + if not self._is_safe_url(img_url): + logger.error(f"URL failed validation check: {img_url}") + return img_url, img_url, None + # Still need to get image object for OCR processing # Get proxy settings from environment variables http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") @@ -727,6 +788,11 @@ class BaseParser(ABC): # Normal remote URL download handling else: + # Validate URL to prevent SSRF attacks + if not self._is_safe_url(img_url): + logger.error(f"URL failed validation check: {img_url}") + return img_url, img_url, None + # Get proxy settings from environment variables http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY") diff --git a/internal/mcp/client.go b/internal/mcp/client.go index 70315c7f..f971412c 100644 --- a/internal/mcp/client.go +++ b/internal/mcp/client.go @@ -4,7 +4,10 @@ import ( "context" "encoding/json" "fmt" + "net" "net/http" + "net/url" + "strings" "time" "github.com/Tencent/WeKnora/internal/logger" @@ -58,6 +61,31 @@ type mcpGoClient struct { initialized bool } +// validateURLScheme validates that authentication endpoints use HTTPS +func validateURLScheme(serviceURL string, hasBasicAuth bool, hasTokenAuth bool) error { + if !hasBasicAuth && !hasTokenAuth { + // No authentication, no HTTPS requirement + return nil + } + + if serviceURL == "" { + return fmt.Errorf("URL is required") + } + + // Parse the URL + parsedURL, err := url.Parse(serviceURL) + if err != nil { + return fmt.Errorf("invalid URL: %w", err) + } + + // Enforce HTTPS for authenticated endpoints + if parsedURL.Scheme != "https" { + return fmt.Errorf("HTTPS is required for authenticated connections, got %s scheme", parsedURL.Scheme) + } + + return nil +} + // NewMCPClient creates a new MCP client based on the transport type func NewMCPClient(config *ClientConfig) (MCPClient, error) { // Create HTTP client with timeout From 389900af7e1b7013f9abe4b9d1d0462d07eadfdc Mon Sep 17 00:00:00 2001 From: begoniezhao Date: Mon, 12 Jan 2026 17:34:01 +0800 Subject: [PATCH 07/15] refactor: Refactor retriever engine checks and expose mapping --- internal/handler/system.go | 26 ++++++++++++++++++++++++-- internal/mcp/client.go | 28 ---------------------------- internal/types/tenant.go | 6 ++++++ 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/internal/handler/system.go b/internal/handler/system.go index 3ab2b243..0f42be8e 100644 --- a/internal/handler/system.go +++ b/internal/handler/system.go @@ -8,6 +8,7 @@ import ( "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" "github.com/gin-gonic/gin" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -102,7 +103,7 @@ func (h *SystemHandler) getKeywordIndexEngine() string { keywordEngines := []string{} for _, driver := range drivers { driver = strings.TrimSpace(driver) - if driver == "postgres" || driver == "elasticsearch_v7" || driver == "elasticsearch_v8" { + if h.supportsRetrieverType(driver, types.KeywordsRetrieverType) { keywordEngines = append(keywordEngines, driver) } } @@ -131,7 +132,7 @@ func (h *SystemHandler) getVectorStoreEngine() string { vectorEngines := []string{} for _, driver := range drivers { driver = strings.TrimSpace(driver) - if driver == "postgres" || driver == "elasticsearch_v8" { + if h.supportsRetrieverType(driver, types.VectorRetrieverType) { vectorEngines = append(vectorEngines, driver) } } @@ -150,6 +151,27 @@ func (h *SystemHandler) getGraphDatabaseEngine() string { return "Neo4j" } +// supportsRetrieverType checks if a driver supports a specific retriever type +// by looking up the retrieverEngineMapping from types package +func (h *SystemHandler) supportsRetrieverType(driver string, retrieverType types.RetrieverType) bool { + // Get the mapping of all supported drivers and their capabilities + mapping := types.GetRetrieverEngineMapping() + + // Check if the driver exists in the mapping + engines, exists := mapping[driver] + if !exists { + return false + } + + // Check if any of the engine configurations support the requested retriever type + for _, engine := range engines { + if engine.RetrieverType == retrieverType { + return true + } + } + return false +} + // isMinioEnabled checks if MinIO is enabled func (h *SystemHandler) isMinioEnabled() bool { // Check if all required MinIO environment variables are set diff --git a/internal/mcp/client.go b/internal/mcp/client.go index f971412c..70315c7f 100644 --- a/internal/mcp/client.go +++ b/internal/mcp/client.go @@ -4,10 +4,7 @@ import ( "context" "encoding/json" "fmt" - "net" "net/http" - "net/url" - "strings" "time" "github.com/Tencent/WeKnora/internal/logger" @@ -61,31 +58,6 @@ type mcpGoClient struct { initialized bool } -// validateURLScheme validates that authentication endpoints use HTTPS -func validateURLScheme(serviceURL string, hasBasicAuth bool, hasTokenAuth bool) error { - if !hasBasicAuth && !hasTokenAuth { - // No authentication, no HTTPS requirement - return nil - } - - if serviceURL == "" { - return fmt.Errorf("URL is required") - } - - // Parse the URL - parsedURL, err := url.Parse(serviceURL) - if err != nil { - return fmt.Errorf("invalid URL: %w", err) - } - - // Enforce HTTPS for authenticated endpoints - if parsedURL.Scheme != "https" { - return fmt.Errorf("HTTPS is required for authenticated connections, got %s scheme", parsedURL.Scheme) - } - - return nil -} - // NewMCPClient creates a new MCP client based on the transport type func NewMCPClient(config *ClientConfig) (MCPClient, error) { // Create HTTP client with timeout diff --git a/internal/types/tenant.go b/internal/types/tenant.go index 67ce9502..088298a9 100644 --- a/internal/types/tenant.go +++ b/internal/types/tenant.go @@ -29,6 +29,12 @@ var retrieverEngineMapping = map[string][]RetrieverEngineParams{ }, } +// GetRetrieverEngineMapping returns the retriever engine mapping +// This allows other packages to access the driver capabilities +func GetRetrieverEngineMapping() map[string][]RetrieverEngineParams { + return retrieverEngineMapping +} + // GetDefaultRetrieverEngines returns the default retriever engines based on RETRIEVE_DRIVER env func GetDefaultRetrieverEngines() []RetrieverEngineParams { result := []RetrieverEngineParams{} From 6393fcf87b22de1820e99d9b37a96728704bc424 Mon Sep 17 00:00:00 2001 From: begoniezhao Date: Mon, 12 Jan 2026 20:15:25 +0800 Subject: [PATCH 08/15] docs: Rewrite Q&A and add MinIO and docreader diagnostics --- docs/QA.md | 106 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 74 insertions(+), 32 deletions(-) diff --git a/docs/QA.md b/docs/QA.md index ccfaa78f..8c3c80a0 100644 --- a/docs/QA.md +++ b/docs/QA.md @@ -57,46 +57,88 @@ INIT_RERANK_MODEL_API_KEY=your_rerank_model_api_key 2. 查看主服务日志，是否有`ERROR`日志输出 -## 4. 如何开启多模态功能？ -1. 确保 `.env` 如下配置被正确设置: +## 4. 多模态功能显示无效的图片链接？ + +当使用多模态功能时，如果遇到图片无法显示或显示无效链接的问题，请按照以下步骤排查： + +### 1. 确认多模态功能已正确配置 + +在知识库设置中开启**高级设置 - 多模态功能**，并在界面中配置相应的多模态模型。 + +### 2. 确认 MinIO 服务已启动 + +如果多模态功能配置使用的是 MinIO 存储，需要确保 MinIO 镜像已正确启动： + ```bash -# VLM_MODEL_NAME 使用的多模态模型名称 -VLM_MODEL_NAME=your_vlm_model_name +# 启动 MinIO 服务 +docker-compose --profile minio up -d -# VLM_MODEL_BASE_URL 使用的多模态模型访问地址 -VLM_MODEL_BASE_URL=your_vlm_model_base_url - -# VLM_MODEL_API_KEY 使用的多模态模型API密钥 -VLM_MODEL_API_KEY=your_vlm_model_api_key +# 或者启动完整服务（包括 MinIO、Jaeger、Neo4j、Qdrant） +docker-compose --profile full up -d ``` -注：多模态大模型当前仅支持remote api访问，固需要提供`VLM_MODEL_BASE_URL`和`VLM_MODEL_API_KEY` -2. 解析后的文件需要上传到COS中，确保 `.env` 中 `COS` 信息正确设置： +### 3. 检查 MinIO Bucket 权限 + +确保 MinIO 对应的 bucket 具有正确的读写权限： + +1. 访问 MinIO 控制台：`http://localhost:9001`（默认端口） +2. 使用 `.env` 中配置的 `MINIO_ACCESS_KEY_ID` 和 `MINIO_SECRET_ACCESS_KEY` 登录 +3. 进入对应的 bucket，检查并设置访问策略为**公开读取**或**公开读写** + +**重要提示**： +- Bucket 名称不要包含特殊字符（包括中文），建议使用小写字母、数字和连字符 +- 如果无法修改现有 bucket 的权限，可以在配置中填入一个不存在的 bucket 名称，本项目会自动创建对应的 bucket 并设置好正确的权限 + +### 4. 配置 MINIO_PUBLIC_ENDPOINT + +在 `docker-compose.yml` 文件中，`MINIO_PUBLIC_ENDPOINT` 变量默认配置为 `http://localhost:9000`。 + +**重要提示**：如果你需要从其他设备或容器访问图片，`localhost` 可能无法正常工作，需要将其替换为本机的实际 IP 地址： + +### 5. 验证配置 + +完成以上配置后，重启相关服务： + ```bash -# 腾讯云COS的访问密钥ID -COS_SECRET_ID=your_cos_secret_id - -# 腾讯云COS的密钥 -COS_SECRET_KEY=your_cos_secret_key - -# 腾讯云COS的区域，例如 ap-guangzhou -COS_REGION=your_cos_region - -# 腾讯云COS的桶名称 -COS_BUCKET_NAME=your_cos_bucket_name - -# 腾讯云COS的应用ID -COS_APP_ID=your_cos_app_id - -# 腾讯云COS的路径前缀，用于存储文件 -COS_PATH_PREFIX=your_cos_path_prefix +docker-compose restart docreader app ``` -重要：务必将COS中文件的权限设置为**公有读**，否则文档解析模块无法正常解析文件 -3. 查看文档解析模块日志，查看OCR和Caption是否正确解析和打印 +然后查看文档解析模块日志，确认 OCR 和 Caption 是否正确解析： + +```bash +docker-compose logs -f docreader +``` -## 5. 如何使用数据分析功能？ +## 5. docreader 服务无法启动？ + +如果 docreader 服务启动失败，日志中出现类似以下信息： + +``` +2026-01-12 xx:xx:xx.xxx [no-req-id] INFO __main__ | Initializing OCR engine with backend: paddle +Initializing server logging +``` + +这通常是因为 PaddleOCR 启动失败导致的。 + +### 解决方案 + +在 `docker-compose.yml` 文件的 `docreader` 服务中，已经配置了 `OCR_BACKEND` 环境变量： + +```yaml +environment: + - OCR_BACKEND=${OCR_BACKEND:-no_ocr} +``` + +然后重启 docreader 服务： + +```bash +docker-compose restart docreader +``` + +**注意**：设置为 `no_ocr` 后，文档解析将不会使用 OCR 功能，这可能会影响图片和扫描文档的文字识别效果。 + +## 6. 如何使用数据分析功能？ 在使用数据分析功能前，请确保智能体已配置相关工具： @@ -118,4 +160,4 @@ COS_PATH_PREFIX=your_cos_path_prefix ## P.S. -如果以上方式未解决问题，请在issue中描述您的问题，并提供必要的日志信息辅助我们进行问题排查 \ No newline at end of file +如果以上方式未解决问题，请在issue中描述您的问题，并提供必要的日志信息辅助我们进行问题排查 From 17f9a97ce43e534f898316c75c19dcfa0debcca4 Mon Sep 17 00:00:00 2001 From: joeyczheng Date: Mon, 12 Jan 2026 18:00:31 +0800 Subject: [PATCH 09/15] feat: add google provider for web search Signed-off-by: joeyczheng --- README.md | 34 +-- README_CN.md | 34 +-- README_JA.md | 34 +-- go.mod | 20 +- go.sum | 42 ++- internal/application/service/web_search.go | 2 + .../application/service/web_search/google.go | 92 ++++++ .../service/web_search/google_test.go | 278 ++++++++++++++++++ 8 files changed, 468 insertions(+), 68 deletions(-) create mode 100644 internal/application/service/web_search/google.go create mode 100644 internal/application/service/web_search/google_test.go diff --git a/README.md b/README.md index 3a982cb1..575c91a7 100644 --- a/README.md +++ b/README.md @@ -101,24 +101,24 @@ WeKnora employs a modern modular design to build a complete document understandi ## 🧩 Feature Matrix -| Module | Support | Description | -|---------|---------|------| -| Agent Mode | ✅ ReACT Agent Mode | Support for using built-in tools to retrieve knowledge bases, MCP tools, and web search, with cross-knowledge base retrieval and multiple iterations | -| Knowledge Base Types | ✅ FAQ / Document | Support for creating FAQ and document knowledge base types, with folder import, URL import, tag management, and online entry | -| Document Formats | ✅ PDF / Word / Txt / Markdown / Images (with OCR / Caption) | Support for structured and unstructured documents with text extraction from images | -| Model Management | ✅ Centralized configuration, built-in model sharing | Centralized model configuration with model selection in knowledge base settings, support for multi-tenant shared built-in models | -| Embedding Models | ✅ Local models, BGE / GTE APIs, etc. | Customizable embedding models, compatible with local deployment and cloud vector generation APIs | -| Vector DB Integration | ✅ PostgreSQL (pgvector), Elasticsearch | Support for mainstream vector index backends, flexible switching for different retrieval scenarios | -| Retrieval Strategies | ✅ BM25 / Dense Retrieval / GraphRAG | Support for sparse/dense recall and knowledge graph-enhanced retrieval with customizable retrieve-rerank-generate pipelines | -| LLM Integration | ✅ Support for Qwen, DeepSeek, etc., with thinking/non-thinking mode switching | Compatible with local models (e.g., via Ollama) or external API services with flexible inference configuration | +| Module | Support | Description | +|---------|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Agent Mode | ✅ ReACT Agent Mode | Support for using built-in tools to retrieve knowledge bases, MCP tools, and web search, with cross-knowledge base retrieval and multiple iterations | +| Knowledge Base Types | ✅ FAQ / Document | Support for creating FAQ and document knowledge base types, with folder import, URL import, tag management, and online entry | +| Document Formats | ✅ PDF / Word / Txt / Markdown / Images (with OCR / Caption) | Support for structured and unstructured documents with text extraction from images | +| Model Management | ✅ Centralized configuration, built-in model sharing | Centralized model configuration with model selection in knowledge base settings, support for multi-tenant shared built-in models | +| Embedding Models | ✅ Local models, BGE / GTE APIs, etc. | Customizable embedding models, compatible with local deployment and cloud vector generation APIs | +| Vector DB Integration | ✅ PostgreSQL (pgvector), Elasticsearch | Support for mainstream vector index backends, flexible switching for different retrieval scenarios | +| Retrieval Strategies | ✅ BM25 / Dense Retrieval / GraphRAG | Support for sparse/dense recall and knowledge graph-enhanced retrieval with customizable retrieve-rerank-generate pipelines | +| LLM Integration | ✅ Support for Qwen, DeepSeek, etc., with thinking/non-thinking mode switching | Compatible with local models (e.g., via Ollama) or external API services with flexible inference configuration | | Conversation Strategy | ✅ Agent models, normal mode models, retrieval thresholds, Prompt configuration | Support for configuring Agent models, normal mode models, retrieval thresholds, online Prompt configuration, precise control over multi-turn conversation behavior | -| Web Search | ✅ Extensible search engines, DuckDuckGo | Support for extensible web search engines with built-in DuckDuckGo search engine | -| MCP Tools | ✅ uvx, npx launchers, Stdio/HTTP Streamable/SSE | Support for extending Agent capabilities through MCP, with built-in uvx and npx launchers, supporting three transport methods | -| QA Capabilities | ✅ Context-aware, multi-turn dialogue, prompt templates | Support for complex semantic modeling, instruction control and chain-of-thought Q&A with configurable prompts and context windows | -| E2E Testing | ✅ Retrieval+generation process visualization and metric evaluation | End-to-end testing tools for evaluating recall hit rates, answer coverage, BLEU/ROUGE and other metrics | -| Deployment Modes | ✅ Support for local deployment / Docker images | Meets private, offline deployment and flexible operation requirements, with fast development mode support | -| User Interfaces | ✅ Web UI + RESTful API | Interactive interface and standard API endpoints, with Agent mode/normal mode switching and tool call process display | -| Task Management | ✅ MQ async tasks, automatic database migration | MQ-based async task state maintenance, support for automatic database schema and data migration during version upgrades | +| Web Search | ✅ Extensible search engines, DuckDuckGo / Google | Support for extensible web search engines with built-in DuckDuckGo search engine | +| MCP Tools | ✅ uvx, npx launchers, Stdio/HTTP Streamable/SSE | Support for extending Agent capabilities through MCP, with built-in uvx and npx launchers, supporting three transport methods | +| QA Capabilities | ✅ Context-aware, multi-turn dialogue, prompt templates | Support for complex semantic modeling, instruction control and chain-of-thought Q&A with configurable prompts and context windows | +| E2E Testing | ✅ Retrieval+generation process visualization and metric evaluation | End-to-end testing tools for evaluating recall hit rates, answer coverage, BLEU/ROUGE and other metrics | +| Deployment Modes | ✅ Support for local deployment / Docker images | Meets private, offline deployment and flexible operation requirements, with fast development mode support | +| User Interfaces | ✅ Web UI + RESTful API | Interactive interface and standard API endpoints, with Agent mode/normal mode switching and tool call process display | +| Task Management | ✅ MQ async tasks, automatic database migration | MQ-based async task state maintenance, support for automatic database schema and data migration during version upgrades | ## 🚀 Getting Started diff --git a/README_CN.md b/README_CN.md index 3a4a4562..b2795d9b 100644 --- a/README_CN.md +++ b/README_CN.md @@ -101,24 +101,24 @@ WeKnora 采用现代化模块化设计，构建了一条完整的文档理解与 ## 🧩 功能模块能力 -| 功能模块 | 支持情况 | 说明 | -|---------|---------|------| -| Agent模式 | ✅ ReACT Agent模式 | 支持使用内置工具检索知识库、MCP工具和网络搜索，跨知识库检索，多次迭代和反思 | -| 知识库类型 | ✅ FAQ / 文档 | 支持创建FAQ和文档两种类型知识库，支持文件夹导入、URL导入、标签管理和在线录入 | +| 功能模块 | 支持情况 | 说明 | +|---------|-----------------------------------------------------|------| +| Agent模式 | ✅ ReACT Agent模式 | 支持使用内置工具检索知识库、MCP工具和网络搜索，跨知识库检索，多次迭代和反思 | +| 知识库类型 | ✅ FAQ / 文档 | 支持创建FAQ和文档两种类型知识库，支持文件夹导入、URL导入、标签管理和在线录入 | | 文档格式支持 | ✅ PDF / Word / Txt / Markdown / 图片（含 OCR / Caption） | 支持多种结构化与非结构化文档内容解析，支持图文混排与图像文字提取 | -| 模型管理 | ✅ 集中配置、内置模型共享 | 模型集中配置，知识库设置页增加模型选择，支持多租户共享内置模型 | -| 嵌入模型支持 | ✅ 本地模型、BGE / GTE API 等 | 支持自定义 embedding 模型，兼容本地部署与云端向量生成接口 | -| 向量数据库接入 | ✅ PostgreSQL（pgvector）、Elasticsearch | 支持主流向量索引后端，可灵活切换与扩展，适配不同检索场景 | -| 检索机制 | ✅ BM25 / Dense Retrieve / GraphRAG | 支持稠密/稀疏召回、知识图谱增强检索等多种策略，可自由组合召回-重排-生成流程 | -| 大模型集成 | ✅ 支持 Qwen、DeepSeek 等，思考/非思考模式切换 | 可接入本地大模型（如 Ollama 启动）或调用外部 API 服务，支持推理模式灵活配置 | -| 对话策略 | ✅ Agent模型、普通模式模型、检索阈值、Prompt配置 | 支持配置Agent模型、普通模式所需的模型、检索阈值，在线配置Prompt，精确控制多轮对话行为 | -| 网络搜索 | ✅ 可扩展搜索引擎、DuckDuckGo | 支持可扩展的网络搜索引擎，内置DuckDuckGo搜索引擎 | -| MCP工具 | ✅ uvx、npx启动工具，Stdio/HTTP Streamable/SSE | 支持通过MCP扩展Agent能力，内置uvx、npx两种MCP启动工具，支持三种传输方式 | -| 问答能力 | ✅ 上下文感知、多轮对话、提示词模板 | 支持复杂语义建模、指令控制与链式问答，可配置提示词与上下文窗口 | -| 端到端测试支持 | ✅ 检索+生成过程可视化与指标评估 | 提供一体化链路测试工具，支持评估召回命中率、回答覆盖度、BLEU / ROUGE 等主流指标 | -| 部署模式 | ✅ 支持本地部署 / Docker 镜像 | 满足私有化、离线部署与灵活运维的需求，支持快速开发模式 | -| 用户界面 | ✅ Web UI + RESTful API | 提供交互式界面与标准 API 接口，支持Agent模式/普通模式切换，展示工具调用过程 | -| 任务管理 | ✅ MQ异步任务、数据库自动迁移 | 引入MQ对异步任务进行状态维护，支持版本升级时的数据库表结构和数据自动迁移 | +| 模型管理 | ✅ 集中配置、内置模型共享 | 模型集中配置，知识库设置页增加模型选择，支持多租户共享内置模型 | +| 嵌入模型支持 | ✅ 本地模型、BGE / GTE API 等 | 支持自定义 embedding 模型，兼容本地部署与云端向量生成接口 | +| 向量数据库接入 | ✅ PostgreSQL（pgvector）、Elasticsearch | 支持主流向量索引后端，可灵活切换与扩展，适配不同检索场景 | +| 检索机制 | ✅ BM25 / Dense Retrieve / GraphRAG | 支持稠密/稀疏召回、知识图谱增强检索等多种策略，可自由组合召回-重排-生成流程 | +| 大模型集成 | ✅ 支持 Qwen、DeepSeek 等，思考/非思考模式切换 | 可接入本地大模型（如 Ollama 启动）或调用外部 API 服务，支持推理模式灵活配置 | +| 对话策略 | ✅ Agent模型、普通模式模型、检索阈值、Prompt配置 | 支持配置Agent模型、普通模式所需的模型、检索阈值，在线配置Prompt，精确控制多轮对话行为 | +| 网络搜索 | ✅ 可扩展搜索引擎、DuckDuckGo / Google | 支持可扩展的网络搜索引擎，内置DuckDuckGo搜索引擎 | +| MCP工具 | ✅ uvx、npx启动工具，Stdio/HTTP Streamable/SSE | 支持通过MCP扩展Agent能力，内置uvx、npx两种MCP启动工具，支持三种传输方式 | +| 问答能力 | ✅ 上下文感知、多轮对话、提示词模板 | 支持复杂语义建模、指令控制与链式问答，可配置提示词与上下文窗口 | +| 端到端测试支持 | ✅ 检索+生成过程可视化与指标评估 | 提供一体化链路测试工具，支持评估召回命中率、回答覆盖度、BLEU / ROUGE 等主流指标 | +| 部署模式 | ✅ 支持本地部署 / Docker 镜像 | 满足私有化、离线部署与灵活运维的需求，支持快速开发模式 | +| 用户界面 | ✅ Web UI + RESTful API | 提供交互式界面与标准 API 接口，支持Agent模式/普通模式切换，展示工具调用过程 | +| 任务管理 | ✅ MQ异步任务、数据库自动迁移 | 引入MQ对异步任务进行状态维护，支持版本升级时的数据库表结构和数据自动迁移 | ## 🚀 快速开始 diff --git a/README_JA.md b/README_JA.md index c70a181f..e00ad667 100644 --- a/README_JA.md +++ b/README_JA.md @@ -101,24 +101,24 @@ WeKnoraは現代的なモジュラー設計を採用し、完全な文書理解 ## 🧩 機能モジュール能力 -| 機能モジュール | サポート状況 | 説明 | -|---------|---------|------| -| Agentモード | ✅ ReACT Agentモード | 組み込みツールでナレッジベースを検索、MCPツールとWeb検索を使用、クロスナレッジベース検索、複数回の反復とリフレクションをサポート | -| ナレッジベースタイプ | ✅ FAQ / ドキュメント | FAQとドキュメントの2種類のナレッジベースの作成をサポート、フォルダーインポート、URLインポート、タグ管理、オンライン入力機能 | +| 機能モジュール | サポート状況 | 説明 | +|---------|-----------------------------------------------------|------| +| Agentモード | ✅ ReACT Agentモード | 組み込みツールでナレッジベースを検索、MCPツールとWeb検索を使用、クロスナレッジベース検索、複数回の反復とリフレクションをサポート | +| ナレッジベースタイプ | ✅ FAQ / ドキュメント | FAQとドキュメントの2種類のナレッジベースの作成をサポート、フォルダーインポート、URLインポート、タグ管理、オンライン入力機能 | | 文書フォーマットサポート | ✅ PDF / Word / Txt / Markdown / 画像（OCR / Caption含む） | 様々な構造化・非構造化文書コンテンツの解析をサポート、図文混在と画像文字抽出をサポート | -| モデル管理 | ✅ 集中設定、組み込みモデル共有 | モデルの集中設定、ナレッジベース設定ページにモデル選択を追加、マルチテナント間での組み込みモデル共有をサポート | -| 埋め込みモデルサポート | ✅ ローカルモデル、BGE / GTE API等 | カスタムembeddingモデルをサポート、ローカルデプロイとクラウドベクトル生成インターフェースに対応 | -| ベクトルデータベース接続 | ✅ PostgreSQL（pgvector）、Elasticsearch | 主流のベクトルインデックスバックエンドをサポート、柔軟な切り替えと拡張が可能、異なる検索シナリオに適応 | -| 検索メカニズム | ✅ BM25 / Dense Retrieve / GraphRAG | 密・疎検索、ナレッジグラフ強化検索など複数の戦略をサポート、検索-再ランキング-生成プロセスを自由に組み合わせ可能 | -| 大規模モデル統合 | ✅ Qwen、DeepSeek等をサポート、思考/非思考モード切り替え | ローカル大規模モデル（Ollama起動など）に接続可能、または外部APIサービスを呼び出し、推論モードの柔軟な設定をサポート | -| 対話戦略 | ✅ Agentモデル、通常モードモデル、検索閾値、Prompt設定 | Agentモデル、通常モードに必要なモデル、検索閾値の設定をサポート、オンラインPrompt設定、マルチターン対話の動作を精密に制御 | -| Web検索 | ✅ 拡張可能な検索エンジン、DuckDuckGo | 拡張可能なWeb検索エンジンをサポート、DuckDuckGo検索エンジンを組み込み | -| MCPツール | ✅ uvx、npx起動ツール、Stdio/HTTP Streamable/SSE | MCPを通じてAgent機能を拡張、uvx、npxの2種類のMCP起動ツールを組み込み、3種類の転送方式をサポート | -| Q&A能力 | ✅ コンテキスト認識、マルチターン対話、プロンプトテンプレート | 複雑な意味モデリング、指示制御、チェーンQ&Aをサポート、プロンプトとコンテキストウィンドウを設定可能 | -| エンドツーエンドテストサポート | ✅ 検索+生成プロセスの可視化と指標評価 | 一体化されたリンクテストツールを提供、リコール的中率、回答カバレッジ、BLEU / ROUGE等の主流指標の評価をサポート | -| デプロイメントモード | ✅ ローカルデプロイメント / Dockerイメージ | プライベート化、オフラインデプロイメント、柔軟な運用保守のニーズに対応、高速開発モードをサポート | -| ユーザーインターフェース | ✅ Web UI + RESTful API | インタラクティブインターフェースと標準APIインターフェースを提供、Agentモード/通常モードの切り替え、ツール呼び出しプロセスの表示をサポート | -| タスク管理 | ✅ MQ非同期タスク、データベース自動マイグレーション | MQによる非同期タスクの状態維持を導入、バージョンアップ時のデータベーステーブル構造とデータの自動マイグレーションをサポート | +| モデル管理 | ✅ 集中設定、組み込みモデル共有 | モデルの集中設定、ナレッジベース設定ページにモデル選択を追加、マルチテナント間での組み込みモデル共有をサポート | +| 埋め込みモデルサポート | ✅ ローカルモデル、BGE / GTE API等 | カスタムembeddingモデルをサポート、ローカルデプロイとクラウドベクトル生成インターフェースに対応 | +| ベクトルデータベース接続 | ✅ PostgreSQL（pgvector）、Elasticsearch | 主流のベクトルインデックスバックエンドをサポート、柔軟な切り替えと拡張が可能、異なる検索シナリオに適応 | +| 検索メカニズム | ✅ BM25 / Dense Retrieve / GraphRAG | 密・疎検索、ナレッジグラフ強化検索など複数の戦略をサポート、検索-再ランキング-生成プロセスを自由に組み合わせ可能 | +| 大規模モデル統合 | ✅ Qwen、DeepSeek等をサポート、思考/非思考モード切り替え | ローカル大規模モデル（Ollama起動など）に接続可能、または外部APIサービスを呼び出し、推論モードの柔軟な設定をサポート | +| 対話戦略 | ✅ Agentモデル、通常モードモデル、検索閾値、Prompt設定 | Agentモデル、通常モードに必要なモデル、検索閾値の設定をサポート、オンラインPrompt設定、マルチターン対話の動作を精密に制御 | +| Web検索 | ✅ 拡張可能な検索エンジン、DuckDuckGo / Google | 拡張可能なWeb検索エンジンをサポート、DuckDuckGo検索エンジンを組み込み | +| MCPツール | ✅ uvx、npx起動ツール、Stdio/HTTP Streamable/SSE | MCPを通じてAgent機能を拡張、uvx、npxの2種類のMCP起動ツールを組み込み、3種類の転送方式をサポート | +| Q&A能力 | ✅ コンテキスト認識、マルチターン対話、プロンプトテンプレート | 複雑な意味モデリング、指示制御、チェーンQ&Aをサポート、プロンプトとコンテキストウィンドウを設定可能 | +| エンドツーエンドテストサポート | ✅ 検索+生成プロセスの可視化と指標評価 | 一体化されたリンクテストツールを提供、リコール的中率、回答カバレッジ、BLEU / ROUGE等の主流指標の評価をサポート | +| デプロイメントモード | ✅ ローカルデプロイメント / Dockerイメージ | プライベート化、オフラインデプロイメント、柔軟な運用保守のニーズに対応、高速開発モードをサポート | +| ユーザーインターフェース | ✅ Web UI + RESTful API | インタラクティブインターフェースと標準APIインターフェースを提供、Agentモード/通常モードの切り替え、ツール呼び出しプロセスの表示をサポート | +| タスク管理 | ✅ MQ非同期タスク、データベース自動マイグレーション | MQによる非同期タスクの状態維持を導入、バージョンアップ時のデータベーステーブル構造とデータの自動マイグレーションをサポート | ## 🚀 クイックスタート diff --git a/go.mod b/go.mod index 49694a64..09e00f00 100644 --- a/go.mod +++ b/go.mod @@ -41,19 +41,23 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0 - go.opentelemetry.io/otel/sdk v1.37.0 + go.opentelemetry.io/otel/sdk v1.38.0 go.opentelemetry.io/otel/trace v1.38.0 go.uber.org/dig v1.18.1 golang.org/x/crypto v0.46.0 golang.org/x/sync v0.19.0 - google.golang.org/grpc v1.76.0 - google.golang.org/protobuf v1.36.10 + google.golang.org/api v0.259.0 + google.golang.org/grpc v1.78.0 + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.11 gorm.io/gorm v1.25.12 ) require ( + cloud.google.com/go/auth v0.18.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect @@ -82,6 +86,7 @@ require ( github.com/duckdb/duckdb-go/mapping v0.0.27 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/elastic/elastic-transport-go/v8 v8.7.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/gabriel-vasile/mimetype v1.4.8 // indirect github.com/gin-contrib/sse v1.1.0 // indirect @@ -103,6 +108,9 @@ require ( github.com/goccy/go-yaml v1.18.0 // indirect github.com/google/flatbuffers v25.9.23+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect + github.com/googleapis/gax-go/v2 v2.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -149,6 +157,7 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect go.opentelemetry.io/otel/metric v1.38.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/mock v0.5.0 // indirect @@ -157,13 +166,14 @@ require ( golang.org/x/exp v0.0.0-20251209150349-8475f28825e9 // indirect golang.org/x/mod v0.31.0 // indirect golang.org/x/net v0.48.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sys v0.39.0 // indirect golang.org/x/telemetry v0.0.0-20251208220230-2638a1023523 // indirect golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.14.0 // indirect golang.org/x/tools v0.40.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 5f2e4b61..11706d90 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,9 @@ +cloud.google.com/go/auth v0.18.0 h1:wnqy5hrv7p3k7cShwAU/Br3nzod7fxoqG+k0VZ+/Pk0= +cloud.google.com/go/auth v0.18.0/go.mod h1:wwkPM1AgE1f2u6dG443MiWoD8C3BtOywNsUMcUTVDRo= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ= entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= @@ -176,9 +182,15 @@ github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.16.0 h1:iHbQmKLLZrexmb0OSsNGTeSTS0HO4YvFOG8g5E4Zd0Y= +github.com/googleapis/gax-go/v2 v2.16.0/go.mod h1:o1vfQjjNZn4+dPnRdl/4ZD7S9414Y4xA+a/6Icj6l14= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -402,10 +414,10 @@ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0 h1:T0Ec2E+3YZf5bgT go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.35.0/go.mod h1:30v2gqH+vYGJsesLWFov8u47EpYTcIQcBjKpI6pJThg= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= @@ -450,6 +462,8 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -516,16 +530,20 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b h1:ULiyYQ0FdsJhwwZUwbaXpZF5yUE3h+RA+gxvBu37ucc= -google.golang.org/genproto/googleapis/api v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:oDOGiMSXHL4sDTJvFvIB9nRQCGdLP1o/iVaqQK8zB+M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= -google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/api v0.259.0 h1:90TaGVIxScrh1Vn/XI2426kRpBqHwWIzVBzJsVZ5XrQ= +google.golang.org/api v0.259.0/go.mod h1:LC2ISWGWbRoyQVpxGntWwLWN/vLNxxKBK9KuJRI8Te4= +google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217 h1:GvESR9BIyHUahIb0NcTum6itIWtdoglGX+rnGxm2934= +google.golang.org/genproto v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:yJ2HH4EHEDTd3JiLmhds6NkJ17ITVYOdV3m3VKOnws0= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b h1:Mv8VFug0MP9e5vUxfBcE3vUkV6CImK3cMNMIDFjmzxU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/application/service/web_search.go b/internal/application/service/web_search.go index 44e4827d..f00f262a 100644 --- a/internal/application/service/web_search.go +++ b/internal/application/service/web_search.go @@ -304,6 +304,8 @@ func NewWebSearchService(cfg *config.Config) (interfaces.WebSearchService, error switch providerConfig.ID { case "duckduckgo": provider, err = web_search.NewDuckDuckGoProvider(providerConfig) + case "google": + provider, err = web_search.NewGoogleProvider(providerConfig) default: return nil, fmt.Errorf("unknown web search provider: %s", providerConfig.ID) } diff --git a/internal/application/service/web_search/google.go b/internal/application/service/web_search/google.go new file mode 100644 index 00000000..a241c737 --- /dev/null +++ b/internal/application/service/web_search/google.go @@ -0,0 +1,92 @@ +package web_search + +import ( + "context" + "fmt" + "net/url" + + "google.golang.org/api/customsearch/v1" + "google.golang.org/api/option" + + "github.com/Tencent/WeKnora/internal/config" + "github.com/Tencent/WeKnora/internal/types" + "github.com/Tencent/WeKnora/internal/types/interfaces" +) + +// GoogleProvider implements web search using Google Custom Search Engine API +type GoogleProvider struct { + srv *customsearch.Service + apiKey string + engineID string + baseURL string +} + +// NewGoogleProvider creates a new Google provider +func NewGoogleProvider(cfg config.WebSearchProviderConfig) (interfaces.WebSearchProvider, error) { + u, err := url.Parse(cfg.APIURL) + if err != nil { + return nil, err + } + engineID := u.Query().Get("engine_id") + if engineID == "" { + return nil, fmt.Errorf("engine_id is empty") + } + apiKey := u.Query().Get("api_key") + if apiKey == "" { + return nil, fmt.Errorf("api_key is empty") + } + clientOpts := make([]option.ClientOption, 0) + clientOpts = append(clientOpts, option.WithAPIKey(apiKey)) + clientOpts = append(clientOpts, option.WithEndpoint(u.Scheme+"://"+u.Host)) + srv, err := customsearch.NewService(context.Background(), clientOpts...) + if err != nil { + return nil, err + } + return &GoogleProvider{ + srv: srv, + apiKey: apiKey, + engineID: engineID, + baseURL: cfg.APIURL, + }, nil +} + +// Name returns the provider name +func (p *GoogleProvider) Name() string { + return "google" +} + +// Search performs a web search using Google Custom Search Engine API +func (p *GoogleProvider) Search( + ctx context.Context, + query string, + maxResults int, + includeDate bool, +) ([]*types.WebSearchResult, error) { + if len(query) == 0 { + return nil, fmt.Errorf("query is empty") + } + cseCall := p.srv.Cse.List().Context(ctx).Cx(p.engineID).Q(query) + + if maxResults > 0 { + cseCall = cseCall.Num(int64(maxResults)) + } else { + cseCall = cseCall.Num(5) + } + cseCall = cseCall.Hl("ch-zh") + + resp, err := cseCall.Do() + if err != nil { + return nil, err + } + results := make([]*types.WebSearchResult, 0) + for _, item := range resp.Items { + result := &types.WebSearchResult{ + Title: item.Title, + URL: item.Link, + Snippet: item.Snippet, + Source: "google", + } + results = append(results, result) + } + return results, nil +} diff --git a/internal/application/service/web_search/google_test.go b/internal/application/service/web_search/google_test.go new file mode 100644 index 00000000..058cf2a7 --- /dev/null +++ b/internal/application/service/web_search/google_test.go @@ -0,0 +1,278 @@ +package web_search + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/config" +) + +func TestNewGoogleProvider(t *testing.T) { + testCases := []struct { + name string + cfg config.WebSearchProviderConfig + expected error + }{ + { + name: "valid config", + cfg: config.WebSearchProviderConfig{ + APIURL: "https://customsearch.googleapis.com/customsearch/v1?api_key=test&engine_id=test", + }, + expected: nil, + }, + { + name: "missing engine id", + cfg: config.WebSearchProviderConfig{ + APIURL: "https://customsearch.googleapis.com/customsearch/v1?api_key=test", + }, + expected: fmt.Errorf("engine_id is empty"), + }, + { + name: "missing api key", + cfg: config.WebSearchProviderConfig{ + APIURL: "https://customsearch.googleapis.com/customsearch/v1?engine_id=test", + }, + expected: fmt.Errorf("api_key is empty"), + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := NewGoogleProvider(tc.cfg) + if tc.expected == nil { + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + } else { + if err == nil { + t.Fatalf("expected error %v, got nil", tc.expected) + } + if !strings.Contains(err.Error(), tc.expected.Error()) { + t.Fatalf("expected error %v, got %v", tc.expected, err) + } + } + }) + } +} + +func TestGoogleProvider_Name(t *testing.T) { + cfg := config.WebSearchProviderConfig{ + APIURL: "https://customsearch.googleapis.com/customsearch/v1?api_key=test&engine_id=test", + } + p, err := NewGoogleProvider(cfg) + if err != nil { + t.Fatalf("failed to create Google provider: %v", err) + } + if p.Name() != "google" { + t.Fatalf("expected provider name google, got %s", p.Name()) + } +} + +func TestGoogleProvider_Search(t *testing.T) { + mockResponse := map[string]interface{}{ + "items": []map[string]interface{}{ + { + "title": "Example Search Result One", + "link": "https://example.com/page1", + "snippet": "This is the first search result snippet describing the content.", + }, + { + "title": "Example Search Result Two", + "link": "https://example.org/page2", + "snippet": "This is the second search result snippet with more details.", + }, + }, + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/customsearch/v1" { + t.Fatalf("unexpected request path: %s", r.URL.Path) + } + + query := r.URL.Query().Get("q") + if query != "weknora" { + t.Fatalf("unexpected query: %s", query) + } + + cx := r.URL.Query().Get("cx") + if cx != "test-engine-id" { + t.Fatalf("unexpected engine ID: %s", cx) + } + + num := r.URL.Query().Get("num") + if num != "5" { + t.Fatalf("unexpected num parameter: %s", num) + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + enc := json.NewEncoder(w) + _ = enc.Encode(mockResponse) + })) + defer ts.Close() + + cfg := config.WebSearchProviderConfig{ + APIURL: fmt.Sprintf("%s/customsearch/v1?api_key=test-key&engine_id=test-engine-id", ts.URL), + } + prov, err := NewGoogleProvider(cfg) + if err != nil { + t.Fatalf("failed to create Google provider: %v", err) + } + + gp := prov.(*GoogleProvider) + if gp == nil { + t.Fatalf("failed to cast to GoogleProvider") + } + + ctx := context.Background() + results, err := prov.Search(ctx, "weknora", 5, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(results) != 2 { + t.Fatalf("expected 2 results, got %d", len(results)) + } + + if results[0].Title != "Example Search Result One" || + results[0].URL != "https://example.com/page1" || + results[0].Snippet != "This is the first search result snippet describing the content." || + results[0].Source != "google" { + t.Fatalf("unexpected first result: %+v", results[0]) + } + + if results[1].Title != "Example Search Result Two" || + results[1].URL != "https://example.org/page2" || + results[1].Snippet != "This is the second search result snippet with more details." || + results[1].Source != "google" { + t.Fatalf("unexpected second result: %+v", results[1]) + } +} + +func TestGoogleProvider_Search_EmptyQuery(t *testing.T) { + cfg := config.WebSearchProviderConfig{ + APIURL: "https://customsearch.googleapis.com/customsearch/v1?api_key=test&engine_id=test", + } + prov, err := NewGoogleProvider(cfg) + if err != nil { + t.Fatalf("failed to create Google provider: %v", err) + } + + ctx := context.Background() + results, err := prov.Search(ctx, "", 5, false) + if err == nil { + t.Fatal("expected error for empty query, got nil") + } + if !strings.Contains(err.Error(), "query is empty") { + t.Fatalf("expected 'query is empty' error, got: %v", err) + } + if results != nil { + t.Fatalf("expected nil results for empty query, got: %v", results) + } +} + +func TestGoogleProvider_Search_NoResults(t *testing.T) { + mockResponse := map[string]interface{}{ + "items": []map[string]interface{}{}, + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + enc := json.NewEncoder(w) + _ = enc.Encode(mockResponse) + })) + defer ts.Close() + + cfg := config.WebSearchProviderConfig{ + APIURL: fmt.Sprintf("%s/customsearch/v1?api_key=test-key&engine_id=test-engine-id", ts.URL), + } + prov, err := NewGoogleProvider(cfg) + if err != nil { + t.Fatalf("failed to create Google provider: %v", err) + } + + ctx := context.Background() + results, err := prov.Search(ctx, "nonexistent", 5, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(results) != 0 { + t.Fatalf("expected 0 results, got %d", len(results)) + } +} + +func TestGoogleProvider_Search_ErrorResponse(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Internal Server Error")) + })) + defer ts.Close() + + cfg := config.WebSearchProviderConfig{ + APIURL: fmt.Sprintf("%s/customsearch/v1?api_key=test-key&engine_id=test-engine-id", ts.URL), + } + prov, err := NewGoogleProvider(cfg) + if err != nil { + t.Fatalf("failed to create Google provider: %v", err) + } + + ctx := context.Background() + results, err := prov.Search(ctx, "test", 5, false) + if err == nil { + t.Fatal("expected error for server error response, got nil") + } + if results != nil { + t.Fatalf("expected nil results for error response, got: %v", results) + } +} + +func TestGoogleProvider_Search_MaxResults(t *testing.T) { + mockResponse := map[string]interface{}{ + "items": []map[string]interface{}{ + {"title": "Result 1", "link": "https://example.com/1", "snippet": "Snippet 1"}, + {"title": "Result 2", "link": "https://example.com/2", "snippet": "Snippet 2"}, + {"title": "Result 3", "link": "https://example.com/3", "snippet": "Snippet 3"}, + }, + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + num := r.URL.Query().Get("num") + if num != "2" { + t.Fatalf("expected num=2, got %s", num) + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + enc := json.NewEncoder(w) + _ = enc.Encode(mockResponse) + })) + defer ts.Close() + + cfg := config.WebSearchProviderConfig{ + APIURL: fmt.Sprintf("%s/customsearch/v1?api_key=test-key&engine_id=test-engine-id", ts.URL), + } + prov, err := NewGoogleProvider(cfg) + if err != nil { + t.Fatalf("failed to create Google provider: %v", err) + } + + ctx := context.Background() + results, err := prov.Search(ctx, "test", 2, false) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(results) != 3 { + t.Fatalf("expected 3 results, got %d", len(results)) + } + + if results[0].Title != "Result 1" || results[1].Title != "Result 2" || results[2].Title != "Result 3" { + t.Fatalf("unexpected results order or content") + } +} From bd0fb766f8f718a579161f7f46c9392c2858dbcb Mon Sep 17 00:00:00 2001 From: begoniezhao Date: Wed, 14 Jan 2026 14:15:08 +0800 Subject: [PATCH 10/15] refactor: Improve error handling and remove unused code --- docreader/parser/chain_parser.py | 10 +++++++++- internal/handler/knowledgebase.go | 2 -- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docreader/parser/chain_parser.py b/docreader/parser/chain_parser.py index 0b5dec48..6e5698d0 100644 --- a/docreader/parser/chain_parser.py +++ b/docreader/parser/chain_parser.py @@ -57,7 +57,15 @@ class FirstParser(BaseParser): """ for p in self._parsers: logger.info(f"FirstParser: using parser {p.__class__.__name__}") - document = p.parse_into_text(content) + try: + document = p.parse_into_text(content) + except Exception: + logger.exception( + "FirstParser: parser %s raised exception; trying next parser", + p.__class__.__name__, + ) + continue + if document.is_valid(): logger.info(f"FirstParser: parser {p.__class__.__name__} succeeded") return document diff --git a/internal/handler/knowledgebase.go b/internal/handler/knowledgebase.go index 775640eb..f98c0717 100644 --- a/internal/handler/knowledgebase.go +++ b/internal/handler/knowledgebase.go @@ -1,7 +1,6 @@ package handler import ( - "context" "encoding/json" "net/http" "strconv" @@ -470,7 +469,6 @@ func (h *KnowledgeBaseHandler) GetKBCloneProgress(c *gin.Context) { // validateExtractConfig validates the graph configuration parameters func validateExtractConfig(config *types.ExtractConfig) error { - logger.Errorf(context.Background(), "Validating extract configuration: %+v", config) if config == nil { return nil } From 6c90480e824138f681e37cc59f9c15ce11e3b83d Mon Sep 17 00:00:00 2001 From: begoniezhao Date: Wed, 14 Jan 2026 18:10:30 +0800 Subject: [PATCH 11/15] refactor: Improve chunk merge/split logic and add validation --- docreader/splitter/splitter.py | 175 +++++++++++++++++- .../application/service/chat_pipline/merge.go | 5 +- 2 files changed, 171 insertions(+), 9 deletions(-) diff --git a/docreader/splitter/splitter.py b/docreader/splitter/splitter.py index 4454b3fd..cac6ea2a 100644 --- a/docreader/splitter/splitter.py +++ b/docreader/splitter/splitter.py @@ -75,9 +75,9 @@ class TextSplitter(BaseModel, Generic[T]): # link - Markdown link syntax [text](url) r"\[.*?\]\(.*?\)", # table header - Markdown table header with separator line - r"(?:\|[^|\n]*)+\|[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|[\r\n]+", + r"[ ]*(?:\|[^|\n]*)+\|[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|[\r\n]+", # table body - Markdown table rows - r"(?:\|[^|\n]*)+\|[\r\n]+", + r"[ ]*(?:\|[^|\n]*)+\|[\r\n]+", # code header - Code block start with language identifier r"```(?:\w+)[\r\n]+[^\r\n]*", ], @@ -137,6 +137,10 @@ class TextSplitter(BaseModel, Generic[T]): # Step 4: Merge splits into final chunks with overlap chunks = self._merge(splits) + + # Step 5: Validate chunks and test restoration + # self._validate_chunks(chunks, text) + return chunks def _split(self, text: str) -> List[str]: @@ -250,10 +254,17 @@ class TextSplitter(BaseModel, Generic[T]): cur_len > self.chunk_overlap or cur_len + split_len + cur_headers_len > self.chunk_size ): - # Remove the first element to reduce overlap + # Remove the first element to reduce overlap. + # If the first element is a prepended header (start==end), also remove it. first_chunk = cur_chunk.pop(0) cur_len -= self.len_function(first_chunk[2]) + # If we just popped a real content piece, there may be a header right after it + # (depending on previous iterations). Pop it only if it is actually a header. + if cur_chunk and first_chunk[0] == first_chunk[1]: + first_chunk = cur_chunk.pop(0) + cur_len -= self.len_function(first_chunk[2]) + # Prepend headers to new chunk if: # 1. Headers exist # 2. Headers + split fit in chunk size @@ -264,10 +275,8 @@ class TextSplitter(BaseModel, Generic[T]): and cur_headers not in split ): next_start = cur_chunk[0][0] if cur_chunk else cur_start - header_start = max(0, next_start - cur_headers_len) - header_end = cur_end - cur_chunk.insert(0, (header_start, header_end, cur_headers)) + cur_chunk.insert(0, (next_start, next_start, cur_headers)) cur_len += cur_headers_len # Add current split to the chunk @@ -397,6 +406,154 @@ class TextSplitter(BaseModel, Generic[T]): start = end return res + def _validate_chunks( + self, chunks: List[Tuple[int, int, str]], original_text: str + ) -> None: + """Validate chunks order and test text restoration. + + This method performs two validations: + 1. Checks if chunk start positions are in ascending order + 2. Tests if the original text can be restored from chunks + + If validation fails, saves debug information to /tmp/chunk_error_.md + + Args: + chunks: List of tuples (start_pos, end_pos, chunk_text) to validate + original_text: The original text that was split + """ + import datetime + + errors = [] + + # Validation 1: Check if start positions are in ascending order + for i in range(1, len(chunks)): + prev_start = chunks[i - 1][0] + curr_start = chunks[i][0] + if curr_start < prev_start: + error_msg = ( + f"Chunk order error: chunk[{i}] start position ({curr_start}) " + f"is less than chunk[{i - 1}] start position ({prev_start})" + ) + errors.append(error_msg) + logger.error(error_msg) + + # Validation 2: Test text restoration + try: + restored_text = self.restore_text(chunks) + if restored_text != original_text: + error_msg = ( + f"Restoration failed: restored text differs from original. " + f"Original length: {len(original_text)}, " + f"Restored length: {len(restored_text)}" + ) + errors.append(error_msg) + logger.error(error_msg) + + # Find first difference position + min_len = min(len(original_text), len(restored_text)) + diff_pos = -1 + for i in range(min_len): + if original_text[i] != restored_text[i]: + diff_pos = i + break + + if diff_pos >= 0: + context_start = max(0, diff_pos - 50) + context_end = min(len(original_text), diff_pos + 50) + errors.append( + f"First difference at position {diff_pos}:\n" + f"Original: {repr(original_text[context_start:context_end])}\n" + f"Restored: {repr(restored_text[context_start:context_end])}" + ) + elif len(original_text) != len(restored_text): + errors.append( + f"Texts match up to position {min_len}, but lengths differ" + ) + except Exception as e: + error_msg = f"Restoration exception: {str(e)}" + errors.append(error_msg) + logger.error(error_msg) + + # If there are errors, save debug information to file + if errors: + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + error_file = f"/tmp/chunk_error_{timestamp}.md" + + with open(error_file, "w", encoding="utf-8") as f: + f.write("# Chunk Validation Error Report\n\n") + f.write(f"Timestamp: {timestamp}\n\n") + + f.write("## Errors\n\n") + for error in errors: + f.write(f"- {error}\n\n") + + f.write("\n## Original Text\n\n") + f.write(f"Length: {len(original_text)}\n\n") + f.write("```\n") + f.write(original_text) + f.write("\n```\n\n") + + f.write("\n## Chunks Information\n\n") + f.write(f"Total chunks: {len(chunks)}\n\n") + for i, (start, end, chunk_text) in enumerate(chunks): + f.write(f"### Chunk {i}\n\n") + f.write(f"- Position: [{start}:{end}]\n") + f.write(f"- Length: {len(chunk_text)}\n") + f.write(f"- Content:\n\n```\n{chunk_text}\n```\n\n") + + try: + restored_text = self.restore_text(chunks) + f.write("\n## Restored Text\n\n") + f.write(f"Length: {len(restored_text)}\n\n") + f.write("```\n") + f.write(restored_text) + f.write("\n```\n") + except Exception as e: + f.write("\n## Restoration Failed\n\n") + f.write(f"Error: {str(e)}\n") + + logger.error(f"Validation errors saved to: {error_file}") + + def restore_text(self, chunks: List[Tuple[int, int, str]]) -> str: + """Restore original text from chunks with overlap handling. + + This method reconstructs the original text from chunks that may contain: + - Overlapping content between consecutive chunks + - Prepended headers that were added during merging (headers have start==end position) + + The algorithm: + 1. Sort chunks by their start position (and end position as tiebreaker) + 2. Track the maximum end position seen so far + 3. For each chunk, extract only the new content (after max_end_pos) + 4. Concatenate all new content pieces + + Args: + chunks: List of tuples (start_pos, end_pos, chunk_text) from split_text() + + Returns: + The restored original text + + Example: + >>> splitter = TextSplitter(chunk_size=10, chunk_overlap=3) + >>> chunks = splitter.split_text("Hello World!") + >>> restored = splitter.restore_text(chunks) + >>> assert restored == "Hello World!" + """ + if not chunks: + return "" + + # Sort chunks by start position, then by end position + sorted_chunks = sorted(chunks, key=lambda x: (x[1], x[0])) + + result_parts = [] + last_end = 0 + + for start_pos, end_pos, chunk_text in sorted_chunks: + result_parts.append(chunk_text[last_end - end_pos :]) + last_end = end_pos + + return "".join(result_parts) + if __name__ == "__main__": s = """ @@ -415,7 +572,11 @@ if __name__ == "__main__": """ - sp = TextSplitter(chunk_size=200, chunk_overlap=2) + sp = TextSplitter( + chunk_size=200, + chunk_overlap=10, + separators=["\n\n", "\n", "。", "？", "！", "，", "；", "："], + ) ck = sp.split_text(s) for c in ck: print("------", len(c)) diff --git a/internal/application/service/chat_pipline/merge.go b/internal/application/service/chat_pipline/merge.go index 0ed92be4..68d1b77c 100644 --- a/internal/application/service/chat_pipline/merge.go +++ b/internal/application/service/chat_pipline/merge.go @@ -101,8 +101,9 @@ func (p *PluginMerge) OnEvent(ctx context.Context, } // Merge overlapping chunks if chunks[i].EndAt > lastChunk.EndAt { - lastChunk.Content = lastChunk.Content + - string([]rune(chunks[i].Content)[lastChunk.EndAt-chunks[i].StartAt:]) + content := string([]rune(chunks[i].Content)) + offset := len(content) - (chunks[i].EndAt - lastChunk.EndAt) + lastChunk.Content = lastChunk.Content + content[offset:] lastChunk.EndAt = chunks[i].EndAt lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID) From cec722aa43c08f9c94366cad07475c516662dc19 Mon Sep 17 00:00:00 2001 From: Xia Dong Date: Wed, 14 Jan 2026 18:51:12 +0800 Subject: [PATCH 12/15] feat(knowledge): Add tag_id support for knowledge creation and file uploads - Introduced tag_id parameter in various knowledge-related API functions to allow categorization of knowledge entries. - Updated frontend components to handle tag_id during knowledge creation and file uploads. - Enhanced UI store to manage the selected tag ID for better user experience in categorizing knowledge. --- frontend/src/api/knowledge-base/index.ts | 235 +++++++++--------- .../components/manual-knowledge-editor.vue | 7 +- frontend/src/components/menu.vue | 8 +- frontend/src/hooks/useKnowledgeBase.ts | 7 +- frontend/src/stores/ui.ts | 7 + .../src/views/knowledge/KnowledgeBase.vue | 11 +- internal/application/service/knowledge.go | 8 +- internal/handler/knowledge.go | 14 +- internal/types/interfaces/knowledge.go | 4 + internal/types/knowledge.go | 1 + 10 files changed, 178 insertions(+), 124 deletions(-) diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts index 15aa4c28..8a59da2a 100644 --- a/frontend/src/api/knowledge-base/index.ts +++ b/frontend/src/api/knowledge-base/index.ts @@ -2,175 +2,184 @@ import { get, post, put, del, postUpload, getDown } from "../../utils/request"; // 知识库管理 API（列表、创建、获取、更新、删除、复制） export function listKnowledgeBases() { - return get(`/api/v1/knowledge-bases`); + return get(`/api/v1/knowledge-bases`); } -export function createKnowledgeBase(data: { - name: string; - description?: string; - type?: 'document' | 'faq'; - chunking_config?: any; - embedding_model_id?: string; - summary_model_id?: string; - vlm_config?: { - enabled: boolean; - model_id?: string; - }; - cos_config?: any; - extract_config?: any; - faq_config?: { index_mode: string; question_index_mode?: string }; +export function createKnowledgeBase(data: { + name: string; + description?: string; + type?: 'document' | 'faq'; + chunking_config?: any; + embedding_model_id?: string; + summary_model_id?: string; + vlm_config?: { + enabled: boolean; + model_id?: string; + }; + cos_config?: any; + extract_config?: any; + faq_config?: { index_mode: string; question_index_mode?: string }; }) { - return post(`/api/v1/knowledge-bases`, data); + return post(`/api/v1/knowledge-bases`, data); } export function getKnowledgeBaseById(id: string) { - return get(`/api/v1/knowledge-bases/${id}`); + return get(`/api/v1/knowledge-bases/${id}`); } export function updateKnowledgeBase(id: string, data: { name: string; description?: string; config: any }) { - return put(`/api/v1/knowledge-bases/${id}` , data); + return put(`/api/v1/knowledge-bases/${id}`, data); } export function deleteKnowledgeBase(id: string) { - return del(`/api/v1/knowledge-bases/${id}`); + return del(`/api/v1/knowledge-bases/${id}`); } export function copyKnowledgeBase(data: { source_id: string; target_id?: string }) { - return post(`/api/v1/knowledge-bases/copy`, data); + return post(`/api/v1/knowledge-bases/copy`, data); } // 知识文件 API（基于具体知识库） -export function uploadKnowledgeFile(kbId: string, data = {}, onProgress?: (progressEvent: any) => void) { - return postUpload(`/api/v1/knowledge-bases/${kbId}/knowledge/file`, data, onProgress); +// data.tag_id: 可选，指定知识所属的分类ID +export function uploadKnowledgeFile(kbId: string, data: { file: File; tag_id?: string;[key: string]: any } = { file: new File([], '') }, onProgress?: (progressEvent: any) => void) { + const formData = new FormData(); + Object.keys(data).forEach(key => { + if (data[key] !== undefined) formData.append(key, data[key]); + }); + return postUpload(`/api/v1/knowledge-bases/${kbId}/knowledge/file`, formData, onProgress); } -export function createKnowledgeFromURL(kbId: string, data: { url: string; enable_multimodel?: boolean }) { - return post(`/api/v1/knowledge-bases/${kbId}/knowledge/url`, data); +// 从URL创建知识 +// data.tag_id: 可选，指定知识所属的分类ID +export function createKnowledgeFromURL(kbId: string, data: { url: string; enable_multimodel?: boolean; tag_id?: string }) { + return post(`/api/v1/knowledge-bases/${kbId}/knowledge/url`, data); } -export function createManualKnowledge(kbId: string, data: { title: string; content: string; status: string }) { - return post(`/api/v1/knowledge-bases/${kbId}/knowledge/manual`, data); +// 手工创建知识 +// data.tag_id: 可选，指定知识所属的分类ID +export function createManualKnowledge(kbId: string, data: { title: string; content: string; status: string; tag_id?: string }) { + return post(`/api/v1/knowledge-bases/${kbId}/knowledge/manual`, data); } export function listKnowledgeFiles( - kbId: string, - params: { page: number; page_size: number; tag_id?: string; keyword?: string; file_type?: string }, + kbId: string, + params: { page: number; page_size: number; tag_id?: string; keyword?: string; file_type?: string }, ) { - const query = new URLSearchParams(); - query.append('page', String(params.page)); - query.append('page_size', String(params.page_size)); - if (params.tag_id) { - query.append('tag_id', params.tag_id); - } - if (params.keyword) { - query.append('keyword', params.keyword); - } - if (params.file_type) { - query.append('file_type', params.file_type); - } - const qs = query.toString(); - return get(`/api/v1/knowledge-bases/${kbId}/knowledge?${qs}`); + const query = new URLSearchParams(); + query.append('page', String(params.page)); + query.append('page_size', String(params.page_size)); + if (params.tag_id) { + query.append('tag_id', params.tag_id); + } + if (params.keyword) { + query.append('keyword', params.keyword); + } + if (params.file_type) { + query.append('file_type', params.file_type); + } + const qs = query.toString(); + return get(`/api/v1/knowledge-bases/${kbId}/knowledge?${qs}`); } export function getKnowledgeDetails(id: string) { - return get(`/api/v1/knowledge/${id}`); + return get(`/api/v1/knowledge/${id}`); } export function updateManualKnowledge(id: string, data: { title: string; content: string; status: string }) { - return put(`/api/v1/knowledge/manual/${id}`, data); + return put(`/api/v1/knowledge/manual/${id}`, data); } export function delKnowledgeDetails(id: string) { - return del(`/api/v1/knowledge/${id}`); + return del(`/api/v1/knowledge/${id}`); } export function downKnowledgeDetails(id: string) { - return getDown(`/api/v1/knowledge/${id}/download`); + return getDown(`/api/v1/knowledge/${id}/download`); } export function batchQueryKnowledge(idsQueryString: string) { - return get(`/api/v1/knowledge/batch?${idsQueryString}`); + return get(`/api/v1/knowledge/batch?${idsQueryString}`); } export function getKnowledgeDetailsCon(id: string, page: number) { - return get(`/api/v1/chunks/${id}?page=${page}&page_size=25`); + return get(`/api/v1/chunks/${id}?page=${page}&page_size=25`); } // Get chunk by chunk_id only (new endpoint - to be added to backend) export function getChunkByIdOnly(chunkId: string) { - return get(`/api/v1/chunks/by-id/${chunkId}`); + return get(`/api/v1/chunks/by-id/${chunkId}`); } // Delete a single generated question from a chunk by question ID export function deleteGeneratedQuestion(chunkId: string, questionId: string) { - return del(`/api/v1/chunks/by-id/${chunkId}/questions`, { question_id: questionId }); + return del(`/api/v1/chunks/by-id/${chunkId}/questions`, { question_id: questionId }); } export function listKnowledgeTags( - kbId: string, - params?: { page?: number; page_size?: number; keyword?: string }, + kbId: string, + params?: { page?: number; page_size?: number; keyword?: string }, ) { - const query = buildQuery(params); - return get(`/api/v1/knowledge-bases/${kbId}/tags${query}`); + const query = buildQuery(params); + return get(`/api/v1/knowledge-bases/${kbId}/tags${query}`); } export function createKnowledgeBaseTag( - kbId: string, - data: { name: string; color?: string; sort_order?: number }, + kbId: string, + data: { name: string; color?: string; sort_order?: number }, ) { - return post(`/api/v1/knowledge-bases/${kbId}/tags`, data); + return post(`/api/v1/knowledge-bases/${kbId}/tags`, data); } export function updateKnowledgeBaseTag( - kbId: string, - tagId: string, - data: { name?: string; color?: string; sort_order?: number }, + kbId: string, + tagId: string, + data: { name?: string; color?: string; sort_order?: number }, ) { - return put(`/api/v1/knowledge-bases/${kbId}/tags/${tagId}`, data); + return put(`/api/v1/knowledge-bases/${kbId}/tags/${tagId}`, data); } export function deleteKnowledgeBaseTag(kbId: string, tagId: string, params?: { force?: boolean }) { - const forceQuery = params?.force ? '?force=true' : ''; - return del(`/api/v1/knowledge-bases/${kbId}/tags/${tagId}${forceQuery}`); + const forceQuery = params?.force ? '?force=true' : ''; + return del(`/api/v1/knowledge-bases/${kbId}/tags/${tagId}${forceQuery}`); } export function updateKnowledgeTagBatch(data: { updates: Record }) { - return put(`/api/v1/knowledge/tags`, data); + return put(`/api/v1/knowledge/tags`, data); } export function updateFAQEntryTagBatch(kbId: string, data: { updates: Record }) { - return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/tags`, data); + return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/tags`, data); } const buildQuery = (params?: Record) => { - if (!params) return ''; - const query = new URLSearchParams(); - Object.entries(params).forEach(([key, value]) => { - if (value === undefined || value === null || value === '') return; - query.append(key, String(value)); - }); - const queryString = query.toString(); - return queryString ? `?${queryString}` : ''; + if (!params) return ''; + const query = new URLSearchParams(); + Object.entries(params).forEach(([key, value]) => { + if (value === undefined || value === null || value === '') return; + query.append(key, String(value)); + }); + const queryString = query.toString(); + return queryString ? `?${queryString}` : ''; }; export function listFAQEntries( - kbId: string, - params?: { page?: number; page_size?: number; tag_id?: string; keyword?: string }, + kbId: string, + params?: { page?: number; page_size?: number; tag_id?: string; keyword?: string }, ) { - const query = buildQuery(params); - return get(`/api/v1/knowledge-bases/${kbId}/faq/entries${query}`); + const query = buildQuery(params); + return get(`/api/v1/knowledge-bases/${kbId}/faq/entries${query}`); } export function upsertFAQEntries(kbId: string, data: { entries: any[]; mode: 'append' | 'replace' }) { - return post(`/api/v1/knowledge-bases/${kbId}/faq/entries`, data); + return post(`/api/v1/knowledge-bases/${kbId}/faq/entries`, data); } export function createFAQEntry(kbId: string, data: any) { - return post(`/api/v1/knowledge-bases/${kbId}/faq/entry`, data); + return post(`/api/v1/knowledge-bases/${kbId}/faq/entry`, data); } export function updateFAQEntry(kbId: string, entryId: string, data: any) { - return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/${entryId}`, data); + return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/${entryId}`, data); } // Unified batch update API - supports is_enabled, is_recommended, tag_id @@ -178,64 +187,64 @@ export function updateFAQEntry(kbId: string, entryId: string, data: any) { // 1. By entry ID: use by_id field // 2. By Tag: use by_tag field to apply the same update to all entries under a tag export interface FAQEntryFieldsUpdate { - is_enabled?: boolean - is_recommended?: boolean - tag_id?: string | null + is_enabled?: boolean + is_recommended?: boolean + tag_id?: string | null } export interface FAQEntryFieldsBatchRequest { - by_id?: Record - by_tag?: Record + by_id?: Record + by_tag?: Record } export function updateFAQEntryFieldsBatch(kbId: string, data: FAQEntryFieldsBatchRequest) { - return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/fields`, data); + return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/fields`, data); } export function deleteFAQEntries(kbId: string, ids: string[]) { - return del(`/api/v1/knowledge-bases/${kbId}/faq/entries`, { ids }); + return del(`/api/v1/knowledge-bases/${kbId}/faq/entries`, { ids }); } export function searchFAQEntries( - kbId: string, - data: { - query_text: string - vector_threshold?: number - match_count?: number - } + kbId: string, + data: { + query_text: string + vector_threshold?: number + match_count?: number + } ) { - return post(`/api/v1/knowledge-bases/${kbId}/faq/search`, data); + return post(`/api/v1/knowledge-bases/${kbId}/faq/search`, data); } // Export FAQ entries as CSV file export async function exportFAQEntries(kbId: string): Promise { - const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export`); - return response as unknown as Blob; + const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export`); + return response as unknown as Blob; } // FAQ Import Progress API export interface FAQImportProgress { - task_id: string - kb_id: string - knowledge_id: string - status: 'pending' | 'processing' | 'completed' | 'failed' - progress: number - total: number - processed: number - message: string - error: string - created_at: number - updated_at: number + task_id: string + kb_id: string + knowledge_id: string + status: 'pending' | 'processing' | 'completed' | 'failed' + progress: number + total: number + processed: number + message: string + error: string + created_at: number + updated_at: number } export function getFAQImportProgress(taskId: string) { - return get(`/api/v1/faq/import/progress/${taskId}`); + return get(`/api/v1/faq/import/progress/${taskId}`); } export function searchKnowledge(keyword: string, offset = 0, limit = 20, fileTypes?: string[]) { - let url = `/api/v1/knowledge/search?keyword=${encodeURIComponent(keyword)}&offset=${offset}&limit=${limit}`; - if (fileTypes && fileTypes.length > 0) { - url += `&file_types=${encodeURIComponent(fileTypes.join(','))}`; - } - return get(url); + let url = `/api/v1/knowledge/search?keyword=${encodeURIComponent(keyword)}&offset=${offset}&limit=${limit}`; + if (fileTypes && fileTypes.length > 0) { + url += `&file_types=${encodeURIComponent(fileTypes.join(','))}`; + } + return get(url); } diff --git a/frontend/src/components/manual-knowledge-editor.vue b/frontend/src/components/manual-knowledge-editor.vue index 6a22a827..39a0b724 100644 --- a/frontend/src/components/manual-knowledge-editor.vue +++ b/frontend/src/components/manual-knowledge-editor.vue @@ -579,7 +579,7 @@ const handleSave = async (targetStatus: ManualStatus) => { saving.value = true savingAction.value = targetStatus try { - const payload = { + const payload: { title: string; content: string; status: string; tag_id?: string } = { title: form.title.trim(), content: form.content, status: targetStatus, @@ -591,6 +591,11 @@ const handleSave = async (targetStatus: ManualStatus) => { if (mode.value === 'edit' && knowledgeId.value) { response = await updateManualKnowledge(knowledgeId.value, payload) } else { + // 创建新知识时，从 store 获取当前选中的分类ID + const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined + if (tagIdToUpload) { + payload.tag_id = tagIdToUpload + } response = await createManualKnowledge(form.kbId, payload) knowledgeID = response?.data?.id || knowledgeID kbId = form.kbId diff --git a/frontend/src/components/menu.vue b/frontend/src/components/menu.vue index d7566329..46cc72e0 100644 --- a/frontend/src/components/menu.vue +++ b/frontend/src/components/menu.vue @@ -823,9 +823,11 @@ const handleDocFileChange = async (event: Event) => { })) try { + // 获取当前选中的分类ID + const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined await uploadKnowledgeFile( kbId, - { file }, + { file, tag_id: tagIdToUpload }, (progressEvent: any) => { if (progressEvent.total) { progress = Math.round((progressEvent.loaded * 100) / progressEvent.total) @@ -1067,9 +1069,11 @@ const handleDocFolderChange = async (event: Event) => { })) try { + // 获取当前选中的分类ID + const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined await uploadKnowledgeFile( kbId, - { file, fileName }, + { file, fileName, tag_id: tagIdToUpload }, (progressEvent: any) => { if (progressEvent?.total) { progress = Math.round((progressEvent.loaded * 100) / progressEvent.total) diff --git a/frontend/src/hooks/useKnowledgeBase.ts b/frontend/src/hooks/useKnowledgeBase.ts index 19d63629..a1b4de2f 100644 --- a/frontend/src/hooks/useKnowledgeBase.ts +++ b/frontend/src/hooks/useKnowledgeBase.ts @@ -10,6 +10,7 @@ import { getKnowledgeDetailsCon, } from "@/api/knowledge-base/index"; import { knowledgeStore } from "@/stores/knowledge"; +import { useUIStore } from "@/stores/ui"; import { useRoute } from 'vue-router'; const usemenuStore = knowledgeStore(); @@ -110,7 +111,11 @@ export default function (knowledgeBaseId?: string) { return; } - uploadKnowledgeFile(currentKbId, { file }) + // 获取当前选中的分类ID + const uiStore = useUIStore(); + const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined; + + uploadKnowledgeFile(currentKbId, { file, tag_id: tagIdToUpload }) .then((result: any) => { if (result.success) { MessagePlugin.info("上传成功！"); diff --git a/frontend/src/stores/ui.ts b/frontend/src/stores/ui.ts index 0b79bbaf..20f17bff 100644 --- a/frontend/src/stores/ui.ts +++ b/frontend/src/stores/ui.ts @@ -7,6 +7,8 @@ export const useUIStore = defineStore('ui', { kbEditorMode: 'create' as 'create' | 'edit', currentKBId: null as string | null, kbEditorType: 'document' as 'document' | 'faq', + // 当前选中的分类ID，用于文件上传时传递 + selectedTagId: '__untagged__' as string, kbEditorInitialSection: null as string | null, settingsInitialSection: null as string | null, settingsInitialSubSection: null as string | null, @@ -101,6 +103,11 @@ export const useUIStore = defineStore('ui', { } } this.manualEditorOnSuccess = null + }, + + // 设置当前选中的分类ID + setSelectedTagId(tagId: string) { + this.selectedTagId = tagId } } }) diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index 4adce1bd..f37846d4 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -220,6 +220,8 @@ const loadTags = async (kbIdValue: string, reset = false) => { const handleTagFilterChange = (value: string) => { selectedTagId.value = value; + // 同步更新 store 中的 selectedTagId，供 menu.vue 上传时使用 + uiStore.setSelectedTagId(value); page = 1; loadKnowledgeFiles(kbId.value); }; @@ -657,9 +659,12 @@ const handleDocumentUpload = async (event: Event) => { let failCount = 0; const totalCount = validFiles.length; + // 获取当前选中的分类ID（如果不是"未分类"则传递） + const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined; + for (const file of validFiles) { try { - const responseData: any = await uploadKnowledgeFile(kbId.value, { file }); + const responseData: any = await uploadKnowledgeFile(kbId.value, { file, tag_id: tagIdToUpload }); const isSuccess = responseData?.success || responseData?.code === 200 || responseData?.status === 'success' || (!responseData?.error && responseData); if (isSuccess) { successCount++; @@ -763,7 +768,9 @@ const handleURLImportConfirm = async () => { urlImporting.value = true; try { - const responseData: any = await createKnowledgeFromURL(kbId.value, { url }); + // 获取当前选中的分类ID + const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined; + const responseData: any = await createKnowledgeFromURL(kbId.value, { url, tag_id: tagIdToUpload }); window.dispatchEvent(new CustomEvent('knowledgeFileUploaded', { detail: { kbId: kbId.value } })); diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 1c9f286d..a08ec815 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -142,7 +142,7 @@ func (s *knowledgeService) isKnowledgeDeleting(ctx context.Context, tenantID uin // CreateKnowledgeFromFile creates a knowledge entry from an uploaded file func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, - kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string, + kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string, tagID string, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating knowledge from file") @@ -261,6 +261,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, knowledge := &types.Knowledge{ TenantID: tenantID, KnowledgeBaseID: kbID, + TagID: tagID, // 设置分类ID，用于知识分类管理 Type: "file", Title: safeFilename, FileName: safeFilename, @@ -358,8 +359,9 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, } // CreateKnowledgeFromURL creates a knowledge entry from a URL source +// tagID is optional - when provided, the knowledge will be assigned to the specified tag/category. func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context, - kbID string, url string, enableMultimodel *bool, title string, + kbID string, url string, enableMultimodel *bool, title string, tagID string, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating knowledge from URL") logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, url) @@ -426,6 +428,7 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context, CreatedAt: time.Now(), UpdatedAt: time.Now(), EmbeddingModelID: kb.EmbeddingModelID, + TagID: tagID, // 设置分类ID，用于知识分类管理 } // Save knowledge record @@ -557,6 +560,7 @@ func (s *knowledgeService) CreateKnowledgeFromManual(ctx context.Context, EmbeddingModelID: kb.EmbeddingModelID, FileName: fileName, FileType: types.KnowledgeTypeManual, + TagID: payload.TagID, // 设置分类ID，用于知识分类管理 } if err := knowledge.SetManualMetadata(meta); err != nil { logger.Errorf(ctx, "Failed to set manual metadata: %v", err) diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index c46f1606..734e3c90 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -164,8 +164,15 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) { enableMultimodel = &parseBool } + // 获取分类ID（如果提供），用于知识分类管理 + tagID := c.PostForm("tag_id") + // 过滤特殊值，空字符串或 "__untagged__" 表示未分类 + if tagID == "__untagged__" || tagID == "" { + tagID = "" + } + // Create knowledge entry from the file - knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName) + knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName, tagID) // Check for duplicate knowledge error if err != nil { if h.handleDuplicateKnowledgeError(c, err, knowledge, "file") { @@ -199,7 +206,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) { // @Accept json // @Produce json // @Param id path string true "知识库ID" -// @Param request body object{url=string,enable_multimodel=bool,title=string} true "URL请求" +// @Param request body object{url=string,enable_multimodel=bool,title=string,tag_id=string} true "URL请求" // @Success 201 {object} map[string]interface{} "创建的知识" // @Failure 400 {object} errors.AppError "请求参数错误" // @Failure 409 {object} map[string]interface{} "URL重复" @@ -222,6 +229,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) { URL string `json:"url" binding:"required"` EnableMultimodel *bool `json:"enable_multimodel"` Title string `json:"title"` + TagID string `json:"tag_id"` } if err := c.ShouldBindJSON(&req); err != nil { logger.Error(ctx, "Failed to parse URL request", err) @@ -238,7 +246,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) { ) // Create knowledge entry from the URL - knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.EnableMultimodel, req.Title) + knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.EnableMultimodel, req.Title, req.TagID) // Check for duplicate knowledge error if err != nil { if h.handleDuplicateKnowledgeError(c, err, knowledge, "url") { diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index b4f0d80d..530f8e6a 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -12,6 +12,7 @@ import ( // KnowledgeService defines the interface for knowledge services. type KnowledgeService interface { // CreateKnowledgeFromFile creates knowledge from a file. + // tagID is optional - when provided, the file will be assigned to the specified tag/category. CreateKnowledgeFromFile( ctx context.Context, kbID string, @@ -19,14 +20,17 @@ type KnowledgeService interface { metadata map[string]string, enableMultimodel *bool, customFileName string, + tagID string, ) (*types.Knowledge, error) // CreateKnowledgeFromURL creates knowledge from a URL. + // tagID is optional - when provided, the knowledge will be assigned to the specified tag/category. CreateKnowledgeFromURL( ctx context.Context, kbID string, url string, enableMultimodel *bool, title string, + tagID string, ) (*types.Knowledge, error) // CreateKnowledgeFromPassage creates knowledge from text passages. CreateKnowledgeFromPassage(ctx context.Context, kbID string, passage []string) (*types.Knowledge, error) diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go index ba38d0ed..012ff457 100644 --- a/internal/types/knowledge.go +++ b/internal/types/knowledge.go @@ -142,6 +142,7 @@ type ManualKnowledgePayload struct { Title string `json:"title"` Content string `json:"content"` Status string `json:"status"` + TagID string `json:"tag_id"` } // NewManualKnowledgeMetadata creates a new ManualKnowledgeMetadata instance. From 928d1c3d441943d936c2119e766beaff4ceffb7e Mon Sep 17 00:00:00 2001 From: Xia Dong Date: Wed, 14 Jan 2026 19:04:55 +0800 Subject: [PATCH 13/15] refactor(knowledge): Standardize formatting and improve code readability - Reformatted the API functions in the knowledge base module for consistent indentation and spacing. - Enhanced overall code readability without altering functionality. --- frontend/src/api/knowledge-base/index.ts | 230 +++++++++++------------ 1 file changed, 115 insertions(+), 115 deletions(-) diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts index 8a59da2a..31efab91 100644 --- a/frontend/src/api/knowledge-base/index.ts +++ b/frontend/src/api/knowledge-base/index.ts @@ -2,184 +2,184 @@ import { get, post, put, del, postUpload, getDown } from "../../utils/request"; // 知识库管理 API（列表、创建、获取、更新、删除、复制） export function listKnowledgeBases() { - return get(`/api/v1/knowledge-bases`); + return get(`/api/v1/knowledge-bases`); } -export function createKnowledgeBase(data: { - name: string; - description?: string; - type?: 'document' | 'faq'; - chunking_config?: any; - embedding_model_id?: string; - summary_model_id?: string; - vlm_config?: { - enabled: boolean; - model_id?: string; - }; - cos_config?: any; - extract_config?: any; - faq_config?: { index_mode: string; question_index_mode?: string }; +export function createKnowledgeBase(data: { + name: string; + description?: string; + type?: 'document' | 'faq'; + chunking_config?: any; + embedding_model_id?: string; + summary_model_id?: string; + vlm_config?: { + enabled: boolean; + model_id?: string; + }; + cos_config?: any; + extract_config?: any; + faq_config?: { index_mode: string; question_index_mode?: string }; }) { - return post(`/api/v1/knowledge-bases`, data); + return post(`/api/v1/knowledge-bases`, data); } export function getKnowledgeBaseById(id: string) { - return get(`/api/v1/knowledge-bases/${id}`); + return get(`/api/v1/knowledge-bases/${id}`); } export function updateKnowledgeBase(id: string, data: { name: string; description?: string; config: any }) { - return put(`/api/v1/knowledge-bases/${id}`, data); + return put(`/api/v1/knowledge-bases/${id}` , data); } export function deleteKnowledgeBase(id: string) { - return del(`/api/v1/knowledge-bases/${id}`); + return del(`/api/v1/knowledge-bases/${id}`); } export function copyKnowledgeBase(data: { source_id: string; target_id?: string }) { - return post(`/api/v1/knowledge-bases/copy`, data); + return post(`/api/v1/knowledge-bases/copy`, data); } // 知识文件 API（基于具体知识库） // data.tag_id: 可选，指定知识所属的分类ID -export function uploadKnowledgeFile(kbId: string, data: { file: File; tag_id?: string;[key: string]: any } = { file: new File([], '') }, onProgress?: (progressEvent: any) => void) { - const formData = new FormData(); - Object.keys(data).forEach(key => { - if (data[key] !== undefined) formData.append(key, data[key]); - }); - return postUpload(`/api/v1/knowledge-bases/${kbId}/knowledge/file`, formData, onProgress); +export function uploadKnowledgeFile(kbId: string, data: { file: File; tag_id?: string; [key: string]: any } = { file: new File([], '') }, onProgress?: (progressEvent: any) => void) { + const formData = new FormData(); + Object.keys(data).forEach(key => { + if (data[key] !== undefined) formData.append(key, data[key]); + }); + return postUpload(`/api/v1/knowledge-bases/${kbId}/knowledge/file`, formData, onProgress); } // 从URL创建知识 // data.tag_id: 可选，指定知识所属的分类ID export function createKnowledgeFromURL(kbId: string, data: { url: string; enable_multimodel?: boolean; tag_id?: string }) { - return post(`/api/v1/knowledge-bases/${kbId}/knowledge/url`, data); + return post(`/api/v1/knowledge-bases/${kbId}/knowledge/url`, data); } // 手工创建知识 // data.tag_id: 可选，指定知识所属的分类ID export function createManualKnowledge(kbId: string, data: { title: string; content: string; status: string; tag_id?: string }) { - return post(`/api/v1/knowledge-bases/${kbId}/knowledge/manual`, data); + return post(`/api/v1/knowledge-bases/${kbId}/knowledge/manual`, data); } export function listKnowledgeFiles( - kbId: string, - params: { page: number; page_size: number; tag_id?: string; keyword?: string; file_type?: string }, + kbId: string, + params: { page: number; page_size: number; tag_id?: string; keyword?: string; file_type?: string }, ) { - const query = new URLSearchParams(); - query.append('page', String(params.page)); - query.append('page_size', String(params.page_size)); - if (params.tag_id) { - query.append('tag_id', params.tag_id); - } - if (params.keyword) { - query.append('keyword', params.keyword); - } - if (params.file_type) { - query.append('file_type', params.file_type); - } - const qs = query.toString(); - return get(`/api/v1/knowledge-bases/${kbId}/knowledge?${qs}`); + const query = new URLSearchParams(); + query.append('page', String(params.page)); + query.append('page_size', String(params.page_size)); + if (params.tag_id) { + query.append('tag_id', params.tag_id); + } + if (params.keyword) { + query.append('keyword', params.keyword); + } + if (params.file_type) { + query.append('file_type', params.file_type); + } + const qs = query.toString(); + return get(`/api/v1/knowledge-bases/${kbId}/knowledge?${qs}`); } export function getKnowledgeDetails(id: string) { - return get(`/api/v1/knowledge/${id}`); + return get(`/api/v1/knowledge/${id}`); } export function updateManualKnowledge(id: string, data: { title: string; content: string; status: string }) { - return put(`/api/v1/knowledge/manual/${id}`, data); + return put(`/api/v1/knowledge/manual/${id}`, data); } export function delKnowledgeDetails(id: string) { - return del(`/api/v1/knowledge/${id}`); + return del(`/api/v1/knowledge/${id}`); } export function downKnowledgeDetails(id: string) { - return getDown(`/api/v1/knowledge/${id}/download`); + return getDown(`/api/v1/knowledge/${id}/download`); } export function batchQueryKnowledge(idsQueryString: string) { - return get(`/api/v1/knowledge/batch?${idsQueryString}`); + return get(`/api/v1/knowledge/batch?${idsQueryString}`); } export function getKnowledgeDetailsCon(id: string, page: number) { - return get(`/api/v1/chunks/${id}?page=${page}&page_size=25`); + return get(`/api/v1/chunks/${id}?page=${page}&page_size=25`); } // Get chunk by chunk_id only (new endpoint - to be added to backend) export function getChunkByIdOnly(chunkId: string) { - return get(`/api/v1/chunks/by-id/${chunkId}`); + return get(`/api/v1/chunks/by-id/${chunkId}`); } // Delete a single generated question from a chunk by question ID export function deleteGeneratedQuestion(chunkId: string, questionId: string) { - return del(`/api/v1/chunks/by-id/${chunkId}/questions`, { question_id: questionId }); + return del(`/api/v1/chunks/by-id/${chunkId}/questions`, { question_id: questionId }); } export function listKnowledgeTags( - kbId: string, - params?: { page?: number; page_size?: number; keyword?: string }, + kbId: string, + params?: { page?: number; page_size?: number; keyword?: string }, ) { - const query = buildQuery(params); - return get(`/api/v1/knowledge-bases/${kbId}/tags${query}`); + const query = buildQuery(params); + return get(`/api/v1/knowledge-bases/${kbId}/tags${query}`); } export function createKnowledgeBaseTag( - kbId: string, - data: { name: string; color?: string; sort_order?: number }, + kbId: string, + data: { name: string; color?: string; sort_order?: number }, ) { - return post(`/api/v1/knowledge-bases/${kbId}/tags`, data); + return post(`/api/v1/knowledge-bases/${kbId}/tags`, data); } export function updateKnowledgeBaseTag( - kbId: string, - tagId: string, - data: { name?: string; color?: string; sort_order?: number }, + kbId: string, + tagId: string, + data: { name?: string; color?: string; sort_order?: number }, ) { - return put(`/api/v1/knowledge-bases/${kbId}/tags/${tagId}`, data); + return put(`/api/v1/knowledge-bases/${kbId}/tags/${tagId}`, data); } export function deleteKnowledgeBaseTag(kbId: string, tagId: string, params?: { force?: boolean }) { - const forceQuery = params?.force ? '?force=true' : ''; - return del(`/api/v1/knowledge-bases/${kbId}/tags/${tagId}${forceQuery}`); + const forceQuery = params?.force ? '?force=true' : ''; + return del(`/api/v1/knowledge-bases/${kbId}/tags/${tagId}${forceQuery}`); } export function updateKnowledgeTagBatch(data: { updates: Record }) { - return put(`/api/v1/knowledge/tags`, data); + return put(`/api/v1/knowledge/tags`, data); } export function updateFAQEntryTagBatch(kbId: string, data: { updates: Record }) { - return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/tags`, data); + return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/tags`, data); } const buildQuery = (params?: Record) => { - if (!params) return ''; - const query = new URLSearchParams(); - Object.entries(params).forEach(([key, value]) => { - if (value === undefined || value === null || value === '') return; - query.append(key, String(value)); - }); - const queryString = query.toString(); - return queryString ? `?${queryString}` : ''; + if (!params) return ''; + const query = new URLSearchParams(); + Object.entries(params).forEach(([key, value]) => { + if (value === undefined || value === null || value === '') return; + query.append(key, String(value)); + }); + const queryString = query.toString(); + return queryString ? `?${queryString}` : ''; }; export function listFAQEntries( - kbId: string, - params?: { page?: number; page_size?: number; tag_id?: string; keyword?: string }, + kbId: string, + params?: { page?: number; page_size?: number; tag_id?: string; keyword?: string }, ) { - const query = buildQuery(params); - return get(`/api/v1/knowledge-bases/${kbId}/faq/entries${query}`); + const query = buildQuery(params); + return get(`/api/v1/knowledge-bases/${kbId}/faq/entries${query}`); } export function upsertFAQEntries(kbId: string, data: { entries: any[]; mode: 'append' | 'replace' }) { - return post(`/api/v1/knowledge-bases/${kbId}/faq/entries`, data); + return post(`/api/v1/knowledge-bases/${kbId}/faq/entries`, data); } export function createFAQEntry(kbId: string, data: any) { - return post(`/api/v1/knowledge-bases/${kbId}/faq/entry`, data); + return post(`/api/v1/knowledge-bases/${kbId}/faq/entry`, data); } export function updateFAQEntry(kbId: string, entryId: string, data: any) { - return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/${entryId}`, data); + return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/${entryId}`, data); } // Unified batch update API - supports is_enabled, is_recommended, tag_id @@ -187,64 +187,64 @@ export function updateFAQEntry(kbId: string, entryId: string, data: any) { // 1. By entry ID: use by_id field // 2. By Tag: use by_tag field to apply the same update to all entries under a tag export interface FAQEntryFieldsUpdate { - is_enabled?: boolean - is_recommended?: boolean - tag_id?: string | null + is_enabled?: boolean + is_recommended?: boolean + tag_id?: string | null } export interface FAQEntryFieldsBatchRequest { - by_id?: Record - by_tag?: Record + by_id?: Record + by_tag?: Record } export function updateFAQEntryFieldsBatch(kbId: string, data: FAQEntryFieldsBatchRequest) { - return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/fields`, data); + return put(`/api/v1/knowledge-bases/${kbId}/faq/entries/fields`, data); } export function deleteFAQEntries(kbId: string, ids: string[]) { - return del(`/api/v1/knowledge-bases/${kbId}/faq/entries`, { ids }); + return del(`/api/v1/knowledge-bases/${kbId}/faq/entries`, { ids }); } export function searchFAQEntries( - kbId: string, - data: { - query_text: string - vector_threshold?: number - match_count?: number - } + kbId: string, + data: { + query_text: string + vector_threshold?: number + match_count?: number + } ) { - return post(`/api/v1/knowledge-bases/${kbId}/faq/search`, data); + return post(`/api/v1/knowledge-bases/${kbId}/faq/search`, data); } // Export FAQ entries as CSV file export async function exportFAQEntries(kbId: string): Promise { - const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export`); - return response as unknown as Blob; + const response = await getDown(`/api/v1/knowledge-bases/${kbId}/faq/entries/export`); + return response as unknown as Blob; } // FAQ Import Progress API export interface FAQImportProgress { - task_id: string - kb_id: string - knowledge_id: string - status: 'pending' | 'processing' | 'completed' | 'failed' - progress: number - total: number - processed: number - message: string - error: string - created_at: number - updated_at: number + task_id: string + kb_id: string + knowledge_id: string + status: 'pending' | 'processing' | 'completed' | 'failed' + progress: number + total: number + processed: number + message: string + error: string + created_at: number + updated_at: number } export function getFAQImportProgress(taskId: string) { - return get(`/api/v1/faq/import/progress/${taskId}`); + return get(`/api/v1/faq/import/progress/${taskId}`); } export function searchKnowledge(keyword: string, offset = 0, limit = 20, fileTypes?: string[]) { - let url = `/api/v1/knowledge/search?keyword=${encodeURIComponent(keyword)}&offset=${offset}&limit=${limit}`; - if (fileTypes && fileTypes.length > 0) { - url += `&file_types=${encodeURIComponent(fileTypes.join(','))}`; - } - return get(url); + let url = `/api/v1/knowledge/search?keyword=${encodeURIComponent(keyword)}&offset=${offset}&limit=${limit}`; + if (fileTypes && fileTypes.length > 0) { + url += `&file_types=${encodeURIComponent(fileTypes.join(','))}`; + } + return get(url); } From 1abdaa5d5c1fe9ee90e643287208eefeff8a4845 Mon Sep 17 00:00:00 2001 From: begoniezhao Date: Thu, 15 Jan 2026 10:42:48 +0800 Subject: [PATCH 14/15] feat: Make OCR and task concurrency configurable --- .env.example | 7 ++++ docker-compose.dev.yml | 2 + docker-compose.yml | 2 + docreader/parser/base_parser.py | 66 ++++++++++++++++----------------- docreader/parser/parser.py | 7 +++- 5 files changed, 49 insertions(+), 35 deletions(-) diff --git a/.env.example b/.env.example index 855ee71f..32ea96da 100644 --- a/.env.example +++ b/.env.example @@ -82,6 +82,13 @@ JWT_SECRET=weknora-jwt-secret # Embedding并发数，出现429错误时，可调小此参数 CONCURRENCY_POOL_SIZE=5 +# Docreader 并发任务数（图片OCR/Caption等异步任务），默认 1 +# 默认使用的 paddleocr 在高并发场景下，会出现异常，请谨慎设置 +# IMAGE_MAX_CONCURRENT=1 + +# Docreader OCR后端(no_ocr, paddle, nanonets) +# OCR_BACKEND=paddle + # 如果使用ElasticSearch作为向量存储，需要配置以下参数 # ElasticSearch地址，例如 http://localhost:9200 # ELASTICSEARCH_ADDR=your_elasticsearch_addr diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index a4dc7902..7d166748 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -120,6 +120,8 @@ services: - WEB_PROXY=${WEB_PROXY:-} # File size limit (in MB) - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} + - IMAGE_MAX_CONCURRENT=${IMAGE_MAX_CONCURRENT:-1} + - OCR_BACKEND=${OCR_BACKEND:-paddle} healthcheck: test: ["CMD", "grpc_health_probe", "-addr=:50051"] interval: 30s diff --git a/docker-compose.yml b/docker-compose.yml index 00a6d93e..14b28fe5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -148,6 +148,8 @@ services: - MINERU_ENDPOINT=${MINERU_ENDPOINT:-} # File size limit (in MB) - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} + - IMAGE_MAX_CONCURRENT=${IMAGE_MAX_CONCURRENT:-1} + - OCR_BACKEND=${OCR_BACKEND:-paddle} healthcheck: test: ["CMD", "grpc_health_probe", "-addr=:50051"] interval: 30s diff --git a/docreader/parser/base_parser.py b/docreader/parser/base_parser.py index 4a7fb19a..781830da 100644 --- a/docreader/parser/base_parser.py +++ b/docreader/parser/base_parser.py @@ -35,34 +35,38 @@ class BaseParser(ABC): @staticmethod def _is_safe_url(url: str) -> bool: """Validate URL to prevent SSRF attacks - + Args: url: URL to validate - + Returns: True if URL is safe to fetch, False otherwise """ try: parsed_url = urlparse(url) - + # Only allow http and https schemes - if parsed_url.scheme not in ['http', 'https']: + if parsed_url.scheme not in ["http", "https"]: logger.warning(f"Rejected URL with invalid scheme: {parsed_url.scheme}") return False - + # Extract hostname hostname = parsed_url.hostname if not hostname: logger.warning("No hostname found in URL") return False - + # Try to parse as IP address try: ip = ipaddress.ip_address(hostname) # Reject private, loopback, link-local, multicast addresses - if (ip.is_private or ip.is_loopback or - ip.is_link_local or ip.is_multicast or - ip.is_reserved): + if ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + ): logger.warning(f"Rejected URL with restricted IP: {ip}") return False except ValueError: @@ -70,17 +74,19 @@ class BaseParser(ABC): # Reject localhost and common internal hostnames hostname_lower = hostname.lower() restricted_hostnames = [ - 'localhost', - '127.0.0.1', - '::1', - 'metadata.google.internal', - 'metadata.tencentyun.com', - '169.254.169.254', # AWS metadata endpoint + "localhost", + "127.0.0.1", + "::1", + "metadata.google.internal", + "metadata.tencentyun.com", + "169.254.169.254", # AWS metadata endpoint ] - if hostname_lower in restricted_hostnames or hostname_lower.endswith('.local'): + if hostname_lower in restricted_hostnames or hostname_lower.endswith( + ".local" + ): logger.warning(f"Rejected URL with restricted hostname: {hostname}") return False - + return True except Exception as e: logger.warning(f"Error validating URL: {str(e)}") @@ -121,7 +127,7 @@ class BaseParser(ABC): chunk_size: int = 1000, chunk_overlap: int = 200, separators: list[str] = ["\n\n", "\n", "。"], - ocr_backend: str = "paddle", + ocr_backend: str = "no_ocr", ocr_config: dict = {}, max_image_size: int = 1920, # Maximum image size max_concurrent_tasks: int = 5, # Max concurrent tasks @@ -151,7 +157,7 @@ class BaseParser(ABC): self.chunk_size = chunk_size self.chunk_overlap = chunk_overlap self.separators = separators - self.ocr_backend = os.getenv("OCR_BACKEND", ocr_backend) + self.ocr_backend = ocr_backend self.ocr_config = ocr_config self.max_image_size = max_image_size self.max_concurrent_tasks = max_concurrent_tasks @@ -161,13 +167,14 @@ class BaseParser(ABC): self.chunking_config.storage_config if self.chunking_config else None ) - logger.info(f"Initializing parser for file: {file_name}, type: {file_type}") logger.info( - f"Parser config: chunk_size={chunk_size}, " + f"Initializing parser for file={file_name}, type={file_type}, " + f"parser config: chunk_size={chunk_size}, " f"overlap={chunk_overlap}, " f"multimodal={enable_multimodal}, " + f"max_chunks={max_chunks}, " f"ocr_backend={ocr_backend}, " - f"max_chunks={max_chunks}" + f"max_concurrent_tasks={max_concurrent_tasks}" ) # Only initialize Caption service if multimodal is enabled vlm_config = self.chunking_config.vlm_config if self.chunking_config else None @@ -305,13 +312,8 @@ class BaseParser(ABC): logger.warning("No image data to process") return [] - # Set max concurrency, reduce concurrency to avoid resource contention - max_concurrency = min( - self.max_concurrent_tasks, 1 - ) # Reduce concurrency to prevent excessive memory usage - # Use semaphore to limit concurrency - semaphore = asyncio.Semaphore(max_concurrency) + semaphore = asyncio.Semaphore(self.max_concurrent_tasks) # Store results to avoid overall failure due to task failure results = [] @@ -741,7 +743,7 @@ class BaseParser(ABC): if not self._is_safe_url(img_url): logger.error(f"URL failed validation check: {img_url}") return img_url, img_url, None - + # Still need to get image object for OCR processing # Get proxy settings from environment variables http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") @@ -792,7 +794,7 @@ class BaseParser(ABC): if not self._is_safe_url(img_url): logger.error(f"URL failed validation check: {img_url}") return img_url, img_url, None - + # Get proxy settings from environment variables http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY") @@ -952,10 +954,8 @@ class BaseParser(ABC): # Create and run all Chunk concurrent processing tasks async def process_all_chunks(): - # Set max concurrency, reduce concurrency to avoid resource contention - max_concurrency = min(self.max_concurrent_tasks, 1) # Reduce concurrency # Use semaphore to limit concurrency - semaphore = asyncio.Semaphore(max_concurrency) + semaphore = asyncio.Semaphore(self.max_concurrent_tasks) async def process_with_limit(chunk, idx, total): """Use semaphore to control concurrent processing of Chunks""" diff --git a/docreader/parser/parser.py b/docreader/parser/parser.py index 2cb0da22..20436485 100644 --- a/docreader/parser/parser.py +++ b/docreader/parser/parser.py @@ -1,4 +1,5 @@ import logging +import os from typing import Dict, Type from docreader.models.document import Document @@ -109,8 +110,9 @@ class Parser: separators=config.separators, # Text separators for chunking enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing max_image_size=1920, # Limit image size to 1920px for performance - max_concurrent_tasks=5, # Limit concurrent tasks to 5 to avoid resource exhaustion chunking_config=config, # Pass the entire chunking config for advanced options + max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")), + ocr_backend=os.getenv("OCR_BACKEND", "paddle"), ) logger.info(f"Starting to parse file content, size: {len(content)} bytes") @@ -155,8 +157,9 @@ class Parser: separators=config.separators, # Text separators for chunking enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing max_image_size=1920, # Limit image size to 1920px for performance - max_concurrent_tasks=5, # Limit concurrent tasks to avoid resource exhaustion chunking_config=config, # Pass the entire chunking config + max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")), + ocr_backend=os.getenv("OCR_BACKEND", "paddle"), ) logger.info("Starting to parse URL content") From b648bd472cf0edffd990856a81c84d6e3f72f29f Mon Sep 17 00:00:00 2001 From: Xia Dong Date: Thu, 15 Jan 2026 10:48:26 +0800 Subject: [PATCH 15/15] feat(knowledge): Enhance delete functionality with success callback - Updated the delKnowledge function to accept an optional onSuccess callback for post-deletion actions. - Modified the delCardConfirm function to refresh the document list and tag counts upon successful deletion. --- frontend/src/hooks/useKnowledgeBase.ts | 13 ++++++++++--- frontend/src/views/knowledge/KnowledgeBase.vue | 6 +++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/frontend/src/hooks/useKnowledgeBase.ts b/frontend/src/hooks/useKnowledgeBase.ts index a1b4de2f..325b5859 100644 --- a/frontend/src/hooks/useKnowledgeBase.ts +++ b/frontend/src/hooks/useKnowledgeBase.ts @@ -63,20 +63,27 @@ export default function (knowledgeBaseId?: string) { }) .catch(() => {}); }; - const delKnowledge = (index: number, item: any) => { + const delKnowledge = (index: number, item: any, onSuccess?: () => void) => { cardList.value[index].isMore = false; moreIndex.value = -1; - delKnowledgeDetails(item.id) + return delKnowledgeDetails(item.id) .then((result: any) => { if (result.success) { MessagePlugin.info("知识删除成功！"); - getKnowled(); + if (onSuccess) { + onSuccess(); + } else { + getKnowled(); + } + return true; } else { MessagePlugin.error("知识删除失败！"); + return false; } }) .catch(() => { MessagePlugin.error("知识删除失败！"); + return false; }); }; const openMore = (index: number) => { diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index f37846d4..1c01745f 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -866,7 +866,11 @@ const getDoc = (page: number) => { const delCardConfirm = () => { delDialog.value = false; - delKnowledge(knowledgeIndex.value, knowledge.value); + delKnowledge(knowledgeIndex.value, knowledge.value, () => { + // 删除成功后刷新文档列表和分类数量 + loadKnowledgeFiles(kbId.value); + loadTags(kbId.value); + }); }; // 处理知识库编辑成功后的回调