diff --git a/.env.example b/.env.example index 855ee71f..32ea96da 100644 --- a/.env.example +++ b/.env.example @@ -82,6 +82,13 @@ JWT_SECRET=weknora-jwt-secret # Embedding并发数，出现429错误时，可调小此参数 CONCURRENCY_POOL_SIZE=5 +# Docreader 并发任务数（图片OCR/Caption等异步任务），默认 1 +# 默认使用的 paddleocr 在高并发场景下，会出现异常，请谨慎设置 +# IMAGE_MAX_CONCURRENT=1 + +# Docreader OCR后端(no_ocr, paddle, nanonets) +# OCR_BACKEND=paddle + # 如果使用ElasticSearch作为向量存储，需要配置以下参数 # ElasticSearch地址，例如 http://localhost:9200 # ELASTICSEARCH_ADDR=your_elasticsearch_addr diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index a4dc7902..7d166748 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -120,6 +120,8 @@ services: - WEB_PROXY=${WEB_PROXY:-} # File size limit (in MB) - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} + - IMAGE_MAX_CONCURRENT=${IMAGE_MAX_CONCURRENT:-1} + - OCR_BACKEND=${OCR_BACKEND:-paddle} healthcheck: test: ["CMD", "grpc_health_probe", "-addr=:50051"] interval: 30s diff --git a/docker-compose.yml b/docker-compose.yml index 00a6d93e..14b28fe5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -148,6 +148,8 @@ services: - MINERU_ENDPOINT=${MINERU_ENDPOINT:-} # File size limit (in MB) - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} + - IMAGE_MAX_CONCURRENT=${IMAGE_MAX_CONCURRENT:-1} + - OCR_BACKEND=${OCR_BACKEND:-paddle} healthcheck: test: ["CMD", "grpc_health_probe", "-addr=:50051"] interval: 30s diff --git a/docreader/parser/base_parser.py b/docreader/parser/base_parser.py index 4a7fb19a..f93778d6 100644 --- a/docreader/parser/base_parser.py +++ b/docreader/parser/base_parser.py @@ -35,34 +35,37 @@ class BaseParser(ABC): @staticmethod def _is_safe_url(url: str) -> bool: """Validate URL to prevent SSRF attacks - + Args: url: URL to validate - + Returns: True if URL is safe to fetch, False otherwise """ try: parsed_url = urlparse(url) - + # Only allow http and https schemes - if parsed_url.scheme not in ['http', 'https']: + if parsed_url.scheme not in ["http", "https"]: logger.warning(f"Rejected URL with invalid scheme: {parsed_url.scheme}") return False - + # Extract hostname hostname = parsed_url.hostname if not hostname: logger.warning("No hostname found in URL") return False - # Try to parse as IP address try: ip = ipaddress.ip_address(hostname) # Reject private, loopback, link-local, multicast addresses - if (ip.is_private or ip.is_loopback or - ip.is_link_local or ip.is_multicast or - ip.is_reserved): + if ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + ): logger.warning(f"Rejected URL with restricted IP: {ip}") return False except ValueError: @@ -70,17 +73,19 @@ class BaseParser(ABC): # Reject localhost and common internal hostnames hostname_lower = hostname.lower() restricted_hostnames = [ - 'localhost', - '127.0.0.1', - '::1', - 'metadata.google.internal', - 'metadata.tencentyun.com', - '169.254.169.254', # AWS metadata endpoint + "localhost", + "127.0.0.1", + "::1", + "metadata.google.internal", + "metadata.tencentyun.com", + "169.254.169.254", # AWS metadata endpoint ] - if hostname_lower in restricted_hostnames or hostname_lower.endswith('.local'): + if hostname_lower in restricted_hostnames or hostname_lower.endswith( + ".local" + ): logger.warning(f"Rejected URL with restricted hostname: {hostname}") return False - + return True except Exception as e: logger.warning(f"Error validating URL: {str(e)}") @@ -121,7 +126,7 @@ class BaseParser(ABC): chunk_size: int = 1000, chunk_overlap: int = 200, separators: list[str] = ["\n\n", "\n", "。"], - ocr_backend: str = "paddle", + ocr_backend: str = "no_ocr", ocr_config: dict = {}, max_image_size: int = 1920, # Maximum image size max_concurrent_tasks: int = 5, # Max concurrent tasks @@ -151,7 +156,7 @@ class BaseParser(ABC): self.chunk_size = chunk_size self.chunk_overlap = chunk_overlap self.separators = separators - self.ocr_backend = os.getenv("OCR_BACKEND", ocr_backend) + self.ocr_backend = ocr_backend self.ocr_config = ocr_config self.max_image_size = max_image_size self.max_concurrent_tasks = max_concurrent_tasks @@ -161,13 +166,14 @@ class BaseParser(ABC): self.chunking_config.storage_config if self.chunking_config else None ) - logger.info(f"Initializing parser for file: {file_name}, type: {file_type}") logger.info( - f"Parser config: chunk_size={chunk_size}, " + f"Initializing parser for file={file_name}, type={file_type}, " + f"parser config: chunk_size={chunk_size}, " f"overlap={chunk_overlap}, " f"multimodal={enable_multimodal}, " + f"max_chunks={max_chunks}, " f"ocr_backend={ocr_backend}, " - f"max_chunks={max_chunks}" + f"max_concurrent_tasks={max_concurrent_tasks}" ) # Only initialize Caption service if multimodal is enabled vlm_config = self.chunking_config.vlm_config if self.chunking_config else None @@ -305,13 +311,8 @@ class BaseParser(ABC): logger.warning("No image data to process") return [] - # Set max concurrency, reduce concurrency to avoid resource contention - max_concurrency = min( - self.max_concurrent_tasks, 1 - ) # Reduce concurrency to prevent excessive memory usage - # Use semaphore to limit concurrency - semaphore = asyncio.Semaphore(max_concurrency) + semaphore = asyncio.Semaphore(self.max_concurrent_tasks) # Store results to avoid overall failure due to task failure results = [] @@ -741,7 +742,6 @@ class BaseParser(ABC): if not self._is_safe_url(img_url): logger.error(f"URL failed validation check: {img_url}") return img_url, img_url, None - # Still need to get image object for OCR processing # Get proxy settings from environment variables http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") @@ -792,7 +792,6 @@ class BaseParser(ABC): if not self._is_safe_url(img_url): logger.error(f"URL failed validation check: {img_url}") return img_url, img_url, None - # Get proxy settings from environment variables http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY") @@ -952,10 +951,8 @@ class BaseParser(ABC): # Create and run all Chunk concurrent processing tasks async def process_all_chunks(): - # Set max concurrency, reduce concurrency to avoid resource contention - max_concurrency = min(self.max_concurrent_tasks, 1) # Reduce concurrency # Use semaphore to limit concurrency - semaphore = asyncio.Semaphore(max_concurrency) + semaphore = asyncio.Semaphore(self.max_concurrent_tasks) async def process_with_limit(chunk, idx, total): """Use semaphore to control concurrent processing of Chunks""" diff --git a/docreader/parser/parser.py b/docreader/parser/parser.py index 2cb0da22..20436485 100644 --- a/docreader/parser/parser.py +++ b/docreader/parser/parser.py @@ -1,4 +1,5 @@ import logging +import os from typing import Dict, Type from docreader.models.document import Document @@ -109,8 +110,9 @@ class Parser: separators=config.separators, # Text separators for chunking enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing max_image_size=1920, # Limit image size to 1920px for performance - max_concurrent_tasks=5, # Limit concurrent tasks to 5 to avoid resource exhaustion chunking_config=config, # Pass the entire chunking config for advanced options + max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")), + ocr_backend=os.getenv("OCR_BACKEND", "paddle"), ) logger.info(f"Starting to parse file content, size: {len(content)} bytes") @@ -155,8 +157,9 @@ class Parser: separators=config.separators, # Text separators for chunking enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing max_image_size=1920, # Limit image size to 1920px for performance - max_concurrent_tasks=5, # Limit concurrent tasks to avoid resource exhaustion chunking_config=config, # Pass the entire chunking config + max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")), + ocr_backend=os.getenv("OCR_BACKEND", "paddle"), ) logger.info("Starting to parse URL content") diff --git a/docreader/splitter/splitter.py b/docreader/splitter/splitter.py index 4454b3fd..cac6ea2a 100644 --- a/docreader/splitter/splitter.py +++ b/docreader/splitter/splitter.py @@ -75,9 +75,9 @@ class TextSplitter(BaseModel, Generic[T]): # link - Markdown link syntax [text](url) r"\[.*?\]\(.*?\)", # table header - Markdown table header with separator line - r"(?:\|[^|\n]*)+\|[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|[\r\n]+", + r"[ ]*(?:\|[^|\n]*)+\|[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|[\r\n]+", # table body - Markdown table rows - r"(?:\|[^|\n]*)+\|[\r\n]+", + r"[ ]*(?:\|[^|\n]*)+\|[\r\n]+", # code header - Code block start with language identifier r"```(?:\w+)[\r\n]+[^\r\n]*", ], @@ -137,6 +137,10 @@ class TextSplitter(BaseModel, Generic[T]): # Step 4: Merge splits into final chunks with overlap chunks = self._merge(splits) + + # Step 5: Validate chunks and test restoration + # self._validate_chunks(chunks, text) + return chunks def _split(self, text: str) -> List[str]: @@ -250,10 +254,17 @@ class TextSplitter(BaseModel, Generic[T]): cur_len > self.chunk_overlap or cur_len + split_len + cur_headers_len > self.chunk_size ): - # Remove the first element to reduce overlap + # Remove the first element to reduce overlap. + # If the first element is a prepended header (start==end), also remove it. first_chunk = cur_chunk.pop(0) cur_len -= self.len_function(first_chunk[2]) + # If we just popped a real content piece, there may be a header right after it + # (depending on previous iterations). Pop it only if it is actually a header. + if cur_chunk and first_chunk[0] == first_chunk[1]: + first_chunk = cur_chunk.pop(0) + cur_len -= self.len_function(first_chunk[2]) + # Prepend headers to new chunk if: # 1. Headers exist # 2. Headers + split fit in chunk size @@ -264,10 +275,8 @@ class TextSplitter(BaseModel, Generic[T]): and cur_headers not in split ): next_start = cur_chunk[0][0] if cur_chunk else cur_start - header_start = max(0, next_start - cur_headers_len) - header_end = cur_end - cur_chunk.insert(0, (header_start, header_end, cur_headers)) + cur_chunk.insert(0, (next_start, next_start, cur_headers)) cur_len += cur_headers_len # Add current split to the chunk @@ -397,6 +406,154 @@ class TextSplitter(BaseModel, Generic[T]): start = end return res + def _validate_chunks( + self, chunks: List[Tuple[int, int, str]], original_text: str + ) -> None: + """Validate chunks order and test text restoration. + + This method performs two validations: + 1. Checks if chunk start positions are in ascending order + 2. Tests if the original text can be restored from chunks + + If validation fails, saves debug information to /tmp/chunk_error_.md + + Args: + chunks: List of tuples (start_pos, end_pos, chunk_text) to validate + original_text: The original text that was split + """ + import datetime + + errors = [] + + # Validation 1: Check if start positions are in ascending order + for i in range(1, len(chunks)): + prev_start = chunks[i - 1][0] + curr_start = chunks[i][0] + if curr_start < prev_start: + error_msg = ( + f"Chunk order error: chunk[{i}] start position ({curr_start}) " + f"is less than chunk[{i - 1}] start position ({prev_start})" + ) + errors.append(error_msg) + logger.error(error_msg) + + # Validation 2: Test text restoration + try: + restored_text = self.restore_text(chunks) + if restored_text != original_text: + error_msg = ( + f"Restoration failed: restored text differs from original. " + f"Original length: {len(original_text)}, " + f"Restored length: {len(restored_text)}" + ) + errors.append(error_msg) + logger.error(error_msg) + + # Find first difference position + min_len = min(len(original_text), len(restored_text)) + diff_pos = -1 + for i in range(min_len): + if original_text[i] != restored_text[i]: + diff_pos = i + break + + if diff_pos >= 0: + context_start = max(0, diff_pos - 50) + context_end = min(len(original_text), diff_pos + 50) + errors.append( + f"First difference at position {diff_pos}:\n" + f"Original: {repr(original_text[context_start:context_end])}\n" + f"Restored: {repr(restored_text[context_start:context_end])}" + ) + elif len(original_text) != len(restored_text): + errors.append( + f"Texts match up to position {min_len}, but lengths differ" + ) + except Exception as e: + error_msg = f"Restoration exception: {str(e)}" + errors.append(error_msg) + logger.error(error_msg) + + # If there are errors, save debug information to file + if errors: + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + error_file = f"/tmp/chunk_error_{timestamp}.md" + + with open(error_file, "w", encoding="utf-8") as f: + f.write("# Chunk Validation Error Report\n\n") + f.write(f"Timestamp: {timestamp}\n\n") + + f.write("## Errors\n\n") + for error in errors: + f.write(f"- {error}\n\n") + + f.write("\n## Original Text\n\n") + f.write(f"Length: {len(original_text)}\n\n") + f.write("```\n") + f.write(original_text) + f.write("\n```\n\n") + + f.write("\n## Chunks Information\n\n") + f.write(f"Total chunks: {len(chunks)}\n\n") + for i, (start, end, chunk_text) in enumerate(chunks): + f.write(f"### Chunk {i}\n\n") + f.write(f"- Position: [{start}:{end}]\n") + f.write(f"- Length: {len(chunk_text)}\n") + f.write(f"- Content:\n\n```\n{chunk_text}\n```\n\n") + + try: + restored_text = self.restore_text(chunks) + f.write("\n## Restored Text\n\n") + f.write(f"Length: {len(restored_text)}\n\n") + f.write("```\n") + f.write(restored_text) + f.write("\n```\n") + except Exception as e: + f.write("\n## Restoration Failed\n\n") + f.write(f"Error: {str(e)}\n") + + logger.error(f"Validation errors saved to: {error_file}") + + def restore_text(self, chunks: List[Tuple[int, int, str]]) -> str: + """Restore original text from chunks with overlap handling. + + This method reconstructs the original text from chunks that may contain: + - Overlapping content between consecutive chunks + - Prepended headers that were added during merging (headers have start==end position) + + The algorithm: + 1. Sort chunks by their start position (and end position as tiebreaker) + 2. Track the maximum end position seen so far + 3. For each chunk, extract only the new content (after max_end_pos) + 4. Concatenate all new content pieces + + Args: + chunks: List of tuples (start_pos, end_pos, chunk_text) from split_text() + + Returns: + The restored original text + + Example: + >>> splitter = TextSplitter(chunk_size=10, chunk_overlap=3) + >>> chunks = splitter.split_text("Hello World!") + >>> restored = splitter.restore_text(chunks) + >>> assert restored == "Hello World!" + """ + if not chunks: + return "" + + # Sort chunks by start position, then by end position + sorted_chunks = sorted(chunks, key=lambda x: (x[1], x[0])) + + result_parts = [] + last_end = 0 + + for start_pos, end_pos, chunk_text in sorted_chunks: + result_parts.append(chunk_text[last_end - end_pos :]) + last_end = end_pos + + return "".join(result_parts) + if __name__ == "__main__": s = """ @@ -415,7 +572,11 @@ if __name__ == "__main__": """ - sp = TextSplitter(chunk_size=200, chunk_overlap=2) + sp = TextSplitter( + chunk_size=200, + chunk_overlap=10, + separators=["\n\n", "\n", "。", "？", "！", "，", "；", "："], + ) ck = sp.split_text(s) for c in ck: print("------", len(c)) diff --git a/frontend/src/api/knowledge-base/index.ts b/frontend/src/api/knowledge-base/index.ts index 67ead452..614c9631 100644 --- a/frontend/src/api/knowledge-base/index.ts +++ b/frontend/src/api/knowledge-base/index.ts @@ -40,15 +40,24 @@ export function copyKnowledgeBase(data: { source_id: string; target_id?: string } // 知识文件 API（基于具体知识库） -export function uploadKnowledgeFile(kbId: string, data = {}, onProgress?: (progressEvent: any) => void) { - return postUpload(`/api/v1/knowledge-bases/${kbId}/knowledge/file`, data, onProgress); +// data.tag_id: 可选，指定知识所属的分类ID +export function uploadKnowledgeFile(kbId: string, data: { file: File; tag_id?: string; [key: string]: any } = { file: new File([], '') }, onProgress?: (progressEvent: any) => void) { + const formData = new FormData(); + Object.keys(data).forEach(key => { + if (data[key] !== undefined) formData.append(key, data[key]); + }); + return postUpload(`/api/v1/knowledge-bases/${kbId}/knowledge/file`, formData, onProgress); } -export function createKnowledgeFromURL(kbId: string, data: { url: string; enable_multimodel?: boolean }) { +// 从URL创建知识 +// data.tag_id: 可选，指定知识所属的分类ID +export function createKnowledgeFromURL(kbId: string, data: { url: string; enable_multimodel?: boolean; tag_id?: string }) { return post(`/api/v1/knowledge-bases/${kbId}/knowledge/url`, data); } -export function createManualKnowledge(kbId: string, data: { title: string; content: string; status: string }) { +// 手工创建知识 +// data.tag_id: 可选，指定知识所属的分类ID +export function createManualKnowledge(kbId: string, data: { title: string; content: string; status: string; tag_id?: string }) { return post(`/api/v1/knowledge-bases/${kbId}/knowledge/manual`, data); } diff --git a/frontend/src/components/manual-knowledge-editor.vue b/frontend/src/components/manual-knowledge-editor.vue index 6a22a827..39a0b724 100644 --- a/frontend/src/components/manual-knowledge-editor.vue +++ b/frontend/src/components/manual-knowledge-editor.vue @@ -579,7 +579,7 @@ const handleSave = async (targetStatus: ManualStatus) => { saving.value = true savingAction.value = targetStatus try { - const payload = { + const payload: { title: string; content: string; status: string; tag_id?: string } = { title: form.title.trim(), content: form.content, status: targetStatus, @@ -591,6 +591,11 @@ const handleSave = async (targetStatus: ManualStatus) => { if (mode.value === 'edit' && knowledgeId.value) { response = await updateManualKnowledge(knowledgeId.value, payload) } else { + // 创建新知识时，从 store 获取当前选中的分类ID + const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined + if (tagIdToUpload) { + payload.tag_id = tagIdToUpload + } response = await createManualKnowledge(form.kbId, payload) knowledgeID = response?.data?.id || knowledgeID kbId = form.kbId diff --git a/frontend/src/components/menu.vue b/frontend/src/components/menu.vue index d7566329..46cc72e0 100644 --- a/frontend/src/components/menu.vue +++ b/frontend/src/components/menu.vue @@ -823,9 +823,11 @@ const handleDocFileChange = async (event: Event) => { })) try { + // 获取当前选中的分类ID + const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined await uploadKnowledgeFile( kbId, - { file }, + { file, tag_id: tagIdToUpload }, (progressEvent: any) => { if (progressEvent.total) { progress = Math.round((progressEvent.loaded * 100) / progressEvent.total) @@ -1067,9 +1069,11 @@ const handleDocFolderChange = async (event: Event) => { })) try { + // 获取当前选中的分类ID + const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined await uploadKnowledgeFile( kbId, - { file, fileName }, + { file, fileName, tag_id: tagIdToUpload }, (progressEvent: any) => { if (progressEvent?.total) { progress = Math.round((progressEvent.loaded * 100) / progressEvent.total) diff --git a/frontend/src/hooks/useKnowledgeBase.ts b/frontend/src/hooks/useKnowledgeBase.ts index 19d63629..325b5859 100644 --- a/frontend/src/hooks/useKnowledgeBase.ts +++ b/frontend/src/hooks/useKnowledgeBase.ts @@ -10,6 +10,7 @@ import { getKnowledgeDetailsCon, } from "@/api/knowledge-base/index"; import { knowledgeStore } from "@/stores/knowledge"; +import { useUIStore } from "@/stores/ui"; import { useRoute } from 'vue-router'; const usemenuStore = knowledgeStore(); @@ -62,20 +63,27 @@ export default function (knowledgeBaseId?: string) { }) .catch(() => {}); }; - const delKnowledge = (index: number, item: any) => { + const delKnowledge = (index: number, item: any, onSuccess?: () => void) => { cardList.value[index].isMore = false; moreIndex.value = -1; - delKnowledgeDetails(item.id) + return delKnowledgeDetails(item.id) .then((result: any) => { if (result.success) { MessagePlugin.info("知识删除成功！"); - getKnowled(); + if (onSuccess) { + onSuccess(); + } else { + getKnowled(); + } + return true; } else { MessagePlugin.error("知识删除失败！"); + return false; } }) .catch(() => { MessagePlugin.error("知识删除失败！"); + return false; }); }; const openMore = (index: number) => { @@ -110,7 +118,11 @@ export default function (knowledgeBaseId?: string) { return; } - uploadKnowledgeFile(currentKbId, { file }) + // 获取当前选中的分类ID + const uiStore = useUIStore(); + const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined; + + uploadKnowledgeFile(currentKbId, { file, tag_id: tagIdToUpload }) .then((result: any) => { if (result.success) { MessagePlugin.info("上传成功！"); diff --git a/frontend/src/stores/ui.ts b/frontend/src/stores/ui.ts index 0b79bbaf..20f17bff 100644 --- a/frontend/src/stores/ui.ts +++ b/frontend/src/stores/ui.ts @@ -7,6 +7,8 @@ export const useUIStore = defineStore('ui', { kbEditorMode: 'create' as 'create' | 'edit', currentKBId: null as string | null, kbEditorType: 'document' as 'document' | 'faq', + // 当前选中的分类ID，用于文件上传时传递 + selectedTagId: '__untagged__' as string, kbEditorInitialSection: null as string | null, settingsInitialSection: null as string | null, settingsInitialSubSection: null as string | null, @@ -101,6 +103,11 @@ export const useUIStore = defineStore('ui', { } } this.manualEditorOnSuccess = null + }, + + // 设置当前选中的分类ID + setSelectedTagId(tagId: string) { + this.selectedTagId = tagId } } }) diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index 7e63c879..7eb8bbfa 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -212,6 +212,8 @@ const loadTags = async (kbIdValue: string, reset = false) => { const handleTagFilterChange = (value: string) => { selectedTagId.value = value; + // 同步更新 store 中的 selectedTagId，供 menu.vue 上传时使用 + uiStore.setSelectedTagId(value); page = 1; loadKnowledgeFiles(kbId.value); }; @@ -635,9 +637,12 @@ const handleDocumentUpload = async (event: Event) => { let failCount = 0; const totalCount = validFiles.length; + // 获取当前选中的分类ID（如果不是"未分类"则传递） + const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined; + for (const file of validFiles) { try { - const responseData: any = await uploadKnowledgeFile(kbId.value, { file }); + const responseData: any = await uploadKnowledgeFile(kbId.value, { file, tag_id: tagIdToUpload }); const isSuccess = responseData?.success || responseData?.code === 200 || responseData?.status === 'success' || (!responseData?.error && responseData); if (isSuccess) { successCount++; @@ -741,7 +746,9 @@ const handleURLImportConfirm = async () => { urlImporting.value = true; try { - const responseData: any = await createKnowledgeFromURL(kbId.value, { url }); + // 获取当前选中的分类ID + const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined; + const responseData: any = await createKnowledgeFromURL(kbId.value, { url, tag_id: tagIdToUpload }); window.dispatchEvent(new CustomEvent('knowledgeFileUploaded', { detail: { kbId: kbId.value } })); @@ -837,7 +844,11 @@ const getDoc = (page: number) => { const delCardConfirm = () => { delDialog.value = false; - delKnowledge(knowledgeIndex.value, knowledge.value); + delKnowledge(knowledgeIndex.value, knowledge.value, () => { + // 删除成功后刷新文档列表和分类数量 + loadKnowledgeFiles(kbId.value); + loadTags(kbId.value); + }); }; // 处理知识库编辑成功后的回调 diff --git a/internal/application/service/chat_pipline/merge.go b/internal/application/service/chat_pipline/merge.go index 0ed92be4..68d1b77c 100644 --- a/internal/application/service/chat_pipline/merge.go +++ b/internal/application/service/chat_pipline/merge.go @@ -101,8 +101,9 @@ func (p *PluginMerge) OnEvent(ctx context.Context, } // Merge overlapping chunks if chunks[i].EndAt > lastChunk.EndAt { - lastChunk.Content = lastChunk.Content + - string([]rune(chunks[i].Content)[lastChunk.EndAt-chunks[i].StartAt:]) + content := string([]rune(chunks[i].Content)) + offset := len(content) - (chunks[i].EndAt - lastChunk.EndAt) + lastChunk.Content = lastChunk.Content + content[offset:] lastChunk.EndAt = chunks[i].EndAt lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID) diff --git a/internal/application/service/knowledge.go b/internal/application/service/knowledge.go index 4794bb32..adfe98a1 100644 --- a/internal/application/service/knowledge.go +++ b/internal/application/service/knowledge.go @@ -142,7 +142,7 @@ func (s *knowledgeService) isKnowledgeDeleting(ctx context.Context, tenantID uin // CreateKnowledgeFromFile creates a knowledge entry from an uploaded file func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, - kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string, + kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string, tagID string, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating knowledge from file") @@ -261,6 +261,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, knowledge := &types.Knowledge{ TenantID: tenantID, KnowledgeBaseID: kbID, + TagID: tagID, // 设置分类ID，用于知识分类管理 Type: "file", Title: safeFilename, FileName: safeFilename, @@ -358,8 +359,9 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context, } // CreateKnowledgeFromURL creates a knowledge entry from a URL source +// tagID is optional - when provided, the knowledge will be assigned to the specified tag/category. func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context, - kbID string, url string, enableMultimodel *bool, title string, + kbID string, url string, enableMultimodel *bool, title string, tagID string, ) (*types.Knowledge, error) { logger.Info(ctx, "Start creating knowledge from URL") logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, url) @@ -426,6 +428,7 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context, CreatedAt: time.Now(), UpdatedAt: time.Now(), EmbeddingModelID: kb.EmbeddingModelID, + TagID: tagID, // 设置分类ID，用于知识分类管理 } // Save knowledge record @@ -557,6 +560,7 @@ func (s *knowledgeService) CreateKnowledgeFromManual(ctx context.Context, EmbeddingModelID: kb.EmbeddingModelID, FileName: fileName, FileType: types.KnowledgeTypeManual, + TagID: payload.TagID, // 设置分类ID，用于知识分类管理 } if err := knowledge.SetManualMetadata(meta); err != nil { logger.Errorf(ctx, "Failed to set manual metadata: %v", err) diff --git a/internal/handler/knowledge.go b/internal/handler/knowledge.go index c46f1606..734e3c90 100644 --- a/internal/handler/knowledge.go +++ b/internal/handler/knowledge.go @@ -164,8 +164,15 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) { enableMultimodel = &parseBool } + // 获取分类ID（如果提供），用于知识分类管理 + tagID := c.PostForm("tag_id") + // 过滤特殊值，空字符串或 "__untagged__" 表示未分类 + if tagID == "__untagged__" || tagID == "" { + tagID = "" + } + // Create knowledge entry from the file - knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName) + knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName, tagID) // Check for duplicate knowledge error if err != nil { if h.handleDuplicateKnowledgeError(c, err, knowledge, "file") { @@ -199,7 +206,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) { // @Accept json // @Produce json // @Param id path string true "知识库ID" -// @Param request body object{url=string,enable_multimodel=bool,title=string} true "URL请求" +// @Param request body object{url=string,enable_multimodel=bool,title=string,tag_id=string} true "URL请求" // @Success 201 {object} map[string]interface{} "创建的知识" // @Failure 400 {object} errors.AppError "请求参数错误" // @Failure 409 {object} map[string]interface{} "URL重复" @@ -222,6 +229,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) { URL string `json:"url" binding:"required"` EnableMultimodel *bool `json:"enable_multimodel"` Title string `json:"title"` + TagID string `json:"tag_id"` } if err := c.ShouldBindJSON(&req); err != nil { logger.Error(ctx, "Failed to parse URL request", err) @@ -238,7 +246,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) { ) // Create knowledge entry from the URL - knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.EnableMultimodel, req.Title) + knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.EnableMultimodel, req.Title, req.TagID) // Check for duplicate knowledge error if err != nil { if h.handleDuplicateKnowledgeError(c, err, knowledge, "url") { diff --git a/internal/types/interfaces/knowledge.go b/internal/types/interfaces/knowledge.go index 9bffa696..e9f06f25 100644 --- a/internal/types/interfaces/knowledge.go +++ b/internal/types/interfaces/knowledge.go @@ -12,6 +12,7 @@ import ( // KnowledgeService defines the interface for knowledge services. type KnowledgeService interface { // CreateKnowledgeFromFile creates knowledge from a file. + // tagID is optional - when provided, the file will be assigned to the specified tag/category. CreateKnowledgeFromFile( ctx context.Context, kbID string, @@ -19,14 +20,17 @@ type KnowledgeService interface { metadata map[string]string, enableMultimodel *bool, customFileName string, + tagID string, ) (*types.Knowledge, error) // CreateKnowledgeFromURL creates knowledge from a URL. + // tagID is optional - when provided, the knowledge will be assigned to the specified tag/category. CreateKnowledgeFromURL( ctx context.Context, kbID string, url string, enableMultimodel *bool, title string, + tagID string, ) (*types.Knowledge, error) // CreateKnowledgeFromPassage creates knowledge from text passages. CreateKnowledgeFromPassage(ctx context.Context, kbID string, passage []string) (*types.Knowledge, error) diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go index 30e86773..50ef1ccb 100644 --- a/internal/types/knowledge.go +++ b/internal/types/knowledge.go @@ -144,6 +144,7 @@ type ManualKnowledgePayload struct { Title string `json:"title"` Content string `json:"content"` Status string `json:"status"` + TagID string `json:"tag_id"` } // NewManualKnowledgeMetadata creates a new ManualKnowledgeMetadata instance.