Weknora/Enterprise-Knowledge-Base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'github_public/main'

Browse Source
This commit is contained in:
wizardchen
2026-01-15 14:39:35 +08:00
parent d530a02cfe b648bd472c
commit 48a0f9b508
17 changed files with 301 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+7
View File
@@ -82,6 +82,13 @@ JWT_SECRET=weknora-jwt-secret
# Embedding并发数，出现429错误时，可调小此参数
CONCURRENCY_POOL_SIZE=5
# Docreader 并发任务数（图片OCR/Caption等异步任务），默认 1
# 默认使用的 paddleocr 在高并发场景下，会出现异常，请谨慎设置
# IMAGE_MAX_CONCURRENT=1
# Docreader OCR后端(no_ocr, paddle, nanonets)
# OCR_BACKEND=paddle
# 如果使用ElasticSearch作为向量存储，需要配置以下参数
# ElasticSearch地址，例如 http://localhost:9200
# ELASTICSEARCH_ADDR=your_elasticsearch_addr
docker-compose.dev.yml
+2
View File
@@ -120,6 +120,8 @@ services:
- WEB_PROXY=${WEB_PROXY:-}
# File size limit (in MB)
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50}
- IMAGE_MAX_CONCURRENT=${IMAGE_MAX_CONCURRENT:-1}
- OCR_BACKEND=${OCR_BACKEND:-paddle}
healthcheck:
test: ["CMD", "grpc_health_probe", "-addr=:50051"]
interval: 30s
docker-compose.yml
+2
View File
@@ -148,6 +148,8 @@ services:
- MINERU_ENDPOINT=${MINERU_ENDPOINT:-}
# File size limit (in MB)
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50}
- IMAGE_MAX_CONCURRENT=${IMAGE_MAX_CONCURRENT:-1}
- OCR_BACKEND=${OCR_BACKEND:-paddle}
healthcheck:
test: ["CMD", "grpc_health_probe", "-addr=:50051"]
interval: 30s
docreader/parser/base_parser.py
+30 -33
View File
@@ -35,34 +35,37 @@ class BaseParser(ABC):
@staticmethod
def _is_safe_url(url: str) -> bool:
"""Validate URL to prevent SSRF attacks
Args:
url: URL to validate
Returns:
True if URL is safe to fetch, False otherwise
"""
try:
parsed_url = urlparse(url)
# Only allow http and https schemes
if parsed_url.scheme not in ['http', 'https']:
if parsed_url.scheme not in ["http", "https"]:
logger.warning(f"Rejected URL with invalid scheme: {parsed_url.scheme}")
return False
# Extract hostname
hostname = parsed_url.hostname
if not hostname:
logger.warning("No hostname found in URL")
return False
# Try to parse as IP address
try:
ip = ipaddress.ip_address(hostname)
# Reject private, loopback, link-local, multicast addresses
if (ip.is_private or ip.is_loopback or
ip.is_link_local or ip.is_multicast or
ip.is_reserved):
if (
ip.is_private
or ip.is_loopback
or ip.is_link_local
or ip.is_multicast
or ip.is_reserved
):
logger.warning(f"Rejected URL with restricted IP: {ip}")
return False
except ValueError:
@@ -70,17 +73,19 @@ class BaseParser(ABC):
# Reject localhost and common internal hostnames
hostname_lower = hostname.lower()
restricted_hostnames = [
'localhost',
'127.0.0.1',
'::1',
'metadata.google.internal',
'metadata.tencentyun.com',
'169.254.169.254', # AWS metadata endpoint
"localhost",
"127.0.0.1",
"::1",
"metadata.google.internal",
"metadata.tencentyun.com",
"169.254.169.254", # AWS metadata endpoint
]
if hostname_lower in restricted_hostnames or hostname_lower.endswith('.local'):
if hostname_lower in restricted_hostnames or hostname_lower.endswith(
".local"
):
logger.warning(f"Rejected URL with restricted hostname: {hostname}")
return False
return True
except Exception as e:
logger.warning(f"Error validating URL: {str(e)}")
@@ -121,7 +126,7 @@ class BaseParser(ABC):
chunk_size: int = 1000,
chunk_overlap: int = 200,
separators: list[str] = ["\n\n", "\n", "。"],
ocr_backend: str = "paddle",
ocr_backend: str = "no_ocr",
ocr_config: dict = {},
max_image_size: int = 1920, # Maximum image size
max_concurrent_tasks: int = 5, # Max concurrent tasks
@@ -151,7 +156,7 @@ class BaseParser(ABC):
self.chunk_size = chunk_size
self.chunk_overlap = chunk_overlap
self.separators = separators
self.ocr_backend = os.getenv("OCR_BACKEND", ocr_backend)
self.ocr_backend = ocr_backend
self.ocr_config = ocr_config
self.max_image_size = max_image_size
self.max_concurrent_tasks = max_concurrent_tasks
@@ -161,13 +166,14 @@ class BaseParser(ABC):
self.chunking_config.storage_config if self.chunking_config else None
)
logger.info(f"Initializing parser for file: {file_name}, type: {file_type}")
logger.info(
f"Parser config: chunk_size={chunk_size}, "
f"Initializing parser for file={file_name}, type={file_type}, "
f"parser config: chunk_size={chunk_size}, "
f"overlap={chunk_overlap}, "
f"multimodal={enable_multimodal}, "
f"max_chunks={max_chunks}, "
f"ocr_backend={ocr_backend}, "
f"max_chunks={max_chunks}"
f"max_concurrent_tasks={max_concurrent_tasks}"
)
# Only initialize Caption service if multimodal is enabled
vlm_config = self.chunking_config.vlm_config if self.chunking_config else None
@@ -305,13 +311,8 @@ class BaseParser(ABC):
logger.warning("No image data to process")
return []
# Set max concurrency, reduce concurrency to avoid resource contention
max_concurrency = min(
self.max_concurrent_tasks, 1
) # Reduce concurrency to prevent excessive memory usage
# Use semaphore to limit concurrency
semaphore = asyncio.Semaphore(max_concurrency)
semaphore = asyncio.Semaphore(self.max_concurrent_tasks)
# Store results to avoid overall failure due to task failure
results = []
@@ -741,7 +742,6 @@ class BaseParser(ABC):
if not self._is_safe_url(img_url):
logger.error(f"URL failed validation check: {img_url}")
return img_url, img_url, None
# Still need to get image object for OCR processing
# Get proxy settings from environment variables
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
@@ -792,7 +792,6 @@ class BaseParser(ABC):
if not self._is_safe_url(img_url):
logger.error(f"URL failed validation check: {img_url}")
return img_url, img_url, None
# Get proxy settings from environment variables
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY")
@@ -952,10 +951,8 @@ class BaseParser(ABC):
# Create and run all Chunk concurrent processing tasks
async def process_all_chunks():
# Set max concurrency, reduce concurrency to avoid resource contention
max_concurrency = min(self.max_concurrent_tasks, 1) # Reduce concurrency
# Use semaphore to limit concurrency
semaphore = asyncio.Semaphore(max_concurrency)
semaphore = asyncio.Semaphore(self.max_concurrent_tasks)
async def process_with_limit(chunk, idx, total):
"""Use semaphore to control concurrent processing of Chunks"""
docreader/parser/parser.py
+5 -2
View File
@@ -1,4 +1,5 @@
import logging
import os
from typing import Dict, Type
from docreader.models.document import Document
@@ -109,8 +110,9 @@ class Parser:
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
max_concurrent_tasks=5, # Limit concurrent tasks to 5 to avoid resource exhaustion
chunking_config=config, # Pass the entire chunking config for advanced options
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
)
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
@@ -155,8 +157,9 @@ class Parser:
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
max_concurrent_tasks=5, # Limit concurrent tasks to avoid resource exhaustion
chunking_config=config, # Pass the entire chunking config
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
)
logger.info("Starting to parse URL content")
docreader/splitter/splitter.py
+168 -7
View File
@@ -75,9 +75,9 @@ class TextSplitter(BaseModel, Generic[T]):
# link - Markdown link syntax [text](url)
r"\[.*?\]\(.*?\)",
# table header - Markdown table header with separator line
r"(?:\|[^|\n]*)+\|[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|[\r\n]+",
r"[ ]*(?:\|[^|\n]*)+\|[\r\n]+\s*(?:\|\s*:?-{3,}:?\s*)+\|[\r\n]+",
# table body - Markdown table rows
r"(?:\|[^|\n]*)+\|[\r\n]+",
r"[ ]*(?:\|[^|\n]*)+\|[\r\n]+",
# code header - Code block start with language identifier
r"```(?:\w+)[\r\n]+[^\r\n]*",
],
@@ -137,6 +137,10 @@ class TextSplitter(BaseModel, Generic[T]):
# Step 4: Merge splits into final chunks with overlap
chunks = self._merge(splits)
# Step 5: Validate chunks and test restoration
# self._validate_chunks(chunks, text)
return chunks
def _split(self, text: str) -> List[str]:
@@ -250,10 +254,17 @@ class TextSplitter(BaseModel, Generic[T]):
cur_len > self.chunk_overlap
or cur_len + split_len + cur_headers_len > self.chunk_size
):
# Remove the first element to reduce overlap
# Remove the first element to reduce overlap.
# If the first element is a prepended header (start==end), also remove it.
first_chunk = cur_chunk.pop(0)
cur_len -= self.len_function(first_chunk[2])
# If we just popped a real content piece, there may be a header right after it
# (depending on previous iterations). Pop it only if it is actually a header.
if cur_chunk and first_chunk[0] == first_chunk[1]:
first_chunk = cur_chunk.pop(0)
cur_len -= self.len_function(first_chunk[2])
# Prepend headers to new chunk if:
# 1. Headers exist
# 2. Headers + split fit in chunk size
@@ -264,10 +275,8 @@ class TextSplitter(BaseModel, Generic[T]):
and cur_headers not in split
):
next_start = cur_chunk[0][0] if cur_chunk else cur_start
header_start = max(0, next_start - cur_headers_len)
header_end = cur_end
cur_chunk.insert(0, (header_start, header_end, cur_headers))
cur_chunk.insert(0, (next_start, next_start, cur_headers))
cur_len += cur_headers_len
# Add current split to the chunk
@@ -397,6 +406,154 @@ class TextSplitter(BaseModel, Generic[T]):
start = end
return res
def _validate_chunks(
self, chunks: List[Tuple[int, int, str]], original_text: str
) -> None:
"""Validate chunks order and test text restoration.
This method performs two validations:
1. Checks if chunk start positions are in ascending order
2. Tests if the original text can be restored from chunks
If validation fails, saves debug information to /tmp/chunk_error_<timestamp>.md
Args:
chunks: List of tuples (start_pos, end_pos, chunk_text) to validate
original_text: The original text that was split
"""
import datetime
errors = []
# Validation 1: Check if start positions are in ascending order
for i in range(1, len(chunks)):
prev_start = chunks[i - 1][0]
curr_start = chunks[i][0]
if curr_start < prev_start:
error_msg = (
f"Chunk order error: chunk[{i}] start position ({curr_start}) "
f"is less than chunk[{i - 1}] start position ({prev_start})"
)
errors.append(error_msg)
logger.error(error_msg)
# Validation 2: Test text restoration
try:
restored_text = self.restore_text(chunks)
if restored_text != original_text:
error_msg = (
f"Restoration failed: restored text differs from original. "
f"Original length: {len(original_text)}, "
f"Restored length: {len(restored_text)}"
)
errors.append(error_msg)
logger.error(error_msg)
# Find first difference position
min_len = min(len(original_text), len(restored_text))
diff_pos = -1
for i in range(min_len):
if original_text[i] != restored_text[i]:
diff_pos = i
break
if diff_pos >= 0:
context_start = max(0, diff_pos - 50)
context_end = min(len(original_text), diff_pos + 50)
errors.append(
f"First difference at position {diff_pos}:\n"
f"Original: {repr(original_text[context_start:context_end])}\n"
f"Restored: {repr(restored_text[context_start:context_end])}"
)
elif len(original_text) != len(restored_text):
errors.append(
f"Texts match up to position {min_len}, but lengths differ"
)
except Exception as e:
error_msg = f"Restoration exception: {str(e)}"
errors.append(error_msg)
logger.error(error_msg)
# If there are errors, save debug information to file
if errors:
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
error_file = f"/tmp/chunk_error_{timestamp}.md"
with open(error_file, "w", encoding="utf-8") as f:
f.write("# Chunk Validation Error Report\n\n")
f.write(f"Timestamp: {timestamp}\n\n")
f.write("## Errors\n\n")
for error in errors:
f.write(f"- {error}\n\n")
f.write("\n## Original Text\n\n")
f.write(f"Length: {len(original_text)}\n\n")
f.write("```\n")
f.write(original_text)
f.write("\n```\n\n")
f.write("\n## Chunks Information\n\n")
f.write(f"Total chunks: {len(chunks)}\n\n")
for i, (start, end, chunk_text) in enumerate(chunks):
f.write(f"### Chunk {i}\n\n")
f.write(f"- Position: [{start}:{end}]\n")
f.write(f"- Length: {len(chunk_text)}\n")
f.write(f"- Content:\n\n```\n{chunk_text}\n```\n\n")
try:
restored_text = self.restore_text(chunks)
f.write("\n## Restored Text\n\n")
f.write(f"Length: {len(restored_text)}\n\n")
f.write("```\n")
f.write(restored_text)
f.write("\n```\n")
except Exception as e:
f.write("\n## Restoration Failed\n\n")
f.write(f"Error: {str(e)}\n")
logger.error(f"Validation errors saved to: {error_file}")
def restore_text(self, chunks: List[Tuple[int, int, str]]) -> str:
"""Restore original text from chunks with overlap handling.
This method reconstructs the original text from chunks that may contain:
- Overlapping content between consecutive chunks
- Prepended headers that were added during merging (headers have start==end position)
The algorithm:
1. Sort chunks by their start position (and end position as tiebreaker)
2. Track the maximum end position seen so far
3. For each chunk, extract only the new content (after max_end_pos)
4. Concatenate all new content pieces
Args:
chunks: List of tuples (start_pos, end_pos, chunk_text) from split_text()
Returns:
The restored original text
Example:
>>> splitter = TextSplitter(chunk_size=10, chunk_overlap=3)
>>> chunks = splitter.split_text("Hello World!")
>>> restored = splitter.restore_text(chunks)
>>> assert restored == "Hello World!"
"""
if not chunks:
return ""
# Sort chunks by start position, then by end position
sorted_chunks = sorted(chunks, key=lambda x: (x[1], x[0]))
result_parts = []
last_end = 0
for start_pos, end_pos, chunk_text in sorted_chunks:
result_parts.append(chunk_text[last_end - end_pos :])
last_end = end_pos
return "".join(result_parts)
if __name__ == "__main__":
s = """
@@ -415,7 +572,11 @@ if __name__ == "__main__":
"""
sp = TextSplitter(chunk_size=200, chunk_overlap=2)
sp = TextSplitter(
chunk_size=200,
chunk_overlap=10,
separators=["\n\n", "\n", "。", "？", "！", "，", "；", "："],
)
ck = sp.split_text(s)
for c in ck:
print("------", len(c))
frontend/src/api/knowledge-base/index.ts
+13 -4
View File
@@ -40,15 +40,24 @@ export function copyKnowledgeBase(data: { source_id: string; target_id?: string
}
// 知识文件 API（基于具体知识库）
export function uploadKnowledgeFile(kbId: string, data = {}, onProgress?: (progressEvent: any) => void) {
return postUpload(`/api/v1/knowledge-bases/${kbId}/knowledge/file`, data, onProgress);
// data.tag_id: 可选，指定知识所属的分类ID
export function uploadKnowledgeFile(kbId: string, data: { file: File; tag_id?: string; [key: string]: any } = { file: new File([], '') }, onProgress?: (progressEvent: any) => void) {
const formData = new FormData();
Object.keys(data).forEach(key => {
if (data[key] !== undefined) formData.append(key, data[key]);
});
return postUpload(`/api/v1/knowledge-bases/${kbId}/knowledge/file`, formData, onProgress);
}
export function createKnowledgeFromURL(kbId: string, data: { url: string; enable_multimodel?: boolean }) {
// 从URL创建知识
// data.tag_id: 可选，指定知识所属的分类ID
export function createKnowledgeFromURL(kbId: string, data: { url: string; enable_multimodel?: boolean; tag_id?: string }) {
return post(`/api/v1/knowledge-bases/${kbId}/knowledge/url`, data);
}
export function createManualKnowledge(kbId: string, data: { title: string; content: string; status: string }) {
// 手工创建知识
// data.tag_id: 可选，指定知识所属的分类ID
export function createManualKnowledge(kbId: string, data: { title: string; content: string; status: string; tag_id?: string }) {
return post(`/api/v1/knowledge-bases/${kbId}/knowledge/manual`, data);
}
frontend/src/components/manual-knowledge-editor.vue
+6 -1
View File
@@ -579,7 +579,7 @@ const handleSave = async (targetStatus: ManualStatus) => {
saving.value = true
savingAction.value = targetStatus
try {
const payload = {
const payload: { title: string; content: string; status: string; tag_id?: string } = {
title: form.title.trim(),
content: form.content,
status: targetStatus,
@@ -591,6 +591,11 @@ const handleSave = async (targetStatus: ManualStatus) => {
if (mode.value === 'edit' && knowledgeId.value) {
response = await updateManualKnowledge(knowledgeId.value, payload)
} else {
// 创建新知识时，从 store 获取当前选中的分类ID
const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined
if (tagIdToUpload) {
payload.tag_id = tagIdToUpload
}
response = await createManualKnowledge(form.kbId, payload)
knowledgeID = response?.data?.id || knowledgeID
kbId = form.kbId
frontend/src/components/menu.vue
+6 -2
View File
@@ -823,9 +823,11 @@ const handleDocFileChange = async (event: Event) => {
}))
try {
// 获取当前选中的分类ID
const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined
await uploadKnowledgeFile(
kbId,
{ file },
{ file, tag_id: tagIdToUpload },
(progressEvent: any) => {
if (progressEvent.total) {
progress = Math.round((progressEvent.loaded * 100) / progressEvent.total)
@@ -1067,9 +1069,11 @@ const handleDocFolderChange = async (event: Event) => {
}))
try {
// 获取当前选中的分类ID
const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined
await uploadKnowledgeFile(
kbId,
{ file, fileName },
{ file, fileName, tag_id: tagIdToUpload },
(progressEvent: any) => {
if (progressEvent?.total) {
progress = Math.round((progressEvent.loaded * 100) / progressEvent.total)
frontend/src/hooks/useKnowledgeBase.ts
+16 -4
View File
@@ -10,6 +10,7 @@ import {
getKnowledgeDetailsCon,
} from "@/api/knowledge-base/index";
import { knowledgeStore } from "@/stores/knowledge";
import { useUIStore } from "@/stores/ui";
import { useRoute } from 'vue-router';
const usemenuStore = knowledgeStore();
@@ -62,20 +63,27 @@ export default function (knowledgeBaseId?: string) {
})
.catch(() => {});
};
const delKnowledge = (index: number, item: any) => {
const delKnowledge = (index: number, item: any, onSuccess?: () => void) => {
cardList.value[index].isMore = false;
moreIndex.value = -1;
delKnowledgeDetails(item.id)
return delKnowledgeDetails(item.id)
.then((result: any) => {
if (result.success) {
MessagePlugin.info("知识删除成功！");
getKnowled();
if (onSuccess) {
onSuccess();
} else {
getKnowled();
}
return true;
} else {
MessagePlugin.error("知识删除失败！");
return false;
}
})
.catch(() => {
MessagePlugin.error("知识删除失败！");
return false;
});
};
const openMore = (index: number) => {
@@ -110,7 +118,11 @@ export default function (knowledgeBaseId?: string) {
return;
}
uploadKnowledgeFile(currentKbId, { file })
// 获取当前选中的分类ID
const uiStore = useUIStore();
const tagIdToUpload = uiStore.selectedTagId !== '__untagged__' ? uiStore.selectedTagId : undefined;
uploadKnowledgeFile(currentKbId, { file, tag_id: tagIdToUpload })
.then((result: any) => {
if (result.success) {
MessagePlugin.info("上传成功！");
frontend/src/stores/ui.ts
+7
View File
@@ -7,6 +7,8 @@ export const useUIStore = defineStore('ui', {
kbEditorMode: 'create' as 'create' | 'edit',
currentKBId: null as string | null,
kbEditorType: 'document' as 'document' | 'faq',
// 当前选中的分类ID，用于文件上传时传递
selectedTagId: '__untagged__' as string,
kbEditorInitialSection: null as string | null,
settingsInitialSection: null as string | null,
settingsInitialSubSection: null as string | null,
@@ -101,6 +103,11 @@ export const useUIStore = defineStore('ui', {
}
}
this.manualEditorOnSuccess = null
},
// 设置当前选中的分类ID
setSelectedTagId(tagId: string) {
this.selectedTagId = tagId
}
}
})
frontend/src/views/knowledge/KnowledgeBase.vue
+14 -3
View File
@@ -212,6 +212,8 @@ const loadTags = async (kbIdValue: string, reset = false) => {
const handleTagFilterChange = (value: string) => {
selectedTagId.value = value;
// 同步更新 store 中的 selectedTagId，供 menu.vue 上传时使用
uiStore.setSelectedTagId(value);
page = 1;
loadKnowledgeFiles(kbId.value);
};
@@ -635,9 +637,12 @@ const handleDocumentUpload = async (event: Event) => {
let failCount = 0;
const totalCount = validFiles.length;
// 获取当前选中的分类ID（如果不是"未分类"则传递）
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
for (const file of validFiles) {
try {
const responseData: any = await uploadKnowledgeFile(kbId.value, { file });
const responseData: any = await uploadKnowledgeFile(kbId.value, { file, tag_id: tagIdToUpload });
const isSuccess = responseData?.success || responseData?.code === 200 || responseData?.status === 'success' || (!responseData?.error && responseData);
if (isSuccess) {
successCount++;
@@ -741,7 +746,9 @@ const handleURLImportConfirm = async () => {
urlImporting.value = true;
try {
const responseData: any = await createKnowledgeFromURL(kbId.value, { url });
// 获取当前选中的分类ID
const tagIdToUpload = selectedTagId.value !== '__untagged__' ? selectedTagId.value : undefined;
const responseData: any = await createKnowledgeFromURL(kbId.value, { url, tag_id: tagIdToUpload });
window.dispatchEvent(new CustomEvent('knowledgeFileUploaded', {
detail: { kbId: kbId.value }
}));
@@ -837,7 +844,11 @@ const getDoc = (page: number) => {
const delCardConfirm = () => {
delDialog.value = false;
delKnowledge(knowledgeIndex.value, knowledge.value);
delKnowledge(knowledgeIndex.value, knowledge.value, () => {
// 删除成功后刷新文档列表和分类数量
loadKnowledgeFiles(kbId.value);
loadTags(kbId.value);
});
};
// 处理知识库编辑成功后的回调
internal/application/service/chat_pipline/merge.go
+3 -2
View File
@@ -101,8 +101,9 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
}
// Merge overlapping chunks
if chunks[i].EndAt > lastChunk.EndAt {
lastChunk.Content = lastChunk.Content +
string([]rune(chunks[i].Content)[lastChunk.EndAt-chunks[i].StartAt:])
content := string([]rune(chunks[i].Content))
offset := len(content) - (chunks[i].EndAt - lastChunk.EndAt)
lastChunk.Content = lastChunk.Content + content[offset:]
lastChunk.EndAt = chunks[i].EndAt
lastChunk.SubChunkID = append(lastChunk.SubChunkID, chunks[i].ID)
internal/application/service/knowledge.go
+6 -2
View File
@@ -142,7 +142,7 @@ func (s *knowledgeService) isKnowledgeDeleting(ctx context.Context, tenantID uin
// CreateKnowledgeFromFile creates a knowledge entry from an uploaded file
func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string,
kbID string, file *multipart.FileHeader, metadata map[string]string, enableMultimodel *bool, customFileName string, tagID string,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from file")
@@ -261,6 +261,7 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
knowledge := &types.Knowledge{
TenantID: tenantID,
KnowledgeBaseID: kbID,
TagID: tagID, // 设置分类ID，用于知识分类管理
Type: "file",
Title: safeFilename,
FileName: safeFilename,
@@ -358,8 +359,9 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
}
// CreateKnowledgeFromURL creates a knowledge entry from a URL source
// tagID is optional - when provided, the knowledge will be assigned to the specified tag/category.
func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
kbID string, url string, enableMultimodel *bool, title string,
kbID string, url string, enableMultimodel *bool, title string, tagID string,
) (*types.Knowledge, error) {
logger.Info(ctx, "Start creating knowledge from URL")
logger.Infof(ctx, "Knowledge base ID: %s, URL: %s", kbID, url)
@@ -426,6 +428,7 @@ func (s *knowledgeService) CreateKnowledgeFromURL(ctx context.Context,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
EmbeddingModelID: kb.EmbeddingModelID,
TagID: tagID, // 设置分类ID，用于知识分类管理
}
// Save knowledge record
@@ -557,6 +560,7 @@ func (s *knowledgeService) CreateKnowledgeFromManual(ctx context.Context,
EmbeddingModelID: kb.EmbeddingModelID,
FileName: fileName,
FileType: types.KnowledgeTypeManual,
TagID: payload.TagID, // 设置分类ID，用于知识分类管理
}
if err := knowledge.SetManualMetadata(meta); err != nil {
logger.Errorf(ctx, "Failed to set manual metadata: %v", err)
internal/handler/knowledge.go
+11 -3
View File
@@ -164,8 +164,15 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) {
enableMultimodel = &parseBool
}
// 获取分类ID（如果提供），用于知识分类管理
tagID := c.PostForm("tag_id")
// 过滤特殊值，空字符串或 "__untagged__" 表示未分类
if tagID == "__untagged__" || tagID == "" {
tagID = ""
}
// Create knowledge entry from the file
knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName)
knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName, tagID)
// Check for duplicate knowledge error
if err != nil {
if h.handleDuplicateKnowledgeError(c, err, knowledge, "file") {
@@ -199,7 +206,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) {
// @Accept json
// @Produce json
// @Param id path string true "知识库ID"
// @Param request body object{url=string,enable_multimodel=bool,title=string} true "URL请求"
// @Param request body object{url=string,enable_multimodel=bool,title=string,tag_id=string} true "URL请求"
// @Success 201 {object} map[string]interface{} "创建的知识"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 409 {object} map[string]interface{} "URL重复"
@@ -222,6 +229,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
URL string `json:"url" binding:"required"`
EnableMultimodel *bool `json:"enable_multimodel"`
Title string `json:"title"`
TagID string `json:"tag_id"`
}
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "Failed to parse URL request", err)
@@ -238,7 +246,7 @@ func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
)
// Create knowledge entry from the URL
knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.EnableMultimodel, req.Title)
knowledge, err := h.kgService.CreateKnowledgeFromURL(ctx, kbID, req.URL, req.EnableMultimodel, req.Title, req.TagID)
// Check for duplicate knowledge error
if err != nil {
if h.handleDuplicateKnowledgeError(c, err, knowledge, "url") {
internal/types/interfaces/knowledge.go
+4
View File
@@ -12,6 +12,7 @@ import (
// KnowledgeService defines the interface for knowledge services.
type KnowledgeService interface {
// CreateKnowledgeFromFile creates knowledge from a file.
// tagID is optional - when provided, the file will be assigned to the specified tag/category.
CreateKnowledgeFromFile(
ctx context.Context,
kbID string,
@@ -19,14 +20,17 @@ type KnowledgeService interface {
metadata map[string]string,
enableMultimodel *bool,
customFileName string,
tagID string,
) (*types.Knowledge, error)
// CreateKnowledgeFromURL creates knowledge from a URL.
// tagID is optional - when provided, the knowledge will be assigned to the specified tag/category.
CreateKnowledgeFromURL(
ctx context.Context,
kbID string,
url string,
enableMultimodel *bool,
title string,
tagID string,
) (*types.Knowledge, error)
// CreateKnowledgeFromPassage creates knowledge from text passages.
CreateKnowledgeFromPassage(ctx context.Context, kbID string, passage []string) (*types.Knowledge, error)
internal/types/knowledge.go
+1
View File
@@ -144,6 +144,7 @@ type ManualKnowledgePayload struct {
Title string `json:"title"`
Content string `json:"content"`
Status string `json:"status"`
TagID string `json:"tag_id"`
}
// NewManualKnowledgeMetadata creates a new ManualKnowledgeMetadata instance.