chore(docreader): 重新组织模块文件

This commit is contained in:
begoniezhao
2025-11-05 11:33:50 +08:00
parent 0d790ffedc
commit c1f731e026
43 changed files with 3435 additions and 2398 deletions
+21
View File
@@ -0,0 +1,21 @@
from dataclasses import dataclass, field
@dataclass
class ChunkingConfig:
"""
Configuration for text chunking process.
Controls how documents are split into smaller pieces for processing.
"""
chunk_size: int = 512 # Maximum size of each chunk in tokens/chars
chunk_overlap: int = 50 # Number of tokens/chars to overlap between chunks
separators: list = field(
default_factory=lambda: ["\n\n", "\n", "。"]
) # Text separators in order of priority
enable_multimodal: bool = (
False # Whether to enable multimodal processing (text + images)
)
storage_config: dict = None # Preferred field name going forward
vlm_config: dict = None # VLM configuration for image captioning