diff --git a/.env.example b/.env.example index 0a38d4fa..5ee62a9d 100644 --- a/.env.example +++ b/.env.example @@ -291,6 +291,10 @@ DOCREADER_ADDR=docreader:50051 # Docreader 连接方式 DOCREADER_TRANSPORT=grpc +# Docreader 中 DOCX 解析的最大页数,默认 100 +# 用于限制超大 Word 文档的解析开销;超过页数的内容将不会继续解析 +# DOCREADER_DOCX_MAX_PAGES=100 + # 如果使用Weaviate作为向量存储,需要配置以下参数 # 注意:容器内访问请使用 service:port(不要用 localhost,也不要用宿主机映射端口) # Weaviate HTTP 地址(Docker 内:weaviate:8080;宿主机访问:localhost:9035) diff --git a/docreader/config.py b/docreader/config.py index 845e46fd..7cc7de5a 100644 --- a/docreader/config.py +++ b/docreader/config.py @@ -52,6 +52,9 @@ class DocReaderConfig: grpc_max_file_size_mb: int grpc_port: int + # Parser + docx_max_pages: int + # Proxy external_http_proxy: str external_https_proxy: str @@ -70,6 +73,7 @@ def load_config() -> DocReaderConfig: * 1024 ) grpc_port = _get_int(["DOCREADER_GRPC_PORT", "PORT"], 50051) + docx_max_pages = _get_int(["DOCREADER_DOCX_MAX_PAGES"], 100) external_http_proxy = _get_str( ["DOCREADER_EXTERNAL_HTTP_PROXY", "EXTERNAL_HTTP_PROXY"], "" @@ -86,6 +90,7 @@ def load_config() -> DocReaderConfig: grpc_max_workers=grpc_max_workers, grpc_max_file_size_mb=grpc_max_file_size_mb, grpc_port=grpc_port, + docx_max_pages=docx_max_pages, external_http_proxy=external_http_proxy, external_https_proxy=external_https_proxy, image_output_dir=image_output_dir, @@ -101,6 +106,7 @@ def dump_config(mask_secrets: bool = True) -> Dict[str, Any]: "DOCREADER_GRPC_MAX_WORKERS": cfg.grpc_max_workers, "DOCREADER_GRPC_MAX_FILE_SIZE_MB": cfg.grpc_max_file_size_mb, "DOCREADER_GRPC_PORT": cfg.grpc_port, + "DOCREADER_DOCX_MAX_PAGES": cfg.docx_max_pages, "DOCREADER_EXTERNAL_HTTP_PROXY": cfg.external_http_proxy, "DOCREADER_EXTERNAL_HTTPS_PROXY": cfg.external_https_proxy, "DOCREADER_IMAGE_OUTPUT_DIR": cfg.image_output_dir, diff --git a/docreader/parser/docx_parser.py b/docreader/parser/docx_parser.py index 7faf1cac..e334d931 100644 --- a/docreader/parser/docx_parser.py +++ b/docreader/parser/docx_parser.py @@ -41,6 +41,7 @@ from docx.image.exceptions import ( ) from PIL import Image +from docreader.config import CONFIG from docreader.models.document import Document as DocumentModel from docreader.parser.base_parser import BaseParser from docreader.utils import endecode @@ -76,7 +77,7 @@ class DocxParser(BaseParser): def __init__( self, - max_pages: int = 100, # Maximum number of pages to process + max_pages: Optional[int] = None, # Maximum number of pages to process **kwargs, ): """Initialize DOCX document parser @@ -95,8 +96,8 @@ class DocxParser(BaseParser): max_pages: Maximum number of pages to process """ super().__init__(**kwargs) - self.max_pages = max_pages - logger.info(f"DocxParser initialized with max_pages={max_pages}") + self.max_pages = CONFIG.docx_max_pages if max_pages is None else max_pages + logger.info(f"DocxParser initialized with max_pages={self.max_pages}") def parse_into_text(self, content: bytes) -> DocumentModel: """Parse DOCX document, extract text content and image Markdown links"""