feat: add docx max pages env config
This commit is contained in:
@@ -291,6 +291,10 @@ DOCREADER_ADDR=docreader:50051
|
||||
# Docreader 连接方式
|
||||
DOCREADER_TRANSPORT=grpc
|
||||
|
||||
# Docreader 中 DOCX 解析的最大页数,默认 100
|
||||
# 用于限制超大 Word 文档的解析开销;超过页数的内容将不会继续解析
|
||||
# DOCREADER_DOCX_MAX_PAGES=100
|
||||
|
||||
# 如果使用Weaviate作为向量存储,需要配置以下参数
|
||||
# 注意:容器内访问请使用 service:port(不要用 localhost,也不要用宿主机映射端口)
|
||||
# Weaviate HTTP 地址(Docker 内:weaviate:8080;宿主机访问:localhost:9035)
|
||||
|
||||
@@ -52,6 +52,9 @@ class DocReaderConfig:
|
||||
grpc_max_file_size_mb: int
|
||||
grpc_port: int
|
||||
|
||||
# Parser
|
||||
docx_max_pages: int
|
||||
|
||||
# Proxy
|
||||
external_http_proxy: str
|
||||
external_https_proxy: str
|
||||
@@ -70,6 +73,7 @@ def load_config() -> DocReaderConfig:
|
||||
* 1024
|
||||
)
|
||||
grpc_port = _get_int(["DOCREADER_GRPC_PORT", "PORT"], 50051)
|
||||
docx_max_pages = _get_int(["DOCREADER_DOCX_MAX_PAGES"], 100)
|
||||
|
||||
external_http_proxy = _get_str(
|
||||
["DOCREADER_EXTERNAL_HTTP_PROXY", "EXTERNAL_HTTP_PROXY"], ""
|
||||
@@ -86,6 +90,7 @@ def load_config() -> DocReaderConfig:
|
||||
grpc_max_workers=grpc_max_workers,
|
||||
grpc_max_file_size_mb=grpc_max_file_size_mb,
|
||||
grpc_port=grpc_port,
|
||||
docx_max_pages=docx_max_pages,
|
||||
external_http_proxy=external_http_proxy,
|
||||
external_https_proxy=external_https_proxy,
|
||||
image_output_dir=image_output_dir,
|
||||
@@ -101,6 +106,7 @@ def dump_config(mask_secrets: bool = True) -> Dict[str, Any]:
|
||||
"DOCREADER_GRPC_MAX_WORKERS": cfg.grpc_max_workers,
|
||||
"DOCREADER_GRPC_MAX_FILE_SIZE_MB": cfg.grpc_max_file_size_mb,
|
||||
"DOCREADER_GRPC_PORT": cfg.grpc_port,
|
||||
"DOCREADER_DOCX_MAX_PAGES": cfg.docx_max_pages,
|
||||
"DOCREADER_EXTERNAL_HTTP_PROXY": cfg.external_http_proxy,
|
||||
"DOCREADER_EXTERNAL_HTTPS_PROXY": cfg.external_https_proxy,
|
||||
"DOCREADER_IMAGE_OUTPUT_DIR": cfg.image_output_dir,
|
||||
|
||||
@@ -41,6 +41,7 @@ from docx.image.exceptions import (
|
||||
)
|
||||
from PIL import Image
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document as DocumentModel
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.utils import endecode
|
||||
@@ -76,7 +77,7 @@ class DocxParser(BaseParser):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
max_pages: int = 100, # Maximum number of pages to process
|
||||
max_pages: Optional[int] = None, # Maximum number of pages to process
|
||||
**kwargs,
|
||||
):
|
||||
"""Initialize DOCX document parser
|
||||
@@ -95,8 +96,8 @@ class DocxParser(BaseParser):
|
||||
max_pages: Maximum number of pages to process
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
self.max_pages = max_pages
|
||||
logger.info(f"DocxParser initialized with max_pages={max_pages}")
|
||||
self.max_pages = CONFIG.docx_max_pages if max_pages is None else max_pages
|
||||
logger.info(f"DocxParser initialized with max_pages={self.max_pages}")
|
||||
|
||||
def parse_into_text(self, content: bytes) -> DocumentModel:
|
||||
"""Parse DOCX document, extract text content and image Markdown links"""
|
||||
|
||||
Reference in New Issue
Block a user