fix(docreader): throttle heavy parser concurrency

This commit is contained in:
wolfkill
2026-05-07 16:05:21 +08:00
committed by lyingbug
parent 56058287dd
commit 450a5bd2dd
12 changed files with 281 additions and 27 deletions
+9 -7
View File
@@ -3,9 +3,11 @@ import logging
from markitdown import MarkItDown
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
from docreader.parser.concurrency import parser_worker_limit
from docreader.parser.markdown_parser import MarkdownParser
logger = logging.getLogger(__name__)
@@ -33,14 +35,14 @@ class StdMarkitdownParser(BaseParser):
if ext and not ext.startswith('.'):
ext = '.' + ext
# 直接调用 convert,移除 try-catch,让异常由上层 PipelineParser 统一捕获
result = self.markitdown.convert(
io.BytesIO(content),
file_extension=ext,
keep_data_uris=True
)
with parser_worker_limit("markitdown", CONFIG.markitdown_max_workers):
result = self.markitdown.convert(
io.BytesIO(content),
file_extension=ext,
keep_data_uris=True
)
return Document(content=result.text_content)
class MarkitdownParser(PipelineParser):
_parser_cls = (StdMarkitdownParser, MarkdownParser)
_parser_cls = (StdMarkitdownParser, MarkdownParser)