fix(docreader): throttle heavy parser concurrency

This commit is contained in:
wolfkill
2026-05-07 16:05:21 +08:00
committed by lyingbug
parent 56058287dd
commit 450a5bd2dd
12 changed files with 281 additions and 27 deletions
+39
View File
@@ -0,0 +1,39 @@
import logging
import threading
from contextlib import contextmanager
from typing import Dict, Iterator
logger = logging.getLogger(__name__)
_LIMITERS: Dict[str, threading.BoundedSemaphore] = {}
_LIMITERS_LOCK = threading.Lock()
def _get_limiter(name: str, max_workers: int) -> threading.BoundedSemaphore:
with _LIMITERS_LOCK:
limiter = _LIMITERS.get(name)
if limiter is None:
limiter = threading.BoundedSemaphore(max_workers)
_LIMITERS[name] = limiter
return limiter
@contextmanager
def parser_worker_limit(name: str, max_workers: int) -> Iterator[None]:
"""Limit concurrent access to heavy, process-wide parser operations.
Set max_workers <= 0 to disable throttling for deployments that have enough
CPU/GPU resources and know the parser backend is safe under concurrency.
"""
if max_workers <= 0:
yield
return
limiter = _get_limiter(name, max_workers)
logger.debug("Waiting for %s parser slot (max_workers=%d)", name, max_workers)
limiter.acquire()
try:
yield
finally:
limiter.release()
+9 -7
View File
@@ -3,9 +3,11 @@ import logging
from markitdown import MarkItDown
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
from docreader.parser.concurrency import parser_worker_limit
from docreader.parser.markdown_parser import MarkdownParser
logger = logging.getLogger(__name__)
@@ -33,14 +35,14 @@ class StdMarkitdownParser(BaseParser):
if ext and not ext.startswith('.'):
ext = '.' + ext
# 直接调用 convert,移除 try-catch,让异常由上层 PipelineParser 统一捕获
result = self.markitdown.convert(
io.BytesIO(content),
file_extension=ext,
keep_data_uris=True
)
with parser_worker_limit("markitdown", CONFIG.markitdown_max_workers):
result = self.markitdown.convert(
io.BytesIO(content),
file_extension=ext,
keep_data_uris=True
)
return Document(content=result.text_content)
class MarkitdownParser(PipelineParser):
_parser_cls = (StdMarkitdownParser, MarkdownParser)
_parser_cls = (StdMarkitdownParser, MarkdownParser)
+58 -18
View File
@@ -1,6 +1,8 @@
from docreader.models.document import Document
from docreader.config import CONFIG
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import FirstParser
from docreader.parser.concurrency import parser_worker_limit
from docreader.parser.markitdown_parser import MarkitdownParser
import io
@@ -10,36 +12,74 @@ import logging
logger = logging.getLogger(__name__)
def _close_pdfium_resource(resource) -> None:
close = getattr(resource, "close", None)
if close:
close()
def _normalize_image_quality(quality: int) -> int:
return min(95, max(1, quality))
class PDFScannedParser(BaseParser):
"""Fallback parser for scanned PDFs.
If the primary parser extracts no text (e.g. Markitdown on a scanned PDF),
this parser converts each page into an image. The Go App will then perform
OCR on the extracted images.
this parser converts each page into a JPEG image. The Go App will then
perform OCR on the extracted images.
"""
def parse_into_text(self, content: bytes) -> Document:
import pdfplumber
import pypdfium2 as pdfium
images = {}
markdown_lines = []
base_name = os.path.splitext(self.file_name or "document")[0]
logger.info("PDFScannedParser: Attempting to convert PDF pages to images for %s", self.file_name)
logger.info(
"PDFScannedParser: Rendering PDF pages to JPEG images for %s",
self.file_name,
)
try:
with pdfplumber.open(io.BytesIO(content)) as pdf:
for i, page in enumerate(pdf.pages):
img_obj = page.to_image(resolution=150).original
img_byte_arr = io.BytesIO()
img_obj.save(img_byte_arr, format="PNG")
img_bytes = img_byte_arr.getvalue()
page_filename = f"{base_name}_page_{i+1}.png"
ref_path = f"images/{page_filename}"
markdown_lines.append(f"![{page_filename}]({ref_path})")
images[ref_path] = base64.b64encode(img_bytes).decode("utf-8")
with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers):
pdf = pdfium.PdfDocument(content)
try:
page_count = len(pdf)
scale = max(1, CONFIG.pdf_render_dpi) / 72
quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality)
for i in range(page_count):
page = pdf[i]
bitmap = None
try:
bitmap = page.render(scale=scale)
img_obj = bitmap.to_pil()
if img_obj.mode != "RGB":
img_obj = img_obj.convert("RGB")
img_byte_arr = io.BytesIO()
img_obj.save(
img_byte_arr,
format="JPEG",
quality=quality,
optimize=True,
)
img_bytes = img_byte_arr.getvalue()
finally:
_close_pdfium_resource(bitmap)
_close_pdfium_resource(page)
page_filename = f"{base_name}_page_{i+1}.jpg"
ref_path = f"images/{page_filename}"
markdown_lines.append(f"![{page_filename}]({ref_path})")
images[ref_path] = base64.b64encode(img_bytes).decode("utf-8")
finally:
_close_pdfium_resource(pdf)
text = "\n\n".join(markdown_lines)
return Document(
@@ -47,11 +87,11 @@ class PDFScannedParser(BaseParser):
images=images,
metadata={
"image_source_type": "scanned_pdf",
"page_count": len(pdf.pages)
"page_count": page_count
}
)
except Exception as e:
logger.exception("PDFScannedParser failed to parse PDF: %v", e)
logger.exception("PDFScannedParser failed to parse PDF: %s", e)
raise e
class PDFParser(FirstParser):