fix(docreader): throttle heavy parser concurrency
This commit is contained in:
@@ -0,0 +1,39 @@
|
||||
import logging
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from typing import Dict, Iterator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_LIMITERS: Dict[str, threading.BoundedSemaphore] = {}
|
||||
_LIMITERS_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _get_limiter(name: str, max_workers: int) -> threading.BoundedSemaphore:
|
||||
with _LIMITERS_LOCK:
|
||||
limiter = _LIMITERS.get(name)
|
||||
if limiter is None:
|
||||
limiter = threading.BoundedSemaphore(max_workers)
|
||||
_LIMITERS[name] = limiter
|
||||
return limiter
|
||||
|
||||
|
||||
@contextmanager
|
||||
def parser_worker_limit(name: str, max_workers: int) -> Iterator[None]:
|
||||
"""Limit concurrent access to heavy, process-wide parser operations.
|
||||
|
||||
Set max_workers <= 0 to disable throttling for deployments that have enough
|
||||
CPU/GPU resources and know the parser backend is safe under concurrency.
|
||||
"""
|
||||
|
||||
if max_workers <= 0:
|
||||
yield
|
||||
return
|
||||
|
||||
limiter = _get_limiter(name, max_workers)
|
||||
logger.debug("Waiting for %s parser slot (max_workers=%d)", name, max_workers)
|
||||
limiter.acquire()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
limiter.release()
|
||||
@@ -3,9 +3,11 @@ import logging
|
||||
|
||||
from markitdown import MarkItDown
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.chain_parser import PipelineParser
|
||||
from docreader.parser.concurrency import parser_worker_limit
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -33,14 +35,14 @@ class StdMarkitdownParser(BaseParser):
|
||||
if ext and not ext.startswith('.'):
|
||||
ext = '.' + ext
|
||||
|
||||
# 直接调用 convert,移除 try-catch,让异常由上层 PipelineParser 统一捕获
|
||||
result = self.markitdown.convert(
|
||||
io.BytesIO(content),
|
||||
file_extension=ext,
|
||||
keep_data_uris=True
|
||||
)
|
||||
with parser_worker_limit("markitdown", CONFIG.markitdown_max_workers):
|
||||
result = self.markitdown.convert(
|
||||
io.BytesIO(content),
|
||||
file_extension=ext,
|
||||
keep_data_uris=True
|
||||
)
|
||||
return Document(content=result.text_content)
|
||||
|
||||
|
||||
class MarkitdownParser(PipelineParser):
|
||||
_parser_cls = (StdMarkitdownParser, MarkdownParser)
|
||||
_parser_cls = (StdMarkitdownParser, MarkdownParser)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
from docreader.models.document import Document
|
||||
from docreader.config import CONFIG
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.chain_parser import FirstParser
|
||||
from docreader.parser.concurrency import parser_worker_limit
|
||||
from docreader.parser.markitdown_parser import MarkitdownParser
|
||||
|
||||
import io
|
||||
@@ -10,36 +12,74 @@ import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _close_pdfium_resource(resource) -> None:
|
||||
close = getattr(resource, "close", None)
|
||||
if close:
|
||||
close()
|
||||
|
||||
|
||||
def _normalize_image_quality(quality: int) -> int:
|
||||
return min(95, max(1, quality))
|
||||
|
||||
|
||||
class PDFScannedParser(BaseParser):
|
||||
"""Fallback parser for scanned PDFs.
|
||||
|
||||
If the primary parser extracts no text (e.g. Markitdown on a scanned PDF),
|
||||
this parser converts each page into an image. The Go App will then perform
|
||||
OCR on the extracted images.
|
||||
this parser converts each page into a JPEG image. The Go App will then
|
||||
perform OCR on the extracted images.
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
import pdfplumber
|
||||
import pypdfium2 as pdfium
|
||||
|
||||
images = {}
|
||||
markdown_lines = []
|
||||
|
||||
base_name = os.path.splitext(self.file_name or "document")[0]
|
||||
|
||||
logger.info("PDFScannedParser: Attempting to convert PDF pages to images for %s", self.file_name)
|
||||
logger.info(
|
||||
"PDFScannedParser: Rendering PDF pages to JPEG images for %s",
|
||||
self.file_name,
|
||||
)
|
||||
|
||||
try:
|
||||
with pdfplumber.open(io.BytesIO(content)) as pdf:
|
||||
for i, page in enumerate(pdf.pages):
|
||||
img_obj = page.to_image(resolution=150).original
|
||||
img_byte_arr = io.BytesIO()
|
||||
img_obj.save(img_byte_arr, format="PNG")
|
||||
img_bytes = img_byte_arr.getvalue()
|
||||
|
||||
page_filename = f"{base_name}_page_{i+1}.png"
|
||||
ref_path = f"images/{page_filename}"
|
||||
|
||||
markdown_lines.append(f"")
|
||||
images[ref_path] = base64.b64encode(img_bytes).decode("utf-8")
|
||||
with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers):
|
||||
pdf = pdfium.PdfDocument(content)
|
||||
try:
|
||||
page_count = len(pdf)
|
||||
scale = max(1, CONFIG.pdf_render_dpi) / 72
|
||||
quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality)
|
||||
|
||||
for i in range(page_count):
|
||||
page = pdf[i]
|
||||
bitmap = None
|
||||
try:
|
||||
bitmap = page.render(scale=scale)
|
||||
img_obj = bitmap.to_pil()
|
||||
if img_obj.mode != "RGB":
|
||||
img_obj = img_obj.convert("RGB")
|
||||
|
||||
img_byte_arr = io.BytesIO()
|
||||
img_obj.save(
|
||||
img_byte_arr,
|
||||
format="JPEG",
|
||||
quality=quality,
|
||||
optimize=True,
|
||||
)
|
||||
img_bytes = img_byte_arr.getvalue()
|
||||
finally:
|
||||
_close_pdfium_resource(bitmap)
|
||||
_close_pdfium_resource(page)
|
||||
|
||||
page_filename = f"{base_name}_page_{i+1}.jpg"
|
||||
ref_path = f"images/{page_filename}"
|
||||
|
||||
markdown_lines.append(f"")
|
||||
images[ref_path] = base64.b64encode(img_bytes).decode("utf-8")
|
||||
finally:
|
||||
_close_pdfium_resource(pdf)
|
||||
|
||||
text = "\n\n".join(markdown_lines)
|
||||
return Document(
|
||||
@@ -47,11 +87,11 @@ class PDFScannedParser(BaseParser):
|
||||
images=images,
|
||||
metadata={
|
||||
"image_source_type": "scanned_pdf",
|
||||
"page_count": len(pdf.pages)
|
||||
"page_count": page_count
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("PDFScannedParser failed to parse PDF: %v", e)
|
||||
logger.exception("PDFScannedParser failed to parse PDF: %s", e)
|
||||
raise e
|
||||
|
||||
class PDFParser(FirstParser):
|
||||
|
||||
Reference in New Issue
Block a user