feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements
Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing. Optional odl-hybrid sidecar stays local-build only and is excluded from default dev-start and full profiles.
This commit is contained in:
@@ -9,6 +9,11 @@ from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.chain_parser import PipelineParser
|
||||
from docreader.parser.concurrency import parser_worker_limit
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
from docreader.parser.ppt_convert import normalize_ppt_bytes
|
||||
from docreader.parser.pptx_media import (
|
||||
attach_pptx_media_to_markdown,
|
||||
markdown_needs_pptx_media_attach,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -32,16 +37,41 @@ class StdMarkitdownParser(BaseParser):
|
||||
Uses self.file_type (inherited from BaseParser) to hint the stream format.
|
||||
"""
|
||||
ext = self.file_type
|
||||
if ext and not ext.startswith('.'):
|
||||
ext = '.' + ext
|
||||
ft = (ext or "").lstrip(".").lower()
|
||||
pptx_bytes: bytes | None = None
|
||||
if ft in ("ppt", "pptx"):
|
||||
content, ext = normalize_ppt_bytes(content, ft)
|
||||
pptx_bytes = content
|
||||
ft = "pptx"
|
||||
elif ext and not ext.startswith("."):
|
||||
ext = "." + ext
|
||||
|
||||
with parser_worker_limit("markitdown", CONFIG.markitdown_max_workers):
|
||||
result = self.markitdown.convert(
|
||||
result = self._convert_markitdown(content, ext, keep_data_uris=True)
|
||||
if result is None:
|
||||
logger.warning(
|
||||
"MarkItDown failed with embedded images for %s; retrying without data URIs",
|
||||
ft or ext,
|
||||
)
|
||||
result = self._convert_markitdown(content, ext, keep_data_uris=False)
|
||||
|
||||
text = result.text_content
|
||||
images: dict[str, str] = {}
|
||||
if pptx_bytes is not None and markdown_needs_pptx_media_attach(text):
|
||||
text, images = attach_pptx_media_to_markdown(text, pptx_bytes)
|
||||
return Document(content=text, images=images)
|
||||
|
||||
def _convert_markitdown(self, content: bytes, ext: str | None, *, keep_data_uris: bool):
|
||||
try:
|
||||
return self.markitdown.convert(
|
||||
io.BytesIO(content),
|
||||
file_extension=ext,
|
||||
keep_data_uris=True
|
||||
keep_data_uris=keep_data_uris,
|
||||
)
|
||||
return Document(content=result.text_content)
|
||||
except Exception:
|
||||
if keep_data_uris:
|
||||
return None
|
||||
raise
|
||||
|
||||
|
||||
class MarkitdownParser(PipelineParser):
|
||||
|
||||
Reference in New Issue
Block a user