feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements

Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level
settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing.
Optional odl-hybrid sidecar stays local-build only and is excluded from
default dev-start and full profiles.
This commit is contained in:
wizardchen
2026-06-03 12:00:09 +08:00
committed by lyingbug
parent 7b1bb1054f
commit ef1047bf67
50 changed files with 4352 additions and 304 deletions
+35 -5
View File
@@ -9,6 +9,11 @@ from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
from docreader.parser.concurrency import parser_worker_limit
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.ppt_convert import normalize_ppt_bytes
from docreader.parser.pptx_media import (
attach_pptx_media_to_markdown,
markdown_needs_pptx_media_attach,
)
logger = logging.getLogger(__name__)
@@ -32,16 +37,41 @@ class StdMarkitdownParser(BaseParser):
Uses self.file_type (inherited from BaseParser) to hint the stream format.
"""
ext = self.file_type
if ext and not ext.startswith('.'):
ext = '.' + ext
ft = (ext or "").lstrip(".").lower()
pptx_bytes: bytes | None = None
if ft in ("ppt", "pptx"):
content, ext = normalize_ppt_bytes(content, ft)
pptx_bytes = content
ft = "pptx"
elif ext and not ext.startswith("."):
ext = "." + ext
with parser_worker_limit("markitdown", CONFIG.markitdown_max_workers):
result = self.markitdown.convert(
result = self._convert_markitdown(content, ext, keep_data_uris=True)
if result is None:
logger.warning(
"MarkItDown failed with embedded images for %s; retrying without data URIs",
ft or ext,
)
result = self._convert_markitdown(content, ext, keep_data_uris=False)
text = result.text_content
images: dict[str, str] = {}
if pptx_bytes is not None and markdown_needs_pptx_media_attach(text):
text, images = attach_pptx_media_to_markdown(text, pptx_bytes)
return Document(content=text, images=images)
def _convert_markitdown(self, content: bytes, ext: str | None, *, keep_data_uris: bool):
try:
return self.markitdown.convert(
io.BytesIO(content),
file_extension=ext,
keep_data_uris=True
keep_data_uris=keep_data_uris,
)
return Document(content=result.text_content)
except Exception:
if keep_data_uris:
return None
raise
class MarkitdownParser(PipelineParser):