feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements

Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level
settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing.
Optional odl-hybrid sidecar stays local-build only and is excluded from
default dev-start and full profiles.
This commit is contained in:
wizardchen
2026-06-03 12:00:09 +08:00
committed by lyingbug
parent 7b1bb1054f
commit ef1047bf67
50 changed files with 4352 additions and 304 deletions
+29
View File
@@ -55,6 +55,12 @@ class DocReaderConfig:
# Parser
docx_max_pages: int
markitdown_max_workers: int
odl_max_workers: int
odl_hybrid: str
odl_hybrid_url: str
odl_hybrid_mode: str
odl_hybrid_fallback: bool
odl_markdown_with_html: bool
pdf_render_max_workers: int
pdf_render_parallelism: int
pdf_render_dpi: int
@@ -81,6 +87,17 @@ def load_config() -> DocReaderConfig:
grpc_port = _get_int(["DOCREADER_GRPC_PORT", "PORT"], 50051)
docx_max_pages = _get_int(["DOCREADER_DOCX_MAX_PAGES"], 0)
markitdown_max_workers = _get_int(["DOCREADER_MARKITDOWN_MAX_WORKERS"], 1)
odl_max_workers = _get_int(["DOCREADER_ODL_MAX_WORKERS"], 1)
odl_hybrid = _get_str(["DOCREADER_ODL_HYBRID"], "off")
odl_hybrid_url = _get_str(
["DOCREADER_ODL_HYBRID_URL"],
"http://127.0.0.1:5002",
)
odl_hybrid_mode = _get_str(["DOCREADER_ODL_HYBRID_MODE"], "auto")
odl_hybrid_fallback = _get_bool(["DOCREADER_ODL_HYBRID_FALLBACK"], False)
odl_markdown_with_html = _get_bool(
["DOCREADER_ODL_MARKDOWN_WITH_HTML"], False
)
pdf_render_max_workers = _get_int(["DOCREADER_PDF_RENDER_MAX_WORKERS"], 1)
# Intra-document render parallelism: how many worker processes render the
# scanned pages of a SINGLE PDF in parallel. pdfium is not thread-safe, so
@@ -117,6 +134,12 @@ def load_config() -> DocReaderConfig:
grpc_port=grpc_port,
docx_max_pages=docx_max_pages,
markitdown_max_workers=markitdown_max_workers,
odl_max_workers=odl_max_workers,
odl_hybrid=odl_hybrid,
odl_hybrid_url=odl_hybrid_url,
odl_hybrid_mode=odl_hybrid_mode,
odl_hybrid_fallback=odl_hybrid_fallback,
odl_markdown_with_html=odl_markdown_with_html,
pdf_render_max_workers=pdf_render_max_workers,
pdf_render_parallelism=pdf_render_parallelism,
pdf_render_dpi=pdf_render_dpi,
@@ -139,6 +162,12 @@ def dump_config(mask_secrets: bool = True) -> Dict[str, Any]:
"DOCREADER_GRPC_PORT": cfg.grpc_port,
"DOCREADER_DOCX_MAX_PAGES": cfg.docx_max_pages,
"DOCREADER_MARKITDOWN_MAX_WORKERS": cfg.markitdown_max_workers,
"DOCREADER_ODL_MAX_WORKERS": cfg.odl_max_workers,
"DOCREADER_ODL_HYBRID": cfg.odl_hybrid,
"DOCREADER_ODL_HYBRID_URL": cfg.odl_hybrid_url,
"DOCREADER_ODL_HYBRID_MODE": cfg.odl_hybrid_mode,
"DOCREADER_ODL_HYBRID_FALLBACK": cfg.odl_hybrid_fallback,
"DOCREADER_ODL_MARKDOWN_WITH_HTML": cfg.odl_markdown_with_html,
"DOCREADER_PDF_RENDER_MAX_WORKERS": cfg.pdf_render_max_workers,
"DOCREADER_PDF_RENDER_PARALLELISM": cfg.pdf_render_parallelism,
"DOCREADER_PDF_RENDER_DPI": cfg.pdf_render_dpi,