feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements

Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level
settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing.
Optional odl-hybrid sidecar stays local-build only and is excluded from
default dev-start and full profiles.
This commit is contained in:
wizardchen
2026-06-03 12:00:09 +08:00
committed by lyingbug
parent 7b1bb1054f
commit ef1047bf67
50 changed files with 4352 additions and 304 deletions
+27 -1
View File
@@ -248,8 +248,13 @@ services:
- docreader-tmp-dev:/tmp/docreader
environment:
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
- MINERU_ENDPOINT=${MINERU_ENDPOINT:-}
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
- DOCREADER_ODL_MAX_WORKERS=${DOCREADER_ODL_MAX_WORKERS:-1}
- DOCREADER_ODL_HYBRID=${DOCREADER_ODL_HYBRID:-off}
- DOCREADER_ODL_HYBRID_URL=${DOCREADER_ODL_HYBRID_URL:-http://odl-hybrid:5002}
- DOCREADER_ODL_HYBRID_MODE=${DOCREADER_ODL_HYBRID_MODE:-auto}
- DOCREADER_ODL_HYBRID_FALLBACK=${DOCREADER_ODL_HYBRID_FALLBACK:-false}
- DOCREADER_ODL_MARKDOWN_WITH_HTML=${DOCREADER_ODL_MARKDOWN_WITH_HTML:-false}
- DOCREADER_MARKITDOWN_MAX_WORKERS=${DOCREADER_MARKITDOWN_MAX_WORKERS:-1}
- DOCREADER_PDF_RENDER_MAX_WORKERS=${DOCREADER_PDF_RENDER_MAX_WORKERS:-1}
- DOCREADER_PDF_RENDER_DPI=${DOCREADER_PDF_RENDER_DPI:-200}
@@ -272,6 +277,27 @@ services:
extra_hosts:
- "host.docker.internal:host-gateway"
# OpenDataLoader hybrid backend (optional). Enable profile "odl-hybrid" and set
# DOCREADER_ODL_HYBRID=docling-fast on docreader. Default --no-ocr (no EasyOCR).
# Local build only — not published to Docker Hub.
odl-hybrid:
build:
context: .
dockerfile: docker/Dockerfile.odl-hybrid
image: weknora-odl-hybrid:local
container_name: WeKnora-odl-hybrid
profiles:
- odl-hybrid
ports:
- "${ODL_HYBRID_PORT:-5002}:5002"
environment:
# Default --no-ocr (digital PDFs). Scanned PDFs: use builtin OCR / MinerU, or
# ODL_HYBRID_EXTRA_ARGS="--force-ocr" (needs EasyOCR + libGL in image).
- ODL_HYBRID_EXTRA_ARGS=${ODL_HYBRID_EXTRA_ARGS:---no-ocr}
networks:
- WeKnora-network-dev
restart: unless-stopped
jaeger:
image: jaegertracing/all-in-one:latest
container_name: WeKnora-jaeger-dev