feat(parser): add OpenDataLoader, PaddleOCR-VL engines, and parser improvements

Introduce opendataloader and PaddleOCR-VL parser engines with tenant-level
settings UI, replace liteparse, and harden Excel/PPT/Markdown parsing.
Optional odl-hybrid sidecar stays local-build only and is excluded from
default dev-start and full profiles.
This commit is contained in:
wizardchen
2026-06-03 12:00:09 +08:00
committed by lyingbug
parent 7b1bb1054f
commit ef1047bf67
50 changed files with 4352 additions and 304 deletions
+24 -7
View File
@@ -241,6 +241,12 @@ services:
environment:
- DOCREADER_IMAGE_OUTPUT_DIR=/tmp/docreader
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
- DOCREADER_ODL_MAX_WORKERS=${DOCREADER_ODL_MAX_WORKERS:-1}
- DOCREADER_ODL_HYBRID=${DOCREADER_ODL_HYBRID:-off}
- DOCREADER_ODL_HYBRID_URL=${DOCREADER_ODL_HYBRID_URL:-http://odl-hybrid:5002}
- DOCREADER_ODL_HYBRID_MODE=${DOCREADER_ODL_HYBRID_MODE:-auto}
- DOCREADER_ODL_HYBRID_FALLBACK=${DOCREADER_ODL_HYBRID_FALLBACK:-false}
- DOCREADER_ODL_MARKDOWN_WITH_HTML=${DOCREADER_ODL_MARKDOWN_WITH_HTML:-false}
- DOCREADER_MARKITDOWN_MAX_WORKERS=${DOCREADER_MARKITDOWN_MAX_WORKERS:-1}
- DOCREADER_PDF_RENDER_MAX_WORKERS=${DOCREADER_PDF_RENDER_MAX_WORKERS:-1}
- DOCREADER_PDF_RENDER_DPI=${DOCREADER_PDF_RENDER_DPI:-200}
@@ -250,13 +256,6 @@ services:
- GRPC_TLS_KEY=${GRPC_TLS_KEY:-}
- GRPC_TLS_CA=${GRPC_TLS_CA:-}
- GRPC_AUTH_TOKEN=${GRPC_AUTH_TOKEN:-}
- OBS_ENDPOINT=${OBS_ENDPOINT:-}
- OBS_REGION=${OBS_REGION:-}
- OBS_ACCESS_KEY=${OBS_ACCESS_KEY:-}
- OBS_SECRET_KEY=${OBS_SECRET_KEY:-}
- OBS_BUCKET_NAME=${OBS_BUCKET_NAME:-}
- OBS_PATH_PREFIX=${OBS_PATH_PREFIX:-}
- OBS_PROXY_DOMAIN=${OBS_PROXY_DOMAIN:-}
healthcheck:
test: ["CMD", "grpc_health_probe", "-addr=localhost:50051"]
interval: 30s
@@ -269,6 +268,24 @@ services:
extra_hosts:
- "host.docker.internal:host-gateway"
# OpenDataLoader hybrid backend (optional). Default --no-ocr (no EasyOCR/libGL).
# Local build only — not published to Docker Hub; use --profile odl-hybrid --build.
odl-hybrid:
build:
context: .
dockerfile: docker/Dockerfile.odl-hybrid
image: weknora-odl-hybrid:local
container_name: WeKnora-odl-hybrid
profiles:
- odl-hybrid
expose:
- "5002"
environment:
- ODL_HYBRID_EXTRA_ARGS=${ODL_HYBRID_EXTRA_ARGS:---no-ocr}
networks:
- WeKnora-network
restart: unless-stopped
# 修改的PostgreSQL配置
postgres:
image: paradedb/paradedb:v0.22.2-pg17