From 88fd42cbc36b9efe5f3937258437aa78d518a40f Mon Sep 17 00:00:00 2001 From: begoniezhao Date: Fri, 16 Jan 2026 15:05:18 +0800 Subject: [PATCH] refactor: Restructure OCR module and centralize config --- docker-compose.dev.yml | 23 +- docker-compose.yml | 23 +- docreader/config.py | 284 ++++++++++++++++++ docreader/main.py | 53 +--- docreader/ocr/__init__.py | 37 +++ docreader/ocr/base.py | 31 ++ .../{parser/ocr_engine.py => ocr/paddle.py} | 163 +--------- docreader/ocr/vlm.py | 87 ++++++ docreader/parser/base_parser.py | 31 +- docreader/parser/caption.py | 20 +- docreader/parser/doc_parser.py | 3 +- docreader/parser/image_parser.py | 4 + docreader/parser/markdown_parser.py | 76 ++--- docreader/parser/mineru_parser.py | 33 +- docreader/parser/parser.py | 10 +- docreader/parser/storage.py | 54 ++-- docreader/parser/web_parser.py | 6 +- 17 files changed, 586 insertions(+), 352 deletions(-) create mode 100644 docreader/config.py create mode 100644 docreader/ocr/__init__.py create mode 100644 docreader/ocr/base.py rename docreader/{parser/ocr_engine.py => ocr/paddle.py} (55%) create mode 100644 docreader/ocr/vlm.py diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 7d166748..e974ae98 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -100,28 +100,11 @@ services: ports: - "${DOCREADER_PORT:-50051}:50051" environment: - - COS_SECRET_ID=${COS_SECRET_ID:-} - - COS_SECRET_KEY=${COS_SECRET_KEY:-} - - COS_REGION=${COS_REGION:-} - - COS_BUCKET_NAME=${COS_BUCKET_NAME:-} - - COS_APP_ID=${COS_APP_ID:-} - - COS_PATH_PREFIX=${COS_PATH_PREFIX:-} - - COS_ENABLE_OLD_DOMAIN=${COS_ENABLE_OLD_DOMAIN:-} - - VLM_MODEL_BASE_URL=${VLM_MODEL_BASE_URL:-} - - VLM_MODEL_NAME=${VLM_MODEL_NAME:-} - - VLM_MODEL_API_KEY=${VLM_MODEL_API_KEY:-} + - MINIO_ENDPOINT=minio:9000 - STORAGE_TYPE=${STORAGE_TYPE:-} - MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000} - - MINIO_ENDPOINT=minio:9000 - - MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin} - - MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin} - - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME:-} - - MINIO_USE_SSL=${MINIO_USE_SSL:-} - - WEB_PROXY=${WEB_PROXY:-} - # File size limit (in MB) - - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} - - IMAGE_MAX_CONCURRENT=${IMAGE_MAX_CONCURRENT:-1} - - OCR_BACKEND=${OCR_BACKEND:-paddle} + - MINERU_ENDPOINT=${MINERU_ENDPOINT:-} + - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}} healthcheck: test: ["CMD", "grpc_health_probe", "-addr=:50051"] interval: 30s diff --git a/docker-compose.yml b/docker-compose.yml index 14b28fe5..aba430b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -127,29 +127,10 @@ services: ports: - "${DOCREADER_PORT:-50051}:50051" environment: - - COS_SECRET_ID=${COS_SECRET_ID:-} - - COS_SECRET_KEY=${COS_SECRET_KEY:-} - - COS_REGION=${COS_REGION:-} - - COS_BUCKET_NAME=${COS_BUCKET_NAME:-} - - COS_APP_ID=${COS_APP_ID:-} - - COS_PATH_PREFIX=${COS_PATH_PREFIX:-} - - COS_ENABLE_OLD_DOMAIN=${COS_ENABLE_OLD_DOMAIN:-} - - VLM_MODEL_BASE_URL=${VLM_MODEL_BASE_URL:-} - - VLM_MODEL_NAME=${VLM_MODEL_NAME:-} - - VLM_MODEL_API_KEY=${VLM_MODEL_API_KEY:-} - - STORAGE_TYPE=${STORAGE_TYPE:-} - - MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000} - MINIO_ENDPOINT=minio:9000 - - MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin} - - MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin} - - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME:-} - - MINIO_USE_SSL=${MINIO_USE_SSL:-} - - WEB_PROXY=${WEB_PROXY:-} + - MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000} - MINERU_ENDPOINT=${MINERU_ENDPOINT:-} - # File size limit (in MB) - - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50} - - IMAGE_MAX_CONCURRENT=${IMAGE_MAX_CONCURRENT:-1} - - OCR_BACKEND=${OCR_BACKEND:-paddle} + - MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-} healthcheck: test: ["CMD", "grpc_health_probe", "-addr=:50051"] interval: 30s diff --git a/docreader/config.py b/docreader/config.py new file mode 100644 index 00000000..791363cb --- /dev/null +++ b/docreader/config.py @@ -0,0 +1,284 @@ +import logging +import os +from dataclasses import dataclass +from typing import Any, Dict, Iterable, Optional, Tuple + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def _get_first_env(keys: Iterable[str]) -> Tuple[Optional[str], Optional[str]]: + """Return (value, key) for the first existing env var in keys.""" + for k in keys: + if k in os.environ: + return os.environ.get(k), k + return None, None + + +def _get_str(keys: Iterable[str], default: str = "") -> str: + v, _ = _get_first_env(keys) + return default if v is None else str(v) + + +def _get_int(keys: Iterable[str], default: int) -> int: + v, _ = _get_first_env(keys) + if v is None or str(v).strip() == "": + return default + try: + return int(str(v).strip()) + except Exception: + return default + + +def _get_bool(keys: Iterable[str], default: bool) -> bool: + v, _ = _get_first_env(keys) + if v is None or str(v).strip() == "": + return default + return str(v).strip().lower() in {"1", "true", "yes", "y", "on"} + + +def _mask_secret(v: str) -> str: + if not v: + return "" + if len(v) <= 6: + return "***" + return f"{v[:2]}***{v[-2:]}" + + +@dataclass(frozen=True) +class DocReaderConfig: + # gRPC + grpc_max_workers: int + grpc_max_file_size_mb: int + grpc_port: int + + # Image processing + image_max_concurrent: int + + # Proxy + external_http_proxy: str + external_https_proxy: str + + # OCR + ocr_backend: str + ocr_api_base_url: str + ocr_api_key: str + ocr_model: str + + # VLM Caption + vlm_model_base_url: str + vlm_model_name: str + vlm_model_api_key: str + vlm_interface_type: str + + # Storage + storage_type: str + + cos_secret_id: str + cos_secret_key: str + cos_region: str + cos_bucket_name: str + cos_app_id: str + cos_path_prefix: str + cos_enable_old_domain: bool + + minio_access_key_id: str + minio_secret_access_key: str + minio_bucket_name: str + minio_path_prefix: str + minio_endpoint: str + minio_public_endpoint: str + minio_use_ssl: bool + + local_storage_base_dir: str + + # Other + mineru_endpoint: str + + +def load_config() -> DocReaderConfig: + """Load config from environment variables. + + Naming convention (new): DOCREADER_* + Backward compatible keys are supported. + """ + + # gRPC + grpc_max_workers = _get_int(["DOCREADER_GRPC_MAX_WORKERS", "GRPC_MAX_WORKERS"], 4) + grpc_max_file_size_mb = ( + _get_int(["DOCREADER_GRPC_MAX_FILE_SIZE_MB", "MAX_FILE_SIZE_MB"], 50) + * 1024 + * 1024 + ) + grpc_port = _get_int(["DOCREADER_GRPC_PORT", "PORT"], 50051) + + # Image processing + image_max_concurrent = _get_int( + ["DOCREADER_IMAGE_MAX_CONCURRENT", "IMAGE_MAX_CONCURRENT"], 1 + ) + + # Proxies + external_http_proxy = _get_str( + ["DOCREADER_EXTERNAL_HTTP_PROXY", "EXTERNAL_HTTP_PROXY"], "" + ) + external_https_proxy = _get_str( + ["DOCREADER_EXTERNAL_HTTPS_PROXY", "EXTERNAL_HTTPS_PROXY"], "" + ) + + # OCR + ocr_backend = _get_str(["DOCREADER_OCR_BACKEND", "OCR_BACKEND"], "") + ocr_api_base_url = _get_str(["DOCREADER_OCR_API_BASE_URL", "OCR_API_BASE_URL"], "") + ocr_api_key = _get_str(["DOCREADER_OCR_API_KEY", "OCR_API_KEY"], "") + ocr_model = _get_str(["DOCREADER_OCR_MODEL", "OCR_MODEL"], "") + + # VLM Caption + vlm_model_base_url = _get_str( + ["DOCREADER_VLM_MODEL_BASE_URL", "VLM_MODEL_BASE_URL"], "" + ) + vlm_model_name = _get_str(["DOCREADER_VLM_MODEL_NAME", "VLM_MODEL_NAME"], "") + vlm_model_api_key = _get_str( + ["DOCREADER_VLM_MODEL_API_KEY", "VLM_MODEL_API_KEY"], "" + ) + vlm_interface_type = _get_str( + ["DOCREADER_VLM_INTERFACE_TYPE", "VLM_INTERFACE_TYPE"], "openai" + ).lower() + + # Storage + storage_type = _get_str(["DOCREADER_STORAGE_TYPE", "STORAGE_TYPE"], "cos").lower() + + # COS + cos_secret_id = _get_str(["DOCREADER_COS_SECRET_ID", "COS_SECRET_ID"], "") + cos_secret_key = _get_str(["DOCREADER_COS_SECRET_KEY", "COS_SECRET_KEY"], "") + cos_region = _get_str(["DOCREADER_COS_REGION", "COS_REGION"], "") + cos_bucket_name = _get_str(["DOCREADER_COS_BUCKET_NAME", "COS_BUCKET_NAME"], "") + cos_app_id = _get_str(["DOCREADER_COS_APP_ID", "COS_APP_ID"], "") + cos_path_prefix = _get_str(["DOCREADER_COS_PATH_PREFIX", "COS_PATH_PREFIX"], "") + cos_enable_old_domain = _get_bool( + ["DOCREADER_COS_ENABLE_OLD_DOMAIN", "COS_ENABLE_OLD_DOMAIN"], True + ) + + # MinIO + minio_access_key_id = _get_str( + ["DOCREADER_MINIO_ACCESS_KEY_ID", "MINIO_ACCESS_KEY_ID"], "minioadmin" + ) + minio_secret_access_key = _get_str( + ["DOCREADER_MINIO_SECRET_ACCESS_KEY", "MINIO_SECRET_ACCESS_KEY"], "minioadmin" + ) + minio_bucket_name = _get_str( + ["DOCREADER_MINIO_BUCKET_NAME", "MINIO_BUCKET_NAME"], "WeKnora" + ) + minio_path_prefix = _get_str( + ["DOCREADER_MINIO_PATH_PREFIX", "MINIO_PATH_PREFIX"], "" + ) + minio_endpoint = _get_str(["DOCREADER_MINIO_ENDPOINT", "MINIO_ENDPOINT"], "") + minio_public_endpoint = _get_str( + ["DOCREADER_MINIO_PUBLIC_ENDPOINT", "MINIO_PUBLIC_ENDPOINT"], "" + ) + minio_use_ssl = _get_bool(["DOCREADER_MINIO_USE_SSL", "MINIO_USE_SSL"], False) + + # Local storage + local_storage_base_dir = "./data/files" + + # Other + mineru_endpoint = _get_str(["DOCREADER_MINERU_ENDPOINT", "MINERU_ENDPOINT"], "") + + return DocReaderConfig( + grpc_max_workers=grpc_max_workers, + grpc_max_file_size_mb=grpc_max_file_size_mb, + grpc_port=grpc_port, + image_max_concurrent=image_max_concurrent, + external_http_proxy=external_http_proxy, + external_https_proxy=external_https_proxy, + ocr_backend=ocr_backend, + ocr_api_base_url=ocr_api_base_url, + ocr_api_key=ocr_api_key, + ocr_model=ocr_model, + vlm_model_base_url=vlm_model_base_url, + vlm_model_name=vlm_model_name, + vlm_model_api_key=vlm_model_api_key, + vlm_interface_type=vlm_interface_type, + storage_type=storage_type, + cos_secret_id=cos_secret_id, + cos_secret_key=cos_secret_key, + cos_region=cos_region, + cos_bucket_name=cos_bucket_name, + cos_app_id=cos_app_id, + cos_path_prefix=cos_path_prefix, + cos_enable_old_domain=cos_enable_old_domain, + minio_access_key_id=minio_access_key_id, + minio_secret_access_key=minio_secret_access_key, + minio_bucket_name=minio_bucket_name, + minio_path_prefix=minio_path_prefix, + minio_endpoint=minio_endpoint, + minio_public_endpoint=minio_public_endpoint, + minio_use_ssl=minio_use_ssl, + local_storage_base_dir=local_storage_base_dir, + mineru_endpoint=mineru_endpoint, + ) + + +CONFIG = load_config() + + +def dump_config(mask_secrets: bool = True) -> Dict[str, Any]: + cfg = CONFIG + d: Dict[str, Any] = { + # gRPC + "DOCREADER_GRPC_MAX_WORKERS": cfg.grpc_max_workers, + "DOCREADER_GRPC_MAX_FILE_SIZE_MB": cfg.grpc_max_file_size_mb, + "DOCREADER_GRPC_PORT": cfg.grpc_port, + # Image processing + "DOCREADER_IMAGE_MAX_CONCURRENT": cfg.image_max_concurrent, + # Proxy + "DOCREADER_EXTERNAL_HTTP_PROXY": cfg.external_http_proxy, + "DOCREADER_EXTERNAL_HTTPS_PROXY": cfg.external_https_proxy, + # OCR + "DOCREADER_OCR_BACKEND": cfg.ocr_backend, + "DOCREADER_OCR_API_BASE_URL": cfg.ocr_api_base_url, + "DOCREADER_OCR_API_KEY": _mask_secret(cfg.ocr_api_key) + if mask_secrets + else cfg.ocr_api_key, + "DOCREADER_OCR_MODEL": cfg.ocr_model, + # VLM + "DOCREADER_VLM_MODEL_BASE_URL": cfg.vlm_model_base_url, + "DOCREADER_VLM_MODEL_NAME": cfg.vlm_model_name, + "DOCREADER_VLM_MODEL_API_KEY": _mask_secret(cfg.vlm_model_api_key) + if mask_secrets + else cfg.vlm_model_api_key, + "DOCREADER_VLM_INTERFACE_TYPE": cfg.vlm_interface_type, + # Storage + "DOCREADER_STORAGE_TYPE": cfg.storage_type, + "DOCREADER_COS_SECRET_ID": _mask_secret(cfg.cos_secret_id) + if mask_secrets + else cfg.cos_secret_id, + "DOCREADER_COS_SECRET_KEY": _mask_secret(cfg.cos_secret_key) + if mask_secrets + else cfg.cos_secret_key, + "DOCREADER_COS_REGION": cfg.cos_region, + "DOCREADER_COS_BUCKET_NAME": cfg.cos_bucket_name, + "DOCREADER_COS_APP_ID": cfg.cos_app_id, + "DOCREADER_COS_PATH_PREFIX": cfg.cos_path_prefix, + "DOCREADER_COS_ENABLE_OLD_DOMAIN": cfg.cos_enable_old_domain, + "DOCREADER_MINIO_ACCESS_KEY_ID": _mask_secret(cfg.minio_access_key_id) + if mask_secrets + else cfg.minio_access_key_id, + "DOCREADER_MINIO_SECRET_ACCESS_KEY": _mask_secret(cfg.minio_secret_access_key) + if mask_secrets + else cfg.minio_secret_access_key, + "DOCREADER_MINIO_BUCKET_NAME": cfg.minio_bucket_name, + "DOCREADER_MINIO_PATH_PREFIX": cfg.minio_path_prefix, + "DOCREADER_MINIO_ENDPOINT": cfg.minio_endpoint, + "DOCREADER_MINIO_PUBLIC_ENDPOINT": cfg.minio_public_endpoint, + "DOCREADER_MINIO_USE_SSL": cfg.minio_use_ssl, + "DOCREADER_LOCAL_STORAGE_BASE_DIR": cfg.local_storage_base_dir, + # Other + "DOCREADER_MINERU_ENDPOINT": cfg.mineru_endpoint, + } + return d + + +def print_config() -> None: + d = dump_config(mask_secrets=True) + logger.info("DocReader env/config (effective values):") + for k in sorted(d.keys()): + logger.info("%s=%s", k, d[k]) diff --git a/docreader/main.py b/docreader/main.py index 28c57cc7..2f2c9a20 100644 --- a/docreader/main.py +++ b/docreader/main.py @@ -11,9 +11,10 @@ import grpc from grpc_health.v1 import health_pb2_grpc from grpc_health.v1.health import HealthServicer +from docreader import config +from docreader.config import CONFIG from docreader.models.read_config import ChunkingConfig from docreader.parser import Parser -from docreader.parser.ocr_engine import OCREngine from docreader.proto import docreader_pb2_grpc from docreader.proto.docreader_pb2 import ( Chunk, @@ -22,6 +23,7 @@ from docreader.proto.docreader_pb2 import ( ReadFromFileRequest, ReadFromURLRequest, ReadResponse, + StorageProvider, ) from docreader.utils.request import init_logging_request_id, request_id_context @@ -58,23 +60,6 @@ logger.info("Initializing server logging") init_logging_request_id() -def get_max_message_length() -> int: - """Get max gRPC message length from environment variable. - Default is 50MB, can be configured via MAX_FILE_SIZE_MB. - """ - try: - size_mb = int(os.environ.get("MAX_FILE_SIZE_MB", "50")) - if size_mb > 0: - return size_mb * 1024 * 1024 - except ValueError: - pass - return 50 * 1024 * 1024 # default 50MB - - -# Set max message size (default 50MB, configurable via MAX_FILE_SIZE_MB) -MAX_MESSAGE_LENGTH = get_max_message_length() - - parser = Parser() @@ -104,7 +89,7 @@ def create_chunking_config(read_config: ReadConfig): # Extract storage config sc = read_config.storage_config storage_config = { - "provider": "minio" if sc.provider == 2 else "cos", + "provider": StorageProvider.Name(sc.provider), "region": sc.region, "bucket_name": sc.bucket_name, "access_key_id": sc.access_key_id, @@ -121,9 +106,10 @@ def create_chunking_config(read_config: ReadConfig): vlm_config = { "model_name": read_config.vlm_config.model_name, "base_url": read_config.vlm_config.base_url, - "api_key": read_config.vlm_config.api_key or "", + "api_key": read_config.vlm_config.api_key, "interface_type": read_config.vlm_config.interface_type or "openai", } + logger.info( f"Using VLM config: model={vlm_config['model_name']}, " f"base_url={vlm_config['base_url']}, " @@ -287,29 +273,16 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer): return proto_chunk -def init_ocr_engine(ocr_backend: Optional[str] = None, **kwargs): - """Initialize OCR engine""" - backend_type = ocr_backend or os.getenv("OCR_BACKEND", "paddle") - logger.info(f"Initializing OCR engine with backend: {backend_type}") - OCREngine.get_instance(backend_type=backend_type, **kwargs) - - def main(): - init_ocr_engine() - - # Set max number of worker threads - max_workers = int(os.environ.get("GRPC_MAX_WORKERS", "4")) - logger.info(f"Starting DocReader service with {max_workers} worker threads") - - # Get port number - port = os.environ.get("GRPC_PORT", "50051") + # Print effective env/config at startup + config.print_config() # Create server server = grpc.server( - futures.ThreadPoolExecutor(max_workers=max_workers), + futures.ThreadPoolExecutor(max_workers=CONFIG.grpc_max_workers), options=[ - ("grpc.max_send_message_length", MAX_MESSAGE_LENGTH), - ("grpc.max_receive_message_length", MAX_MESSAGE_LENGTH), + ("grpc.max_send_message_length", CONFIG.grpc_max_file_size_mb), + ("grpc.max_receive_message_length", CONFIG.grpc_max_file_size_mb), ], ) @@ -321,12 +294,12 @@ def main(): health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server) # Set listen address - server.add_insecure_port(f"[::]:{port}") + server.add_insecure_port(f"[::]:{CONFIG.grpc_port}") # Start service server.start() - logger.info(f"Server started on port {port}") + logger.info(f"Server started on port {CONFIG.grpc_port}") logger.info("Server is ready to accept connections") try: diff --git a/docreader/ocr/__init__.py b/docreader/ocr/__init__.py new file mode 100644 index 00000000..692dceb0 --- /dev/null +++ b/docreader/ocr/__init__.py @@ -0,0 +1,37 @@ +import logging +import threading +from typing import Dict + +from docreader.ocr.base import DummyOCRBackend, OCRBackend +from docreader.ocr.paddle import PaddleOCRBackend +from docreader.ocr.vlm import VLMOCRBackend + +logger = logging.getLogger(__name__) + + +class OCREngine: + """OCR Engine factory class for managing different OCR backend instances""" + + _instances: Dict[str, OCRBackend] = {} + _lock = threading.Lock() + + @classmethod + def get_instance(cls, backend_type: str) -> OCRBackend: + backend_type = (backend_type or "dummy").lower() + + with cls._lock: + inst = cls._instances.get(backend_type) + if inst is not None: + return inst + + logger.info(f"Creating OCR engine instance for backend: {backend_type}") + + if backend_type == "paddle": + inst = PaddleOCRBackend() + elif backend_type == "vlm": + inst = VLMOCRBackend() + else: + inst = DummyOCRBackend() + + cls._instances[backend_type] = inst + return inst diff --git a/docreader/ocr/base.py b/docreader/ocr/base.py new file mode 100644 index 00000000..4c6fb073 --- /dev/null +++ b/docreader/ocr/base.py @@ -0,0 +1,31 @@ +import logging +from abc import ABC, abstractmethod +from typing import Union + +from PIL import Image + +logger = logging.getLogger(__name__) + + +class OCRBackend(ABC): + """Base class for OCR backends""" + + @abstractmethod + def predict(self, image: Union[str, bytes, Image.Image]) -> str: + """Extract text from an image + + Args: + image: Image file path, bytes, or PIL Image object + + Returns: + Extracted text + """ + pass + + +class DummyOCRBackend(OCRBackend): + """Dummy OCR backend implementation""" + + def predict(self, image: Union[str, bytes, Image.Image]) -> str: + logger.warning("Dummy OCR backend is used") + return "" diff --git a/docreader/parser/ocr_engine.py b/docreader/ocr/paddle.py similarity index 55% rename from docreader/parser/ocr_engine.py rename to docreader/ocr/paddle.py index 7ca25cb6..249bebc2 100644 --- a/docreader/parser/ocr_engine.py +++ b/docreader/ocr/paddle.py @@ -3,42 +3,16 @@ import logging import os import platform import subprocess -from abc import ABC, abstractmethod -from typing import Dict, Union +from typing import Union import numpy as np -from openai import OpenAI from PIL import Image -from docreader.utils import endecode +from docreader.ocr.base import OCRBackend logger = logging.getLogger(__name__) -class OCRBackend(ABC): - """Base class for OCR backends""" - - @abstractmethod - def predict(self, image: Union[str, bytes, Image.Image]) -> str: - """Extract text from an image - - Args: - image: Image file path, bytes, or PIL Image object - - Returns: - Extracted text - """ - pass - - -class DummyOCRBackend(OCRBackend): - """Dummy OCR backend implementation""" - - def predict(self, image: Union[str, bytes, Image.Image]) -> str: - logger.warning("Dummy OCR backend is used") - return "" - - class PaddleOCRBackend(OCRBackend): """PaddleOCR backend implementation""" @@ -199,136 +173,3 @@ class PaddleOCRBackend(OCRBackend): except Exception as e: logger.error(f"OCR recognition error: {str(e)}") return "" - - -class NanonetsOCRBackend(OCRBackend): - """Nanonets OCR backend implementation using OpenAI API format""" - - def __init__(self): - """Initialize Nanonets OCR backend - - Args: - api_key: API key for OpenAI API - base_url: Base URL for OpenAI API - model: Model name - """ - # Load configuration from environment variables - base_url = os.getenv("OCR_API_BASE_URL", "http://localhost:8000/v1") - api_key = os.getenv("OCR_API_KEY", "123") - timeout = 30 - self.client = OpenAI(api_key=api_key, base_url=base_url, timeout=timeout) - - self.model = os.getenv("OCR_MODEL", "nanonets/Nanonets-OCR-s") - logger.info(f"Nanonets OCR engine initialized with model: {self.model}") - self.temperature = 0.0 - self.max_tokens = 15000 - # Prompt for OCR text extraction with specific formatting requirements - self.prompt = """## 任务说明 - -请从上传的文档中提取文字内容,严格按自然阅读顺序(从上到下,从左到右)输出,并遵循以下格式规范。 - -### 1. **文本处理** - -* 按正常阅读顺序提取文字,语句流畅自然。 - -### 2. **表格** - -* 所有表格统一转换为 **Markdown 表格格式**。 -* 内容保持清晰、对齐整齐,便于阅读。 - -### 3. **公式** - -* 所有公式转换为 **LaTeX 格式**,使用 `$$公式$$` 包裹。 - -### 4. **图片** - -* 忽略图片信息 - -### 5. **链接** - -* 不要猜测或补全不确定的链接地址。 -""" - - def predict(self, image: Union[str, bytes, Image.Image]) -> str: - """Extract text from an image using Nanonets OCR - - Args: - image: Image file path, bytes, or PIL Image object - - Returns: - Extracted text - """ - if self.client is None: - logger.error("Nanonets OCR client not initialized") - return "" - - try: - # Encode image to base64 format for API transmission - img_base64 = endecode.decode_image(image) - if not img_base64: - return "" - - # Call Nanonets OCR API using OpenAI-compatible format - logger.info(f"Calling Nanonets OCR API with model: {self.model}") - response = self.client.chat.completions.create( - model=self.model, - messages=[ - { - "role": "user", - "content": [ - { - "type": "image_url", - "image_url": { - "url": f"data:image/png;base64,{img_base64}" - }, - }, - { - "type": "text", - "text": self.prompt, - }, - ], - } - ], - temperature=self.temperature, - max_tokens=self.max_tokens, - ) - return response.choices[0].message.content or "" - except Exception as e: - logger.error(f"Nanonets OCR prediction error: {str(e)}") - return "" - - -class OCREngine: - """OCR Engine factory class for managing different OCR backend instances""" - - # Singleton pattern: cache instances for each backend type - _instance: Dict[str, OCRBackend] = {} - - @classmethod - def get_instance(cls, backend_type: str) -> OCRBackend: - """Get OCR engine instance using factory pattern - - Args: - backend_type: OCR backend type, one of: "paddle", "nanonets" - **kwargs: Additional arguments for the backend - - Returns: - OCR engine instance or None if initialization fails - """ - backend_type = backend_type.lower() - # Return cached instance if already initialized - if cls._instance.get(backend_type): - return cls._instance[backend_type] - - logger.info(f"Initializing OCR engine with backend: {backend_type}") - - if backend_type == "paddle": - cls._instance[backend_type] = PaddleOCRBackend() - - elif backend_type == "nanonets": - cls._instance[backend_type] = NanonetsOCRBackend() - - else: - cls._instance[backend_type] = DummyOCRBackend() - - return cls._instance[backend_type] diff --git a/docreader/ocr/vlm.py b/docreader/ocr/vlm.py new file mode 100644 index 00000000..9f4e25d0 --- /dev/null +++ b/docreader/ocr/vlm.py @@ -0,0 +1,87 @@ +import logging +from typing import Union + +from openai import OpenAI +from PIL import Image + +from docreader.config import CONFIG +from docreader.ocr.base import OCRBackend +from docreader.utils import endecode + +logger = logging.getLogger(__name__) + + +class VLMOCRBackend(OCRBackend): + """VLM OCR backend implementation using OpenAI API format""" + + def __init__(self): + """Initialize VLM OCR backend + + Args: + api_key: API key for OpenAI API + base_url: Base URL for OpenAI API + model: Model name + """ + self.model = CONFIG.ocr_model + self.client = OpenAI( + api_key=CONFIG.ocr_api_key, + base_url=CONFIG.ocr_api_base_url, + timeout=30, + ) + self.temperature = 0.0 + self.max_tokens = 5000 + + # Prompt for OCR text extraction with specific formatting requirements + self.prompt = "提取文档图片中正文的所有信息用markdown格式表示," + "其中页眉、页脚部分忽略," + "表格用html格式表达," + "文档中公式用latex格式表示," + "按照阅读顺序组织进行解析。" + + def predict(self, image: Union[str, bytes, Image.Image]) -> str: + """Extract text from an image using VLM OCR + + Args: + image: Image file path, bytes, or PIL Image object + + Returns: + Extracted text + """ + if self.client is None: + logger.error("VLM OCR client not initialized") + return "" + + try: + # Encode image to base64 format for API transmission + img_base64 = endecode.decode_image(image) + if not img_base64: + return "" + + # Call VLM OCR API using OpenAI-compatible format + logger.info(f"Calling VLM OCR API with model: {self.model}") + response = self.client.chat.completions.create( + model=self.model, + messages=[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{img_base64}" + }, + }, + { + "type": "text", + "text": self.prompt, + }, + ], + } + ], + temperature=self.temperature, + max_tokens=self.max_tokens, + ) + return response.choices[0].message.content or "" + except Exception as e: + logger.error(f"VLM OCR prediction error: {str(e)}") + return "" diff --git a/docreader/parser/base_parser.py b/docreader/parser/base_parser.py index f93778d6..30033987 100644 --- a/docreader/parser/base_parser.py +++ b/docreader/parser/base_parser.py @@ -13,10 +13,11 @@ from urllib.parse import urlparse import requests from PIL import Image +from docreader.config import CONFIG from docreader.models.document import Chunk, Document from docreader.models.read_config import ChunkingConfig +from docreader.ocr import OCREngine from docreader.parser.caption import Caption -from docreader.parser.ocr_engine import OCREngine from docreader.parser.storage import create_storage from docreader.splitter.splitter import TextSplitter from docreader.utils import endecode @@ -243,7 +244,7 @@ class BaseParser(ABC): return image async def process_image_async(self, image: Image.Image, image_url: str): - """Asynchronously process image: first perform OCR, then get caption + """Asynchronously process image: perform OCR only (caption is generated asynchronously by Go backend) Args: image: Image object (PIL.Image or numpy array) @@ -252,10 +253,12 @@ class BaseParser(ABC): Returns: tuple: (ocr_text, caption, image_url) - ocr_text: OCR extracted text - - caption: Image description (if OCR has text) or empty string + - caption: Always empty string (caption generation moved to async task in Go backend) - image_url: Image URL (if provided) """ - logger.info("Starting asynchronous image processing (OCR + optional caption)") + logger.info( + "Starting asynchronous image processing (OCR only, caption deferred to async task)" + ) # Resize image resized_image = self._resize_image_if_needed(image) @@ -744,13 +747,11 @@ class BaseParser(ABC): return img_url, img_url, None # Still need to get image object for OCR processing # Get proxy settings from environment variables - http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") - https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY") proxies = {} - if http_proxy: - proxies["http"] = http_proxy - if https_proxy: - proxies["https"] = https_proxy + if CONFIG.external_http_proxy: + proxies["http"] = CONFIG.external_http_proxy + if CONFIG.external_https_proxy: + proxies["https"] = CONFIG.external_https_proxy response = requests.get(img_url, timeout=5, proxies=proxies) if response.status_code == 200: @@ -793,13 +794,11 @@ class BaseParser(ABC): logger.error(f"URL failed validation check: {img_url}") return img_url, img_url, None # Get proxy settings from environment variables - http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY") - https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY") proxies = {} - if http_proxy: - proxies["http"] = http_proxy - if https_proxy: - proxies["https"] = https_proxy + if CONFIG.external_http_proxy: + proxies["http"] = CONFIG.external_http_proxy + if CONFIG.external_https_proxy: + proxies["https"] = CONFIG.external_https_proxy logger.info(f"Downloading image {img_url}, using proxy: {proxies}") response = requests.get(img_url, timeout=5, proxies=proxies) diff --git a/docreader/parser/caption.py b/docreader/parser/caption.py index 548381f9..52ae415c 100644 --- a/docreader/parser/caption.py +++ b/docreader/parser/caption.py @@ -1,6 +1,5 @@ import json import logging -import os import time from dataclasses import dataclass, field from typing import Dict, List, Optional, Union @@ -8,6 +7,8 @@ from typing import Dict, List, Optional, Union import ollama import requests +from docreader.config import CONFIG + logger = logging.getLogger(__name__) @@ -197,15 +198,10 @@ class Caption: self.interface_type = vlm_config.get("interface_type", "openai").lower() else: # Fall back to environment variables if config not provided - base_url = os.getenv("VLM_MODEL_BASE_URL") - model_name = os.getenv("VLM_MODEL_NAME") - if not base_url or not model_name: - logger.error("VLM_MODEL_BASE_URL or VLM_MODEL_NAME is not set") - return - self.completion_url = base_url + "/chat/completions" - self.model = model_name - self.api_key = os.getenv("VLM_MODEL_API_KEY", "") - self.interface_type = os.getenv("VLM_INTERFACE_TYPE", "openai").lower() + self.completion_url = CONFIG.vlm_model_base_url + "/chat/completions" + self.model = CONFIG.vlm_model_name + self.api_key = CONFIG.vlm_model_api_key + self.interface_type = CONFIG.vlm_interface_type # Validate interface type - must be either "ollama" or "openai" if self.interface_type not in ["ollama", "openai"]: @@ -259,7 +255,9 @@ class Caption: model=self.model, prompt="简单凝炼的描述图片的主要内容", images=[image_base64], # Pass base64 encoded image data - options={"temperature": 0.1}, # Low temperature for more deterministic output + options={ + "temperature": 0.1 + }, # Low temperature for more deterministic output stream=False, ) diff --git a/docreader/parser/doc_parser.py b/docreader/parser/doc_parser.py index 578305c8..69da4350 100644 --- a/docreader/parser/doc_parser.py +++ b/docreader/parser/doc_parser.py @@ -5,6 +5,7 @@ from typing import List, Optional import textract +from docreader.config import CONFIG from docreader.models.document import Document from docreader.parser.docx2_parser import Docx2Parser from docreader.utils.tempfile import TempDirContext, TempFileContext @@ -24,7 +25,7 @@ class SandboxExecutor: """ # Get proxy from parameter, environment variable, or use default blocking proxy # Use 'or None' to convert empty string to None, then apply default value - self.proxy = proxy or os.environ.get("WEB_PROXY") or "http://128.0.0.1:1" + self.proxy = proxy or CONFIG.external_https_proxy or "http://128.0.0.1:1" self.default_timeout = default_timeout def execute_in_sandbox(self, cmd: List[str]) -> tuple: diff --git a/docreader/parser/image_parser.py b/docreader/parser/image_parser.py index 5c054bcc..62f2fecf 100644 --- a/docreader/parser/image_parser.py +++ b/docreader/parser/image_parser.py @@ -36,6 +36,10 @@ class ImageParser(BaseParser): image_url = self.storage.upload_bytes(content, file_ext=ext) logger.info(f"Successfully uploaded image, URL: {image_url[:50]}...") + if not image_url: + logger.warning(f"Failed to upload image: {self.file_name}") + return Document(content=f"{self.file_name}", images={}) + # Generate markdown text text = f"![{self.file_name}]({image_url})" images = {image_url: base64.b64encode(content).decode()} diff --git a/docreader/parser/markdown_parser.py b/docreader/parser/markdown_parser.py index 7b979b99..38d7bdf0 100644 --- a/docreader/parser/markdown_parser.py +++ b/docreader/parser/markdown_parser.py @@ -10,6 +10,7 @@ This module provides comprehensive Markdown parsing functionality including: The parser uses a pipeline approach to process Markdown content through multiple stages: table formatting -> image processing. """ + import base64 import logging import os @@ -28,23 +29,23 @@ logger = logging.getLogger(__name__) class MarkdownTableUtil: """Utility class for formatting Markdown tables. - + This class standardizes Markdown table formatting by: - Normalizing column alignment markers (e.g., :---, :---:, ---:) - Adding consistent spacing around pipes (|) - Preserving indentation levels - Handling both header rows and data rows - + Example: Input: |姓名|年龄|城市| |:---|---:|:---:| |张三|25|北京| - + Output: | 姓名 | 年龄 | 城市 | | :--- | ---: | :---: | | 张三 | 25 | 北京 | """ - + def __init__(self): # Pattern to match alignment row (e.g., |:---|---:|:---:|) self.align_pattern = re.compile( @@ -59,14 +60,14 @@ class MarkdownTableUtil: def format_table(self, content: str) -> str: """Format all Markdown tables in the content. - + Args: content: Raw Markdown text containing tables - + Returns: Formatted Markdown text with standardized table formatting """ - + def process_align(match: Match[str]) -> str: """Process alignment row to standardize format.""" # Split by | and remove empty strings @@ -125,10 +126,10 @@ class MarkdownTableUtil: class MarkdownTableFormatter(BaseParser): """Parser for formatting Markdown tables. - + This parser standardizes the formatting of all Markdown tables in the document to ensure consistent spacing and alignment markers. - + Example: >>> formatter = MarkdownTableFormatter() >>> content = b"|Name|Age|\n|---|---|\n|John|30|" @@ -138,17 +139,17 @@ class MarkdownTableFormatter(BaseParser): | --- | --- | | John | 30 | """ - + def __init__(self, **kwargs): super().__init__(**kwargs) self.table_helper = MarkdownTableUtil() def parse_into_text(self, content: bytes) -> Document: """Parse and format Markdown tables. - + Args: content: Raw Markdown content as bytes - + Returns: Document with formatted table content """ @@ -161,18 +162,18 @@ class MarkdownTableFormatter(BaseParser): class MarkdownImageUtil: """Utility class for handling images in Markdown. - + This class provides functionality to: - Extract base64-encoded images from Markdown - Extract image paths from Markdown - Replace image paths with new URLs - Convert base64 images to binary format - + Supported formats: - Base64 embedded images: ![alt](data:image/png;base64,iVBORw0...) - Regular image links: ![alt](path/to/image.png) """ - + def __init__(self): # Pattern to match base64 embedded images # Captures: (1) alt text, (2) image format, (3) base64 data @@ -191,15 +192,15 @@ class MarkdownImageUtil: replace: bool = True, ) -> Tuple[str, List[str]]: """Extract image paths from Markdown content. - + Args: content: Markdown text containing images path_prefix: Optional prefix to add to image paths replace: Whether to replace image syntax in content - + Returns: Tuple of (processed_text, list_of_image_paths) - + Example: >>> util = MarkdownImageUtil() >>> text, images = util.extract_image("![logo](img/logo.png)") @@ -213,7 +214,7 @@ class MarkdownImageUtil: """Replacement function for each image match.""" title = match.group(1) # Alt text image_path = match.group(2) # Image path - + # Add prefix if specified if path_prefix: image_path = f"{path_prefix}/{image_path}" @@ -238,19 +239,19 @@ class MarkdownImageUtil: replace: bool = True, ) -> Tuple[str, Dict[str, bytes]]: """Extract and decode base64 embedded images from Markdown. - + This method finds all base64-encoded images in the Markdown content, decodes them to binary format, generates unique filenames, and optionally replaces them with file path references. - + Args: content: Markdown text containing base64 images path_prefix: Optional directory prefix for generated paths replace: Whether to replace base64 syntax with file paths - + Returns: Tuple of (processed_text, dict_of_path_to_bytes) - + Example: >>> util = MarkdownImageUtil() >>> text = "![logo](data:image/png;base64,iVBORw0KGg...)" @@ -294,17 +295,17 @@ class MarkdownImageUtil: def replace_path(self, content: str, images: Dict[str, str]) -> str: """Replace image paths in Markdown with new URLs. - + This method is typically used to replace local file paths with uploaded URLs after images have been stored. - + Args: content: Markdown text with image references images: Mapping of old paths to new URLs - + Returns: Markdown text with updated image URLs - + Example: >>> util = MarkdownImageUtil() >>> content = "![logo](temp/img.png)" @@ -320,7 +321,7 @@ class MarkdownImageUtil: """Replacement function for each image match.""" title = match.group(1) # Alt text image_path = match.group(2) # Current image path - + # Only replace if path exists in mapping if image_path not in images: return match.group(0) # Keep original @@ -328,7 +329,7 @@ class MarkdownImageUtil: content_replace.add(image_path) # Get new URL from mapping image_path = images[image_path] - return f"![{title}]({image_path})" + return f"![{title}]({image_path})" if image_path else title text = self.replace_pattern.sub(repl, content) logger.debug(f"Replaced {len(content_replace)} images in markdown") @@ -348,16 +349,16 @@ class MarkdownImageUtil: class MarkdownImageBase64(BaseParser): """Parser for extracting and uploading base64 images from Markdown. - + This parser: 1. Extracts base64-encoded images from Markdown content 2. Uploads the decoded images to storage 3. Replaces base64 data with uploaded URLs 4. Returns a Document with clean Markdown and image mappings - + Requires: - self.storage: Storage backend for uploading images - + Example: >>> parser = MarkdownImageBase64(storage=my_storage) >>> content = b"![logo](data:image/png;base64,iVBORw0...)" @@ -365,17 +366,17 @@ class MarkdownImageBase64(BaseParser): >>> print(doc.content) ![logo](https://storage.com/uuid.png) """ - + def __init__(self, **kwargs): super().__init__(**kwargs) self.image_helper = MarkdownImageUtil() def parse_into_text(self, content: bytes) -> Document: """Parse Markdown and process base64 images. - + Args: content: Raw Markdown content as bytes - + Returns: Document with: - content: Markdown with base64 replaced by URLs @@ -411,14 +412,15 @@ class MarkdownImageBase64(BaseParser): class MarkdownParser(PipelineParser): """Complete Markdown parser using pipeline approach. - + This parser processes Markdown content through multiple stages: 1. MarkdownTableFormatter: Standardizes table formatting 2. MarkdownImageBase64: Extracts and uploads base64 images - + The pipeline ensures that content flows through each parser in sequence, with each stage's output becoming the next stage's input. """ + _parser_cls = (MarkdownTableFormatter, MarkdownImageBase64) diff --git a/docreader/parser/mineru_parser.py b/docreader/parser/mineru_parser.py index c7eba34e..5ab6683d 100644 --- a/docreader/parser/mineru_parser.py +++ b/docreader/parser/mineru_parser.py @@ -1,11 +1,11 @@ import logging -import os import re from typing import Dict import markdownify import requests +from docreader.config import CONFIG from docreader.models.document import Document from docreader.parser.base_parser import BaseParser from docreader.parser.chain_parser import PipelineParser @@ -18,19 +18,19 @@ logger = logging.getLogger(__name__) class StdMinerUParser(BaseParser): """ Standard MinerU Parser for document parsing. - + This parser uses MinerU API to parse documents (especially PDFs) into markdown format, with support for tables, formulas, and images extraction. """ + def __init__( self, enable_markdownify: bool = True, - mineru_endpoint: str = "", **kwargs, ): """ Initialize MinerU parser. - + Args: enable_markdownify: Whether to convert HTML tables to markdown format mineru_endpoint: MinerU API endpoint URL @@ -38,7 +38,7 @@ class StdMinerUParser(BaseParser): """ super().__init__(**kwargs) # Get MinerU endpoint from environment variable or parameter - self.minerU = os.getenv("MINERU_ENDPOINT", mineru_endpoint) + self.minerU = CONFIG.mineru_endpoint self.enable_markdownify = enable_markdownify # Helper for processing markdown images self.image_helper = MarkdownImageUtil() @@ -50,10 +50,10 @@ class StdMinerUParser(BaseParser): def ping(self, timeout: int = 5) -> bool: """ Check if MinerU API is available. - + Args: timeout: Request timeout in seconds - + Returns: True if API is available, False otherwise """ @@ -69,10 +69,10 @@ class StdMinerUParser(BaseParser): def parse_into_text(self, content: bytes) -> Document: """ Parse document content into text using MinerU API. - + Args: content: Raw document content in bytes - + Returns: Document object containing parsed text and images """ @@ -165,24 +165,27 @@ class StdMinerUParser(BaseParser): class MinerUParser(PipelineParser): """ MinerU Parser with pipeline processing. - - This parser combines StdMinerUParser for document parsing and + + This parser combines StdMinerUParser for document parsing and MarkdownTableFormatter for table formatting in a pipeline. """ + _parser_cls = (StdMinerUParser, MarkdownTableFormatter) if __name__ == "__main__": + import os + # Example usage for testing logging.basicConfig(level=logging.DEBUG) # Configure your file path and MinerU endpoint your_file = "/path/to/your/file.pdf" - your_mineru = "http://host.docker.internal:9987" - + os.environ["MINERU_ENDPOINT"] = "http://host.docker.internal:9987" + # Create parser instance - parser = MinerUParser(mineru_endpoint=your_mineru) - + parser = MinerUParser() + # Parse PDF file with open(your_file, "rb") as f: content = f.read() diff --git a/docreader/parser/parser.py b/docreader/parser/parser.py index 20436485..3db36fd7 100644 --- a/docreader/parser/parser.py +++ b/docreader/parser/parser.py @@ -1,7 +1,7 @@ import logging -import os from typing import Dict, Type +from docreader.config import CONFIG from docreader.models.document import Document from docreader.models.read_config import ChunkingConfig from docreader.parser.base_parser import BaseParser @@ -111,8 +111,8 @@ class Parser: enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing max_image_size=1920, # Limit image size to 1920px for performance chunking_config=config, # Pass the entire chunking config for advanced options - max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")), - ocr_backend=os.getenv("OCR_BACKEND", "paddle"), + max_concurrent_tasks=CONFIG.image_max_concurrent, + ocr_backend=CONFIG.ocr_backend, ) logger.info(f"Starting to parse file content, size: {len(content)} bytes") @@ -158,8 +158,8 @@ class Parser: enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing max_image_size=1920, # Limit image size to 1920px for performance chunking_config=config, # Pass the entire chunking config - max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")), - ocr_backend=os.getenv("OCR_BACKEND", "paddle"), + max_concurrent_tasks=CONFIG.image_max_concurrent, + ocr_backend=CONFIG.ocr_backend, ) logger.info("Starting to parse URL content") diff --git a/docreader/parser/storage.py b/docreader/parser/storage.py index dd9f51ad..20ab944c 100644 --- a/docreader/parser/storage.py +++ b/docreader/parser/storage.py @@ -10,6 +10,7 @@ from typing import Dict from minio import Minio from qcloud_cos import CosConfig, CosS3Client +from docreader.config import CONFIG from docreader.utils import endecode logger = logging.getLogger(__name__) @@ -73,16 +74,14 @@ class CosStorage(Storage): prefix = cos_config.get("path_prefix", "") else: # Get COS configuration from environment variables - secret_id = os.getenv("COS_SECRET_ID") - secret_key = os.getenv("COS_SECRET_KEY") - region = os.getenv("COS_REGION") - bucket_name = os.getenv("COS_BUCKET_NAME") - appid = os.getenv("COS_APP_ID") - prefix = os.getenv("COS_PATH_PREFIX") + secret_id = CONFIG.cos_secret_id + secret_key = CONFIG.cos_secret_key + region = CONFIG.cos_region + bucket_name = CONFIG.cos_bucket_name + appid = CONFIG.cos_app_id + prefix = CONFIG.cos_path_prefix - enable_old_domain = ( - os.getenv("COS_ENABLE_OLD_DOMAIN", "true").lower() == "true" - ) + enable_old_domain = CONFIG.cos_enable_old_domain if not all([secret_id, secret_key, region, bucket_name, appid]): logger.error( @@ -220,27 +219,27 @@ class MinioStorage(Storage): access_key = ( self.storage_config.get("access_key_id") if self.storage_config and self.storage_config.get("access_key_id") - else os.getenv("MINIO_ACCESS_KEY_ID") + else CONFIG.minio_access_key_id ) secret_key = ( self.storage_config.get("secret_access_key") if self.storage_config and self.storage_config.get("secret_access_key") - else os.getenv("MINIO_SECRET_ACCESS_KEY") + else CONFIG.minio_secret_access_key ) bucket_name = ( self.storage_config.get("bucket_name") if self.storage_config and self.storage_config.get("bucket_name") - else os.getenv("MINIO_BUCKET_NAME", "") + else CONFIG.minio_bucket_name ) path_prefix_raw = ( self.storage_config.get("path_prefix") if self.storage_config and self.storage_config.get("path_prefix") - else os.getenv("MINIO_PATH_PREFIX", "") + else CONFIG.minio_path_prefix ) path_prefix = path_prefix_raw.strip().strip("/") if path_prefix_raw else "" - endpoint = os.getenv("MINIO_ENDPOINT", "") - use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true" + endpoint = CONFIG.minio_endpoint + use_ssl = CONFIG.minio_use_ssl if not all([endpoint, access_key, secret_key, bucket_name]): logger.error( @@ -283,7 +282,7 @@ class MinioStorage(Storage): If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint. """ # 1. Use public endpoint if provided - endpoint = os.getenv("MINIO_PUBLIC_ENDPOINT") + endpoint = CONFIG.minio_public_endpoint if endpoint: return f"{endpoint}/{self.bucket_name}/{object_key}" @@ -381,9 +380,7 @@ class LocalStorage(Storage): def __init__(self, storage_config: Dict[str, str] = {}): self.storage_config = storage_config - base_dir = storage_config.get( - "base_dir", os.getenv("LOCAL_STORAGE_BASE_DIR", "") - ) + base_dir = storage_config.get("base_dir", CONFIG.local_storage_base_dir) self.image_dir = os.path.join(base_dir, "images") os.makedirs(self.image_dir, exist_ok=True) @@ -410,6 +407,20 @@ class Base64Storage(Storage): return f"data:image/{file_ext};base64,{endecode.decode_image(content)}" +class DummyStorage(Storage): + """Dummy storage implementation. + + It is used in tests or in environments where object storage is disabled. + All upload methods return empty string. + """ + + def upload_file(self, file_path: str) -> str: + return "" + + def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str: + return "" + + def create_storage(storage_config: Dict[str, str] | None = None) -> Storage: """Create a storage instance based on configuration or environment variables @@ -419,7 +430,7 @@ def create_storage(storage_config: Dict[str, str] | None = None) -> Storage: Returns: Storage instance """ - storage_type = os.getenv("STORAGE_TYPE", "cos").lower() + storage_type = CONFIG.storage_type if storage_config: storage_type = str(storage_config.get("provider", storage_type)).lower() logger.info(f"Creating {storage_type} storage instance") @@ -432,5 +443,4 @@ def create_storage(storage_config: Dict[str, str] | None = None) -> Storage: return LocalStorage(storage_config or {}) elif storage_type == "base64": return Base64Storage() - - raise ValueError(f"Invalid storage type: {storage_type}") + return DummyStorage() diff --git a/docreader/parser/web_parser.py b/docreader/parser/web_parser.py index 48e8f24c..100399d6 100644 --- a/docreader/parser/web_parser.py +++ b/docreader/parser/web_parser.py @@ -1,10 +1,10 @@ import asyncio import logging -import os from playwright.async_api import async_playwright from trafilatura import extract +from docreader.config import CONFIG from docreader.models.document import Document from docreader.parser.base_parser import BaseParser from docreader.parser.chain_parser import PipelineParser @@ -30,8 +30,8 @@ class StdWebParser(BaseParser): **kwargs: Additional arguments passed to BaseParser """ self.title = title - # Get proxy configuration from environment variable if available - self.proxy = os.environ.get("WEB_PROXY", "") + # Get proxy configuration from config if available + self.proxy = CONFIG.external_https_proxy super().__init__(file_name=title, **kwargs) logger.info(f"Initialized WebParser with title: {title}")