refactor: Restructure OCR module and centralize config

This commit is contained in:
begoniezhao
2026-01-16 15:05:18 +08:00
parent 74bf76eb3f
commit 88fd42cbc3
17 changed files with 586 additions and 352 deletions
+3 -20
View File
@@ -100,28 +100,11 @@ services:
ports:
- "${DOCREADER_PORT:-50051}:50051"
environment:
- COS_SECRET_ID=${COS_SECRET_ID:-}
- COS_SECRET_KEY=${COS_SECRET_KEY:-}
- COS_REGION=${COS_REGION:-}
- COS_BUCKET_NAME=${COS_BUCKET_NAME:-}
- COS_APP_ID=${COS_APP_ID:-}
- COS_PATH_PREFIX=${COS_PATH_PREFIX:-}
- COS_ENABLE_OLD_DOMAIN=${COS_ENABLE_OLD_DOMAIN:-}
- VLM_MODEL_BASE_URL=${VLM_MODEL_BASE_URL:-}
- VLM_MODEL_NAME=${VLM_MODEL_NAME:-}
- VLM_MODEL_API_KEY=${VLM_MODEL_API_KEY:-}
- MINIO_ENDPOINT=minio:9000
- STORAGE_TYPE=${STORAGE_TYPE:-}
- MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000}
- MINIO_ENDPOINT=minio:9000
- MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin}
- MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin}
- MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME:-}
- MINIO_USE_SSL=${MINIO_USE_SSL:-}
- WEB_PROXY=${WEB_PROXY:-}
# File size limit (in MB)
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50}
- IMAGE_MAX_CONCURRENT=${IMAGE_MAX_CONCURRENT:-1}
- OCR_BACKEND=${OCR_BACKEND:-paddle}
- MINERU_ENDPOINT=${MINERU_ENDPOINT:-}
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}}
healthcheck:
test: ["CMD", "grpc_health_probe", "-addr=:50051"]
interval: 30s
+2 -21
View File
@@ -127,29 +127,10 @@ services:
ports:
- "${DOCREADER_PORT:-50051}:50051"
environment:
- COS_SECRET_ID=${COS_SECRET_ID:-}
- COS_SECRET_KEY=${COS_SECRET_KEY:-}
- COS_REGION=${COS_REGION:-}
- COS_BUCKET_NAME=${COS_BUCKET_NAME:-}
- COS_APP_ID=${COS_APP_ID:-}
- COS_PATH_PREFIX=${COS_PATH_PREFIX:-}
- COS_ENABLE_OLD_DOMAIN=${COS_ENABLE_OLD_DOMAIN:-}
- VLM_MODEL_BASE_URL=${VLM_MODEL_BASE_URL:-}
- VLM_MODEL_NAME=${VLM_MODEL_NAME:-}
- VLM_MODEL_API_KEY=${VLM_MODEL_API_KEY:-}
- STORAGE_TYPE=${STORAGE_TYPE:-}
- MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000}
- MINIO_ENDPOINT=minio:9000
- MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin}
- MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin}
- MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME:-}
- MINIO_USE_SSL=${MINIO_USE_SSL:-}
- WEB_PROXY=${WEB_PROXY:-}
- MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000}
- MINERU_ENDPOINT=${MINERU_ENDPOINT:-}
# File size limit (in MB)
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-50}
- IMAGE_MAX_CONCURRENT=${IMAGE_MAX_CONCURRENT:-1}
- OCR_BACKEND=${OCR_BACKEND:-paddle}
- MAX_FILE_SIZE_MB=${MAX_FILE_SIZE_MB:-}
healthcheck:
test: ["CMD", "grpc_health_probe", "-addr=:50051"]
interval: 30s
+284
View File
@@ -0,0 +1,284 @@
import logging
import os
from dataclasses import dataclass
from typing import Any, Dict, Iterable, Optional, Tuple
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
def _get_first_env(keys: Iterable[str]) -> Tuple[Optional[str], Optional[str]]:
"""Return (value, key) for the first existing env var in keys."""
for k in keys:
if k in os.environ:
return os.environ.get(k), k
return None, None
def _get_str(keys: Iterable[str], default: str = "") -> str:
v, _ = _get_first_env(keys)
return default if v is None else str(v)
def _get_int(keys: Iterable[str], default: int) -> int:
v, _ = _get_first_env(keys)
if v is None or str(v).strip() == "":
return default
try:
return int(str(v).strip())
except Exception:
return default
def _get_bool(keys: Iterable[str], default: bool) -> bool:
v, _ = _get_first_env(keys)
if v is None or str(v).strip() == "":
return default
return str(v).strip().lower() in {"1", "true", "yes", "y", "on"}
def _mask_secret(v: str) -> str:
if not v:
return ""
if len(v) <= 6:
return "***"
return f"{v[:2]}***{v[-2:]}"
@dataclass(frozen=True)
class DocReaderConfig:
# gRPC
grpc_max_workers: int
grpc_max_file_size_mb: int
grpc_port: int
# Image processing
image_max_concurrent: int
# Proxy
external_http_proxy: str
external_https_proxy: str
# OCR
ocr_backend: str
ocr_api_base_url: str
ocr_api_key: str
ocr_model: str
# VLM Caption
vlm_model_base_url: str
vlm_model_name: str
vlm_model_api_key: str
vlm_interface_type: str
# Storage
storage_type: str
cos_secret_id: str
cos_secret_key: str
cos_region: str
cos_bucket_name: str
cos_app_id: str
cos_path_prefix: str
cos_enable_old_domain: bool
minio_access_key_id: str
minio_secret_access_key: str
minio_bucket_name: str
minio_path_prefix: str
minio_endpoint: str
minio_public_endpoint: str
minio_use_ssl: bool
local_storage_base_dir: str
# Other
mineru_endpoint: str
def load_config() -> DocReaderConfig:
"""Load config from environment variables.
Naming convention (new): DOCREADER_*
Backward compatible keys are supported.
"""
# gRPC
grpc_max_workers = _get_int(["DOCREADER_GRPC_MAX_WORKERS", "GRPC_MAX_WORKERS"], 4)
grpc_max_file_size_mb = (
_get_int(["DOCREADER_GRPC_MAX_FILE_SIZE_MB", "MAX_FILE_SIZE_MB"], 50)
* 1024
* 1024
)
grpc_port = _get_int(["DOCREADER_GRPC_PORT", "PORT"], 50051)
# Image processing
image_max_concurrent = _get_int(
["DOCREADER_IMAGE_MAX_CONCURRENT", "IMAGE_MAX_CONCURRENT"], 1
)
# Proxies
external_http_proxy = _get_str(
["DOCREADER_EXTERNAL_HTTP_PROXY", "EXTERNAL_HTTP_PROXY"], ""
)
external_https_proxy = _get_str(
["DOCREADER_EXTERNAL_HTTPS_PROXY", "EXTERNAL_HTTPS_PROXY"], ""
)
# OCR
ocr_backend = _get_str(["DOCREADER_OCR_BACKEND", "OCR_BACKEND"], "")
ocr_api_base_url = _get_str(["DOCREADER_OCR_API_BASE_URL", "OCR_API_BASE_URL"], "")
ocr_api_key = _get_str(["DOCREADER_OCR_API_KEY", "OCR_API_KEY"], "")
ocr_model = _get_str(["DOCREADER_OCR_MODEL", "OCR_MODEL"], "")
# VLM Caption
vlm_model_base_url = _get_str(
["DOCREADER_VLM_MODEL_BASE_URL", "VLM_MODEL_BASE_URL"], ""
)
vlm_model_name = _get_str(["DOCREADER_VLM_MODEL_NAME", "VLM_MODEL_NAME"], "")
vlm_model_api_key = _get_str(
["DOCREADER_VLM_MODEL_API_KEY", "VLM_MODEL_API_KEY"], ""
)
vlm_interface_type = _get_str(
["DOCREADER_VLM_INTERFACE_TYPE", "VLM_INTERFACE_TYPE"], "openai"
).lower()
# Storage
storage_type = _get_str(["DOCREADER_STORAGE_TYPE", "STORAGE_TYPE"], "cos").lower()
# COS
cos_secret_id = _get_str(["DOCREADER_COS_SECRET_ID", "COS_SECRET_ID"], "")
cos_secret_key = _get_str(["DOCREADER_COS_SECRET_KEY", "COS_SECRET_KEY"], "")
cos_region = _get_str(["DOCREADER_COS_REGION", "COS_REGION"], "")
cos_bucket_name = _get_str(["DOCREADER_COS_BUCKET_NAME", "COS_BUCKET_NAME"], "")
cos_app_id = _get_str(["DOCREADER_COS_APP_ID", "COS_APP_ID"], "")
cos_path_prefix = _get_str(["DOCREADER_COS_PATH_PREFIX", "COS_PATH_PREFIX"], "")
cos_enable_old_domain = _get_bool(
["DOCREADER_COS_ENABLE_OLD_DOMAIN", "COS_ENABLE_OLD_DOMAIN"], True
)
# MinIO
minio_access_key_id = _get_str(
["DOCREADER_MINIO_ACCESS_KEY_ID", "MINIO_ACCESS_KEY_ID"], "minioadmin"
)
minio_secret_access_key = _get_str(
["DOCREADER_MINIO_SECRET_ACCESS_KEY", "MINIO_SECRET_ACCESS_KEY"], "minioadmin"
)
minio_bucket_name = _get_str(
["DOCREADER_MINIO_BUCKET_NAME", "MINIO_BUCKET_NAME"], "WeKnora"
)
minio_path_prefix = _get_str(
["DOCREADER_MINIO_PATH_PREFIX", "MINIO_PATH_PREFIX"], ""
)
minio_endpoint = _get_str(["DOCREADER_MINIO_ENDPOINT", "MINIO_ENDPOINT"], "")
minio_public_endpoint = _get_str(
["DOCREADER_MINIO_PUBLIC_ENDPOINT", "MINIO_PUBLIC_ENDPOINT"], ""
)
minio_use_ssl = _get_bool(["DOCREADER_MINIO_USE_SSL", "MINIO_USE_SSL"], False)
# Local storage
local_storage_base_dir = "./data/files"
# Other
mineru_endpoint = _get_str(["DOCREADER_MINERU_ENDPOINT", "MINERU_ENDPOINT"], "")
return DocReaderConfig(
grpc_max_workers=grpc_max_workers,
grpc_max_file_size_mb=grpc_max_file_size_mb,
grpc_port=grpc_port,
image_max_concurrent=image_max_concurrent,
external_http_proxy=external_http_proxy,
external_https_proxy=external_https_proxy,
ocr_backend=ocr_backend,
ocr_api_base_url=ocr_api_base_url,
ocr_api_key=ocr_api_key,
ocr_model=ocr_model,
vlm_model_base_url=vlm_model_base_url,
vlm_model_name=vlm_model_name,
vlm_model_api_key=vlm_model_api_key,
vlm_interface_type=vlm_interface_type,
storage_type=storage_type,
cos_secret_id=cos_secret_id,
cos_secret_key=cos_secret_key,
cos_region=cos_region,
cos_bucket_name=cos_bucket_name,
cos_app_id=cos_app_id,
cos_path_prefix=cos_path_prefix,
cos_enable_old_domain=cos_enable_old_domain,
minio_access_key_id=minio_access_key_id,
minio_secret_access_key=minio_secret_access_key,
minio_bucket_name=minio_bucket_name,
minio_path_prefix=minio_path_prefix,
minio_endpoint=minio_endpoint,
minio_public_endpoint=minio_public_endpoint,
minio_use_ssl=minio_use_ssl,
local_storage_base_dir=local_storage_base_dir,
mineru_endpoint=mineru_endpoint,
)
CONFIG = load_config()
def dump_config(mask_secrets: bool = True) -> Dict[str, Any]:
cfg = CONFIG
d: Dict[str, Any] = {
# gRPC
"DOCREADER_GRPC_MAX_WORKERS": cfg.grpc_max_workers,
"DOCREADER_GRPC_MAX_FILE_SIZE_MB": cfg.grpc_max_file_size_mb,
"DOCREADER_GRPC_PORT": cfg.grpc_port,
# Image processing
"DOCREADER_IMAGE_MAX_CONCURRENT": cfg.image_max_concurrent,
# Proxy
"DOCREADER_EXTERNAL_HTTP_PROXY": cfg.external_http_proxy,
"DOCREADER_EXTERNAL_HTTPS_PROXY": cfg.external_https_proxy,
# OCR
"DOCREADER_OCR_BACKEND": cfg.ocr_backend,
"DOCREADER_OCR_API_BASE_URL": cfg.ocr_api_base_url,
"DOCREADER_OCR_API_KEY": _mask_secret(cfg.ocr_api_key)
if mask_secrets
else cfg.ocr_api_key,
"DOCREADER_OCR_MODEL": cfg.ocr_model,
# VLM
"DOCREADER_VLM_MODEL_BASE_URL": cfg.vlm_model_base_url,
"DOCREADER_VLM_MODEL_NAME": cfg.vlm_model_name,
"DOCREADER_VLM_MODEL_API_KEY": _mask_secret(cfg.vlm_model_api_key)
if mask_secrets
else cfg.vlm_model_api_key,
"DOCREADER_VLM_INTERFACE_TYPE": cfg.vlm_interface_type,
# Storage
"DOCREADER_STORAGE_TYPE": cfg.storage_type,
"DOCREADER_COS_SECRET_ID": _mask_secret(cfg.cos_secret_id)
if mask_secrets
else cfg.cos_secret_id,
"DOCREADER_COS_SECRET_KEY": _mask_secret(cfg.cos_secret_key)
if mask_secrets
else cfg.cos_secret_key,
"DOCREADER_COS_REGION": cfg.cos_region,
"DOCREADER_COS_BUCKET_NAME": cfg.cos_bucket_name,
"DOCREADER_COS_APP_ID": cfg.cos_app_id,
"DOCREADER_COS_PATH_PREFIX": cfg.cos_path_prefix,
"DOCREADER_COS_ENABLE_OLD_DOMAIN": cfg.cos_enable_old_domain,
"DOCREADER_MINIO_ACCESS_KEY_ID": _mask_secret(cfg.minio_access_key_id)
if mask_secrets
else cfg.minio_access_key_id,
"DOCREADER_MINIO_SECRET_ACCESS_KEY": _mask_secret(cfg.minio_secret_access_key)
if mask_secrets
else cfg.minio_secret_access_key,
"DOCREADER_MINIO_BUCKET_NAME": cfg.minio_bucket_name,
"DOCREADER_MINIO_PATH_PREFIX": cfg.minio_path_prefix,
"DOCREADER_MINIO_ENDPOINT": cfg.minio_endpoint,
"DOCREADER_MINIO_PUBLIC_ENDPOINT": cfg.minio_public_endpoint,
"DOCREADER_MINIO_USE_SSL": cfg.minio_use_ssl,
"DOCREADER_LOCAL_STORAGE_BASE_DIR": cfg.local_storage_base_dir,
# Other
"DOCREADER_MINERU_ENDPOINT": cfg.mineru_endpoint,
}
return d
def print_config() -> None:
d = dump_config(mask_secrets=True)
logger.info("DocReader env/config (effective values):")
for k in sorted(d.keys()):
logger.info("%s=%s", k, d[k])
+13 -40
View File
@@ -11,9 +11,10 @@ import grpc
from grpc_health.v1 import health_pb2_grpc
from grpc_health.v1.health import HealthServicer
from docreader import config
from docreader.config import CONFIG
from docreader.models.read_config import ChunkingConfig
from docreader.parser import Parser
from docreader.parser.ocr_engine import OCREngine
from docreader.proto import docreader_pb2_grpc
from docreader.proto.docreader_pb2 import (
Chunk,
@@ -22,6 +23,7 @@ from docreader.proto.docreader_pb2 import (
ReadFromFileRequest,
ReadFromURLRequest,
ReadResponse,
StorageProvider,
)
from docreader.utils.request import init_logging_request_id, request_id_context
@@ -58,23 +60,6 @@ logger.info("Initializing server logging")
init_logging_request_id()
def get_max_message_length() -> int:
"""Get max gRPC message length from environment variable.
Default is 50MB, can be configured via MAX_FILE_SIZE_MB.
"""
try:
size_mb = int(os.environ.get("MAX_FILE_SIZE_MB", "50"))
if size_mb > 0:
return size_mb * 1024 * 1024
except ValueError:
pass
return 50 * 1024 * 1024 # default 50MB
# Set max message size (default 50MB, configurable via MAX_FILE_SIZE_MB)
MAX_MESSAGE_LENGTH = get_max_message_length()
parser = Parser()
@@ -104,7 +89,7 @@ def create_chunking_config(read_config: ReadConfig):
# Extract storage config
sc = read_config.storage_config
storage_config = {
"provider": "minio" if sc.provider == 2 else "cos",
"provider": StorageProvider.Name(sc.provider),
"region": sc.region,
"bucket_name": sc.bucket_name,
"access_key_id": sc.access_key_id,
@@ -121,9 +106,10 @@ def create_chunking_config(read_config: ReadConfig):
vlm_config = {
"model_name": read_config.vlm_config.model_name,
"base_url": read_config.vlm_config.base_url,
"api_key": read_config.vlm_config.api_key or "",
"api_key": read_config.vlm_config.api_key,
"interface_type": read_config.vlm_config.interface_type or "openai",
}
logger.info(
f"Using VLM config: model={vlm_config['model_name']}, "
f"base_url={vlm_config['base_url']}, "
@@ -287,29 +273,16 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
return proto_chunk
def init_ocr_engine(ocr_backend: Optional[str] = None, **kwargs):
"""Initialize OCR engine"""
backend_type = ocr_backend or os.getenv("OCR_BACKEND", "paddle")
logger.info(f"Initializing OCR engine with backend: {backend_type}")
OCREngine.get_instance(backend_type=backend_type, **kwargs)
def main():
init_ocr_engine()
# Set max number of worker threads
max_workers = int(os.environ.get("GRPC_MAX_WORKERS", "4"))
logger.info(f"Starting DocReader service with {max_workers} worker threads")
# Get port number
port = os.environ.get("GRPC_PORT", "50051")
# Print effective env/config at startup
config.print_config()
# Create server
server = grpc.server(
futures.ThreadPoolExecutor(max_workers=max_workers),
futures.ThreadPoolExecutor(max_workers=CONFIG.grpc_max_workers),
options=[
("grpc.max_send_message_length", MAX_MESSAGE_LENGTH),
("grpc.max_receive_message_length", MAX_MESSAGE_LENGTH),
("grpc.max_send_message_length", CONFIG.grpc_max_file_size_mb),
("grpc.max_receive_message_length", CONFIG.grpc_max_file_size_mb),
],
)
@@ -321,12 +294,12 @@ def main():
health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
# Set listen address
server.add_insecure_port(f"[::]:{port}")
server.add_insecure_port(f"[::]:{CONFIG.grpc_port}")
# Start service
server.start()
logger.info(f"Server started on port {port}")
logger.info(f"Server started on port {CONFIG.grpc_port}")
logger.info("Server is ready to accept connections")
try:
+37
View File
@@ -0,0 +1,37 @@
import logging
import threading
from typing import Dict
from docreader.ocr.base import DummyOCRBackend, OCRBackend
from docreader.ocr.paddle import PaddleOCRBackend
from docreader.ocr.vlm import VLMOCRBackend
logger = logging.getLogger(__name__)
class OCREngine:
"""OCR Engine factory class for managing different OCR backend instances"""
_instances: Dict[str, OCRBackend] = {}
_lock = threading.Lock()
@classmethod
def get_instance(cls, backend_type: str) -> OCRBackend:
backend_type = (backend_type or "dummy").lower()
with cls._lock:
inst = cls._instances.get(backend_type)
if inst is not None:
return inst
logger.info(f"Creating OCR engine instance for backend: {backend_type}")
if backend_type == "paddle":
inst = PaddleOCRBackend()
elif backend_type == "vlm":
inst = VLMOCRBackend()
else:
inst = DummyOCRBackend()
cls._instances[backend_type] = inst
return inst
+31
View File
@@ -0,0 +1,31 @@
import logging
from abc import ABC, abstractmethod
from typing import Union
from PIL import Image
logger = logging.getLogger(__name__)
class OCRBackend(ABC):
"""Base class for OCR backends"""
@abstractmethod
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
pass
class DummyOCRBackend(OCRBackend):
"""Dummy OCR backend implementation"""
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
logger.warning("Dummy OCR backend is used")
return ""
@@ -3,42 +3,16 @@ import logging
import os
import platform
import subprocess
from abc import ABC, abstractmethod
from typing import Dict, Union
from typing import Union
import numpy as np
from openai import OpenAI
from PIL import Image
from docreader.utils import endecode
from docreader.ocr.base import OCRBackend
logger = logging.getLogger(__name__)
class OCRBackend(ABC):
"""Base class for OCR backends"""
@abstractmethod
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
pass
class DummyOCRBackend(OCRBackend):
"""Dummy OCR backend implementation"""
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
logger.warning("Dummy OCR backend is used")
return ""
class PaddleOCRBackend(OCRBackend):
"""PaddleOCR backend implementation"""
@@ -199,136 +173,3 @@ class PaddleOCRBackend(OCRBackend):
except Exception as e:
logger.error(f"OCR recognition error: {str(e)}")
return ""
class NanonetsOCRBackend(OCRBackend):
"""Nanonets OCR backend implementation using OpenAI API format"""
def __init__(self):
"""Initialize Nanonets OCR backend
Args:
api_key: API key for OpenAI API
base_url: Base URL for OpenAI API
model: Model name
"""
# Load configuration from environment variables
base_url = os.getenv("OCR_API_BASE_URL", "http://localhost:8000/v1")
api_key = os.getenv("OCR_API_KEY", "123")
timeout = 30
self.client = OpenAI(api_key=api_key, base_url=base_url, timeout=timeout)
self.model = os.getenv("OCR_MODEL", "nanonets/Nanonets-OCR-s")
logger.info(f"Nanonets OCR engine initialized with model: {self.model}")
self.temperature = 0.0
self.max_tokens = 15000
# Prompt for OCR text extraction with specific formatting requirements
self.prompt = """## 任务说明
请从上传的文档中提取文字内容,严格按自然阅读顺序(从上到下,从左到右)输出,并遵循以下格式规范。
### 1. **文本处理**
* 按正常阅读顺序提取文字,语句流畅自然。
### 2. **表格**
* 所有表格统一转换为 **Markdown 表格格式**。
* 内容保持清晰、对齐整齐,便于阅读。
### 3. **公式**
* 所有公式转换为 **LaTeX 格式**,使用 `$$公式$$` 包裹。
### 4. **图片**
* 忽略图片信息
### 5. **链接**
* 不要猜测或补全不确定的链接地址。
"""
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image using Nanonets OCR
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
if self.client is None:
logger.error("Nanonets OCR client not initialized")
return ""
try:
# Encode image to base64 format for API transmission
img_base64 = endecode.decode_image(image)
if not img_base64:
return ""
# Call Nanonets OCR API using OpenAI-compatible format
logger.info(f"Calling Nanonets OCR API with model: {self.model}")
response = self.client.chat.completions.create(
model=self.model,
messages=[
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{img_base64}"
},
},
{
"type": "text",
"text": self.prompt,
},
],
}
],
temperature=self.temperature,
max_tokens=self.max_tokens,
)
return response.choices[0].message.content or ""
except Exception as e:
logger.error(f"Nanonets OCR prediction error: {str(e)}")
return ""
class OCREngine:
"""OCR Engine factory class for managing different OCR backend instances"""
# Singleton pattern: cache instances for each backend type
_instance: Dict[str, OCRBackend] = {}
@classmethod
def get_instance(cls, backend_type: str) -> OCRBackend:
"""Get OCR engine instance using factory pattern
Args:
backend_type: OCR backend type, one of: "paddle", "nanonets"
**kwargs: Additional arguments for the backend
Returns:
OCR engine instance or None if initialization fails
"""
backend_type = backend_type.lower()
# Return cached instance if already initialized
if cls._instance.get(backend_type):
return cls._instance[backend_type]
logger.info(f"Initializing OCR engine with backend: {backend_type}")
if backend_type == "paddle":
cls._instance[backend_type] = PaddleOCRBackend()
elif backend_type == "nanonets":
cls._instance[backend_type] = NanonetsOCRBackend()
else:
cls._instance[backend_type] = DummyOCRBackend()
return cls._instance[backend_type]
+87
View File
@@ -0,0 +1,87 @@
import logging
from typing import Union
from openai import OpenAI
from PIL import Image
from docreader.config import CONFIG
from docreader.ocr.base import OCRBackend
from docreader.utils import endecode
logger = logging.getLogger(__name__)
class VLMOCRBackend(OCRBackend):
"""VLM OCR backend implementation using OpenAI API format"""
def __init__(self):
"""Initialize VLM OCR backend
Args:
api_key: API key for OpenAI API
base_url: Base URL for OpenAI API
model: Model name
"""
self.model = CONFIG.ocr_model
self.client = OpenAI(
api_key=CONFIG.ocr_api_key,
base_url=CONFIG.ocr_api_base_url,
timeout=30,
)
self.temperature = 0.0
self.max_tokens = 5000
# Prompt for OCR text extraction with specific formatting requirements
self.prompt = "提取文档图片中正文的所有信息用markdown格式表示,"
"其中页眉、页脚部分忽略,"
"表格用html格式表达,"
"文档中公式用latex格式表示,"
"按照阅读顺序组织进行解析。"
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image using VLM OCR
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
if self.client is None:
logger.error("VLM OCR client not initialized")
return ""
try:
# Encode image to base64 format for API transmission
img_base64 = endecode.decode_image(image)
if not img_base64:
return ""
# Call VLM OCR API using OpenAI-compatible format
logger.info(f"Calling VLM OCR API with model: {self.model}")
response = self.client.chat.completions.create(
model=self.model,
messages=[
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{img_base64}"
},
},
{
"type": "text",
"text": self.prompt,
},
],
}
],
temperature=self.temperature,
max_tokens=self.max_tokens,
)
return response.choices[0].message.content or ""
except Exception as e:
logger.error(f"VLM OCR prediction error: {str(e)}")
return ""
+15 -16
View File
@@ -13,10 +13,11 @@ from urllib.parse import urlparse
import requests
from PIL import Image
from docreader.config import CONFIG
from docreader.models.document import Chunk, Document
from docreader.models.read_config import ChunkingConfig
from docreader.ocr import OCREngine
from docreader.parser.caption import Caption
from docreader.parser.ocr_engine import OCREngine
from docreader.parser.storage import create_storage
from docreader.splitter.splitter import TextSplitter
from docreader.utils import endecode
@@ -243,7 +244,7 @@ class BaseParser(ABC):
return image
async def process_image_async(self, image: Image.Image, image_url: str):
"""Asynchronously process image: first perform OCR, then get caption
"""Asynchronously process image: perform OCR only (caption is generated asynchronously by Go backend)
Args:
image: Image object (PIL.Image or numpy array)
@@ -252,10 +253,12 @@ class BaseParser(ABC):
Returns:
tuple: (ocr_text, caption, image_url)
- ocr_text: OCR extracted text
- caption: Image description (if OCR has text) or empty string
- caption: Always empty string (caption generation moved to async task in Go backend)
- image_url: Image URL (if provided)
"""
logger.info("Starting asynchronous image processing (OCR + optional caption)")
logger.info(
"Starting asynchronous image processing (OCR only, caption deferred to async task)"
)
# Resize image
resized_image = self._resize_image_if_needed(image)
@@ -744,13 +747,11 @@ class BaseParser(ABC):
return img_url, img_url, None
# Still need to get image object for OCR processing
# Get proxy settings from environment variables
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY")
proxies = {}
if http_proxy:
proxies["http"] = http_proxy
if https_proxy:
proxies["https"] = https_proxy
if CONFIG.external_http_proxy:
proxies["http"] = CONFIG.external_http_proxy
if CONFIG.external_https_proxy:
proxies["https"] = CONFIG.external_https_proxy
response = requests.get(img_url, timeout=5, proxies=proxies)
if response.status_code == 200:
@@ -793,13 +794,11 @@ class BaseParser(ABC):
logger.error(f"URL failed validation check: {img_url}")
return img_url, img_url, None
# Get proxy settings from environment variables
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY")
proxies = {}
if http_proxy:
proxies["http"] = http_proxy
if https_proxy:
proxies["https"] = https_proxy
if CONFIG.external_http_proxy:
proxies["http"] = CONFIG.external_http_proxy
if CONFIG.external_https_proxy:
proxies["https"] = CONFIG.external_https_proxy
logger.info(f"Downloading image {img_url}, using proxy: {proxies}")
response = requests.get(img_url, timeout=5, proxies=proxies)
+9 -11
View File
@@ -1,6 +1,5 @@
import json
import logging
import os
import time
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Union
@@ -8,6 +7,8 @@ from typing import Dict, List, Optional, Union
import ollama
import requests
from docreader.config import CONFIG
logger = logging.getLogger(__name__)
@@ -197,15 +198,10 @@ class Caption:
self.interface_type = vlm_config.get("interface_type", "openai").lower()
else:
# Fall back to environment variables if config not provided
base_url = os.getenv("VLM_MODEL_BASE_URL")
model_name = os.getenv("VLM_MODEL_NAME")
if not base_url or not model_name:
logger.error("VLM_MODEL_BASE_URL or VLM_MODEL_NAME is not set")
return
self.completion_url = base_url + "/chat/completions"
self.model = model_name
self.api_key = os.getenv("VLM_MODEL_API_KEY", "")
self.interface_type = os.getenv("VLM_INTERFACE_TYPE", "openai").lower()
self.completion_url = CONFIG.vlm_model_base_url + "/chat/completions"
self.model = CONFIG.vlm_model_name
self.api_key = CONFIG.vlm_model_api_key
self.interface_type = CONFIG.vlm_interface_type
# Validate interface type - must be either "ollama" or "openai"
if self.interface_type not in ["ollama", "openai"]:
@@ -259,7 +255,9 @@ class Caption:
model=self.model,
prompt="简单凝炼的描述图片的主要内容",
images=[image_base64], # Pass base64 encoded image data
options={"temperature": 0.1}, # Low temperature for more deterministic output
options={
"temperature": 0.1
}, # Low temperature for more deterministic output
stream=False,
)
+2 -1
View File
@@ -5,6 +5,7 @@ from typing import List, Optional
import textract
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.docx2_parser import Docx2Parser
from docreader.utils.tempfile import TempDirContext, TempFileContext
@@ -24,7 +25,7 @@ class SandboxExecutor:
"""
# Get proxy from parameter, environment variable, or use default blocking proxy
# Use 'or None' to convert empty string to None, then apply default value
self.proxy = proxy or os.environ.get("WEB_PROXY") or "http://128.0.0.1:1"
self.proxy = proxy or CONFIG.external_https_proxy or "http://128.0.0.1:1"
self.default_timeout = default_timeout
def execute_in_sandbox(self, cmd: List[str]) -> tuple:
+4
View File
@@ -36,6 +36,10 @@ class ImageParser(BaseParser):
image_url = self.storage.upload_bytes(content, file_ext=ext)
logger.info(f"Successfully uploaded image, URL: {image_url[:50]}...")
if not image_url:
logger.warning(f"Failed to upload image: {self.file_name}")
return Document(content=f"{self.file_name}", images={})
# Generate markdown text
text = f"![{self.file_name}]({image_url})"
images = {image_url: base64.b64encode(content).decode()}
+39 -37
View File
@@ -10,6 +10,7 @@ This module provides comprehensive Markdown parsing functionality including:
The parser uses a pipeline approach to process Markdown content through
multiple stages: table formatting -> image processing.
"""
import base64
import logging
import os
@@ -28,23 +29,23 @@ logger = logging.getLogger(__name__)
class MarkdownTableUtil:
"""Utility class for formatting Markdown tables.
This class standardizes Markdown table formatting by:
- Normalizing column alignment markers (e.g., :---, :---:, ---:)
- Adding consistent spacing around pipes (|)
- Preserving indentation levels
- Handling both header rows and data rows
Example:
Input: |姓名|年龄|城市|
|:---|---:|:---:|
|张三|25|北京|
Output: | 姓名 | 年龄 | 城市 |
| :--- | ---: | :---: |
| 张三 | 25 | 北京 |
"""
def __init__(self):
# Pattern to match alignment row (e.g., |:---|---:|:---:|)
self.align_pattern = re.compile(
@@ -59,14 +60,14 @@ class MarkdownTableUtil:
def format_table(self, content: str) -> str:
"""Format all Markdown tables in the content.
Args:
content: Raw Markdown text containing tables
Returns:
Formatted Markdown text with standardized table formatting
"""
def process_align(match: Match[str]) -> str:
"""Process alignment row to standardize format."""
# Split by | and remove empty strings
@@ -125,10 +126,10 @@ class MarkdownTableUtil:
class MarkdownTableFormatter(BaseParser):
"""Parser for formatting Markdown tables.
This parser standardizes the formatting of all Markdown tables in the
document to ensure consistent spacing and alignment markers.
Example:
>>> formatter = MarkdownTableFormatter()
>>> content = b"|Name|Age|\n|---|---|\n|John|30|"
@@ -138,17 +139,17 @@ class MarkdownTableFormatter(BaseParser):
| --- | --- |
| John | 30 |
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.table_helper = MarkdownTableUtil()
def parse_into_text(self, content: bytes) -> Document:
"""Parse and format Markdown tables.
Args:
content: Raw Markdown content as bytes
Returns:
Document with formatted table content
"""
@@ -161,18 +162,18 @@ class MarkdownTableFormatter(BaseParser):
class MarkdownImageUtil:
"""Utility class for handling images in Markdown.
This class provides functionality to:
- Extract base64-encoded images from Markdown
- Extract image paths from Markdown
- Replace image paths with new URLs
- Convert base64 images to binary format
Supported formats:
- Base64 embedded images: ![alt](data:image/png;base64,iVBORw0...)
- Regular image links: ![alt](path/to/image.png)
"""
def __init__(self):
# Pattern to match base64 embedded images
# Captures: (1) alt text, (2) image format, (3) base64 data
@@ -191,15 +192,15 @@ class MarkdownImageUtil:
replace: bool = True,
) -> Tuple[str, List[str]]:
"""Extract image paths from Markdown content.
Args:
content: Markdown text containing images
path_prefix: Optional prefix to add to image paths
replace: Whether to replace image syntax in content
Returns:
Tuple of (processed_text, list_of_image_paths)
Example:
>>> util = MarkdownImageUtil()
>>> text, images = util.extract_image("![logo](img/logo.png)")
@@ -213,7 +214,7 @@ class MarkdownImageUtil:
"""Replacement function for each image match."""
title = match.group(1) # Alt text
image_path = match.group(2) # Image path
# Add prefix if specified
if path_prefix:
image_path = f"{path_prefix}/{image_path}"
@@ -238,19 +239,19 @@ class MarkdownImageUtil:
replace: bool = True,
) -> Tuple[str, Dict[str, bytes]]:
"""Extract and decode base64 embedded images from Markdown.
This method finds all base64-encoded images in the Markdown content,
decodes them to binary format, generates unique filenames, and
optionally replaces them with file path references.
Args:
content: Markdown text containing base64 images
path_prefix: Optional directory prefix for generated paths
replace: Whether to replace base64 syntax with file paths
Returns:
Tuple of (processed_text, dict_of_path_to_bytes)
Example:
>>> util = MarkdownImageUtil()
>>> text = "![logo](data:image/png;base64,iVBORw0KGg...)"
@@ -294,17 +295,17 @@ class MarkdownImageUtil:
def replace_path(self, content: str, images: Dict[str, str]) -> str:
"""Replace image paths in Markdown with new URLs.
This method is typically used to replace local file paths with
uploaded URLs after images have been stored.
Args:
content: Markdown text with image references
images: Mapping of old paths to new URLs
Returns:
Markdown text with updated image URLs
Example:
>>> util = MarkdownImageUtil()
>>> content = "![logo](temp/img.png)"
@@ -320,7 +321,7 @@ class MarkdownImageUtil:
"""Replacement function for each image match."""
title = match.group(1) # Alt text
image_path = match.group(2) # Current image path
# Only replace if path exists in mapping
if image_path not in images:
return match.group(0) # Keep original
@@ -328,7 +329,7 @@ class MarkdownImageUtil:
content_replace.add(image_path)
# Get new URL from mapping
image_path = images[image_path]
return f"![{title}]({image_path})"
return f"![{title}]({image_path})" if image_path else title
text = self.replace_pattern.sub(repl, content)
logger.debug(f"Replaced {len(content_replace)} images in markdown")
@@ -348,16 +349,16 @@ class MarkdownImageUtil:
class MarkdownImageBase64(BaseParser):
"""Parser for extracting and uploading base64 images from Markdown.
This parser:
1. Extracts base64-encoded images from Markdown content
2. Uploads the decoded images to storage
3. Replaces base64 data with uploaded URLs
4. Returns a Document with clean Markdown and image mappings
Requires:
- self.storage: Storage backend for uploading images
Example:
>>> parser = MarkdownImageBase64(storage=my_storage)
>>> content = b"![logo](data:image/png;base64,iVBORw0...)"
@@ -365,17 +366,17 @@ class MarkdownImageBase64(BaseParser):
>>> print(doc.content)
![logo](https://storage.com/uuid.png)
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.image_helper = MarkdownImageUtil()
def parse_into_text(self, content: bytes) -> Document:
"""Parse Markdown and process base64 images.
Args:
content: Raw Markdown content as bytes
Returns:
Document with:
- content: Markdown with base64 replaced by URLs
@@ -411,14 +412,15 @@ class MarkdownImageBase64(BaseParser):
class MarkdownParser(PipelineParser):
"""Complete Markdown parser using pipeline approach.
This parser processes Markdown content through multiple stages:
1. MarkdownTableFormatter: Standardizes table formatting
2. MarkdownImageBase64: Extracts and uploads base64 images
The pipeline ensures that content flows through each parser in sequence,
with each stage's output becoming the next stage's input.
"""
_parser_cls = (MarkdownTableFormatter, MarkdownImageBase64)
+18 -15
View File
@@ -1,11 +1,11 @@
import logging
import os
import re
from typing import Dict
import markdownify
import requests
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
@@ -18,19 +18,19 @@ logger = logging.getLogger(__name__)
class StdMinerUParser(BaseParser):
"""
Standard MinerU Parser for document parsing.
This parser uses MinerU API to parse documents (especially PDFs) into markdown format,
with support for tables, formulas, and images extraction.
"""
def __init__(
self,
enable_markdownify: bool = True,
mineru_endpoint: str = "",
**kwargs,
):
"""
Initialize MinerU parser.
Args:
enable_markdownify: Whether to convert HTML tables to markdown format
mineru_endpoint: MinerU API endpoint URL
@@ -38,7 +38,7 @@ class StdMinerUParser(BaseParser):
"""
super().__init__(**kwargs)
# Get MinerU endpoint from environment variable or parameter
self.minerU = os.getenv("MINERU_ENDPOINT", mineru_endpoint)
self.minerU = CONFIG.mineru_endpoint
self.enable_markdownify = enable_markdownify
# Helper for processing markdown images
self.image_helper = MarkdownImageUtil()
@@ -50,10 +50,10 @@ class StdMinerUParser(BaseParser):
def ping(self, timeout: int = 5) -> bool:
"""
Check if MinerU API is available.
Args:
timeout: Request timeout in seconds
Returns:
True if API is available, False otherwise
"""
@@ -69,10 +69,10 @@ class StdMinerUParser(BaseParser):
def parse_into_text(self, content: bytes) -> Document:
"""
Parse document content into text using MinerU API.
Args:
content: Raw document content in bytes
Returns:
Document object containing parsed text and images
"""
@@ -165,24 +165,27 @@ class StdMinerUParser(BaseParser):
class MinerUParser(PipelineParser):
"""
MinerU Parser with pipeline processing.
This parser combines StdMinerUParser for document parsing and
This parser combines StdMinerUParser for document parsing and
MarkdownTableFormatter for table formatting in a pipeline.
"""
_parser_cls = (StdMinerUParser, MarkdownTableFormatter)
if __name__ == "__main__":
import os
# Example usage for testing
logging.basicConfig(level=logging.DEBUG)
# Configure your file path and MinerU endpoint
your_file = "/path/to/your/file.pdf"
your_mineru = "http://host.docker.internal:9987"
os.environ["MINERU_ENDPOINT"] = "http://host.docker.internal:9987"
# Create parser instance
parser = MinerUParser(mineru_endpoint=your_mineru)
parser = MinerUParser()
# Parse PDF file
with open(your_file, "rb") as f:
content = f.read()
+5 -5
View File
@@ -1,7 +1,7 @@
import logging
import os
from typing import Dict, Type
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.models.read_config import ChunkingConfig
from docreader.parser.base_parser import BaseParser
@@ -111,8 +111,8 @@ class Parser:
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
chunking_config=config, # Pass the entire chunking config for advanced options
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
max_concurrent_tasks=CONFIG.image_max_concurrent,
ocr_backend=CONFIG.ocr_backend,
)
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
@@ -158,8 +158,8 @@ class Parser:
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
chunking_config=config, # Pass the entire chunking config
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
max_concurrent_tasks=CONFIG.image_max_concurrent,
ocr_backend=CONFIG.ocr_backend,
)
logger.info("Starting to parse URL content")
+32 -22
View File
@@ -10,6 +10,7 @@ from typing import Dict
from minio import Minio
from qcloud_cos import CosConfig, CosS3Client
from docreader.config import CONFIG
from docreader.utils import endecode
logger = logging.getLogger(__name__)
@@ -73,16 +74,14 @@ class CosStorage(Storage):
prefix = cos_config.get("path_prefix", "")
else:
# Get COS configuration from environment variables
secret_id = os.getenv("COS_SECRET_ID")
secret_key = os.getenv("COS_SECRET_KEY")
region = os.getenv("COS_REGION")
bucket_name = os.getenv("COS_BUCKET_NAME")
appid = os.getenv("COS_APP_ID")
prefix = os.getenv("COS_PATH_PREFIX")
secret_id = CONFIG.cos_secret_id
secret_key = CONFIG.cos_secret_key
region = CONFIG.cos_region
bucket_name = CONFIG.cos_bucket_name
appid = CONFIG.cos_app_id
prefix = CONFIG.cos_path_prefix
enable_old_domain = (
os.getenv("COS_ENABLE_OLD_DOMAIN", "true").lower() == "true"
)
enable_old_domain = CONFIG.cos_enable_old_domain
if not all([secret_id, secret_key, region, bucket_name, appid]):
logger.error(
@@ -220,27 +219,27 @@ class MinioStorage(Storage):
access_key = (
self.storage_config.get("access_key_id")
if self.storage_config and self.storage_config.get("access_key_id")
else os.getenv("MINIO_ACCESS_KEY_ID")
else CONFIG.minio_access_key_id
)
secret_key = (
self.storage_config.get("secret_access_key")
if self.storage_config and self.storage_config.get("secret_access_key")
else os.getenv("MINIO_SECRET_ACCESS_KEY")
else CONFIG.minio_secret_access_key
)
bucket_name = (
self.storage_config.get("bucket_name")
if self.storage_config and self.storage_config.get("bucket_name")
else os.getenv("MINIO_BUCKET_NAME", "")
else CONFIG.minio_bucket_name
)
path_prefix_raw = (
self.storage_config.get("path_prefix")
if self.storage_config and self.storage_config.get("path_prefix")
else os.getenv("MINIO_PATH_PREFIX", "")
else CONFIG.minio_path_prefix
)
path_prefix = path_prefix_raw.strip().strip("/") if path_prefix_raw else ""
endpoint = os.getenv("MINIO_ENDPOINT", "")
use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true"
endpoint = CONFIG.minio_endpoint
use_ssl = CONFIG.minio_use_ssl
if not all([endpoint, access_key, secret_key, bucket_name]):
logger.error(
@@ -283,7 +282,7 @@ class MinioStorage(Storage):
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
"""
# 1. Use public endpoint if provided
endpoint = os.getenv("MINIO_PUBLIC_ENDPOINT")
endpoint = CONFIG.minio_public_endpoint
if endpoint:
return f"{endpoint}/{self.bucket_name}/{object_key}"
@@ -381,9 +380,7 @@ class LocalStorage(Storage):
def __init__(self, storage_config: Dict[str, str] = {}):
self.storage_config = storage_config
base_dir = storage_config.get(
"base_dir", os.getenv("LOCAL_STORAGE_BASE_DIR", "")
)
base_dir = storage_config.get("base_dir", CONFIG.local_storage_base_dir)
self.image_dir = os.path.join(base_dir, "images")
os.makedirs(self.image_dir, exist_ok=True)
@@ -410,6 +407,20 @@ class Base64Storage(Storage):
return f"data:image/{file_ext};base64,{endecode.decode_image(content)}"
class DummyStorage(Storage):
"""Dummy storage implementation.
It is used in tests or in environments where object storage is disabled.
All upload methods return empty string.
"""
def upload_file(self, file_path: str) -> str:
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
return ""
def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
"""Create a storage instance based on configuration or environment variables
@@ -419,7 +430,7 @@ def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
Returns:
Storage instance
"""
storage_type = os.getenv("STORAGE_TYPE", "cos").lower()
storage_type = CONFIG.storage_type
if storage_config:
storage_type = str(storage_config.get("provider", storage_type)).lower()
logger.info(f"Creating {storage_type} storage instance")
@@ -432,5 +443,4 @@ def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
return LocalStorage(storage_config or {})
elif storage_type == "base64":
return Base64Storage()
raise ValueError(f"Invalid storage type: {storage_type}")
return DummyStorage()
+3 -3
View File
@@ -1,10 +1,10 @@
import asyncio
import logging
import os
from playwright.async_api import async_playwright
from trafilatura import extract
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
@@ -30,8 +30,8 @@ class StdWebParser(BaseParser):
**kwargs: Additional arguments passed to BaseParser
"""
self.title = title
# Get proxy configuration from environment variable if available
self.proxy = os.environ.get("WEB_PROXY", "")
# Get proxy configuration from config if available
self.proxy = CONFIG.external_https_proxy
super().__init__(file_name=title, **kwargs)
logger.info(f"Initialized WebParser with title: {title}")