refactor: Restructure OCR module and centralize config

This commit is contained in:
begoniezhao
2026-01-16 15:05:18 +08:00
parent 74bf76eb3f
commit 88fd42cbc3
17 changed files with 586 additions and 352 deletions
+15 -16
View File
@@ -13,10 +13,11 @@ from urllib.parse import urlparse
import requests
from PIL import Image
from docreader.config import CONFIG
from docreader.models.document import Chunk, Document
from docreader.models.read_config import ChunkingConfig
from docreader.ocr import OCREngine
from docreader.parser.caption import Caption
from docreader.parser.ocr_engine import OCREngine
from docreader.parser.storage import create_storage
from docreader.splitter.splitter import TextSplitter
from docreader.utils import endecode
@@ -243,7 +244,7 @@ class BaseParser(ABC):
return image
async def process_image_async(self, image: Image.Image, image_url: str):
"""Asynchronously process image: first perform OCR, then get caption
"""Asynchronously process image: perform OCR only (caption is generated asynchronously by Go backend)
Args:
image: Image object (PIL.Image or numpy array)
@@ -252,10 +253,12 @@ class BaseParser(ABC):
Returns:
tuple: (ocr_text, caption, image_url)
- ocr_text: OCR extracted text
- caption: Image description (if OCR has text) or empty string
- caption: Always empty string (caption generation moved to async task in Go backend)
- image_url: Image URL (if provided)
"""
logger.info("Starting asynchronous image processing (OCR + optional caption)")
logger.info(
"Starting asynchronous image processing (OCR only, caption deferred to async task)"
)
# Resize image
resized_image = self._resize_image_if_needed(image)
@@ -744,13 +747,11 @@ class BaseParser(ABC):
return img_url, img_url, None
# Still need to get image object for OCR processing
# Get proxy settings from environment variables
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY")
proxies = {}
if http_proxy:
proxies["http"] = http_proxy
if https_proxy:
proxies["https"] = https_proxy
if CONFIG.external_http_proxy:
proxies["http"] = CONFIG.external_http_proxy
if CONFIG.external_https_proxy:
proxies["https"] = CONFIG.external_https_proxy
response = requests.get(img_url, timeout=5, proxies=proxies)
if response.status_code == 200:
@@ -793,13 +794,11 @@ class BaseParser(ABC):
logger.error(f"URL failed validation check: {img_url}")
return img_url, img_url, None
# Get proxy settings from environment variables
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY")
proxies = {}
if http_proxy:
proxies["http"] = http_proxy
if https_proxy:
proxies["https"] = https_proxy
if CONFIG.external_http_proxy:
proxies["http"] = CONFIG.external_http_proxy
if CONFIG.external_https_proxy:
proxies["https"] = CONFIG.external_https_proxy
logger.info(f"Downloading image {img_url}, using proxy: {proxies}")
response = requests.get(img_url, timeout=5, proxies=proxies)
+9 -11
View File
@@ -1,6 +1,5 @@
import json
import logging
import os
import time
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Union
@@ -8,6 +7,8 @@ from typing import Dict, List, Optional, Union
import ollama
import requests
from docreader.config import CONFIG
logger = logging.getLogger(__name__)
@@ -197,15 +198,10 @@ class Caption:
self.interface_type = vlm_config.get("interface_type", "openai").lower()
else:
# Fall back to environment variables if config not provided
base_url = os.getenv("VLM_MODEL_BASE_URL")
model_name = os.getenv("VLM_MODEL_NAME")
if not base_url or not model_name:
logger.error("VLM_MODEL_BASE_URL or VLM_MODEL_NAME is not set")
return
self.completion_url = base_url + "/chat/completions"
self.model = model_name
self.api_key = os.getenv("VLM_MODEL_API_KEY", "")
self.interface_type = os.getenv("VLM_INTERFACE_TYPE", "openai").lower()
self.completion_url = CONFIG.vlm_model_base_url + "/chat/completions"
self.model = CONFIG.vlm_model_name
self.api_key = CONFIG.vlm_model_api_key
self.interface_type = CONFIG.vlm_interface_type
# Validate interface type - must be either "ollama" or "openai"
if self.interface_type not in ["ollama", "openai"]:
@@ -259,7 +255,9 @@ class Caption:
model=self.model,
prompt="简单凝炼的描述图片的主要内容",
images=[image_base64], # Pass base64 encoded image data
options={"temperature": 0.1}, # Low temperature for more deterministic output
options={
"temperature": 0.1
}, # Low temperature for more deterministic output
stream=False,
)
+2 -1
View File
@@ -5,6 +5,7 @@ from typing import List, Optional
import textract
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.docx2_parser import Docx2Parser
from docreader.utils.tempfile import TempDirContext, TempFileContext
@@ -24,7 +25,7 @@ class SandboxExecutor:
"""
# Get proxy from parameter, environment variable, or use default blocking proxy
# Use 'or None' to convert empty string to None, then apply default value
self.proxy = proxy or os.environ.get("WEB_PROXY") or "http://128.0.0.1:1"
self.proxy = proxy or CONFIG.external_https_proxy or "http://128.0.0.1:1"
self.default_timeout = default_timeout
def execute_in_sandbox(self, cmd: List[str]) -> tuple:
+4
View File
@@ -36,6 +36,10 @@ class ImageParser(BaseParser):
image_url = self.storage.upload_bytes(content, file_ext=ext)
logger.info(f"Successfully uploaded image, URL: {image_url[:50]}...")
if not image_url:
logger.warning(f"Failed to upload image: {self.file_name}")
return Document(content=f"{self.file_name}", images={})
# Generate markdown text
text = f"![{self.file_name}]({image_url})"
images = {image_url: base64.b64encode(content).decode()}
+39 -37
View File
@@ -10,6 +10,7 @@ This module provides comprehensive Markdown parsing functionality including:
The parser uses a pipeline approach to process Markdown content through
multiple stages: table formatting -> image processing.
"""
import base64
import logging
import os
@@ -28,23 +29,23 @@ logger = logging.getLogger(__name__)
class MarkdownTableUtil:
"""Utility class for formatting Markdown tables.
This class standardizes Markdown table formatting by:
- Normalizing column alignment markers (e.g., :---, :---:, ---:)
- Adding consistent spacing around pipes (|)
- Preserving indentation levels
- Handling both header rows and data rows
Example:
Input: |姓名|年龄|城市|
|:---|---:|:---:|
|张三|25|北京|
Output: | 姓名 | 年龄 | 城市 |
| :--- | ---: | :---: |
| 张三 | 25 | 北京 |
"""
def __init__(self):
# Pattern to match alignment row (e.g., |:---|---:|:---:|)
self.align_pattern = re.compile(
@@ -59,14 +60,14 @@ class MarkdownTableUtil:
def format_table(self, content: str) -> str:
"""Format all Markdown tables in the content.
Args:
content: Raw Markdown text containing tables
Returns:
Formatted Markdown text with standardized table formatting
"""
def process_align(match: Match[str]) -> str:
"""Process alignment row to standardize format."""
# Split by | and remove empty strings
@@ -125,10 +126,10 @@ class MarkdownTableUtil:
class MarkdownTableFormatter(BaseParser):
"""Parser for formatting Markdown tables.
This parser standardizes the formatting of all Markdown tables in the
document to ensure consistent spacing and alignment markers.
Example:
>>> formatter = MarkdownTableFormatter()
>>> content = b"|Name|Age|\n|---|---|\n|John|30|"
@@ -138,17 +139,17 @@ class MarkdownTableFormatter(BaseParser):
| --- | --- |
| John | 30 |
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.table_helper = MarkdownTableUtil()
def parse_into_text(self, content: bytes) -> Document:
"""Parse and format Markdown tables.
Args:
content: Raw Markdown content as bytes
Returns:
Document with formatted table content
"""
@@ -161,18 +162,18 @@ class MarkdownTableFormatter(BaseParser):
class MarkdownImageUtil:
"""Utility class for handling images in Markdown.
This class provides functionality to:
- Extract base64-encoded images from Markdown
- Extract image paths from Markdown
- Replace image paths with new URLs
- Convert base64 images to binary format
Supported formats:
- Base64 embedded images: ![alt](data:image/png;base64,iVBORw0...)
- Regular image links: ![alt](path/to/image.png)
"""
def __init__(self):
# Pattern to match base64 embedded images
# Captures: (1) alt text, (2) image format, (3) base64 data
@@ -191,15 +192,15 @@ class MarkdownImageUtil:
replace: bool = True,
) -> Tuple[str, List[str]]:
"""Extract image paths from Markdown content.
Args:
content: Markdown text containing images
path_prefix: Optional prefix to add to image paths
replace: Whether to replace image syntax in content
Returns:
Tuple of (processed_text, list_of_image_paths)
Example:
>>> util = MarkdownImageUtil()
>>> text, images = util.extract_image("![logo](img/logo.png)")
@@ -213,7 +214,7 @@ class MarkdownImageUtil:
"""Replacement function for each image match."""
title = match.group(1) # Alt text
image_path = match.group(2) # Image path
# Add prefix if specified
if path_prefix:
image_path = f"{path_prefix}/{image_path}"
@@ -238,19 +239,19 @@ class MarkdownImageUtil:
replace: bool = True,
) -> Tuple[str, Dict[str, bytes]]:
"""Extract and decode base64 embedded images from Markdown.
This method finds all base64-encoded images in the Markdown content,
decodes them to binary format, generates unique filenames, and
optionally replaces them with file path references.
Args:
content: Markdown text containing base64 images
path_prefix: Optional directory prefix for generated paths
replace: Whether to replace base64 syntax with file paths
Returns:
Tuple of (processed_text, dict_of_path_to_bytes)
Example:
>>> util = MarkdownImageUtil()
>>> text = "![logo](data:image/png;base64,iVBORw0KGg...)"
@@ -294,17 +295,17 @@ class MarkdownImageUtil:
def replace_path(self, content: str, images: Dict[str, str]) -> str:
"""Replace image paths in Markdown with new URLs.
This method is typically used to replace local file paths with
uploaded URLs after images have been stored.
Args:
content: Markdown text with image references
images: Mapping of old paths to new URLs
Returns:
Markdown text with updated image URLs
Example:
>>> util = MarkdownImageUtil()
>>> content = "![logo](temp/img.png)"
@@ -320,7 +321,7 @@ class MarkdownImageUtil:
"""Replacement function for each image match."""
title = match.group(1) # Alt text
image_path = match.group(2) # Current image path
# Only replace if path exists in mapping
if image_path not in images:
return match.group(0) # Keep original
@@ -328,7 +329,7 @@ class MarkdownImageUtil:
content_replace.add(image_path)
# Get new URL from mapping
image_path = images[image_path]
return f"![{title}]({image_path})"
return f"![{title}]({image_path})" if image_path else title
text = self.replace_pattern.sub(repl, content)
logger.debug(f"Replaced {len(content_replace)} images in markdown")
@@ -348,16 +349,16 @@ class MarkdownImageUtil:
class MarkdownImageBase64(BaseParser):
"""Parser for extracting and uploading base64 images from Markdown.
This parser:
1. Extracts base64-encoded images from Markdown content
2. Uploads the decoded images to storage
3. Replaces base64 data with uploaded URLs
4. Returns a Document with clean Markdown and image mappings
Requires:
- self.storage: Storage backend for uploading images
Example:
>>> parser = MarkdownImageBase64(storage=my_storage)
>>> content = b"![logo](data:image/png;base64,iVBORw0...)"
@@ -365,17 +366,17 @@ class MarkdownImageBase64(BaseParser):
>>> print(doc.content)
![logo](https://storage.com/uuid.png)
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.image_helper = MarkdownImageUtil()
def parse_into_text(self, content: bytes) -> Document:
"""Parse Markdown and process base64 images.
Args:
content: Raw Markdown content as bytes
Returns:
Document with:
- content: Markdown with base64 replaced by URLs
@@ -411,14 +412,15 @@ class MarkdownImageBase64(BaseParser):
class MarkdownParser(PipelineParser):
"""Complete Markdown parser using pipeline approach.
This parser processes Markdown content through multiple stages:
1. MarkdownTableFormatter: Standardizes table formatting
2. MarkdownImageBase64: Extracts and uploads base64 images
The pipeline ensures that content flows through each parser in sequence,
with each stage's output becoming the next stage's input.
"""
_parser_cls = (MarkdownTableFormatter, MarkdownImageBase64)
+18 -15
View File
@@ -1,11 +1,11 @@
import logging
import os
import re
from typing import Dict
import markdownify
import requests
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
@@ -18,19 +18,19 @@ logger = logging.getLogger(__name__)
class StdMinerUParser(BaseParser):
"""
Standard MinerU Parser for document parsing.
This parser uses MinerU API to parse documents (especially PDFs) into markdown format,
with support for tables, formulas, and images extraction.
"""
def __init__(
self,
enable_markdownify: bool = True,
mineru_endpoint: str = "",
**kwargs,
):
"""
Initialize MinerU parser.
Args:
enable_markdownify: Whether to convert HTML tables to markdown format
mineru_endpoint: MinerU API endpoint URL
@@ -38,7 +38,7 @@ class StdMinerUParser(BaseParser):
"""
super().__init__(**kwargs)
# Get MinerU endpoint from environment variable or parameter
self.minerU = os.getenv("MINERU_ENDPOINT", mineru_endpoint)
self.minerU = CONFIG.mineru_endpoint
self.enable_markdownify = enable_markdownify
# Helper for processing markdown images
self.image_helper = MarkdownImageUtil()
@@ -50,10 +50,10 @@ class StdMinerUParser(BaseParser):
def ping(self, timeout: int = 5) -> bool:
"""
Check if MinerU API is available.
Args:
timeout: Request timeout in seconds
Returns:
True if API is available, False otherwise
"""
@@ -69,10 +69,10 @@ class StdMinerUParser(BaseParser):
def parse_into_text(self, content: bytes) -> Document:
"""
Parse document content into text using MinerU API.
Args:
content: Raw document content in bytes
Returns:
Document object containing parsed text and images
"""
@@ -165,24 +165,27 @@ class StdMinerUParser(BaseParser):
class MinerUParser(PipelineParser):
"""
MinerU Parser with pipeline processing.
This parser combines StdMinerUParser for document parsing and
This parser combines StdMinerUParser for document parsing and
MarkdownTableFormatter for table formatting in a pipeline.
"""
_parser_cls = (StdMinerUParser, MarkdownTableFormatter)
if __name__ == "__main__":
import os
# Example usage for testing
logging.basicConfig(level=logging.DEBUG)
# Configure your file path and MinerU endpoint
your_file = "/path/to/your/file.pdf"
your_mineru = "http://host.docker.internal:9987"
os.environ["MINERU_ENDPOINT"] = "http://host.docker.internal:9987"
# Create parser instance
parser = MinerUParser(mineru_endpoint=your_mineru)
parser = MinerUParser()
# Parse PDF file
with open(your_file, "rb") as f:
content = f.read()
-334
View File
@@ -1,334 +0,0 @@
import io
import logging
import os
import platform
import subprocess
from abc import ABC, abstractmethod
from typing import Dict, Union
import numpy as np
from openai import OpenAI
from PIL import Image
from docreader.utils import endecode
logger = logging.getLogger(__name__)
class OCRBackend(ABC):
"""Base class for OCR backends"""
@abstractmethod
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
pass
class DummyOCRBackend(OCRBackend):
"""Dummy OCR backend implementation"""
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
logger.warning("Dummy OCR backend is used")
return ""
class PaddleOCRBackend(OCRBackend):
"""PaddleOCR backend implementation"""
def __init__(self):
"""Initialize PaddleOCR backend"""
self.ocr = None
try:
import paddle
# Set PaddlePaddle to use CPU and disable GPU
os.environ["CUDA_VISIBLE_DEVICES"] = ""
paddle.device.set_device("cpu")
# Try to detect if CPU supports AVX instruction set
# 尝试检测CPU是否支持AVX指令集
try:
# Detect if CPU supports AVX
# 检测CPU是否支持AVX
if platform.system() == "Linux":
try:
result = subprocess.run(
["grep", "-o", "avx", "/proc/cpuinfo"],
capture_output=True,
text=True,
timeout=5,
)
has_avx = "avx" in result.stdout.lower()
if not has_avx:
logger.warning(
"CPU does not support AVX instructions, "
"using compatibility mode"
)
# Further restrict instruction set usage
# 进一步限制指令集使用
os.environ["FLAGS_use_avx2"] = "0"
os.environ["FLAGS_use_avx"] = "1"
except (
subprocess.TimeoutExpired,
FileNotFoundError,
subprocess.SubprocessError,
):
logger.warning(
"Could not detect AVX support, using compatibility mode"
)
os.environ["FLAGS_use_avx2"] = "0"
os.environ["FLAGS_use_avx"] = "1"
except Exception as e:
logger.warning(
f"Error detecting CPU capabilities: {e}, using compatibility mode"
)
os.environ["FLAGS_use_avx2"] = "0"
os.environ["FLAGS_use_avx"] = "1"
from paddleocr import PaddleOCR
# OCR configuration with text orientation classification enabled
ocr_config = {
"use_gpu": False,
"text_det_limit_type": "max",
"text_det_limit_side_len": 960,
"use_doc_orientation_classify": True, # Enable document orientation classification / 启用文档方向分类
"use_doc_unwarping": False,
"use_textline_orientation": True, # Enable text line orientation detection / 启用文本行方向检测
"text_recognition_model_name": "PP-OCRv4_server_rec",
"text_detection_model_name": "PP-OCRv4_server_det",
"text_det_thresh": 0.3,
"text_det_box_thresh": 0.6,
"text_det_unclip_ratio": 1.5,
"text_rec_score_thresh": 0.0,
"ocr_version": "PP-OCRv4",
"lang": "ch",
"show_log": False,
"use_dilation": True, # improves accuracy
"det_db_score_mode": "slow", # improves accuracy
}
self.ocr = PaddleOCR(**ocr_config)
logger.info("PaddleOCR engine initialized successfully")
except ImportError as e:
logger.error(
f"Failed to import paddleocr: {str(e)}. "
"Please install it with 'pip install paddleocr'"
)
except OSError as e:
if "Illegal instruction" in str(e) or "core dumped" in str(e):
logger.error(
f"PaddlePaddle crashed due to CPU instruction set incompatibility:"
f"{e}"
)
logger.error(
"This happens when the CPU doesn't support AVX instructions. "
"Try install CPU-only version of PaddlePaddle, "
"or use a different OCR backend."
)
else:
logger.error(
f"Failed to initialize PaddleOCR due to OS error: {str(e)}"
)
except Exception as e:
logger.error(f"Failed to initialize PaddleOCR: {str(e)}")
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
if isinstance(image, str):
image = Image.open(image)
elif isinstance(image, bytes):
image = Image.open(io.BytesIO(image))
if not isinstance(image, Image.Image):
raise TypeError("image must be a string, bytes, or PIL Image object")
return self._predict(image)
def _predict(self, image: Image.Image) -> str:
"""Perform OCR recognition on the image
Args:
image: Image object (PIL.Image or numpy array)
Returns:
Extracted text string
"""
if self.ocr is None:
logger.error("PaddleOCR engine not initialized")
return ""
try:
# Ensure image is in RGB format
if image.mode != "RGB":
image = image.convert("RGB")
# Convert to numpy array for PaddleOCR processing
image_array = np.array(image)
# Perform OCR recognition
ocr_result = self.ocr.ocr(image_array, cls=False)
# Extract and concatenate text from OCR results
ocr_text = ""
if ocr_result and ocr_result[0]:
text = [
line[1][0] if line and len(line) >= 2 and line[1] else ""
for line in ocr_result[0]
]
text = [t.strip() for t in text if t]
ocr_text = " ".join(text)
logger.info(f"OCR extracted {len(ocr_text)} characters")
return ocr_text
except Exception as e:
logger.error(f"OCR recognition error: {str(e)}")
return ""
class NanonetsOCRBackend(OCRBackend):
"""Nanonets OCR backend implementation using OpenAI API format"""
def __init__(self):
"""Initialize Nanonets OCR backend
Args:
api_key: API key for OpenAI API
base_url: Base URL for OpenAI API
model: Model name
"""
# Load configuration from environment variables
base_url = os.getenv("OCR_API_BASE_URL", "http://localhost:8000/v1")
api_key = os.getenv("OCR_API_KEY", "123")
timeout = 30
self.client = OpenAI(api_key=api_key, base_url=base_url, timeout=timeout)
self.model = os.getenv("OCR_MODEL", "nanonets/Nanonets-OCR-s")
logger.info(f"Nanonets OCR engine initialized with model: {self.model}")
self.temperature = 0.0
self.max_tokens = 15000
# Prompt for OCR text extraction with specific formatting requirements
self.prompt = """## 任务说明
请从上传的文档中提取文字内容,严格按自然阅读顺序(从上到下,从左到右)输出,并遵循以下格式规范。
### 1. **文本处理**
* 按正常阅读顺序提取文字,语句流畅自然。
### 2. **表格**
* 所有表格统一转换为 **Markdown 表格格式**。
* 内容保持清晰、对齐整齐,便于阅读。
### 3. **公式**
* 所有公式转换为 **LaTeX 格式**,使用 `$$公式$$` 包裹。
### 4. **图片**
* 忽略图片信息
### 5. **链接**
* 不要猜测或补全不确定的链接地址。
"""
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image using Nanonets OCR
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
if self.client is None:
logger.error("Nanonets OCR client not initialized")
return ""
try:
# Encode image to base64 format for API transmission
img_base64 = endecode.decode_image(image)
if not img_base64:
return ""
# Call Nanonets OCR API using OpenAI-compatible format
logger.info(f"Calling Nanonets OCR API with model: {self.model}")
response = self.client.chat.completions.create(
model=self.model,
messages=[
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{img_base64}"
},
},
{
"type": "text",
"text": self.prompt,
},
],
}
],
temperature=self.temperature,
max_tokens=self.max_tokens,
)
return response.choices[0].message.content or ""
except Exception as e:
logger.error(f"Nanonets OCR prediction error: {str(e)}")
return ""
class OCREngine:
"""OCR Engine factory class for managing different OCR backend instances"""
# Singleton pattern: cache instances for each backend type
_instance: Dict[str, OCRBackend] = {}
@classmethod
def get_instance(cls, backend_type: str) -> OCRBackend:
"""Get OCR engine instance using factory pattern
Args:
backend_type: OCR backend type, one of: "paddle", "nanonets"
**kwargs: Additional arguments for the backend
Returns:
OCR engine instance or None if initialization fails
"""
backend_type = backend_type.lower()
# Return cached instance if already initialized
if cls._instance.get(backend_type):
return cls._instance[backend_type]
logger.info(f"Initializing OCR engine with backend: {backend_type}")
if backend_type == "paddle":
cls._instance[backend_type] = PaddleOCRBackend()
elif backend_type == "nanonets":
cls._instance[backend_type] = NanonetsOCRBackend()
else:
cls._instance[backend_type] = DummyOCRBackend()
return cls._instance[backend_type]
+5 -5
View File
@@ -1,7 +1,7 @@
import logging
import os
from typing import Dict, Type
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.models.read_config import ChunkingConfig
from docreader.parser.base_parser import BaseParser
@@ -111,8 +111,8 @@ class Parser:
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
chunking_config=config, # Pass the entire chunking config for advanced options
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
max_concurrent_tasks=CONFIG.image_max_concurrent,
ocr_backend=CONFIG.ocr_backend,
)
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
@@ -158,8 +158,8 @@ class Parser:
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
chunking_config=config, # Pass the entire chunking config
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
max_concurrent_tasks=CONFIG.image_max_concurrent,
ocr_backend=CONFIG.ocr_backend,
)
logger.info("Starting to parse URL content")
+32 -22
View File
@@ -10,6 +10,7 @@ from typing import Dict
from minio import Minio
from qcloud_cos import CosConfig, CosS3Client
from docreader.config import CONFIG
from docreader.utils import endecode
logger = logging.getLogger(__name__)
@@ -73,16 +74,14 @@ class CosStorage(Storage):
prefix = cos_config.get("path_prefix", "")
else:
# Get COS configuration from environment variables
secret_id = os.getenv("COS_SECRET_ID")
secret_key = os.getenv("COS_SECRET_KEY")
region = os.getenv("COS_REGION")
bucket_name = os.getenv("COS_BUCKET_NAME")
appid = os.getenv("COS_APP_ID")
prefix = os.getenv("COS_PATH_PREFIX")
secret_id = CONFIG.cos_secret_id
secret_key = CONFIG.cos_secret_key
region = CONFIG.cos_region
bucket_name = CONFIG.cos_bucket_name
appid = CONFIG.cos_app_id
prefix = CONFIG.cos_path_prefix
enable_old_domain = (
os.getenv("COS_ENABLE_OLD_DOMAIN", "true").lower() == "true"
)
enable_old_domain = CONFIG.cos_enable_old_domain
if not all([secret_id, secret_key, region, bucket_name, appid]):
logger.error(
@@ -220,27 +219,27 @@ class MinioStorage(Storage):
access_key = (
self.storage_config.get("access_key_id")
if self.storage_config and self.storage_config.get("access_key_id")
else os.getenv("MINIO_ACCESS_KEY_ID")
else CONFIG.minio_access_key_id
)
secret_key = (
self.storage_config.get("secret_access_key")
if self.storage_config and self.storage_config.get("secret_access_key")
else os.getenv("MINIO_SECRET_ACCESS_KEY")
else CONFIG.minio_secret_access_key
)
bucket_name = (
self.storage_config.get("bucket_name")
if self.storage_config and self.storage_config.get("bucket_name")
else os.getenv("MINIO_BUCKET_NAME", "")
else CONFIG.minio_bucket_name
)
path_prefix_raw = (
self.storage_config.get("path_prefix")
if self.storage_config and self.storage_config.get("path_prefix")
else os.getenv("MINIO_PATH_PREFIX", "")
else CONFIG.minio_path_prefix
)
path_prefix = path_prefix_raw.strip().strip("/") if path_prefix_raw else ""
endpoint = os.getenv("MINIO_ENDPOINT", "")
use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true"
endpoint = CONFIG.minio_endpoint
use_ssl = CONFIG.minio_use_ssl
if not all([endpoint, access_key, secret_key, bucket_name]):
logger.error(
@@ -283,7 +282,7 @@ class MinioStorage(Storage):
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
"""
# 1. Use public endpoint if provided
endpoint = os.getenv("MINIO_PUBLIC_ENDPOINT")
endpoint = CONFIG.minio_public_endpoint
if endpoint:
return f"{endpoint}/{self.bucket_name}/{object_key}"
@@ -381,9 +380,7 @@ class LocalStorage(Storage):
def __init__(self, storage_config: Dict[str, str] = {}):
self.storage_config = storage_config
base_dir = storage_config.get(
"base_dir", os.getenv("LOCAL_STORAGE_BASE_DIR", "")
)
base_dir = storage_config.get("base_dir", CONFIG.local_storage_base_dir)
self.image_dir = os.path.join(base_dir, "images")
os.makedirs(self.image_dir, exist_ok=True)
@@ -410,6 +407,20 @@ class Base64Storage(Storage):
return f"data:image/{file_ext};base64,{endecode.decode_image(content)}"
class DummyStorage(Storage):
"""Dummy storage implementation.
It is used in tests or in environments where object storage is disabled.
All upload methods return empty string.
"""
def upload_file(self, file_path: str) -> str:
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
return ""
def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
"""Create a storage instance based on configuration or environment variables
@@ -419,7 +430,7 @@ def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
Returns:
Storage instance
"""
storage_type = os.getenv("STORAGE_TYPE", "cos").lower()
storage_type = CONFIG.storage_type
if storage_config:
storage_type = str(storage_config.get("provider", storage_type)).lower()
logger.info(f"Creating {storage_type} storage instance")
@@ -432,5 +443,4 @@ def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
return LocalStorage(storage_config or {})
elif storage_type == "base64":
return Base64Storage()
raise ValueError(f"Invalid storage type: {storage_type}")
return DummyStorage()
+3 -3
View File
@@ -1,10 +1,10 @@
import asyncio
import logging
import os
from playwright.async_api import async_playwright
from trafilatura import extract
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
@@ -30,8 +30,8 @@ class StdWebParser(BaseParser):
**kwargs: Additional arguments passed to BaseParser
"""
self.title = title
# Get proxy configuration from environment variable if available
self.proxy = os.environ.get("WEB_PROXY", "")
# Get proxy configuration from config if available
self.proxy = CONFIG.external_https_proxy
super().__init__(file_name=title, **kwargs)
logger.info(f"Initialized WebParser with title: {title}")