refactor: Restructure OCR module and centralize config
This commit is contained in:
@@ -13,10 +13,11 @@ from urllib.parse import urlparse
|
||||
import requests
|
||||
from PIL import Image
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Chunk, Document
|
||||
from docreader.models.read_config import ChunkingConfig
|
||||
from docreader.ocr import OCREngine
|
||||
from docreader.parser.caption import Caption
|
||||
from docreader.parser.ocr_engine import OCREngine
|
||||
from docreader.parser.storage import create_storage
|
||||
from docreader.splitter.splitter import TextSplitter
|
||||
from docreader.utils import endecode
|
||||
@@ -243,7 +244,7 @@ class BaseParser(ABC):
|
||||
return image
|
||||
|
||||
async def process_image_async(self, image: Image.Image, image_url: str):
|
||||
"""Asynchronously process image: first perform OCR, then get caption
|
||||
"""Asynchronously process image: perform OCR only (caption is generated asynchronously by Go backend)
|
||||
|
||||
Args:
|
||||
image: Image object (PIL.Image or numpy array)
|
||||
@@ -252,10 +253,12 @@ class BaseParser(ABC):
|
||||
Returns:
|
||||
tuple: (ocr_text, caption, image_url)
|
||||
- ocr_text: OCR extracted text
|
||||
- caption: Image description (if OCR has text) or empty string
|
||||
- caption: Always empty string (caption generation moved to async task in Go backend)
|
||||
- image_url: Image URL (if provided)
|
||||
"""
|
||||
logger.info("Starting asynchronous image processing (OCR + optional caption)")
|
||||
logger.info(
|
||||
"Starting asynchronous image processing (OCR only, caption deferred to async task)"
|
||||
)
|
||||
|
||||
# Resize image
|
||||
resized_image = self._resize_image_if_needed(image)
|
||||
@@ -744,13 +747,11 @@ class BaseParser(ABC):
|
||||
return img_url, img_url, None
|
||||
# Still need to get image object for OCR processing
|
||||
# Get proxy settings from environment variables
|
||||
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
|
||||
https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY")
|
||||
proxies = {}
|
||||
if http_proxy:
|
||||
proxies["http"] = http_proxy
|
||||
if https_proxy:
|
||||
proxies["https"] = https_proxy
|
||||
if CONFIG.external_http_proxy:
|
||||
proxies["http"] = CONFIG.external_http_proxy
|
||||
if CONFIG.external_https_proxy:
|
||||
proxies["https"] = CONFIG.external_https_proxy
|
||||
|
||||
response = requests.get(img_url, timeout=5, proxies=proxies)
|
||||
if response.status_code == 200:
|
||||
@@ -793,13 +794,11 @@ class BaseParser(ABC):
|
||||
logger.error(f"URL failed validation check: {img_url}")
|
||||
return img_url, img_url, None
|
||||
# Get proxy settings from environment variables
|
||||
http_proxy = os.environ.get("EXTERNAL_HTTP_PROXY")
|
||||
https_proxy = os.environ.get("EXTERNAL_HTTPS_PROXY")
|
||||
proxies = {}
|
||||
if http_proxy:
|
||||
proxies["http"] = http_proxy
|
||||
if https_proxy:
|
||||
proxies["https"] = https_proxy
|
||||
if CONFIG.external_http_proxy:
|
||||
proxies["http"] = CONFIG.external_http_proxy
|
||||
if CONFIG.external_https_proxy:
|
||||
proxies["https"] = CONFIG.external_https_proxy
|
||||
|
||||
logger.info(f"Downloading image {img_url}, using proxy: {proxies}")
|
||||
response = requests.get(img_url, timeout=5, proxies=proxies)
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Union
|
||||
@@ -8,6 +7,8 @@ from typing import Dict, List, Optional, Union
|
||||
import ollama
|
||||
import requests
|
||||
|
||||
from docreader.config import CONFIG
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -197,15 +198,10 @@ class Caption:
|
||||
self.interface_type = vlm_config.get("interface_type", "openai").lower()
|
||||
else:
|
||||
# Fall back to environment variables if config not provided
|
||||
base_url = os.getenv("VLM_MODEL_BASE_URL")
|
||||
model_name = os.getenv("VLM_MODEL_NAME")
|
||||
if not base_url or not model_name:
|
||||
logger.error("VLM_MODEL_BASE_URL or VLM_MODEL_NAME is not set")
|
||||
return
|
||||
self.completion_url = base_url + "/chat/completions"
|
||||
self.model = model_name
|
||||
self.api_key = os.getenv("VLM_MODEL_API_KEY", "")
|
||||
self.interface_type = os.getenv("VLM_INTERFACE_TYPE", "openai").lower()
|
||||
self.completion_url = CONFIG.vlm_model_base_url + "/chat/completions"
|
||||
self.model = CONFIG.vlm_model_name
|
||||
self.api_key = CONFIG.vlm_model_api_key
|
||||
self.interface_type = CONFIG.vlm_interface_type
|
||||
|
||||
# Validate interface type - must be either "ollama" or "openai"
|
||||
if self.interface_type not in ["ollama", "openai"]:
|
||||
@@ -259,7 +255,9 @@ class Caption:
|
||||
model=self.model,
|
||||
prompt="简单凝炼的描述图片的主要内容",
|
||||
images=[image_base64], # Pass base64 encoded image data
|
||||
options={"temperature": 0.1}, # Low temperature for more deterministic output
|
||||
options={
|
||||
"temperature": 0.1
|
||||
}, # Low temperature for more deterministic output
|
||||
stream=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import List, Optional
|
||||
|
||||
import textract
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.docx2_parser import Docx2Parser
|
||||
from docreader.utils.tempfile import TempDirContext, TempFileContext
|
||||
@@ -24,7 +25,7 @@ class SandboxExecutor:
|
||||
"""
|
||||
# Get proxy from parameter, environment variable, or use default blocking proxy
|
||||
# Use 'or None' to convert empty string to None, then apply default value
|
||||
self.proxy = proxy or os.environ.get("WEB_PROXY") or "http://128.0.0.1:1"
|
||||
self.proxy = proxy or CONFIG.external_https_proxy or "http://128.0.0.1:1"
|
||||
self.default_timeout = default_timeout
|
||||
|
||||
def execute_in_sandbox(self, cmd: List[str]) -> tuple:
|
||||
|
||||
@@ -36,6 +36,10 @@ class ImageParser(BaseParser):
|
||||
image_url = self.storage.upload_bytes(content, file_ext=ext)
|
||||
logger.info(f"Successfully uploaded image, URL: {image_url[:50]}...")
|
||||
|
||||
if not image_url:
|
||||
logger.warning(f"Failed to upload image: {self.file_name}")
|
||||
return Document(content=f"{self.file_name}", images={})
|
||||
|
||||
# Generate markdown text
|
||||
text = f""
|
||||
images = {image_url: base64.b64encode(content).decode()}
|
||||
|
||||
@@ -10,6 +10,7 @@ This module provides comprehensive Markdown parsing functionality including:
|
||||
The parser uses a pipeline approach to process Markdown content through
|
||||
multiple stages: table formatting -> image processing.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
@@ -28,23 +29,23 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class MarkdownTableUtil:
|
||||
"""Utility class for formatting Markdown tables.
|
||||
|
||||
|
||||
This class standardizes Markdown table formatting by:
|
||||
- Normalizing column alignment markers (e.g., :---, :---:, ---:)
|
||||
- Adding consistent spacing around pipes (|)
|
||||
- Preserving indentation levels
|
||||
- Handling both header rows and data rows
|
||||
|
||||
|
||||
Example:
|
||||
Input: |姓名|年龄|城市|
|
||||
|:---|---:|:---:|
|
||||
|张三|25|北京|
|
||||
|
||||
|
||||
Output: | 姓名 | 年龄 | 城市 |
|
||||
| :--- | ---: | :---: |
|
||||
| 张三 | 25 | 北京 |
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self):
|
||||
# Pattern to match alignment row (e.g., |:---|---:|:---:|)
|
||||
self.align_pattern = re.compile(
|
||||
@@ -59,14 +60,14 @@ class MarkdownTableUtil:
|
||||
|
||||
def format_table(self, content: str) -> str:
|
||||
"""Format all Markdown tables in the content.
|
||||
|
||||
|
||||
Args:
|
||||
content: Raw Markdown text containing tables
|
||||
|
||||
|
||||
Returns:
|
||||
Formatted Markdown text with standardized table formatting
|
||||
"""
|
||||
|
||||
|
||||
def process_align(match: Match[str]) -> str:
|
||||
"""Process alignment row to standardize format."""
|
||||
# Split by | and remove empty strings
|
||||
@@ -125,10 +126,10 @@ class MarkdownTableUtil:
|
||||
|
||||
class MarkdownTableFormatter(BaseParser):
|
||||
"""Parser for formatting Markdown tables.
|
||||
|
||||
|
||||
This parser standardizes the formatting of all Markdown tables in the
|
||||
document to ensure consistent spacing and alignment markers.
|
||||
|
||||
|
||||
Example:
|
||||
>>> formatter = MarkdownTableFormatter()
|
||||
>>> content = b"|Name|Age|\n|---|---|\n|John|30|"
|
||||
@@ -138,17 +139,17 @@ class MarkdownTableFormatter(BaseParser):
|
||||
| --- | --- |
|
||||
| John | 30 |
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.table_helper = MarkdownTableUtil()
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse and format Markdown tables.
|
||||
|
||||
|
||||
Args:
|
||||
content: Raw Markdown content as bytes
|
||||
|
||||
|
||||
Returns:
|
||||
Document with formatted table content
|
||||
"""
|
||||
@@ -161,18 +162,18 @@ class MarkdownTableFormatter(BaseParser):
|
||||
|
||||
class MarkdownImageUtil:
|
||||
"""Utility class for handling images in Markdown.
|
||||
|
||||
|
||||
This class provides functionality to:
|
||||
- Extract base64-encoded images from Markdown
|
||||
- Extract image paths from Markdown
|
||||
- Replace image paths with new URLs
|
||||
- Convert base64 images to binary format
|
||||
|
||||
|
||||
Supported formats:
|
||||
- Base64 embedded images: 
|
||||
- Regular image links: 
|
||||
"""
|
||||
|
||||
|
||||
def __init__(self):
|
||||
# Pattern to match base64 embedded images
|
||||
# Captures: (1) alt text, (2) image format, (3) base64 data
|
||||
@@ -191,15 +192,15 @@ class MarkdownImageUtil:
|
||||
replace: bool = True,
|
||||
) -> Tuple[str, List[str]]:
|
||||
"""Extract image paths from Markdown content.
|
||||
|
||||
|
||||
Args:
|
||||
content: Markdown text containing images
|
||||
path_prefix: Optional prefix to add to image paths
|
||||
replace: Whether to replace image syntax in content
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (processed_text, list_of_image_paths)
|
||||
|
||||
|
||||
Example:
|
||||
>>> util = MarkdownImageUtil()
|
||||
>>> text, images = util.extract_image("")
|
||||
@@ -213,7 +214,7 @@ class MarkdownImageUtil:
|
||||
"""Replacement function for each image match."""
|
||||
title = match.group(1) # Alt text
|
||||
image_path = match.group(2) # Image path
|
||||
|
||||
|
||||
# Add prefix if specified
|
||||
if path_prefix:
|
||||
image_path = f"{path_prefix}/{image_path}"
|
||||
@@ -238,19 +239,19 @@ class MarkdownImageUtil:
|
||||
replace: bool = True,
|
||||
) -> Tuple[str, Dict[str, bytes]]:
|
||||
"""Extract and decode base64 embedded images from Markdown.
|
||||
|
||||
|
||||
This method finds all base64-encoded images in the Markdown content,
|
||||
decodes them to binary format, generates unique filenames, and
|
||||
optionally replaces them with file path references.
|
||||
|
||||
|
||||
Args:
|
||||
content: Markdown text containing base64 images
|
||||
path_prefix: Optional directory prefix for generated paths
|
||||
replace: Whether to replace base64 syntax with file paths
|
||||
|
||||
|
||||
Returns:
|
||||
Tuple of (processed_text, dict_of_path_to_bytes)
|
||||
|
||||
|
||||
Example:
|
||||
>>> util = MarkdownImageUtil()
|
||||
>>> text = ""
|
||||
@@ -294,17 +295,17 @@ class MarkdownImageUtil:
|
||||
|
||||
def replace_path(self, content: str, images: Dict[str, str]) -> str:
|
||||
"""Replace image paths in Markdown with new URLs.
|
||||
|
||||
|
||||
This method is typically used to replace local file paths with
|
||||
uploaded URLs after images have been stored.
|
||||
|
||||
|
||||
Args:
|
||||
content: Markdown text with image references
|
||||
images: Mapping of old paths to new URLs
|
||||
|
||||
|
||||
Returns:
|
||||
Markdown text with updated image URLs
|
||||
|
||||
|
||||
Example:
|
||||
>>> util = MarkdownImageUtil()
|
||||
>>> content = ""
|
||||
@@ -320,7 +321,7 @@ class MarkdownImageUtil:
|
||||
"""Replacement function for each image match."""
|
||||
title = match.group(1) # Alt text
|
||||
image_path = match.group(2) # Current image path
|
||||
|
||||
|
||||
# Only replace if path exists in mapping
|
||||
if image_path not in images:
|
||||
return match.group(0) # Keep original
|
||||
@@ -328,7 +329,7 @@ class MarkdownImageUtil:
|
||||
content_replace.add(image_path)
|
||||
# Get new URL from mapping
|
||||
image_path = images[image_path]
|
||||
return f""
|
||||
return f"" if image_path else title
|
||||
|
||||
text = self.replace_pattern.sub(repl, content)
|
||||
logger.debug(f"Replaced {len(content_replace)} images in markdown")
|
||||
@@ -348,16 +349,16 @@ class MarkdownImageUtil:
|
||||
|
||||
class MarkdownImageBase64(BaseParser):
|
||||
"""Parser for extracting and uploading base64 images from Markdown.
|
||||
|
||||
|
||||
This parser:
|
||||
1. Extracts base64-encoded images from Markdown content
|
||||
2. Uploads the decoded images to storage
|
||||
3. Replaces base64 data with uploaded URLs
|
||||
4. Returns a Document with clean Markdown and image mappings
|
||||
|
||||
|
||||
Requires:
|
||||
- self.storage: Storage backend for uploading images
|
||||
|
||||
|
||||
Example:
|
||||
>>> parser = MarkdownImageBase64(storage=my_storage)
|
||||
>>> content = b""
|
||||
@@ -365,17 +366,17 @@ class MarkdownImageBase64(BaseParser):
|
||||
>>> print(doc.content)
|
||||

|
||||
"""
|
||||
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.image_helper = MarkdownImageUtil()
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse Markdown and process base64 images.
|
||||
|
||||
|
||||
Args:
|
||||
content: Raw Markdown content as bytes
|
||||
|
||||
|
||||
Returns:
|
||||
Document with:
|
||||
- content: Markdown with base64 replaced by URLs
|
||||
@@ -411,14 +412,15 @@ class MarkdownImageBase64(BaseParser):
|
||||
|
||||
class MarkdownParser(PipelineParser):
|
||||
"""Complete Markdown parser using pipeline approach.
|
||||
|
||||
|
||||
This parser processes Markdown content through multiple stages:
|
||||
1. MarkdownTableFormatter: Standardizes table formatting
|
||||
2. MarkdownImageBase64: Extracts and uploads base64 images
|
||||
|
||||
|
||||
The pipeline ensures that content flows through each parser in sequence,
|
||||
with each stage's output becoming the next stage's input.
|
||||
"""
|
||||
|
||||
_parser_cls = (MarkdownTableFormatter, MarkdownImageBase64)
|
||||
|
||||
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Dict
|
||||
|
||||
import markdownify
|
||||
import requests
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.chain_parser import PipelineParser
|
||||
@@ -18,19 +18,19 @@ logger = logging.getLogger(__name__)
|
||||
class StdMinerUParser(BaseParser):
|
||||
"""
|
||||
Standard MinerU Parser for document parsing.
|
||||
|
||||
|
||||
This parser uses MinerU API to parse documents (especially PDFs) into markdown format,
|
||||
with support for tables, formulas, and images extraction.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
enable_markdownify: bool = True,
|
||||
mineru_endpoint: str = "",
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Initialize MinerU parser.
|
||||
|
||||
|
||||
Args:
|
||||
enable_markdownify: Whether to convert HTML tables to markdown format
|
||||
mineru_endpoint: MinerU API endpoint URL
|
||||
@@ -38,7 +38,7 @@ class StdMinerUParser(BaseParser):
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
# Get MinerU endpoint from environment variable or parameter
|
||||
self.minerU = os.getenv("MINERU_ENDPOINT", mineru_endpoint)
|
||||
self.minerU = CONFIG.mineru_endpoint
|
||||
self.enable_markdownify = enable_markdownify
|
||||
# Helper for processing markdown images
|
||||
self.image_helper = MarkdownImageUtil()
|
||||
@@ -50,10 +50,10 @@ class StdMinerUParser(BaseParser):
|
||||
def ping(self, timeout: int = 5) -> bool:
|
||||
"""
|
||||
Check if MinerU API is available.
|
||||
|
||||
|
||||
Args:
|
||||
timeout: Request timeout in seconds
|
||||
|
||||
|
||||
Returns:
|
||||
True if API is available, False otherwise
|
||||
"""
|
||||
@@ -69,10 +69,10 @@ class StdMinerUParser(BaseParser):
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse document content into text using MinerU API.
|
||||
|
||||
|
||||
Args:
|
||||
content: Raw document content in bytes
|
||||
|
||||
|
||||
Returns:
|
||||
Document object containing parsed text and images
|
||||
"""
|
||||
@@ -165,24 +165,27 @@ class StdMinerUParser(BaseParser):
|
||||
class MinerUParser(PipelineParser):
|
||||
"""
|
||||
MinerU Parser with pipeline processing.
|
||||
|
||||
This parser combines StdMinerUParser for document parsing and
|
||||
|
||||
This parser combines StdMinerUParser for document parsing and
|
||||
MarkdownTableFormatter for table formatting in a pipeline.
|
||||
"""
|
||||
|
||||
_parser_cls = (StdMinerUParser, MarkdownTableFormatter)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import os
|
||||
|
||||
# Example usage for testing
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
# Configure your file path and MinerU endpoint
|
||||
your_file = "/path/to/your/file.pdf"
|
||||
your_mineru = "http://host.docker.internal:9987"
|
||||
|
||||
os.environ["MINERU_ENDPOINT"] = "http://host.docker.internal:9987"
|
||||
|
||||
# Create parser instance
|
||||
parser = MinerUParser(mineru_endpoint=your_mineru)
|
||||
|
||||
parser = MinerUParser()
|
||||
|
||||
# Parse PDF file
|
||||
with open(your_file, "rb") as f:
|
||||
content = f.read()
|
||||
|
||||
@@ -1,334 +0,0 @@
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import subprocess
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, Union
|
||||
|
||||
import numpy as np
|
||||
from openai import OpenAI
|
||||
from PIL import Image
|
||||
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OCRBackend(ABC):
|
||||
"""Base class for OCR backends"""
|
||||
|
||||
@abstractmethod
|
||||
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
|
||||
"""Extract text from an image
|
||||
|
||||
Args:
|
||||
image: Image file path, bytes, or PIL Image object
|
||||
|
||||
Returns:
|
||||
Extracted text
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class DummyOCRBackend(OCRBackend):
|
||||
"""Dummy OCR backend implementation"""
|
||||
|
||||
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
|
||||
logger.warning("Dummy OCR backend is used")
|
||||
return ""
|
||||
|
||||
|
||||
class PaddleOCRBackend(OCRBackend):
|
||||
"""PaddleOCR backend implementation"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize PaddleOCR backend"""
|
||||
self.ocr = None
|
||||
try:
|
||||
import paddle
|
||||
|
||||
# Set PaddlePaddle to use CPU and disable GPU
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = ""
|
||||
paddle.device.set_device("cpu")
|
||||
|
||||
# Try to detect if CPU supports AVX instruction set
|
||||
# 尝试检测CPU是否支持AVX指令集
|
||||
try:
|
||||
# Detect if CPU supports AVX
|
||||
# 检测CPU是否支持AVX
|
||||
if platform.system() == "Linux":
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["grep", "-o", "avx", "/proc/cpuinfo"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
has_avx = "avx" in result.stdout.lower()
|
||||
if not has_avx:
|
||||
logger.warning(
|
||||
"CPU does not support AVX instructions, "
|
||||
"using compatibility mode"
|
||||
)
|
||||
# Further restrict instruction set usage
|
||||
# 进一步限制指令集使用
|
||||
os.environ["FLAGS_use_avx2"] = "0"
|
||||
os.environ["FLAGS_use_avx"] = "1"
|
||||
except (
|
||||
subprocess.TimeoutExpired,
|
||||
FileNotFoundError,
|
||||
subprocess.SubprocessError,
|
||||
):
|
||||
logger.warning(
|
||||
"Could not detect AVX support, using compatibility mode"
|
||||
)
|
||||
os.environ["FLAGS_use_avx2"] = "0"
|
||||
os.environ["FLAGS_use_avx"] = "1"
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Error detecting CPU capabilities: {e}, using compatibility mode"
|
||||
)
|
||||
os.environ["FLAGS_use_avx2"] = "0"
|
||||
os.environ["FLAGS_use_avx"] = "1"
|
||||
|
||||
from paddleocr import PaddleOCR
|
||||
|
||||
# OCR configuration with text orientation classification enabled
|
||||
ocr_config = {
|
||||
"use_gpu": False,
|
||||
"text_det_limit_type": "max",
|
||||
"text_det_limit_side_len": 960,
|
||||
"use_doc_orientation_classify": True, # Enable document orientation classification / 启用文档方向分类
|
||||
"use_doc_unwarping": False,
|
||||
"use_textline_orientation": True, # Enable text line orientation detection / 启用文本行方向检测
|
||||
"text_recognition_model_name": "PP-OCRv4_server_rec",
|
||||
"text_detection_model_name": "PP-OCRv4_server_det",
|
||||
"text_det_thresh": 0.3,
|
||||
"text_det_box_thresh": 0.6,
|
||||
"text_det_unclip_ratio": 1.5,
|
||||
"text_rec_score_thresh": 0.0,
|
||||
"ocr_version": "PP-OCRv4",
|
||||
"lang": "ch",
|
||||
"show_log": False,
|
||||
"use_dilation": True, # improves accuracy
|
||||
"det_db_score_mode": "slow", # improves accuracy
|
||||
}
|
||||
|
||||
self.ocr = PaddleOCR(**ocr_config)
|
||||
logger.info("PaddleOCR engine initialized successfully")
|
||||
|
||||
except ImportError as e:
|
||||
logger.error(
|
||||
f"Failed to import paddleocr: {str(e)}. "
|
||||
"Please install it with 'pip install paddleocr'"
|
||||
)
|
||||
except OSError as e:
|
||||
if "Illegal instruction" in str(e) or "core dumped" in str(e):
|
||||
logger.error(
|
||||
f"PaddlePaddle crashed due to CPU instruction set incompatibility:"
|
||||
f"{e}"
|
||||
)
|
||||
logger.error(
|
||||
"This happens when the CPU doesn't support AVX instructions. "
|
||||
"Try install CPU-only version of PaddlePaddle, "
|
||||
"or use a different OCR backend."
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
f"Failed to initialize PaddleOCR due to OS error: {str(e)}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize PaddleOCR: {str(e)}")
|
||||
|
||||
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
|
||||
"""Extract text from an image
|
||||
|
||||
Args:
|
||||
image: Image file path, bytes, or PIL Image object
|
||||
|
||||
Returns:
|
||||
Extracted text
|
||||
"""
|
||||
if isinstance(image, str):
|
||||
image = Image.open(image)
|
||||
elif isinstance(image, bytes):
|
||||
image = Image.open(io.BytesIO(image))
|
||||
|
||||
if not isinstance(image, Image.Image):
|
||||
raise TypeError("image must be a string, bytes, or PIL Image object")
|
||||
|
||||
return self._predict(image)
|
||||
|
||||
def _predict(self, image: Image.Image) -> str:
|
||||
"""Perform OCR recognition on the image
|
||||
|
||||
Args:
|
||||
image: Image object (PIL.Image or numpy array)
|
||||
|
||||
Returns:
|
||||
Extracted text string
|
||||
"""
|
||||
if self.ocr is None:
|
||||
logger.error("PaddleOCR engine not initialized")
|
||||
return ""
|
||||
try:
|
||||
# Ensure image is in RGB format
|
||||
if image.mode != "RGB":
|
||||
image = image.convert("RGB")
|
||||
|
||||
# Convert to numpy array for PaddleOCR processing
|
||||
image_array = np.array(image)
|
||||
|
||||
# Perform OCR recognition
|
||||
ocr_result = self.ocr.ocr(image_array, cls=False)
|
||||
|
||||
# Extract and concatenate text from OCR results
|
||||
ocr_text = ""
|
||||
if ocr_result and ocr_result[0]:
|
||||
text = [
|
||||
line[1][0] if line and len(line) >= 2 and line[1] else ""
|
||||
for line in ocr_result[0]
|
||||
]
|
||||
text = [t.strip() for t in text if t]
|
||||
ocr_text = " ".join(text)
|
||||
|
||||
logger.info(f"OCR extracted {len(ocr_text)} characters")
|
||||
return ocr_text
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"OCR recognition error: {str(e)}")
|
||||
return ""
|
||||
|
||||
|
||||
class NanonetsOCRBackend(OCRBackend):
|
||||
"""Nanonets OCR backend implementation using OpenAI API format"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize Nanonets OCR backend
|
||||
|
||||
Args:
|
||||
api_key: API key for OpenAI API
|
||||
base_url: Base URL for OpenAI API
|
||||
model: Model name
|
||||
"""
|
||||
# Load configuration from environment variables
|
||||
base_url = os.getenv("OCR_API_BASE_URL", "http://localhost:8000/v1")
|
||||
api_key = os.getenv("OCR_API_KEY", "123")
|
||||
timeout = 30
|
||||
self.client = OpenAI(api_key=api_key, base_url=base_url, timeout=timeout)
|
||||
|
||||
self.model = os.getenv("OCR_MODEL", "nanonets/Nanonets-OCR-s")
|
||||
logger.info(f"Nanonets OCR engine initialized with model: {self.model}")
|
||||
self.temperature = 0.0
|
||||
self.max_tokens = 15000
|
||||
# Prompt for OCR text extraction with specific formatting requirements
|
||||
self.prompt = """## 任务说明
|
||||
|
||||
请从上传的文档中提取文字内容,严格按自然阅读顺序(从上到下,从左到右)输出,并遵循以下格式规范。
|
||||
|
||||
### 1. **文本处理**
|
||||
|
||||
* 按正常阅读顺序提取文字,语句流畅自然。
|
||||
|
||||
### 2. **表格**
|
||||
|
||||
* 所有表格统一转换为 **Markdown 表格格式**。
|
||||
* 内容保持清晰、对齐整齐,便于阅读。
|
||||
|
||||
### 3. **公式**
|
||||
|
||||
* 所有公式转换为 **LaTeX 格式**,使用 `$$公式$$` 包裹。
|
||||
|
||||
### 4. **图片**
|
||||
|
||||
* 忽略图片信息
|
||||
|
||||
### 5. **链接**
|
||||
|
||||
* 不要猜测或补全不确定的链接地址。
|
||||
"""
|
||||
|
||||
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
|
||||
"""Extract text from an image using Nanonets OCR
|
||||
|
||||
Args:
|
||||
image: Image file path, bytes, or PIL Image object
|
||||
|
||||
Returns:
|
||||
Extracted text
|
||||
"""
|
||||
if self.client is None:
|
||||
logger.error("Nanonets OCR client not initialized")
|
||||
return ""
|
||||
|
||||
try:
|
||||
# Encode image to base64 format for API transmission
|
||||
img_base64 = endecode.decode_image(image)
|
||||
if not img_base64:
|
||||
return ""
|
||||
|
||||
# Call Nanonets OCR API using OpenAI-compatible format
|
||||
logger.info(f"Calling Nanonets OCR API with model: {self.model}")
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": f"data:image/png;base64,{img_base64}"
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": self.prompt,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
temperature=self.temperature,
|
||||
max_tokens=self.max_tokens,
|
||||
)
|
||||
return response.choices[0].message.content or ""
|
||||
except Exception as e:
|
||||
logger.error(f"Nanonets OCR prediction error: {str(e)}")
|
||||
return ""
|
||||
|
||||
|
||||
class OCREngine:
|
||||
"""OCR Engine factory class for managing different OCR backend instances"""
|
||||
|
||||
# Singleton pattern: cache instances for each backend type
|
||||
_instance: Dict[str, OCRBackend] = {}
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls, backend_type: str) -> OCRBackend:
|
||||
"""Get OCR engine instance using factory pattern
|
||||
|
||||
Args:
|
||||
backend_type: OCR backend type, one of: "paddle", "nanonets"
|
||||
**kwargs: Additional arguments for the backend
|
||||
|
||||
Returns:
|
||||
OCR engine instance or None if initialization fails
|
||||
"""
|
||||
backend_type = backend_type.lower()
|
||||
# Return cached instance if already initialized
|
||||
if cls._instance.get(backend_type):
|
||||
return cls._instance[backend_type]
|
||||
|
||||
logger.info(f"Initializing OCR engine with backend: {backend_type}")
|
||||
|
||||
if backend_type == "paddle":
|
||||
cls._instance[backend_type] = PaddleOCRBackend()
|
||||
|
||||
elif backend_type == "nanonets":
|
||||
cls._instance[backend_type] = NanonetsOCRBackend()
|
||||
|
||||
else:
|
||||
cls._instance[backend_type] = DummyOCRBackend()
|
||||
|
||||
return cls._instance[backend_type]
|
||||
@@ -1,7 +1,7 @@
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, Type
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.models.read_config import ChunkingConfig
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
@@ -111,8 +111,8 @@ class Parser:
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
chunking_config=config, # Pass the entire chunking config for advanced options
|
||||
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
|
||||
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
|
||||
max_concurrent_tasks=CONFIG.image_max_concurrent,
|
||||
ocr_backend=CONFIG.ocr_backend,
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
|
||||
@@ -158,8 +158,8 @@ class Parser:
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
chunking_config=config, # Pass the entire chunking config
|
||||
max_concurrent_tasks=int(os.getenv("IMAGE_MAX_CONCURRENT", "1")),
|
||||
ocr_backend=os.getenv("OCR_BACKEND", "paddle"),
|
||||
max_concurrent_tasks=CONFIG.image_max_concurrent,
|
||||
ocr_backend=CONFIG.ocr_backend,
|
||||
)
|
||||
|
||||
logger.info("Starting to parse URL content")
|
||||
|
||||
+32
-22
@@ -10,6 +10,7 @@ from typing import Dict
|
||||
from minio import Minio
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -73,16 +74,14 @@ class CosStorage(Storage):
|
||||
prefix = cos_config.get("path_prefix", "")
|
||||
else:
|
||||
# Get COS configuration from environment variables
|
||||
secret_id = os.getenv("COS_SECRET_ID")
|
||||
secret_key = os.getenv("COS_SECRET_KEY")
|
||||
region = os.getenv("COS_REGION")
|
||||
bucket_name = os.getenv("COS_BUCKET_NAME")
|
||||
appid = os.getenv("COS_APP_ID")
|
||||
prefix = os.getenv("COS_PATH_PREFIX")
|
||||
secret_id = CONFIG.cos_secret_id
|
||||
secret_key = CONFIG.cos_secret_key
|
||||
region = CONFIG.cos_region
|
||||
bucket_name = CONFIG.cos_bucket_name
|
||||
appid = CONFIG.cos_app_id
|
||||
prefix = CONFIG.cos_path_prefix
|
||||
|
||||
enable_old_domain = (
|
||||
os.getenv("COS_ENABLE_OLD_DOMAIN", "true").lower() == "true"
|
||||
)
|
||||
enable_old_domain = CONFIG.cos_enable_old_domain
|
||||
|
||||
if not all([secret_id, secret_key, region, bucket_name, appid]):
|
||||
logger.error(
|
||||
@@ -220,27 +219,27 @@ class MinioStorage(Storage):
|
||||
access_key = (
|
||||
self.storage_config.get("access_key_id")
|
||||
if self.storage_config and self.storage_config.get("access_key_id")
|
||||
else os.getenv("MINIO_ACCESS_KEY_ID")
|
||||
else CONFIG.minio_access_key_id
|
||||
)
|
||||
secret_key = (
|
||||
self.storage_config.get("secret_access_key")
|
||||
if self.storage_config and self.storage_config.get("secret_access_key")
|
||||
else os.getenv("MINIO_SECRET_ACCESS_KEY")
|
||||
else CONFIG.minio_secret_access_key
|
||||
)
|
||||
bucket_name = (
|
||||
self.storage_config.get("bucket_name")
|
||||
if self.storage_config and self.storage_config.get("bucket_name")
|
||||
else os.getenv("MINIO_BUCKET_NAME", "")
|
||||
else CONFIG.minio_bucket_name
|
||||
)
|
||||
path_prefix_raw = (
|
||||
self.storage_config.get("path_prefix")
|
||||
if self.storage_config and self.storage_config.get("path_prefix")
|
||||
else os.getenv("MINIO_PATH_PREFIX", "")
|
||||
else CONFIG.minio_path_prefix
|
||||
)
|
||||
path_prefix = path_prefix_raw.strip().strip("/") if path_prefix_raw else ""
|
||||
|
||||
endpoint = os.getenv("MINIO_ENDPOINT", "")
|
||||
use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true"
|
||||
endpoint = CONFIG.minio_endpoint
|
||||
use_ssl = CONFIG.minio_use_ssl
|
||||
|
||||
if not all([endpoint, access_key, secret_key, bucket_name]):
|
||||
logger.error(
|
||||
@@ -283,7 +282,7 @@ class MinioStorage(Storage):
|
||||
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
|
||||
"""
|
||||
# 1. Use public endpoint if provided
|
||||
endpoint = os.getenv("MINIO_PUBLIC_ENDPOINT")
|
||||
endpoint = CONFIG.minio_public_endpoint
|
||||
if endpoint:
|
||||
return f"{endpoint}/{self.bucket_name}/{object_key}"
|
||||
|
||||
@@ -381,9 +380,7 @@ class LocalStorage(Storage):
|
||||
|
||||
def __init__(self, storage_config: Dict[str, str] = {}):
|
||||
self.storage_config = storage_config
|
||||
base_dir = storage_config.get(
|
||||
"base_dir", os.getenv("LOCAL_STORAGE_BASE_DIR", "")
|
||||
)
|
||||
base_dir = storage_config.get("base_dir", CONFIG.local_storage_base_dir)
|
||||
self.image_dir = os.path.join(base_dir, "images")
|
||||
os.makedirs(self.image_dir, exist_ok=True)
|
||||
|
||||
@@ -410,6 +407,20 @@ class Base64Storage(Storage):
|
||||
return f"data:image/{file_ext};base64,{endecode.decode_image(content)}"
|
||||
|
||||
|
||||
class DummyStorage(Storage):
|
||||
"""Dummy storage implementation.
|
||||
|
||||
It is used in tests or in environments where object storage is disabled.
|
||||
All upload methods return empty string.
|
||||
"""
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
return ""
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
|
||||
"""Create a storage instance based on configuration or environment variables
|
||||
|
||||
@@ -419,7 +430,7 @@ def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
|
||||
Returns:
|
||||
Storage instance
|
||||
"""
|
||||
storage_type = os.getenv("STORAGE_TYPE", "cos").lower()
|
||||
storage_type = CONFIG.storage_type
|
||||
if storage_config:
|
||||
storage_type = str(storage_config.get("provider", storage_type)).lower()
|
||||
logger.info(f"Creating {storage_type} storage instance")
|
||||
@@ -432,5 +443,4 @@ def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
|
||||
return LocalStorage(storage_config or {})
|
||||
elif storage_type == "base64":
|
||||
return Base64Storage()
|
||||
|
||||
raise ValueError(f"Invalid storage type: {storage_type}")
|
||||
return DummyStorage()
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
from trafilatura import extract
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.chain_parser import PipelineParser
|
||||
@@ -30,8 +30,8 @@ class StdWebParser(BaseParser):
|
||||
**kwargs: Additional arguments passed to BaseParser
|
||||
"""
|
||||
self.title = title
|
||||
# Get proxy configuration from environment variable if available
|
||||
self.proxy = os.environ.get("WEB_PROXY", "")
|
||||
# Get proxy configuration from config if available
|
||||
self.proxy = CONFIG.external_https_proxy
|
||||
super().__init__(file_name=title, **kwargs)
|
||||
logger.info(f"Initialized WebParser with title: {title}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user