Weknora/Enterprise-Knowledge-Base
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 新增文档模型类，调整配置与解析逻辑，优化日志及导入

Browse Source 
移除日志设置与冗余代码，优化导入、类型提示及OCR后端管理
统一调整各文件模块导入路径为绝对导入
调整导入路径，移除部分导入，优化日志及注释
升级文档解析器为 Docx2Parser，优化超时与图片处理逻辑
This commit is contained in:
begoniezhao
2025-11-07 10:30:02 +08:00
committed by lyingbug
parent af620806e0
commit 2d66abedf0
39 changed files with 2676 additions and 1570 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docreader/parser/__init__.py
+6 -13
View File
@@ -13,22 +13,18 @@ The parsers extract content from documents and can split them into
meaningful chunks for further processing and indexing.
"""
from .base_parser import BaseParser, ParseResult
from .docx_parser import DocxParser
from .doc_parser import DocParser
from .pdf_parser import PDFParser
from .markdown_parser import MarkdownParser
from .text_parser import TextParser
from .docx2_parser import Docx2Parser
from .image_parser import ImageParser
from .web_parser import WebParser
from .markdown_parser import MarkdownParser
from .parser import Parser
from .config import ChunkingConfig
from .ocr_engine import OCREngine
from .pdf_parser import PDFParser
from .text_parser import TextParser
from .web_parser import WebParser
# Export public classes and modules
__all__ = [
"BaseParser", # Base parser class that all format parsers inherit from
"DocxParser", # Parser for .docx files (modern Word documents)
"Docx2Parser", # Parser for .docx files (modern Word documents)
"DocParser", # Parser for .doc files (legacy Word documents)
"PDFParser", # Parser for PDF documents
"MarkdownParser", # Parser for Markdown text files
@@ -36,7 +32,4 @@ __all__ = [
"ImageParser", # Parser for images with text content
"WebParser", # Parser for web pages
"Parser", # Main parser factory that selects the appropriate parser
"ChunkingConfig", # Configuration for text chunking behavior
"ParseResult", # Standard result format returned by all parsers
"OCREngine", # OCR engine for extracting text from images
]
docreader/parser/base_parser.py
+171 -474
View File
@@ -1,65 +1,28 @@
# -*- coding: utf-8 -*-
import re
import os
import asyncio
from typing import List, Dict, Any, Optional, Tuple, Union
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
import logging
import sys
import traceback
import numpy as np
import time
import io
import json
from .ocr_engine import OCREngine
from .image_utils import image_to_base64
from .config import ChunkingConfig
from .storage import create_storage
import logging
import os
import re
import time
from abc import ABC, abstractmethod
from typing import Dict, List, Optional, Tuple
import requests
from PIL import Image
# Add parent directory to Python path for src imports
current_dir = os.path.dirname(os.path.abspath(__file__))
parent_dir = os.path.dirname(current_dir)
if parent_dir not in sys.path:
sys.path.insert(0, parent_dir)
try:
from services.docreader.src.parser.caption import Caption
except ImportError:
# Fallback: try relative import
try:
from .caption import Caption
except ImportError:
# If both imports fail, set to None
Caption = None
logging.warning(
"Failed to import Caption, image captioning will be unavailable"
)
from docreader.models.document import Chunk, Document
from docreader.models.read_config import ChunkingConfig
from docreader.parser.caption import Caption
from docreader.parser.ocr_engine import OCREngine
from docreader.parser.storage import create_storage
from docreader.splitter.splitter import TextSplitter
from docreader.utils import endecode
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@dataclass
class Chunk:
"""Chunk result"""
content: str # Chunk content
seq: int # Chunk sequence number
start: int # Chunk start position
end: int # Chunk end position
images: List[Dict[str, Any]] = field(default_factory=list) # Images in the chunk
@dataclass
class ParseResult:
"""Parse result"""
text: str # Extracted text content
chunks: Optional[List[Chunk]] = None # Chunk results
class BaseParser(ABC):
"""Base parser interface"""
@@ -97,17 +60,17 @@ class BaseParser(ABC):
def __init__(
self,
file_name: str = "",
file_type: str = None,
file_type: Optional[str] = None,
enable_multimodal: bool = True,
chunk_size: int = 1000,
chunk_overlap: int = 200,
separators: list = ["\n\n", "\n", "。"],
separators: list[str] = ["\n\n", "\n", "。"],
ocr_backend: str = "paddle",
ocr_config: dict = None,
ocr_config: dict = {},
max_image_size: int = 1920, # Maximum image size
max_concurrent_tasks: int = 5, # Max concurrent tasks
max_chunks: int = 1000, # Max number of returned chunks
chunking_config: ChunkingConfig = None, # Chunking configuration object
chunking_config: Optional[ChunkingConfig] = None,
):
"""Initialize parser
@@ -125,7 +88,6 @@ class BaseParser(ABC):
max_chunks: Max number of returned chunks
"""
# Storage client instance
self._storage = None
self.file_name = file_name
self.file_type = file_type or os.path.splitext(file_name)[1]
self.enable_multimodal = enable_multimodal
@@ -133,15 +95,16 @@ class BaseParser(ABC):
self.chunk_overlap = chunk_overlap
self.separators = separators
self.ocr_backend = os.getenv("OCR_BACKEND", ocr_backend)
self.ocr_config = ocr_config or {}
self.ocr_config = ocr_config
self.max_image_size = max_image_size
self.max_concurrent_tasks = max_concurrent_tasks
self.max_chunks = max_chunks
self.chunking_config = chunking_config
logger.info(
f"Initializing {self.__class__.__name__} for file: {file_name}, type: {self.file_type}"
self.storage = create_storage(
self.chunking_config.storage_config if self.chunking_config else None
)
logger.info(f"Initializing parser for file: {file_name}, type: {file_type}")
logger.info(
f"Parser config: chunk_size={chunk_size}, "
f"overlap={chunk_overlap}, "
@@ -150,16 +113,24 @@ class BaseParser(ABC):
f"max_chunks={max_chunks}"
)
# Only initialize Caption service if multimodal is enabled
if self.enable_multimodal:
try:
self.caption_parser = Caption(self.chunking_config.vlm_config)
except Exception as e:
logger.warning(f"Failed to initialize Caption service: {str(e)}")
self.caption_parser = None
else:
self.caption_parser = None
vlm_config = self.chunking_config.vlm_config if self.chunking_config else None
self.caption_parser = (
Caption(vlm_config=vlm_config) if self.enable_multimodal else None
)
def perform_ocr(self, image):
@abstractmethod
def parse_into_text(self, content: bytes) -> Document:
"""Parse document content
Args:
content: Document content
Returns:
Either a string containing the parsed text, or a tuple of (text, image_map)
where image_map is a dict mapping image URLs to Image objects
"""
def perform_ocr(self, image: Image.Image):
"""Execute OCR recognition on the image
Args:
@@ -170,53 +141,23 @@ class BaseParser(ABC):
"""
start_time = time.time()
logger.info("Starting OCR recognition")
resized_image = None
try:
# Resize image to avoid processing large images
resized_image = self._resize_image_if_needed(image)
# Resize image to avoid processing large images
resized_image = self._resize_image_if_needed(image)
# Get OCR engine
ocr_engine = self.get_ocr_engine(
backend_type=self.ocr_backend, **self.ocr_config
)
if ocr_engine is None:
logger.error(
f"OCR engine ({self.ocr_backend}) initialization failed or unavailable, "
"skipping OCR recognition"
)
return ""
# Get OCR engine
ocr_engine = OCREngine.get_instance(self.ocr_backend)
# Execute OCR prediction
logger.info(f"Executing OCR prediction (using {self.ocr_backend} engine)")
# Add extra exception handling
try:
ocr_result = ocr_engine.predict(resized_image)
except RuntimeError as e:
# Handle common CUDA memory issues or other runtime errors
logger.error(f"OCR prediction runtime error: {str(e)}")
return ""
except Exception as e:
# Handle other prediction errors
logger.error(f"Unexpected OCR prediction error: {str(e)}")
return ""
# Execute OCR prediction
logger.info(f"Executing OCR prediction (using {self.ocr_backend} engine)")
ocr_result = ocr_engine.predict(resized_image)
process_time = time.time() - start_time
logger.info(f"OCR recognition completed, time: {process_time:.2f} seconds")
return ocr_result
except Exception as e:
process_time = time.time() - start_time
logger.error(
f"OCR recognition error: {str(e)}, time: {process_time:.2f} seconds"
)
return ""
finally:
# Release image resources
if resized_image is not image and hasattr(resized_image, "close"):
# Only close the new image we created, not the original image
resized_image.close()
process_time = time.time() - start_time
logger.info(f"OCR recognition completed, time: {process_time:.2f} seconds")
def _resize_image_if_needed(self, image):
return ocr_result
def _resize_image_if_needed(self, image: Image.Image) -> Image.Image:
"""Resize image if it exceeds maximum size limit
Args:
@@ -225,102 +166,21 @@ class BaseParser(ABC):
Returns:
Resized image object
"""
try:
# If it's a PIL Image
if hasattr(image, "size"):
width, height = image.size
if width > self.max_image_size or height > self.max_image_size:
logger.info(f"Resizing PIL image, original size: {width}x{height}")
scale = min(
self.max_image_size / width, self.max_image_size / height
)
new_width = int(width * scale)
new_height = int(height * scale)
resized_image = image.resize((new_width, new_height))
logger.info(f"Resized to: {new_width}x{new_height}")
return resized_image
else:
logger.info(
f"PIL image size {width}x{height} is within limits, no resizing needed"
)
return image
# If it's a numpy array
elif hasattr(image, "shape"):
height, width = image.shape[:2]
if width > self.max_image_size or height > self.max_image_size:
logger.info(
f"Resizing numpy image, original size: {width}x{height}"
)
scale = min(
self.max_image_size / width, self.max_image_size / height
)
new_width = int(width * scale)
new_height = int(height * scale)
# Use PIL for resizing numpy arrays
pil_image = Image.fromarray(image)
resized_pil = pil_image.resize((new_width, new_height))
resized_image = np.array(resized_pil)
logger.info(f"Resized to: {new_width}x{new_height}")
return resized_image
else:
logger.info(
f"Numpy image size {width}x{height} is within limits, no resizing needed"
)
return image
else:
logger.warning(f"Unknown image type: {type(image)}, cannot resize")
return image
except Exception as e:
logger.error(f"Error resizing image: {str(e)}")
return image
width, height = image.size
if width > self.max_image_size or height > self.max_image_size:
logger.info(f"Resizing PIL image, original size: {width}x{height}")
scale = min(self.max_image_size / width, self.max_image_size / height)
new_width = int(width * scale)
new_height = int(height * scale)
resized_image = image.resize((new_width, new_height))
logger.info(f"Resized to: {new_width}x{new_height}")
return resized_image
def process_image(self, image, image_url=None):
"""Process image: first perform OCR, then get caption if text is available
logger.info(f"PIL image size is {width}x{height}, no resizing needed")
return image
Args:
image: Image object (PIL.Image or numpy array)
image_url: Image URL (if uploaded)
Returns:
tuple: (ocr_text, caption, image_url)
- ocr_text: OCR extracted text
- caption: Image description (if OCR has text) or empty string
- image_url: Image URL (if provided)
"""
logger.info("Starting image processing (OCR + optional caption)")
# Resize image
image = self._resize_image_if_needed(image)
# Perform OCR recognition
ocr_text = self.perform_ocr(image)
caption = ""
if self.caption_parser:
logger.info(
f"OCR successfully extracted {len(ocr_text)} characters, continuing to get caption"
)
# Convert image to base64 for caption generation
img_base64 = image_to_base64(image)
if img_base64:
caption = self.get_image_caption(img_base64)
if caption:
logger.info(f"Successfully obtained image caption: {caption}")
else:
logger.warning("Failed to get caption")
else:
logger.warning("Failed to convert image to base64")
caption = ""
else:
logger.info("Caption service not initialized, skipping caption retrieval")
# Release image resources
del image
return ocr_text, caption, image_url
async def process_image_async(self, image, image_url=None):
"""Asynchronously process image: first perform OCR, then get caption if text is available
async def process_image_async(self, image: Image.Image, image_url: str):
"""Asynchronously process image: first perform OCR, then get caption
Args:
image: Image object (PIL.Image or numpy array)
@@ -333,84 +193,47 @@ class BaseParser(ABC):
- image_url: Image URL (if provided)
"""
logger.info("Starting asynchronous image processing (OCR + optional caption)")
resized_image = None
# Resize image
resized_image = self._resize_image_if_needed(image)
try:
# Resize image
resized_image = self._resize_image_if_needed(image)
# Perform OCR recognition (using run_in_executor to execute synchronous operations in the event loop)
# Perform OCR recognition
loop = asyncio.get_event_loop()
try:
# Add timeout mechanism to avoid infinite blocking (30 seconds timeout)
ocr_task = loop.run_in_executor(None, self.perform_ocr, resized_image)
ocr_text = await asyncio.wait_for(ocr_task, timeout=30.0)
except asyncio.TimeoutError:
logger.error(
"OCR processing timed out (30 seconds), skipping this image"
)
ocr_text = ""
except Exception as e:
logger.error(f"OCR processing error: {str(e)}")
logger.error(f"OCR processing error, skipping this image: {str(e)}")
ocr_text = ""
logger.info(
f"OCR successfully extracted {len(ocr_text)} characters, continuing to get caption"
)
caption = ""
if self.caption_parser:
try:
# Convert image to base64 for caption generation
img_base64 = image_to_base64(resized_image)
if img_base64:
# Add timeout to avoid blocking caption retrieval (30 seconds timeout)
caption_task = self.get_image_caption_async(img_base64)
image_data, caption = await asyncio.wait_for(
caption_task, timeout=30.0
)
if caption:
logger.info(
f"Successfully obtained image caption: {caption}"
)
else:
logger.warning("Failed to get caption")
else:
logger.warning("Failed to convert image to base64")
caption = ""
except asyncio.TimeoutError:
logger.warning("Caption retrieval timed out, skipping")
except Exception as e:
logger.error(f"Failed to get caption: {str(e)}")
else:
logger.info(
"Caption service not initialized, skipping caption retrieval"
)
logger.info(f"Successfully obtained image ocr: {ocr_text}")
img_base64 = endecode.decode_image(resized_image)
caption = self.get_image_caption(img_base64)
logger.info(f"Successfully obtained image caption: {caption}")
return ocr_text, caption, image_url
finally:
# Release image resources
if resized_image is not image and hasattr(resized_image, "close"):
# Only close the new image we created, not the original image
resized_image.close()
resized_image.close()
async def process_with_limit(self, idx, image, url, semaphore):
async def process_with_limit(
self, idx: int, image: Image.Image, url: str, semaphore: asyncio.Semaphore
):
"""Function to process a single image using a semaphore"""
try:
logger.info(f"Waiting to process image {idx+1}")
logger.info(f"Waiting to process image {idx + 1}")
async with semaphore: # Use semaphore to control concurrency
logger.info(f"Starting to process image {idx+1}")
logger.info(f"Starting to process image {idx + 1}")
result = await self.process_image_async(image, url)
logger.info(f"Completed processing image {idx+1}")
logger.info(f"Completed processing image {idx + 1}")
return result
except Exception as e:
logger.error(f"Error processing image {idx+1}: {str(e)}")
logger.error(f"Error processing image {idx + 1}: {str(e)}")
return ("", "", url) # Return empty result to avoid overall failure
finally:
# Manually release image resources
if hasattr(image, "close"):
image.close()
image.close()
async def process_multiple_images(self, images_data):
async def process_multiple_images(self, images_data: List[Tuple[Image.Image, str]]):
"""Process multiple images concurrently
Args:
@@ -450,7 +273,7 @@ class BaseParser(ABC):
for i, result in enumerate(completed_results):
if isinstance(result, Exception):
logger.error(
f"Image {i+1} processing returned an exception: {str(result)}"
f"Image {i + 1} processing returned an exception: {str(result)}"
)
# For exceptions, add empty results
if i < len(images_data):
@@ -467,47 +290,10 @@ class BaseParser(ABC):
logger.info("Image processing resource cleanup complete")
logger.info(
f"Completed concurrent processing of {len(results)}/{len(images_data)} images"
f"Concurrent processing of {len(results)}/{len(images_data)} images"
)
return results
def decode_bytes(self, content: bytes) -> str:
"""Intelligently decode byte stream, supports multiple encodings
Tries to decode in common encodings, if all fail, uses latin-1 as fallback
Args:
content: Byte stream to decode
Returns:
Decoded string
"""
logger.info(f"Attempting to decode bytes of length: {len(content)}")
# Common encodings, sorted by priority
encodings = ["utf-8", "gb18030", "gb2312", "gbk", "big5", "ascii", "latin-1"]
text = None
# Try decoding with each encoding format
for encoding in encodings:
try:
text = content.decode(encoding)
logger.info(f"Successfully decoded content using {encoding} encoding")
break
except UnicodeDecodeError:
logger.info(f"Failed to decode using {encoding} encoding")
continue
# If all encodings fail, use latin-1 as fallback
if text is None:
text = content.decode("latin-1")
logger.warning(
f"Unable to determine correct encoding, using latin-1 as fallback. "
f"This may cause character issues."
)
logger.info(f"Decoded text length: {len(text)} characters")
return text
def get_image_caption(self, image_data: str) -> str:
"""Get image description
@@ -517,6 +303,9 @@ class BaseParser(ABC):
Returns:
Image description
"""
if not self.caption_parser:
logger.warning("Caption parser not initialized")
return ""
start_time = time.time()
logger.info(
f"Getting caption for image: {image_data[:250]}..."
@@ -533,80 +322,7 @@ class BaseParser(ABC):
logger.warning("Failed to get caption for image")
return caption
async def get_image_caption_async(self, image_data: str) -> Tuple[str, str]:
"""Asynchronously get image description
Args:
image_data: Image data (base64 encoded string or URL)
Returns:
Tuple[str, str]: Image data and corresponding description
"""
caption = self.get_image_caption(image_data)
return image_data, caption
def __init_storage(self):
"""Initialize storage client based on configuration"""
if self._storage is None:
storage_config = (
self.chunking_config.storage_config if self.chunking_config else None
)
self._storage = create_storage(storage_config)
logger.info(
f"Initialized storage client: {self._storage.__class__.__name__}"
)
return self._storage
def upload_file(self, file_path: str) -> str:
"""Upload file to object storage
Args:
file_path: File path
Returns:
File URL
"""
logger.info(f"Uploading file: {file_path}")
try:
storage = self.__init_storage()
return storage.upload_file(file_path)
except Exception as e:
logger.error(f"Failed to upload file: {str(e)}")
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to object storage
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
logger.info(f"Uploading bytes content, size: {len(content)} bytes")
try:
storage = self.__init_storage()
return storage.upload_bytes(content, file_ext)
except Exception as e:
logger.error(f"Failed to upload bytes to storage: {str(e)}")
traceback.print_exc()
return ""
@abstractmethod
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""Parse document content
Args:
content: Document content
Returns:
Either a string containing the parsed text, or a tuple of (text, image_map)
where image_map is a dict mapping image URLs to Image objects
"""
pass
def parse(self, content: bytes) -> ParseResult:
def parse(self, content: bytes) -> Document:
"""Parse document content
Args:
@@ -616,17 +332,19 @@ class BaseParser(ABC):
Parse result
"""
logger.info(
f"Parsing document with {self.__class__.__name__}, content size: {len(content)} bytes"
f"Parsing document with {self.__class__.__name__}, bytes: {len(content)}"
)
parse_result = self.parse_into_text(content)
if isinstance(parse_result, tuple):
text, image_map = parse_result
else:
text = parse_result
image_map = {}
logger.info(f"Extracted {len(text)} characters of text from {self.file_name}")
logger.info(f"Beginning chunking process for text")
chunks = self.chunk_text(text)
document = self.parse_into_text(content)
logger.info(
f"Extracted {len(document.content)} characters from {self.file_name}"
)
splitter = TextSplitter(
chunk_size=self.chunk_size,
chunk_overlap=self.chunk_overlap,
separators=self.separators,
)
chunk_str = splitter.split_text(document.content)
chunks = self._str_to_chunk(chunk_str)
logger.info(f"Created {len(chunks)} chunks from document")
# Limit the number of returned chunks
@@ -636,7 +354,7 @@ class BaseParser(ABC):
)
chunks = chunks[: self.max_chunks]
# If multimodal is enabled and file type is supported, process images in each chunk
# If multimodal is enabled and file type is supported, process images
if self.enable_multimodal:
# Get file extension and convert to lowercase
file_ext = (
@@ -647,11 +365,12 @@ class BaseParser(ABC):
# Define allowed file types for image processing
allowed_types = [
".pdf", # PDF files
# Text files
".pdf",
".md",
".markdown", # Markdown files
".markdown",
".doc",
".docx", # Word documents
".docx",
# Image files
".jpg",
".jpeg",
@@ -666,13 +385,21 @@ class BaseParser(ABC):
logger.info(
f"Processing images in each chunk for file type: {file_ext}"
)
chunks = self.process_chunks_images(chunks, image_map)
chunks = self.process_chunks_images(chunks, document.images)
else:
logger.info(
f"Skipping image processing for unsupported file type: {file_ext}"
)
return ParseResult(text=text, chunks=chunks)
document.chunks = chunks
return document
def _str_to_chunk(self, text: List[Tuple[int, int, str]]) -> List[Chunk]:
"""Convert string to Chunk object"""
return [
Chunk(seq=i, content=t, start=start, end=end)
for i, (start, end, t) in enumerate(text)
]
def _split_into_units(self, text: str) -> List[str]:
"""
@@ -682,9 +409,7 @@ class BaseParser(ABC):
Returns:
基本单元的列表
"""
logger.info(
f"Splitting text into basic units with robust structure protection, text length: {len(text)}"
)
logger.info(f"Splitting text into basic units, text length: {len(text)}")
# 定义所有需要作为整体保护的结构模式 ---
table_pattern = r"(?m)(^\|.*\|[ \t]*\r?\n(?:[ \t]*\r?\n)?^\|\s*:?--+.*\r?\n(?:^\|.*\|\r?\n?)*)"
@@ -710,7 +435,8 @@ class BaseParser(ABC):
# 按起始位置排序
protected_ranges.sort(key=lambda x: x[0])
logger.info(
f"Found {len(protected_ranges)} protected structures (tables, code, formulas, images, links)."
f"Found {len(protected_ranges)} protected structures "
"(tables, code, formulas, images, links)."
)
# 合并可能重叠的保护范围 ---
@@ -731,7 +457,7 @@ class BaseParser(ABC):
merged_ranges.append((current_start, current_end))
protected_ranges = merged_ranges
logger.info(
f"After merging overlaps, {len(protected_ranges)} protected ranges remain."
f"After overlaps, {len(protected_ranges)} protected ranges remain."
)
# 根据保护范围和分隔符来分割文本 ---
@@ -749,7 +475,7 @@ class BaseParser(ABC):
segments = re.split(separator_pattern, pre_text)
units.extend([s for s in segments if s]) # 添加所有非空部分
# b. 将整个受保护的块（例如，一个完整的表格）作为一个单独的、不可分割的单元添加
# b. 将整个受保护的块（例如，一个完整的表格）作为一个不可分割的单元添加
protected_text = text[start:end]
units.append(protected_text)
@@ -764,38 +490,6 @@ class BaseParser(ABC):
logger.info(f"Text splitting complete, created {len(units)} final basic units.")
return units
def _find_complete_units(self, units: List[str], target_size: int) -> List[str]:
"""Find a list of complete units that do not exceed the target size
Args:
units: List of units
target_size: Target size
Returns:
List of complete units
"""
logger.info(f"Finding complete units with target size: {target_size}")
result = []
current_size = 0
for unit in units:
unit_size = len(unit)
if current_size + unit_size > target_size and result:
logger.info(
f"Reached target size limit at {current_size} characters, stopping"
)
break
result.append(unit)
current_size += unit_size
logger.info(
f"Added unit of size {unit_size}, current total: {current_size}/{target_size}"
)
logger.info(
f"Found {len(result)} complete units totaling {current_size} characters"
)
return result
def chunk_text(self, text: str) -> List[Chunk]:
"""Chunk text, preserving Markdown structure
@@ -825,7 +519,7 @@ class BaseParser(ABC):
for i, unit in enumerate(units):
unit_size = len(unit)
logger.info(f"Processing unit {i+1}/{len(units)}, size: {unit_size}")
logger.info(f"Processing unit {i + 1}/{len(units)}, size: {unit_size}")
# If current chunk plus new unit exceeds size limit, create new chunk
if current_size + unit_size > self.chunk_size and current_chunk:
@@ -855,14 +549,12 @@ class BaseParser(ABC):
for u in reversed(current_chunk):
if overlap_size + len(u) > overlap_target:
logger.info(
f"Reached overlap target ({overlap_size}/{overlap_target})"
f"Overlap target ({overlap_size}/{overlap_target})"
)
break
overlap_units.insert(0, u)
overlap_size += len(u)
logger.info(
f"Added unit to overlap, current overlap size: {overlap_size}"
)
logger.info(f"Added unit to overlap, size: {overlap_size}")
# Remove elements from overlap that are included in separators
start_index = 0
@@ -883,7 +575,7 @@ class BaseParser(ABC):
overlap_units = overlap_units[start_index:]
logger.info(
f"Final overlap: {len(overlap_units)} units, {overlap_size} characters"
f"Overlap: {len(overlap_units)} units, {overlap_size} size"
)
current_chunk = overlap_units
@@ -899,7 +591,7 @@ class BaseParser(ABC):
current_chunk.append(unit)
current_size += unit_size
logger.info(
f"Added unit to current chunk, now at {current_size}/{self.chunk_size} characters"
f"Added unit to current chunk, at {current_size}/{self.chunk_size}"
)
# Add the last chunk
@@ -925,12 +617,13 @@ class BaseParser(ABC):
chunk: Document chunk
Returns:
List of image information, each element contains image URL and match position
List of image information
"""
logger.info(f"Extracting image information from Chunk #{chunk.seq}")
text = chunk.content
# Regex to extract image information from text, supporting Markdown images and HTML images
# Regex to extract image information from text,
# support: Markdown images, HTML images
img_pattern = r'!\[([^\]]*)\]\(([^)]+)\)|<img [^>]*src="([^"]+)" [^>]*>'
# Extract image information
@@ -954,28 +647,28 @@ class BaseParser(ABC):
images_info.append(image_info)
logger.info(
f"Image in Chunk #{chunk.seq} {match_idx+1}: " f"URL={img_url[:50]}..."
f"Image in Chunk #{chunk.seq} {match_idx + 1}: URL={img_url[:50]}..."
if len(img_url) > 50
else f"Image in Chunk #{chunk.seq} {match_idx+1}: URL={img_url}"
else f"Image in Chunk #{chunk.seq} {match_idx + 1}: URL={img_url}"
)
return images_info
async def download_and_upload_image(self, img_url: str):
"""Download image and upload to object storage, if it's already an object storage path or local path, use directly
async def download_and_upload_image(
self, img_url: str
) -> Tuple[str, str, Image.Image | None]:
"""Download image and upload to object storage,
if it's already an object storage path or local path, use directly
Args:
img_url: Image URL or local path
Returns:
tuple: (original URL, storage URL, image object), if failed returns (original URL, None, None)
tuple: (original URL, storage URL, image object),
if failed returns (original URL, None, None)
"""
try:
import requests
from PIL import Image
import io
# Check if it's already a storage URL (COS or MinIO)
is_storage_url = any(
pattern in img_url
@@ -997,12 +690,7 @@ class BaseParser(ABC):
response = requests.get(img_url, timeout=5, proxies=proxies)
if response.status_code == 200:
image = Image.open(io.BytesIO(response.content))
try:
return img_url, img_url, image
finally:
# Ensure image resources are also released after the function returns
# Image will be closed by the caller
pass
return img_url, img_url, image
else:
logger.warning(
f"Failed to get storage image: {response.status_code}"
@@ -1022,7 +710,7 @@ class BaseParser(ABC):
# Upload to storage
with open(img_url, "rb") as f:
content = f.read()
storage_url = self.upload_bytes(content)
storage_url = self.storage.upload_bytes(content)
logger.info(
f"Successfully uploaded local image to storage: {storage_url}"
)
@@ -1031,7 +719,7 @@ class BaseParser(ABC):
logger.error(f"Error processing local image: {str(e)}")
if image and hasattr(image, "close"):
image.close()
return img_url, None, None
return img_url, img_url, None
# Normal remote URL download handling
else:
@@ -1044,9 +732,7 @@ class BaseParser(ABC):
if https_proxy:
proxies["https"] = https_proxy
logger.info(
f"Downloading image {img_url}, using proxy: {proxies if proxies else 'None'}"
)
logger.info(f"Downloading image {img_url}, using proxy: {proxies}")
response = requests.get(img_url, timeout=5, proxies=proxies)
if response.status_code == 200:
@@ -1054,7 +740,7 @@ class BaseParser(ABC):
image = Image.open(io.BytesIO(response.content))
try:
# Upload to storage using the method in BaseParser
storage_url = self.upload_bytes(response.content)
storage_url = self.storage.upload_bytes(response.content)
logger.info(
f"Successfully uploaded image to storage: {storage_url}"
)
@@ -1064,11 +750,11 @@ class BaseParser(ABC):
pass
else:
logger.warning(f"Failed to download image: {response.status_code}")
return img_url, None, None
return img_url, img_url, None
except Exception as e:
logger.error(f"Error downloading or processing image: {str(e)}")
return img_url, None, None
return img_url, img_url, None
async def process_chunk_images_async(
self, chunk, chunk_idx, total_chunks, image_map=None
@@ -1086,18 +772,19 @@ class BaseParser(ABC):
"""
logger.info(
f"Starting to process images in Chunk #{chunk_idx+1}/{total_chunks}"
f"Starting to process images in Chunk #{chunk_idx + 1}/{total_chunks}"
)
# Extract image information from the Chunk
images_info = self.extract_images_from_chunk(chunk)
if not images_info:
logger.info(f"Chunk #{chunk_idx+1} found no images")
logger.info(f"Chunk #{chunk_idx + 1} found no images")
return chunk
# Prepare images that need to be downloaded and processed
images_to_process = []
url_to_info_map = {} # Map URL to image information
# Map URL to image information
url_to_info_map = {}
# Record all image URLs that need to be processed
for img_info in images_info:
@@ -1106,14 +793,21 @@ class BaseParser(ABC):
results = []
download_tasks = []
for img_url in url_to_info_map.keys(): # Check if image is already in the image_map
# Check if image is already in the image_map
for img_url in url_to_info_map.keys():
if image_map and img_url in image_map:
logger.info(f"Image already in image_map: {img_url}, using cached object")
results.append((img_url, img_url, image_map[img_url]))
logger.info(
f"Image already in image_map: {img_url}, using cached object"
)
image = Image.open(
io.BytesIO(endecode.encode_image(image_map[img_url]))
)
results.append((img_url, img_url, image))
else:
download_task = self.download_and_upload_image(img_url)
download_tasks.append(download_task)
# Concurrent download and upload of images, ignore images that are already in the image_map
# Concurrent download and upload of images,
# ignore images that are already in the image_map
results.extend(await asyncio.gather(*download_tasks))
# Process download results, prepare for OCR processing
@@ -1123,16 +817,17 @@ class BaseParser(ABC):
img_info["cos_url"] = cos_url
images_to_process.append((image, cos_url))
# If no images were successfully downloaded and uploaded, return the original Chunk
# If no images were successfully downloaded and uploaded,
# return the original Chunk
if not images_to_process:
logger.info(
f"Chunk #{chunk_idx+1} found no successfully downloaded and uploaded images"
f"Chunk #{chunk_idx + 1} not found downloaded and uploaded images"
)
return chunk
# Concurrent processing of all images (OCR + caption)
logger.info(
f"Processing {len(images_to_process)} images in Chunk #{chunk_idx+1}"
f"Processing {len(images_to_process)} images in Chunk #{chunk_idx + 1}"
)
# Concurrent processing of all images
@@ -1163,10 +858,12 @@ class BaseParser(ABC):
# Update image information in the Chunk
chunk.images = processed_images
logger.info(f"Completed image processing in Chunk #{chunk_idx+1}")
logger.info(f"Completed image processing in Chunk #{chunk_idx + 1}")
return chunk
def process_chunks_images(self, chunks: List[Chunk], image_map=None) -> List[Chunk]:
def process_chunks_images(
self, chunks: List[Chunk], image_map: Dict[str, str] = {}
) -> List[Chunk]:
"""Concurrent processing of images in all Chunks
Args:
@@ -1210,7 +907,7 @@ class BaseParser(ABC):
processed_chunks = []
for i, result in enumerate(results):
if isinstance(result, Exception):
logger.error(f"Error processing Chunk {i+1}: {str(result)}")
logger.error(f"Error processing Chunk {i + 1}: {str(result)}")
# Keep original Chunk
if i < len(chunks):
processed_chunks.append(chunks[i])
@@ -1235,7 +932,7 @@ class BaseParser(ABC):
# Execute processing for all Chunks
processed_chunks = loop.run_until_complete(process_all_chunks())
logger.info(
f"Successfully completed concurrent processing of {len(processed_chunks)}/{len(chunks)} chunks"
f"Completed processing of {len(processed_chunks)}/{len(chunks)} chunks"
)
return processed_chunks
docreader/parser/caption.py
+59 -46
View File
@@ -3,11 +3,10 @@ import logging
import os
import time
from dataclasses import dataclass, field
from typing import List, Optional, Union
from typing import Dict, List, Optional, Union
import requests
import ollama
import requests
logger = logging.getLogger(__name__)
@@ -158,11 +157,16 @@ class CaptionChatResp:
Returns:
The content string from the first choice, or empty string if no choices
"""
if self.choices:
logger.info("Retrieving content from first choice")
return self.choices[0].message.content
logger.warning("No choices available in response")
return ""
if (
not self.choices
or not self.choices[0]
or not self.choices[0].message
or not self.choices[0].message.content
):
logger.warning("No choices available in response")
return ""
logger.info("Retrieving content from first choice")
return self.choices[0].message.content
class Caption:
@@ -171,33 +175,43 @@ class Caption:
Uses an external API to process images and return textual descriptions.
"""
def __init__(self, vlm_config=None):
"""Initialize the Caption service with configuration from parameters or environment variables."""
def __init__(self, vlm_config: Optional[Dict[str, str]] = None):
"""
Initialize the Caption service with configuration
from parameters or environment variables.
"""
logger.info("Initializing Caption service")
self.prompt = """简单凝炼的描述图片的主要内容"""
# Use provided VLM config if available, otherwise fall back to environment variables
self.timeout = 30
# Use provided VLM config if available,
# otherwise fall back to environment variables
if vlm_config and vlm_config.get("base_url") and vlm_config.get("model_name"):
self.completion_url = vlm_config.get("base_url", "") + "/chat/completions"
self.model = vlm_config.get("model_name", "")
self.api_key = vlm_config.get("api_key", "")
self.interface_type = vlm_config.get("interface_type", "openai").lower()
else:
if os.getenv("VLM_MODEL_BASE_URL") == "" or os.getenv("VLM_MODEL_NAME") == "":
base_url = os.getenv("VLM_MODEL_BASE_URL")
model_name = os.getenv("VLM_MODEL_NAME")
if not base_url or not model_name:
logger.error("VLM_MODEL_BASE_URL or VLM_MODEL_NAME is not set")
return
self.completion_url = os.getenv("VLM_MODEL_BASE_URL") + "/chat/completions"
self.model = os.getenv("VLM_MODEL_NAME")
self.api_key = os.getenv("VLM_MODEL_API_KEY")
self.completion_url = base_url + "/chat/completions"
self.model = model_name
self.api_key = os.getenv("VLM_MODEL_API_KEY", "")
self.interface_type = os.getenv("VLM_INTERFACE_TYPE", "openai").lower()
# 验证接口类型
if self.interface_type not in ["ollama", "openai"]:
logger.warning(f"Unknown interface type: {self.interface_type}, defaulting to openai")
logger.warning(
f"Unknown interface type: {self.interface_type}, defaulting to openai"
)
self.interface_type = "openai"
logger.info(
f"Service configured with model: {self.model}, endpoint: {self.completion_url}, interface: {self.interface_type}"
f"Configured with model: {self.model}, "
f"endpoint: {self.completion_url}, interface: {self.interface_type}"
)
def _call_caption_api(self, image_data: str) -> Optional[CaptionChatResp]:
@@ -210,8 +224,8 @@ class Caption:
Returns:
CaptionChatResp object if successful, None otherwise
"""
logger.info(f"Calling Caption API for image captioning")
logger.info(f"Processing image data: {image_data[:50] if len(image_data) > 50 else image_data}")
logger.info("Calling Caption API for image captioning")
logger.info(f"Processing image data: {image_data[:50]}...")
# 根据接口类型选择调用方式
if self.interface_type == "ollama":
@@ -226,39 +240,35 @@ class Caption:
client = ollama.Client(
host=host,
timeout=self.timeout,
)
try:
logger.info(f"Calling Ollama API with model: {self.model}")
# 调用Ollama API，使用images参数传递base64编码的图片
response = client.generate(
model=self.model,
prompt="简单凝炼的描述图片的主要内容",
images=[image_base64], # image_base64是base64编码的图片数据
images=[image_base64], # image_base64是base64编码的图片数据
options={"temperature": 0.1},
stream=False,
)
# 构造响应对象
caption_resp = CaptionChatResp(
id="ollama_response",
created=int(time.time()),
model=self.model,
model=Model(id=self.model),
object="chat.completion",
choices=[
Choice(
message=Message(
role="assistant",
content=response.response
)
)
]
Choice(message=Message(role="assistant", content=response.response))
],
)
logger.info("Successfully received response from Ollama API")
return caption_resp
except Exception as e:
logger.error(f"Error calling Ollama API: {e}")
return None
@@ -266,13 +276,16 @@ class Caption:
def _call_openai_api(self, image_base64: str) -> Optional[CaptionChatResp]:
"""Call OpenAI-compatible API for image captioning."""
logger.info(f"Calling OpenAI-compatible API with model: {self.model}")
user_msg = UserMessage(
role="user",
content=[
Content(type="text", text=self.prompt),
Content(
type="image_url", image_url=ImageUrl(url="data:image/png;base64," + image_base64, detail="auto")
type="image_url",
image_url=ImageUrl(
url="data:image/png;base64," + image_base64, detail="auto"
),
),
],
)
@@ -295,23 +308,23 @@ class Caption:
headers["Authorization"] = f"Bearer {self.api_key}"
try:
logger.info(f"Sending request to OpenAI-compatible API with model: {self.model}")
logger.info(
f"Sending request to OpenAI-compatible API with model: {self.model}"
)
response = requests.post(
self.completion_url,
data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4),
headers=headers,
timeout=30,
timeout=self.timeout,
)
if response.status_code != 200:
logger.error(
f"OpenAI-compatible API returned non-200 status code: {response.status_code}"
f"OpenAI API returned non-200 status code: {response.status_code}"
)
response.raise_for_status()
logger.info(
f"Successfully received response from OpenAI-compatible API with status: {response.status_code}"
)
logger.info(f"Converting response to CaptionChatResp object")
logger.info(f"Received from OpenAI with status: {response.status_code}")
logger.info("Converting response to CaptionChatResp object")
caption_resp = CaptionChatResp.from_json(response.json())
if caption_resp.usage:
@@ -322,7 +335,7 @@ class Caption:
return caption_resp
except requests.exceptions.Timeout:
logger.error(f"Timeout while calling OpenAI-compatible API after 30 seconds")
logger.error("Timeout while calling OpenAI-compatible API after 30 seconds")
return None
except requests.exceptions.RequestException as e:
logger.error(f"Request error calling OpenAI-compatible API: {e}")
docreader/parser/chain_parser.py
+70
View File
@@ -0,0 +1,70 @@
import logging
from typing import List, Tuple, Type
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.utils import endecode
logger = logging.getLogger(__name__)
class FirstParser(BaseParser):
_parser_cls: Tuple[Type["BaseParser"], ...] = ()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._parsers: List[BaseParser] = []
for parser_cls in self._parser_cls:
try:
parser = parser_cls(*args, **kwargs)
self._parsers.append(parser)
except Exception as e:
logger.error(f"Failed to create parser {parser_cls.__name__}: {e}")
def parse_into_text(self, content: bytes) -> Document:
for p in self._parsers:
document = p.parse_into_text(content)
if document.is_valid():
return document
return Document()
@classmethod
def create(cls, *parser_classes: Type["BaseParser"]) -> Type["FirstParser"]:
names = "_".join([p.__name__ for p in parser_classes])
return type(f"FirstParser_{names}", (cls,), {"_parser_cls": parser_classes})
class PipelineParser(BaseParser):
_parser_cls: Tuple[Type["BaseParser"], ...] = ()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._parsers: List[BaseParser] = []
for parser_cls in self._parser_cls:
try:
parser = parser_cls(*args, **kwargs)
self._parsers.append(parser)
except Exception as e:
logger.error(f"Failed to create parser {parser_cls.__name__}: {e}")
def parse_into_text(self, content: bytes) -> Document:
document = Document()
for p in self._parsers:
document = p.parse_into_text(content)
content = endecode.encode_bytes(document.content)
return document
@classmethod
def create(cls, *parser_classes: Type["BaseParser"]) -> Type["PipelineParser"]:
names = "_".join([p.__name__ for p in parser_classes])
return type(f"PipelineParser_{names}", (cls,), {"_parser_cls": parser_classes})
if __name__ == "__main__":
from docreader.parser.markdown_parser import MarkdownParser
cls = FirstParser.create(MarkdownParser)
parser = cls()
print(parser.parse_into_text(b"aaa"))
docreader/parser/config.py
-21
View File
@@ -1,21 +0,0 @@
from dataclasses import dataclass, field
@dataclass
class ChunkingConfig:
"""
Configuration for text chunking process.
Controls how documents are split into smaller pieces for processing.
"""
chunk_size: int = 512 # Maximum size of each chunk in tokens/chars
chunk_overlap: int = 50 # Number of tokens/chars to overlap between chunks
separators: list = field(
default_factory=lambda: ["\n\n", "\n", "。"]
) # Text separators in order of priority
enable_multimodal: bool = (
False # Whether to enable multimodal processing (text + images)
)
storage_config: dict = None # Preferred field name going forward
vlm_config: dict = None # VLM configuration for image captioning
docreader/parser/doc_parser.py
+141 -209
View File
@@ -1,134 +1,88 @@
import asyncio
import logging
import re
import tempfile
import os
import subprocess
import shutil
from io import BytesIO
from typing import Optional, List, Tuple
import textract
from PIL import Image
import zipfile
import xml.etree.ElementTree as ET
from typing import List, Optional
from .base_parser import BaseParser
from .docx_parser import DocxParser, Docx
import textract
from docreader.models.document import Document
from docreader.parser.docx2_parser import Docx2Parser
from docreader.utils.tempfile import TempDirContext, TempFileContext
logger = logging.getLogger(__name__)
class DocParser(BaseParser):
class DocParser(Docx2Parser):
"""DOC document parser"""
def parse_into_text(self, content: bytes) -> str:
"""Parse DOC document
Args:
content: DOC document content
Returns:
Parse result
"""
def parse_into_text(self, content: bytes) -> Document:
logger.info(f"Parsing DOC document, content size: {len(content)} bytes")
handle_chain = [
# 1. Try to convert to docx format to extract images
self._parse_with_docx,
# 2. If image extraction is not needed or conversion failed,
# try using antiword to extract text
self._parse_with_antiword,
# 3. If antiword extraction fails, use textract
self._parse_with_textract,
]
# Save byte content as a temporary file
with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as temp_file:
temp_file_path = temp_file.name
temp_file.write(content)
temp_file.flush()
logger.info(f"Saved DOC content to temporary file: {temp_file_path}")
with TempFileContext(content, ".doc") as temp_file_path:
for handle in handle_chain:
try:
document = handle(temp_file_path)
if document:
return document
except Exception as e:
logger.warning(f"Failed to parse DOC with {handle.__name__} {e}")
try:
# First try to convert to docx format to extract images
if self.enable_multimodal:
logger.info("Multimodal enabled, attempting to extract images from DOC")
docx_content = self._convert_doc_to_docx(temp_file_path)
return Document(content="")
if docx_content:
logger.info("Successfully converted DOC to DOCX, using DocxParser")
# Use existing DocxParser to parse the converted docx
docx_parser = DocxParser(
file_name=self.file_name,
file_type="docx",
enable_multimodal=self.enable_multimodal,
chunk_size=self.chunk_size,
chunk_overlap=self.chunk_overlap,
chunking_config=self.chunking_config,
separators=self.separators,
)
text = docx_parser.parse_into_text(docx_content)
logger.info(f"Extracted {len(text)} characters using DocxParser")
def _parse_with_docx(self, temp_file_path: str) -> Document:
logger.info("Multimodal enabled, attempting to extract images from DOC")
# Clean up temporary file
os.unlink(temp_file_path)
logger.info(f"Deleted temporary file: {temp_file_path}")
docx_content = self._try_convert_doc_to_docx(temp_file_path)
if not docx_content:
raise RuntimeError("Failed to convert DOC to DOCX")
return text
else:
logger.warning(
"Failed to convert DOC to DOCX, falling back to text-only extraction"
)
logger.info("Successfully converted DOC to DOCX, using DocxParser")
# Use existing DocxParser to parse the converted docx
document = super(Docx2Parser, self).parse_into_text(docx_content)
logger.info(f"Extracted {len(document.content)} characters using DocxParser")
return document
# If image extraction is not needed or conversion failed, try using antiword to extract text
try:
logger.info("Attempting to parse DOC file with antiword")
# Check if antiword is installed
antiword_path = self._find_antiword_path()
def _parse_with_antiword(self, temp_file_path: str) -> Document:
logger.info("Attempting to parse DOC file with antiword")
if antiword_path:
# Use antiword to extract text directly
logger.info(f"Using antiword at {antiword_path} to extract text")
process = subprocess.Popen(
[antiword_path, temp_file_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
# Check if antiword is installed
antiword_path = self._try_find_antiword()
if not antiword_path:
raise RuntimeError("antiword not found in PATH")
if process.returncode == 0:
text = stdout.decode("utf-8", errors="ignore")
logger.info(
f"Successfully extracted {len(text)} characters using antiword"
)
# Clean up temporary file
os.unlink(temp_file_path)
logger.info(f"Deleted temporary file: {temp_file_path}")
return text
else:
logger.warning(
f"antiword extraction failed: {stderr.decode('utf-8', errors='ignore')}"
)
else:
logger.warning("antiword not found, falling back to textract")
except Exception as e:
logger.warning(
f"Error using antiword: {str(e)}, falling back to textract"
)
# If antiword fails, try using textract
logger.info("Parsing DOC file with textract")
text = textract.process(temp_file_path, method="antiword").decode("utf-8")
logger.info(
f"Successfully extracted {len(text)} characters of text from DOC document using textract"
# Use antiword to extract text directly
process = subprocess.Popen(
[antiword_path, temp_file_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
if process.returncode != 0:
raise RuntimeError(
f"antiword extraction failed: {stderr.decode('utf-8', errors='ignore')}"
)
text = stdout.decode("utf-8", errors="ignore")
logger.info(f"Successfully extracted {len(text)} characters using antiword")
return Document(content=text)
# Clean up temporary file
os.unlink(temp_file_path)
logger.info(f"Deleted temporary file: {temp_file_path}")
def _parse_with_textract(self, temp_file_path: str) -> Document:
logger.info(f"Parsing DOC file with textract: {temp_file_path}")
text = textract.process(temp_file_path, method="antiword").decode("utf-8")
logger.info(f"Successfully extracted {len(text)} bytes of DOC using textract")
return Document(content=str(text))
return text
except Exception as e:
logger.error(f"Error parsing DOC document: {str(e)}")
# Ensure temporary file is cleaned up
if os.path.exists(temp_file_path):
os.unlink(temp_file_path)
logger.info(f"Deleted temporary file after error: {temp_file_path}")
return ""
def _convert_doc_to_docx(self, doc_path: str) -> Optional[bytes]:
def _try_convert_doc_to_docx(self, doc_path: str) -> Optional[bytes]:
"""Convert DOC file to DOCX format
Uses LibreOffice/OpenOffice for conversion
@@ -141,21 +95,16 @@ class DocParser(BaseParser):
"""
logger.info(f"Converting DOC to DOCX: {doc_path}")
# Check if LibreOffice or OpenOffice is installed
soffice_path = self._try_find_soffice()
if not soffice_path:
return None
# Execute conversion command
logger.info(f"Using {soffice_path} to convert DOC to DOCX")
# Create a temporary directory to store the converted file
temp_dir = tempfile.mkdtemp()
docx_path = os.path.join(temp_dir, "converted.docx")
try:
# Check if LibreOffice or OpenOffice is installed
soffice_path = self._find_soffice_path()
if not soffice_path:
logger.error(
"LibreOffice/OpenOffice not found, cannot convert DOC to DOCX"
)
return None
# Execute conversion command
logger.info(f"Using {soffice_path} to convert DOC to DOCX")
with TempDirContext() as temp_dir:
cmd = [
soffice_path,
"--headless",
@@ -165,7 +114,6 @@ class DocParser(BaseParser):
temp_dir,
doc_path,
]
logger.info(f"Running command: {' '.join(cmd)}")
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
@@ -173,41 +121,68 @@ class DocParser(BaseParser):
stdout, stderr = process.communicate()
if process.returncode != 0:
logger.error(
f"Error converting DOC to DOCX: {stderr.decode('utf-8', errors='ignore')}"
logger.warning(
f"Error converting DOC to DOCX: {stderr.decode('utf-8')}"
)
return None
# Find the converted file
for file in os.listdir(temp_dir):
if file.endswith(".docx"):
converted_file = os.path.join(temp_dir, file)
logger.info(f"Found converted file: {converted_file}")
# Read the converted file content
with open(converted_file, "rb") as f:
docx_content = f.read()
docx_file = [
file for file in os.listdir(temp_dir) if file.endswith(".docx")
]
logger.info(f"Found {len(docx_file)} DOCX file(s) in temporary directory")
for file in docx_file:
converted_file = os.path.join(temp_dir, file)
logger.info(f"Found converted file: {converted_file}")
# Read the converted file content
with open(converted_file, "rb") as f:
docx_content = f.read()
logger.info(
f"Successfully read converted DOCX file, size: {len(docx_content)} bytes"
f"Successfully read DOCX file, size: {len(docx_content)}"
)
return docx_content
return None
logger.error("No DOCX file found after conversion")
return None
def _try_find_executable_path(
self,
executable_name: str,
possible_path: List[str] = [],
environment_variable: List[str] = [],
) -> Optional[str]:
"""Find executable path
Args:
executable_name: Executable name
possible_path: List of possible paths
environment_variable: List of environment variables to check
Returns:
Executable path, or None if not found
"""
# Common executable paths
paths: List[str] = []
paths.extend(possible_path)
paths.extend(os.environ.get(env_var, "") for env_var in environment_variable)
paths = list(set(paths))
except Exception as e:
logger.error(f"Error during DOC to DOCX conversion: {str(e)}")
return None
finally:
# Clean up temporary directory
try:
shutil.rmtree(temp_dir)
logger.info(f"Cleaned up temporary directory: {temp_dir}")
except Exception as e:
logger.warning(f"Failed to clean up temporary directory: {str(e)}")
# Check if path is set in environment variable
for path in paths:
if os.path.exists(path):
logger.info(f"Found {executable_name} at {path}")
return path
def _find_soffice_path(self) -> Optional[str]:
# Try to find in PATH
result = subprocess.run(
["which", executable_name], capture_output=True, text=True
)
if result.returncode == 0 and result.stdout.strip():
path = result.stdout.strip()
logger.info(f"Found {executable_name} at {path}")
return path
logger.warning(f"Failed to find {executable_name}")
return None
def _try_find_soffice(self) -> Optional[str]:
"""Find LibreOffice/OpenOffice executable path
Returns:
@@ -225,32 +200,13 @@ class DocParser(BaseParser):
"C:\\Program Files\\LibreOffice\\program\\soffice.exe",
"C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe",
]
return self._try_find_executable_path(
executable_name="soffice",
possible_path=possible_paths,
environment_variable=["LIBREOFFICE_PATH"],
)
# Check if path is set in environment variable
if os.environ.get("LIBREOFFICE_PATH"):
possible_paths.insert(0, os.environ.get("LIBREOFFICE_PATH"))
for path in possible_paths:
if os.path.exists(path):
logger.info(f"Found LibreOffice/OpenOffice at: {path}")
return path
# Try to find in PATH
try:
result = subprocess.run(
["which", "soffice"], capture_output=True, text=True
)
if result.returncode == 0 and result.stdout.strip():
path = result.stdout.strip()
logger.info(f"Found LibreOffice/OpenOffice in PATH: {path}")
return path
except Exception:
pass
logger.warning("LibreOffice/OpenOffice not found")
return None
def _find_antiword_path(self) -> Optional[str]:
def _try_find_antiword(self) -> Optional[str]:
"""Find antiword executable path
Returns:
@@ -265,51 +221,27 @@ class DocParser(BaseParser):
"C:\\Program Files\\Antiword\\antiword.exe",
"C:\\Program Files (x86)\\Antiword\\antiword.exe",
]
# Check if path is set in environment variable
if os.environ.get("ANTIWORD_PATH"):
possible_paths.insert(0, os.environ.get("ANTIWORD_PATH"))
for path in possible_paths:
if os.path.exists(path):
logger.info(f"Found antiword at: {path}")
return path
# Try to find in PATH
try:
result = subprocess.run(
["which", "antiword"], capture_output=True, text=True
)
if result.returncode == 0 and result.stdout.strip():
path = result.stdout.strip()
logger.info(f"Found antiword in PATH: {path}")
return path
except Exception:
pass
logger.warning("antiword not found")
return None
return self._try_find_executable_path(
executable_name="antiword",
possible_path=possible_paths,
environment_variable=["ANTIWORD_PATH"],
)
if __name__ == "__main__":
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
logger.info("Running DocParser in standalone mode")
logging.basicConfig(level=logging.DEBUG)
file_name = "/path/to/your/test.doc"
logger.info(f"Processing file: {file_name}")
doc_parser = DocParser(
file_name, enable_multimodal=True, chunk_size=512, chunk_overlap=60
file_name=file_name,
enable_multimodal=True,
chunk_size=512,
chunk_overlap=60,
)
logger.info("Parser initialized, starting processing")
with open(file_name, "rb") as f:
content = f.read()
text = doc_parser.parse_into_text(content)
logger.info(f"Processing complete, extracted text length: {len(text)}")
logger.info(f"Sample text: {text[:200]}...")
document = doc_parser.parse_into_text(content)
logger.info(f"Processing complete, extracted text length: {len(document.content)}")
logger.info(f"Sample text: {document.content[:200]}...")
docreader/parser/docx2_parser.py
+28
View File
@@ -0,0 +1,28 @@
import logging
from docreader.parser.chain_parser import FirstParser
from docreader.parser.docx_parser import DocxParser
from docreader.parser.markitdown_parser import MarkitdownParser
logger = logging.getLogger(__name__)
class Docx2Parser(FirstParser):
_parser_cls = (MarkitdownParser, DocxParser)
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
your_file = "/path/to/your/file.docx"
parser = Docx2Parser(separators=[".", "?", "!", "。", "？", "！"])
with open(your_file, "rb") as f:
content = f.read()
document = parser.parse(content)
for cc in document.chunks:
logger.info(f"chunk: {cc}")
# document = parser.parse_into_text(content)
# logger.info(f"docx content: {document.content}")
# logger.info(f"find images {document.images.keys()}")
docreader/parser/docx_parser.py
+91 -94
View File
@@ -1,37 +1,36 @@
import logging
import tempfile
import os
import sys
import time
from io import BytesIO
from typing import Optional, Dict, Any, Tuple, List, Union
from dataclasses import dataclass, field
from PIL import Image
from docx import Document
from docx.image.exceptions import (
UnrecognizedImageError,
UnexpectedEndOfFileError,
InvalidImageStreamError,
)
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_completed
import re
import tempfile
import threading
import time
import traceback
from concurrent.futures import ProcessPoolExecutor, as_completed
from dataclasses import dataclass, field
from io import BytesIO
from multiprocessing import Manager
import re
from typing import Any, Dict, List, Optional, Tuple
from .base_parser import BaseParser
from docx import Document
from docx.image.exceptions import (
InvalidImageStreamError,
UnexpectedEndOfFileError,
UnrecognizedImageError,
)
from PIL import Image
from docreader.models.document import Document as DocumentModel
from docreader.parser.base_parser import BaseParser
from docreader.utils import endecode
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
# Add thread local storage to track the processing status of each thread
thread_local = threading.local()
class ImageData:
"""Represents a processed image of document content"""
local_path: str = ""
object: Image.Image = None
object: Optional[Image.Image] = None
url: str = ""
@@ -40,7 +39,9 @@ class LineData:
"""Represents a processed line of document content with associated images"""
text: str = "" # Extracted text content
images: List[ImageData] = field(default_factory=list) # List of images or image paths
images: List[ImageData] = field(
default_factory=list
) # List of images or image paths
extra_info: str = "" # Placeholder for additional info (currently unused)
page_num: int = 0 # Page number
content_sequence: List[Tuple[str, Any]] = field(
@@ -53,18 +54,8 @@ class DocxParser(BaseParser):
def __init__(
self,
file_name: str = "",
file_type: str = None,
enable_multimodal: bool = True,
chunk_size: int = 1000,
chunk_overlap: int = 200,
separators: list = ["\n\n", "\n", "。"],
ocr_backend: str = "paddle",
ocr_config: dict = None,
max_image_size: int = 1920,
max_concurrent_tasks: int = 5,
max_pages: int = 100, # Maximum number of pages to process, default to 50 pages
chunking_config=None,
max_pages: int = 100, # Maximum number of pages to process
**kwargs,
):
"""Initialize DOCX document parser
@@ -79,37 +70,16 @@ class DocxParser(BaseParser):
ocr_config: OCR engine configuration
max_image_size: Maximum image size limit
max_concurrent_tasks: Maximum number of concurrent tasks
max_pages: Maximum number of pages to process, if more than this, only process the first max_pages pages
max_pages: Maximum number of pages to process
"""
super().__init__(
file_name=file_name,
file_type=file_type,
enable_multimodal=enable_multimodal,
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=separators,
ocr_backend=ocr_backend,
ocr_config=ocr_config,
max_image_size=max_image_size,
max_concurrent_tasks=max_concurrent_tasks,
chunking_config=chunking_config,
)
super().__init__(**kwargs)
self.max_pages = max_pages
logger.info(f"DocxParser initialized with max_pages={max_pages}")
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""Parse DOCX document, extract text content and image Markdown links
Args:
content: DOCX document content
Returns:
Tuple of (parsed_text, image_map) where image_map maps image URLs to Image objects
All LineData objects are used internally but not returned directly through this interface
"""
def parse_into_text(self, content: bytes) -> DocumentModel:
"""Parse DOCX document, extract text content and image Markdown links"""
logger.info(f"Parsing DOCX document, content size: {len(content)} bytes")
logger.info(f"Max pages limit set to: {self.max_pages}")
logger.info("Converting DOCX content to sections and tables")
start_time = time.time()
# Use concurrent processing to handle the document
@@ -123,7 +93,7 @@ class DocxParser(BaseParser):
docx_processor = Docx(
max_image_size=self.max_image_size,
enable_multimodal=self.enable_multimodal,
upload_file=self.upload_file,
upload_file=self.storage.upload_file,
)
all_lines, tables = docx_processor(
binary=content,
@@ -140,7 +110,7 @@ class DocxParser(BaseParser):
section_start_time = time.time()
text_parts = []
image_parts = {}
image_parts: Dict[str, str] = {}
for sec_idx, line in enumerate(all_lines):
try:
@@ -148,16 +118,19 @@ class DocxParser(BaseParser):
text_parts.append(line.text)
if sec_idx < 3 or sec_idx % 50 == 0:
logger.info(
f"Added section {sec_idx+1} text: {line.text[:50]}..."
f"Added section {sec_idx + 1} text: {line.text[:50]}..."
if len(line.text) > 50
else f"Added section {sec_idx+1} text: {line.text}"
else f"Added section {sec_idx + 1} text: {line.text}"
)
if line.images:
for image_data in line.images:
if image_data.url:
image_parts[image_data.url] = image_data.object
if image_data.url and image_data.object:
image_parts[image_data.url] = endecode.decode_image(
image_data.object
)
image_data.object.close()
except Exception as e:
logger.error(f"Error processing section {sec_idx+1}: {str(e)}")
logger.error(f"Error processing section {sec_idx + 1}: {str(e)}")
logger.error(f"Detailed stack trace: {traceback.format_exc()}")
continue
@@ -176,17 +149,17 @@ class DocxParser(BaseParser):
total_processing_time = time.time() - start_time
logger.info(
f"Parsing complete in {total_processing_time:.2f}s, generated {len(text)} characters of text "
f"Parsing complete in {total_processing_time:.2f}s, "
f"generated {len(text)} characters of text"
)
return text, image_parts
return DocumentModel(content=text, images=image_parts)
except Exception as e:
logger.error(f"Error parsing DOCX document: {str(e)}")
logger.error(f"Detailed stack trace: {traceback.format_exc()}")
fallback_text = self._parse_using_simple_method(content)
return fallback_text, {}
return self._parse_using_simple_method(content)
def _parse_using_simple_method(self, content: bytes) -> str:
def _parse_using_simple_method(self, content: bytes) -> DocumentModel:
"""Parse document using a simplified method, as a fallback
Args:
@@ -201,7 +174,8 @@ class DocxParser(BaseParser):
doc = Document(BytesIO(content))
logger.info(
f"Successfully loaded document in simplified method, "
f"contains {len(doc.paragraphs)} paragraphs and {len(doc.tables)} tables"
f"contains {len(doc.paragraphs)} paragraphs "
f"and {len(doc.tables)} tables"
)
text_parts = []
@@ -211,7 +185,7 @@ class DocxParser(BaseParser):
para_with_text = 0
for i, para in enumerate(doc.paragraphs):
if i % 100 == 0:
logger.info(f"Processing paragraph {i+1}/{para_count}")
logger.info(f"Processing paragraph {i + 1}/{para_count}")
if para.text.strip():
text_parts.append(para.text.strip())
para_with_text += 1
@@ -225,7 +199,7 @@ class DocxParser(BaseParser):
rows_processed = 0
for i, table in enumerate(doc.tables):
if i % 10 == 0:
logger.info(f"Processing table {i+1}/{table_count}")
logger.info(f"Processing table {i + 1}/{table_count}")
table_has_content = False
for row in table.rows:
@@ -256,25 +230,24 @@ class DocxParser(BaseParser):
# If the result is still empty, return an error message
if not result_text:
logger.warning("No text extracted using simplified method")
return "", {}
return DocumentModel()
return result_text, {}
return DocumentModel(content=result_text)
except Exception as backup_error:
processing_time = time.time() - start_time
logger.error(
f"Simplified parsing failed after {processing_time:.2f}s: {str(backup_error)}"
f"Simplified parsing failed {processing_time:.2f}s: {backup_error}"
)
logger.error(f"Detailed traceback: {traceback.format_exc()}")
return "", {}
return DocumentModel()
class Docx:
def __init__(self, max_image_size=1920, enable_multimodal=False, upload_file=None):
logger.info("Initializing DOCX processor")
self.max_image_size = max_image_size # Maximum image size limit
self.picture_cache = (
{}
) # Image cache to avoid processing the same image repeatedly
# Image cache to avoid processing the same image repeatedly
self.picture_cache = {}
self.enable_multimodal = enable_multimodal
self.upload_file = upload_file
@@ -454,7 +427,6 @@ class Docx:
return page_to_paragraphs
def __call__(
self,
binary: Optional[bytes] = None,
@@ -611,7 +583,6 @@ class Docx:
return pages_to_process
def _process_document(
self,
binary,
@@ -806,7 +777,9 @@ class Docx:
# Collect temporary image paths for later cleanup
for line in page_lines:
for image_data in line.images:
if image_data.local_path and image_data.local_path.startswith("/tmp/docx_img_"):
if image_data.local_path and image_data.local_path.startswith(
"/tmp/docx_img_"
):
temp_img_paths.add(image_data.local_path)
results.extend(page_lines)
@@ -876,7 +849,11 @@ class Docx:
# Process all image data objects
for image_data in image_paths:
if image_data.local_path and os.path.exists(image_data.local_path) and image_data.local_path not in image_url_map:
if (
image_data.local_path
and os.path.exists(image_data.local_path)
and image_data.local_path not in image_url_map
):
try:
# Upload the image if it doesn't have a URL yet
if not image_data.url:
@@ -886,12 +863,16 @@ class Docx:
image_data.url = image_url
# Add image URL as Markdown format
markdown_image = f"![]({image_url})"
image_url_map[image_data.local_path] = markdown_image
image_url_map[image_data.local_path] = (
markdown_image
)
logger.info(
f"Added image URL for {image_data.local_path}: {image_url}"
)
else:
logger.warning(f"Failed to upload image: {image_data.local_path}")
logger.warning(
f"Failed to upload image: {image_data.local_path}"
)
else:
# Already has a URL, use it
markdown_image = f"![]({image_data.url})"
@@ -925,12 +906,19 @@ class Docx:
# For ImageData objects, use the URL
if isinstance(content, str) and content in image_url_map:
combined_parts.append(image_url_map[content])
elif hasattr(content, 'local_path') and content.local_path in image_url_map:
elif (
hasattr(content, "local_path")
and content.local_path in image_url_map
):
combined_parts.append(image_url_map[content.local_path])
# Create the final text with proper ordering
final_text = "\n\n".join(part for part in combined_parts if part)
processed_lines.append(LineData(text=final_text, page_num=page_num, images=line_data.images))
processed_lines.append(
LineData(
text=final_text, page_num=page_num, images=line_data.images
)
)
else:
processed_lines = lines
@@ -1003,11 +991,11 @@ class Docx:
logger.info(f"Processing {table_count} tables")
for tb_idx, tb in enumerate(self.doc.tables):
if tb_idx % 10 == 0: # Log only every 10 tables to reduce log volume
logger.info(f"Processing table {tb_idx+1}/{table_count}")
logger.info(f"Processing table {tb_idx + 1}/{table_count}")
# Optimize: Check if table is empty
if len(tb.rows) == 0 or all(len(r.cells) == 0 for r in tb.rows):
logger.info(f"Skipping empty table {tb_idx+1}")
logger.info(f"Skipping empty table {tb_idx + 1}")
continue
table_html = self._convert_table_to_html(tb)
@@ -1111,8 +1099,8 @@ def _save_image_to_temp(logger, image, page_num, img_idx):
if not image:
return None
import tempfile
import os
import tempfile
try:
# Create a temporary file
@@ -1187,8 +1175,15 @@ def process_page_multiprocess(
return []
# Extract page content
combined_text, image_objects, content_sequence = _extract_page_content_in_process(
process_logger, doc, page_num, paragraphs, enable_multimodal, max_image_size
combined_text, image_objects, content_sequence = (
_extract_page_content_in_process(
process_logger,
doc,
page_num,
paragraphs,
enable_multimodal,
max_image_size,
)
)
# Process content sequence to maintain order between processes
@@ -1199,7 +1194,9 @@ def process_page_multiprocess(
if enable_multimodal:
# First pass: save all images to temporary files
for i, image_object in enumerate(image_objects):
img_path = _save_image_to_temp(process_logger, image_object, page_num, i)
img_path = _save_image_to_temp(
process_logger, image_object, page_num, i
)
if img_path:
# Create ImageData object
image_data = ImageData()
docreader/parser/image_parser.py
+20 -44
View File
@@ -1,15 +1,13 @@
import base64
import logging
import os
import asyncio
from PIL import Image
import io
from typing import Dict, Any, Tuple, Union
from .base_parser import BaseParser, ParseResult
import numpy as np
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
# Set up logger for this module
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class ImageParser(BaseParser):
"""
@@ -23,46 +21,24 @@ class ImageParser(BaseParser):
4. Returning a combined result with both text and image reference
"""
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
def parse_into_text(self, content: bytes) -> Document:
"""
Parse image content, upload the image and return Markdown reference along with image map.
Args:
content: Raw image data (bytes)
Returns:
Tuple of (markdown_text, image_map) where image_map maps image URLs to PIL Image objects
Parse image content into markdown text
:param content: bytes content of the image
:return: Document object
"""
logger.info(f"Parsing image content, size: {len(content)} bytes")
image_map = {}
try:
# Upload image to storage service
logger.info("Uploading image to storage")
_, ext = os.path.splitext(self.file_name)
image_url = self.upload_bytes(content, file_ext=ext)
if not image_url:
logger.error("Failed to upload image to storage")
return "", {}
logger.info(
f"Successfully uploaded image, URL: {image_url[:50]}..."
if len(image_url) > 50
else f"Successfully uploaded image, URL: {image_url}"
)
# Create image object and add to map
try:
from PIL import Image
import io
image = Image.open(io.BytesIO(content))
image_map[image_url] = image
logger.info(f"Added image to image_map for URL: {image_url}")
except Exception as img_err:
logger.error(f"Error creating image object: {str(img_err)}")
# Get file extension
ext = os.path.splitext(self.file_name)[1].lower()
markdown_text = f"![{self.file_name}]({image_url})"
return markdown_text, image_map
# Upload image to storage
image_url = self.storage.upload_bytes(content, file_ext=ext)
logger.info(f"Successfully uploaded image, URL: {image_url[:50]}...")
except Exception as e:
logger.error(f"Error parsing image: {str(e)}")
return "", {}
# Generate markdown text
text = f"![{self.file_name}]({image_url})"
images = {image_url: base64.b64encode(content).decode()}
# Create image object and add to map
return Document(content=text, images=images)
docreader/parser/image_utils.py
-43
View File
@@ -1,43 +0,0 @@
import base64
import io
import logging
from typing import Union
from PIL import Image
import numpy as np
logger = logging.getLogger(__name__)
def image_to_base64(image: Union[str, bytes, Image.Image, np.ndarray]) -> str:
"""Convert image to base64 encoded string
Args:
image: Image file path, bytes, PIL Image object, or numpy array
Returns:
Base64 encoded image string, or empty string if conversion fails
"""
try:
if isinstance(image, str):
# It's a file path
with open(image, "rb") as image_file:
return base64.b64encode(image_file.read()).decode("utf-8")
elif isinstance(image, bytes):
# It's bytes data
return base64.b64encode(image).decode("utf-8")
elif isinstance(image, Image.Image):
# It's a PIL Image
buffer = io.BytesIO()
image.save(buffer, format="PNG")
return base64.b64encode(buffer.getvalue()).decode("utf-8")
elif isinstance(image, np.ndarray):
# It's a numpy array
pil_image = Image.fromarray(image)
buffer = io.BytesIO()
pil_image.save(buffer, format="PNG")
return base64.b64encode(buffer.getvalue()).decode("utf-8")
else:
logger.error(f"Unsupported image type: {type(image)}")
return ""
except Exception as e:
logger.error(f"Error converting image to base64: {str(e)}")
return ""
docreader/parser/markdown_image_util.py
+111
View File
@@ -0,0 +1,111 @@
import logging
import re
import uuid
from typing import Dict, List, Match, Optional, Tuple
from docreader.utils import endecode
# Get logger object
logger = logging.getLogger(__name__)
class MarkdownImageUtil:
def __init__(self):
self.b64_pattern = re.compile(
r"!\[([^\]]*)\]\(data:image/(\w+);base64,([^\)]+)\)"
)
self.image_pattern = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
self.replace_pattern = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)")
def extract_image(
self,
content: str,
path_prefix: Optional[str] = None,
replace: bool = True,
) -> Tuple[str, List[str]]:
"""Extract base64 encoded images from Markdown content"""
# image_path => base64 bytes
images: List[str] = []
def repl(match: Match[str]) -> str:
title = match.group(1)
image_path = match.group(2)
if path_prefix:
image_path = f"{path_prefix}/{image_path}"
images.append(image_path)
if not replace:
return match.group(0)
# Replace image path with URL
return f"![{title}]({image_path})"
text = self.image_pattern.sub(repl, content)
logger.debug(f"Extracted {len(images)} images from markdown")
return text, images
def extract_base64(
self,
content: str,
path_prefix: Optional[str] = None,
replace: bool = True,
) -> Tuple[str, Dict[str, bytes]]:
"""Extract base64 encoded images from Markdown content"""
# image_path => base64 bytes
images: Dict[str, bytes] = {}
def repl(match: Match[str]) -> str:
title = match.group(1)
img_ext = match.group(2)
img_b64 = match.group(3)
image_byte = endecode.encode_image(img_b64, errors="ignore")
if not image_byte:
logger.error(f"Failed to decode base64 image skip it: {img_b64}")
return title
image_path = f"{uuid.uuid4()}.{img_ext}"
if path_prefix:
image_path = f"{path_prefix}/{image_path}"
images[image_path] = image_byte
if not replace:
return match.group(0)
# Replace image path with URL
return f"![{title}]({image_path})"
text = self.b64_pattern.sub(repl, content)
logger.debug(f"Extracted {len(images)} base64 images from markdown")
return text, images
def replace_path(self, content: str, images: Dict[str, str]) -> str:
content_replace: set = set()
def repl(match: Match[str]) -> str:
title = match.group(1)
image_path = match.group(2)
if image_path not in images:
return match.group(0)
content_replace.add(image_path)
image_path = images[image_path]
return f"![{title}]({image_path})"
text = self.replace_pattern.sub(repl, content)
logger.debug(f"Replaced {len(content_replace)} images in markdown")
return text
if __name__ == "__main__":
your_content = "test![](data:image/png;base64,iVBORw0KGgoAAAA)test"
image_handle = MarkdownImageUtil()
text, images = image_handle.extract_base64(your_content)
print(text)
for image_url, image_byte in images.items():
with open(image_url, "wb") as f:
f.write(image_byte)
docreader/parser/markdown_parser.py
+43 -23
View File
@@ -1,33 +1,53 @@
import asyncio
import re
import base64
import logging
import numpy as np
import os # Import os module to get environment variables
from typing import Dict, List, Optional, Tuple, Union, Any
from .base_parser import BaseParser
import os
from typing import Dict
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
from docreader.parser.markdown_image_util import MarkdownImageUtil
from docreader.utils import endecode
# Get logger object
logger = logging.getLogger(__name__)
class MarkdownParser(BaseParser):
"""Markdown document parser"""
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""Parse Markdown document, only extract text content, do not process images
Args:
content: Markdown document content
Returns:
Parsed text result
"""
logger.info(f"Parsing Markdown document, content size: {len(content)} bytes")
class MarkdownImageBase64(BaseParser):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.image_helper = MarkdownImageUtil()
def parse_into_text(self, content: bytes) -> Document:
# Convert byte content to string using universal decoding method
text = self.decode_bytes(content)
logger.info(f"Decoded Markdown content, text length: {len(text)} characters")
text = endecode.decode_bytes(content)
text, img_b64 = self.image_helper.extract_base64(text, path_prefix="images")
logger.info(f"Markdown parsing complete, extracted {len(text)} characters of text")
return text
images: Dict[str, str] = {}
image_replace: Dict[str, str] = {}
logger.debug(f"Uploading {len(img_b64)} images from markdown")
for ipath, b64_bytes in img_b64.items():
ext = os.path.splitext(ipath)[1].lower()
image_url = self.storage.upload_bytes(b64_bytes, ext)
image_replace[ipath] = image_url
images[image_url] = base64.b64encode(b64_bytes).decode()
text = self.image_helper.replace_path(text, image_replace)
return Document(content=text, images=images)
class MarkdownParser(PipelineParser):
_parser_cls = (MarkdownImageBase64,)
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
your_content = "test![](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMgA)test"
parser = MarkdownParser()
document = parser.parse_into_text(your_content.encode())
logger.info(document.content)
logger.info(f"Images: {len(document.images)}, name: {document.images.keys()}")
docreader/parser/markitdown_parser.py
+31
View File
@@ -0,0 +1,31 @@
import io
import logging
from markitdown import MarkItDown
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
from docreader.parser.markdown_parser import MarkdownParser
logger = logging.getLogger(__name__)
class StdMarkitdownParser(BaseParser):
"""
PDF Document Parser
This parser handles PDF documents by extracting text content.
It uses the markitdown library for simple text extraction.
"""
def __init__(self, *args, **kwargs):
self.markitdown = MarkItDown()
def parse_into_text(self, content: bytes) -> Document:
result = self.markitdown.convert(io.BytesIO(content), keep_data_uris=True)
return Document(content=result.text_content)
class MarkitdownParser(PipelineParser):
_parser_cls = (StdMarkitdownParser, MarkdownParser)
docreader/parser/mineru_parser.py
+124
View File
@@ -0,0 +1,124 @@
import logging
import os
import re
from typing import Dict
import markdownify
import requests
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.markdown_parser import MarkdownImageUtil
from docreader.utils import endecode
logger = logging.getLogger(__name__)
class MinerUParser(BaseParser):
def __init__(
self,
enable_markdownify: bool = True,
mineru_endpoint: str = "",
**kwargs,
):
super().__init__(**kwargs)
self.minerU = os.getenv("MINERU_ENDPOINT", mineru_endpoint)
self.enable_markdownify = enable_markdownify
self.image_helper = MarkdownImageUtil()
self.base64_pattern = re.compile(r"data:image/(\w+);base64,(.*)")
self.enable = self.ping()
assert self.ping(), "MinerU API is not reachable"
def ping(self, timeout: int = 5) -> bool:
try:
response = requests.get(
self.minerU + "/docs", timeout=timeout, allow_redirects=True
)
response.raise_for_status()
return True
except Exception:
return False
def parse_into_text(self, content: bytes) -> Document:
logger.info(f"Parsing scanned PDF via MinerU API (size: {len(content)} bytes)")
md_content: str = ""
images_b64: Dict[str, str] = {}
try:
response = requests.post(
url=self.minerU + "/file_parse",
data={
"return_md": True,
"return_images": True,
"lang_list": ["ch", "en"],
"table_enable": True,
"formula_enable": True,
"parse_method": "auto",
"start_page_id": 0,
"end_page_id": 99999,
"backend": "pipeline",
"response_format_zip": False,
"return_middle_json": False,
"return_model_output": False,
"return_content_list": False,
},
files={"files": content},
timeout=1000,
)
response.raise_for_status()
result = response.json()["results"]["files"]
md_content = result["md_content"]
images_b64 = result.get("images", {})
except Exception as e:
logger.error(f"MinerU parsing failed: {e}", exc_info=True)
return Document()
# convert table(HTML) in markdown to markdown table
if self.enable_markdownify:
logger.debug("Converting HTML to Markdown")
md_content = markdownify.markdownify(md_content)
images = {}
image_replace = {}
# image in images_bs64 may not be used in md_content
# such as: table ...
# so we need to filter them
for ipath, b64_str in images_b64.items():
if f"images/{ipath}" not in md_content:
logger.debug(f"Image {ipath} not used in markdown")
continue
match = self.base64_pattern.match(b64_str)
if match:
file_ext = match.group(1)
b64_str = match.group(2)
image_bytes = endecode.encode_image(b64_str, errors="ignore")
if not image_bytes:
logger.error("Failed to decode base64 image skip it")
continue
image_url = self.storage.upload_bytes(
image_bytes, file_ext=f".{file_ext}"
)
images[image_url] = b64_str
image_replace[f"images/{ipath}"] = image_url
logger.info(f"Replaced {len(image_replace)} images in markdown")
text = self.image_helper.replace_path(md_content, image_replace)
logger.info(
f"Successfully parsed PDF, text: {len(text)}, images: {len(images)}"
)
return Document(content=text, images=images)
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
your_file = "/path/to/your/file.pdf"
your_mineru = "http://host.docker.internal:9987"
parser = MinerUParser(mineru_endpoint=your_mineru)
with open(your_file, "rb") as f:
content = f.read()
document = parser.parse_into_text(content)
logger.error(document.content)
docreader/parser/ocr_engine.py
+164 -115
View File
@@ -1,71 +1,96 @@
import os
import logging
import base64
from typing import Optional, Union, Dict, Any
from abc import ABC, abstractmethod
from PIL import Image
import io
import logging
import os
import platform
import subprocess
from abc import ABC, abstractmethod
from typing import Dict, Union
import numpy as np
from .image_utils import image_to_base64
from openai import OpenAI
from PIL import Image
from docreader.utils import endecode
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class OCRBackend(ABC):
"""Base class for OCR backends"""
@abstractmethod
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
pass
class DummyOCRBackend(OCRBackend):
"""Dummy OCR backend implementation"""
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
logger.warning("Dummy OCR backend is used")
return ""
class PaddleOCRBackend(OCRBackend):
"""PaddleOCR backend implementation"""
def __init__(self, **kwargs):
def __init__(self):
"""Initialize PaddleOCR backend"""
self.ocr = None
try:
import os
import paddle
# Set PaddlePaddle to use CPU and disable GPU
os.environ['CUDA_VISIBLE_DEVICES'] = ''
paddle.set_device('cpu')
os.environ["CUDA_VISIBLE_DEVICES"] = ""
paddle.device.set_device("cpu")
# 尝试检测CPU是否支持AVX指令集
try:
import subprocess
import platform
# 检测CPU是否支持AVX
if platform.system() == "Linux":
try:
result = subprocess.run(['grep', '-o', 'avx', '/proc/cpuinfo'],
capture_output=True, text=True, timeout=5)
has_avx = 'avx' in result.stdout.lower()
result = subprocess.run(
["grep", "-o", "avx", "/proc/cpuinfo"],
capture_output=True,
text=True,
timeout=5,
)
has_avx = "avx" in result.stdout.lower()
if not has_avx:
logger.warning("CPU does not support AVX instructions, using compatibility mode")
logger.warning(
"CPU does not support AVX instructions, "
"using compatibility mode"
)
# 进一步限制指令集使用
os.environ['FLAGS_use_avx2'] = '0'
os.environ['FLAGS_use_avx'] = '1'
except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
logger.warning("Could not detect AVX support, using compatibility mode")
os.environ['FLAGS_use_avx2'] = '0'
os.environ['FLAGS_use_avx'] = '1'
os.environ["FLAGS_use_avx2"] = "0"
os.environ["FLAGS_use_avx"] = "1"
except (
subprocess.TimeoutExpired,
FileNotFoundError,
subprocess.SubprocessError,
):
logger.warning(
"Could not detect AVX support, using compatibility mode"
)
os.environ["FLAGS_use_avx2"] = "0"
os.environ["FLAGS_use_avx"] = "1"
except Exception as e:
logger.warning(f"Error detecting CPU capabilities: {e}, using compatibility mode")
os.environ['FLAGS_use_avx2'] = '0'
os.environ['FLAGS_use_avx'] = '1'
logger.warning(
f"Error detecting CPU capabilities: {e}, using compatibility mode"
)
os.environ["FLAGS_use_avx2"] = "0"
os.environ["FLAGS_use_avx"] = "1"
from paddleocr import PaddleOCR
# OCR configuration with text orientation classification enabled
ocr_config = {
"use_gpu": False,
@@ -86,23 +111,53 @@ class PaddleOCRBackend(OCRBackend):
"use_dilation": True, # improves accuracy
"det_db_score_mode": "slow", # improves accuracy
}
self.ocr = PaddleOCR(**ocr_config)
logger.info("PaddleOCR engine initialized successfully")
except ImportError as e:
logger.error(f"Failed to import paddleocr: {str(e)}. Please install it with 'pip install paddleocr'")
logger.error(
f"Failed to import paddleocr: {str(e)}. "
"Please install it with 'pip install paddleocr'"
)
except OSError as e:
if "Illegal instruction" in str(e) or "core dumped" in str(e):
logger.error(f"PaddlePaddle crashed due to CPU instruction set incompatibility: {str(e)}")
logger.error("This usually happens when the CPU doesn't support AVX instructions.")
logger.error("Please try installing a CPU-only version of PaddlePaddle or use a different OCR backend.")
logger.error(
f"PaddlePaddle crashed due to CPU instruction set incompatibility:"
f"{e}"
)
logger.error(
"This happens when the CPU doesn't support AVX instructions. "
"Try install CPU-only version of PaddlePaddle, "
"or use a different OCR backend."
)
else:
logger.error(f"Failed to initialize PaddleOCR due to OS error: {str(e)}")
logger.error(
f"Failed to initialize PaddleOCR due to OS error: {str(e)}"
)
except Exception as e:
logger.error(f"Failed to initialize PaddleOCR: {str(e)}")
def predict(self, image):
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
if isinstance(image, str):
image = Image.open(image)
elif isinstance(image, bytes):
image = Image.open(io.BytesIO(image))
if not isinstance(image, Image.Image):
raise TypeError("image must be a string, bytes, or PIL Image object")
return self._predict(image)
def _predict(self, image: Image.Image) -> str:
"""Perform OCR recognition on the image
Args:
@@ -111,63 +166,59 @@ class PaddleOCRBackend(OCRBackend):
Returns:
Extracted text string
"""
if self.ocr is None:
logger.error("PaddleOCR engine not initialized")
return ""
try:
# Ensure image is in RGB format
if hasattr(image, "convert") and image.mode != "RGB":
if image.mode != "RGB":
image = image.convert("RGB")
# Convert to numpy array if needed
if hasattr(image, "convert"):
image_array = np.array(image)
else:
image_array = image
image_array = np.array(image)
# Perform OCR
ocr_result = self.ocr.ocr(image_array, cls=False)
# Extract text
ocr_text = ""
if ocr_result and ocr_result[0]:
for line in ocr_result[0]:
if line and len(line) >= 2:
text = line[1][0] if line[1] else ""
if text:
ocr_text += text + " "
text_length = len(ocr_text.strip())
if text_length > 0:
logger.info(f"OCR extracted {text_length} characters")
return ocr_text.strip()
else:
logger.warning("OCR returned empty result")
return ""
text = [
line[1][0] if line and len(line) >= 2 and line[1] else ""
for line in ocr_result[0]
]
text = [t.strip() for t in text if t]
ocr_text = " ".join(text)
logger.info(f"OCR extracted {len(ocr_text)} characters")
return ocr_text
except Exception as e:
logger.error(f"OCR recognition error: {str(e)}")
return ""
class NanonetsOCRBackend(OCRBackend):
"""Nanonets OCR backend implementation using OpenAI API format"""
def __init__(self, **kwargs):
def __init__(self):
"""Initialize Nanonets OCR backend
Args:
api_key: API key for OpenAI API
base_url: Base URL for OpenAI API
model: Model name
"""
try:
from openai import OpenAI
self.api_key = kwargs.get("api_key", "123")
self.base_url = kwargs.get("base_url", "http://localhost:8000/v1")
self.model = kwargs.get("model", "nanonets/Nanonets-OCR-s")
self.temperature = kwargs.get("temperature", 0.0)
self.max_tokens = kwargs.get("max_tokens", 15000)
self.client = OpenAI(api_key=self.api_key, base_url=self.base_url)
self.prompt = """
## 任务说明
base_url = os.getenv("OCR_API_BASE_URL", "http://localhost:8000/v1")
api_key = os.getenv("OCR_API_KEY", "123")
timeout = 30
self.client = OpenAI(api_key=api_key, base_url=base_url, timeout=timeout)
self.model = os.getenv("OCR_MODEL", "nanonets/Nanonets-OCR-s")
logger.info(f"Nanonets OCR engine initialized with model: {self.model}")
self.temperature = 0.0
self.max_tokens = 15000
self.prompt = """## 任务说明
请从上传的文档中提取文字内容，严格按自然阅读顺序（从上到下，从左到右）输出，并遵循以下格式规范。
@@ -192,33 +243,26 @@ class NanonetsOCRBackend(OCRBackend):
* 不要猜测或补全不确定的链接地址。
"""
logger.info(f"Nanonets OCR engine initialized with model: {self.model}")
except ImportError:
logger.error("Failed to import openai. Please install it with 'pip install openai'")
self.client = None
except Exception as e:
logger.error(f"Failed to initialize Nanonets OCR: {str(e)}")
self.client = None
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image using Nanonets OCR
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
if self.client is None:
logger.error("Nanonets OCR client not initialized")
return ""
try:
# Encode image to base64
img_base64 = image_to_base64(image)
img_base64 = endecode.decode_image(image)
if not img_base64:
return ""
# Call Nanonets OCR API
logger.info(f"Calling Nanonets OCR API with model: {self.model}")
response = self.client.chat.completions.create(
@@ -229,7 +273,9 @@ class NanonetsOCRBackend(OCRBackend):
"content": [
{
"type": "image_url",
"image_url": {"url": f"data:image/png;base64,{img_base64}"},
"image_url": {
"url": f"data:image/png;base64,{img_base64}"
},
},
{
"type": "text",
@@ -239,40 +285,43 @@ class NanonetsOCRBackend(OCRBackend):
}
],
temperature=self.temperature,
max_tokens=self.max_tokens
max_tokens=self.max_tokens,
)
return response.choices[0].message.content
return response.choices[0].message.content or ""
except Exception as e:
logger.error(f"Nanonets OCR prediction error: {str(e)}")
return ""
class OCREngine:
"""OCR Engine factory class"""
_instance = None
_instance: Dict[str, OCRBackend] = {}
@classmethod
def get_instance(cls, backend_type="paddle", **kwargs) -> Optional[OCRBackend]:
def get_instance(cls, backend_type: str) -> OCRBackend:
"""Get OCR engine instance
Args:
backend_type: OCR backend type, one of: "paddle", "nanonets"
**kwargs: Additional arguments for the backend
Returns:
OCR engine instance or None if initialization fails
"""
if cls._instance is None:
logger.info(f"Initializing OCR engine with backend: {backend_type}")
if backend_type.lower() == "paddle":
cls._instance = PaddleOCRBackend(**kwargs)
elif backend_type.lower() == "nanonets":
cls._instance = NanonetsOCRBackend(**kwargs)
else:
logger.error(f"Unknown OCR backend type: {backend_type}")
return None
return cls._instance
backend_type = backend_type.lower()
if cls._instance.get(backend_type):
return cls._instance[backend_type]
logger.info(f"Initializing OCR engine with backend: {backend_type}")
if backend_type == "paddle":
cls._instance[backend_type] = PaddleOCRBackend()
elif backend_type == "nanonets":
cls._instance[backend_type] = NanonetsOCRBackend()
else:
cls._instance[backend_type] = DummyOCRBackend()
return cls._instance[backend_type]
docreader/parser/parser.py
+69 -117
View File
@@ -1,30 +1,19 @@
import logging
from dataclasses import dataclass, field
from typing import Dict, Any, Optional, Type
from typing import Dict, Type
from .base_parser import BaseParser, ParseResult
from .docx_parser import DocxParser
from .doc_parser import DocParser
from .pdf_parser import PDFParser
from .markdown_parser import MarkdownParser
from .text_parser import TextParser
from .image_parser import ImageParser
from .web_parser import WebParser
from .config import ChunkingConfig
import traceback
from docreader.models.document import Document
from docreader.models.read_config import ChunkingConfig
from docreader.parser.base_parser import BaseParser
from docreader.parser.doc_parser import DocParser
from docreader.parser.docx2_parser import Docx2Parser
from docreader.parser.image_parser import ImageParser
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.pdf_parser import PDFParser
from docreader.parser.text_parser import TextParser
from docreader.parser.web_parser import WebParser
logger = logging.getLogger(__name__)
@dataclass
class Chunk:
"""
Represents a single text chunk with associated metadata.
Basic unit for document processing and embedding.
"""
content: str # Text content of the chunk
metadata: Dict[str, Any] = None # Associated metadata (source, page number, etc.)
class Parser:
"""
@@ -33,10 +22,9 @@ class Parser:
"""
def __init__(self):
logger.info("Initializing document parser")
# Initialize all parser types
self.parsers: Dict[str, Type[BaseParser]] = {
"docx": DocxParser,
"docx": Docx2Parser,
"doc": DocParser,
"pdf": PDFParser,
"md": MarkdownParser,
@@ -56,8 +44,7 @@ class Parser:
", ".join(self.parsers.keys()),
)
def get_parser(self, file_type: str) -> Optional[Type[BaseParser]]:
def get_parser(self, file_type: str) -> Type[BaseParser]:
"""
Get parser class for the specified file type.
@@ -67,12 +54,9 @@ class Parser:
Returns:
Parser class for the file type, or None if unsupported
"""
file_type = file_type.lower()
parser = self.parsers.get(file_type)
if parser:
logger.info(f"Found parser for file type: {file_type}")
else:
logger.warning(f"No parser found for file type: {file_type}")
parser = self.parsers.get(file_type.lower())
if not parser:
raise ValueError(f"Unsupported file type: {file_type}")
return parser
def parse_file(
@@ -81,7 +65,7 @@ class Parser:
file_type: str,
content: bytes,
config: ChunkingConfig,
) -> Optional[ParseResult]:
) -> Document:
"""
Parse file content using appropriate parser based on file type.
@@ -96,60 +80,41 @@ class Parser:
"""
logger.info(f"Parsing file: {file_name} with type: {file_type}")
logger.info(
f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, "
f"Chunking config: size={config.chunk_size}, "
f"overlap={config.chunk_overlap}, "
f"multimodal={config.enable_multimodal}"
)
parser_instance = None
try:
# Get appropriate parser for file type
cls = self.get_parser(file_type)
if cls is None:
logger.error(f"Unsupported file type: {file_type}")
return None
# Parse file content
logger.info(f"Creating parser instance for {file_type} file")
parser_instance = cls(
file_name=file_name,
file_type=file_type,
chunk_size=config.chunk_size,
chunk_overlap=config.chunk_overlap,
separators=config.separators,
enable_multimodal=config.enable_multimodal,
max_image_size=1920, # Limit image size to 1920px
max_concurrent_tasks=5, # Limit concurrent tasks to 5
chunking_config=config, # Pass the entire chunking config
)
# Get appropriate parser for file type
cls = self.get_parser(file_type)
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
result = parser_instance.parse(content)
# Parse file content
logger.info(f"Creating parser instance for {file_type} file")
parser = cls(
file_name=file_name,
file_type=file_type,
chunk_size=config.chunk_size,
chunk_overlap=config.chunk_overlap,
separators=config.separators,
enable_multimodal=config.enable_multimodal,
max_image_size=1920, # Limit image size to 1920px
max_concurrent_tasks=5, # Limit concurrent tasks to 5
chunking_config=config, # Pass the entire chunking config
)
if result:
logger.info(
f"Successfully parsed file {file_name}, generated {len(result.chunks)} chunks"
)
if result.chunks and len(result.chunks) > 0:
logger.info(
f"First chunk content length: {len(result.chunks[0].content)}"
)
else:
logger.warning(f"Parser returned empty chunks for file: {file_name}")
else:
logger.warning(f"Parser returned None result for file: {file_name}")
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
result = parser.parse(content)
# Return parse results
return result
if not result.content:
logger.warning(f"Parser returned empty content for file: {file_name}")
elif not result.chunks:
logger.warning(f"Parser returned empty chunks for file: {file_name}")
elif result.chunks[0]:
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
logger.info(f"Parsed file {file_name}, with {len(result.chunks)} chunks")
return result
except Exception as e:
logger.error(f"Error parsing file {file_name}: {str(e)}")
logger.info(f"Detailed traceback: {traceback.format_exc()}")
return None
def parse_url(
self, url: str, title: str, config: ChunkingConfig
) -> Optional[ParseResult]:
def parse_url(self, url: str, title: str, config: ChunkingConfig) -> Document:
"""
Parse content from a URL using the WebParser.
@@ -163,44 +128,31 @@ class Parser:
"""
logger.info(f"Parsing URL: {url}, title: {title}")
logger.info(
f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, "
f"multimodal={config.enable_multimodal}"
f"Chunking config: size={config.chunk_size}, "
f"overlap={config.chunk_overlap}, multimodal={config.enable_multimodal}"
)
parser_instance = None
try:
# Create web parser instance
logger.info("Creating WebParser instance")
parser_instance = WebParser(
title=title,
chunk_size=config.chunk_size,
chunk_overlap=config.chunk_overlap,
separators=config.separators,
enable_multimodal=config.enable_multimodal,
max_image_size=1920, # Limit image size
max_concurrent_tasks=5, # Limit concurrent tasks
chunking_config=config,
)
# Create web parser instance
logger.info("Creating WebParser instance")
parser = WebParser(
title=title,
chunk_size=config.chunk_size,
chunk_overlap=config.chunk_overlap,
separators=config.separators,
enable_multimodal=config.enable_multimodal,
max_image_size=1920, # Limit image size
max_concurrent_tasks=5, # Limit concurrent tasks
chunking_config=config,
)
logger.info(f"Starting to parse URL content")
result = parser_instance.parse(url)
if result:
logger.info(
f"Successfully parsed URL, generated {len(result.chunks)} chunks"
)
logger.info(
f"First chunk content length: {len(result.chunks[0].content) if result.chunks else 0}"
)
else:
logger.warning(f"Parser returned empty result for URL: {url}")
# Return parse results
return result
except Exception as e:
logger.error(f"Error parsing URL {url}: {str(e)}")
logger.info(f"Detailed traceback: {traceback.format_exc()}")
return None
logger.info("Starting to parse URL content")
result = parser.parse(url.encode())
if not result.content:
logger.warning(f"Parser returned empty content for url: {url}")
elif not result.chunks:
logger.warning(f"Parser returned empty chunks for url: {url}")
elif result.chunks[0]:
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
logger.info(f"Parsed url {url}, with {len(result.chunks)} chunks")
return result
docreader/parser/pdf_parser.py
+5 -111
View File
@@ -1,113 +1,7 @@
import logging
import os
import io
from typing import Any, List, Iterator, Optional, Mapping, Tuple, Dict, Union
from docreader.parser.chain_parser import FirstParser
from docreader.parser.markitdown_parser import MarkitdownParser
from docreader.parser.mineru_parser import MinerUParser
import pdfplumber
import tempfile
from .base_parser import BaseParser
logger = logging.getLogger(__name__)
class PDFParser(BaseParser):
"""
PDF Document Parser
This parser handles PDF documents by extracting text content.
It uses the pypdf library for simple text extraction.
"""
def _convert_table_to_markdown(self, table_data: list) -> str:
if not table_data or not table_data[0]: return ""
def clean_cell(cell):
if cell is None: return ""
return str(cell).replace("\n", " <br> ")
try:
markdown = ""
header = [clean_cell(cell) for cell in table_data[0]]
markdown += "| " + " | ".join(header) + " |\n"
markdown += "| " + " | ".join(["---"] * len(header)) + " |\n"
for row in table_data[1:]:
if not row: continue
body_row = [clean_cell(cell) for cell in row]
if len(body_row) != len(header):
logger.warning(f"Skipping malformed table row: {body_row}")
continue
markdown += "| " + " | ".join(body_row) + " |\n"
return markdown
except Exception as e:
logger.error(f"Error converting table to markdown: {e}")
return ""
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
logger.info(f"Parsing PDF with pdfplumber, content size: {len(content)} bytes")
all_page_content = []
temp_pdf = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf")
temp_pdf_path = temp_pdf.name
try:
temp_pdf.write(content)
temp_pdf.close()
logger.info(f"PDF content written to temporary file: {temp_pdf_path}")
with pdfplumber.open(temp_pdf_path) as pdf:
logger.info(f"PDF has {len(pdf.pages)} pages")
for page_num, page in enumerate(pdf.pages):
page_content_parts = []
# Try-fallback strategy for table detection
default_settings = { "vertical_strategy": "lines", "horizontal_strategy": "lines" }
found_tables = page.find_tables(default_settings)
if not found_tables:
logger.info(f"Page {page_num+1}: Default strategy found no tables. Trying fallback strategy.")
fallback_settings = { "vertical_strategy": "text", "horizontal_strategy": "lines" }
found_tables = page.find_tables(fallback_settings)
table_bboxes = [table.bbox for table in found_tables]
# Define a filter function that keeps objects NOT inside any table bbox.
def not_within_bboxes(obj):
"""Check if an object is outside all table bounding boxes."""
for bbox in table_bboxes:
# Check if the object's vertical center is within a bbox
if bbox[1] <= (obj["top"] + obj["bottom"]) / 2 <= bbox[3]:
return False # It's inside a table, so we DON'T keep it
return True # It's outside all tables, so we DO keep it
# that contains only the non-table text.
non_table_page = page.filter(not_within_bboxes)
# Now, extract text from this filtered page view.
text = non_table_page.extract_text(x_tolerance=2)
if text:
page_content_parts.append(text)
# Process and append the structured Markdown tables
if found_tables:
logger.info(f"Found {len(found_tables)} tables on page {page_num + 1}")
for table in found_tables:
markdown_table = self._convert_table_to_markdown(table.extract())
page_content_parts.append(f"\n\n{markdown_table}\n\n")
all_page_content.append("".join(page_content_parts))
final_text = "\n\n--- Page Break ---\n\n".join(all_page_content)
logger.info(f"PDF parsing complete. Extracted {len(final_text)} text chars.")
return final_text
except Exception as e:
logger.error(f"Failed to parse PDF document: {str(e)}")
return ""
finally:
# This block is GUARANTEED to execute, preventing resource leaks.
if os.path.exists(temp_pdf_path):
try:
os.remove(temp_pdf_path)
logger.info(f"Temporary file cleaned up: {temp_pdf_path}")
except OSError as e:
logger.error(f"Error removing temporary file {temp_pdf_path}: {e}")
class PDFParser(FirstParser):
_parser_cls = (MinerUParser, MarkitdownParser)
docreader/parser/storage.py
+163 -109
View File
@@ -1,64 +1,68 @@
# -*- coding: utf-8 -*-
import os
import uuid
import logging
import io
import logging
import os
import traceback
import uuid
from abc import ABC, abstractmethod
from typing import Tuple, Optional
from typing import Dict
from qcloud_cos import CosConfig, CosS3Client
from minio import Minio
from qcloud_cos import CosConfig, CosS3Client
from docreader.utils import endecode
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class Storage(ABC):
"""Abstract base class for object storage operations"""
@abstractmethod
def upload_file(self, file_path: str) -> str:
"""Upload file to object storage
Args:
file_path: File path
Returns:
File URL
"""
pass
@abstractmethod
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to object storage
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
pass
@abstractmethod
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to object storage
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
pass
class CosStorage(Storage):
"""Tencent Cloud COS storage implementation"""
def __init__(self, storage_config=None):
"""Initialize COS storage
Args:
storage_config: Storage configuration
"""
self.storage_config = storage_config
self.client, self.bucket_name, self.region, self.prefix = self._init_cos_client()
self.client, self.bucket_name, self.region, self.prefix = (
self._init_cos_client()
)
def _init_cos_client(self):
"""Initialize Tencent Cloud COS client"""
try:
# Use provided COS config if available, otherwise fall back to environment variables
# Use provided COS config if available,
# otherwise fall back to environment variables
if self.storage_config and self.storage_config.get("access_key_id") != "":
cos_config = self.storage_config
secret_id = cos_config.get("access_key_id")
@@ -75,15 +79,16 @@ class CosStorage(Storage):
bucket_name = os.getenv("COS_BUCKET_NAME")
appid = os.getenv("COS_APP_ID")
prefix = os.getenv("COS_PATH_PREFIX")
enable_old_domain = (
os.getenv("COS_ENABLE_OLD_DOMAIN", "true").lower() == "true"
)
if not all([secret_id, secret_key, region, bucket_name, appid]):
logger.error(
"Incomplete COS configuration, missing required environment variables"
f"secret_id: {secret_id}, secret_key: {secret_key}, region: {region}, bucket_name: {bucket_name}, appid: {appid}"
"Incomplete COS configuration, missing environment variables"
f"secret_id: {secret_id}, secret_key: {secret_key}, "
f"region: {region}, bucket_name: {bucket_name}, appid: {appid}"
)
return None, None, None, None
@@ -105,27 +110,26 @@ class CosStorage(Storage):
except Exception as e:
logger.error(f"Failed to initialize COS client: {str(e)}")
return None, None, None, None
def _get_download_url(self, bucket_name, region, object_key):
"""Generate COS object URL
Args:
bucket_name: Bucket name
region: Region
object_key: Object key
Returns:
File URL
"""
return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to Tencent Cloud COS
Args:
file_path: File path
Returns:
File URL
"""
@@ -135,16 +139,16 @@ class CosStorage(Storage):
return ""
# Generate object key, use UUID to avoid conflicts
file_name = os.path.basename(file_path)
object_key = (
f"{self.prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
)
file_ext = os.path.splitext(file_path)[1]
object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
logger.info(f"Generated object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to COS")
response = self.client.upload_file(
Bucket=self.bucket_name, LocalFilePath=file_path, Key=object_key
self.client.upload_file(
Bucket=self.bucket_name,
LocalFilePath=file_path,
Key=object_key,
)
# Get file URL
@@ -156,14 +160,14 @@ class CosStorage(Storage):
except Exception as e:
logger.error(f"Failed to upload file to COS: {str(e)}")
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to Tencent Cloud COS
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
@@ -171,10 +175,16 @@ class CosStorage(Storage):
logger.info(f"Uploading bytes content to COS, size: {len(content)} bytes")
if not self.client:
return ""
object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}" if self.prefix else f"images/{uuid.uuid4().hex}{file_ext}"
object_key = (
f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
if self.prefix
else f"images/{uuid.uuid4().hex}{file_ext}"
)
logger.info(f"Generated object key: {object_key}")
self.client.put_object(Bucket=self.bucket_name, Body=content, Key=object_key)
self.client.put_object(
Bucket=self.bucket_name, Body=content, Key=object_key
)
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
logger.info(f"Successfully uploaded bytes to COS: {file_url}")
return file_url
@@ -186,16 +196,18 @@ class CosStorage(Storage):
class MinioStorage(Storage):
"""MinIO storage implementation"""
def __init__(self, storage_config=None):
"""Initialize MinIO storage
Args:
storage_config: Storage configuration
"""
self.storage_config = storage_config
self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = self._init_minio_client()
self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = (
self._init_minio_client()
)
def _init_minio_client(self):
"""Initialize MinIO client from environment variables or injected config.
@@ -203,58 +215,69 @@ class MinioStorage(Storage):
prefer those values to override envs.
"""
try:
endpoint = os.getenv("MINIO_ENDPOINT")
endpoint = os.getenv("MINIO_ENDPOINT", "")
use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true"
if self.storage_config and self.storage_config.get("bucket_name"):
storage_config = self.storage_config
bucket_name = storage_config.get("bucket_name")
bucket_name = storage_config.get("bucket_name", "")
path_prefix = storage_config.get("path_prefix").strip().strip("/")
access_key = storage_config.get("access_key_id")
secret_key = storage_config.get("secret_access_key")
else:
access_key = os.getenv("MINIO_ACCESS_KEY_ID")
secret_key = os.getenv("MINIO_SECRET_ACCESS_KEY")
bucket_name = os.getenv("MINIO_BUCKET_NAME")
bucket_name = os.getenv("MINIO_BUCKET_NAME", "")
path_prefix = os.getenv("MINIO_PATH_PREFIX", "").strip().strip("/")
if not all([endpoint, access_key, secret_key, bucket_name]):
logger.error("Incomplete MinIO configuration, missing required environment variables")
logger.error(
"Incomplete MinIO configuration, missing environment variables"
)
return None, None, None, None, None
# Initialize client
client = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl)
client = Minio(
endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl
)
# Ensure bucket exists
found = client.bucket_exists(bucket_name)
if not found:
client.make_bucket(bucket_name)
policy = '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}' % (bucket_name, bucket_name)
policy = (
'{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}'
% (bucket_name, bucket_name)
)
client.set_bucket_policy(bucket_name, policy)
return client, bucket_name, use_ssl, endpoint, path_prefix
except Exception as e:
logger.error(f"Failed to initialize MinIO client: {str(e)}")
return None, None, None, None, None
def _get_download_url(self, bucket_name: str, object_key: str, use_ssl: bool, endpoint: str, public_endpoint: str = None):
def _get_download_url(self, object_key: str):
"""Construct a public URL for MinIO object.
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
"""
if public_endpoint:
base = public_endpoint
else:
scheme = "https" if use_ssl else "http"
base = f"{scheme}://{endpoint}"
# Path-style URL for MinIO
return f"{base}/{bucket_name}/{object_key}"
# 1. Use public endpoint if provided
endpoint = os.getenv("MINIO_PUBLIC_ENDPOINT")
if endpoint:
return f"{endpoint}/{self.bucket_name}/{object_key}"
# 2. Use SSL if enabled
if self.use_ssl:
return f"https://{self.endpoint}/{self.bucket_name}/{object_key}"
# 3. Use HTTP default
return f"http://{self.endpoint}/{self.bucket_name}/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to MinIO
Args:
file_path: File path
Returns:
File URL
"""
@@ -265,29 +288,27 @@ class MinioStorage(Storage):
# Generate object key, use UUID to avoid conflicts
file_name = os.path.basename(file_path)
object_key = f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}" if self.path_prefix else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
object_key = (
f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
if self.path_prefix
else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
)
logger.info(f"Generated MinIO object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to MinIO")
with open(file_path, 'rb') as file_data:
with open(file_path, "rb") as file_data:
file_size = os.path.getsize(file_path)
self.client.put_object(
bucket_name=self.bucket_name,
bucket_name=self.bucket_name or "",
object_name=object_key,
data=file_data,
length=file_size,
content_type='application/octet-stream'
content_type="application/octet-stream",
)
# Get file URL
file_url = self._get_download_url(
self.bucket_name,
object_key,
self.use_ssl,
self.endpoint,
os.getenv("MINIO_PUBLIC_ENDPOINT", None)
)
file_url = self._get_download_url(object_key)
logger.info(f"Successfully uploaded file to MinIO: {file_url}")
return file_url
@@ -295,14 +316,14 @@ class MinioStorage(Storage):
except Exception as e:
logger.error(f"Failed to upload file to MinIO: {str(e)}")
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to MinIO
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
@@ -310,23 +331,21 @@ class MinioStorage(Storage):
logger.info(f"Uploading bytes content to MinIO, size: {len(content)} bytes")
if not self.client:
return ""
object_key = f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}" if self.path_prefix else f"images/{uuid.uuid4().hex}{file_ext}"
object_key = (
f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}"
if self.path_prefix
else f"images/{uuid.uuid4().hex}{file_ext}"
)
logger.info(f"Generated MinIO object key: {object_key}")
self.client.put_object(
self.bucket_name,
object_key,
data=io.BytesIO(content),
length=len(content),
content_type="application/octet-stream"
)
file_url = self._get_download_url(
self.bucket_name,
object_key,
self.use_ssl,
self.endpoint,
os.getenv("MINIO_PUBLIC_ENDPOINT", None)
self.bucket_name or "",
object_key,
data=io.BytesIO(content),
length=len(content),
content_type="application/octet-stream",
)
file_url = self._get_download_url(object_key)
logger.info(f"Successfully uploaded bytes to MinIO: {file_url}")
return file_url
except Exception as e:
@@ -335,26 +354,61 @@ class MinioStorage(Storage):
return ""
def create_storage(storage_config=None) -> Storage:
class LocalStorage(Storage):
"""Local file system storage implementation"""
def __init__(self, storage_config: Dict[str, str] = {}):
self.storage_config = storage_config
base_dir = storage_config.get(
"base_dir", os.getenv("LOCAL_STORAGE_BASE_DIR", "")
)
self.image_dir = os.path.join(base_dir, "images")
os.makedirs(self.image_dir, exist_ok=True)
def upload_file(self, file_path: str) -> str:
logger.info(f"Uploading file to local storage: {file_path}")
return file_path
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
logger.info(f"Uploading file to local storage: {len(content)} bytes")
fname = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
with open(fname, "wb") as f:
f.write(content)
return fname
class Base64Storage(Storage):
def upload_file(self, file_path: str) -> str:
logger.info(f"Uploading file to base64 storage: {file_path}")
return file_path
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
logger.info(f"Uploading file to base64 storage: {len(content)} bytes")
file_ext = file_ext.lstrip(".")
return f"data:image/{file_ext};base64,{endecode.decode_image(content)}"
def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
"""Create a storage instance based on configuration or environment variables
Args:
storage_config: Storage configuration dictionary
Returns:
Storage instance
"""
storage_type = os.getenv("STORAGE_TYPE", "cos").lower()
if storage_config:
storage_type = str(storage_config.get("provider", storage_type)).lower()
logger.info(f"Creating {storage_type} storage instance")
if storage_type == "minio":
return MinioStorage(storage_config)
elif storage_type == "cos":
# Default to COS
return CosStorage(storage_config)
else:
return None
elif storage_type == "local":
return LocalStorage(storage_config or {})
elif storage_type == "base64":
return Base64Storage()
raise ValueError(f"Invalid storage type: {storage_type}")
docreader/parser/text_parser.py
+8 -11
View File
@@ -1,6 +1,8 @@
import logging
from .base_parser import BaseParser
from typing import Dict, Any, Tuple, Union
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.utils import endecode
logger = logging.getLogger(__name__)
@@ -11,7 +13,7 @@ class TextParser(BaseParser):
This parser handles text extraction and chunking from plain text documents.
"""
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
def parse_into_text(self, content: bytes) -> Document:
"""
Parse text document content by decoding bytes to string.
@@ -25,20 +27,15 @@ class TextParser(BaseParser):
Parsed text content as string
"""
logger.info(f"Parsing text document, content size: {len(content)} bytes")
text = self.decode_bytes(content)
text = endecode.decode_bytes(content)
logger.info(
f"Successfully parsed text document, extracted {len(text)} characters"
)
return text
return Document(content=text)
if __name__ == "__main__":
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
logger.info("Running TextParser in standalone mode")
logger = logging.getLogger(__name__)
# Sample text for testing
text = """## 标题1
docreader/parser/web_parser.py
+15 -12
View File
@@ -1,11 +1,14 @@
from typing import Any, Optional, Tuple, Dict, Union
import os
from playwright.async_api import async_playwright
from bs4 import BeautifulSoup
from .base_parser import BaseParser, ParseResult
import logging
import asyncio
import logging
import os
from typing import Any
from bs4 import BeautifulSoup
from playwright.async_api import async_playwright
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.utils import endecode
logger = logging.getLogger(__name__)
@@ -59,7 +62,7 @@ class WebParser(BaseParser):
# Return empty BeautifulSoup object on error
return BeautifulSoup("", "html.parser")
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
def parse_into_text(self, content: bytes) -> Document:
"""Parse web page
Args:
@@ -78,10 +81,10 @@ class WebParser(BaseParser):
# Run async method
# Handle content possibly being a string
if isinstance(content, bytes):
url = self.decode_bytes(content)
url = endecode.decode_bytes(content)
logger.info(f"Decoded URL from bytes: {url}")
else:
url = content
url = str(content)
logger.info(f"Using content as URL directly: {url}")
logger.info(f"Scraping web page: {url}")
@@ -118,11 +121,11 @@ class WebParser(BaseParser):
logger.info(
f"Web page parsing complete, total content: {len(result)} characters"
)
return result
return Document(content=result)
except Exception as e:
logger.error(f"Error parsing web page: {str(e)}")
return f"Error parsing web page: {str(e)}"
return Document(content=f"Error parsing web page: {str(e)}")
finally:
# Close event loop