chore(docreader): 重新组织模块文件

This commit is contained in:
begoniezhao
2025-11-05 11:33:50 +08:00
parent 0d790ffedc
commit c1f731e026
43 changed files with 3435 additions and 2398 deletions
+42
View File
@@ -0,0 +1,42 @@
"""
Parser module for WeKnora document processing system.
This module provides document parsers for various file formats including:
- Microsoft Word documents (.doc, .docx)
- PDF documents
- Markdown files
- Plain text files
- Images with text content
- Web pages
The parsers extract content from documents and can split them into
meaningful chunks for further processing and indexing.
"""
from .base_parser import BaseParser, ParseResult
from .docx_parser import DocxParser
from .doc_parser import DocParser
from .pdf_parser import PDFParser
from .markdown_parser import MarkdownParser
from .text_parser import TextParser
from .image_parser import ImageParser
from .web_parser import WebParser
from .parser import Parser
from .config import ChunkingConfig
from .ocr_engine import OCREngine
# Export public classes and modules
__all__ = [
"BaseParser", # Base parser class that all format parsers inherit from
"DocxParser", # Parser for .docx files (modern Word documents)
"DocParser", # Parser for .doc files (legacy Word documents)
"PDFParser", # Parser for PDF documents
"MarkdownParser", # Parser for Markdown text files
"TextParser", # Parser for plain text files
"ImageParser", # Parser for images with text content
"WebParser", # Parser for web pages
"Parser", # Main parser factory that selects the appropriate parser
"ChunkingConfig", # Configuration for text chunking behavior
"ParseResult", # Standard result format returned by all parsers
"OCREngine", # OCR engine for extracting text from images
]
File diff suppressed because it is too large Load Diff
+360
View File
@@ -0,0 +1,360 @@
import json
import logging
import os
import time
from dataclasses import dataclass, field
from typing import List, Optional, Union
import requests
import ollama
logger = logging.getLogger(__name__)
@dataclass
class ImageUrl:
"""Image URL data structure for caption requests."""
url: Optional[str] = None
detail: Optional[str] = None
@dataclass
class Content:
"""Content data structure that can contain text or image URL."""
type: Optional[str] = None
text: Optional[str] = None
image_url: Optional[ImageUrl] = None
@dataclass
class SystemMessage:
"""System message for VLM model requests."""
role: Optional[str] = None
content: Optional[str] = None
@dataclass
class UserMessage:
"""User message for VLM model requests, can contain multiple content items."""
role: Optional[str] = None
content: List[Content] = field(default_factory=list)
@dataclass
class CompletionRequest:
"""Request structure for VLM model completion API."""
model: str
temperature: float
top_p: float
messages: List[Union[SystemMessage, UserMessage]]
user: str
@dataclass
class Model:
"""Model identifier structure."""
id: str
@dataclass
class ModelsResp:
"""Response structure for available models API."""
data: List[Model] = field(default_factory=list)
@dataclass
class Message:
"""Message structure in API response."""
role: Optional[str] = None
content: Optional[str] = None
tool_calls: Optional[str] = None
@dataclass
class Choice:
"""Choice structure in API response."""
message: Optional[Message] = None
@dataclass
class Usage:
"""Token usage information in API response."""
prompt_tokens: Optional[int] = 0
total_tokens: Optional[int] = 0
completion_tokens: Optional[int] = 0
@dataclass
class CaptionChatResp:
"""Response structure for caption chat API."""
id: Optional[str] = None
created: Optional[int] = None
model: Optional[Model] = None
object: Optional[str] = None
choices: List[Choice] = field(default_factory=list)
usage: Optional[Usage] = None
@staticmethod
def from_json(json_data: dict) -> "CaptionChatResp":
"""
Parse API response JSON into a CaptionChatResp object.
Args:
json_data: The JSON response from the API
Returns:
A parsed CaptionChatResp object
"""
logger.info("Parsing CaptionChatResp from JSON")
# Manually parse nested fields with safe field extraction
choices = []
for choice in json_data.get("choices", []):
message_data = choice.get("message", {})
message = Message(
role=message_data.get("role"),
content=message_data.get("content"),
tool_calls=message_data.get("tool_calls"),
)
choices.append(Choice(message=message))
# Handle usage with safe field extraction
usage_data = json_data.get("usage", {})
usage = None
if usage_data:
usage = Usage(
prompt_tokens=usage_data.get("prompt_tokens", 0),
total_tokens=usage_data.get("total_tokens", 0),
completion_tokens=usage_data.get("completion_tokens", 0),
)
logger.info(
f"Parsed {len(choices)} choices and usage data: {usage is not None}"
)
return CaptionChatResp(
id=json_data.get("id"),
created=json_data.get("created"),
model=json_data.get("model"),
object=json_data.get("object"),
choices=choices,
usage=usage,
)
def choice_data(self) -> str:
"""
Extract the content from the first choice in the response.
Returns:
The content string from the first choice, or empty string if no choices
"""
if self.choices:
logger.info("Retrieving content from first choice")
return self.choices[0].message.content
logger.warning("No choices available in response")
return ""
class Caption:
"""
Service for generating captions for images using a Vision Language Model.
Uses an external API to process images and return textual descriptions.
"""
def __init__(self, vlm_config=None):
"""Initialize the Caption service with configuration from parameters or environment variables."""
logger.info("Initializing Caption service")
self.prompt = """简单凝炼的描述图片的主要内容"""
# Use provided VLM config if available, otherwise fall back to environment variables
if vlm_config and vlm_config.get("base_url") and vlm_config.get("model_name"):
self.completion_url = vlm_config.get("base_url", "") + "/chat/completions"
self.model = vlm_config.get("model_name", "")
self.api_key = vlm_config.get("api_key", "")
self.interface_type = vlm_config.get("interface_type", "openai").lower()
else:
if os.getenv("VLM_MODEL_BASE_URL") == "" or os.getenv("VLM_MODEL_NAME") == "":
logger.error("VLM_MODEL_BASE_URL or VLM_MODEL_NAME is not set")
return
self.completion_url = os.getenv("VLM_MODEL_BASE_URL") + "/chat/completions"
self.model = os.getenv("VLM_MODEL_NAME")
self.api_key = os.getenv("VLM_MODEL_API_KEY")
self.interface_type = os.getenv("VLM_INTERFACE_TYPE", "openai").lower()
# 验证接口类型
if self.interface_type not in ["ollama", "openai"]:
logger.warning(f"Unknown interface type: {self.interface_type}, defaulting to openai")
self.interface_type = "openai"
logger.info(
f"Service configured with model: {self.model}, endpoint: {self.completion_url}, interface: {self.interface_type}"
)
def _call_caption_api(self, image_data: str) -> Optional[CaptionChatResp]:
"""
Call the Caption API to generate a description for the given image.
Args:
image_data: URL of the image or base64 encoded image data
Returns:
CaptionChatResp object if successful, None otherwise
"""
logger.info(f"Calling Caption API for image captioning")
logger.info(f"Processing image data: {image_data[:50] if len(image_data) > 50 else image_data}")
# 根据接口类型选择调用方式
if self.interface_type == "ollama":
return self._call_ollama_api(image_data)
else:
return self._call_openai_api(image_data)
def _call_ollama_api(self, image_base64: str) -> Optional[CaptionChatResp]:
"""Call Ollama API for image captioning using base64 encoded image data."""
host = self.completion_url.replace("/v1/chat/completions", "")
client = ollama.Client(
host=host,
)
try:
logger.info(f"Calling Ollama API with model: {self.model}")
# 调用Ollama API,使用images参数传递base64编码的图片
response = client.generate(
model=self.model,
prompt="简单凝炼的描述图片的主要内容",
images=[image_base64], # image_base64是base64编码的图片数据
options={"temperature": 0.1},
stream=False,
)
# 构造响应对象
caption_resp = CaptionChatResp(
id="ollama_response",
created=int(time.time()),
model=self.model,
object="chat.completion",
choices=[
Choice(
message=Message(
role="assistant",
content=response.response
)
)
]
)
logger.info("Successfully received response from Ollama API")
return caption_resp
except Exception as e:
logger.error(f"Error calling Ollama API: {e}")
return None
def _call_openai_api(self, image_base64: str) -> Optional[CaptionChatResp]:
"""Call OpenAI-compatible API for image captioning."""
logger.info(f"Calling OpenAI-compatible API with model: {self.model}")
user_msg = UserMessage(
role="user",
content=[
Content(type="text", text=self.prompt),
Content(
type="image_url", image_url=ImageUrl(url="data:image/png;base64," + image_base64, detail="auto")
),
],
)
gpt_req = CompletionRequest(
model=self.model,
temperature=0.3,
top_p=0.8,
messages=[user_msg],
user="abc",
)
headers = {
"Content-Type": "application/json",
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
try:
logger.info(f"Sending request to OpenAI-compatible API with model: {self.model}")
response = requests.post(
self.completion_url,
data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4),
headers=headers,
timeout=30,
)
if response.status_code != 200:
logger.error(
f"OpenAI-compatible API returned non-200 status code: {response.status_code}"
)
response.raise_for_status()
logger.info(
f"Successfully received response from OpenAI-compatible API with status: {response.status_code}"
)
logger.info(f"Converting response to CaptionChatResp object")
caption_resp = CaptionChatResp.from_json(response.json())
if caption_resp.usage:
logger.info(
f"API usage: prompt_tokens={caption_resp.usage.prompt_tokens}, "
f"completion_tokens={caption_resp.usage.completion_tokens}"
)
return caption_resp
except requests.exceptions.Timeout:
logger.error(f"Timeout while calling OpenAI-compatible API after 30 seconds")
return None
except requests.exceptions.RequestException as e:
logger.error(f"Request error calling OpenAI-compatible API: {e}")
return None
except Exception as e:
logger.error(f"Unexpected error calling OpenAI-compatible API: {e}")
return None
def get_caption(self, image_data: str) -> str:
"""
Get a caption for the provided image data.
Args:
image_data: URL of the image or base64 encoded image data
Returns:
Caption text as string, or empty string if captioning failed
"""
logger.info("Getting caption for image")
if not image_data or self.completion_url is None:
logger.error("Image data is not set")
return ""
caption_resp = self._call_caption_api(image_data)
if caption_resp:
caption = caption_resp.choice_data()
caption_length = len(caption)
logger.info(f"Successfully generated caption of length {caption_length}")
logger.info(
f"Caption: {caption[:50]}..."
if caption_length > 50
else f"Caption: {caption}"
)
return caption
logger.warning("Failed to get caption from Caption API")
return ""
+21
View File
@@ -0,0 +1,21 @@
from dataclasses import dataclass, field
@dataclass
class ChunkingConfig:
"""
Configuration for text chunking process.
Controls how documents are split into smaller pieces for processing.
"""
chunk_size: int = 512 # Maximum size of each chunk in tokens/chars
chunk_overlap: int = 50 # Number of tokens/chars to overlap between chunks
separators: list = field(
default_factory=lambda: ["\n\n", "\n", "。"]
) # Text separators in order of priority
enable_multimodal: bool = (
False # Whether to enable multimodal processing (text + images)
)
storage_config: dict = None # Preferred field name going forward
vlm_config: dict = None # VLM configuration for image captioning
+315
View File
@@ -0,0 +1,315 @@
import asyncio
import logging
import re
import tempfile
import os
import subprocess
import shutil
from io import BytesIO
from typing import Optional, List, Tuple
import textract
from PIL import Image
import zipfile
import xml.etree.ElementTree as ET
from .base_parser import BaseParser
from .docx_parser import DocxParser, Docx
logger = logging.getLogger(__name__)
class DocParser(BaseParser):
"""DOC document parser"""
def parse_into_text(self, content: bytes) -> str:
"""Parse DOC document
Args:
content: DOC document content
Returns:
Parse result
"""
logger.info(f"Parsing DOC document, content size: {len(content)} bytes")
# Save byte content as a temporary file
with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as temp_file:
temp_file_path = temp_file.name
temp_file.write(content)
temp_file.flush()
logger.info(f"Saved DOC content to temporary file: {temp_file_path}")
try:
# First try to convert to docx format to extract images
if self.enable_multimodal:
logger.info("Multimodal enabled, attempting to extract images from DOC")
docx_content = self._convert_doc_to_docx(temp_file_path)
if docx_content:
logger.info("Successfully converted DOC to DOCX, using DocxParser")
# Use existing DocxParser to parse the converted docx
docx_parser = DocxParser(
file_name=self.file_name,
file_type="docx",
enable_multimodal=self.enable_multimodal,
chunk_size=self.chunk_size,
chunk_overlap=self.chunk_overlap,
chunking_config=self.chunking_config,
separators=self.separators,
)
text = docx_parser.parse_into_text(docx_content)
logger.info(f"Extracted {len(text)} characters using DocxParser")
# Clean up temporary file
os.unlink(temp_file_path)
logger.info(f"Deleted temporary file: {temp_file_path}")
return text
else:
logger.warning(
"Failed to convert DOC to DOCX, falling back to text-only extraction"
)
# If image extraction is not needed or conversion failed, try using antiword to extract text
try:
logger.info("Attempting to parse DOC file with antiword")
# Check if antiword is installed
antiword_path = self._find_antiword_path()
if antiword_path:
# Use antiword to extract text directly
logger.info(f"Using antiword at {antiword_path} to extract text")
process = subprocess.Popen(
[antiword_path, temp_file_path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
if process.returncode == 0:
text = stdout.decode("utf-8", errors="ignore")
logger.info(
f"Successfully extracted {len(text)} characters using antiword"
)
# Clean up temporary file
os.unlink(temp_file_path)
logger.info(f"Deleted temporary file: {temp_file_path}")
return text
else:
logger.warning(
f"antiword extraction failed: {stderr.decode('utf-8', errors='ignore')}"
)
else:
logger.warning("antiword not found, falling back to textract")
except Exception as e:
logger.warning(
f"Error using antiword: {str(e)}, falling back to textract"
)
# If antiword fails, try using textract
logger.info("Parsing DOC file with textract")
text = textract.process(temp_file_path, method="antiword").decode("utf-8")
logger.info(
f"Successfully extracted {len(text)} characters of text from DOC document using textract"
)
# Clean up temporary file
os.unlink(temp_file_path)
logger.info(f"Deleted temporary file: {temp_file_path}")
return text
except Exception as e:
logger.error(f"Error parsing DOC document: {str(e)}")
# Ensure temporary file is cleaned up
if os.path.exists(temp_file_path):
os.unlink(temp_file_path)
logger.info(f"Deleted temporary file after error: {temp_file_path}")
return ""
def _convert_doc_to_docx(self, doc_path: str) -> Optional[bytes]:
"""Convert DOC file to DOCX format
Uses LibreOffice/OpenOffice for conversion
Args:
doc_path: DOC file path
Returns:
Byte stream of DOCX file content, or None if conversion fails
"""
logger.info(f"Converting DOC to DOCX: {doc_path}")
# Create a temporary directory to store the converted file
temp_dir = tempfile.mkdtemp()
docx_path = os.path.join(temp_dir, "converted.docx")
try:
# Check if LibreOffice or OpenOffice is installed
soffice_path = self._find_soffice_path()
if not soffice_path:
logger.error(
"LibreOffice/OpenOffice not found, cannot convert DOC to DOCX"
)
return None
# Execute conversion command
logger.info(f"Using {soffice_path} to convert DOC to DOCX")
cmd = [
soffice_path,
"--headless",
"--convert-to",
"docx",
"--outdir",
temp_dir,
doc_path,
]
logger.info(f"Running command: {' '.join(cmd)}")
process = subprocess.Popen(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
stdout, stderr = process.communicate()
if process.returncode != 0:
logger.error(
f"Error converting DOC to DOCX: {stderr.decode('utf-8', errors='ignore')}"
)
return None
# Find the converted file
for file in os.listdir(temp_dir):
if file.endswith(".docx"):
converted_file = os.path.join(temp_dir, file)
logger.info(f"Found converted file: {converted_file}")
# Read the converted file content
with open(converted_file, "rb") as f:
docx_content = f.read()
logger.info(
f"Successfully read converted DOCX file, size: {len(docx_content)} bytes"
)
return docx_content
logger.error("No DOCX file found after conversion")
return None
except Exception as e:
logger.error(f"Error during DOC to DOCX conversion: {str(e)}")
return None
finally:
# Clean up temporary directory
try:
shutil.rmtree(temp_dir)
logger.info(f"Cleaned up temporary directory: {temp_dir}")
except Exception as e:
logger.warning(f"Failed to clean up temporary directory: {str(e)}")
def _find_soffice_path(self) -> Optional[str]:
"""Find LibreOffice/OpenOffice executable path
Returns:
Executable path, or None if not found
"""
# Common LibreOffice/OpenOffice executable paths
possible_paths = [
# Linux
"/usr/bin/soffice",
"/usr/lib/libreoffice/program/soffice",
"/opt/libreoffice25.2/program/soffice",
# macOS
"/Applications/LibreOffice.app/Contents/MacOS/soffice",
# Windows
"C:\\Program Files\\LibreOffice\\program\\soffice.exe",
"C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe",
]
# Check if path is set in environment variable
if os.environ.get("LIBREOFFICE_PATH"):
possible_paths.insert(0, os.environ.get("LIBREOFFICE_PATH"))
for path in possible_paths:
if os.path.exists(path):
logger.info(f"Found LibreOffice/OpenOffice at: {path}")
return path
# Try to find in PATH
try:
result = subprocess.run(
["which", "soffice"], capture_output=True, text=True
)
if result.returncode == 0 and result.stdout.strip():
path = result.stdout.strip()
logger.info(f"Found LibreOffice/OpenOffice in PATH: {path}")
return path
except Exception:
pass
logger.warning("LibreOffice/OpenOffice not found")
return None
def _find_antiword_path(self) -> Optional[str]:
"""Find antiword executable path
Returns:
Executable path, or None if not found
"""
# Common antiword executable paths
possible_paths = [
# Linux/macOS
"/usr/bin/antiword",
"/usr/local/bin/antiword",
# Windows
"C:\\Program Files\\Antiword\\antiword.exe",
"C:\\Program Files (x86)\\Antiword\\antiword.exe",
]
# Check if path is set in environment variable
if os.environ.get("ANTIWORD_PATH"):
possible_paths.insert(0, os.environ.get("ANTIWORD_PATH"))
for path in possible_paths:
if os.path.exists(path):
logger.info(f"Found antiword at: {path}")
return path
# Try to find in PATH
try:
result = subprocess.run(
["which", "antiword"], capture_output=True, text=True
)
if result.returncode == 0 and result.stdout.strip():
path = result.stdout.strip()
logger.info(f"Found antiword in PATH: {path}")
return path
except Exception:
pass
logger.warning("antiword not found")
return None
if __name__ == "__main__":
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
logger.info("Running DocParser in standalone mode")
file_name = "/path/to/your/test.doc"
logger.info(f"Processing file: {file_name}")
doc_parser = DocParser(
file_name, enable_multimodal=True, chunk_size=512, chunk_overlap=60
)
logger.info("Parser initialized, starting processing")
with open(file_name, "rb") as f:
content = f.read()
text = doc_parser.parse_into_text(content)
logger.info(f"Processing complete, extracted text length: {len(text)}")
logger.info(f"Sample text: {text[:200]}...")
File diff suppressed because it is too large Load Diff
+68
View File
@@ -0,0 +1,68 @@
import logging
import os
import asyncio
from PIL import Image
import io
from typing import Dict, Any, Tuple, Union
from .base_parser import BaseParser, ParseResult
import numpy as np
# Set up logger for this module
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class ImageParser(BaseParser):
"""
Parser for image files with OCR capability.
Extracts text from images and generates captions.
This parser handles image processing by:
1. Uploading the image to storage
2. Generating a descriptive caption
3. Performing OCR to extract text content
4. Returning a combined result with both text and image reference
"""
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""
Parse image content, upload the image and return Markdown reference along with image map.
Args:
content: Raw image data (bytes)
Returns:
Tuple of (markdown_text, image_map) where image_map maps image URLs to PIL Image objects
"""
logger.info(f"Parsing image content, size: {len(content)} bytes")
image_map = {}
try:
# Upload image to storage service
logger.info("Uploading image to storage")
_, ext = os.path.splitext(self.file_name)
image_url = self.upload_bytes(content, file_ext=ext)
if not image_url:
logger.error("Failed to upload image to storage")
return "", {}
logger.info(
f"Successfully uploaded image, URL: {image_url[:50]}..."
if len(image_url) > 50
else f"Successfully uploaded image, URL: {image_url}"
)
# Create image object and add to map
try:
from PIL import Image
import io
image = Image.open(io.BytesIO(content))
image_map[image_url] = image
logger.info(f"Added image to image_map for URL: {image_url}")
except Exception as img_err:
logger.error(f"Error creating image object: {str(img_err)}")
markdown_text = f"![{self.file_name}]({image_url})"
return markdown_text, image_map
except Exception as e:
logger.error(f"Error parsing image: {str(e)}")
return "", {}
+43
View File
@@ -0,0 +1,43 @@
import base64
import io
import logging
from typing import Union
from PIL import Image
import numpy as np
logger = logging.getLogger(__name__)
def image_to_base64(image: Union[str, bytes, Image.Image, np.ndarray]) -> str:
"""Convert image to base64 encoded string
Args:
image: Image file path, bytes, PIL Image object, or numpy array
Returns:
Base64 encoded image string, or empty string if conversion fails
"""
try:
if isinstance(image, str):
# It's a file path
with open(image, "rb") as image_file:
return base64.b64encode(image_file.read()).decode("utf-8")
elif isinstance(image, bytes):
# It's bytes data
return base64.b64encode(image).decode("utf-8")
elif isinstance(image, Image.Image):
# It's a PIL Image
buffer = io.BytesIO()
image.save(buffer, format="PNG")
return base64.b64encode(buffer.getvalue()).decode("utf-8")
elif isinstance(image, np.ndarray):
# It's a numpy array
pil_image = Image.fromarray(image)
buffer = io.BytesIO()
pil_image.save(buffer, format="PNG")
return base64.b64encode(buffer.getvalue()).decode("utf-8")
else:
logger.error(f"Unsupported image type: {type(image)}")
return ""
except Exception as e:
logger.error(f"Error converting image to base64: {str(e)}")
return ""
+33
View File
@@ -0,0 +1,33 @@
import asyncio
import re
import logging
import numpy as np
import os # Import os module to get environment variables
from typing import Dict, List, Optional, Tuple, Union, Any
from .base_parser import BaseParser
# Get logger object
logger = logging.getLogger(__name__)
class MarkdownParser(BaseParser):
"""Markdown document parser"""
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""Parse Markdown document, only extract text content, do not process images
Args:
content: Markdown document content
Returns:
Parsed text result
"""
logger.info(f"Parsing Markdown document, content size: {len(content)} bytes")
# Convert byte content to string using universal decoding method
text = self.decode_bytes(content)
logger.info(f"Decoded Markdown content, text length: {len(text)} characters")
logger.info(f"Markdown parsing complete, extracted {len(text)} characters of text")
return text
+278
View File
@@ -0,0 +1,278 @@
import os
import logging
import base64
from typing import Optional, Union, Dict, Any
from abc import ABC, abstractmethod
from PIL import Image
import io
import numpy as np
from .image_utils import image_to_base64
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class OCRBackend(ABC):
"""Base class for OCR backends"""
@abstractmethod
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
pass
class PaddleOCRBackend(OCRBackend):
"""PaddleOCR backend implementation"""
def __init__(self, **kwargs):
"""Initialize PaddleOCR backend"""
self.ocr = None
try:
import os
import paddle
# Set PaddlePaddle to use CPU and disable GPU
os.environ['CUDA_VISIBLE_DEVICES'] = ''
paddle.set_device('cpu')
# 尝试检测CPU是否支持AVX指令集
try:
import subprocess
import platform
# 检测CPU是否支持AVX
if platform.system() == "Linux":
try:
result = subprocess.run(['grep', '-o', 'avx', '/proc/cpuinfo'],
capture_output=True, text=True, timeout=5)
has_avx = 'avx' in result.stdout.lower()
if not has_avx:
logger.warning("CPU does not support AVX instructions, using compatibility mode")
# 进一步限制指令集使用
os.environ['FLAGS_use_avx2'] = '0'
os.environ['FLAGS_use_avx'] = '1'
except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
logger.warning("Could not detect AVX support, using compatibility mode")
os.environ['FLAGS_use_avx2'] = '0'
os.environ['FLAGS_use_avx'] = '1'
except Exception as e:
logger.warning(f"Error detecting CPU capabilities: {e}, using compatibility mode")
os.environ['FLAGS_use_avx2'] = '0'
os.environ['FLAGS_use_avx'] = '1'
from paddleocr import PaddleOCR
# OCR configuration with text orientation classification enabled
ocr_config = {
"use_gpu": False,
"text_det_limit_type": "max",
"text_det_limit_side_len": 960,
"use_doc_orientation_classify": True, # 启用文档方向分类
"use_doc_unwarping": False,
"use_textline_orientation": True, # 启用文本行方向检测
"text_recognition_model_name": "PP-OCRv4_server_rec",
"text_detection_model_name": "PP-OCRv4_server_det",
"text_det_thresh": 0.3,
"text_det_box_thresh": 0.6,
"text_det_unclip_ratio": 1.5,
"text_rec_score_thresh": 0.0,
"ocr_version": "PP-OCRv4",
"lang": "ch",
"show_log": False,
"use_dilation": True, # improves accuracy
"det_db_score_mode": "slow", # improves accuracy
}
self.ocr = PaddleOCR(**ocr_config)
logger.info("PaddleOCR engine initialized successfully")
except ImportError as e:
logger.error(f"Failed to import paddleocr: {str(e)}. Please install it with 'pip install paddleocr'")
except OSError as e:
if "Illegal instruction" in str(e) or "core dumped" in str(e):
logger.error(f"PaddlePaddle crashed due to CPU instruction set incompatibility: {str(e)}")
logger.error("This usually happens when the CPU doesn't support AVX instructions.")
logger.error("Please try installing a CPU-only version of PaddlePaddle or use a different OCR backend.")
else:
logger.error(f"Failed to initialize PaddleOCR due to OS error: {str(e)}")
except Exception as e:
logger.error(f"Failed to initialize PaddleOCR: {str(e)}")
def predict(self, image):
"""Perform OCR recognition on the image
Args:
image: Image object (PIL.Image or numpy array)
Returns:
Extracted text string
"""
try:
# Ensure image is in RGB format
if hasattr(image, "convert") and image.mode != "RGB":
image = image.convert("RGB")
# Convert to numpy array if needed
if hasattr(image, "convert"):
image_array = np.array(image)
else:
image_array = image
# Perform OCR
ocr_result = self.ocr.ocr(image_array, cls=False)
# Extract text
ocr_text = ""
if ocr_result and ocr_result[0]:
for line in ocr_result[0]:
if line and len(line) >= 2:
text = line[1][0] if line[1] else ""
if text:
ocr_text += text + " "
text_length = len(ocr_text.strip())
if text_length > 0:
logger.info(f"OCR extracted {text_length} characters")
return ocr_text.strip()
else:
logger.warning("OCR returned empty result")
return ""
except Exception as e:
logger.error(f"OCR recognition error: {str(e)}")
return ""
class NanonetsOCRBackend(OCRBackend):
"""Nanonets OCR backend implementation using OpenAI API format"""
def __init__(self, **kwargs):
"""Initialize Nanonets OCR backend
Args:
api_key: API key for OpenAI API
base_url: Base URL for OpenAI API
model: Model name
"""
try:
from openai import OpenAI
self.api_key = kwargs.get("api_key", "123")
self.base_url = kwargs.get("base_url", "http://localhost:8000/v1")
self.model = kwargs.get("model", "nanonets/Nanonets-OCR-s")
self.temperature = kwargs.get("temperature", 0.0)
self.max_tokens = kwargs.get("max_tokens", 15000)
self.client = OpenAI(api_key=self.api_key, base_url=self.base_url)
self.prompt = """
## 任务说明
请从上传的文档中提取文字内容,严格按自然阅读顺序(从上到下,从左到右)输出,并遵循以下格式规范。
### 1. **文本处理**
* 按正常阅读顺序提取文字,语句流畅自然。
### 2. **表格**
* 所有表格统一转换为 **Markdown 表格格式**。
* 内容保持清晰、对齐整齐,便于阅读。
### 3. **公式**
* 所有公式转换为 **LaTeX 格式**,使用 `$$公式$$` 包裹。
### 4. **图片**
* 忽略图片信息
### 5. **链接**
* 不要猜测或补全不确定的链接地址。
"""
logger.info(f"Nanonets OCR engine initialized with model: {self.model}")
except ImportError:
logger.error("Failed to import openai. Please install it with 'pip install openai'")
self.client = None
except Exception as e:
logger.error(f"Failed to initialize Nanonets OCR: {str(e)}")
self.client = None
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
"""Extract text from an image using Nanonets OCR
Args:
image: Image file path, bytes, or PIL Image object
Returns:
Extracted text
"""
if self.client is None:
logger.error("Nanonets OCR client not initialized")
return ""
try:
# Encode image to base64
img_base64 = image_to_base64(image)
if not img_base64:
return ""
# Call Nanonets OCR API
logger.info(f"Calling Nanonets OCR API with model: {self.model}")
response = self.client.chat.completions.create(
model=self.model,
messages=[
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {"url": f"data:image/png;base64,{img_base64}"},
},
{
"type": "text",
"text": self.prompt,
},
],
}
],
temperature=self.temperature,
max_tokens=self.max_tokens
)
return response.choices[0].message.content
except Exception as e:
logger.error(f"Nanonets OCR prediction error: {str(e)}")
return ""
class OCREngine:
"""OCR Engine factory class"""
_instance = None
@classmethod
def get_instance(cls, backend_type="paddle", **kwargs) -> Optional[OCRBackend]:
"""Get OCR engine instance
Args:
backend_type: OCR backend type, one of: "paddle", "nanonets"
**kwargs: Additional arguments for the backend
Returns:
OCR engine instance or None if initialization fails
"""
if cls._instance is None:
logger.info(f"Initializing OCR engine with backend: {backend_type}")
if backend_type.lower() == "paddle":
cls._instance = PaddleOCRBackend(**kwargs)
elif backend_type.lower() == "nanonets":
cls._instance = NanonetsOCRBackend(**kwargs)
else:
logger.error(f"Unknown OCR backend type: {backend_type}")
return None
return cls._instance
+206
View File
@@ -0,0 +1,206 @@
import logging
from dataclasses import dataclass, field
from typing import Dict, Any, Optional, Type
from .base_parser import BaseParser, ParseResult
from .docx_parser import DocxParser
from .doc_parser import DocParser
from .pdf_parser import PDFParser
from .markdown_parser import MarkdownParser
from .text_parser import TextParser
from .image_parser import ImageParser
from .web_parser import WebParser
from .config import ChunkingConfig
import traceback
logger = logging.getLogger(__name__)
@dataclass
class Chunk:
"""
Represents a single text chunk with associated metadata.
Basic unit for document processing and embedding.
"""
content: str # Text content of the chunk
metadata: Dict[str, Any] = None # Associated metadata (source, page number, etc.)
class Parser:
"""
Document parser facade that integrates all specialized parsers.
Provides a unified interface for parsing various document types.
"""
def __init__(self):
logger.info("Initializing document parser")
# Initialize all parser types
self.parsers: Dict[str, Type[BaseParser]] = {
"docx": DocxParser,
"doc": DocParser,
"pdf": PDFParser,
"md": MarkdownParser,
"txt": TextParser,
"jpg": ImageParser,
"jpeg": ImageParser,
"png": ImageParser,
"gif": ImageParser,
"bmp": ImageParser,
"tiff": ImageParser,
"webp": ImageParser,
"markdown": MarkdownParser,
}
logger.info(
"Parser initialized with %d parsers: %s",
len(self.parsers),
", ".join(self.parsers.keys()),
)
def get_parser(self, file_type: str) -> Optional[Type[BaseParser]]:
"""
Get parser class for the specified file type.
Args:
file_type: The file extension or type identifier
Returns:
Parser class for the file type, or None if unsupported
"""
file_type = file_type.lower()
parser = self.parsers.get(file_type)
if parser:
logger.info(f"Found parser for file type: {file_type}")
else:
logger.warning(f"No parser found for file type: {file_type}")
return parser
def parse_file(
self,
file_name: str,
file_type: str,
content: bytes,
config: ChunkingConfig,
) -> Optional[ParseResult]:
"""
Parse file content using appropriate parser based on file type.
Args:
file_name: Name of the file being parsed
file_type: Type/extension of the file
content: Raw file content as bytes
config: Configuration for chunking process
Returns:
ParseResult containing chunks and metadata, or None if parsing failed
"""
logger.info(f"Parsing file: {file_name} with type: {file_type}")
logger.info(
f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, "
f"multimodal={config.enable_multimodal}"
)
parser_instance = None
try:
# Get appropriate parser for file type
cls = self.get_parser(file_type)
if cls is None:
logger.error(f"Unsupported file type: {file_type}")
return None
# Parse file content
logger.info(f"Creating parser instance for {file_type} file")
parser_instance = cls(
file_name=file_name,
file_type=file_type,
chunk_size=config.chunk_size,
chunk_overlap=config.chunk_overlap,
separators=config.separators,
enable_multimodal=config.enable_multimodal,
max_image_size=1920, # Limit image size to 1920px
max_concurrent_tasks=5, # Limit concurrent tasks to 5
chunking_config=config, # Pass the entire chunking config
)
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
result = parser_instance.parse(content)
if result:
logger.info(
f"Successfully parsed file {file_name}, generated {len(result.chunks)} chunks"
)
if result.chunks and len(result.chunks) > 0:
logger.info(
f"First chunk content length: {len(result.chunks[0].content)}"
)
else:
logger.warning(f"Parser returned empty chunks for file: {file_name}")
else:
logger.warning(f"Parser returned None result for file: {file_name}")
# Return parse results
return result
except Exception as e:
logger.error(f"Error parsing file {file_name}: {str(e)}")
logger.info(f"Detailed traceback: {traceback.format_exc()}")
return None
def parse_url(
self, url: str, title: str, config: ChunkingConfig
) -> Optional[ParseResult]:
"""
Parse content from a URL using the WebParser.
Args:
url: URL to parse
title: Title of the webpage (for metadata)
config: Configuration for chunking process
Returns:
ParseResult containing chunks and metadata, or None if parsing failed
"""
logger.info(f"Parsing URL: {url}, title: {title}")
logger.info(
f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, "
f"multimodal={config.enable_multimodal}"
)
parser_instance = None
try:
# Create web parser instance
logger.info("Creating WebParser instance")
parser_instance = WebParser(
title=title,
chunk_size=config.chunk_size,
chunk_overlap=config.chunk_overlap,
separators=config.separators,
enable_multimodal=config.enable_multimodal,
max_image_size=1920, # Limit image size
max_concurrent_tasks=5, # Limit concurrent tasks
chunking_config=config,
)
logger.info(f"Starting to parse URL content")
result = parser_instance.parse(url)
if result:
logger.info(
f"Successfully parsed URL, generated {len(result.chunks)} chunks"
)
logger.info(
f"First chunk content length: {len(result.chunks[0].content) if result.chunks else 0}"
)
else:
logger.warning(f"Parser returned empty result for URL: {url}")
# Return parse results
return result
except Exception as e:
logger.error(f"Error parsing URL {url}: {str(e)}")
logger.info(f"Detailed traceback: {traceback.format_exc()}")
return None
+113
View File
@@ -0,0 +1,113 @@
import logging
import os
import io
from typing import Any, List, Iterator, Optional, Mapping, Tuple, Dict, Union
import pdfplumber
import tempfile
from .base_parser import BaseParser
logger = logging.getLogger(__name__)
class PDFParser(BaseParser):
"""
PDF Document Parser
This parser handles PDF documents by extracting text content.
It uses the pypdf library for simple text extraction.
"""
def _convert_table_to_markdown(self, table_data: list) -> str:
if not table_data or not table_data[0]: return ""
def clean_cell(cell):
if cell is None: return ""
return str(cell).replace("\n", " <br> ")
try:
markdown = ""
header = [clean_cell(cell) for cell in table_data[0]]
markdown += "| " + " | ".join(header) + " |\n"
markdown += "| " + " | ".join(["---"] * len(header)) + " |\n"
for row in table_data[1:]:
if not row: continue
body_row = [clean_cell(cell) for cell in row]
if len(body_row) != len(header):
logger.warning(f"Skipping malformed table row: {body_row}")
continue
markdown += "| " + " | ".join(body_row) + " |\n"
return markdown
except Exception as e:
logger.error(f"Error converting table to markdown: {e}")
return ""
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
logger.info(f"Parsing PDF with pdfplumber, content size: {len(content)} bytes")
all_page_content = []
temp_pdf = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf")
temp_pdf_path = temp_pdf.name
try:
temp_pdf.write(content)
temp_pdf.close()
logger.info(f"PDF content written to temporary file: {temp_pdf_path}")
with pdfplumber.open(temp_pdf_path) as pdf:
logger.info(f"PDF has {len(pdf.pages)} pages")
for page_num, page in enumerate(pdf.pages):
page_content_parts = []
# Try-fallback strategy for table detection
default_settings = { "vertical_strategy": "lines", "horizontal_strategy": "lines" }
found_tables = page.find_tables(default_settings)
if not found_tables:
logger.info(f"Page {page_num+1}: Default strategy found no tables. Trying fallback strategy.")
fallback_settings = { "vertical_strategy": "text", "horizontal_strategy": "lines" }
found_tables = page.find_tables(fallback_settings)
table_bboxes = [table.bbox for table in found_tables]
# Define a filter function that keeps objects NOT inside any table bbox.
def not_within_bboxes(obj):
"""Check if an object is outside all table bounding boxes."""
for bbox in table_bboxes:
# Check if the object's vertical center is within a bbox
if bbox[1] <= (obj["top"] + obj["bottom"]) / 2 <= bbox[3]:
return False # It's inside a table, so we DON'T keep it
return True # It's outside all tables, so we DO keep it
# that contains only the non-table text.
non_table_page = page.filter(not_within_bboxes)
# Now, extract text from this filtered page view.
text = non_table_page.extract_text(x_tolerance=2)
if text:
page_content_parts.append(text)
# Process and append the structured Markdown tables
if found_tables:
logger.info(f"Found {len(found_tables)} tables on page {page_num + 1}")
for table in found_tables:
markdown_table = self._convert_table_to_markdown(table.extract())
page_content_parts.append(f"\n\n{markdown_table}\n\n")
all_page_content.append("".join(page_content_parts))
final_text = "\n\n--- Page Break ---\n\n".join(all_page_content)
logger.info(f"PDF parsing complete. Extracted {len(final_text)} text chars.")
return final_text
except Exception as e:
logger.error(f"Failed to parse PDF document: {str(e)}")
return ""
finally:
# This block is GUARANTEED to execute, preventing resource leaks.
if os.path.exists(temp_pdf_path):
try:
os.remove(temp_pdf_path)
logger.info(f"Temporary file cleaned up: {temp_pdf_path}")
except OSError as e:
logger.error(f"Error removing temporary file {temp_pdf_path}: {e}")
+360
View File
@@ -0,0 +1,360 @@
# -*- coding: utf-8 -*-
import os
import uuid
import logging
import io
import traceback
from abc import ABC, abstractmethod
from typing import Tuple, Optional
from qcloud_cos import CosConfig, CosS3Client
from minio import Minio
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class Storage(ABC):
"""Abstract base class for object storage operations"""
@abstractmethod
def upload_file(self, file_path: str) -> str:
"""Upload file to object storage
Args:
file_path: File path
Returns:
File URL
"""
pass
@abstractmethod
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to object storage
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
pass
class CosStorage(Storage):
"""Tencent Cloud COS storage implementation"""
def __init__(self, storage_config=None):
"""Initialize COS storage
Args:
storage_config: Storage configuration
"""
self.storage_config = storage_config
self.client, self.bucket_name, self.region, self.prefix = self._init_cos_client()
def _init_cos_client(self):
"""Initialize Tencent Cloud COS client"""
try:
# Use provided COS config if available, otherwise fall back to environment variables
if self.storage_config and self.storage_config.get("access_key_id") != "":
cos_config = self.storage_config
secret_id = cos_config.get("access_key_id")
secret_key = cos_config.get("secret_access_key")
region = cos_config.get("region")
bucket_name = cos_config.get("bucket_name")
appid = cos_config.get("app_id")
prefix = cos_config.get("path_prefix", "")
else:
# Get COS configuration from environment variables
secret_id = os.getenv("COS_SECRET_ID")
secret_key = os.getenv("COS_SECRET_KEY")
region = os.getenv("COS_REGION")
bucket_name = os.getenv("COS_BUCKET_NAME")
appid = os.getenv("COS_APP_ID")
prefix = os.getenv("COS_PATH_PREFIX")
enable_old_domain = (
os.getenv("COS_ENABLE_OLD_DOMAIN", "true").lower() == "true"
)
if not all([secret_id, secret_key, region, bucket_name, appid]):
logger.error(
"Incomplete COS configuration, missing required environment variables"
f"secret_id: {secret_id}, secret_key: {secret_key}, region: {region}, bucket_name: {bucket_name}, appid: {appid}"
)
return None, None, None, None
# Initialize COS configuration
logger.info(
f"Initializing COS client with region: {region}, bucket: {bucket_name}"
)
config = CosConfig(
Appid=appid,
Region=region,
SecretId=secret_id,
SecretKey=secret_key,
EnableOldDomain=enable_old_domain,
)
# Create client
client = CosS3Client(config)
return client, bucket_name, region, prefix
except Exception as e:
logger.error(f"Failed to initialize COS client: {str(e)}")
return None, None, None, None
def _get_download_url(self, bucket_name, region, object_key):
"""Generate COS object URL
Args:
bucket_name: Bucket name
region: Region
object_key: Object key
Returns:
File URL
"""
return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to Tencent Cloud COS
Args:
file_path: File path
Returns:
File URL
"""
logger.info(f"Uploading file to COS: {file_path}")
try:
if not self.client:
return ""
# Generate object key, use UUID to avoid conflicts
file_name = os.path.basename(file_path)
object_key = (
f"{self.prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
)
logger.info(f"Generated object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to COS")
response = self.client.upload_file(
Bucket=self.bucket_name, LocalFilePath=file_path, Key=object_key
)
# Get file URL
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
logger.info(f"Successfully uploaded file to COS: {file_url}")
return file_url
except Exception as e:
logger.error(f"Failed to upload file to COS: {str(e)}")
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to Tencent Cloud COS
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
try:
logger.info(f"Uploading bytes content to COS, size: {len(content)} bytes")
if not self.client:
return ""
object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}" if self.prefix else f"images/{uuid.uuid4().hex}{file_ext}"
logger.info(f"Generated object key: {object_key}")
self.client.put_object(Bucket=self.bucket_name, Body=content, Key=object_key)
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
logger.info(f"Successfully uploaded bytes to COS: {file_url}")
return file_url
except Exception as e:
logger.error(f"Failed to upload bytes to COS: {str(e)}")
traceback.print_exc()
return ""
class MinioStorage(Storage):
"""MinIO storage implementation"""
def __init__(self, storage_config=None):
"""Initialize MinIO storage
Args:
storage_config: Storage configuration
"""
self.storage_config = storage_config
self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = self._init_minio_client()
def _init_minio_client(self):
"""Initialize MinIO client from environment variables or injected config.
If storage_config.path_prefix contains JSON from server (for minio case),
prefer those values to override envs.
"""
try:
endpoint = os.getenv("MINIO_ENDPOINT")
use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true"
if self.storage_config and self.storage_config.get("bucket_name"):
storage_config = self.storage_config
bucket_name = storage_config.get("bucket_name")
path_prefix = storage_config.get("path_prefix").strip().strip("/")
access_key = storage_config.get("access_key_id")
secret_key = storage_config.get("secret_access_key")
else:
access_key = os.getenv("MINIO_ACCESS_KEY_ID")
secret_key = os.getenv("MINIO_SECRET_ACCESS_KEY")
bucket_name = os.getenv("MINIO_BUCKET_NAME")
path_prefix = os.getenv("MINIO_PATH_PREFIX", "").strip().strip("/")
if not all([endpoint, access_key, secret_key, bucket_name]):
logger.error("Incomplete MinIO configuration, missing required environment variables")
return None, None, None, None, None
# Initialize client
client = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl)
# Ensure bucket exists
found = client.bucket_exists(bucket_name)
if not found:
client.make_bucket(bucket_name)
policy = '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}' % (bucket_name, bucket_name)
client.set_bucket_policy(bucket_name, policy)
return client, bucket_name, use_ssl, endpoint, path_prefix
except Exception as e:
logger.error(f"Failed to initialize MinIO client: {str(e)}")
return None, None, None, None, None
def _get_download_url(self, bucket_name: str, object_key: str, use_ssl: bool, endpoint: str, public_endpoint: str = None):
"""Construct a public URL for MinIO object.
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
"""
if public_endpoint:
base = public_endpoint
else:
scheme = "https" if use_ssl else "http"
base = f"{scheme}://{endpoint}"
# Path-style URL for MinIO
return f"{base}/{bucket_name}/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to MinIO
Args:
file_path: File path
Returns:
File URL
"""
logger.info(f"Uploading file to MinIO: {file_path}")
try:
if not self.client:
return ""
# Generate object key, use UUID to avoid conflicts
file_name = os.path.basename(file_path)
object_key = f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}" if self.path_prefix else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
logger.info(f"Generated MinIO object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to MinIO")
with open(file_path, 'rb') as file_data:
file_size = os.path.getsize(file_path)
self.client.put_object(
bucket_name=self.bucket_name,
object_name=object_key,
data=file_data,
length=file_size,
content_type='application/octet-stream'
)
# Get file URL
file_url = self._get_download_url(
self.bucket_name,
object_key,
self.use_ssl,
self.endpoint,
os.getenv("MINIO_PUBLIC_ENDPOINT", None)
)
logger.info(f"Successfully uploaded file to MinIO: {file_url}")
return file_url
except Exception as e:
logger.error(f"Failed to upload file to MinIO: {str(e)}")
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to MinIO
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
try:
logger.info(f"Uploading bytes content to MinIO, size: {len(content)} bytes")
if not self.client:
return ""
object_key = f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}" if self.path_prefix else f"images/{uuid.uuid4().hex}{file_ext}"
logger.info(f"Generated MinIO object key: {object_key}")
self.client.put_object(
self.bucket_name,
object_key,
data=io.BytesIO(content),
length=len(content),
content_type="application/octet-stream"
)
file_url = self._get_download_url(
self.bucket_name,
object_key,
self.use_ssl,
self.endpoint,
os.getenv("MINIO_PUBLIC_ENDPOINT", None)
)
logger.info(f"Successfully uploaded bytes to MinIO: {file_url}")
return file_url
except Exception as e:
logger.error(f"Failed to upload bytes to MinIO: {str(e)}")
traceback.print_exc()
return ""
def create_storage(storage_config=None) -> Storage:
"""Create a storage instance based on configuration or environment variables
Args:
storage_config: Storage configuration dictionary
Returns:
Storage instance
"""
storage_type = os.getenv("STORAGE_TYPE", "cos").lower()
if storage_config:
storage_type = str(storage_config.get("provider", storage_type)).lower()
logger.info(f"Creating {storage_type} storage instance")
if storage_type == "minio":
return MinioStorage(storage_config)
elif storage_type == "cos":
# Default to COS
return CosStorage(storage_config)
else:
return None
+58
View File
@@ -0,0 +1,58 @@
import logging
from .base_parser import BaseParser
from typing import Dict, Any, Tuple, Union
logger = logging.getLogger(__name__)
class TextParser(BaseParser):
"""
Text document parser for processing plain text files.
This parser handles text extraction and chunking from plain text documents.
"""
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""
Parse text document content by decoding bytes to string.
This is a straightforward parser that simply converts the binary content
to text using appropriate character encoding.
Args:
content: Raw document content as bytes
Returns:
Parsed text content as string
"""
logger.info(f"Parsing text document, content size: {len(content)} bytes")
text = self.decode_bytes(content)
logger.info(
f"Successfully parsed text document, extracted {len(text)} characters"
)
return text
if __name__ == "__main__":
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
logger.info("Running TextParser in standalone mode")
# Sample text for testing
text = """## 标题1
![alt text](image.png)
## 标题2
![alt text](image2.png)
## 标题3
![alt text](image3.png)"""
logger.info(f"Test text content: {text}")
# Define separators for text splitting
seperators = ["\n\n", "\n", "。"]
parser = TextParser(separators=seperators)
logger.info("Splitting text into units")
units = parser._split_into_units(text)
logger.info(f"Split text into {len(units)} units")
logger.info(f"Units: {units}")
+130
View File
@@ -0,0 +1,130 @@
from typing import Any, Optional, Tuple, Dict, Union
import os
from playwright.async_api import async_playwright
from bs4 import BeautifulSoup
from .base_parser import BaseParser, ParseResult
import logging
import asyncio
logger = logging.getLogger(__name__)
class WebParser(BaseParser):
"""Web page parser"""
def __init__(self, title: str, **kwargs):
self.title = title
self.proxy = os.environ.get("WEB_PROXY", "")
super().__init__(file_name=title, **kwargs)
logger.info(f"Initialized WebParser with title: {title}")
async def scrape(self, url: str) -> Any:
logger.info(f"Starting web page scraping for URL: {url}")
try:
async with async_playwright() as p:
kwargs = {}
if self.proxy:
kwargs["proxy"] = {"server": self.proxy}
logger.info("Launching WebKit browser")
browser = await p.webkit.launch(**kwargs)
page = await browser.new_page()
logger.info(f"Navigating to URL: {url}")
try:
await page.goto(url, timeout=30000)
logger.info("Initial page load complete")
except Exception as e:
logger.error(f"Error navigating to URL: {str(e)}")
await browser.close()
return BeautifulSoup(
"", "html.parser"
) # Return empty soup on navigation error
logger.info("Retrieving page HTML content")
content = await page.content()
logger.info(f"Retrieved {len(content)} bytes of HTML content")
await browser.close()
logger.info("Browser closed")
# Parse HTML content with BeautifulSoup
logger.info("Parsing HTML with BeautifulSoup")
soup = BeautifulSoup(content, "html.parser")
logger.info("Successfully parsed HTML content")
return soup
except Exception as e:
logger.error(f"Failed to scrape web page: {str(e)}")
# Return empty BeautifulSoup object on error
return BeautifulSoup("", "html.parser")
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""Parse web page
Args:
content: Web page content
Returns:
Parse result
"""
logger.info("Starting web page parsing")
# Call async method synchronously
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
# Run async method
# Handle content possibly being a string
if isinstance(content, bytes):
url = self.decode_bytes(content)
logger.info(f"Decoded URL from bytes: {url}")
else:
url = content
logger.info(f"Using content as URL directly: {url}")
logger.info(f"Scraping web page: {url}")
soup = loop.run_until_complete(self.scrape(url))
# Extract page text
logger.info("Extracting text from web page")
text = soup.get_text("\n")
logger.info(f"Extracted {len(text)} characters of text from URL: {url}")
# Get title, usually in <title> or <h1> tag
if self.title != "":
title = self.title
logger.info(f"Using provided title: {title}")
else:
title = soup.title.string if soup.title else None
logger.info(f"Found title tag: {title}")
if not title: # If <title> tag does not exist or is empty, try <h1> tag
h1_tag = soup.find("h1")
if h1_tag:
title = h1_tag.get_text()
logger.info(f"Using h1 tag as title: {title}")
else:
title = "Untitled Web Page"
logger.info("No title found, using default")
logger.info(f"Web page title: {title}")
text = "\n".join(
(line.strip() for line in text.splitlines() if line.strip())
)
result = title + "\n\n" + text
logger.info(
f"Web page parsing complete, total content: {len(result)} characters"
)
return result
except Exception as e:
logger.error(f"Error parsing web page: {str(e)}")
return f"Error parsing web page: {str(e)}"
finally:
# Close event loop
logger.info("Closing event loop")
loop.close()