chore(docreader): 重新组织模块文件
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
Parser module for WeKnora document processing system.
|
||||
|
||||
This module provides document parsers for various file formats including:
|
||||
- Microsoft Word documents (.doc, .docx)
|
||||
- PDF documents
|
||||
- Markdown files
|
||||
- Plain text files
|
||||
- Images with text content
|
||||
- Web pages
|
||||
|
||||
The parsers extract content from documents and can split them into
|
||||
meaningful chunks for further processing and indexing.
|
||||
"""
|
||||
|
||||
from .base_parser import BaseParser, ParseResult
|
||||
from .docx_parser import DocxParser
|
||||
from .doc_parser import DocParser
|
||||
from .pdf_parser import PDFParser
|
||||
from .markdown_parser import MarkdownParser
|
||||
from .text_parser import TextParser
|
||||
from .image_parser import ImageParser
|
||||
from .web_parser import WebParser
|
||||
from .parser import Parser
|
||||
from .config import ChunkingConfig
|
||||
from .ocr_engine import OCREngine
|
||||
|
||||
# Export public classes and modules
|
||||
__all__ = [
|
||||
"BaseParser", # Base parser class that all format parsers inherit from
|
||||
"DocxParser", # Parser for .docx files (modern Word documents)
|
||||
"DocParser", # Parser for .doc files (legacy Word documents)
|
||||
"PDFParser", # Parser for PDF documents
|
||||
"MarkdownParser", # Parser for Markdown text files
|
||||
"TextParser", # Parser for plain text files
|
||||
"ImageParser", # Parser for images with text content
|
||||
"WebParser", # Parser for web pages
|
||||
"Parser", # Main parser factory that selects the appropriate parser
|
||||
"ChunkingConfig", # Configuration for text chunking behavior
|
||||
"ParseResult", # Standard result format returned by all parsers
|
||||
"OCREngine", # OCR engine for extracting text from images
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,360 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import requests
|
||||
import ollama
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageUrl:
|
||||
"""Image URL data structure for caption requests."""
|
||||
|
||||
url: Optional[str] = None
|
||||
detail: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Content:
|
||||
"""Content data structure that can contain text or image URL."""
|
||||
|
||||
type: Optional[str] = None
|
||||
text: Optional[str] = None
|
||||
image_url: Optional[ImageUrl] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemMessage:
|
||||
"""System message for VLM model requests."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class UserMessage:
|
||||
"""User message for VLM model requests, can contain multiple content items."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: List[Content] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompletionRequest:
|
||||
"""Request structure for VLM model completion API."""
|
||||
|
||||
model: str
|
||||
temperature: float
|
||||
top_p: float
|
||||
messages: List[Union[SystemMessage, UserMessage]]
|
||||
user: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Model:
|
||||
"""Model identifier structure."""
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelsResp:
|
||||
"""Response structure for available models API."""
|
||||
|
||||
data: List[Model] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Message:
|
||||
"""Message structure in API response."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
tool_calls: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Choice:
|
||||
"""Choice structure in API response."""
|
||||
|
||||
message: Optional[Message] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Usage:
|
||||
"""Token usage information in API response."""
|
||||
|
||||
prompt_tokens: Optional[int] = 0
|
||||
total_tokens: Optional[int] = 0
|
||||
completion_tokens: Optional[int] = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CaptionChatResp:
|
||||
"""Response structure for caption chat API."""
|
||||
|
||||
id: Optional[str] = None
|
||||
created: Optional[int] = None
|
||||
model: Optional[Model] = None
|
||||
object: Optional[str] = None
|
||||
choices: List[Choice] = field(default_factory=list)
|
||||
usage: Optional[Usage] = None
|
||||
|
||||
@staticmethod
|
||||
def from_json(json_data: dict) -> "CaptionChatResp":
|
||||
"""
|
||||
Parse API response JSON into a CaptionChatResp object.
|
||||
|
||||
Args:
|
||||
json_data: The JSON response from the API
|
||||
|
||||
Returns:
|
||||
A parsed CaptionChatResp object
|
||||
"""
|
||||
logger.info("Parsing CaptionChatResp from JSON")
|
||||
# Manually parse nested fields with safe field extraction
|
||||
choices = []
|
||||
for choice in json_data.get("choices", []):
|
||||
message_data = choice.get("message", {})
|
||||
message = Message(
|
||||
role=message_data.get("role"),
|
||||
content=message_data.get("content"),
|
||||
tool_calls=message_data.get("tool_calls"),
|
||||
)
|
||||
choices.append(Choice(message=message))
|
||||
|
||||
# Handle usage with safe field extraction
|
||||
usage_data = json_data.get("usage", {})
|
||||
usage = None
|
||||
if usage_data:
|
||||
usage = Usage(
|
||||
prompt_tokens=usage_data.get("prompt_tokens", 0),
|
||||
total_tokens=usage_data.get("total_tokens", 0),
|
||||
completion_tokens=usage_data.get("completion_tokens", 0),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Parsed {len(choices)} choices and usage data: {usage is not None}"
|
||||
)
|
||||
return CaptionChatResp(
|
||||
id=json_data.get("id"),
|
||||
created=json_data.get("created"),
|
||||
model=json_data.get("model"),
|
||||
object=json_data.get("object"),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
def choice_data(self) -> str:
|
||||
"""
|
||||
Extract the content from the first choice in the response.
|
||||
|
||||
Returns:
|
||||
The content string from the first choice, or empty string if no choices
|
||||
"""
|
||||
if self.choices:
|
||||
logger.info("Retrieving content from first choice")
|
||||
return self.choices[0].message.content
|
||||
logger.warning("No choices available in response")
|
||||
return ""
|
||||
|
||||
|
||||
class Caption:
|
||||
"""
|
||||
Service for generating captions for images using a Vision Language Model.
|
||||
Uses an external API to process images and return textual descriptions.
|
||||
"""
|
||||
|
||||
def __init__(self, vlm_config=None):
|
||||
"""Initialize the Caption service with configuration from parameters or environment variables."""
|
||||
logger.info("Initializing Caption service")
|
||||
self.prompt = """简单凝炼的描述图片的主要内容"""
|
||||
|
||||
# Use provided VLM config if available, otherwise fall back to environment variables
|
||||
if vlm_config and vlm_config.get("base_url") and vlm_config.get("model_name"):
|
||||
self.completion_url = vlm_config.get("base_url", "") + "/chat/completions"
|
||||
self.model = vlm_config.get("model_name", "")
|
||||
self.api_key = vlm_config.get("api_key", "")
|
||||
self.interface_type = vlm_config.get("interface_type", "openai").lower()
|
||||
else:
|
||||
if os.getenv("VLM_MODEL_BASE_URL") == "" or os.getenv("VLM_MODEL_NAME") == "":
|
||||
logger.error("VLM_MODEL_BASE_URL or VLM_MODEL_NAME is not set")
|
||||
return
|
||||
self.completion_url = os.getenv("VLM_MODEL_BASE_URL") + "/chat/completions"
|
||||
self.model = os.getenv("VLM_MODEL_NAME")
|
||||
self.api_key = os.getenv("VLM_MODEL_API_KEY")
|
||||
self.interface_type = os.getenv("VLM_INTERFACE_TYPE", "openai").lower()
|
||||
|
||||
# 验证接口类型
|
||||
if self.interface_type not in ["ollama", "openai"]:
|
||||
logger.warning(f"Unknown interface type: {self.interface_type}, defaulting to openai")
|
||||
self.interface_type = "openai"
|
||||
|
||||
logger.info(
|
||||
f"Service configured with model: {self.model}, endpoint: {self.completion_url}, interface: {self.interface_type}"
|
||||
)
|
||||
|
||||
def _call_caption_api(self, image_data: str) -> Optional[CaptionChatResp]:
|
||||
"""
|
||||
Call the Caption API to generate a description for the given image.
|
||||
|
||||
Args:
|
||||
image_data: URL of the image or base64 encoded image data
|
||||
|
||||
Returns:
|
||||
CaptionChatResp object if successful, None otherwise
|
||||
"""
|
||||
logger.info(f"Calling Caption API for image captioning")
|
||||
logger.info(f"Processing image data: {image_data[:50] if len(image_data) > 50 else image_data}")
|
||||
|
||||
# 根据接口类型选择调用方式
|
||||
if self.interface_type == "ollama":
|
||||
return self._call_ollama_api(image_data)
|
||||
else:
|
||||
return self._call_openai_api(image_data)
|
||||
|
||||
def _call_ollama_api(self, image_base64: str) -> Optional[CaptionChatResp]:
|
||||
"""Call Ollama API for image captioning using base64 encoded image data."""
|
||||
|
||||
host = self.completion_url.replace("/v1/chat/completions", "")
|
||||
|
||||
client = ollama.Client(
|
||||
host=host,
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"Calling Ollama API with model: {self.model}")
|
||||
|
||||
# 调用Ollama API,使用images参数传递base64编码的图片
|
||||
response = client.generate(
|
||||
model=self.model,
|
||||
prompt="简单凝炼的描述图片的主要内容",
|
||||
images=[image_base64], # image_base64是base64编码的图片数据
|
||||
options={"temperature": 0.1},
|
||||
stream=False,
|
||||
)
|
||||
|
||||
# 构造响应对象
|
||||
caption_resp = CaptionChatResp(
|
||||
id="ollama_response",
|
||||
created=int(time.time()),
|
||||
model=self.model,
|
||||
object="chat.completion",
|
||||
choices=[
|
||||
Choice(
|
||||
message=Message(
|
||||
role="assistant",
|
||||
content=response.response
|
||||
)
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
logger.info("Successfully received response from Ollama API")
|
||||
return caption_resp
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calling Ollama API: {e}")
|
||||
return None
|
||||
|
||||
def _call_openai_api(self, image_base64: str) -> Optional[CaptionChatResp]:
|
||||
"""Call OpenAI-compatible API for image captioning."""
|
||||
logger.info(f"Calling OpenAI-compatible API with model: {self.model}")
|
||||
|
||||
user_msg = UserMessage(
|
||||
role="user",
|
||||
content=[
|
||||
Content(type="text", text=self.prompt),
|
||||
Content(
|
||||
type="image_url", image_url=ImageUrl(url="data:image/png;base64," + image_base64, detail="auto")
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
gpt_req = CompletionRequest(
|
||||
model=self.model,
|
||||
temperature=0.3,
|
||||
top_p=0.8,
|
||||
messages=[user_msg],
|
||||
user="abc",
|
||||
)
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
if self.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.api_key}"
|
||||
|
||||
try:
|
||||
logger.info(f"Sending request to OpenAI-compatible API with model: {self.model}")
|
||||
response = requests.post(
|
||||
self.completion_url,
|
||||
data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4),
|
||||
headers=headers,
|
||||
timeout=30,
|
||||
)
|
||||
if response.status_code != 200:
|
||||
logger.error(
|
||||
f"OpenAI-compatible API returned non-200 status code: {response.status_code}"
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
logger.info(
|
||||
f"Successfully received response from OpenAI-compatible API with status: {response.status_code}"
|
||||
)
|
||||
logger.info(f"Converting response to CaptionChatResp object")
|
||||
caption_resp = CaptionChatResp.from_json(response.json())
|
||||
|
||||
if caption_resp.usage:
|
||||
logger.info(
|
||||
f"API usage: prompt_tokens={caption_resp.usage.prompt_tokens}, "
|
||||
f"completion_tokens={caption_resp.usage.completion_tokens}"
|
||||
)
|
||||
|
||||
return caption_resp
|
||||
except requests.exceptions.Timeout:
|
||||
logger.error(f"Timeout while calling OpenAI-compatible API after 30 seconds")
|
||||
return None
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Request error calling OpenAI-compatible API: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error calling OpenAI-compatible API: {e}")
|
||||
return None
|
||||
|
||||
def get_caption(self, image_data: str) -> str:
|
||||
"""
|
||||
Get a caption for the provided image data.
|
||||
|
||||
Args:
|
||||
image_data: URL of the image or base64 encoded image data
|
||||
|
||||
Returns:
|
||||
Caption text as string, or empty string if captioning failed
|
||||
"""
|
||||
logger.info("Getting caption for image")
|
||||
if not image_data or self.completion_url is None:
|
||||
logger.error("Image data is not set")
|
||||
return ""
|
||||
caption_resp = self._call_caption_api(image_data)
|
||||
if caption_resp:
|
||||
caption = caption_resp.choice_data()
|
||||
caption_length = len(caption)
|
||||
logger.info(f"Successfully generated caption of length {caption_length}")
|
||||
logger.info(
|
||||
f"Caption: {caption[:50]}..."
|
||||
if caption_length > 50
|
||||
else f"Caption: {caption}"
|
||||
)
|
||||
return caption
|
||||
logger.warning("Failed to get caption from Caption API")
|
||||
return ""
|
||||
@@ -0,0 +1,21 @@
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkingConfig:
|
||||
"""
|
||||
Configuration for text chunking process.
|
||||
Controls how documents are split into smaller pieces for processing.
|
||||
"""
|
||||
|
||||
chunk_size: int = 512 # Maximum size of each chunk in tokens/chars
|
||||
chunk_overlap: int = 50 # Number of tokens/chars to overlap between chunks
|
||||
separators: list = field(
|
||||
default_factory=lambda: ["\n\n", "\n", "。"]
|
||||
) # Text separators in order of priority
|
||||
enable_multimodal: bool = (
|
||||
False # Whether to enable multimodal processing (text + images)
|
||||
)
|
||||
storage_config: dict = None # Preferred field name going forward
|
||||
vlm_config: dict = None # VLM configuration for image captioning
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import tempfile
|
||||
import os
|
||||
import subprocess
|
||||
import shutil
|
||||
from io import BytesIO
|
||||
from typing import Optional, List, Tuple
|
||||
import textract
|
||||
from PIL import Image
|
||||
import zipfile
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from .base_parser import BaseParser
|
||||
from .docx_parser import DocxParser, Docx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DocParser(BaseParser):
|
||||
"""DOC document parser"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> str:
|
||||
"""Parse DOC document
|
||||
|
||||
Args:
|
||||
content: DOC document content
|
||||
|
||||
Returns:
|
||||
Parse result
|
||||
"""
|
||||
logger.info(f"Parsing DOC document, content size: {len(content)} bytes")
|
||||
|
||||
# Save byte content as a temporary file
|
||||
with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as temp_file:
|
||||
temp_file_path = temp_file.name
|
||||
temp_file.write(content)
|
||||
temp_file.flush()
|
||||
logger.info(f"Saved DOC content to temporary file: {temp_file_path}")
|
||||
|
||||
try:
|
||||
# First try to convert to docx format to extract images
|
||||
if self.enable_multimodal:
|
||||
logger.info("Multimodal enabled, attempting to extract images from DOC")
|
||||
docx_content = self._convert_doc_to_docx(temp_file_path)
|
||||
|
||||
if docx_content:
|
||||
logger.info("Successfully converted DOC to DOCX, using DocxParser")
|
||||
# Use existing DocxParser to parse the converted docx
|
||||
docx_parser = DocxParser(
|
||||
file_name=self.file_name,
|
||||
file_type="docx",
|
||||
enable_multimodal=self.enable_multimodal,
|
||||
chunk_size=self.chunk_size,
|
||||
chunk_overlap=self.chunk_overlap,
|
||||
chunking_config=self.chunking_config,
|
||||
separators=self.separators,
|
||||
)
|
||||
text = docx_parser.parse_into_text(docx_content)
|
||||
logger.info(f"Extracted {len(text)} characters using DocxParser")
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file: {temp_file_path}")
|
||||
|
||||
return text
|
||||
else:
|
||||
logger.warning(
|
||||
"Failed to convert DOC to DOCX, falling back to text-only extraction"
|
||||
)
|
||||
|
||||
# If image extraction is not needed or conversion failed, try using antiword to extract text
|
||||
try:
|
||||
logger.info("Attempting to parse DOC file with antiword")
|
||||
# Check if antiword is installed
|
||||
antiword_path = self._find_antiword_path()
|
||||
|
||||
if antiword_path:
|
||||
# Use antiword to extract text directly
|
||||
logger.info(f"Using antiword at {antiword_path} to extract text")
|
||||
process = subprocess.Popen(
|
||||
[antiword_path, temp_file_path],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = process.communicate()
|
||||
|
||||
if process.returncode == 0:
|
||||
text = stdout.decode("utf-8", errors="ignore")
|
||||
logger.info(
|
||||
f"Successfully extracted {len(text)} characters using antiword"
|
||||
)
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file: {temp_file_path}")
|
||||
|
||||
return text
|
||||
else:
|
||||
logger.warning(
|
||||
f"antiword extraction failed: {stderr.decode('utf-8', errors='ignore')}"
|
||||
)
|
||||
else:
|
||||
logger.warning("antiword not found, falling back to textract")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Error using antiword: {str(e)}, falling back to textract"
|
||||
)
|
||||
|
||||
# If antiword fails, try using textract
|
||||
logger.info("Parsing DOC file with textract")
|
||||
text = textract.process(temp_file_path, method="antiword").decode("utf-8")
|
||||
logger.info(
|
||||
f"Successfully extracted {len(text)} characters of text from DOC document using textract"
|
||||
)
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file: {temp_file_path}")
|
||||
|
||||
return text
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing DOC document: {str(e)}")
|
||||
# Ensure temporary file is cleaned up
|
||||
if os.path.exists(temp_file_path):
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file after error: {temp_file_path}")
|
||||
return ""
|
||||
|
||||
def _convert_doc_to_docx(self, doc_path: str) -> Optional[bytes]:
|
||||
"""Convert DOC file to DOCX format
|
||||
|
||||
Uses LibreOffice/OpenOffice for conversion
|
||||
|
||||
Args:
|
||||
doc_path: DOC file path
|
||||
|
||||
Returns:
|
||||
Byte stream of DOCX file content, or None if conversion fails
|
||||
"""
|
||||
logger.info(f"Converting DOC to DOCX: {doc_path}")
|
||||
|
||||
# Create a temporary directory to store the converted file
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
docx_path = os.path.join(temp_dir, "converted.docx")
|
||||
|
||||
try:
|
||||
# Check if LibreOffice or OpenOffice is installed
|
||||
soffice_path = self._find_soffice_path()
|
||||
if not soffice_path:
|
||||
logger.error(
|
||||
"LibreOffice/OpenOffice not found, cannot convert DOC to DOCX"
|
||||
)
|
||||
return None
|
||||
|
||||
# Execute conversion command
|
||||
logger.info(f"Using {soffice_path} to convert DOC to DOCX")
|
||||
cmd = [
|
||||
soffice_path,
|
||||
"--headless",
|
||||
"--convert-to",
|
||||
"docx",
|
||||
"--outdir",
|
||||
temp_dir,
|
||||
doc_path,
|
||||
]
|
||||
|
||||
logger.info(f"Running command: {' '.join(cmd)}")
|
||||
process = subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
||||
)
|
||||
stdout, stderr = process.communicate()
|
||||
|
||||
if process.returncode != 0:
|
||||
logger.error(
|
||||
f"Error converting DOC to DOCX: {stderr.decode('utf-8', errors='ignore')}"
|
||||
)
|
||||
return None
|
||||
|
||||
# Find the converted file
|
||||
for file in os.listdir(temp_dir):
|
||||
if file.endswith(".docx"):
|
||||
converted_file = os.path.join(temp_dir, file)
|
||||
logger.info(f"Found converted file: {converted_file}")
|
||||
|
||||
# Read the converted file content
|
||||
with open(converted_file, "rb") as f:
|
||||
docx_content = f.read()
|
||||
|
||||
logger.info(
|
||||
f"Successfully read converted DOCX file, size: {len(docx_content)} bytes"
|
||||
)
|
||||
return docx_content
|
||||
|
||||
logger.error("No DOCX file found after conversion")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during DOC to DOCX conversion: {str(e)}")
|
||||
return None
|
||||
finally:
|
||||
# Clean up temporary directory
|
||||
try:
|
||||
shutil.rmtree(temp_dir)
|
||||
logger.info(f"Cleaned up temporary directory: {temp_dir}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up temporary directory: {str(e)}")
|
||||
|
||||
def _find_soffice_path(self) -> Optional[str]:
|
||||
"""Find LibreOffice/OpenOffice executable path
|
||||
|
||||
Returns:
|
||||
Executable path, or None if not found
|
||||
"""
|
||||
# Common LibreOffice/OpenOffice executable paths
|
||||
possible_paths = [
|
||||
# Linux
|
||||
"/usr/bin/soffice",
|
||||
"/usr/lib/libreoffice/program/soffice",
|
||||
"/opt/libreoffice25.2/program/soffice",
|
||||
# macOS
|
||||
"/Applications/LibreOffice.app/Contents/MacOS/soffice",
|
||||
# Windows
|
||||
"C:\\Program Files\\LibreOffice\\program\\soffice.exe",
|
||||
"C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe",
|
||||
]
|
||||
|
||||
# Check if path is set in environment variable
|
||||
if os.environ.get("LIBREOFFICE_PATH"):
|
||||
possible_paths.insert(0, os.environ.get("LIBREOFFICE_PATH"))
|
||||
|
||||
for path in possible_paths:
|
||||
if os.path.exists(path):
|
||||
logger.info(f"Found LibreOffice/OpenOffice at: {path}")
|
||||
return path
|
||||
|
||||
# Try to find in PATH
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["which", "soffice"], capture_output=True, text=True
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
path = result.stdout.strip()
|
||||
logger.info(f"Found LibreOffice/OpenOffice in PATH: {path}")
|
||||
return path
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.warning("LibreOffice/OpenOffice not found")
|
||||
return None
|
||||
|
||||
def _find_antiword_path(self) -> Optional[str]:
|
||||
"""Find antiword executable path
|
||||
|
||||
Returns:
|
||||
Executable path, or None if not found
|
||||
"""
|
||||
# Common antiword executable paths
|
||||
possible_paths = [
|
||||
# Linux/macOS
|
||||
"/usr/bin/antiword",
|
||||
"/usr/local/bin/antiword",
|
||||
# Windows
|
||||
"C:\\Program Files\\Antiword\\antiword.exe",
|
||||
"C:\\Program Files (x86)\\Antiword\\antiword.exe",
|
||||
]
|
||||
|
||||
# Check if path is set in environment variable
|
||||
if os.environ.get("ANTIWORD_PATH"):
|
||||
possible_paths.insert(0, os.environ.get("ANTIWORD_PATH"))
|
||||
|
||||
for path in possible_paths:
|
||||
if os.path.exists(path):
|
||||
logger.info(f"Found antiword at: {path}")
|
||||
return path
|
||||
|
||||
# Try to find in PATH
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["which", "antiword"], capture_output=True, text=True
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
path = result.stdout.strip()
|
||||
logger.info(f"Found antiword in PATH: {path}")
|
||||
return path
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.warning("antiword not found")
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
logger.info("Running DocParser in standalone mode")
|
||||
|
||||
file_name = "/path/to/your/test.doc"
|
||||
logger.info(f"Processing file: {file_name}")
|
||||
|
||||
doc_parser = DocParser(
|
||||
file_name, enable_multimodal=True, chunk_size=512, chunk_overlap=60
|
||||
)
|
||||
logger.info("Parser initialized, starting processing")
|
||||
|
||||
with open(file_name, "rb") as f:
|
||||
content = f.read()
|
||||
|
||||
text = doc_parser.parse_into_text(content)
|
||||
logger.info(f"Processing complete, extracted text length: {len(text)}")
|
||||
logger.info(f"Sample text: {text[:200]}...")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,68 @@
|
||||
import logging
|
||||
import os
|
||||
import asyncio
|
||||
from PIL import Image
|
||||
import io
|
||||
from typing import Dict, Any, Tuple, Union
|
||||
from .base_parser import BaseParser, ParseResult
|
||||
import numpy as np
|
||||
|
||||
# Set up logger for this module
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
class ImageParser(BaseParser):
|
||||
"""
|
||||
Parser for image files with OCR capability.
|
||||
Extracts text from images and generates captions.
|
||||
|
||||
This parser handles image processing by:
|
||||
1. Uploading the image to storage
|
||||
2. Generating a descriptive caption
|
||||
3. Performing OCR to extract text content
|
||||
4. Returning a combined result with both text and image reference
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
|
||||
"""
|
||||
Parse image content, upload the image and return Markdown reference along with image map.
|
||||
|
||||
Args:
|
||||
content: Raw image data (bytes)
|
||||
|
||||
Returns:
|
||||
Tuple of (markdown_text, image_map) where image_map maps image URLs to PIL Image objects
|
||||
"""
|
||||
logger.info(f"Parsing image content, size: {len(content)} bytes")
|
||||
image_map = {}
|
||||
|
||||
try:
|
||||
# Upload image to storage service
|
||||
logger.info("Uploading image to storage")
|
||||
_, ext = os.path.splitext(self.file_name)
|
||||
image_url = self.upload_bytes(content, file_ext=ext)
|
||||
if not image_url:
|
||||
logger.error("Failed to upload image to storage")
|
||||
return "", {}
|
||||
logger.info(
|
||||
f"Successfully uploaded image, URL: {image_url[:50]}..."
|
||||
if len(image_url) > 50
|
||||
else f"Successfully uploaded image, URL: {image_url}"
|
||||
)
|
||||
|
||||
# Create image object and add to map
|
||||
try:
|
||||
from PIL import Image
|
||||
import io
|
||||
image = Image.open(io.BytesIO(content))
|
||||
image_map[image_url] = image
|
||||
logger.info(f"Added image to image_map for URL: {image_url}")
|
||||
except Exception as img_err:
|
||||
logger.error(f"Error creating image object: {str(img_err)}")
|
||||
|
||||
markdown_text = f""
|
||||
return markdown_text, image_map
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing image: {str(e)}")
|
||||
return "", {}
|
||||
@@ -0,0 +1,43 @@
|
||||
import base64
|
||||
import io
|
||||
import logging
|
||||
from typing import Union
|
||||
from PIL import Image
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def image_to_base64(image: Union[str, bytes, Image.Image, np.ndarray]) -> str:
|
||||
"""Convert image to base64 encoded string
|
||||
|
||||
Args:
|
||||
image: Image file path, bytes, PIL Image object, or numpy array
|
||||
|
||||
Returns:
|
||||
Base64 encoded image string, or empty string if conversion fails
|
||||
"""
|
||||
try:
|
||||
if isinstance(image, str):
|
||||
# It's a file path
|
||||
with open(image, "rb") as image_file:
|
||||
return base64.b64encode(image_file.read()).decode("utf-8")
|
||||
elif isinstance(image, bytes):
|
||||
# It's bytes data
|
||||
return base64.b64encode(image).decode("utf-8")
|
||||
elif isinstance(image, Image.Image):
|
||||
# It's a PIL Image
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="PNG")
|
||||
return base64.b64encode(buffer.getvalue()).decode("utf-8")
|
||||
elif isinstance(image, np.ndarray):
|
||||
# It's a numpy array
|
||||
pil_image = Image.fromarray(image)
|
||||
buffer = io.BytesIO()
|
||||
pil_image.save(buffer, format="PNG")
|
||||
return base64.b64encode(buffer.getvalue()).decode("utf-8")
|
||||
else:
|
||||
logger.error(f"Unsupported image type: {type(image)}")
|
||||
return ""
|
||||
except Exception as e:
|
||||
logger.error(f"Error converting image to base64: {str(e)}")
|
||||
return ""
|
||||
@@ -0,0 +1,33 @@
|
||||
import asyncio
|
||||
import re
|
||||
import logging
|
||||
import numpy as np
|
||||
import os # Import os module to get environment variables
|
||||
from typing import Dict, List, Optional, Tuple, Union, Any
|
||||
from .base_parser import BaseParser
|
||||
|
||||
# Get logger object
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MarkdownParser(BaseParser):
|
||||
"""Markdown document parser"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
|
||||
"""Parse Markdown document, only extract text content, do not process images
|
||||
|
||||
Args:
|
||||
content: Markdown document content
|
||||
|
||||
Returns:
|
||||
Parsed text result
|
||||
"""
|
||||
logger.info(f"Parsing Markdown document, content size: {len(content)} bytes")
|
||||
|
||||
# Convert byte content to string using universal decoding method
|
||||
text = self.decode_bytes(content)
|
||||
logger.info(f"Decoded Markdown content, text length: {len(text)} characters")
|
||||
|
||||
logger.info(f"Markdown parsing complete, extracted {len(text)} characters of text")
|
||||
return text
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
import os
|
||||
import logging
|
||||
import base64
|
||||
from typing import Optional, Union, Dict, Any
|
||||
from abc import ABC, abstractmethod
|
||||
from PIL import Image
|
||||
import io
|
||||
import numpy as np
|
||||
from .image_utils import image_to_base64
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
class OCRBackend(ABC):
|
||||
"""Base class for OCR backends"""
|
||||
|
||||
@abstractmethod
|
||||
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
|
||||
"""Extract text from an image
|
||||
|
||||
Args:
|
||||
image: Image file path, bytes, or PIL Image object
|
||||
|
||||
Returns:
|
||||
Extracted text
|
||||
"""
|
||||
pass
|
||||
|
||||
class PaddleOCRBackend(OCRBackend):
|
||||
"""PaddleOCR backend implementation"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
"""Initialize PaddleOCR backend"""
|
||||
self.ocr = None
|
||||
try:
|
||||
import os
|
||||
import paddle
|
||||
|
||||
# Set PaddlePaddle to use CPU and disable GPU
|
||||
os.environ['CUDA_VISIBLE_DEVICES'] = ''
|
||||
paddle.set_device('cpu')
|
||||
|
||||
# 尝试检测CPU是否支持AVX指令集
|
||||
try:
|
||||
import subprocess
|
||||
import platform
|
||||
|
||||
# 检测CPU是否支持AVX
|
||||
if platform.system() == "Linux":
|
||||
try:
|
||||
result = subprocess.run(['grep', '-o', 'avx', '/proc/cpuinfo'],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
has_avx = 'avx' in result.stdout.lower()
|
||||
if not has_avx:
|
||||
logger.warning("CPU does not support AVX instructions, using compatibility mode")
|
||||
# 进一步限制指令集使用
|
||||
os.environ['FLAGS_use_avx2'] = '0'
|
||||
os.environ['FLAGS_use_avx'] = '1'
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
|
||||
logger.warning("Could not detect AVX support, using compatibility mode")
|
||||
os.environ['FLAGS_use_avx2'] = '0'
|
||||
os.environ['FLAGS_use_avx'] = '1'
|
||||
except Exception as e:
|
||||
logger.warning(f"Error detecting CPU capabilities: {e}, using compatibility mode")
|
||||
os.environ['FLAGS_use_avx2'] = '0'
|
||||
os.environ['FLAGS_use_avx'] = '1'
|
||||
|
||||
from paddleocr import PaddleOCR
|
||||
# OCR configuration with text orientation classification enabled
|
||||
ocr_config = {
|
||||
"use_gpu": False,
|
||||
"text_det_limit_type": "max",
|
||||
"text_det_limit_side_len": 960,
|
||||
"use_doc_orientation_classify": True, # 启用文档方向分类
|
||||
"use_doc_unwarping": False,
|
||||
"use_textline_orientation": True, # 启用文本行方向检测
|
||||
"text_recognition_model_name": "PP-OCRv4_server_rec",
|
||||
"text_detection_model_name": "PP-OCRv4_server_det",
|
||||
"text_det_thresh": 0.3,
|
||||
"text_det_box_thresh": 0.6,
|
||||
"text_det_unclip_ratio": 1.5,
|
||||
"text_rec_score_thresh": 0.0,
|
||||
"ocr_version": "PP-OCRv4",
|
||||
"lang": "ch",
|
||||
"show_log": False,
|
||||
"use_dilation": True, # improves accuracy
|
||||
"det_db_score_mode": "slow", # improves accuracy
|
||||
}
|
||||
|
||||
self.ocr = PaddleOCR(**ocr_config)
|
||||
logger.info("PaddleOCR engine initialized successfully")
|
||||
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import paddleocr: {str(e)}. Please install it with 'pip install paddleocr'")
|
||||
except OSError as e:
|
||||
if "Illegal instruction" in str(e) or "core dumped" in str(e):
|
||||
logger.error(f"PaddlePaddle crashed due to CPU instruction set incompatibility: {str(e)}")
|
||||
logger.error("This usually happens when the CPU doesn't support AVX instructions.")
|
||||
logger.error("Please try installing a CPU-only version of PaddlePaddle or use a different OCR backend.")
|
||||
else:
|
||||
logger.error(f"Failed to initialize PaddleOCR due to OS error: {str(e)}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize PaddleOCR: {str(e)}")
|
||||
|
||||
def predict(self, image):
|
||||
"""Perform OCR recognition on the image
|
||||
|
||||
Args:
|
||||
image: Image object (PIL.Image or numpy array)
|
||||
|
||||
Returns:
|
||||
Extracted text string
|
||||
"""
|
||||
try:
|
||||
# Ensure image is in RGB format
|
||||
if hasattr(image, "convert") and image.mode != "RGB":
|
||||
image = image.convert("RGB")
|
||||
|
||||
# Convert to numpy array if needed
|
||||
if hasattr(image, "convert"):
|
||||
image_array = np.array(image)
|
||||
else:
|
||||
image_array = image
|
||||
|
||||
# Perform OCR
|
||||
ocr_result = self.ocr.ocr(image_array, cls=False)
|
||||
|
||||
# Extract text
|
||||
ocr_text = ""
|
||||
if ocr_result and ocr_result[0]:
|
||||
for line in ocr_result[0]:
|
||||
if line and len(line) >= 2:
|
||||
text = line[1][0] if line[1] else ""
|
||||
if text:
|
||||
ocr_text += text + " "
|
||||
|
||||
text_length = len(ocr_text.strip())
|
||||
if text_length > 0:
|
||||
logger.info(f"OCR extracted {text_length} characters")
|
||||
return ocr_text.strip()
|
||||
else:
|
||||
logger.warning("OCR returned empty result")
|
||||
return ""
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"OCR recognition error: {str(e)}")
|
||||
return ""
|
||||
|
||||
class NanonetsOCRBackend(OCRBackend):
|
||||
"""Nanonets OCR backend implementation using OpenAI API format"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
"""Initialize Nanonets OCR backend
|
||||
|
||||
Args:
|
||||
api_key: API key for OpenAI API
|
||||
base_url: Base URL for OpenAI API
|
||||
model: Model name
|
||||
"""
|
||||
try:
|
||||
from openai import OpenAI
|
||||
self.api_key = kwargs.get("api_key", "123")
|
||||
self.base_url = kwargs.get("base_url", "http://localhost:8000/v1")
|
||||
self.model = kwargs.get("model", "nanonets/Nanonets-OCR-s")
|
||||
self.temperature = kwargs.get("temperature", 0.0)
|
||||
self.max_tokens = kwargs.get("max_tokens", 15000)
|
||||
|
||||
self.client = OpenAI(api_key=self.api_key, base_url=self.base_url)
|
||||
self.prompt = """
|
||||
## 任务说明
|
||||
|
||||
请从上传的文档中提取文字内容,严格按自然阅读顺序(从上到下,从左到右)输出,并遵循以下格式规范。
|
||||
|
||||
### 1. **文本处理**
|
||||
|
||||
* 按正常阅读顺序提取文字,语句流畅自然。
|
||||
|
||||
### 2. **表格**
|
||||
|
||||
* 所有表格统一转换为 **Markdown 表格格式**。
|
||||
* 内容保持清晰、对齐整齐,便于阅读。
|
||||
|
||||
### 3. **公式**
|
||||
|
||||
* 所有公式转换为 **LaTeX 格式**,使用 `$$公式$$` 包裹。
|
||||
|
||||
### 4. **图片**
|
||||
|
||||
* 忽略图片信息
|
||||
|
||||
### 5. **链接**
|
||||
|
||||
* 不要猜测或补全不确定的链接地址。
|
||||
"""
|
||||
logger.info(f"Nanonets OCR engine initialized with model: {self.model}")
|
||||
except ImportError:
|
||||
logger.error("Failed to import openai. Please install it with 'pip install openai'")
|
||||
self.client = None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize Nanonets OCR: {str(e)}")
|
||||
self.client = None
|
||||
|
||||
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
|
||||
"""Extract text from an image using Nanonets OCR
|
||||
|
||||
Args:
|
||||
image: Image file path, bytes, or PIL Image object
|
||||
|
||||
Returns:
|
||||
Extracted text
|
||||
"""
|
||||
if self.client is None:
|
||||
logger.error("Nanonets OCR client not initialized")
|
||||
return ""
|
||||
|
||||
try:
|
||||
# Encode image to base64
|
||||
img_base64 = image_to_base64(image)
|
||||
if not img_base64:
|
||||
return ""
|
||||
|
||||
# Call Nanonets OCR API
|
||||
logger.info(f"Calling Nanonets OCR API with model: {self.model}")
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/png;base64,{img_base64}"},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": self.prompt,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
temperature=self.temperature,
|
||||
max_tokens=self.max_tokens
|
||||
)
|
||||
|
||||
return response.choices[0].message.content
|
||||
except Exception as e:
|
||||
logger.error(f"Nanonets OCR prediction error: {str(e)}")
|
||||
return ""
|
||||
|
||||
class OCREngine:
|
||||
"""OCR Engine factory class"""
|
||||
|
||||
_instance = None
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls, backend_type="paddle", **kwargs) -> Optional[OCRBackend]:
|
||||
"""Get OCR engine instance
|
||||
|
||||
Args:
|
||||
backend_type: OCR backend type, one of: "paddle", "nanonets"
|
||||
**kwargs: Additional arguments for the backend
|
||||
|
||||
Returns:
|
||||
OCR engine instance or None if initialization fails
|
||||
"""
|
||||
if cls._instance is None:
|
||||
logger.info(f"Initializing OCR engine with backend: {backend_type}")
|
||||
|
||||
if backend_type.lower() == "paddle":
|
||||
cls._instance = PaddleOCRBackend(**kwargs)
|
||||
elif backend_type.lower() == "nanonets":
|
||||
cls._instance = NanonetsOCRBackend(**kwargs)
|
||||
else:
|
||||
logger.error(f"Unknown OCR backend type: {backend_type}")
|
||||
return None
|
||||
|
||||
return cls._instance
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Any, Optional, Type
|
||||
|
||||
from .base_parser import BaseParser, ParseResult
|
||||
from .docx_parser import DocxParser
|
||||
from .doc_parser import DocParser
|
||||
from .pdf_parser import PDFParser
|
||||
from .markdown_parser import MarkdownParser
|
||||
from .text_parser import TextParser
|
||||
from .image_parser import ImageParser
|
||||
from .web_parser import WebParser
|
||||
from .config import ChunkingConfig
|
||||
import traceback
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass
|
||||
class Chunk:
|
||||
"""
|
||||
Represents a single text chunk with associated metadata.
|
||||
Basic unit for document processing and embedding.
|
||||
"""
|
||||
|
||||
content: str # Text content of the chunk
|
||||
metadata: Dict[str, Any] = None # Associated metadata (source, page number, etc.)
|
||||
|
||||
|
||||
class Parser:
|
||||
"""
|
||||
Document parser facade that integrates all specialized parsers.
|
||||
Provides a unified interface for parsing various document types.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
logger.info("Initializing document parser")
|
||||
# Initialize all parser types
|
||||
self.parsers: Dict[str, Type[BaseParser]] = {
|
||||
"docx": DocxParser,
|
||||
"doc": DocParser,
|
||||
"pdf": PDFParser,
|
||||
"md": MarkdownParser,
|
||||
"txt": TextParser,
|
||||
"jpg": ImageParser,
|
||||
"jpeg": ImageParser,
|
||||
"png": ImageParser,
|
||||
"gif": ImageParser,
|
||||
"bmp": ImageParser,
|
||||
"tiff": ImageParser,
|
||||
"webp": ImageParser,
|
||||
"markdown": MarkdownParser,
|
||||
}
|
||||
logger.info(
|
||||
"Parser initialized with %d parsers: %s",
|
||||
len(self.parsers),
|
||||
", ".join(self.parsers.keys()),
|
||||
)
|
||||
|
||||
|
||||
def get_parser(self, file_type: str) -> Optional[Type[BaseParser]]:
|
||||
"""
|
||||
Get parser class for the specified file type.
|
||||
|
||||
Args:
|
||||
file_type: The file extension or type identifier
|
||||
|
||||
Returns:
|
||||
Parser class for the file type, or None if unsupported
|
||||
"""
|
||||
file_type = file_type.lower()
|
||||
parser = self.parsers.get(file_type)
|
||||
if parser:
|
||||
logger.info(f"Found parser for file type: {file_type}")
|
||||
else:
|
||||
logger.warning(f"No parser found for file type: {file_type}")
|
||||
return parser
|
||||
|
||||
def parse_file(
|
||||
self,
|
||||
file_name: str,
|
||||
file_type: str,
|
||||
content: bytes,
|
||||
config: ChunkingConfig,
|
||||
) -> Optional[ParseResult]:
|
||||
"""
|
||||
Parse file content using appropriate parser based on file type.
|
||||
|
||||
Args:
|
||||
file_name: Name of the file being parsed
|
||||
file_type: Type/extension of the file
|
||||
content: Raw file content as bytes
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing file: {file_name} with type: {file_type}")
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, "
|
||||
f"multimodal={config.enable_multimodal}"
|
||||
)
|
||||
|
||||
parser_instance = None
|
||||
|
||||
try:
|
||||
# Get appropriate parser for file type
|
||||
cls = self.get_parser(file_type)
|
||||
if cls is None:
|
||||
logger.error(f"Unsupported file type: {file_type}")
|
||||
return None
|
||||
|
||||
# Parse file content
|
||||
logger.info(f"Creating parser instance for {file_type} file")
|
||||
parser_instance = cls(
|
||||
file_name=file_name,
|
||||
file_type=file_type,
|
||||
chunk_size=config.chunk_size,
|
||||
chunk_overlap=config.chunk_overlap,
|
||||
separators=config.separators,
|
||||
enable_multimodal=config.enable_multimodal,
|
||||
max_image_size=1920, # Limit image size to 1920px
|
||||
max_concurrent_tasks=5, # Limit concurrent tasks to 5
|
||||
chunking_config=config, # Pass the entire chunking config
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
|
||||
result = parser_instance.parse(content)
|
||||
|
||||
if result:
|
||||
logger.info(
|
||||
f"Successfully parsed file {file_name}, generated {len(result.chunks)} chunks"
|
||||
)
|
||||
if result.chunks and len(result.chunks) > 0:
|
||||
logger.info(
|
||||
f"First chunk content length: {len(result.chunks[0].content)}"
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Parser returned empty chunks for file: {file_name}")
|
||||
else:
|
||||
logger.warning(f"Parser returned None result for file: {file_name}")
|
||||
|
||||
# Return parse results
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing file {file_name}: {str(e)}")
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
return None
|
||||
|
||||
def parse_url(
|
||||
self, url: str, title: str, config: ChunkingConfig
|
||||
) -> Optional[ParseResult]:
|
||||
"""
|
||||
Parse content from a URL using the WebParser.
|
||||
|
||||
Args:
|
||||
url: URL to parse
|
||||
title: Title of the webpage (for metadata)
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing URL: {url}, title: {title}")
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, "
|
||||
f"multimodal={config.enable_multimodal}"
|
||||
)
|
||||
|
||||
parser_instance = None
|
||||
|
||||
try:
|
||||
# Create web parser instance
|
||||
logger.info("Creating WebParser instance")
|
||||
parser_instance = WebParser(
|
||||
title=title,
|
||||
chunk_size=config.chunk_size,
|
||||
chunk_overlap=config.chunk_overlap,
|
||||
separators=config.separators,
|
||||
enable_multimodal=config.enable_multimodal,
|
||||
max_image_size=1920, # Limit image size
|
||||
max_concurrent_tasks=5, # Limit concurrent tasks
|
||||
chunking_config=config,
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse URL content")
|
||||
result = parser_instance.parse(url)
|
||||
|
||||
if result:
|
||||
logger.info(
|
||||
f"Successfully parsed URL, generated {len(result.chunks)} chunks"
|
||||
)
|
||||
logger.info(
|
||||
f"First chunk content length: {len(result.chunks[0].content) if result.chunks else 0}"
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Parser returned empty result for URL: {url}")
|
||||
|
||||
# Return parse results
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing URL {url}: {str(e)}")
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import logging
|
||||
import os
|
||||
import io
|
||||
from typing import Any, List, Iterator, Optional, Mapping, Tuple, Dict, Union
|
||||
|
||||
import pdfplumber
|
||||
import tempfile
|
||||
from .base_parser import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class PDFParser(BaseParser):
|
||||
"""
|
||||
PDF Document Parser
|
||||
|
||||
This parser handles PDF documents by extracting text content.
|
||||
It uses the pypdf library for simple text extraction.
|
||||
"""
|
||||
def _convert_table_to_markdown(self, table_data: list) -> str:
|
||||
|
||||
if not table_data or not table_data[0]: return ""
|
||||
def clean_cell(cell):
|
||||
if cell is None: return ""
|
||||
return str(cell).replace("\n", " <br> ")
|
||||
try:
|
||||
markdown = ""
|
||||
header = [clean_cell(cell) for cell in table_data[0]]
|
||||
markdown += "| " + " | ".join(header) + " |\n"
|
||||
markdown += "| " + " | ".join(["---"] * len(header)) + " |\n"
|
||||
for row in table_data[1:]:
|
||||
if not row: continue
|
||||
body_row = [clean_cell(cell) for cell in row]
|
||||
if len(body_row) != len(header):
|
||||
logger.warning(f"Skipping malformed table row: {body_row}")
|
||||
continue
|
||||
markdown += "| " + " | ".join(body_row) + " |\n"
|
||||
return markdown
|
||||
except Exception as e:
|
||||
logger.error(f"Error converting table to markdown: {e}")
|
||||
return ""
|
||||
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
|
||||
|
||||
logger.info(f"Parsing PDF with pdfplumber, content size: {len(content)} bytes")
|
||||
|
||||
all_page_content = []
|
||||
|
||||
|
||||
temp_pdf = tempfile.NamedTemporaryFile(delete=False, suffix=".pdf")
|
||||
temp_pdf_path = temp_pdf.name
|
||||
|
||||
try:
|
||||
temp_pdf.write(content)
|
||||
temp_pdf.close()
|
||||
logger.info(f"PDF content written to temporary file: {temp_pdf_path}")
|
||||
|
||||
with pdfplumber.open(temp_pdf_path) as pdf:
|
||||
logger.info(f"PDF has {len(pdf.pages)} pages")
|
||||
|
||||
for page_num, page in enumerate(pdf.pages):
|
||||
page_content_parts = []
|
||||
|
||||
# Try-fallback strategy for table detection
|
||||
default_settings = { "vertical_strategy": "lines", "horizontal_strategy": "lines" }
|
||||
found_tables = page.find_tables(default_settings)
|
||||
if not found_tables:
|
||||
logger.info(f"Page {page_num+1}: Default strategy found no tables. Trying fallback strategy.")
|
||||
fallback_settings = { "vertical_strategy": "text", "horizontal_strategy": "lines" }
|
||||
found_tables = page.find_tables(fallback_settings)
|
||||
|
||||
table_bboxes = [table.bbox for table in found_tables]
|
||||
# Define a filter function that keeps objects NOT inside any table bbox.
|
||||
def not_within_bboxes(obj):
|
||||
"""Check if an object is outside all table bounding boxes."""
|
||||
for bbox in table_bboxes:
|
||||
# Check if the object's vertical center is within a bbox
|
||||
if bbox[1] <= (obj["top"] + obj["bottom"]) / 2 <= bbox[3]:
|
||||
return False # It's inside a table, so we DON'T keep it
|
||||
return True # It's outside all tables, so we DO keep it
|
||||
|
||||
# that contains only the non-table text.
|
||||
non_table_page = page.filter(not_within_bboxes)
|
||||
|
||||
# Now, extract text from this filtered page view.
|
||||
text = non_table_page.extract_text(x_tolerance=2)
|
||||
if text:
|
||||
page_content_parts.append(text)
|
||||
|
||||
# Process and append the structured Markdown tables
|
||||
if found_tables:
|
||||
logger.info(f"Found {len(found_tables)} tables on page {page_num + 1}")
|
||||
for table in found_tables:
|
||||
markdown_table = self._convert_table_to_markdown(table.extract())
|
||||
page_content_parts.append(f"\n\n{markdown_table}\n\n")
|
||||
|
||||
|
||||
all_page_content.append("".join(page_content_parts))
|
||||
|
||||
final_text = "\n\n--- Page Break ---\n\n".join(all_page_content)
|
||||
logger.info(f"PDF parsing complete. Extracted {len(final_text)} text chars.")
|
||||
|
||||
return final_text
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse PDF document: {str(e)}")
|
||||
return ""
|
||||
finally:
|
||||
# This block is GUARANTEED to execute, preventing resource leaks.
|
||||
if os.path.exists(temp_pdf_path):
|
||||
try:
|
||||
os.remove(temp_pdf_path)
|
||||
logger.info(f"Temporary file cleaned up: {temp_pdf_path}")
|
||||
except OSError as e:
|
||||
logger.error(f"Error removing temporary file {temp_pdf_path}: {e}")
|
||||
@@ -0,0 +1,360 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import uuid
|
||||
import logging
|
||||
import io
|
||||
import traceback
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Tuple, Optional
|
||||
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
from minio import Minio
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
|
||||
class Storage(ABC):
|
||||
"""Abstract base class for object storage operations"""
|
||||
|
||||
@abstractmethod
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to object storage
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to object storage
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class CosStorage(Storage):
|
||||
"""Tencent Cloud COS storage implementation"""
|
||||
|
||||
def __init__(self, storage_config=None):
|
||||
"""Initialize COS storage
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration
|
||||
"""
|
||||
self.storage_config = storage_config
|
||||
self.client, self.bucket_name, self.region, self.prefix = self._init_cos_client()
|
||||
|
||||
def _init_cos_client(self):
|
||||
"""Initialize Tencent Cloud COS client"""
|
||||
try:
|
||||
# Use provided COS config if available, otherwise fall back to environment variables
|
||||
if self.storage_config and self.storage_config.get("access_key_id") != "":
|
||||
cos_config = self.storage_config
|
||||
secret_id = cos_config.get("access_key_id")
|
||||
secret_key = cos_config.get("secret_access_key")
|
||||
region = cos_config.get("region")
|
||||
bucket_name = cos_config.get("bucket_name")
|
||||
appid = cos_config.get("app_id")
|
||||
prefix = cos_config.get("path_prefix", "")
|
||||
else:
|
||||
# Get COS configuration from environment variables
|
||||
secret_id = os.getenv("COS_SECRET_ID")
|
||||
secret_key = os.getenv("COS_SECRET_KEY")
|
||||
region = os.getenv("COS_REGION")
|
||||
bucket_name = os.getenv("COS_BUCKET_NAME")
|
||||
appid = os.getenv("COS_APP_ID")
|
||||
prefix = os.getenv("COS_PATH_PREFIX")
|
||||
|
||||
enable_old_domain = (
|
||||
os.getenv("COS_ENABLE_OLD_DOMAIN", "true").lower() == "true"
|
||||
)
|
||||
|
||||
if not all([secret_id, secret_key, region, bucket_name, appid]):
|
||||
logger.error(
|
||||
"Incomplete COS configuration, missing required environment variables"
|
||||
f"secret_id: {secret_id}, secret_key: {secret_key}, region: {region}, bucket_name: {bucket_name}, appid: {appid}"
|
||||
)
|
||||
return None, None, None, None
|
||||
|
||||
# Initialize COS configuration
|
||||
logger.info(
|
||||
f"Initializing COS client with region: {region}, bucket: {bucket_name}"
|
||||
)
|
||||
config = CosConfig(
|
||||
Appid=appid,
|
||||
Region=region,
|
||||
SecretId=secret_id,
|
||||
SecretKey=secret_key,
|
||||
EnableOldDomain=enable_old_domain,
|
||||
)
|
||||
|
||||
# Create client
|
||||
client = CosS3Client(config)
|
||||
return client, bucket_name, region, prefix
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize COS client: {str(e)}")
|
||||
return None, None, None, None
|
||||
|
||||
def _get_download_url(self, bucket_name, region, object_key):
|
||||
"""Generate COS object URL
|
||||
|
||||
Args:
|
||||
bucket_name: Bucket name
|
||||
region: Region
|
||||
object_key: Object key
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}"
|
||||
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to Tencent Cloud COS
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
logger.info(f"Uploading file to COS: {file_path}")
|
||||
try:
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
# Generate object key, use UUID to avoid conflicts
|
||||
file_name = os.path.basename(file_path)
|
||||
object_key = (
|
||||
f"{self.prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
|
||||
)
|
||||
logger.info(f"Generated object key: {object_key}")
|
||||
|
||||
# Upload file
|
||||
logger.info("Attempting to upload file to COS")
|
||||
response = self.client.upload_file(
|
||||
Bucket=self.bucket_name, LocalFilePath=file_path, Key=object_key
|
||||
)
|
||||
|
||||
# Get file URL
|
||||
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
|
||||
|
||||
logger.info(f"Successfully uploaded file to COS: {file_url}")
|
||||
return file_url
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload file to COS: {str(e)}")
|
||||
return ""
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to Tencent Cloud COS
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Uploading bytes content to COS, size: {len(content)} bytes")
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}" if self.prefix else f"images/{uuid.uuid4().hex}{file_ext}"
|
||||
logger.info(f"Generated object key: {object_key}")
|
||||
self.client.put_object(Bucket=self.bucket_name, Body=content, Key=object_key)
|
||||
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
|
||||
logger.info(f"Successfully uploaded bytes to COS: {file_url}")
|
||||
return file_url
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload bytes to COS: {str(e)}")
|
||||
traceback.print_exc()
|
||||
return ""
|
||||
|
||||
|
||||
class MinioStorage(Storage):
|
||||
"""MinIO storage implementation"""
|
||||
|
||||
def __init__(self, storage_config=None):
|
||||
"""Initialize MinIO storage
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration
|
||||
"""
|
||||
self.storage_config = storage_config
|
||||
self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = self._init_minio_client()
|
||||
|
||||
def _init_minio_client(self):
|
||||
"""Initialize MinIO client from environment variables or injected config.
|
||||
|
||||
If storage_config.path_prefix contains JSON from server (for minio case),
|
||||
prefer those values to override envs.
|
||||
"""
|
||||
try:
|
||||
endpoint = os.getenv("MINIO_ENDPOINT")
|
||||
use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true"
|
||||
if self.storage_config and self.storage_config.get("bucket_name"):
|
||||
storage_config = self.storage_config
|
||||
bucket_name = storage_config.get("bucket_name")
|
||||
path_prefix = storage_config.get("path_prefix").strip().strip("/")
|
||||
access_key = storage_config.get("access_key_id")
|
||||
secret_key = storage_config.get("secret_access_key")
|
||||
else:
|
||||
access_key = os.getenv("MINIO_ACCESS_KEY_ID")
|
||||
secret_key = os.getenv("MINIO_SECRET_ACCESS_KEY")
|
||||
bucket_name = os.getenv("MINIO_BUCKET_NAME")
|
||||
path_prefix = os.getenv("MINIO_PATH_PREFIX", "").strip().strip("/")
|
||||
|
||||
if not all([endpoint, access_key, secret_key, bucket_name]):
|
||||
logger.error("Incomplete MinIO configuration, missing required environment variables")
|
||||
return None, None, None, None, None
|
||||
|
||||
# Initialize client
|
||||
client = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl)
|
||||
|
||||
# Ensure bucket exists
|
||||
found = client.bucket_exists(bucket_name)
|
||||
if not found:
|
||||
client.make_bucket(bucket_name)
|
||||
policy = '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}' % (bucket_name, bucket_name)
|
||||
client.set_bucket_policy(bucket_name, policy)
|
||||
|
||||
return client, bucket_name, use_ssl, endpoint, path_prefix
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize MinIO client: {str(e)}")
|
||||
return None, None, None, None, None
|
||||
|
||||
def _get_download_url(self, bucket_name: str, object_key: str, use_ssl: bool, endpoint: str, public_endpoint: str = None):
|
||||
"""Construct a public URL for MinIO object.
|
||||
|
||||
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
|
||||
"""
|
||||
if public_endpoint:
|
||||
base = public_endpoint
|
||||
else:
|
||||
scheme = "https" if use_ssl else "http"
|
||||
base = f"{scheme}://{endpoint}"
|
||||
# Path-style URL for MinIO
|
||||
return f"{base}/{bucket_name}/{object_key}"
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to MinIO
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
logger.info(f"Uploading file to MinIO: {file_path}")
|
||||
try:
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
# Generate object key, use UUID to avoid conflicts
|
||||
file_name = os.path.basename(file_path)
|
||||
object_key = f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}" if self.path_prefix else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
|
||||
logger.info(f"Generated MinIO object key: {object_key}")
|
||||
|
||||
# Upload file
|
||||
logger.info("Attempting to upload file to MinIO")
|
||||
with open(file_path, 'rb') as file_data:
|
||||
file_size = os.path.getsize(file_path)
|
||||
self.client.put_object(
|
||||
bucket_name=self.bucket_name,
|
||||
object_name=object_key,
|
||||
data=file_data,
|
||||
length=file_size,
|
||||
content_type='application/octet-stream'
|
||||
)
|
||||
|
||||
# Get file URL
|
||||
file_url = self._get_download_url(
|
||||
self.bucket_name,
|
||||
object_key,
|
||||
self.use_ssl,
|
||||
self.endpoint,
|
||||
os.getenv("MINIO_PUBLIC_ENDPOINT", None)
|
||||
)
|
||||
|
||||
logger.info(f"Successfully uploaded file to MinIO: {file_url}")
|
||||
return file_url
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload file to MinIO: {str(e)}")
|
||||
return ""
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to MinIO
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Uploading bytes content to MinIO, size: {len(content)} bytes")
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
object_key = f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}" if self.path_prefix else f"images/{uuid.uuid4().hex}{file_ext}"
|
||||
logger.info(f"Generated MinIO object key: {object_key}")
|
||||
self.client.put_object(
|
||||
self.bucket_name,
|
||||
object_key,
|
||||
data=io.BytesIO(content),
|
||||
length=len(content),
|
||||
content_type="application/octet-stream"
|
||||
)
|
||||
file_url = self._get_download_url(
|
||||
self.bucket_name,
|
||||
object_key,
|
||||
self.use_ssl,
|
||||
self.endpoint,
|
||||
os.getenv("MINIO_PUBLIC_ENDPOINT", None)
|
||||
)
|
||||
logger.info(f"Successfully uploaded bytes to MinIO: {file_url}")
|
||||
return file_url
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload bytes to MinIO: {str(e)}")
|
||||
traceback.print_exc()
|
||||
return ""
|
||||
|
||||
|
||||
def create_storage(storage_config=None) -> Storage:
|
||||
"""Create a storage instance based on configuration or environment variables
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration dictionary
|
||||
|
||||
Returns:
|
||||
Storage instance
|
||||
"""
|
||||
storage_type = os.getenv("STORAGE_TYPE", "cos").lower()
|
||||
|
||||
if storage_config:
|
||||
storage_type = str(storage_config.get("provider", storage_type)).lower()
|
||||
|
||||
logger.info(f"Creating {storage_type} storage instance")
|
||||
|
||||
if storage_type == "minio":
|
||||
return MinioStorage(storage_config)
|
||||
elif storage_type == "cos":
|
||||
# Default to COS
|
||||
return CosStorage(storage_config)
|
||||
else:
|
||||
return None
|
||||
@@ -0,0 +1,58 @@
|
||||
import logging
|
||||
from .base_parser import BaseParser
|
||||
from typing import Dict, Any, Tuple, Union
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TextParser(BaseParser):
|
||||
"""
|
||||
Text document parser for processing plain text files.
|
||||
This parser handles text extraction and chunking from plain text documents.
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
|
||||
"""
|
||||
Parse text document content by decoding bytes to string.
|
||||
|
||||
This is a straightforward parser that simply converts the binary content
|
||||
to text using appropriate character encoding.
|
||||
|
||||
Args:
|
||||
content: Raw document content as bytes
|
||||
|
||||
Returns:
|
||||
Parsed text content as string
|
||||
"""
|
||||
logger.info(f"Parsing text document, content size: {len(content)} bytes")
|
||||
text = self.decode_bytes(content)
|
||||
logger.info(
|
||||
f"Successfully parsed text document, extracted {len(text)} characters"
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
logger.info("Running TextParser in standalone mode")
|
||||
|
||||
# Sample text for testing
|
||||
text = """## 标题1
|
||||

|
||||
## 标题2
|
||||

|
||||
## 标题3
|
||||
"""
|
||||
logger.info(f"Test text content: {text}")
|
||||
|
||||
# Define separators for text splitting
|
||||
seperators = ["\n\n", "\n", "。"]
|
||||
parser = TextParser(separators=seperators)
|
||||
logger.info("Splitting text into units")
|
||||
units = parser._split_into_units(text)
|
||||
logger.info(f"Split text into {len(units)} units")
|
||||
logger.info(f"Units: {units}")
|
||||
@@ -0,0 +1,130 @@
|
||||
from typing import Any, Optional, Tuple, Dict, Union
|
||||
import os
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
from bs4 import BeautifulSoup
|
||||
from .base_parser import BaseParser, ParseResult
|
||||
import logging
|
||||
import asyncio
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WebParser(BaseParser):
|
||||
"""Web page parser"""
|
||||
|
||||
def __init__(self, title: str, **kwargs):
|
||||
self.title = title
|
||||
self.proxy = os.environ.get("WEB_PROXY", "")
|
||||
super().__init__(file_name=title, **kwargs)
|
||||
logger.info(f"Initialized WebParser with title: {title}")
|
||||
|
||||
async def scrape(self, url: str) -> Any:
|
||||
logger.info(f"Starting web page scraping for URL: {url}")
|
||||
try:
|
||||
async with async_playwright() as p:
|
||||
kwargs = {}
|
||||
if self.proxy:
|
||||
kwargs["proxy"] = {"server": self.proxy}
|
||||
logger.info("Launching WebKit browser")
|
||||
browser = await p.webkit.launch(**kwargs)
|
||||
page = await browser.new_page()
|
||||
|
||||
logger.info(f"Navigating to URL: {url}")
|
||||
try:
|
||||
await page.goto(url, timeout=30000)
|
||||
logger.info("Initial page load complete")
|
||||
except Exception as e:
|
||||
logger.error(f"Error navigating to URL: {str(e)}")
|
||||
await browser.close()
|
||||
return BeautifulSoup(
|
||||
"", "html.parser"
|
||||
) # Return empty soup on navigation error
|
||||
|
||||
logger.info("Retrieving page HTML content")
|
||||
content = await page.content()
|
||||
logger.info(f"Retrieved {len(content)} bytes of HTML content")
|
||||
|
||||
await browser.close()
|
||||
logger.info("Browser closed")
|
||||
|
||||
# Parse HTML content with BeautifulSoup
|
||||
logger.info("Parsing HTML with BeautifulSoup")
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
logger.info("Successfully parsed HTML content")
|
||||
return soup
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to scrape web page: {str(e)}")
|
||||
# Return empty BeautifulSoup object on error
|
||||
return BeautifulSoup("", "html.parser")
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
|
||||
"""Parse web page
|
||||
|
||||
Args:
|
||||
content: Web page content
|
||||
|
||||
Returns:
|
||||
Parse result
|
||||
"""
|
||||
logger.info("Starting web page parsing")
|
||||
|
||||
# Call async method synchronously
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
try:
|
||||
# Run async method
|
||||
# Handle content possibly being a string
|
||||
if isinstance(content, bytes):
|
||||
url = self.decode_bytes(content)
|
||||
logger.info(f"Decoded URL from bytes: {url}")
|
||||
else:
|
||||
url = content
|
||||
logger.info(f"Using content as URL directly: {url}")
|
||||
|
||||
logger.info(f"Scraping web page: {url}")
|
||||
soup = loop.run_until_complete(self.scrape(url))
|
||||
|
||||
# Extract page text
|
||||
logger.info("Extracting text from web page")
|
||||
text = soup.get_text("\n")
|
||||
logger.info(f"Extracted {len(text)} characters of text from URL: {url}")
|
||||
|
||||
# Get title, usually in <title> or <h1> tag
|
||||
if self.title != "":
|
||||
title = self.title
|
||||
logger.info(f"Using provided title: {title}")
|
||||
else:
|
||||
title = soup.title.string if soup.title else None
|
||||
logger.info(f"Found title tag: {title}")
|
||||
|
||||
if not title: # If <title> tag does not exist or is empty, try <h1> tag
|
||||
h1_tag = soup.find("h1")
|
||||
if h1_tag:
|
||||
title = h1_tag.get_text()
|
||||
logger.info(f"Using h1 tag as title: {title}")
|
||||
else:
|
||||
title = "Untitled Web Page"
|
||||
logger.info("No title found, using default")
|
||||
|
||||
logger.info(f"Web page title: {title}")
|
||||
text = "\n".join(
|
||||
(line.strip() for line in text.splitlines() if line.strip())
|
||||
)
|
||||
|
||||
result = title + "\n\n" + text
|
||||
logger.info(
|
||||
f"Web page parsing complete, total content: {len(result)} characters"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing web page: {str(e)}")
|
||||
return f"Error parsing web page: {str(e)}"
|
||||
|
||||
finally:
|
||||
# Close event loop
|
||||
logger.info("Closing event loop")
|
||||
loop.close()
|
||||
Reference in New Issue
Block a user