feat: 新增文档模型类，调整配置与解析逻辑，优化日志及导入
移除日志设置与冗余代码，优化导入、类型提示及OCR后端管理 统一调整各文件模块导入路径为绝对导入 调整导入路径，移除部分导入，优化日志及注释 升级文档解析器为 Docx2Parser，优化超时与图片处理逻辑
This commit is contained in:
begoniezhao
+141
-209
@@ -1,134 +1,88 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import tempfile
|
||||
import os
|
||||
import subprocess
|
||||
import shutil
|
||||
from io import BytesIO
|
||||
from typing import Optional, List, Tuple
|
||||
import textract
|
||||
from PIL import Image
|
||||
import zipfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import List, Optional
|
||||
|
||||
from .base_parser import BaseParser
|
||||
from .docx_parser import DocxParser, Docx
|
||||
import textract
|
||||
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.docx2_parser import Docx2Parser
|
||||
from docreader.utils.tempfile import TempDirContext, TempFileContext
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DocParser(BaseParser):
|
||||
class DocParser(Docx2Parser):
|
||||
"""DOC document parser"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> str:
|
||||
"""Parse DOC document
|
||||
|
||||
Args:
|
||||
content: DOC document content
|
||||
|
||||
Returns:
|
||||
Parse result
|
||||
"""
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
logger.info(f"Parsing DOC document, content size: {len(content)} bytes")
|
||||
|
||||
handle_chain = [
|
||||
# 1. Try to convert to docx format to extract images
|
||||
self._parse_with_docx,
|
||||
# 2. If image extraction is not needed or conversion failed,
|
||||
# try using antiword to extract text
|
||||
self._parse_with_antiword,
|
||||
# 3. If antiword extraction fails, use textract
|
||||
self._parse_with_textract,
|
||||
]
|
||||
|
||||
# Save byte content as a temporary file
|
||||
with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as temp_file:
|
||||
temp_file_path = temp_file.name
|
||||
temp_file.write(content)
|
||||
temp_file.flush()
|
||||
logger.info(f"Saved DOC content to temporary file: {temp_file_path}")
|
||||
with TempFileContext(content, ".doc") as temp_file_path:
|
||||
for handle in handle_chain:
|
||||
try:
|
||||
document = handle(temp_file_path)
|
||||
if document:
|
||||
return document
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to parse DOC with {handle.__name__} {e}")
|
||||
|
||||
try:
|
||||
# First try to convert to docx format to extract images
|
||||
if self.enable_multimodal:
|
||||
logger.info("Multimodal enabled, attempting to extract images from DOC")
|
||||
docx_content = self._convert_doc_to_docx(temp_file_path)
|
||||
return Document(content="")
|
||||
|
||||
if docx_content:
|
||||
logger.info("Successfully converted DOC to DOCX, using DocxParser")
|
||||
# Use existing DocxParser to parse the converted docx
|
||||
docx_parser = DocxParser(
|
||||
file_name=self.file_name,
|
||||
file_type="docx",
|
||||
enable_multimodal=self.enable_multimodal,
|
||||
chunk_size=self.chunk_size,
|
||||
chunk_overlap=self.chunk_overlap,
|
||||
chunking_config=self.chunking_config,
|
||||
separators=self.separators,
|
||||
)
|
||||
text = docx_parser.parse_into_text(docx_content)
|
||||
logger.info(f"Extracted {len(text)} characters using DocxParser")
|
||||
def _parse_with_docx(self, temp_file_path: str) -> Document:
|
||||
logger.info("Multimodal enabled, attempting to extract images from DOC")
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file: {temp_file_path}")
|
||||
docx_content = self._try_convert_doc_to_docx(temp_file_path)
|
||||
if not docx_content:
|
||||
raise RuntimeError("Failed to convert DOC to DOCX")
|
||||
|
||||
return text
|
||||
else:
|
||||
logger.warning(
|
||||
"Failed to convert DOC to DOCX, falling back to text-only extraction"
|
||||
)
|
||||
logger.info("Successfully converted DOC to DOCX, using DocxParser")
|
||||
# Use existing DocxParser to parse the converted docx
|
||||
document = super(Docx2Parser, self).parse_into_text(docx_content)
|
||||
logger.info(f"Extracted {len(document.content)} characters using DocxParser")
|
||||
return document
|
||||
|
||||
# If image extraction is not needed or conversion failed, try using antiword to extract text
|
||||
try:
|
||||
logger.info("Attempting to parse DOC file with antiword")
|
||||
# Check if antiword is installed
|
||||
antiword_path = self._find_antiword_path()
|
||||
def _parse_with_antiword(self, temp_file_path: str) -> Document:
|
||||
logger.info("Attempting to parse DOC file with antiword")
|
||||
|
||||
if antiword_path:
|
||||
# Use antiword to extract text directly
|
||||
logger.info(f"Using antiword at {antiword_path} to extract text")
|
||||
process = subprocess.Popen(
|
||||
[antiword_path, temp_file_path],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = process.communicate()
|
||||
# Check if antiword is installed
|
||||
antiword_path = self._try_find_antiword()
|
||||
if not antiword_path:
|
||||
raise RuntimeError("antiword not found in PATH")
|
||||
|
||||
if process.returncode == 0:
|
||||
text = stdout.decode("utf-8", errors="ignore")
|
||||
logger.info(
|
||||
f"Successfully extracted {len(text)} characters using antiword"
|
||||
)
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file: {temp_file_path}")
|
||||
|
||||
return text
|
||||
else:
|
||||
logger.warning(
|
||||
f"antiword extraction failed: {stderr.decode('utf-8', errors='ignore')}"
|
||||
)
|
||||
else:
|
||||
logger.warning("antiword not found, falling back to textract")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Error using antiword: {str(e)}, falling back to textract"
|
||||
)
|
||||
|
||||
# If antiword fails, try using textract
|
||||
logger.info("Parsing DOC file with textract")
|
||||
text = textract.process(temp_file_path, method="antiword").decode("utf-8")
|
||||
logger.info(
|
||||
f"Successfully extracted {len(text)} characters of text from DOC document using textract"
|
||||
# Use antiword to extract text directly
|
||||
process = subprocess.Popen(
|
||||
[antiword_path, temp_file_path],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = process.communicate()
|
||||
if process.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"antiword extraction failed: {stderr.decode('utf-8', errors='ignore')}"
|
||||
)
|
||||
text = stdout.decode("utf-8", errors="ignore")
|
||||
logger.info(f"Successfully extracted {len(text)} characters using antiword")
|
||||
return Document(content=text)
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file: {temp_file_path}")
|
||||
def _parse_with_textract(self, temp_file_path: str) -> Document:
|
||||
logger.info(f"Parsing DOC file with textract: {temp_file_path}")
|
||||
text = textract.process(temp_file_path, method="antiword").decode("utf-8")
|
||||
logger.info(f"Successfully extracted {len(text)} bytes of DOC using textract")
|
||||
return Document(content=str(text))
|
||||
|
||||
return text
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing DOC document: {str(e)}")
|
||||
# Ensure temporary file is cleaned up
|
||||
if os.path.exists(temp_file_path):
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file after error: {temp_file_path}")
|
||||
return ""
|
||||
|
||||
def _convert_doc_to_docx(self, doc_path: str) -> Optional[bytes]:
|
||||
def _try_convert_doc_to_docx(self, doc_path: str) -> Optional[bytes]:
|
||||
"""Convert DOC file to DOCX format
|
||||
|
||||
Uses LibreOffice/OpenOffice for conversion
|
||||
@@ -141,21 +95,16 @@ class DocParser(BaseParser):
|
||||
"""
|
||||
logger.info(f"Converting DOC to DOCX: {doc_path}")
|
||||
|
||||
# Check if LibreOffice or OpenOffice is installed
|
||||
soffice_path = self._try_find_soffice()
|
||||
if not soffice_path:
|
||||
return None
|
||||
|
||||
# Execute conversion command
|
||||
logger.info(f"Using {soffice_path} to convert DOC to DOCX")
|
||||
|
||||
# Create a temporary directory to store the converted file
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
docx_path = os.path.join(temp_dir, "converted.docx")
|
||||
|
||||
try:
|
||||
# Check if LibreOffice or OpenOffice is installed
|
||||
soffice_path = self._find_soffice_path()
|
||||
if not soffice_path:
|
||||
logger.error(
|
||||
"LibreOffice/OpenOffice not found, cannot convert DOC to DOCX"
|
||||
)
|
||||
return None
|
||||
|
||||
# Execute conversion command
|
||||
logger.info(f"Using {soffice_path} to convert DOC to DOCX")
|
||||
with TempDirContext() as temp_dir:
|
||||
cmd = [
|
||||
soffice_path,
|
||||
"--headless",
|
||||
@@ -165,7 +114,6 @@ class DocParser(BaseParser):
|
||||
temp_dir,
|
||||
doc_path,
|
||||
]
|
||||
|
||||
logger.info(f"Running command: {' '.join(cmd)}")
|
||||
process = subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
||||
@@ -173,41 +121,68 @@ class DocParser(BaseParser):
|
||||
stdout, stderr = process.communicate()
|
||||
|
||||
if process.returncode != 0:
|
||||
logger.error(
|
||||
f"Error converting DOC to DOCX: {stderr.decode('utf-8', errors='ignore')}"
|
||||
logger.warning(
|
||||
f"Error converting DOC to DOCX: {stderr.decode('utf-8')}"
|
||||
)
|
||||
return None
|
||||
|
||||
# Find the converted file
|
||||
for file in os.listdir(temp_dir):
|
||||
if file.endswith(".docx"):
|
||||
converted_file = os.path.join(temp_dir, file)
|
||||
logger.info(f"Found converted file: {converted_file}")
|
||||
|
||||
# Read the converted file content
|
||||
with open(converted_file, "rb") as f:
|
||||
docx_content = f.read()
|
||||
docx_file = [
|
||||
file for file in os.listdir(temp_dir) if file.endswith(".docx")
|
||||
]
|
||||
logger.info(f"Found {len(docx_file)} DOCX file(s) in temporary directory")
|
||||
for file in docx_file:
|
||||
converted_file = os.path.join(temp_dir, file)
|
||||
logger.info(f"Found converted file: {converted_file}")
|
||||
|
||||
# Read the converted file content
|
||||
with open(converted_file, "rb") as f:
|
||||
docx_content = f.read()
|
||||
logger.info(
|
||||
f"Successfully read converted DOCX file, size: {len(docx_content)} bytes"
|
||||
f"Successfully read DOCX file, size: {len(docx_content)}"
|
||||
)
|
||||
return docx_content
|
||||
return None
|
||||
|
||||
logger.error("No DOCX file found after conversion")
|
||||
return None
|
||||
def _try_find_executable_path(
|
||||
self,
|
||||
executable_name: str,
|
||||
possible_path: List[str] = [],
|
||||
environment_variable: List[str] = [],
|
||||
) -> Optional[str]:
|
||||
"""Find executable path
|
||||
Args:
|
||||
executable_name: Executable name
|
||||
possible_path: List of possible paths
|
||||
environment_variable: List of environment variables to check
|
||||
Returns:
|
||||
Executable path, or None if not found
|
||||
"""
|
||||
# Common executable paths
|
||||
paths: List[str] = []
|
||||
paths.extend(possible_path)
|
||||
paths.extend(os.environ.get(env_var, "") for env_var in environment_variable)
|
||||
paths = list(set(paths))
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during DOC to DOCX conversion: {str(e)}")
|
||||
return None
|
||||
finally:
|
||||
# Clean up temporary directory
|
||||
try:
|
||||
shutil.rmtree(temp_dir)
|
||||
logger.info(f"Cleaned up temporary directory: {temp_dir}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up temporary directory: {str(e)}")
|
||||
# Check if path is set in environment variable
|
||||
for path in paths:
|
||||
if os.path.exists(path):
|
||||
logger.info(f"Found {executable_name} at {path}")
|
||||
return path
|
||||
|
||||
def _find_soffice_path(self) -> Optional[str]:
|
||||
# Try to find in PATH
|
||||
result = subprocess.run(
|
||||
["which", executable_name], capture_output=True, text=True
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
path = result.stdout.strip()
|
||||
logger.info(f"Found {executable_name} at {path}")
|
||||
return path
|
||||
|
||||
logger.warning(f"Failed to find {executable_name}")
|
||||
return None
|
||||
|
||||
def _try_find_soffice(self) -> Optional[str]:
|
||||
"""Find LibreOffice/OpenOffice executable path
|
||||
|
||||
Returns:
|
||||
@@ -225,32 +200,13 @@ class DocParser(BaseParser):
|
||||
"C:\\Program Files\\LibreOffice\\program\\soffice.exe",
|
||||
"C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe",
|
||||
]
|
||||
return self._try_find_executable_path(
|
||||
executable_name="soffice",
|
||||
possible_path=possible_paths,
|
||||
environment_variable=["LIBREOFFICE_PATH"],
|
||||
)
|
||||
|
||||
# Check if path is set in environment variable
|
||||
if os.environ.get("LIBREOFFICE_PATH"):
|
||||
possible_paths.insert(0, os.environ.get("LIBREOFFICE_PATH"))
|
||||
|
||||
for path in possible_paths:
|
||||
if os.path.exists(path):
|
||||
logger.info(f"Found LibreOffice/OpenOffice at: {path}")
|
||||
return path
|
||||
|
||||
# Try to find in PATH
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["which", "soffice"], capture_output=True, text=True
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
path = result.stdout.strip()
|
||||
logger.info(f"Found LibreOffice/OpenOffice in PATH: {path}")
|
||||
return path
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.warning("LibreOffice/OpenOffice not found")
|
||||
return None
|
||||
|
||||
def _find_antiword_path(self) -> Optional[str]:
|
||||
def _try_find_antiword(self) -> Optional[str]:
|
||||
"""Find antiword executable path
|
||||
|
||||
Returns:
|
||||
@@ -265,51 +221,27 @@ class DocParser(BaseParser):
|
||||
"C:\\Program Files\\Antiword\\antiword.exe",
|
||||
"C:\\Program Files (x86)\\Antiword\\antiword.exe",
|
||||
]
|
||||
|
||||
# Check if path is set in environment variable
|
||||
if os.environ.get("ANTIWORD_PATH"):
|
||||
possible_paths.insert(0, os.environ.get("ANTIWORD_PATH"))
|
||||
|
||||
for path in possible_paths:
|
||||
if os.path.exists(path):
|
||||
logger.info(f"Found antiword at: {path}")
|
||||
return path
|
||||
|
||||
# Try to find in PATH
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["which", "antiword"], capture_output=True, text=True
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
path = result.stdout.strip()
|
||||
logger.info(f"Found antiword in PATH: {path}")
|
||||
return path
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.warning("antiword not found")
|
||||
return None
|
||||
return self._try_find_executable_path(
|
||||
executable_name="antiword",
|
||||
possible_path=possible_paths,
|
||||
environment_variable=["ANTIWORD_PATH"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
logger.info("Running DocParser in standalone mode")
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
file_name = "/path/to/your/test.doc"
|
||||
logger.info(f"Processing file: {file_name}")
|
||||
|
||||
doc_parser = DocParser(
|
||||
file_name, enable_multimodal=True, chunk_size=512, chunk_overlap=60
|
||||
file_name=file_name,
|
||||
enable_multimodal=True,
|
||||
chunk_size=512,
|
||||
chunk_overlap=60,
|
||||
)
|
||||
logger.info("Parser initialized, starting processing")
|
||||
|
||||
with open(file_name, "rb") as f:
|
||||
content = f.read()
|
||||
|
||||
text = doc_parser.parse_into_text(content)
|
||||
logger.info(f"Processing complete, extracted text length: {len(text)}")
|
||||
logger.info(f"Sample text: {text[:200]}...")
|
||||
document = doc_parser.parse_into_text(content)
|
||||
logger.info(f"Processing complete, extracted text length: {len(document.content)}")
|
||||
logger.info(f"Sample text: {document.content[:200]}...")
|
||||
|
||||
Reference in New Issue
Block a user