chore(docreader): 重新组织模块文件
This commit is contained in:
@@ -0,0 +1,206 @@
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Any, Optional, Type
|
||||
|
||||
from .base_parser import BaseParser, ParseResult
|
||||
from .docx_parser import DocxParser
|
||||
from .doc_parser import DocParser
|
||||
from .pdf_parser import PDFParser
|
||||
from .markdown_parser import MarkdownParser
|
||||
from .text_parser import TextParser
|
||||
from .image_parser import ImageParser
|
||||
from .web_parser import WebParser
|
||||
from .config import ChunkingConfig
|
||||
import traceback
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass
|
||||
class Chunk:
|
||||
"""
|
||||
Represents a single text chunk with associated metadata.
|
||||
Basic unit for document processing and embedding.
|
||||
"""
|
||||
|
||||
content: str # Text content of the chunk
|
||||
metadata: Dict[str, Any] = None # Associated metadata (source, page number, etc.)
|
||||
|
||||
|
||||
class Parser:
|
||||
"""
|
||||
Document parser facade that integrates all specialized parsers.
|
||||
Provides a unified interface for parsing various document types.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
logger.info("Initializing document parser")
|
||||
# Initialize all parser types
|
||||
self.parsers: Dict[str, Type[BaseParser]] = {
|
||||
"docx": DocxParser,
|
||||
"doc": DocParser,
|
||||
"pdf": PDFParser,
|
||||
"md": MarkdownParser,
|
||||
"txt": TextParser,
|
||||
"jpg": ImageParser,
|
||||
"jpeg": ImageParser,
|
||||
"png": ImageParser,
|
||||
"gif": ImageParser,
|
||||
"bmp": ImageParser,
|
||||
"tiff": ImageParser,
|
||||
"webp": ImageParser,
|
||||
"markdown": MarkdownParser,
|
||||
}
|
||||
logger.info(
|
||||
"Parser initialized with %d parsers: %s",
|
||||
len(self.parsers),
|
||||
", ".join(self.parsers.keys()),
|
||||
)
|
||||
|
||||
|
||||
def get_parser(self, file_type: str) -> Optional[Type[BaseParser]]:
|
||||
"""
|
||||
Get parser class for the specified file type.
|
||||
|
||||
Args:
|
||||
file_type: The file extension or type identifier
|
||||
|
||||
Returns:
|
||||
Parser class for the file type, or None if unsupported
|
||||
"""
|
||||
file_type = file_type.lower()
|
||||
parser = self.parsers.get(file_type)
|
||||
if parser:
|
||||
logger.info(f"Found parser for file type: {file_type}")
|
||||
else:
|
||||
logger.warning(f"No parser found for file type: {file_type}")
|
||||
return parser
|
||||
|
||||
def parse_file(
|
||||
self,
|
||||
file_name: str,
|
||||
file_type: str,
|
||||
content: bytes,
|
||||
config: ChunkingConfig,
|
||||
) -> Optional[ParseResult]:
|
||||
"""
|
||||
Parse file content using appropriate parser based on file type.
|
||||
|
||||
Args:
|
||||
file_name: Name of the file being parsed
|
||||
file_type: Type/extension of the file
|
||||
content: Raw file content as bytes
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing file: {file_name} with type: {file_type}")
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, "
|
||||
f"multimodal={config.enable_multimodal}"
|
||||
)
|
||||
|
||||
parser_instance = None
|
||||
|
||||
try:
|
||||
# Get appropriate parser for file type
|
||||
cls = self.get_parser(file_type)
|
||||
if cls is None:
|
||||
logger.error(f"Unsupported file type: {file_type}")
|
||||
return None
|
||||
|
||||
# Parse file content
|
||||
logger.info(f"Creating parser instance for {file_type} file")
|
||||
parser_instance = cls(
|
||||
file_name=file_name,
|
||||
file_type=file_type,
|
||||
chunk_size=config.chunk_size,
|
||||
chunk_overlap=config.chunk_overlap,
|
||||
separators=config.separators,
|
||||
enable_multimodal=config.enable_multimodal,
|
||||
max_image_size=1920, # Limit image size to 1920px
|
||||
max_concurrent_tasks=5, # Limit concurrent tasks to 5
|
||||
chunking_config=config, # Pass the entire chunking config
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
|
||||
result = parser_instance.parse(content)
|
||||
|
||||
if result:
|
||||
logger.info(
|
||||
f"Successfully parsed file {file_name}, generated {len(result.chunks)} chunks"
|
||||
)
|
||||
if result.chunks and len(result.chunks) > 0:
|
||||
logger.info(
|
||||
f"First chunk content length: {len(result.chunks[0].content)}"
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Parser returned empty chunks for file: {file_name}")
|
||||
else:
|
||||
logger.warning(f"Parser returned None result for file: {file_name}")
|
||||
|
||||
# Return parse results
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing file {file_name}: {str(e)}")
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
return None
|
||||
|
||||
def parse_url(
|
||||
self, url: str, title: str, config: ChunkingConfig
|
||||
) -> Optional[ParseResult]:
|
||||
"""
|
||||
Parse content from a URL using the WebParser.
|
||||
|
||||
Args:
|
||||
url: URL to parse
|
||||
title: Title of the webpage (for metadata)
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing URL: {url}, title: {title}")
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, "
|
||||
f"multimodal={config.enable_multimodal}"
|
||||
)
|
||||
|
||||
parser_instance = None
|
||||
|
||||
try:
|
||||
# Create web parser instance
|
||||
logger.info("Creating WebParser instance")
|
||||
parser_instance = WebParser(
|
||||
title=title,
|
||||
chunk_size=config.chunk_size,
|
||||
chunk_overlap=config.chunk_overlap,
|
||||
separators=config.separators,
|
||||
enable_multimodal=config.enable_multimodal,
|
||||
max_image_size=1920, # Limit image size
|
||||
max_concurrent_tasks=5, # Limit concurrent tasks
|
||||
chunking_config=config,
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse URL content")
|
||||
result = parser_instance.parse(url)
|
||||
|
||||
if result:
|
||||
logger.info(
|
||||
f"Successfully parsed URL, generated {len(result.chunks)} chunks"
|
||||
)
|
||||
logger.info(
|
||||
f"First chunk content length: {len(result.chunks[0].content) if result.chunks else 0}"
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Parser returned empty result for URL: {url}")
|
||||
|
||||
# Return parse results
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing URL {url}: {str(e)}")
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user