chore(docreader): 重新组织模块文件
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
import logging
|
||||
from .base_parser import BaseParser
|
||||
from typing import Dict, Any, Tuple, Union
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TextParser(BaseParser):
|
||||
"""
|
||||
Text document parser for processing plain text files.
|
||||
This parser handles text extraction and chunking from plain text documents.
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
|
||||
"""
|
||||
Parse text document content by decoding bytes to string.
|
||||
|
||||
This is a straightforward parser that simply converts the binary content
|
||||
to text using appropriate character encoding.
|
||||
|
||||
Args:
|
||||
content: Raw document content as bytes
|
||||
|
||||
Returns:
|
||||
Parsed text content as string
|
||||
"""
|
||||
logger.info(f"Parsing text document, content size: {len(content)} bytes")
|
||||
text = self.decode_bytes(content)
|
||||
logger.info(
|
||||
f"Successfully parsed text document, extracted {len(text)} characters"
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
logger.info("Running TextParser in standalone mode")
|
||||
|
||||
# Sample text for testing
|
||||
text = """## 标题1
|
||||

|
||||
## 标题2
|
||||

|
||||
## 标题3
|
||||
"""
|
||||
logger.info(f"Test text content: {text}")
|
||||
|
||||
# Define separators for text splitting
|
||||
seperators = ["\n\n", "\n", "。"]
|
||||
parser = TextParser(separators=seperators)
|
||||
logger.info("Splitting text into units")
|
||||
units = parser._split_into_units(text)
|
||||
logger.info(f"Split text into {len(units)} units")
|
||||
logger.info(f"Units: {units}")
|
||||
Reference in New Issue
Block a user