begoniezhao
2d66abedf0
移除日志设置与冗余代码，优化导入、类型提示及OCR后端管理 统一调整各文件模块导入路径为绝对导入 调整导入路径，移除部分导入，优化日志及注释 升级文档解析器为 Docx2Parser，优化超时与图片处理逻辑
56 lines
1.6 KiB
Python
56 lines
1.6 KiB
Python
import logging
|
|
|
|
from docreader.models.document import Document
|
|
from docreader.parser.base_parser import BaseParser
|
|
from docreader.utils import endecode
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class TextParser(BaseParser):
|
|
"""
|
|
Text document parser for processing plain text files.
|
|
This parser handles text extraction and chunking from plain text documents.
|
|
"""
|
|
|
|
def parse_into_text(self, content: bytes) -> Document:
|
|
"""
|
|
Parse text document content by decoding bytes to string.
|
|
|
|
This is a straightforward parser that simply converts the binary content
|
|
to text using appropriate character encoding.
|
|
|
|
Args:
|
|
content: Raw document content as bytes
|
|
|
|
Returns:
|
|
Parsed text content as string
|
|
"""
|
|
logger.info(f"Parsing text document, content size: {len(content)} bytes")
|
|
text = endecode.decode_bytes(content)
|
|
logger.info(
|
|
f"Successfully parsed text document, extracted {len(text)} characters"
|
|
)
|
|
return Document(content=text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Sample text for testing
|
|
text = """## 标题1
|
|
|
|
## 标题2
|
|
|
|
## 标题3
|
|
"""
|
|
logger.info(f"Test text content: {text}")
|
|
|
|
# Define separators for text splitting
|
|
seperators = ["\n\n", "\n", "。"]
|
|
parser = TextParser(separators=seperators)
|
|
logger.info("Splitting text into units")
|
|
units = parser._split_into_units(text)
|
|
logger.info(f"Split text into {len(units)} units")
|
|
logger.info(f"Units: {units}")
|