feat: 新增文档模型类,调整配置与解析逻辑,优化日志及导入
移除日志设置与冗余代码,优化导入、类型提示及OCR后端管理 统一调整各文件模块导入路径为绝对导入 调整导入路径,移除部分导入,优化日志及注释 升级文档解析器为 Docx2Parser,优化超时与图片处理逻辑
This commit is contained in:
+33
-80
@@ -1,37 +1,25 @@
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from concurrent import futures
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import traceback
|
||||
import grpc
|
||||
import uuid
|
||||
import atexit
|
||||
from concurrent import futures
|
||||
from typing import Optional
|
||||
|
||||
import grpc
|
||||
from grpc_health.v1 import health_pb2_grpc
|
||||
from grpc_health.v1.health import HealthServicer
|
||||
|
||||
# Add parent directory to Python path
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
parent_dir = os.path.dirname(current_dir)
|
||||
if parent_dir not in sys.path:
|
||||
sys.path.insert(0, parent_dir)
|
||||
from docreader.models.read_config import ChunkingConfig
|
||||
from docreader.parser import Parser
|
||||
from docreader.parser.ocr_engine import OCREngine
|
||||
from docreader.proto import docreader_pb2_grpc
|
||||
from docreader.proto.docreader_pb2 import Chunk, Image, ReadResponse
|
||||
from docreader.utils.request import init_logging_request_id, request_id_context
|
||||
|
||||
from proto.docreader_pb2 import ReadResponse, Chunk, Image
|
||||
from proto import docreader_pb2_grpc
|
||||
from parser import Parser, OCREngine
|
||||
from parser.config import ChunkingConfig
|
||||
from utils.request import request_id_context, init_logging_request_id
|
||||
|
||||
# --- Encoding utilities: sanitize strings to valid UTF-8 and (optionally) multi-encoding read ---
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
try:
|
||||
# Optional dependency for charset detection; install via `pip install charset-normalizer`
|
||||
from charset_normalizer import from_bytes as _cn_from_bytes # type: ignore
|
||||
except Exception: # pragma: no cover
|
||||
_cn_from_bytes = None # type: ignore
|
||||
|
||||
# Surrogate range U+D800..U+DFFF are invalid Unicode scalar values and cannot be encoded to UTF-8
|
||||
# Surrogate range U+D800..U+DFFF are invalid Unicode scalar values
|
||||
# cannot be encoded to UTF-8
|
||||
_SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
|
||||
|
||||
|
||||
@@ -47,29 +35,6 @@ def to_valid_utf8_text(s: Optional[str]) -> str:
|
||||
return s.encode("utf-8", errors="replace").decode("utf-8")
|
||||
|
||||
|
||||
def read_text_with_fallback(file_path: str) -> str:
|
||||
"""Read text from file supporting multiple encodings with graceful fallback.
|
||||
|
||||
This server currently receives bytes over gRPC and delegates decoding to the parser.
|
||||
This helper is provided for future local-file reads if needed.
|
||||
"""
|
||||
with open(file_path, "rb") as f:
|
||||
raw = f.read()
|
||||
if _cn_from_bytes is not None:
|
||||
try:
|
||||
result = _cn_from_bytes(raw).best()
|
||||
if result:
|
||||
return str(result)
|
||||
except Exception:
|
||||
pass
|
||||
for enc in ("utf-8", "gb18030", "latin-1"):
|
||||
try:
|
||||
return raw.decode(enc, errors="replace")
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
return raw.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
# Ensure no existing handlers
|
||||
for handler in logging.root.handlers[:]:
|
||||
logging.root.removeHandler(handler)
|
||||
@@ -113,7 +78,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
request.file_type or os.path.splitext(request.file_name)[1][1:]
|
||||
)
|
||||
logger.info(
|
||||
f"Received ReadFromFile request for file: {request.file_name}, type: {file_type}"
|
||||
f"ReadFromFile for file: {request.file_name}, type: {file_type}"
|
||||
)
|
||||
logger.info(f"File content size: {len(request.file_content)} bytes")
|
||||
|
||||
@@ -124,8 +89,8 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
enable_multimodal = request.read_config.enable_multimodal or False
|
||||
|
||||
logger.info(
|
||||
f"Using chunking config: size={chunk_size}, overlap={chunk_overlap}, "
|
||||
f"multimodal={enable_multimodal}"
|
||||
f"Using chunking config: size={chunk_size}, "
|
||||
f"overlap={chunk_overlap}, multimodal={enable_multimodal}"
|
||||
)
|
||||
|
||||
# Get Storage and VLM config from request
|
||||
@@ -144,7 +109,8 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
"path_prefix": sc.path_prefix,
|
||||
}
|
||||
logger.info(
|
||||
f"Using Storage config: provider={storage_config.get('provider')}, bucket={storage_config['bucket_name']}"
|
||||
f"Using Storage config: provider={storage_config.get('provider')}, "
|
||||
f"bucket={storage_config['bucket_name']}"
|
||||
)
|
||||
|
||||
vlm_config = {
|
||||
@@ -170,7 +136,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
)
|
||||
|
||||
# Parse file
|
||||
logger.info(f"Starting file parsing process")
|
||||
logger.info("Starting file parsing process")
|
||||
result = self.parser.parse_file(
|
||||
request.file_name, file_type, request.file_content, chunking_config
|
||||
)
|
||||
@@ -184,7 +150,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
|
||||
# Convert to protobuf message
|
||||
logger.info(
|
||||
f"Successfully parsed file {request.file_name}, returning {len(result.chunks)} chunks"
|
||||
f"Parsed file {request.file_name}, with {len(result.chunks)} chunks"
|
||||
)
|
||||
|
||||
# Build response, including image info
|
||||
@@ -224,8 +190,8 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
enable_multimodal = request.read_config.enable_multimodal or False
|
||||
|
||||
logger.info(
|
||||
f"Using chunking config: size={chunk_size}, overlap={chunk_overlap}, "
|
||||
f"multimodal={enable_multimodal}"
|
||||
f"Using chunking config: size={chunk_size}, "
|
||||
f"overlap={chunk_overlap}, multimodal={enable_multimodal}"
|
||||
)
|
||||
|
||||
# Get Storage and VLM config from request
|
||||
@@ -243,7 +209,8 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
"path_prefix": sc.path_prefix,
|
||||
}
|
||||
logger.info(
|
||||
f"Using Storage config: provider={storage_config.get('provider')}, bucket={storage_config['bucket_name']}"
|
||||
f"Using Storage config: provider={storage_config.get('provider')}, "
|
||||
f"bucket={storage_config['bucket_name']}"
|
||||
)
|
||||
|
||||
vlm_config = {
|
||||
@@ -269,7 +236,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
)
|
||||
|
||||
# Parse URL
|
||||
logger.info(f"Starting URL parsing process")
|
||||
logger.info("Starting URL parsing process")
|
||||
result = self.parser.parse_url(
|
||||
request.url, request.title, chunking_config
|
||||
)
|
||||
@@ -282,7 +249,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
|
||||
# Convert to protobuf message, including image info
|
||||
logger.info(
|
||||
f"Successfully parsed URL {request.url}, returning {len(result.chunks)} chunks"
|
||||
f"Parsed URL {request.url}, returning {len(result.chunks)} chunks"
|
||||
)
|
||||
|
||||
response = ReadResponse(
|
||||
@@ -335,29 +302,15 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
return proto_chunk
|
||||
|
||||
|
||||
def init_ocr_engine(ocr_backend, ocr_config):
|
||||
def init_ocr_engine(ocr_backend: Optional[str] = None, **kwargs):
|
||||
"""Initialize OCR engine"""
|
||||
try:
|
||||
logger.info(f"Initializing OCR engine with backend: {ocr_backend}")
|
||||
ocr_engine = OCREngine.get_instance(backend_type=ocr_backend, **ocr_config)
|
||||
if ocr_engine:
|
||||
logger.info("OCR engine initialized successfully")
|
||||
return True
|
||||
else:
|
||||
logger.error("OCR engine initialization failed")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error initializing OCR engine: {str(e)}")
|
||||
return False
|
||||
backend_type = ocr_backend or os.getenv("OCR_BACKEND", "paddle")
|
||||
logger.info(f"Initializing OCR engine with backend: {backend_type}")
|
||||
OCREngine.get_instance(backend_type=backend_type, **kwargs)
|
||||
|
||||
|
||||
def main():
|
||||
init_ocr_engine(
|
||||
os.getenv("OCR_BACKEND", "paddle"),
|
||||
{
|
||||
"OCR_API_BASE_URL": os.getenv("OCR_API_BASE_URL", ""),
|
||||
},
|
||||
)
|
||||
init_ocr_engine()
|
||||
|
||||
# Set max number of worker threads
|
||||
max_workers = int(os.environ.get("GRPC_MAX_WORKERS", "4"))
|
||||
|
||||
Reference in New Issue
Block a user