refactor: Introduce create_chunking_config function for better readability and maintainability

This commit is contained in:
begoniezhao
2025-12-01 18:07:43 +08:00
parent 3e31fdeefd
commit 5d6729fc0c
2 changed files with 76 additions and 105 deletions
+75 -104
View File
@@ -15,7 +15,14 @@ from docreader.models.read_config import ChunkingConfig
from docreader.parser import Parser
from docreader.parser.ocr_engine import OCREngine
from docreader.proto import docreader_pb2_grpc
from docreader.proto.docreader_pb2 import Chunk, Image, ReadResponse
from docreader.proto.docreader_pb2 import (
Chunk,
Image,
ReadConfig,
ReadFromFileRequest,
ReadFromURLRequest,
ReadResponse,
)
from docreader.utils.request import init_logging_request_id, request_id_context
# Surrogate range U+D800..U+DFFF are invalid Unicode scalar values
@@ -57,12 +64,75 @@ MAX_MESSAGE_LENGTH = 50 * 1024 * 1024
parser = Parser()
def create_chunking_config(read_config: ReadConfig):
"""Create ChunkingConfig from ReadConfig request.
Args:
read_config: The read_config from the gRPC request
Returns:
ChunkingConfig: Configured chunking configuration object
"""
# Extract chunking parameters
chunk_size = read_config.chunk_size or 512
chunk_overlap = read_config.chunk_overlap or 50
# Convert protobuf RepeatedScalarFieldContainer to list for type compatibility
separators = (
list(read_config.separators) if read_config.separators else ["\n\n", "\n", "。"]
)
enable_multimodal = read_config.enable_multimodal or False
logger.info(
f"Using chunking config: size={chunk_size}, "
f"overlap={chunk_overlap}, multimodal={enable_multimodal}"
)
# Extract storage config
sc = read_config.storage_config
storage_config = {
"provider": "minio" if sc.provider == 2 else "cos",
"region": sc.region,
"bucket_name": sc.bucket_name,
"access_key_id": sc.access_key_id,
"secret_access_key": sc.secret_access_key,
"app_id": sc.app_id,
"path_prefix": sc.path_prefix,
}
logger.info(
f"Using Storage config: provider={storage_config.get('provider')}, "
f"bucket={storage_config['bucket_name']}"
)
# Extract VLM config
vlm_config = {
"model_name": read_config.vlm_config.model_name,
"base_url": read_config.vlm_config.base_url,
"api_key": read_config.vlm_config.api_key or "",
"interface_type": read_config.vlm_config.interface_type or "openai",
}
logger.info(
f"Using VLM config: model={vlm_config['model_name']}, "
f"base_url={vlm_config['base_url']}, "
f"interface_type={vlm_config['interface_type']}"
)
# Create and return ChunkingConfig
return ChunkingConfig(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=separators,
enable_multimodal=enable_multimodal,
storage_config=storage_config,
vlm_config=vlm_config,
)
class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
def __init__(self):
super().__init__()
self.parser = Parser()
def ReadFromFile(self, request, context):
def ReadFromFile(self, request: ReadFromFileRequest, context):
# Get or generate request ID
request_id = (
request.request_id
@@ -83,57 +153,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
logger.info(f"File content size: {len(request.file_content)} bytes")
# Create chunking config
chunk_size = request.read_config.chunk_size or 512
chunk_overlap = request.read_config.chunk_overlap or 50
separators = request.read_config.separators or ["\n\n", "\n", "。"]
enable_multimodal = request.read_config.enable_multimodal or False
logger.info(
f"Using chunking config: size={chunk_size}, "
f"overlap={chunk_overlap}, multimodal={enable_multimodal}"
)
# Get Storage and VLM config from request
storage_config = None
vlm_config = None
sc = request.read_config.storage_config
# Keep parser-side key name as cos_config for backward compatibility
storage_config = {
"provider": "minio" if sc.provider == 2 else "cos",
"region": sc.region,
"bucket_name": sc.bucket_name,
"access_key_id": sc.access_key_id,
"secret_access_key": sc.secret_access_key,
"app_id": sc.app_id,
"path_prefix": sc.path_prefix,
}
logger.info(
f"Using Storage config: provider={storage_config.get('provider')}, "
f"bucket={storage_config['bucket_name']}"
)
vlm_config = {
"model_name": request.read_config.vlm_config.model_name,
"base_url": request.read_config.vlm_config.base_url,
"api_key": request.read_config.vlm_config.api_key or "",
"interface_type": request.read_config.vlm_config.interface_type
or "openai",
}
logger.info(
f"Using VLM config: model={vlm_config['model_name']}, "
f"base_url={vlm_config['base_url']}, "
f"interface_type={vlm_config['interface_type']}"
)
chunking_config = ChunkingConfig(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=separators,
enable_multimodal=enable_multimodal,
storage_config=storage_config,
vlm_config=vlm_config,
)
chunking_config = create_chunking_config(request.read_config)
# Parse file
logger.info("Starting file parsing process")
@@ -170,7 +190,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
context.set_details(str(e))
return ReadResponse(error=str(e))
def ReadFromURL(self, request, context):
def ReadFromURL(self, request: ReadFromURLRequest, context):
# Get or generate request ID
request_id = (
request.request_id
@@ -184,56 +204,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
logger.info(f"Received ReadFromURL request for URL: {request.url}")
# Create chunking config
chunk_size = request.read_config.chunk_size or 512
chunk_overlap = request.read_config.chunk_overlap or 50
separators = request.read_config.separators or ["\n\n", "\n", "。"]
enable_multimodal = request.read_config.enable_multimodal or False
logger.info(
f"Using chunking config: size={chunk_size}, "
f"overlap={chunk_overlap}, multimodal={enable_multimodal}"
)
# Get Storage and VLM config from request
storage_config = None
vlm_config = None
sc = request.read_config.storage_config
storage_config = {
"provider": "minio" if sc.provider == 2 else "cos",
"region": sc.region,
"bucket_name": sc.bucket_name,
"access_key_id": sc.access_key_id,
"secret_access_key": sc.secret_access_key,
"app_id": sc.app_id,
"path_prefix": sc.path_prefix,
}
logger.info(
f"Using Storage config: provider={storage_config.get('provider')}, "
f"bucket={storage_config['bucket_name']}"
)
vlm_config = {
"model_name": request.read_config.vlm_config.model_name,
"base_url": request.read_config.vlm_config.base_url,
"api_key": request.read_config.vlm_config.api_key or "",
"interface_type": request.read_config.vlm_config.interface_type
or "openai",
}
logger.info(
f"Using VLM config: model={vlm_config['model_name']}, "
f"base_url={vlm_config['base_url']}, "
f"interface_type={vlm_config['interface_type']}"
)
chunking_config = ChunkingConfig(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=separators,
enable_multimodal=enable_multimodal,
storage_config=storage_config,
vlm_config=vlm_config,
)
chunking_config = create_chunking_config(request.read_config)
# Parse URL
logger.info("Starting URL parsing process")
+1 -1
View File
@@ -15,7 +15,7 @@ class ChunkingConfig:
chunk_overlap: int = 50
# Text separators in order of priority
separators: list = field(default_factory=lambda: ["\n\n", "\n", "。"])
separators: list[str] = field(default_factory=lambda: ["\n\n", "\n", "。"])
# Whether to enable multimodal processing (text + images)
enable_multimodal: bool = False