feat: support minio storage

This commit is contained in:
wizardchen
2025-08-13 21:56:20 +08:00
committed by lyingbug
parent 621d9aad37
commit 8b43931886
29 changed files with 2735 additions and 1494 deletions
+2 -1
View File
@@ -21,7 +21,8 @@ from .markdown_parser import MarkdownParser
from .text_parser import TextParser
from .image_parser import ImageParser
from .web_parser import WebParser
from .parser import Parser, ChunkingConfig
from .parser import Parser
from .config import ChunkingConfig
from .ocr_engine import OCREngine
# Export public classes and modules
+65 -148
View File
@@ -1,19 +1,22 @@
# -*- coding: utf-8 -*-
import re
import os
import uuid
import asyncio
from typing import List, Dict, Any, Optional, Tuple, Union
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
import logging
from qcloud_cos import CosConfig, CosS3Client
import sys
import traceback
import numpy as np
import time
import io
import json
from .ocr_engine import OCREngine
from .image_utils import image_to_base64
from .config import ChunkingConfig
from .storage import create_storage
from PIL import Image
# Add parent directory to Python path for src imports
current_dir = os.path.dirname(os.path.abspath(__file__))
@@ -101,7 +104,7 @@ class BaseParser(ABC):
max_image_size: int = 1920, # Maximum image size
max_concurrent_tasks: int = 5, # Max concurrent tasks
max_chunks: int = 1000, # Max number of returned chunks
chunking_config: object = None, # Chunking configuration object
chunking_config: ChunkingConfig = None, # Chunking configuration object
):
"""Initialize parser
@@ -118,6 +121,8 @@ class BaseParser(ABC):
max_concurrent_tasks: Max concurrent tasks
max_chunks: Max number of returned chunks
"""
# Storage client instance
self._storage = None
self.file_name = file_name
self.file_type = file_type or os.path.splitext(file_name)[1]
self.enable_multimodal = enable_multimodal
@@ -144,12 +149,7 @@ class BaseParser(ABC):
# Only initialize Caption service if multimodal is enabled
if self.enable_multimodal:
try:
# Get VLM config from chunking config if available
vlm_config = None
if self.chunking_config and hasattr(self.chunking_config, 'vlm_config'):
vlm_config = self.chunking_config.vlm_config
self.caption_parser = Caption(vlm_config)
self.caption_parser = Caption(self.chunking_config.vlm_config)
except Exception as e:
logger.warning(f"Failed to initialize Caption service: {str(e)}")
self.caption_parser = None
@@ -526,70 +526,16 @@ class BaseParser(ABC):
caption = self.get_image_caption(image_data)
return image_data, caption
def _init_cos_client(self, cos_config=None):
"""Initialize Tencent Cloud COS client"""
try:
# Use provided COS config if available, otherwise fall back to environment variables
if cos_config:
secret_id = cos_config.get("secret_id")
secret_key = cos_config.get("secret_key")
region = cos_config.get("region")
bucket_name = cos_config.get("bucket_name")
appid = cos_config.get("app_id")
prefix = cos_config.get("path_prefix", "")
enable_old_domain = cos_config.get("enable_old_domain", "true").lower() == "true"
else:
# Get COS configuration from environment variables
secret_id = os.getenv("COS_SECRET_ID")
secret_key = os.getenv("COS_SECRET_KEY")
region = os.getenv("COS_REGION")
bucket_name = os.getenv("COS_BUCKET_NAME")
appid = os.getenv("COS_APP_ID")
prefix = os.getenv("COS_PATH_PREFIX")
enable_old_domain = (
os.getenv("COS_ENABLE_OLD_DOMAIN", "true").lower() == "true"
)
if not all([secret_id, secret_key, region, bucket_name, appid]):
logger.error(
"Incomplete COS configuration, missing required environment variables"
)
return None, None, None, None
# Initialize COS configuration
logger.info(
f"Initializing COS client with region: {region}, bucket: {bucket_name}"
)
config = CosConfig(
Appid=appid,
Region=region,
SecretId=secret_id,
SecretKey=secret_key,
EnableOldDomain=enable_old_domain,
)
# Create client
client = CosS3Client(config)
return client, bucket_name, region, prefix
except Exception as e:
logger.error(f"Failed to initialize COS client: {str(e)}")
return None, None, None, None
def _get_file_url(self, bucket_name, region, object_key):
"""Generate COS object URL
Args:
bucket_name: Bucket name
region: Region
object_key: Object key
Returns:
File URL
"""
return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}"
def __init_storage(self):
"""Initialize storage client based on configuration"""
if self._storage is None:
storage_config = self.chunking_config.storage_config if self.chunking_config else None
self._storage = create_storage(storage_config)
logger.info(f"Initialized storage client: {self._storage.__class__.__name__}")
return self._storage
def upload_file(self, file_path: str) -> str:
"""Upload file to Tencent Cloud COS
"""Upload file to object storage
Args:
file_path: File path
@@ -597,84 +543,43 @@ class BaseParser(ABC):
Returns:
File URL
"""
logger.info(f"Uploading file to COS: {file_path}")
logger.info(f"Uploading file: {file_path}")
try:
client, bucket_name, region, prefix = self._init_cos_client()
if not client:
return ""
# Generate object key, use UUID to avoid conflicts
file_name = os.path.basename(file_path)
object_key = (
f"{prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
)
logger.info(f"Generated object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to COS")
response = client.upload_file(
Bucket=bucket_name, LocalFilePath=file_path, Key=object_key
)
# Get file URL
file_url = self._get_file_url(bucket_name, region, object_key)
logger.info(f"Successfully uploaded file to COS: {file_url}")
return file_url
storage = self.__init_storage()
return storage.upload_file(file_path)
except Exception as e:
logger.error(f"Failed to upload file to COS: {str(e)}")
logger.error(f"Failed to upload file: {str(e)}")
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png", cos_config=None) -> str:
"""Directly upload file content to Tencent Cloud COS
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to object storage
Args:
content: File byte content
file_ext: File extension, default is .png
cos_config: COS configuration dictionary
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
logger.info(
f"Uploading bytes content to COS, content size: {len(content)} bytes"
)
logger.info(f"Uploading bytes content, size: {len(content)} bytes")
try:
client, bucket_name, region, prefix = self._init_cos_client(cos_config)
if not client:
return ""
# Generate object key, use UUID to avoid conflicts
object_key = f"{prefix}/images/{uuid.uuid4().hex}{file_ext}"
logger.info(f"Generated object key: {object_key}")
# Directly upload file content
logger.info("Attempting to upload bytes content to COS")
response = client.put_object(
Bucket=bucket_name, Body=content, Key=object_key
)
# Get file URL
file_url = self._get_file_url(bucket_name, region, object_key)
logger.info(f"Successfully uploaded bytes to COS: {file_url}")
return file_url
storage = self.__init_storage()
return storage.upload_bytes(content, file_ext)
except Exception as e:
logger.error(f"Failed to upload bytes to COS: {str(e)}")
logger.error(f"Failed to upload bytes to storage: {str(e)}")
traceback.print_exc()
return ""
@abstractmethod
def parse_into_text(self, content: bytes) -> str:
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""Parse document content
Args:
content: Document content
Returns:
Parse result
Either a string containing the parsed text, or a tuple of (text, image_map)
where image_map is a dict mapping image URLs to Image objects
"""
pass
@@ -690,7 +595,12 @@ class BaseParser(ABC):
logger.info(
f"Parsing document with {self.__class__.__name__}, content size: {len(content)} bytes"
)
text = self.parse_into_text(content)
parse_result = self.parse_into_text(content)
if isinstance(parse_result, tuple):
text, image_map = parse_result
else:
text = parse_result
image_map = {}
logger.info(f"Extracted {len(text)} characters of text from {self.file_name}")
logger.info(f"Beginning chunking process for text")
chunks = self.chunk_text(text)
@@ -725,7 +635,7 @@ class BaseParser(ABC):
if file_ext in allowed_types:
logger.info(f"Processing images in each chunk for file type: {file_ext}")
chunks = self.process_chunks_images(chunks)
chunks = self.process_chunks_images(chunks, image_map)
else:
logger.info(f"Skipping image processing for unsupported file type: {file_ext}")
@@ -1053,15 +963,16 @@ class BaseParser(ABC):
return images_info
async def download_and_upload_image(self, img_url: str, current_request_id=None):
"""Download image and upload to COS, if it's already a COS path or local path, use directly
async def download_and_upload_image(self, img_url: str, current_request_id=None, image_map=None):
"""Download image and upload to object storage, if it's already an object storage path or local path, use directly
Args:
img_url: Image URL or local path
current_request_id: Current request ID
image_map: Optional dictionary mapping image URLs to Image objects
Returns:
tuple: (original URL, COS URL, image object), if failed returns (original URL, None, None)
tuple: (original URL, storage URL, image object), if failed returns (original URL, None, None)
"""
# Set request ID context in the asynchronous task
try:
@@ -1077,8 +988,14 @@ class BaseParser(ABC):
from PIL import Image
import io
# Check if it's already a COS path
if "cos" in img_url and "myqcloud.com" in img_url:
# Check if image is already in the image_map
if image_map and img_url in image_map:
logger.info(f"Image already in image_map: {img_url}, using cached object")
return img_url, img_url, image_map[img_url]
# Check if it's already a storage URL (COS or MinIO)
is_storage_url = any(pattern in img_url for pattern in ["cos", "myqcloud.com", "minio", ".s3."])
if is_storage_url:
logger.info(f"Image already on COS: {img_url}, no need to re-upload")
try:
# Still need to get image object for OCR processing
@@ -1101,10 +1018,10 @@ class BaseParser(ABC):
# Image will be closed by the caller
pass
else:
logger.warning(f"Failed to get COS image: {response.status_code}")
logger.warning(f"Failed to get storage image: {response.status_code}")
return img_url, img_url, None
except Exception as e:
logger.error(f"Error getting COS image: {str(e)}")
logger.error(f"Error getting storage image: {str(e)}")
return img_url, img_url, None
# Check if it's a local file path
@@ -1114,12 +1031,12 @@ class BaseParser(ABC):
try:
# Read local image
image = Image.open(img_url)
# Upload to COS
# Upload to storage
with open(img_url, 'rb') as f:
content = f.read()
cos_url = self.upload_bytes(content)
logger.info(f"Successfully uploaded local image to COS: {cos_url}")
return img_url, cos_url, image
storage_url = self.upload_bytes(content)
logger.info(f"Successfully uploaded local image to storage: {storage_url}")
return img_url, storage_url, image
except Exception as e:
logger.error(f"Error processing local image: {str(e)}")
if image and hasattr(image, 'close'):
@@ -1144,10 +1061,10 @@ class BaseParser(ABC):
# Download successful, create image object
image = Image.open(io.BytesIO(response.content))
try:
# Upload to COS using the method in BaseParser
cos_url = self.upload_bytes(response.content)
logger.info(f"Successfully uploaded image to COS: {cos_url}")
return img_url, cos_url, image
# Upload to storage using the method in BaseParser
storage_url = self.upload_bytes(response.content)
logger.info(f"Successfully uploaded image to storage: {storage_url}")
return img_url, storage_url, image
finally:
# Image will be closed by the caller
pass
@@ -1159,7 +1076,7 @@ class BaseParser(ABC):
logger.error(f"Error downloading or processing image: {str(e)}")
return img_url, None, None
async def process_chunk_images_async(self, chunk, chunk_idx, total_chunks, current_request_id=None):
async def process_chunk_images_async(self, chunk, chunk_idx, total_chunks, current_request_id=None, image_map=None):
"""Asynchronously process images in a single Chunk
Args:
@@ -1201,7 +1118,7 @@ class BaseParser(ABC):
loop = asyncio.get_event_loop()
# Concurrent download and upload of images
tasks = [self.download_and_upload_image(url, current_request_id) for url in url_to_info_map.keys()]
tasks = [self.download_and_upload_image(url, current_request_id, image_map) for url in url_to_info_map.keys()]
results = await asyncio.gather(*tasks)
# Process download results, prepare for OCR processing
@@ -1248,7 +1165,7 @@ class BaseParser(ABC):
logger.info(f"Completed image processing in Chunk #{chunk_idx+1}")
return chunk
def process_chunks_images(self, chunks: List[Chunk]) -> List[Chunk]:
def process_chunks_images(self, chunks: List[Chunk], image_map=None) -> List[Chunk]:
"""Concurrent processing of images in all Chunks
Args:
@@ -1282,7 +1199,7 @@ class BaseParser(ABC):
async def process_with_limit(chunk, idx, total):
"""Use semaphore to control concurrent processing of Chunks"""
async with semaphore:
return await self.process_chunk_images_async(chunk, idx, total, current_request_id)
return await self.process_chunk_images_async(chunk, idx, total, current_request_id, image_map)
# Create tasks for all Chunks
tasks = [
+21
View File
@@ -0,0 +1,21 @@
from dataclasses import dataclass, field
@dataclass
class ChunkingConfig:
"""
Configuration for text chunking process.
Controls how documents are split into smaller pieces for processing.
"""
chunk_size: int = 512 # Maximum size of each chunk in tokens/chars
chunk_overlap: int = 50 # Number of tokens/chars to overlap between chunks
separators: list = field(
default_factory=lambda: ["\n\n", "\n", "。"]
) # Text separators in order of priority
enable_multimodal: bool = (
False # Whether to enable multimodal processing (text + images)
)
storage_config: dict = None # Preferred field name going forward
vlm_config: dict = None # VLM configuration for image captioning
File diff suppressed because it is too large Load Diff
+21 -17
View File
@@ -3,12 +3,13 @@ import os
import asyncio
from PIL import Image
import io
from typing import Dict, Any, Tuple, Union
from .base_parser import BaseParser, ParseResult
import numpy as np
# Set up logger for this module
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class ImageParser(BaseParser):
"""
@@ -22,43 +23,46 @@ class ImageParser(BaseParser):
4. Returning a combined result with both text and image reference
"""
def parse_into_text(self, content: bytes) -> str:
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""
Parse image content, only upload the image and return Markdown reference, no OCR or caption processing.
Parse image content, upload the image and return Markdown reference along with image map.
Args:
content: Raw image data (bytes)
Returns:
String containing Markdown image reference
Tuple of (markdown_text, image_map) where image_map maps image URLs to PIL Image objects
"""
logger.info(f"Parsing image content, size: {len(content)} bytes")
image_map = {}
try:
# Upload image to storage service
logger.info("Uploading image to storage")
_, ext = os.path.splitext(self.file_name)
# Get COS config from chunking config if available
cos_config = None
if hasattr(self, 'chunking_config') and self.chunking_config and hasattr(self.chunking_config, 'cos_config'):
cos_config = self.chunking_config.cos_config
image_url = self.upload_bytes(content, file_ext=ext, cos_config=cos_config)
image_url = self.upload_bytes(content, file_ext=ext)
if not image_url:
logger.error("Failed to upload image to storage")
return ""
return "", {}
logger.info(
f"Successfully uploaded image, URL: {image_url[:50]}..."
if len(image_url) > 50
else f"Successfully uploaded image, URL: {image_url}"
)
# Directly generate Markdown image reference, no OCR or caption processing
markdown_text = f"![{self.file_name}]({image_url})"
logger.info("Generated Markdown image reference without OCR or caption processing")
# Create image object and add to map
try:
from PIL import Image
import io
image = Image.open(io.BytesIO(content))
image_map[image_url] = image
logger.info(f"Added image to image_map for URL: {image_url}")
except Exception as img_err:
logger.error(f"Error creating image object: {str(img_err)}")
return markdown_text
markdown_text = f"![{self.file_name}]({image_url})"
return markdown_text, image_map
except Exception as e:
logger.error(f"Error parsing image: {str(e)}")
return ""
return "", {}
@@ -3,7 +3,7 @@ import re
import logging
import numpy as np
import os # Import os module to get environment variables
from typing import Dict, List, Optional, Tuple
from typing import Dict, List, Optional, Tuple, Union, Any
from .base_parser import BaseParser
# Get logger object
@@ -13,7 +13,7 @@ logger = logging.getLogger(__name__)
class MarkdownParser(BaseParser):
"""Markdown document parser"""
def parse_into_text(self, content: bytes) -> str:
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""Parse Markdown document, only extract text content, do not process images
Args:
+1 -21
View File
@@ -10,30 +10,11 @@ from .markdown_parser import MarkdownParser
from .text_parser import TextParser
from .image_parser import ImageParser
from .web_parser import WebParser
from .config import ChunkingConfig
import traceback
logger = logging.getLogger(__name__)
@dataclass
class ChunkingConfig:
"""
Configuration for text chunking process.
Controls how documents are split into smaller pieces for processing.
"""
chunk_size: int = 512 # Maximum size of each chunk in tokens/chars
chunk_overlap: int = 50 # Number of tokens/chars to overlap between chunks
separators: list = field(
default_factory=lambda: ["\n\n", "\n", "。"]
) # Text separators in order of priority
enable_multimodal: bool = (
False # Whether to enable multimodal processing (text + images)
)
cos_config: dict = None # COS configuration for file storage
vlm_config: dict = None # VLM configuration for image captioning
@dataclass
class Chunk:
"""
@@ -100,7 +81,6 @@ class Parser:
file_type: str,
content: bytes,
config: ChunkingConfig,
enable_multimodal: bool = False,
) -> Optional[ParseResult]:
"""
Parse file content using appropriate parser based on file type.
+2 -2
View File
@@ -1,7 +1,7 @@
import logging
import os
import io
from typing import Any, List, Iterator, Optional, Mapping, Tuple, Dict
from typing import Any, List, Iterator, Optional, Mapping, Tuple, Dict, Union
from pypdf import PdfReader
from .base_parser import BaseParser
@@ -16,7 +16,7 @@ class PDFParser(BaseParser):
It uses the pypdf library for simple text extraction.
"""
def parse_into_text(self, content: bytes) -> str:
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""
Parse PDF document content into text
+363
View File
@@ -0,0 +1,363 @@
# -*- coding: utf-8 -*-
import os
import uuid
import logging
import io
import traceback
from abc import ABC, abstractmethod
from typing import Tuple, Optional
from qcloud_cos import CosConfig, CosS3Client
from minio import Minio
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
class Storage(ABC):
"""Abstract base class for object storage operations"""
@abstractmethod
def upload_file(self, file_path: str) -> str:
"""Upload file to object storage
Args:
file_path: File path
Returns:
File URL
"""
pass
@abstractmethod
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to object storage
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
pass
class CosStorage(Storage):
"""Tencent Cloud COS storage implementation"""
def __init__(self, storage_config=None):
"""Initialize COS storage
Args:
storage_config: Storage configuration
"""
self.storage_config = storage_config
self.client, self.bucket_name, self.region, self.prefix = self._init_cos_client()
def _init_cos_client(self):
"""Initialize Tencent Cloud COS client"""
try:
# Use provided COS config if available, otherwise fall back to environment variables
if self.storage_config and self.storage_config.get("access_key_id") != "":
cos_config = self.storage_config
secret_id = cos_config.get("access_key_id")
secret_key = cos_config.get("secret_access_key")
region = cos_config.get("region")
bucket_name = cos_config.get("bucket_name")
appid = cos_config.get("app_id")
prefix = cos_config.get("path_prefix", "")
else:
# Get COS configuration from environment variables
secret_id = os.getenv("COS_SECRET_ID")
secret_key = os.getenv("COS_SECRET_KEY")
region = os.getenv("COS_REGION")
bucket_name = os.getenv("COS_BUCKET_NAME")
appid = os.getenv("COS_APP_ID")
prefix = os.getenv("COS_PATH_PREFIX")
enable_old_domain = (
os.getenv("COS_ENABLE_OLD_DOMAIN", "true").lower() == "true"
)
if not all([secret_id, secret_key, region, bucket_name, appid]):
logger.error(
"Incomplete COS configuration, missing required environment variables"
f"secret_id: {secret_id}, secret_key: {secret_key}, region: {region}, bucket_name: {bucket_name}, appid: {appid}"
)
return None, None, None, None
# Initialize COS configuration
logger.info(
f"Initializing COS client with region: {region}, bucket: {bucket_name}"
)
config = CosConfig(
Appid=appid,
Region=region,
SecretId=secret_id,
SecretKey=secret_key,
EnableOldDomain=enable_old_domain,
)
# Create client
client = CosS3Client(config)
return client, bucket_name, region, prefix
except Exception as e:
logger.error(f"Failed to initialize COS client: {str(e)}")
return None, None, None, None
def _get_download_url(self, bucket_name, region, object_key):
"""Generate COS object URL
Args:
bucket_name: Bucket name
region: Region
object_key: Object key
Returns:
File URL
"""
return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to Tencent Cloud COS
Args:
file_path: File path
Returns:
File URL
"""
logger.info(f"Uploading file to COS: {file_path}")
try:
if not self.client:
return ""
# Generate object key, use UUID to avoid conflicts
file_name = os.path.basename(file_path)
object_key = (
f"{self.prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
)
logger.info(f"Generated object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to COS")
response = self.client.upload_file(
Bucket=self.bucket_name, LocalFilePath=file_path, Key=object_key
)
# Get file URL
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
logger.info(f"Successfully uploaded file to COS: {file_url}")
return file_url
except Exception as e:
logger.error(f"Failed to upload file to COS: {str(e)}")
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to Tencent Cloud COS
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
try:
logger.info(f"Uploading bytes content to COS, size: {len(content)} bytes")
if not self.client:
return ""
object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}" if self.prefix else f"images/{uuid.uuid4().hex}{file_ext}"
logger.info(f"Generated object key: {object_key}")
self.client.put_object(Bucket=self.bucket_name, Body=content, Key=object_key)
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
logger.info(f"Successfully uploaded bytes to COS: {file_url}")
return file_url
except Exception as e:
logger.error(f"Failed to upload bytes to COS: {str(e)}")
traceback.print_exc()
return ""
class MinioStorage(Storage):
"""MinIO storage implementation"""
def __init__(self, storage_config=None):
"""Initialize MinIO storage
Args:
storage_config: Storage configuration
"""
self.storage_config = storage_config
self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = self._init_minio_client()
def _init_minio_client(self):
"""Initialize MinIO client from environment variables or injected config.
If storage_config.path_prefix contains JSON from server (for minio case),
prefer those values to override envs.
"""
try:
endpoint = os.getenv("MINIO_ENDPOINT")
access_key = os.getenv("MINIO_ACCESS_KEY_ID")
secret_key = os.getenv("MINIO_SECRET_ACCESS_KEY")
bucket_name = os.getenv("MINIO_BUCKET_NAME")
use_ssl = os.getenv("MINIO_USE_SSL", "false").lower() == "true"
path_prefix = os.getenv("MINIO_PATH_PREFIX")
# Attempt to override with new generic storage fields
if self.storage_config:
storage_config = self.storage_config
# Direct fields
endpoint = storage_config.get("endpoint", endpoint)
bucket_name = storage_config.get("bucket_name", bucket_name)
path_prefix = storage_config.get("path_prefix", path_prefix)
access_key = storage_config.get("access_key_id", access_key)
secret_key = storage_config.get("secret_access_key", secret_key)
if not all([endpoint, access_key, secret_key, bucket_name]):
logger.error("Incomplete MinIO configuration, missing required environment variables")
return None, None, None, None, None
# Initialize client
client = Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl)
# Ensure bucket exists
found = client.bucket_exists(bucket_name)
if not found:
client.make_bucket(bucket_name)
policy = '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/*"]}]}' % (bucket_name, bucket_name)
client.set_bucket_policy(bucket_name, policy)
return client, bucket_name, use_ssl, endpoint, path_prefix
except Exception as e:
logger.error(f"Failed to initialize MinIO client: {str(e)}")
return None, None, None, None, None
def _get_download_url(self, bucket_name: str, object_key: str, use_ssl: bool, endpoint: str, public_endpoint: str = None):
"""Construct a public URL for MinIO object.
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
"""
if public_endpoint:
base = public_endpoint
else:
scheme = "https" if use_ssl else "http"
base = f"{scheme}://{endpoint}"
# Path-style URL for MinIO
return f"{base}/{bucket_name}/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to MinIO
Args:
file_path: File path
Returns:
File URL
"""
logger.info(f"Uploading file to MinIO: {file_path}")
try:
if not self.client:
return ""
# Generate object key, use UUID to avoid conflicts
file_name = os.path.basename(file_path)
object_key = f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}" if self.path_prefix else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
logger.info(f"Generated MinIO object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to MinIO")
with open(file_path, 'rb') as file_data:
file_size = os.path.getsize(file_path)
self.client.put_object(
bucket_name=self.bucket_name,
object_name=object_key,
data=file_data,
length=file_size,
content_type='application/octet-stream'
)
# Get file URL
file_url = self._get_download_url(
self.bucket_name,
object_key,
self.use_ssl,
self.endpoint,
os.getenv("MINIO_PUBLIC_ENDPOINT", None)
)
logger.info(f"Successfully uploaded file to MinIO: {file_url}")
return file_url
except Exception as e:
logger.error(f"Failed to upload file to MinIO: {str(e)}")
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to MinIO
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
try:
logger.info(f"Uploading bytes content to MinIO, size: {len(content)} bytes")
if not self.client:
return ""
object_key = f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}" if self.path_prefix else f"images/{uuid.uuid4().hex}{file_ext}"
logger.info(f"Generated MinIO object key: {object_key}")
self.client.put_object(
self.bucket_name,
object_key,
data=io.BytesIO(content),
length=len(content),
content_type="application/octet-stream"
)
file_url = self._get_download_url(
self.bucket_name,
object_key,
self.use_ssl,
self.endpoint,
os.getenv("MINIO_PUBLIC_ENDPOINT", None)
)
logger.info(f"Successfully uploaded bytes to MinIO: {file_url}")
return file_url
except Exception as e:
logger.error(f"Failed to upload bytes to MinIO: {str(e)}")
traceback.print_exc()
return ""
def create_storage(storage_config=None) -> Storage:
"""Create a storage instance based on configuration or environment variables
Args:
storage_config: Storage configuration dictionary
Returns:
Storage instance
"""
storage_type = os.getenv("STORAGE_TYPE", "cos").lower()
if storage_config:
storage_type = str(storage_config.get("provider", storage_type)).lower()
logger.info(f"Creating {storage_type} storage instance")
if storage_type == "minio":
return MinioStorage(storage_config)
elif storage_type == "cos":
# Default to COS
return CosStorage(storage_config)
else:
return None
+2 -1
View File
@@ -1,5 +1,6 @@
import logging
from .base_parser import BaseParser
from typing import Dict, Any, Tuple, Union
logger = logging.getLogger(__name__)
@@ -10,7 +11,7 @@ class TextParser(BaseParser):
This parser handles text extraction and chunking from plain text documents.
"""
def parse_into_text(self, content: bytes) -> str:
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""
Parse text document content by decoding bytes to string.
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import Any, Optional
from typing import Any, Optional, Tuple, Dict, Union
from playwright.async_api import async_playwright
from bs4 import BeautifulSoup
@@ -66,7 +66,7 @@ class WebParser(BaseParser):
# Return empty BeautifulSoup object on error
return BeautifulSoup("", "html.parser")
def parse_into_text(self, content: bytes) -> str:
def parse_into_text(self, content: bytes) -> Union[str, Tuple[str, Dict[str, Any]]]:
"""Parse web page
Args:
+140 -75
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.36.7
// protoc-gen-go v1.36.6
// protoc v5.29.3
// source: docreader.proto
@@ -21,33 +21,84 @@ const (
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
// COS 配置
type COSConfig struct {
state protoimpl.MessageState `protogen:"open.v1"`
SecretId string `protobuf:"bytes,1,opt,name=secret_id,json=secretId,proto3" json:"secret_id,omitempty"` // COS Secret ID
SecretKey string `protobuf:"bytes,2,opt,name=secret_key,json=secretKey,proto3" json:"secret_key,omitempty"` // COS Secret Key
Region string `protobuf:"bytes,3,opt,name=region,proto3" json:"region,omitempty"` // COS Region
BucketName string `protobuf:"bytes,4,opt,name=bucket_name,json=bucketName,proto3" json:"bucket_name,omitempty"` // COS Bucket Name
AppId string `protobuf:"bytes,5,opt,name=app_id,json=appId,proto3" json:"app_id,omitempty"` // COS App ID
PathPrefix string `protobuf:"bytes,6,opt,name=path_prefix,json=pathPrefix,proto3" json:"path_prefix,omitempty"` // COS Path Prefix
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
// 对象存储提供方
type StorageProvider int32
const (
StorageProvider_STORAGE_PROVIDER_UNSPECIFIED StorageProvider = 0
StorageProvider_COS StorageProvider = 1 // 腾讯云 COS
StorageProvider_MINIO StorageProvider = 2 // MinIO/S3 兼容
)
// Enum value maps for StorageProvider.
var (
StorageProvider_name = map[int32]string{
0: "STORAGE_PROVIDER_UNSPECIFIED",
1: "COS",
2: "MINIO",
}
StorageProvider_value = map[string]int32{
"STORAGE_PROVIDER_UNSPECIFIED": 0,
"COS": 1,
"MINIO": 2,
}
)
func (x StorageProvider) Enum() *StorageProvider {
p := new(StorageProvider)
*p = x
return p
}
func (x *COSConfig) Reset() {
*x = COSConfig{}
func (x StorageProvider) String() string {
return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
}
func (StorageProvider) Descriptor() protoreflect.EnumDescriptor {
return file_docreader_proto_enumTypes[0].Descriptor()
}
func (StorageProvider) Type() protoreflect.EnumType {
return &file_docreader_proto_enumTypes[0]
}
func (x StorageProvider) Number() protoreflect.EnumNumber {
return protoreflect.EnumNumber(x)
}
// Deprecated: Use StorageProvider.Descriptor instead.
func (StorageProvider) EnumDescriptor() ([]byte, []int) {
return file_docreader_proto_rawDescGZIP(), []int{0}
}
// 通用对象存储配置,兼容 COS 与 MinIO
type StorageConfig struct {
state protoimpl.MessageState `protogen:"open.v1"`
Provider StorageProvider `protobuf:"varint,1,opt,name=provider,proto3,enum=docreader.StorageProvider" json:"provider,omitempty"` // 存储提供方
Region string `protobuf:"bytes,2,opt,name=region,proto3" json:"region,omitempty"` // 区域(COS 使用)
BucketName string `protobuf:"bytes,3,opt,name=bucket_name,json=bucketName,proto3" json:"bucket_name,omitempty"` // 桶名
AccessKeyId string `protobuf:"bytes,4,opt,name=access_key_id,json=accessKeyId,proto3" json:"access_key_id,omitempty"` // 访问密钥 ID(MinIO/S3 使用)
SecretAccessKey string `protobuf:"bytes,5,opt,name=secret_access_key,json=secretAccessKey,proto3" json:"secret_access_key,omitempty"` // 访问密钥 Secret(MinIO/S3 使用)
AppId string `protobuf:"bytes,6,opt,name=app_id,json=appId,proto3" json:"app_id,omitempty"` // 应用 ID(COS 使用)
PathPrefix string `protobuf:"bytes,7,opt,name=path_prefix,json=pathPrefix,proto3" json:"path_prefix,omitempty"` // 路径前缀
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *StorageConfig) Reset() {
*x = StorageConfig{}
mi := &file_docreader_proto_msgTypes[0]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *COSConfig) String() string {
func (x *StorageConfig) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*COSConfig) ProtoMessage() {}
func (*StorageConfig) ProtoMessage() {}
func (x *COSConfig) ProtoReflect() protoreflect.Message {
func (x *StorageConfig) ProtoReflect() protoreflect.Message {
mi := &file_docreader_proto_msgTypes[0]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -59,47 +110,54 @@ func (x *COSConfig) ProtoReflect() protoreflect.Message {
return mi.MessageOf(x)
}
// Deprecated: Use COSConfig.ProtoReflect.Descriptor instead.
func (*COSConfig) Descriptor() ([]byte, []int) {
// Deprecated: Use StorageConfig.ProtoReflect.Descriptor instead.
func (*StorageConfig) Descriptor() ([]byte, []int) {
return file_docreader_proto_rawDescGZIP(), []int{0}
}
func (x *COSConfig) GetSecretId() string {
func (x *StorageConfig) GetProvider() StorageProvider {
if x != nil {
return x.SecretId
return x.Provider
}
return ""
return StorageProvider_STORAGE_PROVIDER_UNSPECIFIED
}
func (x *COSConfig) GetSecretKey() string {
if x != nil {
return x.SecretKey
}
return ""
}
func (x *COSConfig) GetRegion() string {
func (x *StorageConfig) GetRegion() string {
if x != nil {
return x.Region
}
return ""
}
func (x *COSConfig) GetBucketName() string {
func (x *StorageConfig) GetBucketName() string {
if x != nil {
return x.BucketName
}
return ""
}
func (x *COSConfig) GetAppId() string {
func (x *StorageConfig) GetAccessKeyId() string {
if x != nil {
return x.AccessKeyId
}
return ""
}
func (x *StorageConfig) GetSecretAccessKey() string {
if x != nil {
return x.SecretAccessKey
}
return ""
}
func (x *StorageConfig) GetAppId() string {
if x != nil {
return x.AppId
}
return ""
}
func (x *COSConfig) GetPathPrefix() string {
func (x *StorageConfig) GetPathPrefix() string {
if x != nil {
return x.PathPrefix
}
@@ -181,7 +239,7 @@ type ReadConfig struct {
ChunkOverlap int32 `protobuf:"varint,2,opt,name=chunk_overlap,json=chunkOverlap,proto3" json:"chunk_overlap,omitempty"` // 分块重叠
Separators []string `protobuf:"bytes,3,rep,name=separators,proto3" json:"separators,omitempty"` // 分隔符
EnableMultimodal bool `protobuf:"varint,4,opt,name=enable_multimodal,json=enableMultimodal,proto3" json:"enable_multimodal,omitempty"` // 多模态处理
CosConfig *COSConfig `protobuf:"bytes,5,opt,name=cos_config,json=cosConfig,proto3" json:"cos_config,omitempty"` // COS 配置
StorageConfig *StorageConfig `protobuf:"bytes,5,opt,name=storage_config,json=storageConfig,proto3" json:"storage_config,omitempty"` // 对象存储配置(通用)
VlmConfig *VLMConfig `protobuf:"bytes,6,opt,name=vlm_config,json=vlmConfig,proto3" json:"vlm_config,omitempty"` // VLM 配置
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
@@ -245,9 +303,9 @@ func (x *ReadConfig) GetEnableMultimodal() bool {
return false
}
func (x *ReadConfig) GetCosConfig() *COSConfig {
func (x *ReadConfig) GetStorageConfig() *StorageConfig {
if x != nil {
return x.CosConfig
return x.StorageConfig
}
return nil
}
@@ -623,23 +681,23 @@ var File_docreader_proto protoreflect.FileDescriptor
const file_docreader_proto_rawDesc = "" +
"\n" +
"\x0fdocreader.proto\x12\tdocreader\"\xb8\x01\n" +
"\tCOSConfig\x12\x1b\n" +
"\tsecret_id\x18\x01 \x01(\tR\bsecretId\x12\x1d\n" +
"\n" +
"secret_key\x18\x02 \x01(\tR\tsecretKey\x12\x16\n" +
"\x06region\x18\x03 \x01(\tR\x06region\x12\x1f\n" +
"\vbucket_name\x18\x04 \x01(\tR\n" +
"bucketName\x12\x15\n" +
"\x06app_id\x18\x05 \x01(\tR\x05appId\x12\x1f\n" +
"\vpath_prefix\x18\x06 \x01(\tR\n" +
"\x0fdocreader.proto\x12\tdocreader\"\x88\x02\n" +
"\rStorageConfig\x126\n" +
"\bprovider\x18\x01 \x01(\x0e2\x1a.docreader.StorageProviderR\bprovider\x12\x16\n" +
"\x06region\x18\x02 \x01(\tR\x06region\x12\x1f\n" +
"\vbucket_name\x18\x03 \x01(\tR\n" +
"bucketName\x12\"\n" +
"\raccess_key_id\x18\x04 \x01(\tR\vaccessKeyId\x12*\n" +
"\x11secret_access_key\x18\x05 \x01(\tR\x0fsecretAccessKey\x12\x15\n" +
"\x06app_id\x18\x06 \x01(\tR\x05appId\x12\x1f\n" +
"\vpath_prefix\x18\a \x01(\tR\n" +
"pathPrefix\"\x85\x01\n" +
"\tVLMConfig\x12\x1d\n" +
"\n" +
"model_name\x18\x01 \x01(\tR\tmodelName\x12\x19\n" +
"\bbase_url\x18\x02 \x01(\tR\abaseUrl\x12\x17\n" +
"\aapi_key\x18\x03 \x01(\tR\x06apiKey\x12%\n" +
"\x0einterface_type\x18\x04 \x01(\tR\rinterfaceType\"\x87\x02\n" +
"\x0einterface_type\x18\x04 \x01(\tR\rinterfaceType\"\x93\x02\n" +
"\n" +
"ReadConfig\x12\x1d\n" +
"\n" +
@@ -648,9 +706,8 @@ const file_docreader_proto_rawDesc = "" +
"\n" +
"separators\x18\x03 \x03(\tR\n" +
"separators\x12+\n" +
"\x11enable_multimodal\x18\x04 \x01(\bR\x10enableMultimodal\x123\n" +
"\n" +
"cos_config\x18\x05 \x01(\v2\x14.docreader.COSConfigR\tcosConfig\x123\n" +
"\x11enable_multimodal\x18\x04 \x01(\bR\x10enableMultimodal\x12?\n" +
"\x0estorage_config\x18\x05 \x01(\v2\x18.docreader.StorageConfigR\rstorageConfig\x123\n" +
"\n" +
"vlm_config\x18\x06 \x01(\v2\x14.docreader.VLMConfigR\tvlmConfig\"\xc9\x01\n" +
"\x13ReadFromFileRequest\x12!\n" +
@@ -683,7 +740,11 @@ const file_docreader_proto_rawDesc = "" +
"\x06images\x18\x05 \x03(\v2\x10.docreader.ImageR\x06images\"N\n" +
"\fReadResponse\x12(\n" +
"\x06chunks\x18\x01 \x03(\v2\x10.docreader.ChunkR\x06chunks\x12\x14\n" +
"\x05error\x18\x02 \x01(\tR\x05error2\x9f\x01\n" +
"\x05error\x18\x02 \x01(\tR\x05error*G\n" +
"\x0fStorageProvider\x12 \n" +
"\x1cSTORAGE_PROVIDER_UNSPECIFIED\x10\x00\x12\a\n" +
"\x03COS\x10\x01\x12\t\n" +
"\x05MINIO\x10\x022\x9f\x01\n" +
"\tDocReader\x12I\n" +
"\fReadFromFile\x12\x1e.docreader.ReadFromFileRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n" +
"\vReadFromURL\x12\x1d.docreader.ReadFromURLRequest\x1a\x17.docreader.ReadResponse\"\x00B5Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3"
@@ -700,33 +761,36 @@ func file_docreader_proto_rawDescGZIP() []byte {
return file_docreader_proto_rawDescData
}
var file_docreader_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
var file_docreader_proto_msgTypes = make([]protoimpl.MessageInfo, 8)
var file_docreader_proto_goTypes = []any{
(*COSConfig)(nil), // 0: docreader.COSConfig
(*VLMConfig)(nil), // 1: docreader.VLMConfig
(*ReadConfig)(nil), // 2: docreader.ReadConfig
(*ReadFromFileRequest)(nil), // 3: docreader.ReadFromFileRequest
(*ReadFromURLRequest)(nil), // 4: docreader.ReadFromURLRequest
(*Image)(nil), // 5: docreader.Image
(*Chunk)(nil), // 6: docreader.Chunk
(*ReadResponse)(nil), // 7: docreader.ReadResponse
(StorageProvider)(0), // 0: docreader.StorageProvider
(*StorageConfig)(nil), // 1: docreader.StorageConfig
(*VLMConfig)(nil), // 2: docreader.VLMConfig
(*ReadConfig)(nil), // 3: docreader.ReadConfig
(*ReadFromFileRequest)(nil), // 4: docreader.ReadFromFileRequest
(*ReadFromURLRequest)(nil), // 5: docreader.ReadFromURLRequest
(*Image)(nil), // 6: docreader.Image
(*Chunk)(nil), // 7: docreader.Chunk
(*ReadResponse)(nil), // 8: docreader.ReadResponse
}
var file_docreader_proto_depIdxs = []int32{
0, // 0: docreader.ReadConfig.cos_config:type_name -> docreader.COSConfig
1, // 1: docreader.ReadConfig.vlm_config:type_name -> docreader.VLMConfig
2, // 2: docreader.ReadFromFileRequest.read_config:type_name -> docreader.ReadConfig
2, // 3: docreader.ReadFromURLRequest.read_config:type_name -> docreader.ReadConfig
5, // 4: docreader.Chunk.images:type_name -> docreader.Image
6, // 5: docreader.ReadResponse.chunks:type_name -> docreader.Chunk
3, // 6: docreader.DocReader.ReadFromFile:input_type -> docreader.ReadFromFileRequest
4, // 7: docreader.DocReader.ReadFromURL:input_type -> docreader.ReadFromURLRequest
7, // 8: docreader.DocReader.ReadFromFile:output_type -> docreader.ReadResponse
7, // 9: docreader.DocReader.ReadFromURL:output_type -> docreader.ReadResponse
8, // [8:10] is the sub-list for method output_type
6, // [6:8] is the sub-list for method input_type
6, // [6:6] is the sub-list for extension type_name
6, // [6:6] is the sub-list for extension extendee
0, // [0:6] is the sub-list for field type_name
0, // 0: docreader.StorageConfig.provider:type_name -> docreader.StorageProvider
1, // 1: docreader.ReadConfig.storage_config:type_name -> docreader.StorageConfig
2, // 2: docreader.ReadConfig.vlm_config:type_name -> docreader.VLMConfig
3, // 3: docreader.ReadFromFileRequest.read_config:type_name -> docreader.ReadConfig
3, // 4: docreader.ReadFromURLRequest.read_config:type_name -> docreader.ReadConfig
6, // 5: docreader.Chunk.images:type_name -> docreader.Image
7, // 6: docreader.ReadResponse.chunks:type_name -> docreader.Chunk
4, // 7: docreader.DocReader.ReadFromFile:input_type -> docreader.ReadFromFileRequest
5, // 8: docreader.DocReader.ReadFromURL:input_type -> docreader.ReadFromURLRequest
8, // 9: docreader.DocReader.ReadFromFile:output_type -> docreader.ReadResponse
8, // 10: docreader.DocReader.ReadFromURL:output_type -> docreader.ReadResponse
9, // [9:11] is the sub-list for method output_type
7, // [7:9] is the sub-list for method input_type
7, // [7:7] is the sub-list for extension type_name
7, // [7:7] is the sub-list for extension extendee
0, // [0:7] is the sub-list for field type_name
}
func init() { file_docreader_proto_init() }
@@ -739,13 +803,14 @@ func file_docreader_proto_init() {
File: protoimpl.DescBuilder{
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc)),
NumEnums: 0,
NumEnums: 1,
NumMessages: 8,
NumExtensions: 0,
NumServices: 1,
},
GoTypes: file_docreader_proto_goTypes,
DependencyIndexes: file_docreader_proto_depIdxs,
EnumInfos: file_docreader_proto_enumTypes,
MessageInfos: file_docreader_proto_msgTypes,
}.Build()
File_docreader_proto = out.File
+17 -9
View File
@@ -12,14 +12,22 @@ service DocReader {
rpc ReadFromURL(ReadFromURLRequest) returns (ReadResponse) {}
}
// COS 配置
message COSConfig {
string secret_id = 1; // COS Secret ID
string secret_key = 2; // COS Secret Key
string region = 3; // COS Region
string bucket_name = 4; // COS Bucket Name
string app_id = 5; // COS App ID
string path_prefix = 6; // COS Path Prefix
// 对象存储提供方
enum StorageProvider {
STORAGE_PROVIDER_UNSPECIFIED = 0;
COS = 1; // 腾讯云 COS
MINIO = 2; // MinIO/S3 兼容
}
// 通用对象存储配置,兼容 COS 与 MinIO
message StorageConfig {
StorageProvider provider = 1; // 存储提供方
string region = 2; // 区域(COS 使用)
string bucket_name = 3; // 桶名
string access_key_id = 4; // 访问密钥 ID(MinIO/S3 使用)
string secret_access_key = 5; // 访问密钥 Secret(MinIO/S3 使用)
string app_id = 6; // 应用 ID(COS 使用)
string path_prefix = 7; // 路径前缀
}
// VLM 配置
@@ -35,7 +43,7 @@ message ReadConfig {
int32 chunk_overlap = 2; // 分块重叠
repeated string separators = 3; // 分隔符
bool enable_multimodal = 4; // 多模态处理
COSConfig cos_config = 5; // COS 配置
StorageConfig storage_config = 5; // 对象存储配置(通用)
VLMConfig vlm_config = 6; // VLM 配置
}
+49 -47
View File
@@ -14,7 +14,8 @@ if parent_dir not in sys.path:
from proto.docreader_pb2 import ReadResponse, Chunk, Image
from proto import docreader_pb2_grpc
from parser import Parser, ChunkingConfig, OCREngine
from parser import Parser, OCREngine
from parser.config import ChunkingConfig
from utils.request import request_id_context, init_logging_request_id
# Ensure no existing handlers
@@ -74,38 +75,39 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
f"multimodal={enable_multimodal}"
)
# Get COS and VLM config from request
cos_config = None
# Get Storage and VLM config from request
storage_config = None
vlm_config = None
if hasattr(request.read_config, 'cos_config') and request.read_config.cos_config:
cos_config = {
'secret_id': request.read_config.cos_config.secret_id,
'secret_key': request.read_config.cos_config.secret_key,
'region': request.read_config.cos_config.region,
'bucket_name': request.read_config.cos_config.bucket_name,
'app_id': request.read_config.cos_config.app_id,
'path_prefix': request.read_config.cos_config.path_prefix or '',
}
logger.info(f"Using COS config: region={cos_config['region']}, bucket={cos_config['bucket_name']}")
sc = request.read_config.storage_config
# Keep parser-side key name as cos_config for backward compatibility
storage_config = {
'provider': 'minio' if sc.provider == 2 else 'cos',
'region': sc.region,
'bucket_name': sc.bucket_name,
'access_key_id': sc.access_key_id,
'secret_access_key': sc.secret_access_key,
'app_id': sc.app_id,
'path_prefix': sc.path_prefix,
}
logger.info(f"Using Storage config: provider={storage_config.get('provider')}, bucket={storage_config['bucket_name']}")
if hasattr(request.read_config, 'vlm_config') and request.read_config.vlm_config:
vlm_config = {
'model_name': request.read_config.vlm_config.model_name,
'base_url': request.read_config.vlm_config.base_url,
'api_key': request.read_config.vlm_config.api_key or '',
'interface_type': request.read_config.vlm_config.interface_type or 'openai',
}
logger.info(f"Using VLM config: model={vlm_config['model_name']}, "
f"base_url={vlm_config['base_url']}, "
f"interface_type={vlm_config['interface_type']}")
vlm_config = {
'model_name': request.read_config.vlm_config.model_name,
'base_url': request.read_config.vlm_config.base_url,
'api_key': request.read_config.vlm_config.api_key or '',
'interface_type': request.read_config.vlm_config.interface_type or 'openai',
}
logger.info(f"Using VLM config: model={vlm_config['model_name']}, "
f"base_url={vlm_config['base_url']}, "
f"interface_type={vlm_config['interface_type']}")
chunking_config = ChunkingConfig(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=separators,
enable_multimodal=enable_multimodal,
cos_config=cos_config,
storage_config=storage_config,
vlm_config=vlm_config,
)
@@ -166,38 +168,38 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
f"multimodal={enable_multimodal}"
)
# Get COS and VLM config from request
cos_config = None
# Get Storage and VLM config from request
storage_config = None
vlm_config = None
if hasattr(request.read_config, 'cos_config') and request.read_config.cos_config:
cos_config = {
'secret_id': request.read_config.cos_config.secret_id,
'secret_key': request.read_config.cos_config.secret_key,
'region': request.read_config.cos_config.region,
'bucket_name': request.read_config.cos_config.bucket_name,
'app_id': request.read_config.cos_config.app_id,
'path_prefix': request.read_config.cos_config.path_prefix or '',
}
logger.info(f"Using COS config: region={cos_config['region']}, bucket={cos_config['bucket_name']}")
sc = request.read_config.storage_config
storage_config = {
'provider': 'minio' if sc.provider == 2 else 'cos',
'region': sc.region,
'bucket_name': sc.bucket_name,
'access_key_id': sc.access_key_id,
'secret_access_key': sc.secret_access_key,
'app_id': sc.app_id,
'path_prefix': sc.path_prefix,
}
logger.info(f"Using Storage config: provider={storage_config.get('provider')}, bucket={storage_config['bucket_name']}")
if hasattr(request.read_config, 'vlm_config') and request.read_config.vlm_config:
vlm_config = {
'model_name': request.read_config.vlm_config.model_name,
'base_url': request.read_config.vlm_config.base_url,
'api_key': request.read_config.vlm_config.api_key or '',
'interface_type': request.read_config.vlm_config.interface_type or 'openai',
}
logger.info(f"Using VLM config: model={vlm_config['model_name']}, "
f"base_url={vlm_config['base_url']}, "
f"interface_type={vlm_config['interface_type']}")
vlm_config = {
'model_name': request.read_config.vlm_config.model_name,
'base_url': request.read_config.vlm_config.base_url,
'api_key': request.read_config.vlm_config.api_key or '',
'interface_type': request.read_config.vlm_config.interface_type or 'openai',
}
logger.info(f"Using VLM config: model={vlm_config['model_name']}, "
f"base_url={vlm_config['base_url']}, "
f"interface_type={vlm_config['interface_type']}")
chunking_config = ChunkingConfig(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=separators,
enable_multimodal=enable_multimodal,
cos_config=cos_config,
storage_config=storage_config,
vlm_config=vlm_config,
)