feat: introduce WeKnora Lite edition with lightweight configuration and deployment
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template. - Updated `.env.example` to remove deprecated variables and include new Docreader settings. - Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service. - Introduced a Makefile target for building and running the Lite version, along with packaging capabilities. - Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support. - Implemented a new service file for managing the Lite version as a system service. This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
+19
-38
@@ -13,42 +13,35 @@ import (
|
||||
"google.golang.org/grpc/resolver"
|
||||
)
|
||||
|
||||
// getMaxMessageSize returns the maximum gRPC message size in bytes.
|
||||
// Default is 50MB, can be configured via MAX_FILE_SIZE_MB environment variable.
|
||||
func getMaxMessageSize() int {
|
||||
if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" {
|
||||
if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 {
|
||||
return size * 1024 * 1024
|
||||
}
|
||||
}
|
||||
return 50 * 1024 * 1024 // default 50MB
|
||||
return 50 * 1024 * 1024
|
||||
}
|
||||
|
||||
// Logger is the default logger used by the client
|
||||
var Logger = log.New(os.Stdout, "[DocReader] ", log.LstdFlags|log.Lmicroseconds)
|
||||
|
||||
// ImageInfo 表示一个图片的信息
|
||||
type ImageInfo struct {
|
||||
URL string // 图片URL(COS)
|
||||
Caption string // 图片描述
|
||||
OCRText string // OCR提取的文本
|
||||
OriginalURL string // 原始图片URL
|
||||
Start int // 图片在文本中的开始位置
|
||||
End int // 图片在文本中的结束位置
|
||||
// ImageRefInfo represents an image reference from a converted document.
|
||||
type ImageRefInfo struct {
|
||||
Filename string
|
||||
OriginalRef string
|
||||
MimeType string
|
||||
StorageKey string
|
||||
}
|
||||
|
||||
// Client represents a DocReader service client
|
||||
// Client represents a DocReader service client.
|
||||
type Client struct {
|
||||
conn *grpc.ClientConn
|
||||
proto.DocReaderClient
|
||||
debug bool
|
||||
}
|
||||
|
||||
// NewClient creates a new DocReader client with the specified address
|
||||
func NewClient(addr string) (*Client, error) {
|
||||
Logger.Printf("INFO: Creating new DocReader client connecting to %s", addr)
|
||||
|
||||
// 设置消息大小限制 (configurable via GRPC_MAX_MESSAGE_SIZE_MB)
|
||||
maxMsgSize := getMaxMessageSize()
|
||||
opts := []grpc.DialOption{
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
@@ -75,19 +68,15 @@ func NewClient(addr string) (*Client, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Close closes the client connection
|
||||
func (c *Client) Close() error {
|
||||
Logger.Printf("INFO: Closing DocReader client connection")
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// SetDebug enables or disables debug logging
|
||||
func (c *Client) SetDebug(debug bool) {
|
||||
c.debug = debug
|
||||
Logger.Printf("INFO: Debug logging set to %v", debug)
|
||||
}
|
||||
|
||||
// Log logs a message with the appropriate level
|
||||
func (c *Client) Log(level string, format string, args ...interface{}) {
|
||||
if level == "DEBUG" && !c.debug {
|
||||
return
|
||||
@@ -95,28 +84,20 @@ func (c *Client) Log(level string, format string, args ...interface{}) {
|
||||
Logger.Printf("%s: %s", level, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// GetImagesFromChunk 从一个Chunk中提取所有图片信息
|
||||
func GetImagesFromChunk(chunk *proto.Chunk) []ImageInfo {
|
||||
if chunk == nil || len(chunk.Images) == 0 {
|
||||
// GetImageRefsFromResponse extracts image references from a ReadResponse.
|
||||
func GetImageRefsFromResponse(resp *proto.ReadResponse) []ImageRefInfo {
|
||||
if resp == nil || len(resp.ImageRefs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
images := make([]ImageInfo, 0, len(chunk.Images))
|
||||
for _, img := range chunk.Images {
|
||||
images = append(images, ImageInfo{
|
||||
URL: img.Url,
|
||||
Caption: img.Caption,
|
||||
OCRText: img.OcrText,
|
||||
OriginalURL: img.OriginalUrl,
|
||||
Start: int(img.Start),
|
||||
End: int(img.End),
|
||||
refs := make([]ImageRefInfo, 0, len(resp.ImageRefs))
|
||||
for _, ref := range resp.ImageRefs {
|
||||
refs = append(refs, ImageRefInfo{
|
||||
Filename: ref.Filename,
|
||||
OriginalRef: ref.OriginalRef,
|
||||
MimeType: ref.MimeType,
|
||||
StorageKey: ref.StorageKey,
|
||||
})
|
||||
}
|
||||
|
||||
return images
|
||||
}
|
||||
|
||||
// HasImagesInChunk 判断一个Chunk是否包含图片
|
||||
func HasImagesInChunk(chunk *proto.Chunk) bool {
|
||||
return chunk != nil && len(chunk.Images) > 0
|
||||
return refs
|
||||
}
|
||||
|
||||
@@ -11,144 +11,75 @@ import (
|
||||
)
|
||||
|
||||
func init() {
|
||||
// 配置测试日志
|
||||
log.SetOutput(os.Stdout)
|
||||
log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.Lshortfile)
|
||||
log.Println("INFO: Initializing DocReader client tests")
|
||||
}
|
||||
|
||||
func TestReadFromURL(t *testing.T) {
|
||||
log.Println("INFO: Starting TestReadFromURL")
|
||||
|
||||
// 创建测试客户端
|
||||
log.Println("INFO: Creating test client")
|
||||
func TestReadURL(t *testing.T) {
|
||||
client, err := NewClient("localhost:50051")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to create client: %v", err)
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// 启用调试日志
|
||||
client.SetDebug(true)
|
||||
|
||||
// 测试 ReadFromURL 方法
|
||||
log.Println("INFO: Sending ReadFromURL request to server")
|
||||
startTime := time.Now()
|
||||
resp, err := client.ReadFromURL(
|
||||
resp, err := client.Read(
|
||||
context.Background(),
|
||||
&proto.ReadFromURLRequest{
|
||||
&proto.ReadRequest{
|
||||
Url: "https://example.com",
|
||||
Title: "test",
|
||||
ReadConfig: &proto.ReadConfig{
|
||||
ChunkSize: 512,
|
||||
ChunkOverlap: 50,
|
||||
Separators: []string{"\n\n", "\n", "。"},
|
||||
EnableMultimodal: true,
|
||||
},
|
||||
},
|
||||
)
|
||||
log.Printf("INFO: Read(URL) completed in %v", time.Since(startTime))
|
||||
|
||||
requestDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: ReadFromURL failed: %v", err)
|
||||
t.Fatalf("ReadFromURL failed: %v", err)
|
||||
t.Fatalf("Read failed: %v", err)
|
||||
}
|
||||
log.Printf("INFO: ReadFromURL completed in %v", requestDuration)
|
||||
|
||||
// 验证结果
|
||||
chunkCount := len(resp.Chunks)
|
||||
log.Printf("INFO: Received %d chunks from URL parsing", chunkCount)
|
||||
if chunkCount == 0 {
|
||||
log.Println("WARN: Expected non-empty content but received none")
|
||||
t.Error("Expected non-empty content")
|
||||
if resp.Error != "" {
|
||||
t.Fatalf("Read returned error: %s", resp.Error)
|
||||
}
|
||||
|
||||
// 打印结果
|
||||
for i, chunk := range resp.Chunks {
|
||||
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
|
||||
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
|
||||
} else if i == 2 && chunkCount > 4 {
|
||||
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
|
||||
}
|
||||
if resp.MarkdownContent == "" {
|
||||
t.Error("Expected non-empty markdown content")
|
||||
}
|
||||
|
||||
log.Println("INFO: TestReadFromURL completed successfully")
|
||||
log.Printf("INFO: content_len=%d, images=%d", len(resp.MarkdownContent), len(resp.ImageRefs))
|
||||
}
|
||||
|
||||
func TestReadFromFileWithChunking(t *testing.T) {
|
||||
log.Println("INFO: Starting TestReadFromFileWithChunking")
|
||||
|
||||
// 创建测试客户端
|
||||
log.Println("INFO: Creating test client")
|
||||
func TestReadFile(t *testing.T) {
|
||||
client, err := NewClient("localhost:50051")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to create client: %v", err)
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// 启用调试日志
|
||||
client.SetDebug(true)
|
||||
|
||||
// 读取测试文件
|
||||
log.Println("INFO: Reading test file")
|
||||
fileContent, err := os.ReadFile("../testdata/test.md")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to read test file: %v", err)
|
||||
t.Fatalf("Failed to read test file: %v", err)
|
||||
}
|
||||
log.Printf("INFO: Read test file, size: %d bytes", len(fileContent))
|
||||
|
||||
// 测试 ReadFromFile 方法,带分块参数
|
||||
log.Println("INFO: Sending ReadFromFile request to server")
|
||||
startTime := time.Now()
|
||||
resp, err := client.ReadFromFile(
|
||||
resp, err := client.Read(
|
||||
context.Background(),
|
||||
&proto.ReadFromFileRequest{
|
||||
&proto.ReadRequest{
|
||||
FileContent: fileContent,
|
||||
FileName: "test.md",
|
||||
FileType: "md",
|
||||
ReadConfig: &proto.ReadConfig{
|
||||
ChunkSize: 200,
|
||||
ChunkOverlap: 50,
|
||||
Separators: []string{"\n\n", "\n", "。"},
|
||||
EnableMultimodal: true,
|
||||
},
|
||||
},
|
||||
)
|
||||
log.Printf("INFO: Read(file) completed in %v", time.Since(startTime))
|
||||
|
||||
requestDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: ReadFromFile failed: %v", err)
|
||||
t.Fatalf("ReadFromFile failed: %v", err)
|
||||
t.Fatalf("Read failed: %v", err)
|
||||
}
|
||||
log.Printf("INFO: ReadFromFile completed in %v", requestDuration)
|
||||
|
||||
// 验证结果
|
||||
chunkCount := len(resp.Chunks)
|
||||
log.Printf("INFO: Received %d chunks from file parsing", chunkCount)
|
||||
if chunkCount == 0 {
|
||||
log.Println("WARN: Expected non-empty content but received none")
|
||||
t.Error("Expected non-empty content")
|
||||
if resp.Error != "" {
|
||||
t.Fatalf("Read returned error: %s", resp.Error)
|
||||
}
|
||||
if resp.MarkdownContent == "" {
|
||||
t.Error("Expected non-empty markdown content")
|
||||
}
|
||||
|
||||
// 打印结果
|
||||
for i, chunk := range resp.Chunks {
|
||||
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
|
||||
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
|
||||
} else if i == 2 && chunkCount > 4 {
|
||||
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("INFO: TestReadFromFileWithChunking completed successfully")
|
||||
}
|
||||
|
||||
// 截断字符串以供日志打印
|
||||
func truncateString(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
imageRefs := GetImageRefsFromResponse(resp)
|
||||
log.Printf("INFO: content_len=%d, images=%d", len(resp.MarkdownContent), len(imageRefs))
|
||||
}
|
||||
|
||||
+7
-176
@@ -52,58 +52,17 @@ class DocReaderConfig:
|
||||
grpc_max_file_size_mb: int
|
||||
grpc_port: int
|
||||
|
||||
# Image processing
|
||||
image_max_concurrent: int
|
||||
|
||||
# Proxy
|
||||
external_http_proxy: str
|
||||
external_https_proxy: str
|
||||
|
||||
# OCR
|
||||
ocr_backend: str
|
||||
ocr_api_base_url: str
|
||||
ocr_api_key: str
|
||||
ocr_model: str
|
||||
|
||||
# VLM Caption
|
||||
vlm_model_base_url: str
|
||||
vlm_model_name: str
|
||||
vlm_model_api_key: str
|
||||
vlm_interface_type: str
|
||||
|
||||
# Storage
|
||||
storage_type: str
|
||||
|
||||
cos_secret_id: str
|
||||
cos_secret_key: str
|
||||
cos_region: str
|
||||
cos_bucket_name: str
|
||||
cos_app_id: str
|
||||
cos_path_prefix: str
|
||||
cos_enable_old_domain: bool
|
||||
|
||||
minio_access_key_id: str
|
||||
minio_secret_access_key: str
|
||||
minio_bucket_name: str
|
||||
minio_path_prefix: str
|
||||
minio_endpoint: str
|
||||
minio_public_endpoint: str
|
||||
minio_use_ssl: bool
|
||||
|
||||
local_storage_base_dir: str
|
||||
|
||||
# Other
|
||||
mineru_endpoint: str
|
||||
# Temp image output directory (shared with Go app via volume, local mode fallback)
|
||||
image_output_dir: str
|
||||
|
||||
|
||||
def load_config() -> DocReaderConfig:
|
||||
"""Load config from environment variables.
|
||||
"""Load config from environment variables (lightweight version)."""
|
||||
|
||||
Naming convention (new): DOCREADER_*
|
||||
Backward compatible keys are supported.
|
||||
"""
|
||||
|
||||
# gRPC
|
||||
grpc_max_workers = _get_int(["DOCREADER_GRPC_MAX_WORKERS", "GRPC_MAX_WORKERS"], 4)
|
||||
grpc_max_file_size_mb = (
|
||||
_get_int(["DOCREADER_GRPC_MAX_FILE_SIZE_MB", "MAX_FILE_SIZE_MB"], 50)
|
||||
@@ -112,12 +71,6 @@ def load_config() -> DocReaderConfig:
|
||||
)
|
||||
grpc_port = _get_int(["DOCREADER_GRPC_PORT", "PORT"], 50051)
|
||||
|
||||
# Image processing
|
||||
image_max_concurrent = _get_int(
|
||||
["DOCREADER_IMAGE_MAX_CONCURRENT", "IMAGE_MAX_CONCURRENT"], 1
|
||||
)
|
||||
|
||||
# Proxies
|
||||
external_http_proxy = _get_str(
|
||||
["DOCREADER_EXTERNAL_HTTP_PROXY", "EXTERNAL_HTTP_PROXY"], ""
|
||||
)
|
||||
@@ -125,95 +78,17 @@ def load_config() -> DocReaderConfig:
|
||||
["DOCREADER_EXTERNAL_HTTPS_PROXY", "EXTERNAL_HTTPS_PROXY"], ""
|
||||
)
|
||||
|
||||
# OCR
|
||||
ocr_backend = _get_str(["DOCREADER_OCR_BACKEND", "OCR_BACKEND"], "")
|
||||
ocr_api_base_url = _get_str(["DOCREADER_OCR_API_BASE_URL", "OCR_API_BASE_URL"], "")
|
||||
ocr_api_key = _get_str(["DOCREADER_OCR_API_KEY", "OCR_API_KEY"], "")
|
||||
ocr_model = _get_str(["DOCREADER_OCR_MODEL", "OCR_MODEL"], "")
|
||||
|
||||
# VLM Caption
|
||||
vlm_model_base_url = _get_str(
|
||||
["DOCREADER_VLM_MODEL_BASE_URL", "VLM_MODEL_BASE_URL"], ""
|
||||
image_output_dir = _get_str(
|
||||
["DOCREADER_IMAGE_OUTPUT_DIR", "IMAGE_OUTPUT_DIR"], "/tmp/docreader"
|
||||
)
|
||||
vlm_model_name = _get_str(["DOCREADER_VLM_MODEL_NAME", "VLM_MODEL_NAME"], "")
|
||||
vlm_model_api_key = _get_str(
|
||||
["DOCREADER_VLM_MODEL_API_KEY", "VLM_MODEL_API_KEY"], ""
|
||||
)
|
||||
vlm_interface_type = _get_str(
|
||||
["DOCREADER_VLM_INTERFACE_TYPE", "VLM_INTERFACE_TYPE"], "openai"
|
||||
).lower()
|
||||
|
||||
# Storage
|
||||
storage_type = _get_str(["DOCREADER_STORAGE_TYPE", "STORAGE_TYPE"], "cos").lower()
|
||||
|
||||
# COS
|
||||
cos_secret_id = _get_str(["DOCREADER_COS_SECRET_ID", "COS_SECRET_ID"], "")
|
||||
cos_secret_key = _get_str(["DOCREADER_COS_SECRET_KEY", "COS_SECRET_KEY"], "")
|
||||
cos_region = _get_str(["DOCREADER_COS_REGION", "COS_REGION"], "")
|
||||
cos_bucket_name = _get_str(["DOCREADER_COS_BUCKET_NAME", "COS_BUCKET_NAME"], "")
|
||||
cos_app_id = _get_str(["DOCREADER_COS_APP_ID", "COS_APP_ID"], "")
|
||||
cos_path_prefix = _get_str(["DOCREADER_COS_PATH_PREFIX", "COS_PATH_PREFIX"], "")
|
||||
cos_enable_old_domain = _get_bool(
|
||||
["DOCREADER_COS_ENABLE_OLD_DOMAIN", "COS_ENABLE_OLD_DOMAIN"], True
|
||||
)
|
||||
|
||||
# MinIO
|
||||
minio_access_key_id = _get_str(
|
||||
["DOCREADER_MINIO_ACCESS_KEY_ID", "MINIO_ACCESS_KEY_ID"], "minioadmin"
|
||||
)
|
||||
minio_secret_access_key = _get_str(
|
||||
["DOCREADER_MINIO_SECRET_ACCESS_KEY", "MINIO_SECRET_ACCESS_KEY"], "minioadmin"
|
||||
)
|
||||
minio_bucket_name = _get_str(
|
||||
["DOCREADER_MINIO_BUCKET_NAME", "MINIO_BUCKET_NAME"], "WeKnora"
|
||||
)
|
||||
minio_path_prefix = _get_str(
|
||||
["DOCREADER_MINIO_PATH_PREFIX", "MINIO_PATH_PREFIX"], ""
|
||||
)
|
||||
minio_endpoint = _get_str(["DOCREADER_MINIO_ENDPOINT", "MINIO_ENDPOINT"], "")
|
||||
minio_public_endpoint = _get_str(
|
||||
["DOCREADER_MINIO_PUBLIC_ENDPOINT", "MINIO_PUBLIC_ENDPOINT"], ""
|
||||
)
|
||||
minio_use_ssl = _get_bool(["DOCREADER_MINIO_USE_SSL", "MINIO_USE_SSL"], False)
|
||||
|
||||
# Local storage
|
||||
local_storage_base_dir = "./data/files"
|
||||
|
||||
# Other
|
||||
mineru_endpoint = _get_str(["DOCREADER_MINERU_ENDPOINT", "MINERU_ENDPOINT"], "")
|
||||
|
||||
return DocReaderConfig(
|
||||
grpc_max_workers=grpc_max_workers,
|
||||
grpc_max_file_size_mb=grpc_max_file_size_mb,
|
||||
grpc_port=grpc_port,
|
||||
image_max_concurrent=image_max_concurrent,
|
||||
external_http_proxy=external_http_proxy,
|
||||
external_https_proxy=external_https_proxy,
|
||||
ocr_backend=ocr_backend,
|
||||
ocr_api_base_url=ocr_api_base_url,
|
||||
ocr_api_key=ocr_api_key,
|
||||
ocr_model=ocr_model,
|
||||
vlm_model_base_url=vlm_model_base_url,
|
||||
vlm_model_name=vlm_model_name,
|
||||
vlm_model_api_key=vlm_model_api_key,
|
||||
vlm_interface_type=vlm_interface_type,
|
||||
storage_type=storage_type,
|
||||
cos_secret_id=cos_secret_id,
|
||||
cos_secret_key=cos_secret_key,
|
||||
cos_region=cos_region,
|
||||
cos_bucket_name=cos_bucket_name,
|
||||
cos_app_id=cos_app_id,
|
||||
cos_path_prefix=cos_path_prefix,
|
||||
cos_enable_old_domain=cos_enable_old_domain,
|
||||
minio_access_key_id=minio_access_key_id,
|
||||
minio_secret_access_key=minio_secret_access_key,
|
||||
minio_bucket_name=minio_bucket_name,
|
||||
minio_path_prefix=minio_path_prefix,
|
||||
minio_endpoint=minio_endpoint,
|
||||
minio_public_endpoint=minio_public_endpoint,
|
||||
minio_use_ssl=minio_use_ssl,
|
||||
local_storage_base_dir=local_storage_base_dir,
|
||||
mineru_endpoint=mineru_endpoint,
|
||||
image_output_dir=image_output_dir,
|
||||
)
|
||||
|
||||
|
||||
@@ -223,56 +98,12 @@ CONFIG = load_config()
|
||||
def dump_config(mask_secrets: bool = True) -> Dict[str, Any]:
|
||||
cfg = CONFIG
|
||||
d: Dict[str, Any] = {
|
||||
# gRPC
|
||||
"DOCREADER_GRPC_MAX_WORKERS": cfg.grpc_max_workers,
|
||||
"DOCREADER_GRPC_MAX_FILE_SIZE_MB": cfg.grpc_max_file_size_mb,
|
||||
"DOCREADER_GRPC_PORT": cfg.grpc_port,
|
||||
# Image processing
|
||||
"DOCREADER_IMAGE_MAX_CONCURRENT": cfg.image_max_concurrent,
|
||||
# Proxy
|
||||
"DOCREADER_EXTERNAL_HTTP_PROXY": cfg.external_http_proxy,
|
||||
"DOCREADER_EXTERNAL_HTTPS_PROXY": cfg.external_https_proxy,
|
||||
# OCR
|
||||
"DOCREADER_OCR_BACKEND": cfg.ocr_backend,
|
||||
"DOCREADER_OCR_API_BASE_URL": cfg.ocr_api_base_url,
|
||||
"DOCREADER_OCR_API_KEY": _mask_secret(cfg.ocr_api_key)
|
||||
if mask_secrets
|
||||
else cfg.ocr_api_key,
|
||||
"DOCREADER_OCR_MODEL": cfg.ocr_model,
|
||||
# VLM
|
||||
"DOCREADER_VLM_MODEL_BASE_URL": cfg.vlm_model_base_url,
|
||||
"DOCREADER_VLM_MODEL_NAME": cfg.vlm_model_name,
|
||||
"DOCREADER_VLM_MODEL_API_KEY": _mask_secret(cfg.vlm_model_api_key)
|
||||
if mask_secrets
|
||||
else cfg.vlm_model_api_key,
|
||||
"DOCREADER_VLM_INTERFACE_TYPE": cfg.vlm_interface_type,
|
||||
# Storage
|
||||
"DOCREADER_STORAGE_TYPE": cfg.storage_type,
|
||||
"DOCREADER_COS_SECRET_ID": _mask_secret(cfg.cos_secret_id)
|
||||
if mask_secrets
|
||||
else cfg.cos_secret_id,
|
||||
"DOCREADER_COS_SECRET_KEY": _mask_secret(cfg.cos_secret_key)
|
||||
if mask_secrets
|
||||
else cfg.cos_secret_key,
|
||||
"DOCREADER_COS_REGION": cfg.cos_region,
|
||||
"DOCREADER_COS_BUCKET_NAME": cfg.cos_bucket_name,
|
||||
"DOCREADER_COS_APP_ID": cfg.cos_app_id,
|
||||
"DOCREADER_COS_PATH_PREFIX": cfg.cos_path_prefix,
|
||||
"DOCREADER_COS_ENABLE_OLD_DOMAIN": cfg.cos_enable_old_domain,
|
||||
"DOCREADER_MINIO_ACCESS_KEY_ID": _mask_secret(cfg.minio_access_key_id)
|
||||
if mask_secrets
|
||||
else cfg.minio_access_key_id,
|
||||
"DOCREADER_MINIO_SECRET_ACCESS_KEY": _mask_secret(cfg.minio_secret_access_key)
|
||||
if mask_secrets
|
||||
else cfg.minio_secret_access_key,
|
||||
"DOCREADER_MINIO_BUCKET_NAME": cfg.minio_bucket_name,
|
||||
"DOCREADER_MINIO_PATH_PREFIX": cfg.minio_path_prefix,
|
||||
"DOCREADER_MINIO_ENDPOINT": cfg.minio_endpoint,
|
||||
"DOCREADER_MINIO_PUBLIC_ENDPOINT": cfg.minio_public_endpoint,
|
||||
"DOCREADER_MINIO_USE_SSL": cfg.minio_use_ssl,
|
||||
"DOCREADER_LOCAL_STORAGE_BASE_DIR": cfg.local_storage_base_dir,
|
||||
# Other
|
||||
"DOCREADER_MINERU_ENDPOINT": cfg.mineru_endpoint,
|
||||
"DOCREADER_IMAGE_OUTPUT_DIR": cfg.image_output_dir,
|
||||
}
|
||||
return d
|
||||
|
||||
|
||||
+149
-205
@@ -13,118 +13,130 @@ from grpc_health.v1.health import HealthServicer
|
||||
|
||||
from docreader import config
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.read_config import ChunkingConfig
|
||||
from docreader.parser import Parser
|
||||
from docreader.proto import docreader_pb2_grpc
|
||||
from docreader.parser.registry import registry
|
||||
from docreader.proto.docreader_pb2 import (
|
||||
Chunk,
|
||||
Image,
|
||||
ReadConfig,
|
||||
ReadFromFileRequest,
|
||||
ReadFromURLRequest,
|
||||
ReadRequest,
|
||||
ReadResponse,
|
||||
StorageProvider,
|
||||
ImageRef,
|
||||
ListEnginesResponse,
|
||||
ParserEngineInfo,
|
||||
)
|
||||
from docreader.utils.request import init_logging_request_id, request_id_context
|
||||
|
||||
# Surrogate range U+D800..U+DFFF are invalid Unicode scalar values
|
||||
# cannot be encoded to UTF-8
|
||||
_SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
|
||||
|
||||
|
||||
def to_valid_utf8_text(s: Optional[str]) -> str:
|
||||
"""Return a UTF-8 safe string for protobuf.
|
||||
|
||||
- Replace any surrogate code points with U+FFFD
|
||||
- Re-encode with errors='replace' to ensure valid UTF-8
|
||||
"""
|
||||
if not s:
|
||||
return ""
|
||||
s = _SURROGATE_RE.sub("\ufffd", s)
|
||||
return s.encode("utf-8", errors="replace").decode("utf-8")
|
||||
|
||||
|
||||
# Ensure no existing handlers
|
||||
for handler in logging.root.handlers[:]:
|
||||
logging.root.removeHandler(handler)
|
||||
|
||||
# Configure logging - use stdout
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
logging.root.addHandler(handler)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.info("Initializing server logging")
|
||||
_level_name = (os.environ.get("LOG_LEVEL") or "INFO").upper()
|
||||
_level = getattr(logging, _level_name, logging.INFO)
|
||||
logging.root.setLevel(_level)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.info("Initializing server logging, level=%s", _level_name)
|
||||
|
||||
# Initialize request ID logging
|
||||
init_logging_request_id()
|
||||
|
||||
|
||||
parser = Parser()
|
||||
def _resolve_images(images: dict, request_id: str, storage_map: dict | None = None) -> tuple[str, list]:
|
||||
"""Resolve document images with priority: shared storage > temp dir > inline bytes.
|
||||
|
||||
``images`` is a dict of {relative_path: raw_data} where raw_data is
|
||||
base64-encoded string or raw bytes.
|
||||
|
||||
def create_chunking_config(read_config: ReadConfig):
|
||||
"""Create ChunkingConfig from ReadConfig request.
|
||||
``storage_map`` is a dict from the request's config.image_storage map.
|
||||
When it contains a valid "provider" key, images are uploaded to shared object
|
||||
storage and storage_key is set (no inline bytes, saving gRPC message size).
|
||||
|
||||
Args:
|
||||
read_config: The read_config from the gRPC request
|
||||
|
||||
Returns:
|
||||
ChunkingConfig: Configured chunking configuration object
|
||||
Returns (image_dir_path, list[ImageRef]).
|
||||
"""
|
||||
# Extract chunking parameters
|
||||
chunk_size = read_config.chunk_size or 512
|
||||
chunk_overlap = read_config.chunk_overlap or 50
|
||||
# Convert protobuf RepeatedScalarFieldContainer to list for type compatibility
|
||||
separators = (
|
||||
list(read_config.separators) if read_config.separators else ["\n\n", "\n", "。"]
|
||||
)
|
||||
enable_multimodal = read_config.enable_multimodal or False
|
||||
import base64
|
||||
|
||||
logger.info(
|
||||
f"Using chunking config: size={chunk_size}, "
|
||||
f"overlap={chunk_overlap}, multimodal={enable_multimodal}"
|
||||
)
|
||||
if not images:
|
||||
return "", []
|
||||
|
||||
# Extract storage config
|
||||
sc = read_config.storage_config
|
||||
storage_config = {
|
||||
"provider": StorageProvider.Name(sc.provider),
|
||||
"region": sc.region,
|
||||
"bucket_name": sc.bucket_name,
|
||||
"access_key_id": sc.access_key_id,
|
||||
"secret_access_key": sc.secret_access_key,
|
||||
"app_id": sc.app_id,
|
||||
"path_prefix": sc.path_prefix,
|
||||
}
|
||||
logger.info(
|
||||
f"Using Storage config: provider={storage_config.get('provider')}, "
|
||||
f"bucket={storage_config['bucket_name']}"
|
||||
)
|
||||
|
||||
# Extract VLM config
|
||||
vlm_config = {
|
||||
"model_name": read_config.vlm_config.model_name,
|
||||
"base_url": read_config.vlm_config.base_url,
|
||||
"api_key": read_config.vlm_config.api_key,
|
||||
"interface_type": read_config.vlm_config.interface_type or "openai",
|
||||
mime_map = {
|
||||
".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
|
||||
".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
|
||||
}
|
||||
|
||||
logger.info(
|
||||
f"Using VLM config: model={vlm_config['model_name']}, "
|
||||
f"base_url={vlm_config['base_url']}, "
|
||||
f"interface_type={vlm_config['interface_type']}"
|
||||
)
|
||||
storage_client = None
|
||||
use_storage = False
|
||||
provider = (storage_map or {}).get("provider", "")
|
||||
if provider:
|
||||
try:
|
||||
from docreader.parser.storage import create_storage
|
||||
storage_client = create_storage(storage_map)
|
||||
use_storage = True
|
||||
logger.info("Using shared storage (%s) for image upload", provider)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to init shared storage, falling back to inline bytes: %s", e)
|
||||
|
||||
# Create and return ChunkingConfig
|
||||
return ChunkingConfig(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
separators=separators,
|
||||
enable_multimodal=enable_multimodal,
|
||||
storage_config=storage_config,
|
||||
vlm_config=vlm_config,
|
||||
)
|
||||
base_dir = CONFIG.image_output_dir
|
||||
output_dir = os.path.join(base_dir, request_id, "images")
|
||||
wrote_to_dir = False
|
||||
if not use_storage:
|
||||
try:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
wrote_to_dir = True
|
||||
except OSError:
|
||||
logger.warning("Cannot write to image output dir %s", output_dir)
|
||||
|
||||
refs = []
|
||||
for ref_path, b64data in images.items():
|
||||
try:
|
||||
img_bytes = base64.b64decode(b64data)
|
||||
except Exception:
|
||||
img_bytes = b64data.encode("utf-8") if isinstance(b64data, str) else b64data
|
||||
|
||||
fname = os.path.basename(ref_path) or f"{uuid.uuid4().hex}.png"
|
||||
ext = os.path.splitext(fname)[1].lower()
|
||||
mime = mime_map.get(ext, "application/octet-stream")
|
||||
|
||||
if use_storage and storage_client:
|
||||
storage_key = storage_client.upload_bytes(img_bytes, ext)
|
||||
if storage_key:
|
||||
refs.append(ImageRef(
|
||||
filename=fname,
|
||||
original_ref=ref_path,
|
||||
mime_type=mime,
|
||||
storage_key=storage_key,
|
||||
))
|
||||
continue
|
||||
logger.warning("Storage upload failed for %s, falling back to inline bytes", fname)
|
||||
|
||||
if wrote_to_dir:
|
||||
dest = os.path.join(output_dir, fname)
|
||||
try:
|
||||
with open(dest, "wb") as f:
|
||||
f.write(img_bytes)
|
||||
except OSError as e:
|
||||
logger.warning("Failed to write image %s: %s", dest, e)
|
||||
|
||||
refs.append(ImageRef(
|
||||
filename=fname,
|
||||
original_ref=ref_path,
|
||||
mime_type=mime,
|
||||
image_data=img_bytes,
|
||||
))
|
||||
|
||||
image_dir = os.path.join(base_dir, request_id) if wrote_to_dir else ""
|
||||
mode = "storage" if use_storage else ("dir+inline" if wrote_to_dir else "inline")
|
||||
logger.info("Resolved %d images (mode=%s)", len(refs), mode)
|
||||
return image_dir, refs
|
||||
|
||||
|
||||
class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
@@ -132,152 +144,90 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
super().__init__()
|
||||
self.parser = Parser()
|
||||
|
||||
def ReadFromFile(self, request: ReadFromFileRequest, context):
|
||||
# Get or generate request ID
|
||||
request_id = (
|
||||
request.request_id
|
||||
if hasattr(request, "request_id") and request.request_id
|
||||
else str(uuid.uuid4())
|
||||
)
|
||||
def Read(self, request: ReadRequest, context):
|
||||
"""Unified read: file mode (file_content set) or URL mode (url set)."""
|
||||
request_id = request.request_id or str(uuid.uuid4())
|
||||
is_url = bool(request.url)
|
||||
|
||||
# Use request ID context
|
||||
with request_id_context(request_id):
|
||||
try:
|
||||
# Get file type
|
||||
file_type = (
|
||||
request.file_type or os.path.splitext(request.file_name)[1][1:]
|
||||
)
|
||||
logger.info(
|
||||
f"ReadFromFile for file: {request.file_name}, type: {file_type}"
|
||||
)
|
||||
logger.info(f"File content size: {len(request.file_content)} bytes")
|
||||
cfg = request.config
|
||||
parser_engine = cfg.parser_engine if cfg else ""
|
||||
engine_overrides = dict(cfg.parser_engine_overrides) if cfg else {}
|
||||
storage_map = dict(cfg.image_storage) if cfg and cfg.image_storage else None
|
||||
|
||||
# Create chunking config
|
||||
chunking_config = create_chunking_config(request.read_config)
|
||||
if is_url:
|
||||
logger.info("Read(URL): url=%s", request.url)
|
||||
result = self.parser.parse_url(
|
||||
request.url,
|
||||
request.title,
|
||||
parser_engine=parser_engine,
|
||||
engine_overrides=engine_overrides,
|
||||
)
|
||||
source_desc = request.url
|
||||
else:
|
||||
file_type = (
|
||||
request.file_type or os.path.splitext(request.file_name)[1][1:]
|
||||
)
|
||||
logger.info(
|
||||
"Read(File): file=%s, type=%s, size=%d bytes",
|
||||
request.file_name, file_type, len(request.file_content),
|
||||
)
|
||||
result = self.parser.parse_file(
|
||||
request.file_name,
|
||||
file_type,
|
||||
request.file_content,
|
||||
parser_engine=parser_engine,
|
||||
engine_overrides=engine_overrides,
|
||||
)
|
||||
source_desc = request.file_name
|
||||
|
||||
# Parse file
|
||||
logger.info("Starting file parsing process")
|
||||
result = self.parser.parse_file(
|
||||
request.file_name, file_type, request.file_content, chunking_config
|
||||
)
|
||||
|
||||
if not result:
|
||||
error_msg = "Failed to parse file"
|
||||
if not result or not result.content:
|
||||
error_msg = f"Failed to parse: {source_desc}"
|
||||
logger.error(error_msg)
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(error_msg)
|
||||
return ReadResponse()
|
||||
|
||||
# Convert to protobuf message
|
||||
logger.info(
|
||||
f"Parsed file {request.file_name}, with {len(result.chunks)} chunks"
|
||||
)
|
||||
|
||||
# Build response, including image info
|
||||
response = ReadResponse(
|
||||
chunks=[
|
||||
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
|
||||
]
|
||||
)
|
||||
logger.info(f"Response size: {response.ByteSize()} bytes")
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error reading file: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(str(e))
|
||||
return ReadResponse(error=str(e))
|
||||
|
||||
def ReadFromURL(self, request: ReadFromURLRequest, context):
|
||||
# Get or generate request ID
|
||||
request_id = (
|
||||
request.request_id
|
||||
if hasattr(request, "request_id") and request.request_id
|
||||
else str(uuid.uuid4())
|
||||
)
|
||||
|
||||
# Use request ID context
|
||||
with request_id_context(request_id):
|
||||
try:
|
||||
logger.info(f"Received ReadFromURL request for URL: {request.url}")
|
||||
|
||||
# Create chunking config
|
||||
chunking_config = create_chunking_config(request.read_config)
|
||||
|
||||
# Parse URL
|
||||
logger.info("Starting URL parsing process")
|
||||
result = self.parser.parse_url(
|
||||
request.url, request.title, chunking_config
|
||||
)
|
||||
if not result:
|
||||
error_msg = "Failed to parse URL"
|
||||
logger.error(error_msg)
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(error_msg)
|
||||
return ReadResponse(error=error_msg)
|
||||
|
||||
# Convert to protobuf message, including image info
|
||||
logger.info(
|
||||
f"Parsed URL {request.url}, returning {len(result.chunks)} chunks"
|
||||
_c = to_valid_utf8_text
|
||||
image_dir, image_refs = _resolve_images(
|
||||
result.images, request_id, storage_map=storage_map
|
||||
)
|
||||
|
||||
response = ReadResponse(
|
||||
chunks=[
|
||||
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
|
||||
]
|
||||
markdown_content=_c(result.content),
|
||||
image_refs=image_refs,
|
||||
image_dir_path=image_dir,
|
||||
)
|
||||
logger.info(
|
||||
"Read response: content_len=%d, images=%d",
|
||||
len(result.content), len(image_refs),
|
||||
)
|
||||
logger.info(f"Response size: {response.ByteSize()} bytes")
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error reading URL: {str(e)}"
|
||||
error_msg = f"Error reading document: {e}"
|
||||
logger.error(error_msg)
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(str(e))
|
||||
logger.info("Traceback: %s", traceback.format_exc())
|
||||
return ReadResponse(error=str(e))
|
||||
|
||||
def _convert_chunk_to_proto(self, chunk):
|
||||
"""Convert internal Chunk object to protobuf Chunk message
|
||||
Ensures all string fields are valid UTF-8 for protobuf (no lone surrogates).
|
||||
"""
|
||||
# Clean helper for strings
|
||||
_c = to_valid_utf8_text
|
||||
|
||||
proto_chunk = Chunk(
|
||||
content=_c(getattr(chunk, "content", None)),
|
||||
seq=getattr(chunk, "seq", 0),
|
||||
start=getattr(chunk, "start", 0),
|
||||
end=getattr(chunk, "end", 0),
|
||||
)
|
||||
|
||||
# If chunk has images attribute and is not empty, add image info
|
||||
if hasattr(chunk, "images") and chunk.images:
|
||||
logger.info(
|
||||
f"Adding {len(chunk.images)} images to chunk {getattr(chunk, 'seq', 0)}"
|
||||
def ListEngines(self, request, context):
|
||||
overrides = dict(getattr(request, "config_overrides", None) or {})
|
||||
engines_data = registry.list_engines(overrides=overrides or None)
|
||||
engines = [
|
||||
ParserEngineInfo(
|
||||
name=e["name"],
|
||||
description=e["description"],
|
||||
file_types=e["file_types"],
|
||||
available=e.get("available", True),
|
||||
unavailable_reason=e.get("unavailable_reason", ""),
|
||||
)
|
||||
for img_info in chunk.images:
|
||||
# img_info expected as dict
|
||||
proto_image = Image(
|
||||
url=_c(img_info.get("cos_url", "")),
|
||||
caption=_c(img_info.get("caption", "")),
|
||||
ocr_text=_c(img_info.get("ocr_text", "")),
|
||||
original_url=_c(img_info.get("original_url", "")),
|
||||
start=int(img_info.get("start", 0) or 0),
|
||||
end=int(img_info.get("end", 0) or 0),
|
||||
)
|
||||
proto_chunk.images.append(proto_image)
|
||||
|
||||
return proto_chunk
|
||||
for e in engines_data
|
||||
]
|
||||
return ListEnginesResponse(engines=engines)
|
||||
|
||||
|
||||
def main():
|
||||
# Print effective env/config at startup
|
||||
config.print_config()
|
||||
|
||||
# Create server
|
||||
server = grpc.server(
|
||||
futures.ThreadPoolExecutor(max_workers=CONFIG.grpc_max_workers),
|
||||
options=[
|
||||
@@ -286,24 +236,18 @@ def main():
|
||||
],
|
||||
)
|
||||
|
||||
# Register services
|
||||
docreader_pb2_grpc.add_DocReaderServicer_to_server(DocReaderServicer(), server)
|
||||
|
||||
# Register health check service
|
||||
health_servicer = HealthServicer()
|
||||
health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
|
||||
|
||||
# Set listen address
|
||||
server.add_insecure_port(f"[::]:{CONFIG.grpc_port}")
|
||||
|
||||
# Start service
|
||||
server.start()
|
||||
|
||||
logger.info(f"Server started on port {CONFIG.grpc_port}")
|
||||
logger.info("Server started on port %d", CONFIG.grpc_port)
|
||||
logger.info("Server is ready to accept connections")
|
||||
|
||||
try:
|
||||
# Wait for service termination
|
||||
server.wait_for_termination()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Received termination signal, shutting down server")
|
||||
|
||||
@@ -1,27 +1,17 @@
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkingConfig:
|
||||
"""
|
||||
Configuration for text chunking process.
|
||||
Controls how documents are split into smaller pieces for processing.
|
||||
"""Legacy config kept for backward compatibility.
|
||||
|
||||
After the lightweight refactoring, chunking is done in Go.
|
||||
This class is only kept so existing parser constructors don't break.
|
||||
"""
|
||||
|
||||
# Maximum size of each chunk in tokens/chars
|
||||
chunk_size: int = 512
|
||||
|
||||
# Number of tokens/chars to overlap between chunks
|
||||
chunk_overlap: int = 50
|
||||
|
||||
# Text separators in order of priority
|
||||
separators: list[str] = field(default_factory=lambda: ["\n\n", "\n", "。"])
|
||||
|
||||
# Whether to enable multimodal processing (text + images)
|
||||
separators: list[str] | None = None
|
||||
enable_multimodal: bool = False
|
||||
|
||||
# Preferred field name going forward
|
||||
storage_config: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# VLM configuration for image captioning
|
||||
vlm_config: dict[str, str] = field(default_factory=dict)
|
||||
storage_config: dict[str, str] | None = None
|
||||
vlm_config: dict[str, str] | None = None
|
||||
|
||||
@@ -13,7 +13,6 @@ The parsers extract content from documents and can split them into
|
||||
meaningful chunks for further processing and indexing.
|
||||
"""
|
||||
|
||||
from .csv_parser import CSVParser
|
||||
from .doc_parser import DocParser
|
||||
from .docx2_parser import Docx2Parser
|
||||
from .excel_parser import ExcelParser
|
||||
@@ -21,19 +20,19 @@ from .image_parser import ImageParser
|
||||
from .markdown_parser import MarkdownParser
|
||||
from .parser import Parser
|
||||
from .pdf_parser import PDFParser
|
||||
from .text_parser import TextParser
|
||||
from .registry import ParserEngineRegistry, registry
|
||||
from .web_parser import WebParser
|
||||
|
||||
# Export public classes and modules
|
||||
__all__ = [
|
||||
"Docx2Parser", # Parser for .docx files (modern Word documents)
|
||||
"DocParser", # Parser for .doc files (legacy Word documents)
|
||||
"PDFParser", # Parser for PDF documents
|
||||
"MarkdownParser", # Parser for Markdown text files
|
||||
"TextParser", # Parser for plain text files
|
||||
"ImageParser", # Parser for images with text content
|
||||
"WebParser", # Parser for web pages
|
||||
"Parser", # Main parser factory that selects the appropriate parser
|
||||
"CSVParser", # Parser for CSV files
|
||||
"ExcelParser", # Parser for Excel files
|
||||
"Docx2Parser",
|
||||
"DocParser",
|
||||
"PDFParser",
|
||||
"MarkdownParser",
|
||||
"ImageParser",
|
||||
"WebParser",
|
||||
"Parser",
|
||||
"ExcelParser",
|
||||
"ParserEngineRegistry",
|
||||
"registry",
|
||||
]
|
||||
|
||||
+22
-968
File diff suppressed because it is too large
Load Diff
@@ -1,387 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
import ollama
|
||||
import requests
|
||||
|
||||
from docreader.config import CONFIG
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageUrl:
|
||||
"""Image URL data structure for caption requests."""
|
||||
|
||||
url: Optional[str] = None
|
||||
detail: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Content:
|
||||
"""Content data structure that can contain text or image URL."""
|
||||
|
||||
type: Optional[str] = None
|
||||
text: Optional[str] = None
|
||||
image_url: Optional[ImageUrl] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemMessage:
|
||||
"""System message for VLM model requests."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class UserMessage:
|
||||
"""User message for VLM model requests, can contain multiple content items."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: List[Content] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompletionRequest:
|
||||
"""Request structure for VLM model completion API."""
|
||||
|
||||
model: str
|
||||
temperature: float
|
||||
top_p: float
|
||||
messages: List[Union[SystemMessage, UserMessage]]
|
||||
user: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Model:
|
||||
"""Model identifier structure."""
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelsResp:
|
||||
"""Response structure for available models API."""
|
||||
|
||||
data: List[Model] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Message:
|
||||
"""Message structure in API response."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
tool_calls: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Choice:
|
||||
"""Choice structure in API response."""
|
||||
|
||||
message: Optional[Message] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Usage:
|
||||
"""Token usage information in API response."""
|
||||
|
||||
prompt_tokens: Optional[int] = 0
|
||||
total_tokens: Optional[int] = 0
|
||||
completion_tokens: Optional[int] = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CaptionChatResp:
|
||||
"""Response structure for caption chat API."""
|
||||
|
||||
id: Optional[str] = None
|
||||
created: Optional[int] = None
|
||||
model: Optional[Model] = None
|
||||
object: Optional[str] = None
|
||||
choices: List[Choice] = field(default_factory=list)
|
||||
usage: Optional[Usage] = None
|
||||
|
||||
@staticmethod
|
||||
def from_json(json_data: dict) -> "CaptionChatResp":
|
||||
"""
|
||||
Parse API response JSON into a CaptionChatResp object.
|
||||
|
||||
Args:
|
||||
json_data: The JSON response from the API
|
||||
|
||||
Returns:
|
||||
A parsed CaptionChatResp object
|
||||
"""
|
||||
logger.info("Parsing CaptionChatResp from JSON")
|
||||
# Manually parse nested fields with safe field extraction
|
||||
choices = []
|
||||
for choice in json_data.get("choices", []):
|
||||
message_data = choice.get("message", {})
|
||||
message = Message(
|
||||
role=message_data.get("role"),
|
||||
content=message_data.get("content"),
|
||||
tool_calls=message_data.get("tool_calls"),
|
||||
)
|
||||
choices.append(Choice(message=message))
|
||||
|
||||
# Handle usage with safe field extraction
|
||||
usage_data = json_data.get("usage", {})
|
||||
usage = None
|
||||
if usage_data:
|
||||
usage = Usage(
|
||||
prompt_tokens=usage_data.get("prompt_tokens", 0),
|
||||
total_tokens=usage_data.get("total_tokens", 0),
|
||||
completion_tokens=usage_data.get("completion_tokens", 0),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"Parsed {len(choices)} choices and usage data: {usage is not None}"
|
||||
)
|
||||
return CaptionChatResp(
|
||||
id=json_data.get("id"),
|
||||
created=json_data.get("created"),
|
||||
model=json_data.get("model"),
|
||||
object=json_data.get("object"),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
def choice_data(self) -> str:
|
||||
"""
|
||||
Extract the content from the first choice in the response.
|
||||
|
||||
Returns:
|
||||
The content string from the first choice, or empty string if no choices
|
||||
"""
|
||||
if (
|
||||
not self.choices
|
||||
or not self.choices[0]
|
||||
or not self.choices[0].message
|
||||
or not self.choices[0].message.content
|
||||
):
|
||||
logger.warning("No choices available in response")
|
||||
return ""
|
||||
logger.info("Retrieving content from first choice")
|
||||
return self.choices[0].message.content
|
||||
|
||||
|
||||
class Caption:
|
||||
"""
|
||||
Service for generating captions for images using a Vision Language Model.
|
||||
Uses an external API to process images and return textual descriptions.
|
||||
"""
|
||||
|
||||
def __init__(self, vlm_config: Optional[Dict[str, str]] = None):
|
||||
"""
|
||||
Initialize the Caption service with configuration
|
||||
from parameters or environment variables.
|
||||
"""
|
||||
logger.info("Initializing Caption service")
|
||||
# Default prompt for image captioning in Chinese: "Briefly describe the main content of the image"
|
||||
self.prompt = """简单凝炼的描述图片的主要内容"""
|
||||
# API request timeout in seconds
|
||||
self.timeout = 30
|
||||
|
||||
# Use provided VLM config if available,
|
||||
# otherwise fall back to environment variables
|
||||
if vlm_config and vlm_config.get("base_url") and vlm_config.get("model_name"):
|
||||
# Build completion URL from provided base URL
|
||||
self.completion_url = vlm_config.get("base_url", "") + "/chat/completions"
|
||||
self.model = vlm_config.get("model_name", "")
|
||||
self.api_key = vlm_config.get("api_key", "")
|
||||
# Interface type: "ollama" or "openai" (default)
|
||||
self.interface_type = vlm_config.get("interface_type", "openai").lower()
|
||||
else:
|
||||
# Fall back to environment variables if config not provided
|
||||
self.completion_url = CONFIG.vlm_model_base_url + "/chat/completions"
|
||||
self.model = CONFIG.vlm_model_name
|
||||
self.api_key = CONFIG.vlm_model_api_key
|
||||
self.interface_type = CONFIG.vlm_interface_type
|
||||
|
||||
# Validate interface type - must be either "ollama" or "openai"
|
||||
if self.interface_type not in ["ollama", "openai"]:
|
||||
logger.warning(
|
||||
f"Unknown interface type: {self.interface_type}, defaulting to openai"
|
||||
)
|
||||
self.interface_type = "openai"
|
||||
|
||||
logger.info(
|
||||
f"Configured with model: {self.model}, "
|
||||
f"endpoint: {self.completion_url}, interface: {self.interface_type}"
|
||||
)
|
||||
|
||||
def _call_caption_api(self, image_data: str) -> Optional[CaptionChatResp]:
|
||||
"""
|
||||
Call the Caption API to generate a description for the given image.
|
||||
|
||||
Args:
|
||||
image_data: URL of the image or base64 encoded image data
|
||||
|
||||
Returns:
|
||||
CaptionChatResp object if successful, None otherwise
|
||||
"""
|
||||
logger.info("Calling Caption API for image captioning")
|
||||
logger.info(f"Processing image data: {image_data[:50]}...")
|
||||
|
||||
# Route to appropriate API based on interface type
|
||||
if self.interface_type == "ollama":
|
||||
return self._call_ollama_api(image_data)
|
||||
else:
|
||||
return self._call_openai_api(image_data)
|
||||
|
||||
def _call_ollama_api(self, image_base64: str) -> Optional[CaptionChatResp]:
|
||||
"""Call Ollama API for image captioning using base64 encoded image data."""
|
||||
|
||||
# Extract host URL by removing the chat completions endpoint
|
||||
# Handle both "/v1/chat/completions" and "/chat/completions" patterns
|
||||
host = self.completion_url.replace("/v1/chat/completions", "").replace("/chat/completions", "")
|
||||
|
||||
# Initialize Ollama client with host and timeout
|
||||
client = ollama.Client(
|
||||
host=host,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
||||
try:
|
||||
logger.info(f"Calling Ollama API with model: {self.model}")
|
||||
|
||||
# Call Ollama API with base64 encoded image
|
||||
# Prompt: "Briefly describe the main content of the image"
|
||||
response = client.generate(
|
||||
model=self.model,
|
||||
prompt="简单凝炼的描述图片的主要内容",
|
||||
images=[image_base64], # Pass base64 encoded image data
|
||||
options={
|
||||
"temperature": 0.1
|
||||
}, # Low temperature for more deterministic output
|
||||
stream=False,
|
||||
)
|
||||
|
||||
# Construct response object in standard format
|
||||
caption_resp = CaptionChatResp(
|
||||
id="ollama_response",
|
||||
created=int(time.time()),
|
||||
model=Model(id=self.model),
|
||||
object="chat.completion",
|
||||
choices=[
|
||||
Choice(message=Message(role="assistant", content=response.response))
|
||||
],
|
||||
)
|
||||
|
||||
logger.info("Successfully received response from Ollama API")
|
||||
return caption_resp
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error calling Ollama API: {e}")
|
||||
return None
|
||||
|
||||
def _call_openai_api(self, image_base64: str) -> Optional[CaptionChatResp]:
|
||||
"""Call OpenAI-compatible API for image captioning."""
|
||||
logger.info(f"Calling OpenAI-compatible API with model: {self.model}")
|
||||
|
||||
# Construct user message with text prompt and base64 encoded image
|
||||
user_msg = UserMessage(
|
||||
role="user",
|
||||
content=[
|
||||
Content(type="text", text=self.prompt),
|
||||
Content(
|
||||
type="image_url",
|
||||
image_url=ImageUrl(
|
||||
url="data:image/png;base64," + image_base64, detail="auto"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# Build completion request with model parameters
|
||||
gpt_req = CompletionRequest(
|
||||
model=self.model,
|
||||
temperature=0.3, # Moderate randomness for balanced output
|
||||
top_p=0.8, # Nucleus sampling parameter
|
||||
messages=[user_msg],
|
||||
user="abc",
|
||||
)
|
||||
|
||||
# Set up HTTP headers for the API request
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
# Add authorization header if API key is provided
|
||||
if self.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.api_key}"
|
||||
|
||||
try:
|
||||
logger.info(
|
||||
f"Sending request to OpenAI-compatible API with model: {self.model}"
|
||||
)
|
||||
# Send POST request to the API endpoint
|
||||
response = requests.post(
|
||||
self.completion_url,
|
||||
data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4),
|
||||
headers=headers,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
# Check for successful response
|
||||
if response.status_code != 200:
|
||||
logger.error(
|
||||
f"OpenAI API returned non-200 status code: {response.status_code}"
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
logger.info(f"Received from OpenAI with status: {response.status_code}")
|
||||
logger.info("Converting response to CaptionChatResp object")
|
||||
# Parse JSON response into structured object
|
||||
caption_resp = CaptionChatResp.from_json(response.json())
|
||||
|
||||
if caption_resp.usage:
|
||||
logger.info(
|
||||
f"API usage: prompt_tokens={caption_resp.usage.prompt_tokens}, "
|
||||
f"completion_tokens={caption_resp.usage.completion_tokens}"
|
||||
)
|
||||
|
||||
return caption_resp
|
||||
except requests.exceptions.Timeout:
|
||||
logger.error("Timeout while calling OpenAI-compatible API after 30 seconds")
|
||||
return None
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Request error calling OpenAI-compatible API: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error calling OpenAI-compatible API: {e}")
|
||||
return None
|
||||
|
||||
def get_caption(self, image_data: str) -> str:
|
||||
"""
|
||||
Get a caption for the provided image data.
|
||||
|
||||
Args:
|
||||
image_data: URL of the image or base64 encoded image data
|
||||
|
||||
Returns:
|
||||
Caption text as string, or empty string if captioning failed
|
||||
"""
|
||||
logger.info("Getting caption for image")
|
||||
if not image_data or self.completion_url is None:
|
||||
logger.error("Image data is not set")
|
||||
return ""
|
||||
caption_resp = self._call_caption_api(image_data)
|
||||
if caption_resp:
|
||||
caption = caption_resp.choice_data()
|
||||
caption_length = len(caption)
|
||||
logger.info(f"Successfully generated caption of length {caption_length}")
|
||||
logger.info(
|
||||
f"Caption: {caption[:50]}..."
|
||||
if caption_length > 50
|
||||
else f"Caption: {caption}"
|
||||
)
|
||||
return caption
|
||||
logger.warning("Failed to get caption from Caption API")
|
||||
return ""
|
||||
@@ -1,101 +0,0 @@
|
||||
"""
|
||||
CSV Parser Module
|
||||
|
||||
This module provides a parser for CSV (Comma-Separated Values) files.
|
||||
It converts CSV data into a Document with structured chunks, where each row
|
||||
becomes a separate chunk with key-value pairs.
|
||||
"""
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from typing import List
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from docreader.models.document import Chunk, Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CSVParser(BaseParser):
|
||||
"""
|
||||
Parser for CSV files that converts tabular data into structured text.
|
||||
|
||||
This parser reads CSV content and transforms each row into a formatted string
|
||||
with column-value pairs. Each row is stored as a separate Chunk in the Document,
|
||||
allowing for granular access to individual records.
|
||||
|
||||
The output format for each row is:
|
||||
"column1: value1, column2: value2, column3: value3\n"
|
||||
|
||||
Usage:
|
||||
parser = CSVParser()
|
||||
with open("data.csv", "rb") as f:
|
||||
document = parser.parse_into_text(f.read())
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse CSV content into a Document with structured chunks.
|
||||
|
||||
Each row in the CSV is converted into a formatted string and stored as
|
||||
a separate Chunk. The chunks maintain sequential order and track their
|
||||
position in the overall document.
|
||||
|
||||
Args:
|
||||
content: Raw bytes content of the CSV file
|
||||
|
||||
Returns:
|
||||
Document: A Document object containing:
|
||||
- content: Full text with all rows concatenated
|
||||
- chunks: List of Chunk objects, one per CSV row
|
||||
|
||||
Note:
|
||||
Bad lines in the CSV are automatically skipped using pandas'
|
||||
on_bad_lines="skip" parameter.
|
||||
"""
|
||||
chunks: List[Chunk] = []
|
||||
text: List[str] = []
|
||||
start, end = 0, 0
|
||||
|
||||
# Read CSV content into a pandas DataFrame, skipping malformed lines
|
||||
df = pd.read_csv(BytesIO(content), on_bad_lines="skip")
|
||||
|
||||
# Process each row in the DataFrame
|
||||
for i, (idx, row) in enumerate(df.iterrows()):
|
||||
# Format row as "column: value" pairs separated by commas
|
||||
content_row = (
|
||||
",".join(
|
||||
f"{col.strip()}: {str(row[col]).strip()}" for col in df.columns
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
# Update end position for this chunk
|
||||
end += len(content_row)
|
||||
text.append(content_row)
|
||||
|
||||
# Create a chunk for this row with position tracking
|
||||
chunks.append(Chunk(content=content_row, seq=i, start=start, end=end))
|
||||
# Update start position for next chunk
|
||||
start = end
|
||||
|
||||
return Document(
|
||||
content="".join(text),
|
||||
chunks=chunks,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Example usage: Parse a CSV file and display its content
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
your_file = "/path/to/your/file.csv"
|
||||
parser = CSVParser()
|
||||
with open(your_file, "rb") as f:
|
||||
content = f.read()
|
||||
document = parser.parse_into_text(content)
|
||||
# Display full document content
|
||||
logger.error(document.content)
|
||||
|
||||
# Display individual chunks (rows)
|
||||
for chunk in document.chunks:
|
||||
logger.error(chunk.content)
|
||||
@@ -89,11 +89,33 @@ class DocxParser(BaseParser):
|
||||
logger.info(f"Setting max_workers to {max_workers} for document processing")
|
||||
|
||||
try:
|
||||
inline_images: Dict[str, str] = {}
|
||||
|
||||
def _inline_upload(local_path: str) -> str:
|
||||
"""Read temp image file, base64-encode, and return a ref path.
|
||||
|
||||
The Go-side ImageResolver (or main.py _resolve_images) handles
|
||||
actual storage upload from Document.images.
|
||||
"""
|
||||
import base64
|
||||
import uuid as _uuid
|
||||
|
||||
try:
|
||||
with open(local_path, "rb") as f:
|
||||
raw = f.read()
|
||||
ext = os.path.splitext(local_path)[1].lower() or ".png"
|
||||
ref = f"images/{_uuid.uuid4().hex}{ext}"
|
||||
inline_images[ref] = base64.b64encode(raw).decode()
|
||||
return ref
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to read temp image %s: %s", local_path, exc)
|
||||
return ""
|
||||
|
||||
logger.info(f"Starting Docx processing with max_pages={self.max_pages}")
|
||||
docx_processor = Docx(
|
||||
max_image_size=self.max_image_size,
|
||||
enable_multimodal=self.enable_multimodal,
|
||||
upload_file=self.storage.upload_file,
|
||||
max_image_size=1920,
|
||||
enable_multimodal=True,
|
||||
upload_file=_inline_upload,
|
||||
)
|
||||
all_lines, tables = docx_processor(
|
||||
binary=content,
|
||||
@@ -153,6 +175,7 @@ class DocxParser(BaseParser):
|
||||
f"generated {len(text)} characters of text"
|
||||
)
|
||||
|
||||
image_parts.update(inline_images)
|
||||
return DocumentModel(content=text, images=image_parts)
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing DOCX document: {str(e)}")
|
||||
|
||||
@@ -5,44 +5,24 @@ import os
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
|
||||
# Set up logger for this module
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ImageParser(BaseParser):
|
||||
"""
|
||||
Parser for image files with OCR capability.
|
||||
Extracts text from images and generates captions.
|
||||
"""Parser for standalone image files.
|
||||
|
||||
This parser handles image processing by:
|
||||
1. Uploading the image to storage
|
||||
2. Generating a descriptive caption
|
||||
3. Performing OCR to extract text content
|
||||
4. Returning a combined result with both text and image reference
|
||||
Returns the image as a markdown reference with the raw image data
|
||||
in Document.images so that the Go-side ImageResolver (or main.py's
|
||||
_resolve_images) can handle storage upload.
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse image content into markdown text
|
||||
:param content: bytes content of the image
|
||||
:return: Document object
|
||||
"""
|
||||
logger.info(f"Parsing image content, size: {len(content)} bytes")
|
||||
logger.info("Parsing image file=%s, size=%d bytes", self.file_name, len(content))
|
||||
|
||||
# Get file extension
|
||||
ext = os.path.splitext(self.file_name)[1].lower()
|
||||
ext = os.path.splitext(self.file_name)[1].lower() or ".png"
|
||||
ref_path = f"images/{self.file_name}"
|
||||
|
||||
# Upload image to storage
|
||||
image_url = self.storage.upload_bytes(content, file_ext=ext)
|
||||
logger.info(f"Successfully uploaded image, URL: {image_url[:50]}...")
|
||||
text = f""
|
||||
images = {ref_path: base64.b64encode(content).decode()}
|
||||
|
||||
if not image_url:
|
||||
logger.warning(f"Failed to upload image: {self.file_name}")
|
||||
return Document(content=f"{self.file_name}", images={})
|
||||
|
||||
# Generate markdown text
|
||||
text = f""
|
||||
images = {image_url: base64.b64encode(content).decode()}
|
||||
|
||||
# Create image object and add to map
|
||||
return Document(content=text, images=images)
|
||||
|
||||
@@ -348,23 +348,11 @@ class MarkdownImageUtil:
|
||||
|
||||
|
||||
class MarkdownImageBase64(BaseParser):
|
||||
"""Parser for extracting and uploading base64 images from Markdown.
|
||||
"""Parser for extracting base64 images from Markdown.
|
||||
|
||||
This parser:
|
||||
1. Extracts base64-encoded images from Markdown content
|
||||
2. Uploads the decoded images to storage
|
||||
3. Replaces base64 data with uploaded URLs
|
||||
4. Returns a Document with clean Markdown and image mappings
|
||||
|
||||
Requires:
|
||||
- self.storage: Storage backend for uploading images
|
||||
|
||||
Example:
|
||||
>>> parser = MarkdownImageBase64(storage=my_storage)
|
||||
>>> content = b""
|
||||
>>> doc = parser.parse_into_text(content)
|
||||
>>> print(doc.content)
|
||||

|
||||
Extracts base64-encoded images, replaces them with path references,
|
||||
and returns the raw image data in Document.images for the Go-side
|
||||
ImageResolver (or main.py _resolve_images) to handle storage.
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
@@ -372,41 +360,14 @@ class MarkdownImageBase64(BaseParser):
|
||||
self.image_helper = MarkdownImageUtil()
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""Parse Markdown and process base64 images.
|
||||
|
||||
Args:
|
||||
content: Raw Markdown content as bytes
|
||||
|
||||
Returns:
|
||||
Document with:
|
||||
- content: Markdown with base64 replaced by URLs
|
||||
- images: Dict mapping URLs to base64 strings
|
||||
"""
|
||||
# Convert byte content to string using universal decoding method
|
||||
text = endecode.decode_bytes(content)
|
||||
# Extract base64 images and replace with temporary paths
|
||||
text, img_b64 = self.image_helper.extract_base64(text, path_prefix="images")
|
||||
|
||||
# Final image mapping: URL -> base64 string (for Document)
|
||||
images: Dict[str, str] = {}
|
||||
# Temporary mapping: temp_path -> uploaded_URL (for replacement)
|
||||
image_replace: Dict[str, str] = {}
|
||||
for ipath, raw_bytes in img_b64.items():
|
||||
images[ipath] = base64.b64encode(raw_bytes).decode()
|
||||
|
||||
logger.debug(f"Uploading {len(img_b64)} images from markdown")
|
||||
# Upload each extracted image to storage
|
||||
for ipath, b64_bytes in img_b64.items():
|
||||
# Get file extension for proper MIME type
|
||||
ext = os.path.splitext(ipath)[1].lower()
|
||||
# Upload binary data and get back URL
|
||||
image_url = self.storage.upload_bytes(b64_bytes, ext)
|
||||
|
||||
# Map temp path to uploaded URL for replacement
|
||||
image_replace[ipath] = image_url
|
||||
# Store base64 string in final images dict
|
||||
images[image_url] = base64.b64encode(b64_bytes).decode()
|
||||
|
||||
# Replace temporary paths with actual uploaded URLs
|
||||
text = self.image_helper.replace_path(text, image_replace)
|
||||
logger.debug("Extracted %d base64 images from markdown", len(images))
|
||||
return Document(content=text, images=images)
|
||||
|
||||
|
||||
|
||||
@@ -1,329 +0,0 @@
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Dict, Optional
|
||||
|
||||
import markdownify
|
||||
import requests
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.chain_parser import PipelineParser
|
||||
from docreader.parser.markdown_parser import MarkdownImageUtil, MarkdownTableFormatter
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StdMinerUParser(BaseParser):
|
||||
"""
|
||||
Standard MinerU Parser for document parsing.
|
||||
|
||||
This parser uses MinerU API to parse documents (especially PDFs) into markdown format,
|
||||
with support for tables, formulas, and images extraction.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
enable_markdownify: bool = True,
|
||||
mineru_endpoint: Optional[str] = None, # Added: 支持传入自定义 endpoint
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Initialize MinerU parser.
|
||||
|
||||
Args:
|
||||
enable_markdownify: Whether to convert HTML tables to markdown format
|
||||
mineru_endpoint: MinerU API endpoint URL
|
||||
**kwargs: Additional arguments passed to BaseParser
|
||||
"""
|
||||
super().__init__(**kwargs)
|
||||
# Get MinerU endpoint from environment variable or parameter
|
||||
# Modified: 优先使用传入的参数,否则使用 Config
|
||||
base_url = mineru_endpoint if mineru_endpoint else CONFIG.mineru_endpoint
|
||||
self.minerU = base_url.rstrip("/") if base_url else ""
|
||||
|
||||
self.enable_markdownify = enable_markdownify
|
||||
# Helper for processing markdown images
|
||||
self.image_helper = MarkdownImageUtil()
|
||||
# Pattern to match base64 encoded images
|
||||
self.base64_pattern = re.compile(r"data:image/(\w+);base64,(.*)")
|
||||
# Check if MinerU API is available
|
||||
self.enable = self.ping()
|
||||
|
||||
def ping(self, timeout: int = 5) -> bool:
|
||||
"""
|
||||
Check if MinerU API is available.
|
||||
|
||||
Args:
|
||||
timeout: Request timeout in seconds
|
||||
|
||||
Returns:
|
||||
True if API is available, False otherwise
|
||||
"""
|
||||
try:
|
||||
response = requests.get(
|
||||
self.minerU + "/docs", timeout=timeout, allow_redirects=True
|
||||
)
|
||||
response.raise_for_status()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse document content into text using MinerU API.
|
||||
|
||||
Args:
|
||||
content: Raw document content in bytes
|
||||
|
||||
Returns:
|
||||
Document object containing parsed text and images
|
||||
"""
|
||||
if not self.enable:
|
||||
logger.debug("MinerU API is not enabled")
|
||||
return Document()
|
||||
|
||||
logger.info(f"Parsing scanned PDF via MinerU API (size: {len(content)} bytes)")
|
||||
md_content: str = ""
|
||||
images_b64: Dict[str, str] = {}
|
||||
try:
|
||||
# Call MinerU API to parse document
|
||||
response = requests.post(
|
||||
url=self.minerU + "/file_parse",
|
||||
data={
|
||||
"return_md": True, # Return markdown content
|
||||
"return_images": True, # Return extracted images
|
||||
"lang_list": ["ch", "en"], # Support Chinese and English
|
||||
"table_enable": True, # Enable table parsing
|
||||
"formula_enable": True, # Enable formula parsing
|
||||
"parse_method": "auto", # Auto detect parsing method
|
||||
"start_page_id": 0, # Start from first page
|
||||
"end_page_id": 99999, # Parse all pages
|
||||
"backend": "pipeline", # Use pipeline backend
|
||||
"response_format_zip": False, # Return JSON instead of ZIP
|
||||
"return_middle_json": False, # Don't return intermediate JSON
|
||||
"return_model_output": False, # Don't return model output
|
||||
"return_content_list": False, # Don't return content list
|
||||
},
|
||||
files={"files": content},
|
||||
timeout=1000,
|
||||
)
|
||||
response.raise_for_status()
|
||||
result = response.json()["results"]["files"]
|
||||
md_content = result["md_content"]
|
||||
images_b64 = result.get("images", {})
|
||||
except Exception as e:
|
||||
logger.error(f"MinerU parsing failed: {e}", exc_info=True)
|
||||
return Document()
|
||||
|
||||
# Convert HTML tables in markdown to markdown table format
|
||||
if self.enable_markdownify:
|
||||
logger.debug("Converting HTML to Markdown")
|
||||
md_content = markdownify.markdownify(md_content)
|
||||
|
||||
images = {}
|
||||
image_replace = {}
|
||||
# Filter images that are actually used in markdown content
|
||||
# Some images in images_b64 may not be referenced in md_content
|
||||
# (e.g., images embedded in tables), so we need to filter them
|
||||
for ipath, b64_str in images_b64.items():
|
||||
# Skip images that are not referenced in markdown content
|
||||
if f"images/{ipath}" not in md_content:
|
||||
logger.debug(f"Image {ipath} not used in markdown")
|
||||
continue
|
||||
# Parse base64 image data
|
||||
match = self.base64_pattern.match(b64_str)
|
||||
if match:
|
||||
# Extract image format (e.g., png, jpg)
|
||||
file_ext = match.group(1)
|
||||
# Extract base64 encoded data
|
||||
b64_str = match.group(2)
|
||||
|
||||
# Decode base64 string to bytes
|
||||
image_bytes = endecode.encode_image(b64_str, errors="ignore")
|
||||
if not image_bytes:
|
||||
logger.error("Failed to decode base64 image skip it")
|
||||
continue
|
||||
|
||||
# Upload image to storage and get URL
|
||||
image_url = self.storage.upload_bytes(
|
||||
image_bytes, file_ext=f".{file_ext}"
|
||||
)
|
||||
|
||||
# Store image mapping for later use
|
||||
images[image_url] = b64_str
|
||||
# Prepare replacement mapping for markdown content
|
||||
image_replace[f"images/{ipath}"] = image_url
|
||||
|
||||
logger.info(f"Replaced {len(image_replace)} images in markdown")
|
||||
# Replace image paths in markdown with uploaded URLs
|
||||
text = self.image_helper.replace_path(md_content, image_replace)
|
||||
|
||||
logger.info(
|
||||
f"Successfully parsed PDF, text: {len(text)}, images: {len(images)}"
|
||||
)
|
||||
return Document(content=text, images=images)
|
||||
|
||||
|
||||
# Added: 新增 MinerUCloudParser 类,支持异步任务提交
|
||||
class MinerUCloudParser(StdMinerUParser):
|
||||
"""
|
||||
MinerU Parser for REMOTE/CLOUD API (Asynchronous).
|
||||
Uses the /submit -> /status -> /result workflow.
|
||||
"""
|
||||
|
||||
SUBMIT_TIMEOUT = 30
|
||||
POLL_INTERVAL = 2
|
||||
MAX_WAIT_TIME = 600
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse document content using Cloud MinerU API (Async/Polling).
|
||||
"""
|
||||
if not self.enable:
|
||||
return Document()
|
||||
|
||||
logger.info(f"Parsing PDF via Cloud MinerU API (size: {len(content)} bytes)")
|
||||
|
||||
try:
|
||||
# --- Step 1: Submit Task ---
|
||||
submit_url = f"{self.minerU}/submit"
|
||||
logger.info(f"Submitting task to {submit_url}")
|
||||
|
||||
response = requests.post(
|
||||
url=submit_url,
|
||||
files={"files": content},
|
||||
data={
|
||||
"enable_formula": "true",
|
||||
"enable_table": "true",
|
||||
"layout_model": "doclayout_yolo",
|
||||
"backend": "pipeline",
|
||||
},
|
||||
timeout=self.SUBMIT_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
# Robust task_id extraction
|
||||
resp_data = response.json()
|
||||
task_id = resp_data.get("task_id") or resp_data.get("data", {}).get("task_id")
|
||||
|
||||
if not task_id:
|
||||
raise ValueError(f"No task_id in response: {resp_data}")
|
||||
|
||||
logger.info(f"Task submitted, ID: {task_id}, waiting for completion...")
|
||||
|
||||
# --- Step 2: Poll Status ---
|
||||
start_time = time.time()
|
||||
|
||||
while True:
|
||||
if time.time() - start_time > self.MAX_WAIT_TIME:
|
||||
raise TimeoutError(f"Task {task_id} timed out after {self.MAX_WAIT_TIME}s")
|
||||
|
||||
try:
|
||||
status_resp = requests.get(
|
||||
f"{self.minerU}/status/{task_id}",
|
||||
timeout=10
|
||||
)
|
||||
status_resp.raise_for_status()
|
||||
status_data = status_resp.json()
|
||||
except requests.RequestException as e:
|
||||
logger.warning(f"Status check failed for {task_id}: {e}. Retrying...")
|
||||
time.sleep(self.POLL_INTERVAL)
|
||||
continue
|
||||
|
||||
state = status_data.get("status") or status_data.get("state")
|
||||
|
||||
if state in ["done", "success"]:
|
||||
break
|
||||
elif state == "failed":
|
||||
error_msg = status_data.get("error") or "Unknown error"
|
||||
raise RuntimeError(f"Task {task_id} failed: {error_msg}")
|
||||
else:
|
||||
time.sleep(self.POLL_INTERVAL)
|
||||
|
||||
# --- Step 3: Get Result ---
|
||||
result_resp = requests.get(
|
||||
f"{self.minerU}/result/{task_id}",
|
||||
timeout=30
|
||||
)
|
||||
result_resp.raise_for_status()
|
||||
result_json = result_resp.json()
|
||||
|
||||
# Normalize result data
|
||||
result_data = result_json.get("result", result_json)
|
||||
|
||||
md_content = result_data.get("md_content", "")
|
||||
images_b64 = result_data.get("images", {})
|
||||
|
||||
# 使用父类的方法处理图片和Markdown转换 (复用现有逻辑)
|
||||
|
||||
# Convert HTML tables
|
||||
if self.enable_markdownify:
|
||||
md_content = markdownify.markdownify(md_content)
|
||||
|
||||
images = {}
|
||||
image_replace = {}
|
||||
|
||||
for ipath, b64_str in images_b64.items():
|
||||
if f"images/{ipath}" not in md_content:
|
||||
continue
|
||||
match = self.base64_pattern.match(b64_str)
|
||||
if match:
|
||||
file_ext = match.group(1)
|
||||
b64_str_clean = match.group(2)
|
||||
image_bytes = endecode.encode_image(b64_str_clean, errors="ignore")
|
||||
if not image_bytes: continue
|
||||
|
||||
if self.storage:
|
||||
image_url = self.storage.upload_bytes(image_bytes, file_ext=f".{file_ext}")
|
||||
images[image_url] = b64_str_clean
|
||||
image_replace[f"images/{ipath}"] = image_url
|
||||
|
||||
if image_replace:
|
||||
md_content = self.image_helper.replace_path(md_content, image_replace)
|
||||
|
||||
return Document(content=md_content, images=images)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Cloud MinerU parsing failed: {e}", exc_info=True)
|
||||
return Document()
|
||||
|
||||
|
||||
class MinerUParser(PipelineParser):
|
||||
"""
|
||||
MinerU Parser with pipeline processing.
|
||||
|
||||
This parser combines StdMinerUParser for document parsing and
|
||||
MarkdownTableFormatter for table formatting in a pipeline.
|
||||
"""
|
||||
|
||||
_parser_cls = (StdMinerUParser, MarkdownTableFormatter)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import os
|
||||
|
||||
# Example usage for testing
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
# Configure your file path and MinerU endpoint
|
||||
your_file = "/path/to/your/file.pdf"
|
||||
|
||||
# Added: 修改为 Localhost 方便测试
|
||||
test_endpoint = "http://localhost:9987"
|
||||
os.environ["MINERU_ENDPOINT"] = test_endpoint
|
||||
|
||||
# Create parser instance
|
||||
# Modified: 传入 endpoint
|
||||
parser = MinerUParser(mineru_endpoint=test_endpoint)
|
||||
|
||||
# Parse PDF file
|
||||
if os.path.exists(your_file):
|
||||
with open(your_file, "rb") as f:
|
||||
content = f.read()
|
||||
document = parser.parse_into_text(content)
|
||||
logger.error(document.content)
|
||||
else:
|
||||
print(f"File not found: {your_file}")
|
||||
+42
-138
@@ -1,178 +1,82 @@
|
||||
import logging
|
||||
from typing import Dict, Type
|
||||
from typing import Any, Optional
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.models.document import Document
|
||||
from docreader.models.read_config import ChunkingConfig
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.csv_parser import CSVParser
|
||||
from docreader.parser.doc_parser import DocParser
|
||||
from docreader.parser.docx2_parser import Docx2Parser
|
||||
from docreader.parser.excel_parser import ExcelParser
|
||||
from docreader.parser.image_parser import ImageParser
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
from docreader.parser.pdf_parser import PDFParser
|
||||
from docreader.parser.text_parser import TextParser
|
||||
from docreader.parser.registry import registry
|
||||
from docreader.parser.web_parser import WebParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Parser:
|
||||
"""
|
||||
Document parser facade that integrates all specialized parsers.
|
||||
Provides a unified interface for parsing various document types.
|
||||
"""Document parser facade (lightweight version).
|
||||
|
||||
Converts files/URLs to markdown + image references.
|
||||
No chunking, no storage, no OCR, no VLM.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Initialize all parser types - maps file extensions to their corresponding parser classes
|
||||
self.parsers: Dict[str, Type[BaseParser]] = {
|
||||
# Document formats
|
||||
"docx": Docx2Parser,
|
||||
"doc": DocParser,
|
||||
"pdf": PDFParser,
|
||||
"md": MarkdownParser,
|
||||
"txt": TextParser,
|
||||
# Image formats - all use the same ImageParser
|
||||
"jpg": ImageParser,
|
||||
"jpeg": ImageParser,
|
||||
"png": ImageParser,
|
||||
"gif": ImageParser,
|
||||
"bmp": ImageParser,
|
||||
"tiff": ImageParser,
|
||||
"webp": ImageParser,
|
||||
# Alternative markdown extension
|
||||
"markdown": MarkdownParser,
|
||||
# Spreadsheet formats
|
||||
"csv": CSVParser,
|
||||
"xlsx": ExcelParser,
|
||||
"xls": ExcelParser,
|
||||
}
|
||||
self.registry = registry
|
||||
logger.info(
|
||||
"Parser initialized with %d parsers: %s",
|
||||
len(self.parsers),
|
||||
", ".join(self.parsers.keys()),
|
||||
"Parser initialized with engines: %s",
|
||||
", ".join(self.registry.get_engine_names()),
|
||||
)
|
||||
|
||||
def get_parser(self, file_type: str) -> Type[BaseParser]:
|
||||
"""
|
||||
Get parser class for the specified file type.
|
||||
|
||||
Args:
|
||||
file_type: The file extension or type identifier
|
||||
|
||||
Returns:
|
||||
Parser class for the file type, or None if unsupported
|
||||
"""
|
||||
# Look up parser by file type (case-insensitive)
|
||||
parser = self.parsers.get(file_type.lower())
|
||||
if not parser:
|
||||
# Raise error if file type is not supported
|
||||
raise ValueError(f"Unsupported file type: {file_type}")
|
||||
return parser
|
||||
|
||||
def parse_file(
|
||||
self,
|
||||
file_name: str,
|
||||
file_type: str,
|
||||
content: bytes,
|
||||
config: ChunkingConfig,
|
||||
parser_engine: Optional[str] = None,
|
||||
engine_overrides: Optional[dict[str, Any]] = None,
|
||||
) -> Document:
|
||||
"""
|
||||
Parse file content using appropriate parser based on file type.
|
||||
|
||||
Args:
|
||||
file_name: Name of the file being parsed
|
||||
file_type: Type/extension of the file
|
||||
content: Raw file content as bytes
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing file: {file_name} with type: {file_type}")
|
||||
"""Parse file content to markdown."""
|
||||
engine = parser_engine or ""
|
||||
overrides = engine_overrides or {}
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, "
|
||||
f"overlap={config.chunk_overlap}, "
|
||||
f"multimodal={config.enable_multimodal}"
|
||||
"Parsing file: %s, type: %s, engine: %s",
|
||||
file_name,
|
||||
file_type,
|
||||
engine or "builtin",
|
||||
)
|
||||
|
||||
# Get appropriate parser class for the file type
|
||||
cls = self.get_parser(file_type)
|
||||
|
||||
# Create parser instance with configuration
|
||||
logger.info(f"Creating parser instance for {file_type} file")
|
||||
cls = self.registry.get_parser_class(engine, file_type)
|
||||
logger.info(
|
||||
"Creating %s parser instance for %s file",
|
||||
cls.__name__,
|
||||
file_type,
|
||||
)
|
||||
parser = cls(
|
||||
file_name=file_name,
|
||||
file_type=file_type,
|
||||
chunk_size=config.chunk_size, # Size of each text chunk
|
||||
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
|
||||
separators=config.separators, # Text separators for chunking
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
chunking_config=config, # Pass the entire chunking config for advanced options
|
||||
max_concurrent_tasks=CONFIG.image_max_concurrent,
|
||||
ocr_backend=CONFIG.ocr_backend,
|
||||
**overrides,
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
|
||||
# Execute the parsing process
|
||||
logger.info("Starting to parse file content, size: %d bytes", len(content))
|
||||
result = parser.parse(content)
|
||||
|
||||
# Validate parsing results and log warnings if needed
|
||||
if not result.content:
|
||||
logger.warning(f"Parser returned empty content for file: {file_name}")
|
||||
elif not result.chunks:
|
||||
logger.warning(f"Parser returned empty chunks for file: {file_name}")
|
||||
elif result.chunks[0]:
|
||||
# Log first chunk size for debugging
|
||||
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
|
||||
logger.info(f"Parsed file {file_name}, with {len(result.chunks)} chunks")
|
||||
logger.warning("Parser returned empty content for file: %s", file_name)
|
||||
logger.info(
|
||||
"Parsed file %s, content length=%d", file_name, len(result.content)
|
||||
)
|
||||
return result
|
||||
|
||||
def parse_url(self, url: str, title: str, config: ChunkingConfig) -> Document:
|
||||
"""
|
||||
Parse content from a URL using the WebParser.
|
||||
|
||||
Args:
|
||||
url: URL to parse
|
||||
title: Title of the webpage (for metadata)
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing URL: {url}, title: {title}")
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, "
|
||||
f"overlap={config.chunk_overlap}, multimodal={config.enable_multimodal}"
|
||||
)
|
||||
|
||||
# Create web parser instance with configuration
|
||||
logger.info("Creating WebParser instance")
|
||||
parser = WebParser(
|
||||
title=title, # Webpage title for metadata
|
||||
chunk_size=config.chunk_size, # Size of each text chunk
|
||||
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
|
||||
separators=config.separators, # Text separators for chunking
|
||||
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
|
||||
max_image_size=1920, # Limit image size to 1920px for performance
|
||||
chunking_config=config, # Pass the entire chunking config
|
||||
max_concurrent_tasks=CONFIG.image_max_concurrent,
|
||||
ocr_backend=CONFIG.ocr_backend,
|
||||
)
|
||||
def parse_url(
|
||||
self,
|
||||
url: str,
|
||||
title: str,
|
||||
parser_engine: Optional[str] = None,
|
||||
engine_overrides: Optional[dict[str, Any]] = None,
|
||||
) -> Document:
|
||||
"""Parse content from a URL to markdown."""
|
||||
logger.info("Parsing URL: %s, title: %s", url, title)
|
||||
|
||||
parser = WebParser(title=title)
|
||||
logger.info("Starting to parse URL content")
|
||||
# Parse URL content (encode URL string to bytes as required by parser interface)
|
||||
result = parser.parse(url.encode())
|
||||
|
||||
# Validate parsing results and log warnings if needed
|
||||
if not result.content:
|
||||
logger.warning(f"Parser returned empty content for url: {url}")
|
||||
elif not result.chunks:
|
||||
logger.warning(f"Parser returned empty chunks for url: {url}")
|
||||
elif result.chunks[0]:
|
||||
# Log first chunk size for debugging
|
||||
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
|
||||
logger.info(f"Parsed url {url}, with {len(result.chunks)} chunks")
|
||||
logger.warning("Parser returned empty content for url: %s", url)
|
||||
logger.info("Parsed url %s, content length=%d", url, len(result.content))
|
||||
return result
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from docreader.parser.chain_parser import FirstParser
|
||||
from docreader.parser.markitdown_parser import MarkitdownParser
|
||||
from docreader.parser.mineru_parser import MinerUParser
|
||||
|
||||
|
||||
class PDFParser(FirstParser):
|
||||
@@ -13,4 +12,4 @@ class PDFParser(FirstParser):
|
||||
The first successful parser result will be returned.
|
||||
"""
|
||||
# Parser classes to try in order (chain of responsibility pattern)
|
||||
_parser_cls = (MinerUParser, MarkitdownParser)
|
||||
_parser_cls = (MarkitdownParser,)
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import logging
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Type
|
||||
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.parser.doc_parser import DocParser
|
||||
from docreader.parser.docx2_parser import Docx2Parser
|
||||
from docreader.parser.excel_parser import ExcelParser
|
||||
from docreader.parser.image_parser import ImageParser
|
||||
from docreader.parser.markdown_parser import MarkdownParser
|
||||
from docreader.parser.markitdown_parser import MarkitdownParser
|
||||
from docreader.parser.pdf_parser import PDFParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BUILTIN_ENGINE = "builtin"
|
||||
|
||||
|
||||
class ParserEngineRegistry:
|
||||
"""Registry for parser engines.
|
||||
|
||||
Each engine maps file extensions to parser classes.
|
||||
When a requested engine doesn't support a file type, the registry
|
||||
falls back to the builtin engine automatically.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._engines: Dict[str, Dict[str, Type[BaseParser]]] = {}
|
||||
self._descriptions: Dict[str, str] = {}
|
||||
self._check_available: Dict[str, Callable[..., Tuple[bool, str]]] = {}
|
||||
self._unavailable_hint: Dict[str, str] = {}
|
||||
|
||||
def register(
|
||||
self,
|
||||
name: str,
|
||||
file_types: Dict[str, Type[BaseParser]],
|
||||
description: str = "",
|
||||
check_available: Callable[..., Tuple[bool, str]] | None = None,
|
||||
unavailable_hint: str = "",
|
||||
):
|
||||
self._engines[name] = file_types
|
||||
self._descriptions[name] = description
|
||||
if check_available is not None:
|
||||
self._check_available[name] = check_available
|
||||
self._unavailable_hint[name] = unavailable_hint
|
||||
logger.info(
|
||||
"Registered parser engine '%s' with file types: %s",
|
||||
name,
|
||||
", ".join(file_types.keys()),
|
||||
)
|
||||
|
||||
def get_parser_class(self, engine: str, file_type: str) -> Type[BaseParser]:
|
||||
"""Resolve parser class for the given engine and file type.
|
||||
|
||||
Falls back to builtin engine when the requested engine doesn't
|
||||
support the file type.
|
||||
"""
|
||||
ft = file_type.lower()
|
||||
|
||||
if engine and engine in self._engines:
|
||||
cls = self._engines[engine].get(ft)
|
||||
if cls:
|
||||
logger.info("Using engine '%s' for file type '%s'", engine, ft)
|
||||
return cls
|
||||
logger.info(
|
||||
"Engine '%s' does not support '%s', falling back to builtin",
|
||||
engine,
|
||||
ft,
|
||||
)
|
||||
|
||||
builtin = self._engines.get(BUILTIN_ENGINE, {})
|
||||
cls = builtin.get(ft)
|
||||
if cls:
|
||||
return cls
|
||||
|
||||
raise ValueError(f"Unsupported file type: {file_type}")
|
||||
|
||||
def list_engines(self, overrides: Optional[Dict[str, str]] = None) -> List[Dict]:
|
||||
"""Return metadata for all registered engines, including availability.
|
||||
|
||||
Args:
|
||||
overrides: tenant-level config overrides (e.g. mineru_endpoint, mineru_api_key)
|
||||
forwarded to each engine's check_available function.
|
||||
"""
|
||||
result = []
|
||||
for name, parsers in self._engines.items():
|
||||
available = True
|
||||
unavailable_reason = ""
|
||||
check = self._check_available.get(name)
|
||||
if check is not None:
|
||||
try:
|
||||
available, unavailable_reason = check(overrides)
|
||||
except Exception as e:
|
||||
available = False
|
||||
unavailable_reason = str(e) or self._unavailable_hint.get(name, "")
|
||||
if not available and not unavailable_reason:
|
||||
unavailable_reason = self._unavailable_hint.get(name, "不可用")
|
||||
result.append(
|
||||
{
|
||||
"name": name,
|
||||
"description": self._descriptions.get(name, ""),
|
||||
"file_types": sorted(parsers.keys()),
|
||||
"available": available,
|
||||
"unavailable_reason": unavailable_reason,
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
def get_engine_names(self) -> List[str]:
|
||||
return list(self._engines.keys())
|
||||
|
||||
|
||||
def _build_default_registry() -> ParserEngineRegistry:
|
||||
"""Create and populate the default registry with all known engines."""
|
||||
reg = ParserEngineRegistry()
|
||||
|
||||
_image_types = {
|
||||
ext: ImageParser for ext in ("jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp")
|
||||
}
|
||||
|
||||
reg.register(
|
||||
BUILTIN_ENGINE,
|
||||
{
|
||||
"docx": Docx2Parser,
|
||||
"doc": DocParser,
|
||||
"pdf": PDFParser,
|
||||
"md": MarkdownParser,
|
||||
"markdown": MarkdownParser,
|
||||
"xlsx": ExcelParser,
|
||||
"xls": ExcelParser,
|
||||
**_image_types,
|
||||
},
|
||||
description="内置解析引擎",
|
||||
)
|
||||
|
||||
reg.register(
|
||||
"markitdown",
|
||||
{
|
||||
"md": MarkitdownParser,
|
||||
"markdown": MarkitdownParser,
|
||||
"pdf": MarkitdownParser,
|
||||
"docx": MarkitdownParser,
|
||||
"doc": MarkitdownParser,
|
||||
"pptx": MarkitdownParser,
|
||||
"ppt": MarkitdownParser,
|
||||
"xlsx": MarkitdownParser,
|
||||
"xls": MarkitdownParser,
|
||||
"csv": MarkitdownParser,
|
||||
},
|
||||
description="MarkItDown 解析引擎(微软 MarkItDown 库)",
|
||||
)
|
||||
|
||||
# NOTE: Engine listing is managed by Go-side engine registry
|
||||
# (docparser.ListAllEngines). The Python list_engines method is kept for
|
||||
# backward compatibility with the gRPC ListEngines RPC but the Go app
|
||||
# no longer calls it. MinerU engines are handled natively by Go.
|
||||
|
||||
return reg
|
||||
|
||||
|
||||
registry = _build_default_registry()
|
||||
+98
-222
@@ -5,96 +5,70 @@ import os
|
||||
import traceback
|
||||
import uuid
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict
|
||||
from typing import Dict, Optional
|
||||
|
||||
from minio import Minio
|
||||
from qcloud_cos import CosConfig, CosS3Client
|
||||
|
||||
from docreader.config import CONFIG
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _cfg(storage_config: Optional[Dict], key: str, *env_keys: str, default: str = "") -> str:
|
||||
"""Read a value from storage_config dict, falling back to env vars."""
|
||||
if storage_config:
|
||||
v = storage_config.get(key, "")
|
||||
if v:
|
||||
return str(v)
|
||||
for ek in env_keys:
|
||||
v = os.environ.get(ek, "")
|
||||
if v:
|
||||
return v
|
||||
return default
|
||||
|
||||
|
||||
class Storage(ABC):
|
||||
"""Abstract base class for object storage operations"""
|
||||
|
||||
@abstractmethod
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to object storage
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to object storage
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class CosStorage(Storage):
|
||||
"""Tencent Cloud COS storage implementation"""
|
||||
|
||||
def __init__(self, storage_config=None):
|
||||
"""Initialize COS storage
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration
|
||||
"""
|
||||
def __init__(self, storage_config: Optional[Dict] = None):
|
||||
self.storage_config = storage_config
|
||||
self.client, self.bucket_name, self.region, self.prefix = (
|
||||
self._init_cos_client()
|
||||
)
|
||||
|
||||
def _init_cos_client(self):
|
||||
"""Initialize Tencent Cloud COS client"""
|
||||
try:
|
||||
# Use provided COS config if available,
|
||||
# otherwise fall back to environment variables
|
||||
if self.storage_config and self.storage_config.get("access_key_id") != "":
|
||||
cos_config = self.storage_config
|
||||
secret_id = cos_config.get("access_key_id")
|
||||
secret_key = cos_config.get("secret_access_key")
|
||||
region = cos_config.get("region")
|
||||
bucket_name = cos_config.get("bucket_name")
|
||||
appid = cos_config.get("app_id")
|
||||
prefix = cos_config.get("path_prefix", "")
|
||||
else:
|
||||
# Get COS configuration from environment variables
|
||||
secret_id = CONFIG.cos_secret_id
|
||||
secret_key = CONFIG.cos_secret_key
|
||||
region = CONFIG.cos_region
|
||||
bucket_name = CONFIG.cos_bucket_name
|
||||
appid = CONFIG.cos_app_id
|
||||
prefix = CONFIG.cos_path_prefix
|
||||
|
||||
enable_old_domain = CONFIG.cos_enable_old_domain
|
||||
sc = self.storage_config
|
||||
secret_id = _cfg(sc, "access_key_id", "COS_SECRET_ID")
|
||||
secret_key = _cfg(sc, "secret_access_key", "COS_SECRET_KEY")
|
||||
region = _cfg(sc, "region", "COS_REGION")
|
||||
bucket_name = _cfg(sc, "bucket_name", "COS_BUCKET_NAME")
|
||||
appid = _cfg(sc, "app_id", "COS_APP_ID")
|
||||
prefix = _cfg(sc, "path_prefix", "COS_PATH_PREFIX")
|
||||
enable_old_domain = os.environ.get("COS_ENABLE_OLD_DOMAIN", "").lower() in ("1", "true", "yes")
|
||||
|
||||
if not all([secret_id, secret_key, region, bucket_name, appid]):
|
||||
logger.error(
|
||||
"Incomplete COS configuration, missing environment variables"
|
||||
f"secret_id: {secret_id}, secret_key: {secret_key}, "
|
||||
f"region: {region}, bucket_name: {bucket_name}, appid: {appid}"
|
||||
"Incomplete COS configuration: "
|
||||
"secret_id=%s, region=%s, bucket=%s, appid=%s",
|
||||
bool(secret_id), region, bucket_name, appid,
|
||||
)
|
||||
return None, None, None, None
|
||||
|
||||
# Initialize COS configuration
|
||||
logger.info(
|
||||
f"Initializing COS client with region: {region}, bucket: {bucket_name}"
|
||||
)
|
||||
logger.info("Initializing COS client: region=%s, bucket=%s", region, bucket_name)
|
||||
config = CosConfig(
|
||||
Appid=appid,
|
||||
Region=region,
|
||||
@@ -102,93 +76,50 @@ class CosStorage(Storage):
|
||||
SecretKey=secret_key,
|
||||
EnableOldDomain=enable_old_domain,
|
||||
)
|
||||
|
||||
# Create client
|
||||
client = CosS3Client(config)
|
||||
return client, bucket_name, region, prefix
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize COS client: {str(e)}")
|
||||
logger.error("Failed to initialize COS client: %s", e)
|
||||
return None, None, None, None
|
||||
|
||||
def _get_download_url(self, bucket_name, region, object_key):
|
||||
"""Generate COS object URL
|
||||
|
||||
Args:
|
||||
bucket_name: Bucket name
|
||||
region: Region
|
||||
object_key: Object key
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}"
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to Tencent Cloud COS
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
logger.info(f"Uploading file to COS: {file_path}")
|
||||
try:
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
# Generate object key, use UUID to avoid conflicts
|
||||
file_ext = os.path.splitext(file_path)[1]
|
||||
object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
|
||||
logger.info(f"Generated object key: {object_key}")
|
||||
|
||||
# Upload file
|
||||
logger.info("Attempting to upload file to COS")
|
||||
self.client.upload_file(
|
||||
Bucket=self.bucket_name,
|
||||
LocalFilePath=file_path,
|
||||
Key=object_key,
|
||||
)
|
||||
|
||||
# Get file URL
|
||||
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
|
||||
|
||||
logger.info(f"Successfully uploaded file to COS: {file_url}")
|
||||
logger.info("COS upload_file ok: %s", file_url)
|
||||
return file_url
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload file to COS: {str(e)}")
|
||||
logger.error("COS upload_file failed: %s", e)
|
||||
return ""
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to Tencent Cloud COS
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Uploading bytes content to COS, size: {len(content)} bytes")
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
object_key = (
|
||||
f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
|
||||
if self.prefix
|
||||
else f"images/{uuid.uuid4().hex}{file_ext}"
|
||||
)
|
||||
logger.info(f"Generated object key: {object_key}")
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket_name, Body=content, Key=object_key
|
||||
)
|
||||
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
|
||||
logger.info(f"Successfully uploaded bytes to COS: {file_url}")
|
||||
logger.info("COS upload_bytes ok: %s", file_url)
|
||||
return file_url
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload bytes to COS: {str(e)}")
|
||||
logger.error("COS upload_bytes failed: %s", e)
|
||||
traceback.print_exc()
|
||||
return ""
|
||||
|
||||
@@ -196,67 +127,34 @@ class CosStorage(Storage):
|
||||
class MinioStorage(Storage):
|
||||
"""MinIO storage implementation"""
|
||||
|
||||
def __init__(self, storage_config=None):
|
||||
"""Initialize MinIO storage
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration
|
||||
"""
|
||||
def __init__(self, storage_config: Optional[Dict] = None):
|
||||
self.storage_config = storage_config
|
||||
self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = (
|
||||
self._init_minio_client()
|
||||
)
|
||||
|
||||
def _init_minio_client(self):
|
||||
"""Initialize MinIO client from environment variables or injected config.
|
||||
|
||||
If storage_config contains valid configuration, prefer those values
|
||||
to override environment variables.
|
||||
"""
|
||||
try:
|
||||
# Get configuration from storage_config with environment variables as fallback
|
||||
# Each field can independently fall back to environment variables
|
||||
access_key = (
|
||||
self.storage_config.get("access_key_id")
|
||||
if self.storage_config and self.storage_config.get("access_key_id")
|
||||
else CONFIG.minio_access_key_id
|
||||
)
|
||||
secret_key = (
|
||||
self.storage_config.get("secret_access_key")
|
||||
if self.storage_config and self.storage_config.get("secret_access_key")
|
||||
else CONFIG.minio_secret_access_key
|
||||
)
|
||||
bucket_name = (
|
||||
self.storage_config.get("bucket_name")
|
||||
if self.storage_config and self.storage_config.get("bucket_name")
|
||||
else CONFIG.minio_bucket_name
|
||||
)
|
||||
path_prefix_raw = (
|
||||
self.storage_config.get("path_prefix")
|
||||
if self.storage_config and self.storage_config.get("path_prefix")
|
||||
else CONFIG.minio_path_prefix
|
||||
)
|
||||
sc = self.storage_config
|
||||
access_key = _cfg(sc, "access_key_id", "MINIO_ACCESS_KEY_ID")
|
||||
secret_key = _cfg(sc, "secret_access_key", "MINIO_SECRET_ACCESS_KEY")
|
||||
bucket_name = _cfg(sc, "bucket_name", "MINIO_BUCKET_NAME")
|
||||
path_prefix_raw = _cfg(sc, "path_prefix", "MINIO_PATH_PREFIX")
|
||||
path_prefix = path_prefix_raw.strip().strip("/") if path_prefix_raw else ""
|
||||
|
||||
endpoint = CONFIG.minio_endpoint
|
||||
use_ssl = CONFIG.minio_use_ssl
|
||||
endpoint = _cfg(sc, "endpoint", "MINIO_ENDPOINT")
|
||||
use_ssl = os.environ.get("MINIO_USE_SSL", "").lower() in ("1", "true", "yes")
|
||||
|
||||
if not all([endpoint, access_key, secret_key, bucket_name]):
|
||||
logger.error(
|
||||
"Incomplete MinIO configuration, missing environment variables"
|
||||
)
|
||||
logger.error("Incomplete MinIO configuration")
|
||||
return None, None, None, None, None
|
||||
|
||||
# Initialize client
|
||||
client = Minio(
|
||||
endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl
|
||||
)
|
||||
|
||||
# Ensure bucket exists
|
||||
found = client.bucket_exists(bucket_name)
|
||||
if not found:
|
||||
client.make_bucket(bucket_name)
|
||||
# Set public read policy for the bucket
|
||||
policy = (
|
||||
"{"
|
||||
'"Version":"2012-10-17",'
|
||||
@@ -273,51 +171,26 @@ class MinioStorage(Storage):
|
||||
|
||||
return client, bucket_name, use_ssl, endpoint, path_prefix
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize MinIO client: {str(e)}")
|
||||
logger.error("Failed to initialize MinIO client: %s", e)
|
||||
return None, None, None, None, None
|
||||
|
||||
def _get_download_url(self, object_key: str):
|
||||
"""Construct a public URL for MinIO object.
|
||||
|
||||
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
|
||||
"""
|
||||
# 1. Use public endpoint if provided
|
||||
endpoint = CONFIG.minio_public_endpoint
|
||||
if endpoint:
|
||||
return f"{endpoint}/{self.bucket_name}/{object_key}"
|
||||
|
||||
# 2. Use SSL if enabled
|
||||
if self.use_ssl:
|
||||
return f"https://{self.endpoint}/{self.bucket_name}/{object_key}"
|
||||
|
||||
# 3. Use HTTP default
|
||||
return f"http://{self.endpoint}/{self.bucket_name}/{object_key}"
|
||||
public_endpoint = os.environ.get("MINIO_PUBLIC_ENDPOINT", "")
|
||||
if public_endpoint:
|
||||
return f"{public_endpoint}/{self.bucket_name}/{object_key}"
|
||||
scheme = "https" if self.use_ssl else "http"
|
||||
return f"{scheme}://{self.endpoint}/{self.bucket_name}/{object_key}"
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
"""Upload file to MinIO
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
logger.info(f"Uploading file to MinIO: {file_path}")
|
||||
try:
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
# Generate object key, use UUID to avoid conflicts
|
||||
file_name = os.path.basename(file_path)
|
||||
object_key = (
|
||||
f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
|
||||
if self.path_prefix
|
||||
else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
|
||||
)
|
||||
logger.info(f"Generated MinIO object key: {object_key}")
|
||||
|
||||
# Upload file
|
||||
logger.info("Attempting to upload file to MinIO")
|
||||
with open(file_path, "rb") as file_data:
|
||||
file_size = os.path.getsize(file_path)
|
||||
self.client.put_object(
|
||||
@@ -327,38 +200,22 @@ class MinioStorage(Storage):
|
||||
length=file_size,
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
|
||||
# Get file URL
|
||||
file_url = self._get_download_url(object_key)
|
||||
|
||||
logger.info(f"Successfully uploaded file to MinIO: {file_url}")
|
||||
logger.info("MinIO upload_file ok: %s", file_url)
|
||||
return file_url
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload file to MinIO: {str(e)}")
|
||||
logger.error("MinIO upload_file failed: %s", e)
|
||||
return ""
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
"""Upload bytes to MinIO
|
||||
|
||||
Args:
|
||||
content: Byte content to upload
|
||||
file_ext: File extension
|
||||
|
||||
Returns:
|
||||
File URL
|
||||
"""
|
||||
try:
|
||||
logger.info(f"Uploading bytes content to MinIO, size: {len(content)} bytes")
|
||||
if not self.client:
|
||||
return ""
|
||||
|
||||
object_key = (
|
||||
f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}"
|
||||
if self.path_prefix
|
||||
else f"images/{uuid.uuid4().hex}{file_ext}"
|
||||
)
|
||||
logger.info(f"Generated MinIO object key: {object_key}")
|
||||
self.client.put_object(
|
||||
self.bucket_name or "",
|
||||
object_key,
|
||||
@@ -367,52 +224,67 @@ class MinioStorage(Storage):
|
||||
content_type="application/octet-stream",
|
||||
)
|
||||
file_url = self._get_download_url(object_key)
|
||||
logger.info(f"Successfully uploaded bytes to MinIO: {file_url}")
|
||||
logger.info("MinIO upload_bytes ok: %s", file_url)
|
||||
return file_url
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload bytes to MinIO: {str(e)}")
|
||||
logger.error("MinIO upload_bytes failed: %s", e)
|
||||
traceback.print_exc()
|
||||
return ""
|
||||
|
||||
|
||||
class LocalStorage(Storage):
|
||||
"""Local file system storage implementation"""
|
||||
"""Local file system storage implementation.
|
||||
|
||||
def __init__(self, storage_config: Dict[str, str] = {}):
|
||||
self.storage_config = storage_config
|
||||
base_dir = storage_config.get("base_dir", CONFIG.local_storage_base_dir)
|
||||
self.image_dir = os.path.join(base_dir, "images")
|
||||
Saves files under base_dir and returns web-accessible URL paths
|
||||
(e.g. /files/images/uuid.jpg) so that the Go app can serve them.
|
||||
"""
|
||||
|
||||
def __init__(self, storage_config: Optional[Dict] = None):
|
||||
sc = storage_config or {}
|
||||
self.base_dir = (
|
||||
sc.get("base_dir")
|
||||
or os.environ.get("LOCAL_STORAGE_BASE_DIR", "/data/files")
|
||||
)
|
||||
path_prefix = (sc.get("path_prefix") or "").strip().strip("/")
|
||||
if path_prefix:
|
||||
self.image_dir = os.path.join(self.base_dir, path_prefix, "images")
|
||||
else:
|
||||
self.image_dir = os.path.join(self.base_dir, "images")
|
||||
self.url_prefix = (
|
||||
sc.get("url_prefix")
|
||||
or os.environ.get("LOCAL_STORAGE_URL_PREFIX", "/files")
|
||||
)
|
||||
os.makedirs(self.image_dir, exist_ok=True)
|
||||
|
||||
def _to_url(self, fpath: str) -> str:
|
||||
if self.url_prefix:
|
||||
rel = os.path.relpath(fpath, self.base_dir)
|
||||
return f"{self.url_prefix}/{rel}"
|
||||
return fpath
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
logger.info(f"Uploading file to local storage: {file_path}")
|
||||
return file_path
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
logger.info(f"Uploading file to local storage: {len(content)} bytes")
|
||||
fname = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
|
||||
with open(fname, "wb") as f:
|
||||
fpath = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
|
||||
with open(fpath, "wb") as f:
|
||||
f.write(content)
|
||||
return fname
|
||||
url = self._to_url(fpath)
|
||||
logger.info("Local storage saved: %s -> %s", fpath, url)
|
||||
return url
|
||||
|
||||
|
||||
class Base64Storage(Storage):
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
logger.info(f"Uploading file to base64 storage: {file_path}")
|
||||
return file_path
|
||||
|
||||
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
|
||||
logger.info(f"Uploading file to base64 storage: {len(content)} bytes")
|
||||
file_ext = file_ext.lstrip(".")
|
||||
return f"data:image/{file_ext};base64,{endecode.decode_image(content)}"
|
||||
|
||||
|
||||
class DummyStorage(Storage):
|
||||
"""Dummy storage implementation.
|
||||
|
||||
It is used in tests or in environments where object storage is disabled.
|
||||
All upload methods return empty string.
|
||||
"""
|
||||
"""Dummy storage — all uploads return empty string."""
|
||||
|
||||
def upload_file(self, file_path: str) -> str:
|
||||
return ""
|
||||
@@ -421,26 +293,30 @@ class DummyStorage(Storage):
|
||||
return ""
|
||||
|
||||
|
||||
def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
|
||||
"""Create a storage instance based on configuration or environment variables
|
||||
def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage:
|
||||
"""Create a storage instance based on storage_config dict.
|
||||
|
||||
Args:
|
||||
storage_config: Storage configuration dictionary
|
||||
|
||||
Returns:
|
||||
Storage instance
|
||||
The ``provider`` key in storage_config determines the backend:
|
||||
minio, cos, local, base64.
|
||||
Falls back to STORAGE_TYPE env var, then ``local``.
|
||||
"""
|
||||
storage_type = CONFIG.storage_type
|
||||
storage_type = ""
|
||||
if storage_config:
|
||||
storage_type = str(storage_config.get("provider", storage_type)).lower()
|
||||
logger.info(f"Creating {storage_type} storage instance")
|
||||
provider = str(storage_config.get("provider", "")).lower().strip()
|
||||
if provider and provider not in ("unspecified", "storage_provider_unspecified"):
|
||||
storage_type = provider
|
||||
|
||||
if not storage_type:
|
||||
storage_type = os.environ.get("STORAGE_TYPE", "local").lower().strip()
|
||||
|
||||
logger.info("Creating %s storage instance", storage_type)
|
||||
|
||||
if storage_type == "minio":
|
||||
return MinioStorage(storage_config)
|
||||
elif storage_type == "cos":
|
||||
return CosStorage(storage_config)
|
||||
elif storage_type == "local":
|
||||
return LocalStorage(storage_config or {})
|
||||
return LocalStorage(storage_config)
|
||||
elif storage_type == "base64":
|
||||
return Base64Storage()
|
||||
return DummyStorage()
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
import logging
|
||||
|
||||
from docreader.models.document import Document
|
||||
from docreader.parser.base_parser import BaseParser
|
||||
from docreader.utils import endecode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TextParser(BaseParser):
|
||||
"""
|
||||
Text document parser for processing plain text files.
|
||||
This parser handles text extraction and chunking from plain text documents.
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> Document:
|
||||
"""
|
||||
Parse text document content by decoding bytes to string.
|
||||
|
||||
This is a straightforward parser that simply converts the binary content
|
||||
to text using appropriate character encoding.
|
||||
|
||||
Args:
|
||||
content: Raw document content as bytes
|
||||
|
||||
Returns:
|
||||
Parsed text content as string
|
||||
"""
|
||||
logger.info(f"Parsing text document, content size: {len(content)} bytes")
|
||||
text = endecode.decode_bytes(content)
|
||||
logger.info(
|
||||
f"Successfully parsed text document, extracted {len(text)} characters"
|
||||
)
|
||||
return Document(content=text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Sample text for testing
|
||||
text = """## 标题1
|
||||

|
||||
## 标题2
|
||||

|
||||
## 标题3
|
||||
"""
|
||||
logger.info(f"Test text content: {text}")
|
||||
|
||||
# Define separators for text splitting
|
||||
seperators = ["\n\n", "\n", "。"]
|
||||
parser = TextParser(separators=seperators)
|
||||
logger.info("Splitting text into units")
|
||||
units = parser._split_into_units(text)
|
||||
logger.info(f"Split text into {len(units)} units")
|
||||
logger.info(f"Units: {units}")
|
||||
+474
-547
File diff suppressed because it is too large
Load Diff
@@ -4,86 +4,56 @@ package docreader;
|
||||
|
||||
option go_package = "github.com/Tencent/WeKnora/internal/docreader/proto";
|
||||
|
||||
// 文档读取服务
|
||||
service DocReader {
|
||||
// 从文件读取文档
|
||||
rpc ReadFromFile(ReadFromFileRequest) returns (ReadResponse) {}
|
||||
// 从URL读取文档
|
||||
rpc ReadFromURL(ReadFromURLRequest) returns (ReadResponse) {}
|
||||
}
|
||||
|
||||
// 对象存储提供方
|
||||
enum StorageProvider {
|
||||
STORAGE_PROVIDER_UNSPECIFIED = 0;
|
||||
COS = 1; // 腾讯云 COS
|
||||
MINIO = 2; // MinIO/S3 兼容
|
||||
}
|
||||
|
||||
// 通用对象存储配置,兼容 COS 与 MinIO
|
||||
message StorageConfig {
|
||||
StorageProvider provider = 1; // 存储提供方
|
||||
string region = 2; // 区域(COS 使用)
|
||||
string bucket_name = 3; // 桶名
|
||||
string access_key_id = 4; // 访问密钥 ID(MinIO/S3 使用)
|
||||
string secret_access_key = 5; // 访问密钥 Secret(MinIO/S3 使用)
|
||||
string app_id = 6; // 应用 ID(COS 使用)
|
||||
string path_prefix = 7; // 路径前缀
|
||||
}
|
||||
|
||||
// VLM 配置
|
||||
message VLMConfig {
|
||||
string model_name = 1; // VLM Model Name
|
||||
string base_url = 2; // VLM Base URL
|
||||
string api_key = 3; // VLM API Key
|
||||
string interface_type = 4; // VLM Interface Type: "ollama" or "openai"
|
||||
rpc Read(ReadRequest) returns (ReadResponse) {}
|
||||
rpc ListEngines(ListEnginesRequest) returns (ListEnginesResponse) {}
|
||||
}
|
||||
|
||||
message ReadConfig {
|
||||
int32 chunk_size = 1; // 分块大小
|
||||
int32 chunk_overlap = 2; // 分块重叠
|
||||
repeated string separators = 3; // 分隔符
|
||||
bool enable_multimodal = 4; // 多模态处理
|
||||
StorageConfig storage_config = 5; // 对象存储配置(通用)
|
||||
VLMConfig vlm_config = 6; // VLM 配置
|
||||
string parser_engine = 1;
|
||||
map<string, string> parser_engine_overrides = 2;
|
||||
map<string, string> image_storage = 3; // storage provider config (provider, endpoint, bucket, ...)
|
||||
}
|
||||
|
||||
// 从文件读取文档请求
|
||||
message ReadFromFileRequest {
|
||||
bytes file_content = 1; // 文件内容
|
||||
string file_name = 2; // 文件名
|
||||
string file_type = 3; // 文件类型
|
||||
ReadConfig read_config = 4;
|
||||
string request_id = 5;
|
||||
// Unified read request: set file_content for file mode, url for URL mode.
|
||||
message ReadRequest {
|
||||
bytes file_content = 1;
|
||||
string file_name = 2;
|
||||
string file_type = 3;
|
||||
string url = 4;
|
||||
string title = 5;
|
||||
ReadConfig config = 6;
|
||||
string request_id = 7;
|
||||
}
|
||||
|
||||
// 从URL读取文档请求
|
||||
message ReadFromURLRequest {
|
||||
string url = 1; // 文档URL
|
||||
string title = 2; // 标题
|
||||
ReadConfig read_config = 3;
|
||||
string request_id = 4;
|
||||
message ImageRef {
|
||||
string filename = 1;
|
||||
string original_ref = 2;
|
||||
string mime_type = 3;
|
||||
string storage_key = 4; // download URL from shared storage
|
||||
bytes image_data = 5; // inline bytes fallback
|
||||
}
|
||||
|
||||
// 图片信息
|
||||
message Image {
|
||||
string url = 1; // 图片URL
|
||||
string caption = 2; // 图片描述
|
||||
string ocr_text = 3; // OCR提取的文本内容
|
||||
string original_url = 4; // 原始图片URL
|
||||
int32 start = 5; // 图片在文本中的开始位置
|
||||
int32 end = 6; // 图片在文本中的结束位置
|
||||
}
|
||||
|
||||
message Chunk {
|
||||
string content = 1; // 块内容
|
||||
int32 seq = 2; // 块在文档中的次序
|
||||
int32 start = 3; // 块在文档中的起始位置
|
||||
int32 end = 4; // 块在文档中的结束位置
|
||||
repeated Image images = 5; // 块中包含的图片信息
|
||||
}
|
||||
|
||||
// 从URL读取文档响应
|
||||
message ReadResponse {
|
||||
repeated Chunk chunks = 1; // 文档分块
|
||||
string error = 2; // 错误信息
|
||||
}
|
||||
string markdown_content = 1;
|
||||
repeated ImageRef image_refs = 2;
|
||||
string image_dir_path = 3;
|
||||
map<string, string> metadata = 4;
|
||||
string error = 5;
|
||||
}
|
||||
|
||||
message ListEnginesRequest {
|
||||
map<string, string> config_overrides = 1;
|
||||
}
|
||||
|
||||
message ParserEngineInfo {
|
||||
string name = 1;
|
||||
string description = 2;
|
||||
repeated string file_types = 3;
|
||||
bool available = 4;
|
||||
string unavailable_reason = 5;
|
||||
}
|
||||
|
||||
message ListEnginesResponse {
|
||||
repeated ParserEngineInfo engines = 1;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
|
||||
// versions:
|
||||
// - protoc-gen-go-grpc v1.5.1
|
||||
// - protoc v5.29.3
|
||||
// - protoc v6.33.4
|
||||
// source: docreader.proto
|
||||
|
||||
package proto
|
||||
@@ -19,20 +19,18 @@ import (
|
||||
const _ = grpc.SupportPackageIsVersion9
|
||||
|
||||
const (
|
||||
DocReader_ReadFromFile_FullMethodName = "/docreader.DocReader/ReadFromFile"
|
||||
DocReader_ReadFromURL_FullMethodName = "/docreader.DocReader/ReadFromURL"
|
||||
DocReader_Read_FullMethodName = "/docreader.DocReader/Read"
|
||||
DocReader_ListEngines_FullMethodName = "/docreader.DocReader/ListEngines"
|
||||
DocReader_ConvertToPDF_FullMethodName = "/docreader.DocReader/ConvertToPDF"
|
||||
)
|
||||
|
||||
// DocReaderClient is the client API for DocReader service.
|
||||
//
|
||||
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
||||
//
|
||||
// 文档读取服务
|
||||
type DocReaderClient interface {
|
||||
// 从文件读取文档
|
||||
ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error)
|
||||
// 从URL读取文档
|
||||
ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error)
|
||||
Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error)
|
||||
ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error)
|
||||
ConvertToPDF(ctx context.Context, in *ConvertToPDFRequest, opts ...grpc.CallOption) (*ConvertToPDFResponse, error)
|
||||
}
|
||||
|
||||
type docReaderClient struct {
|
||||
@@ -43,20 +41,30 @@ func NewDocReaderClient(cc grpc.ClientConnInterface) DocReaderClient {
|
||||
return &docReaderClient{cc}
|
||||
}
|
||||
|
||||
func (c *docReaderClient) ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
|
||||
func (c *docReaderClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ReadResponse)
|
||||
err := c.cc.Invoke(ctx, DocReader_ReadFromFile_FullMethodName, in, out, cOpts...)
|
||||
err := c.cc.Invoke(ctx, DocReader_Read_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *docReaderClient) ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
|
||||
func (c *docReaderClient) ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ReadResponse)
|
||||
err := c.cc.Invoke(ctx, DocReader_ReadFromURL_FullMethodName, in, out, cOpts...)
|
||||
out := new(ListEnginesResponse)
|
||||
err := c.cc.Invoke(ctx, DocReader_ListEngines_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *docReaderClient) ConvertToPDF(ctx context.Context, in *ConvertToPDFRequest, opts ...grpc.CallOption) (*ConvertToPDFResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ConvertToPDFResponse)
|
||||
err := c.cc.Invoke(ctx, DocReader_ConvertToPDF_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -66,13 +74,10 @@ func (c *docReaderClient) ReadFromURL(ctx context.Context, in *ReadFromURLReques
|
||||
// DocReaderServer is the server API for DocReader service.
|
||||
// All implementations must embed UnimplementedDocReaderServer
|
||||
// for forward compatibility.
|
||||
//
|
||||
// 文档读取服务
|
||||
type DocReaderServer interface {
|
||||
// 从文件读取文档
|
||||
ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error)
|
||||
// 从URL读取文档
|
||||
ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error)
|
||||
Read(context.Context, *ReadRequest) (*ReadResponse, error)
|
||||
ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error)
|
||||
ConvertToPDF(context.Context, *ConvertToPDFRequest) (*ConvertToPDFResponse, error)
|
||||
mustEmbedUnimplementedDocReaderServer()
|
||||
}
|
||||
|
||||
@@ -83,11 +88,14 @@ type DocReaderServer interface {
|
||||
// pointer dereference when methods are called.
|
||||
type UnimplementedDocReaderServer struct{}
|
||||
|
||||
func (UnimplementedDocReaderServer) ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ReadFromFile not implemented")
|
||||
func (UnimplementedDocReaderServer) Read(context.Context, *ReadRequest) (*ReadResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Read not implemented")
|
||||
}
|
||||
func (UnimplementedDocReaderServer) ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ReadFromURL not implemented")
|
||||
func (UnimplementedDocReaderServer) ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ListEngines not implemented")
|
||||
}
|
||||
func (UnimplementedDocReaderServer) ConvertToPDF(context.Context, *ConvertToPDFRequest) (*ConvertToPDFResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ConvertToPDF not implemented")
|
||||
}
|
||||
func (UnimplementedDocReaderServer) mustEmbedUnimplementedDocReaderServer() {}
|
||||
func (UnimplementedDocReaderServer) testEmbeddedByValue() {}
|
||||
@@ -110,38 +118,56 @@ func RegisterDocReaderServer(s grpc.ServiceRegistrar, srv DocReaderServer) {
|
||||
s.RegisterService(&DocReader_ServiceDesc, srv)
|
||||
}
|
||||
|
||||
func _DocReader_ReadFromFile_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ReadFromFileRequest)
|
||||
func _DocReader_Read_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ReadRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(DocReaderServer).ReadFromFile(ctx, in)
|
||||
return srv.(DocReaderServer).Read(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: DocReader_ReadFromFile_FullMethodName,
|
||||
FullMethod: DocReader_Read_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(DocReaderServer).ReadFromFile(ctx, req.(*ReadFromFileRequest))
|
||||
return srv.(DocReaderServer).Read(ctx, req.(*ReadRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _DocReader_ReadFromURL_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ReadFromURLRequest)
|
||||
func _DocReader_ListEngines_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ListEnginesRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(DocReaderServer).ReadFromURL(ctx, in)
|
||||
return srv.(DocReaderServer).ListEngines(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: DocReader_ReadFromURL_FullMethodName,
|
||||
FullMethod: DocReader_ListEngines_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(DocReaderServer).ReadFromURL(ctx, req.(*ReadFromURLRequest))
|
||||
return srv.(DocReaderServer).ListEngines(ctx, req.(*ListEnginesRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _DocReader_ConvertToPDF_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ConvertToPDFRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(DocReaderServer).ConvertToPDF(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: DocReader_ConvertToPDF_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(DocReaderServer).ConvertToPDF(ctx, req.(*ConvertToPDFRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
@@ -154,12 +180,16 @@ var DocReader_ServiceDesc = grpc.ServiceDesc{
|
||||
HandlerType: (*DocReaderServer)(nil),
|
||||
Methods: []grpc.MethodDesc{
|
||||
{
|
||||
MethodName: "ReadFromFile",
|
||||
Handler: _DocReader_ReadFromFile_Handler,
|
||||
MethodName: "Read",
|
||||
Handler: _DocReader_Read_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ReadFromURL",
|
||||
Handler: _DocReader_ReadFromURL_Handler,
|
||||
MethodName: "ListEngines",
|
||||
Handler: _DocReader_ListEngines_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ConvertToPDF",
|
||||
Handler: _DocReader_ConvertToPDF_Handler,
|
||||
},
|
||||
},
|
||||
Streams: []grpc.StreamDesc{},
|
||||
|
||||
@@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xb9\x01\n\rStorageConfig\x12,\n\x08provider\x18\x01 \x01(\x0e\x32\x1a.docreader.StorageProvider\x12\x0e\n\x06region\x18\x02 \x01(\t\x12\x13\n\x0b\x62ucket_name\x18\x03 \x01(\t\x12\x15\n\raccess_key_id\x18\x04 \x01(\t\x12\x19\n\x11secret_access_key\x18\x05 \x01(\t\x12\x0e\n\x06\x61pp_id\x18\x06 \x01(\t\x12\x13\n\x0bpath_prefix\x18\x07 \x01(\t\"Z\n\tVLMConfig\x12\x12\n\nmodel_name\x18\x01 \x01(\t\x12\x10\n\x08\x62\x61se_url\x18\x02 \x01(\t\x12\x0f\n\x07\x61pi_key\x18\x03 \x01(\t\x12\x16\n\x0einterface_type\x18\x04 \x01(\t\"\xc2\x01\n\nReadConfig\x12\x12\n\nchunk_size\x18\x01 \x01(\x05\x12\x15\n\rchunk_overlap\x18\x02 \x01(\x05\x12\x12\n\nseparators\x18\x03 \x03(\t\x12\x19\n\x11\x65nable_multimodal\x18\x04 \x01(\x08\x12\x30\n\x0estorage_config\x18\x05 \x01(\x0b\x32\x18.docreader.StorageConfig\x12(\n\nvlm_config\x18\x06 \x01(\x0b\x32\x14.docreader.VLMConfig\"\x91\x01\n\x13ReadFromFileRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12*\n\x0bread_config\x18\x04 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"p\n\x12ReadFromURLRequest\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\r\n\x05title\x18\x02 \x01(\t\x12*\n\x0bread_config\x18\x03 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x04 \x01(\t\"i\n\x05Image\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0f\n\x07\x63\x61ption\x18\x02 \x01(\t\x12\x10\n\x08ocr_text\x18\x03 \x01(\t\x12\x14\n\x0coriginal_url\x18\x04 \x01(\t\x12\r\n\x05start\x18\x05 \x01(\x05\x12\x0b\n\x03\x65nd\x18\x06 \x01(\x05\"c\n\x05\x43hunk\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\x12\x0b\n\x03seq\x18\x02 \x01(\x05\x12\r\n\x05start\x18\x03 \x01(\x05\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x05\x12 \n\x06images\x18\x05 \x03(\x0b\x32\x10.docreader.Image\"?\n\x0cReadResponse\x12 \n\x06\x63hunks\x18\x01 \x03(\x0b\x32\x10.docreader.Chunk\x12\r\n\x05\x65rror\x18\x02 \x01(\t*G\n\x0fStorageProvider\x12 \n\x1cSTORAGE_PROVIDER_UNSPECIFIED\x10\x00\x12\x07\n\x03\x43OS\x10\x01\x12\t\n\x05MINIO\x10\x02\x32\x9f\x01\n\tDocReader\x12I\n\x0cReadFromFile\x12\x1e.docreader.ReadFromFileRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n\x0bReadFromURL\x12\x1d.docreader.ReadFromURLRequest\x1a\x17.docreader.ReadResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
|
||||
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xa9\x02\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x12>\n\rimage_storage\x18\x03 \x03(\x0b\x32\'.docreader.ReadConfig.ImageStorageEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11ImageStorageEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"\xe2\x01\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo\"Q\n\x13\x43onvertToPDFRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\":\n\x14\x43onvertToPDFResponse\x12\x13\n\x0bpdf_content\x18\x01 \x01(\x0c\x12\r\n\x05\x65rror\x18\x02 \x01(\t2\xe9\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x12Q\n\x0c\x43onvertToPDF\x12\x1e.docreader.ConvertToPDFRequest\x1a\x1f.docreader.ConvertToPDFResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
|
||||
|
||||
_globals = globals()
|
||||
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
||||
@@ -32,24 +32,40 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'docreader_pb2', _globals)
|
||||
if not _descriptor._USE_C_DESCRIPTORS:
|
||||
_globals['DESCRIPTOR']._loaded_options = None
|
||||
_globals['DESCRIPTOR']._serialized_options = b'Z3github.com/Tencent/WeKnora/internal/docreader/proto'
|
||||
_globals['_STORAGEPROVIDER']._serialized_start=1042
|
||||
_globals['_STORAGEPROVIDER']._serialized_end=1113
|
||||
_globals['_STORAGECONFIG']._serialized_start=31
|
||||
_globals['_STORAGECONFIG']._serialized_end=216
|
||||
_globals['_VLMCONFIG']._serialized_start=218
|
||||
_globals['_VLMCONFIG']._serialized_end=308
|
||||
_globals['_READCONFIG']._serialized_start=311
|
||||
_globals['_READCONFIG']._serialized_end=505
|
||||
_globals['_READFROMFILEREQUEST']._serialized_start=508
|
||||
_globals['_READFROMFILEREQUEST']._serialized_end=653
|
||||
_globals['_READFROMURLREQUEST']._serialized_start=655
|
||||
_globals['_READFROMURLREQUEST']._serialized_end=767
|
||||
_globals['_IMAGE']._serialized_start=769
|
||||
_globals['_IMAGE']._serialized_end=874
|
||||
_globals['_CHUNK']._serialized_start=876
|
||||
_globals['_CHUNK']._serialized_end=975
|
||||
_globals['_READRESPONSE']._serialized_start=977
|
||||
_globals['_READRESPONSE']._serialized_end=1040
|
||||
_globals['_DOCREADER']._serialized_start=1116
|
||||
_globals['_DOCREADER']._serialized_end=1275
|
||||
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._loaded_options = None
|
||||
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_options = b'8\001'
|
||||
_globals['_READCONFIG_IMAGESTORAGEENTRY']._loaded_options = None
|
||||
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_options = b'8\001'
|
||||
_globals['_READRESPONSE_METADATAENTRY']._loaded_options = None
|
||||
_globals['_READRESPONSE_METADATAENTRY']._serialized_options = b'8\001'
|
||||
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._loaded_options = None
|
||||
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_options = b'8\001'
|
||||
_globals['_READCONFIG']._serialized_start=31
|
||||
_globals['_READCONFIG']._serialized_end=328
|
||||
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_start=215
|
||||
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_end=275
|
||||
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_start=277
|
||||
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_end=328
|
||||
_globals['_READREQUEST']._serialized_start=331
|
||||
_globals['_READREQUEST']._serialized_end=491
|
||||
_globals['_IMAGEREF']._serialized_start=493
|
||||
_globals['_IMAGEREF']._serialized_end=603
|
||||
_globals['_READRESPONSE']._serialized_start=606
|
||||
_globals['_READRESPONSE']._serialized_end=832
|
||||
_globals['_READRESPONSE_METADATAENTRY']._serialized_start=785
|
||||
_globals['_READRESPONSE_METADATAENTRY']._serialized_end=832
|
||||
_globals['_LISTENGINESREQUEST']._serialized_start=835
|
||||
_globals['_LISTENGINESREQUEST']._serialized_end=989
|
||||
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=935
|
||||
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=989
|
||||
_globals['_PARSERENGINEINFO']._serialized_start=991
|
||||
_globals['_PARSERENGINEINFO']._serialized_end=1111
|
||||
_globals['_LISTENGINESRESPONSE']._serialized_start=1113
|
||||
_globals['_LISTENGINESRESPONSE']._serialized_end=1180
|
||||
_globals['_CONVERTTOPDFREQUEST']._serialized_start=1182
|
||||
_globals['_CONVERTTOPDFREQUEST']._serialized_end=1263
|
||||
_globals['_CONVERTTOPDFRESPONSE']._serialized_start=1265
|
||||
_globals['_CONVERTTOPDFRESPONSE']._serialized_end=1323
|
||||
_globals['_DOCREADER']._serialized_start=1326
|
||||
_globals['_DOCREADER']._serialized_end=1559
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
|
||||
+110
-102
@@ -1,5 +1,4 @@
|
||||
from google.protobuf.internal import containers as _containers
|
||||
from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from collections.abc import Iterable as _Iterable, Mapping as _Mapping
|
||||
@@ -7,121 +6,130 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
|
||||
|
||||
DESCRIPTOR: _descriptor.FileDescriptor
|
||||
|
||||
class StorageProvider(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
STORAGE_PROVIDER_UNSPECIFIED: _ClassVar[StorageProvider]
|
||||
COS: _ClassVar[StorageProvider]
|
||||
MINIO: _ClassVar[StorageProvider]
|
||||
STORAGE_PROVIDER_UNSPECIFIED: StorageProvider
|
||||
COS: StorageProvider
|
||||
MINIO: StorageProvider
|
||||
|
||||
class StorageConfig(_message.Message):
|
||||
__slots__ = ("provider", "region", "bucket_name", "access_key_id", "secret_access_key", "app_id", "path_prefix")
|
||||
PROVIDER_FIELD_NUMBER: _ClassVar[int]
|
||||
REGION_FIELD_NUMBER: _ClassVar[int]
|
||||
BUCKET_NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
ACCESS_KEY_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
SECRET_ACCESS_KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
APP_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
PATH_PREFIX_FIELD_NUMBER: _ClassVar[int]
|
||||
provider: StorageProvider
|
||||
region: str
|
||||
bucket_name: str
|
||||
access_key_id: str
|
||||
secret_access_key: str
|
||||
app_id: str
|
||||
path_prefix: str
|
||||
def __init__(self, provider: _Optional[_Union[StorageProvider, str]] = ..., region: _Optional[str] = ..., bucket_name: _Optional[str] = ..., access_key_id: _Optional[str] = ..., secret_access_key: _Optional[str] = ..., app_id: _Optional[str] = ..., path_prefix: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class VLMConfig(_message.Message):
|
||||
__slots__ = ("model_name", "base_url", "api_key", "interface_type")
|
||||
MODEL_NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
BASE_URL_FIELD_NUMBER: _ClassVar[int]
|
||||
API_KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
INTERFACE_TYPE_FIELD_NUMBER: _ClassVar[int]
|
||||
model_name: str
|
||||
base_url: str
|
||||
api_key: str
|
||||
interface_type: str
|
||||
def __init__(self, model_name: _Optional[str] = ..., base_url: _Optional[str] = ..., api_key: _Optional[str] = ..., interface_type: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ReadConfig(_message.Message):
|
||||
__slots__ = ("chunk_size", "chunk_overlap", "separators", "enable_multimodal", "storage_config", "vlm_config")
|
||||
CHUNK_SIZE_FIELD_NUMBER: _ClassVar[int]
|
||||
CHUNK_OVERLAP_FIELD_NUMBER: _ClassVar[int]
|
||||
SEPARATORS_FIELD_NUMBER: _ClassVar[int]
|
||||
ENABLE_MULTIMODAL_FIELD_NUMBER: _ClassVar[int]
|
||||
STORAGE_CONFIG_FIELD_NUMBER: _ClassVar[int]
|
||||
VLM_CONFIG_FIELD_NUMBER: _ClassVar[int]
|
||||
chunk_size: int
|
||||
chunk_overlap: int
|
||||
separators: _containers.RepeatedScalarFieldContainer[str]
|
||||
enable_multimodal: bool
|
||||
storage_config: StorageConfig
|
||||
vlm_config: VLMConfig
|
||||
def __init__(self, chunk_size: _Optional[int] = ..., chunk_overlap: _Optional[int] = ..., separators: _Optional[_Iterable[str]] = ..., enable_multimodal: bool = ..., storage_config: _Optional[_Union[StorageConfig, _Mapping]] = ..., vlm_config: _Optional[_Union[VLMConfig, _Mapping]] = ...) -> None: ...
|
||||
__slots__ = ("parser_engine", "parser_engine_overrides", "image_storage")
|
||||
class ParserEngineOverridesEntry(_message.Message):
|
||||
__slots__ = ("key", "value")
|
||||
KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
VALUE_FIELD_NUMBER: _ClassVar[int]
|
||||
key: str
|
||||
value: str
|
||||
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
|
||||
class ImageStorageEntry(_message.Message):
|
||||
__slots__ = ("key", "value")
|
||||
KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
VALUE_FIELD_NUMBER: _ClassVar[int]
|
||||
key: str
|
||||
value: str
|
||||
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
|
||||
PARSER_ENGINE_FIELD_NUMBER: _ClassVar[int]
|
||||
PARSER_ENGINE_OVERRIDES_FIELD_NUMBER: _ClassVar[int]
|
||||
IMAGE_STORAGE_FIELD_NUMBER: _ClassVar[int]
|
||||
parser_engine: str
|
||||
parser_engine_overrides: _containers.ScalarMap[str, str]
|
||||
image_storage: _containers.ScalarMap[str, str]
|
||||
def __init__(self, parser_engine: _Optional[str] = ..., parser_engine_overrides: _Optional[_Mapping[str, str]] = ..., image_storage: _Optional[_Mapping[str, str]] = ...) -> None: ...
|
||||
|
||||
class ReadFromFileRequest(_message.Message):
|
||||
__slots__ = ("file_content", "file_name", "file_type", "read_config", "request_id")
|
||||
class ReadRequest(_message.Message):
|
||||
__slots__ = ("file_content", "file_name", "file_type", "url", "title", "config", "request_id")
|
||||
FILE_CONTENT_FIELD_NUMBER: _ClassVar[int]
|
||||
FILE_NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
FILE_TYPE_FIELD_NUMBER: _ClassVar[int]
|
||||
READ_CONFIG_FIELD_NUMBER: _ClassVar[int]
|
||||
URL_FIELD_NUMBER: _ClassVar[int]
|
||||
TITLE_FIELD_NUMBER: _ClassVar[int]
|
||||
CONFIG_FIELD_NUMBER: _ClassVar[int]
|
||||
REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
file_content: bytes
|
||||
file_name: str
|
||||
file_type: str
|
||||
read_config: ReadConfig
|
||||
request_id: str
|
||||
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ..., read_config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ReadFromURLRequest(_message.Message):
|
||||
__slots__ = ("url", "title", "read_config", "request_id")
|
||||
URL_FIELD_NUMBER: _ClassVar[int]
|
||||
TITLE_FIELD_NUMBER: _ClassVar[int]
|
||||
READ_CONFIG_FIELD_NUMBER: _ClassVar[int]
|
||||
REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
url: str
|
||||
title: str
|
||||
read_config: ReadConfig
|
||||
config: ReadConfig
|
||||
request_id: str
|
||||
def __init__(self, url: _Optional[str] = ..., title: _Optional[str] = ..., read_config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
|
||||
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ..., url: _Optional[str] = ..., title: _Optional[str] = ..., config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class Image(_message.Message):
|
||||
__slots__ = ("url", "caption", "ocr_text", "original_url", "start", "end")
|
||||
URL_FIELD_NUMBER: _ClassVar[int]
|
||||
CAPTION_FIELD_NUMBER: _ClassVar[int]
|
||||
OCR_TEXT_FIELD_NUMBER: _ClassVar[int]
|
||||
ORIGINAL_URL_FIELD_NUMBER: _ClassVar[int]
|
||||
START_FIELD_NUMBER: _ClassVar[int]
|
||||
END_FIELD_NUMBER: _ClassVar[int]
|
||||
url: str
|
||||
caption: str
|
||||
ocr_text: str
|
||||
original_url: str
|
||||
start: int
|
||||
end: int
|
||||
def __init__(self, url: _Optional[str] = ..., caption: _Optional[str] = ..., ocr_text: _Optional[str] = ..., original_url: _Optional[str] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class Chunk(_message.Message):
|
||||
__slots__ = ("content", "seq", "start", "end", "images")
|
||||
CONTENT_FIELD_NUMBER: _ClassVar[int]
|
||||
SEQ_FIELD_NUMBER: _ClassVar[int]
|
||||
START_FIELD_NUMBER: _ClassVar[int]
|
||||
END_FIELD_NUMBER: _ClassVar[int]
|
||||
IMAGES_FIELD_NUMBER: _ClassVar[int]
|
||||
content: str
|
||||
seq: int
|
||||
start: int
|
||||
end: int
|
||||
images: _containers.RepeatedCompositeFieldContainer[Image]
|
||||
def __init__(self, content: _Optional[str] = ..., seq: _Optional[int] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., images: _Optional[_Iterable[_Union[Image, _Mapping]]] = ...) -> None: ...
|
||||
class ImageRef(_message.Message):
|
||||
__slots__ = ("filename", "original_ref", "mime_type", "storage_key", "image_data")
|
||||
FILENAME_FIELD_NUMBER: _ClassVar[int]
|
||||
ORIGINAL_REF_FIELD_NUMBER: _ClassVar[int]
|
||||
MIME_TYPE_FIELD_NUMBER: _ClassVar[int]
|
||||
STORAGE_KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
IMAGE_DATA_FIELD_NUMBER: _ClassVar[int]
|
||||
filename: str
|
||||
original_ref: str
|
||||
mime_type: str
|
||||
storage_key: str
|
||||
image_data: bytes
|
||||
def __init__(self, filename: _Optional[str] = ..., original_ref: _Optional[str] = ..., mime_type: _Optional[str] = ..., storage_key: _Optional[str] = ..., image_data: _Optional[bytes] = ...) -> None: ...
|
||||
|
||||
class ReadResponse(_message.Message):
|
||||
__slots__ = ("chunks", "error")
|
||||
CHUNKS_FIELD_NUMBER: _ClassVar[int]
|
||||
__slots__ = ("markdown_content", "image_refs", "image_dir_path", "metadata", "error")
|
||||
class MetadataEntry(_message.Message):
|
||||
__slots__ = ("key", "value")
|
||||
KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
VALUE_FIELD_NUMBER: _ClassVar[int]
|
||||
key: str
|
||||
value: str
|
||||
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
|
||||
MARKDOWN_CONTENT_FIELD_NUMBER: _ClassVar[int]
|
||||
IMAGE_REFS_FIELD_NUMBER: _ClassVar[int]
|
||||
IMAGE_DIR_PATH_FIELD_NUMBER: _ClassVar[int]
|
||||
METADATA_FIELD_NUMBER: _ClassVar[int]
|
||||
ERROR_FIELD_NUMBER: _ClassVar[int]
|
||||
chunks: _containers.RepeatedCompositeFieldContainer[Chunk]
|
||||
markdown_content: str
|
||||
image_refs: _containers.RepeatedCompositeFieldContainer[ImageRef]
|
||||
image_dir_path: str
|
||||
metadata: _containers.ScalarMap[str, str]
|
||||
error: str
|
||||
def __init__(self, chunks: _Optional[_Iterable[_Union[Chunk, _Mapping]]] = ..., error: _Optional[str] = ...) -> None: ...
|
||||
def __init__(self, markdown_content: _Optional[str] = ..., image_refs: _Optional[_Iterable[_Union[ImageRef, _Mapping]]] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ListEnginesRequest(_message.Message):
|
||||
__slots__ = ("config_overrides",)
|
||||
class ConfigOverridesEntry(_message.Message):
|
||||
__slots__ = ("key", "value")
|
||||
KEY_FIELD_NUMBER: _ClassVar[int]
|
||||
VALUE_FIELD_NUMBER: _ClassVar[int]
|
||||
key: str
|
||||
value: str
|
||||
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
|
||||
CONFIG_OVERRIDES_FIELD_NUMBER: _ClassVar[int]
|
||||
config_overrides: _containers.ScalarMap[str, str]
|
||||
def __init__(self, config_overrides: _Optional[_Mapping[str, str]] = ...) -> None: ...
|
||||
|
||||
class ParserEngineInfo(_message.Message):
|
||||
__slots__ = ("name", "description", "file_types", "available", "unavailable_reason")
|
||||
NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
DESCRIPTION_FIELD_NUMBER: _ClassVar[int]
|
||||
FILE_TYPES_FIELD_NUMBER: _ClassVar[int]
|
||||
AVAILABLE_FIELD_NUMBER: _ClassVar[int]
|
||||
UNAVAILABLE_REASON_FIELD_NUMBER: _ClassVar[int]
|
||||
name: str
|
||||
description: str
|
||||
file_types: _containers.RepeatedScalarFieldContainer[str]
|
||||
available: bool
|
||||
unavailable_reason: str
|
||||
def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ..., file_types: _Optional[_Iterable[str]] = ..., available: bool = ..., unavailable_reason: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ListEnginesResponse(_message.Message):
|
||||
__slots__ = ("engines",)
|
||||
ENGINES_FIELD_NUMBER: _ClassVar[int]
|
||||
engines: _containers.RepeatedCompositeFieldContainer[ParserEngineInfo]
|
||||
def __init__(self, engines: _Optional[_Iterable[_Union[ParserEngineInfo, _Mapping]]] = ...) -> None: ...
|
||||
|
||||
class ConvertToPDFRequest(_message.Message):
|
||||
__slots__ = ("file_content", "file_name", "file_type")
|
||||
FILE_CONTENT_FIELD_NUMBER: _ClassVar[int]
|
||||
FILE_NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
FILE_TYPE_FIELD_NUMBER: _ClassVar[int]
|
||||
file_content: bytes
|
||||
file_name: str
|
||||
file_type: str
|
||||
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ConvertToPDFResponse(_message.Message):
|
||||
__slots__ = ("pdf_content", "error")
|
||||
PDF_CONTENT_FIELD_NUMBER: _ClassVar[int]
|
||||
ERROR_FIELD_NUMBER: _ClassVar[int]
|
||||
pdf_content: bytes
|
||||
error: str
|
||||
def __init__(self, pdf_content: _Optional[bytes] = ..., error: _Optional[str] = ...) -> None: ...
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
import grpc
|
||||
import warnings
|
||||
|
||||
import docreader_pb2 as docreader__pb2
|
||||
from docreader.proto import docreader_pb2 as docreader__pb2
|
||||
|
||||
GRPC_GENERATED_VERSION = '1.76.0'
|
||||
GRPC_GENERATED_VERSION = '1.78.0'
|
||||
GRPC_VERSION = grpc.__version__
|
||||
_version_not_supported = False
|
||||
|
||||
@@ -26,8 +26,7 @@ if _version_not_supported:
|
||||
|
||||
|
||||
class DocReaderStub(object):
|
||||
"""文档读取服务
|
||||
"""
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
def __init__(self, channel):
|
||||
"""Constructor.
|
||||
@@ -35,32 +34,40 @@ class DocReaderStub(object):
|
||||
Args:
|
||||
channel: A grpc.Channel.
|
||||
"""
|
||||
self.ReadFromFile = channel.unary_unary(
|
||||
'/docreader.DocReader/ReadFromFile',
|
||||
request_serializer=docreader__pb2.ReadFromFileRequest.SerializeToString,
|
||||
self.Read = channel.unary_unary(
|
||||
'/docreader.DocReader/Read',
|
||||
request_serializer=docreader__pb2.ReadRequest.SerializeToString,
|
||||
response_deserializer=docreader__pb2.ReadResponse.FromString,
|
||||
_registered_method=True)
|
||||
self.ReadFromURL = channel.unary_unary(
|
||||
'/docreader.DocReader/ReadFromURL',
|
||||
request_serializer=docreader__pb2.ReadFromURLRequest.SerializeToString,
|
||||
response_deserializer=docreader__pb2.ReadResponse.FromString,
|
||||
self.ListEngines = channel.unary_unary(
|
||||
'/docreader.DocReader/ListEngines',
|
||||
request_serializer=docreader__pb2.ListEnginesRequest.SerializeToString,
|
||||
response_deserializer=docreader__pb2.ListEnginesResponse.FromString,
|
||||
_registered_method=True)
|
||||
self.ConvertToPDF = channel.unary_unary(
|
||||
'/docreader.DocReader/ConvertToPDF',
|
||||
request_serializer=docreader__pb2.ConvertToPDFRequest.SerializeToString,
|
||||
response_deserializer=docreader__pb2.ConvertToPDFResponse.FromString,
|
||||
_registered_method=True)
|
||||
|
||||
|
||||
class DocReaderServicer(object):
|
||||
"""文档读取服务
|
||||
"""
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
def ReadFromFile(self, request, context):
|
||||
"""从文件读取文档
|
||||
"""
|
||||
def Read(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def ReadFromURL(self, request, context):
|
||||
"""从URL读取文档
|
||||
"""
|
||||
def ListEngines(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def ConvertToPDF(self, request, context):
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
@@ -68,15 +75,20 @@ class DocReaderServicer(object):
|
||||
|
||||
def add_DocReaderServicer_to_server(servicer, server):
|
||||
rpc_method_handlers = {
|
||||
'ReadFromFile': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ReadFromFile,
|
||||
request_deserializer=docreader__pb2.ReadFromFileRequest.FromString,
|
||||
'Read': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Read,
|
||||
request_deserializer=docreader__pb2.ReadRequest.FromString,
|
||||
response_serializer=docreader__pb2.ReadResponse.SerializeToString,
|
||||
),
|
||||
'ReadFromURL': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ReadFromURL,
|
||||
request_deserializer=docreader__pb2.ReadFromURLRequest.FromString,
|
||||
response_serializer=docreader__pb2.ReadResponse.SerializeToString,
|
||||
'ListEngines': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ListEngines,
|
||||
request_deserializer=docreader__pb2.ListEnginesRequest.FromString,
|
||||
response_serializer=docreader__pb2.ListEnginesResponse.SerializeToString,
|
||||
),
|
||||
'ConvertToPDF': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ConvertToPDF,
|
||||
request_deserializer=docreader__pb2.ConvertToPDFRequest.FromString,
|
||||
response_serializer=docreader__pb2.ConvertToPDFResponse.SerializeToString,
|
||||
),
|
||||
}
|
||||
generic_handler = grpc.method_handlers_generic_handler(
|
||||
@@ -87,11 +99,10 @@ def add_DocReaderServicer_to_server(servicer, server):
|
||||
|
||||
# This class is part of an EXPERIMENTAL API.
|
||||
class DocReader(object):
|
||||
"""文档读取服务
|
||||
"""
|
||||
"""Missing associated documentation comment in .proto file."""
|
||||
|
||||
@staticmethod
|
||||
def ReadFromFile(request,
|
||||
def Read(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
@@ -104,8 +115,8 @@ class DocReader(object):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
'/docreader.DocReader/ReadFromFile',
|
||||
docreader__pb2.ReadFromFileRequest.SerializeToString,
|
||||
'/docreader.DocReader/Read',
|
||||
docreader__pb2.ReadRequest.SerializeToString,
|
||||
docreader__pb2.ReadResponse.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
@@ -118,7 +129,7 @@ class DocReader(object):
|
||||
_registered_method=True)
|
||||
|
||||
@staticmethod
|
||||
def ReadFromURL(request,
|
||||
def ListEngines(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
@@ -131,9 +142,36 @@ class DocReader(object):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
'/docreader.DocReader/ReadFromURL',
|
||||
docreader__pb2.ReadFromURLRequest.SerializeToString,
|
||||
docreader__pb2.ReadResponse.FromString,
|
||||
'/docreader.DocReader/ListEngines',
|
||||
docreader__pb2.ListEnginesRequest.SerializeToString,
|
||||
docreader__pb2.ListEnginesResponse.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True)
|
||||
|
||||
@staticmethod
|
||||
def ConvertToPDF(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
'/docreader.DocReader/ConvertToPDF',
|
||||
docreader__pb2.ConvertToPDFRequest.SerializeToString,
|
||||
docreader__pb2.ConvertToPDFResponse.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -x
|
||||
set -ex
|
||||
|
||||
# 设置目录
|
||||
PROTO_DIR="docreader/proto"
|
||||
@@ -13,12 +13,16 @@ python3 -m grpc_tools.protoc -I${PROTO_DIR} \
|
||||
--grpc_python_out=${PYTHON_OUT} \
|
||||
${PROTO_DIR}/docreader.proto
|
||||
|
||||
# 生成Go代码
|
||||
protoc -I${PROTO_DIR} --go_out=${GO_OUT} \
|
||||
--go_opt=paths=source_relative \
|
||||
--go-grpc_out=${GO_OUT} \
|
||||
--go-grpc_opt=paths=source_relative \
|
||||
${PROTO_DIR}/docreader.proto
|
||||
# 生成Go代码(仅在 protoc-gen-go 可用时执行)
|
||||
if command -v protoc-gen-go &> /dev/null; then
|
||||
protoc -I${PROTO_DIR} --go_out=${GO_OUT} \
|
||||
--go_opt=paths=source_relative \
|
||||
--go-grpc_out=${GO_OUT} \
|
||||
--go-grpc_opt=paths=source_relative \
|
||||
${PROTO_DIR}/docreader.proto
|
||||
else
|
||||
echo "protoc-gen-go not found, skipping Go code generation"
|
||||
fi
|
||||
|
||||
# 修复Python导入问题(MacOS兼容版本)
|
||||
if [ "$(uname)" == "Darwin" ]; then
|
||||
|
||||
Reference in New Issue
Block a user