feat: introduce WeKnora Lite edition with lightweight configuration and deployment

- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
wizardchen
2026-02-28 13:22:07 +08:00
committed by lyingbug
parent 5241dbc39e
commit 397689d2f3
140 changed files with 13365 additions and 5413 deletions
+19 -38
View File
@@ -13,42 +13,35 @@ import (
"google.golang.org/grpc/resolver"
)
// getMaxMessageSize returns the maximum gRPC message size in bytes.
// Default is 50MB, can be configured via MAX_FILE_SIZE_MB environment variable.
func getMaxMessageSize() int {
if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" {
if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 {
return size * 1024 * 1024
}
}
return 50 * 1024 * 1024 // default 50MB
return 50 * 1024 * 1024
}
// Logger is the default logger used by the client
var Logger = log.New(os.Stdout, "[DocReader] ", log.LstdFlags|log.Lmicroseconds)
// ImageInfo 表示一个图片的信息
type ImageInfo struct {
URL string // 图片URL(COS)
Caption string // 图片描述
OCRText string // OCR提取的文本
OriginalURL string // 原始图片URL
Start int // 图片在文本中的开始位置
End int // 图片在文本中的结束位置
// ImageRefInfo represents an image reference from a converted document.
type ImageRefInfo struct {
Filename string
OriginalRef string
MimeType string
StorageKey string
}
// Client represents a DocReader service client
// Client represents a DocReader service client.
type Client struct {
conn *grpc.ClientConn
proto.DocReaderClient
debug bool
}
// NewClient creates a new DocReader client with the specified address
func NewClient(addr string) (*Client, error) {
Logger.Printf("INFO: Creating new DocReader client connecting to %s", addr)
// 设置消息大小限制 (configurable via GRPC_MAX_MESSAGE_SIZE_MB)
maxMsgSize := getMaxMessageSize()
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
@@ -75,19 +68,15 @@ func NewClient(addr string) (*Client, error) {
}, nil
}
// Close closes the client connection
func (c *Client) Close() error {
Logger.Printf("INFO: Closing DocReader client connection")
return c.conn.Close()
}
// SetDebug enables or disables debug logging
func (c *Client) SetDebug(debug bool) {
c.debug = debug
Logger.Printf("INFO: Debug logging set to %v", debug)
}
// Log logs a message with the appropriate level
func (c *Client) Log(level string, format string, args ...interface{}) {
if level == "DEBUG" && !c.debug {
return
@@ -95,28 +84,20 @@ func (c *Client) Log(level string, format string, args ...interface{}) {
Logger.Printf("%s: %s", level, fmt.Sprintf(format, args...))
}
// GetImagesFromChunk 从一个Chunk中提取所有图片信息
func GetImagesFromChunk(chunk *proto.Chunk) []ImageInfo {
if chunk == nil || len(chunk.Images) == 0 {
// GetImageRefsFromResponse extracts image references from a ReadResponse.
func GetImageRefsFromResponse(resp *proto.ReadResponse) []ImageRefInfo {
if resp == nil || len(resp.ImageRefs) == 0 {
return nil
}
images := make([]ImageInfo, 0, len(chunk.Images))
for _, img := range chunk.Images {
images = append(images, ImageInfo{
URL: img.Url,
Caption: img.Caption,
OCRText: img.OcrText,
OriginalURL: img.OriginalUrl,
Start: int(img.Start),
End: int(img.End),
refs := make([]ImageRefInfo, 0, len(resp.ImageRefs))
for _, ref := range resp.ImageRefs {
refs = append(refs, ImageRefInfo{
Filename: ref.Filename,
OriginalRef: ref.OriginalRef,
MimeType: ref.MimeType,
StorageKey: ref.StorageKey,
})
}
return images
}
// HasImagesInChunk 判断一个Chunk是否包含图片
func HasImagesInChunk(chunk *proto.Chunk) bool {
return chunk != nil && len(chunk.Images) > 0
return refs
}
+22 -91
View File
@@ -11,144 +11,75 @@ import (
)
func init() {
// 配置测试日志
log.SetOutput(os.Stdout)
log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.Lshortfile)
log.Println("INFO: Initializing DocReader client tests")
}
func TestReadFromURL(t *testing.T) {
log.Println("INFO: Starting TestReadFromURL")
// 创建测试客户端
log.Println("INFO: Creating test client")
func TestReadURL(t *testing.T) {
client, err := NewClient("localhost:50051")
if err != nil {
log.Printf("ERROR: Failed to create client: %v", err)
t.Fatalf("Failed to create client: %v", err)
}
defer client.Close()
// 启用调试日志
client.SetDebug(true)
// 测试 ReadFromURL 方法
log.Println("INFO: Sending ReadFromURL request to server")
startTime := time.Now()
resp, err := client.ReadFromURL(
resp, err := client.Read(
context.Background(),
&proto.ReadFromURLRequest{
&proto.ReadRequest{
Url: "https://example.com",
Title: "test",
ReadConfig: &proto.ReadConfig{
ChunkSize: 512,
ChunkOverlap: 50,
Separators: []string{"\n\n", "\n", "。"},
EnableMultimodal: true,
},
},
)
log.Printf("INFO: Read(URL) completed in %v", time.Since(startTime))
requestDuration := time.Since(startTime)
if err != nil {
log.Printf("ERROR: ReadFromURL failed: %v", err)
t.Fatalf("ReadFromURL failed: %v", err)
t.Fatalf("Read failed: %v", err)
}
log.Printf("INFO: ReadFromURL completed in %v", requestDuration)
// 验证结果
chunkCount := len(resp.Chunks)
log.Printf("INFO: Received %d chunks from URL parsing", chunkCount)
if chunkCount == 0 {
log.Println("WARN: Expected non-empty content but received none")
t.Error("Expected non-empty content")
if resp.Error != "" {
t.Fatalf("Read returned error: %s", resp.Error)
}
// 打印结果
for i, chunk := range resp.Chunks {
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
} else if i == 2 && chunkCount > 4 {
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
}
if resp.MarkdownContent == "" {
t.Error("Expected non-empty markdown content")
}
log.Println("INFO: TestReadFromURL completed successfully")
log.Printf("INFO: content_len=%d, images=%d", len(resp.MarkdownContent), len(resp.ImageRefs))
}
func TestReadFromFileWithChunking(t *testing.T) {
log.Println("INFO: Starting TestReadFromFileWithChunking")
// 创建测试客户端
log.Println("INFO: Creating test client")
func TestReadFile(t *testing.T) {
client, err := NewClient("localhost:50051")
if err != nil {
log.Printf("ERROR: Failed to create client: %v", err)
t.Fatalf("Failed to create client: %v", err)
}
defer client.Close()
// 启用调试日志
client.SetDebug(true)
// 读取测试文件
log.Println("INFO: Reading test file")
fileContent, err := os.ReadFile("../testdata/test.md")
if err != nil {
log.Printf("ERROR: Failed to read test file: %v", err)
t.Fatalf("Failed to read test file: %v", err)
}
log.Printf("INFO: Read test file, size: %d bytes", len(fileContent))
// 测试 ReadFromFile 方法,带分块参数
log.Println("INFO: Sending ReadFromFile request to server")
startTime := time.Now()
resp, err := client.ReadFromFile(
resp, err := client.Read(
context.Background(),
&proto.ReadFromFileRequest{
&proto.ReadRequest{
FileContent: fileContent,
FileName: "test.md",
FileType: "md",
ReadConfig: &proto.ReadConfig{
ChunkSize: 200,
ChunkOverlap: 50,
Separators: []string{"\n\n", "\n", "。"},
EnableMultimodal: true,
},
},
)
log.Printf("INFO: Read(file) completed in %v", time.Since(startTime))
requestDuration := time.Since(startTime)
if err != nil {
log.Printf("ERROR: ReadFromFile failed: %v", err)
t.Fatalf("ReadFromFile failed: %v", err)
t.Fatalf("Read failed: %v", err)
}
log.Printf("INFO: ReadFromFile completed in %v", requestDuration)
// 验证结果
chunkCount := len(resp.Chunks)
log.Printf("INFO: Received %d chunks from file parsing", chunkCount)
if chunkCount == 0 {
log.Println("WARN: Expected non-empty content but received none")
t.Error("Expected non-empty content")
if resp.Error != "" {
t.Fatalf("Read returned error: %s", resp.Error)
}
if resp.MarkdownContent == "" {
t.Error("Expected non-empty markdown content")
}
// 打印结果
for i, chunk := range resp.Chunks {
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
} else if i == 2 && chunkCount > 4 {
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
}
}
log.Println("INFO: TestReadFromFileWithChunking completed successfully")
}
// 截断字符串以供日志打印
func truncateString(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
imageRefs := GetImageRefsFromResponse(resp)
log.Printf("INFO: content_len=%d, images=%d", len(resp.MarkdownContent), len(imageRefs))
}
+7 -176
View File
@@ -52,58 +52,17 @@ class DocReaderConfig:
grpc_max_file_size_mb: int
grpc_port: int
# Image processing
image_max_concurrent: int
# Proxy
external_http_proxy: str
external_https_proxy: str
# OCR
ocr_backend: str
ocr_api_base_url: str
ocr_api_key: str
ocr_model: str
# VLM Caption
vlm_model_base_url: str
vlm_model_name: str
vlm_model_api_key: str
vlm_interface_type: str
# Storage
storage_type: str
cos_secret_id: str
cos_secret_key: str
cos_region: str
cos_bucket_name: str
cos_app_id: str
cos_path_prefix: str
cos_enable_old_domain: bool
minio_access_key_id: str
minio_secret_access_key: str
minio_bucket_name: str
minio_path_prefix: str
minio_endpoint: str
minio_public_endpoint: str
minio_use_ssl: bool
local_storage_base_dir: str
# Other
mineru_endpoint: str
# Temp image output directory (shared with Go app via volume, local mode fallback)
image_output_dir: str
def load_config() -> DocReaderConfig:
"""Load config from environment variables.
"""Load config from environment variables (lightweight version)."""
Naming convention (new): DOCREADER_*
Backward compatible keys are supported.
"""
# gRPC
grpc_max_workers = _get_int(["DOCREADER_GRPC_MAX_WORKERS", "GRPC_MAX_WORKERS"], 4)
grpc_max_file_size_mb = (
_get_int(["DOCREADER_GRPC_MAX_FILE_SIZE_MB", "MAX_FILE_SIZE_MB"], 50)
@@ -112,12 +71,6 @@ def load_config() -> DocReaderConfig:
)
grpc_port = _get_int(["DOCREADER_GRPC_PORT", "PORT"], 50051)
# Image processing
image_max_concurrent = _get_int(
["DOCREADER_IMAGE_MAX_CONCURRENT", "IMAGE_MAX_CONCURRENT"], 1
)
# Proxies
external_http_proxy = _get_str(
["DOCREADER_EXTERNAL_HTTP_PROXY", "EXTERNAL_HTTP_PROXY"], ""
)
@@ -125,95 +78,17 @@ def load_config() -> DocReaderConfig:
["DOCREADER_EXTERNAL_HTTPS_PROXY", "EXTERNAL_HTTPS_PROXY"], ""
)
# OCR
ocr_backend = _get_str(["DOCREADER_OCR_BACKEND", "OCR_BACKEND"], "")
ocr_api_base_url = _get_str(["DOCREADER_OCR_API_BASE_URL", "OCR_API_BASE_URL"], "")
ocr_api_key = _get_str(["DOCREADER_OCR_API_KEY", "OCR_API_KEY"], "")
ocr_model = _get_str(["DOCREADER_OCR_MODEL", "OCR_MODEL"], "")
# VLM Caption
vlm_model_base_url = _get_str(
["DOCREADER_VLM_MODEL_BASE_URL", "VLM_MODEL_BASE_URL"], ""
image_output_dir = _get_str(
["DOCREADER_IMAGE_OUTPUT_DIR", "IMAGE_OUTPUT_DIR"], "/tmp/docreader"
)
vlm_model_name = _get_str(["DOCREADER_VLM_MODEL_NAME", "VLM_MODEL_NAME"], "")
vlm_model_api_key = _get_str(
["DOCREADER_VLM_MODEL_API_KEY", "VLM_MODEL_API_KEY"], ""
)
vlm_interface_type = _get_str(
["DOCREADER_VLM_INTERFACE_TYPE", "VLM_INTERFACE_TYPE"], "openai"
).lower()
# Storage
storage_type = _get_str(["DOCREADER_STORAGE_TYPE", "STORAGE_TYPE"], "cos").lower()
# COS
cos_secret_id = _get_str(["DOCREADER_COS_SECRET_ID", "COS_SECRET_ID"], "")
cos_secret_key = _get_str(["DOCREADER_COS_SECRET_KEY", "COS_SECRET_KEY"], "")
cos_region = _get_str(["DOCREADER_COS_REGION", "COS_REGION"], "")
cos_bucket_name = _get_str(["DOCREADER_COS_BUCKET_NAME", "COS_BUCKET_NAME"], "")
cos_app_id = _get_str(["DOCREADER_COS_APP_ID", "COS_APP_ID"], "")
cos_path_prefix = _get_str(["DOCREADER_COS_PATH_PREFIX", "COS_PATH_PREFIX"], "")
cos_enable_old_domain = _get_bool(
["DOCREADER_COS_ENABLE_OLD_DOMAIN", "COS_ENABLE_OLD_DOMAIN"], True
)
# MinIO
minio_access_key_id = _get_str(
["DOCREADER_MINIO_ACCESS_KEY_ID", "MINIO_ACCESS_KEY_ID"], "minioadmin"
)
minio_secret_access_key = _get_str(
["DOCREADER_MINIO_SECRET_ACCESS_KEY", "MINIO_SECRET_ACCESS_KEY"], "minioadmin"
)
minio_bucket_name = _get_str(
["DOCREADER_MINIO_BUCKET_NAME", "MINIO_BUCKET_NAME"], "WeKnora"
)
minio_path_prefix = _get_str(
["DOCREADER_MINIO_PATH_PREFIX", "MINIO_PATH_PREFIX"], ""
)
minio_endpoint = _get_str(["DOCREADER_MINIO_ENDPOINT", "MINIO_ENDPOINT"], "")
minio_public_endpoint = _get_str(
["DOCREADER_MINIO_PUBLIC_ENDPOINT", "MINIO_PUBLIC_ENDPOINT"], ""
)
minio_use_ssl = _get_bool(["DOCREADER_MINIO_USE_SSL", "MINIO_USE_SSL"], False)
# Local storage
local_storage_base_dir = "./data/files"
# Other
mineru_endpoint = _get_str(["DOCREADER_MINERU_ENDPOINT", "MINERU_ENDPOINT"], "")
return DocReaderConfig(
grpc_max_workers=grpc_max_workers,
grpc_max_file_size_mb=grpc_max_file_size_mb,
grpc_port=grpc_port,
image_max_concurrent=image_max_concurrent,
external_http_proxy=external_http_proxy,
external_https_proxy=external_https_proxy,
ocr_backend=ocr_backend,
ocr_api_base_url=ocr_api_base_url,
ocr_api_key=ocr_api_key,
ocr_model=ocr_model,
vlm_model_base_url=vlm_model_base_url,
vlm_model_name=vlm_model_name,
vlm_model_api_key=vlm_model_api_key,
vlm_interface_type=vlm_interface_type,
storage_type=storage_type,
cos_secret_id=cos_secret_id,
cos_secret_key=cos_secret_key,
cos_region=cos_region,
cos_bucket_name=cos_bucket_name,
cos_app_id=cos_app_id,
cos_path_prefix=cos_path_prefix,
cos_enable_old_domain=cos_enable_old_domain,
minio_access_key_id=minio_access_key_id,
minio_secret_access_key=minio_secret_access_key,
minio_bucket_name=minio_bucket_name,
minio_path_prefix=minio_path_prefix,
minio_endpoint=minio_endpoint,
minio_public_endpoint=minio_public_endpoint,
minio_use_ssl=minio_use_ssl,
local_storage_base_dir=local_storage_base_dir,
mineru_endpoint=mineru_endpoint,
image_output_dir=image_output_dir,
)
@@ -223,56 +98,12 @@ CONFIG = load_config()
def dump_config(mask_secrets: bool = True) -> Dict[str, Any]:
cfg = CONFIG
d: Dict[str, Any] = {
# gRPC
"DOCREADER_GRPC_MAX_WORKERS": cfg.grpc_max_workers,
"DOCREADER_GRPC_MAX_FILE_SIZE_MB": cfg.grpc_max_file_size_mb,
"DOCREADER_GRPC_PORT": cfg.grpc_port,
# Image processing
"DOCREADER_IMAGE_MAX_CONCURRENT": cfg.image_max_concurrent,
# Proxy
"DOCREADER_EXTERNAL_HTTP_PROXY": cfg.external_http_proxy,
"DOCREADER_EXTERNAL_HTTPS_PROXY": cfg.external_https_proxy,
# OCR
"DOCREADER_OCR_BACKEND": cfg.ocr_backend,
"DOCREADER_OCR_API_BASE_URL": cfg.ocr_api_base_url,
"DOCREADER_OCR_API_KEY": _mask_secret(cfg.ocr_api_key)
if mask_secrets
else cfg.ocr_api_key,
"DOCREADER_OCR_MODEL": cfg.ocr_model,
# VLM
"DOCREADER_VLM_MODEL_BASE_URL": cfg.vlm_model_base_url,
"DOCREADER_VLM_MODEL_NAME": cfg.vlm_model_name,
"DOCREADER_VLM_MODEL_API_KEY": _mask_secret(cfg.vlm_model_api_key)
if mask_secrets
else cfg.vlm_model_api_key,
"DOCREADER_VLM_INTERFACE_TYPE": cfg.vlm_interface_type,
# Storage
"DOCREADER_STORAGE_TYPE": cfg.storage_type,
"DOCREADER_COS_SECRET_ID": _mask_secret(cfg.cos_secret_id)
if mask_secrets
else cfg.cos_secret_id,
"DOCREADER_COS_SECRET_KEY": _mask_secret(cfg.cos_secret_key)
if mask_secrets
else cfg.cos_secret_key,
"DOCREADER_COS_REGION": cfg.cos_region,
"DOCREADER_COS_BUCKET_NAME": cfg.cos_bucket_name,
"DOCREADER_COS_APP_ID": cfg.cos_app_id,
"DOCREADER_COS_PATH_PREFIX": cfg.cos_path_prefix,
"DOCREADER_COS_ENABLE_OLD_DOMAIN": cfg.cos_enable_old_domain,
"DOCREADER_MINIO_ACCESS_KEY_ID": _mask_secret(cfg.minio_access_key_id)
if mask_secrets
else cfg.minio_access_key_id,
"DOCREADER_MINIO_SECRET_ACCESS_KEY": _mask_secret(cfg.minio_secret_access_key)
if mask_secrets
else cfg.minio_secret_access_key,
"DOCREADER_MINIO_BUCKET_NAME": cfg.minio_bucket_name,
"DOCREADER_MINIO_PATH_PREFIX": cfg.minio_path_prefix,
"DOCREADER_MINIO_ENDPOINT": cfg.minio_endpoint,
"DOCREADER_MINIO_PUBLIC_ENDPOINT": cfg.minio_public_endpoint,
"DOCREADER_MINIO_USE_SSL": cfg.minio_use_ssl,
"DOCREADER_LOCAL_STORAGE_BASE_DIR": cfg.local_storage_base_dir,
# Other
"DOCREADER_MINERU_ENDPOINT": cfg.mineru_endpoint,
"DOCREADER_IMAGE_OUTPUT_DIR": cfg.image_output_dir,
}
return d
+149 -205
View File
@@ -13,118 +13,130 @@ from grpc_health.v1.health import HealthServicer
from docreader import config
from docreader.config import CONFIG
from docreader.models.read_config import ChunkingConfig
from docreader.parser import Parser
from docreader.proto import docreader_pb2_grpc
from docreader.parser.registry import registry
from docreader.proto.docreader_pb2 import (
Chunk,
Image,
ReadConfig,
ReadFromFileRequest,
ReadFromURLRequest,
ReadRequest,
ReadResponse,
StorageProvider,
ImageRef,
ListEnginesResponse,
ParserEngineInfo,
)
from docreader.utils.request import init_logging_request_id, request_id_context
# Surrogate range U+D800..U+DFFF are invalid Unicode scalar values
# cannot be encoded to UTF-8
_SURROGATE_RE = re.compile(r"[\ud800-\udfff]")
def to_valid_utf8_text(s: Optional[str]) -> str:
"""Return a UTF-8 safe string for protobuf.
- Replace any surrogate code points with U+FFFD
- Re-encode with errors='replace' to ensure valid UTF-8
"""
if not s:
return ""
s = _SURROGATE_RE.sub("\ufffd", s)
return s.encode("utf-8", errors="replace").decode("utf-8")
# Ensure no existing handlers
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
# Configure logging - use stdout
handler = logging.StreamHandler(sys.stdout)
logging.root.addHandler(handler)
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.info("Initializing server logging")
_level_name = (os.environ.get("LOG_LEVEL") or "INFO").upper()
_level = getattr(logging, _level_name, logging.INFO)
logging.root.setLevel(_level)
logger = logging.getLogger(__name__)
logger.info("Initializing server logging, level=%s", _level_name)
# Initialize request ID logging
init_logging_request_id()
parser = Parser()
def _resolve_images(images: dict, request_id: str, storage_map: dict | None = None) -> tuple[str, list]:
"""Resolve document images with priority: shared storage > temp dir > inline bytes.
``images`` is a dict of {relative_path: raw_data} where raw_data is
base64-encoded string or raw bytes.
def create_chunking_config(read_config: ReadConfig):
"""Create ChunkingConfig from ReadConfig request.
``storage_map`` is a dict from the request's config.image_storage map.
When it contains a valid "provider" key, images are uploaded to shared object
storage and storage_key is set (no inline bytes, saving gRPC message size).
Args:
read_config: The read_config from the gRPC request
Returns:
ChunkingConfig: Configured chunking configuration object
Returns (image_dir_path, list[ImageRef]).
"""
# Extract chunking parameters
chunk_size = read_config.chunk_size or 512
chunk_overlap = read_config.chunk_overlap or 50
# Convert protobuf RepeatedScalarFieldContainer to list for type compatibility
separators = (
list(read_config.separators) if read_config.separators else ["\n\n", "\n", "。"]
)
enable_multimodal = read_config.enable_multimodal or False
import base64
logger.info(
f"Using chunking config: size={chunk_size}, "
f"overlap={chunk_overlap}, multimodal={enable_multimodal}"
)
if not images:
return "", []
# Extract storage config
sc = read_config.storage_config
storage_config = {
"provider": StorageProvider.Name(sc.provider),
"region": sc.region,
"bucket_name": sc.bucket_name,
"access_key_id": sc.access_key_id,
"secret_access_key": sc.secret_access_key,
"app_id": sc.app_id,
"path_prefix": sc.path_prefix,
}
logger.info(
f"Using Storage config: provider={storage_config.get('provider')}, "
f"bucket={storage_config['bucket_name']}"
)
# Extract VLM config
vlm_config = {
"model_name": read_config.vlm_config.model_name,
"base_url": read_config.vlm_config.base_url,
"api_key": read_config.vlm_config.api_key,
"interface_type": read_config.vlm_config.interface_type or "openai",
mime_map = {
".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg",
".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
}
logger.info(
f"Using VLM config: model={vlm_config['model_name']}, "
f"base_url={vlm_config['base_url']}, "
f"interface_type={vlm_config['interface_type']}"
)
storage_client = None
use_storage = False
provider = (storage_map or {}).get("provider", "")
if provider:
try:
from docreader.parser.storage import create_storage
storage_client = create_storage(storage_map)
use_storage = True
logger.info("Using shared storage (%s) for image upload", provider)
except Exception as e:
logger.warning("Failed to init shared storage, falling back to inline bytes: %s", e)
# Create and return ChunkingConfig
return ChunkingConfig(
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
separators=separators,
enable_multimodal=enable_multimodal,
storage_config=storage_config,
vlm_config=vlm_config,
)
base_dir = CONFIG.image_output_dir
output_dir = os.path.join(base_dir, request_id, "images")
wrote_to_dir = False
if not use_storage:
try:
os.makedirs(output_dir, exist_ok=True)
wrote_to_dir = True
except OSError:
logger.warning("Cannot write to image output dir %s", output_dir)
refs = []
for ref_path, b64data in images.items():
try:
img_bytes = base64.b64decode(b64data)
except Exception:
img_bytes = b64data.encode("utf-8") if isinstance(b64data, str) else b64data
fname = os.path.basename(ref_path) or f"{uuid.uuid4().hex}.png"
ext = os.path.splitext(fname)[1].lower()
mime = mime_map.get(ext, "application/octet-stream")
if use_storage and storage_client:
storage_key = storage_client.upload_bytes(img_bytes, ext)
if storage_key:
refs.append(ImageRef(
filename=fname,
original_ref=ref_path,
mime_type=mime,
storage_key=storage_key,
))
continue
logger.warning("Storage upload failed for %s, falling back to inline bytes", fname)
if wrote_to_dir:
dest = os.path.join(output_dir, fname)
try:
with open(dest, "wb") as f:
f.write(img_bytes)
except OSError as e:
logger.warning("Failed to write image %s: %s", dest, e)
refs.append(ImageRef(
filename=fname,
original_ref=ref_path,
mime_type=mime,
image_data=img_bytes,
))
image_dir = os.path.join(base_dir, request_id) if wrote_to_dir else ""
mode = "storage" if use_storage else ("dir+inline" if wrote_to_dir else "inline")
logger.info("Resolved %d images (mode=%s)", len(refs), mode)
return image_dir, refs
class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
@@ -132,152 +144,90 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
super().__init__()
self.parser = Parser()
def ReadFromFile(self, request: ReadFromFileRequest, context):
# Get or generate request ID
request_id = (
request.request_id
if hasattr(request, "request_id") and request.request_id
else str(uuid.uuid4())
)
def Read(self, request: ReadRequest, context):
"""Unified read: file mode (file_content set) or URL mode (url set)."""
request_id = request.request_id or str(uuid.uuid4())
is_url = bool(request.url)
# Use request ID context
with request_id_context(request_id):
try:
# Get file type
file_type = (
request.file_type or os.path.splitext(request.file_name)[1][1:]
)
logger.info(
f"ReadFromFile for file: {request.file_name}, type: {file_type}"
)
logger.info(f"File content size: {len(request.file_content)} bytes")
cfg = request.config
parser_engine = cfg.parser_engine if cfg else ""
engine_overrides = dict(cfg.parser_engine_overrides) if cfg else {}
storage_map = dict(cfg.image_storage) if cfg and cfg.image_storage else None
# Create chunking config
chunking_config = create_chunking_config(request.read_config)
if is_url:
logger.info("Read(URL): url=%s", request.url)
result = self.parser.parse_url(
request.url,
request.title,
parser_engine=parser_engine,
engine_overrides=engine_overrides,
)
source_desc = request.url
else:
file_type = (
request.file_type or os.path.splitext(request.file_name)[1][1:]
)
logger.info(
"Read(File): file=%s, type=%s, size=%d bytes",
request.file_name, file_type, len(request.file_content),
)
result = self.parser.parse_file(
request.file_name,
file_type,
request.file_content,
parser_engine=parser_engine,
engine_overrides=engine_overrides,
)
source_desc = request.file_name
# Parse file
logger.info("Starting file parsing process")
result = self.parser.parse_file(
request.file_name, file_type, request.file_content, chunking_config
)
if not result:
error_msg = "Failed to parse file"
if not result or not result.content:
error_msg = f"Failed to parse: {source_desc}"
logger.error(error_msg)
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(error_msg)
return ReadResponse()
# Convert to protobuf message
logger.info(
f"Parsed file {request.file_name}, with {len(result.chunks)} chunks"
)
# Build response, including image info
response = ReadResponse(
chunks=[
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
]
)
logger.info(f"Response size: {response.ByteSize()} bytes")
return response
except Exception as e:
error_msg = f"Error reading file: {str(e)}"
logger.error(error_msg)
logger.info(f"Detailed traceback: {traceback.format_exc()}")
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(str(e))
return ReadResponse(error=str(e))
def ReadFromURL(self, request: ReadFromURLRequest, context):
# Get or generate request ID
request_id = (
request.request_id
if hasattr(request, "request_id") and request.request_id
else str(uuid.uuid4())
)
# Use request ID context
with request_id_context(request_id):
try:
logger.info(f"Received ReadFromURL request for URL: {request.url}")
# Create chunking config
chunking_config = create_chunking_config(request.read_config)
# Parse URL
logger.info("Starting URL parsing process")
result = self.parser.parse_url(
request.url, request.title, chunking_config
)
if not result:
error_msg = "Failed to parse URL"
logger.error(error_msg)
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(error_msg)
return ReadResponse(error=error_msg)
# Convert to protobuf message, including image info
logger.info(
f"Parsed URL {request.url}, returning {len(result.chunks)} chunks"
_c = to_valid_utf8_text
image_dir, image_refs = _resolve_images(
result.images, request_id, storage_map=storage_map
)
response = ReadResponse(
chunks=[
self._convert_chunk_to_proto(chunk) for chunk in result.chunks
]
markdown_content=_c(result.content),
image_refs=image_refs,
image_dir_path=image_dir,
)
logger.info(
"Read response: content_len=%d, images=%d",
len(result.content), len(image_refs),
)
logger.info(f"Response size: {response.ByteSize()} bytes")
return response
except Exception as e:
error_msg = f"Error reading URL: {str(e)}"
error_msg = f"Error reading document: {e}"
logger.error(error_msg)
logger.info(f"Detailed traceback: {traceback.format_exc()}")
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details(str(e))
logger.info("Traceback: %s", traceback.format_exc())
return ReadResponse(error=str(e))
def _convert_chunk_to_proto(self, chunk):
"""Convert internal Chunk object to protobuf Chunk message
Ensures all string fields are valid UTF-8 for protobuf (no lone surrogates).
"""
# Clean helper for strings
_c = to_valid_utf8_text
proto_chunk = Chunk(
content=_c(getattr(chunk, "content", None)),
seq=getattr(chunk, "seq", 0),
start=getattr(chunk, "start", 0),
end=getattr(chunk, "end", 0),
)
# If chunk has images attribute and is not empty, add image info
if hasattr(chunk, "images") and chunk.images:
logger.info(
f"Adding {len(chunk.images)} images to chunk {getattr(chunk, 'seq', 0)}"
def ListEngines(self, request, context):
overrides = dict(getattr(request, "config_overrides", None) or {})
engines_data = registry.list_engines(overrides=overrides or None)
engines = [
ParserEngineInfo(
name=e["name"],
description=e["description"],
file_types=e["file_types"],
available=e.get("available", True),
unavailable_reason=e.get("unavailable_reason", ""),
)
for img_info in chunk.images:
# img_info expected as dict
proto_image = Image(
url=_c(img_info.get("cos_url", "")),
caption=_c(img_info.get("caption", "")),
ocr_text=_c(img_info.get("ocr_text", "")),
original_url=_c(img_info.get("original_url", "")),
start=int(img_info.get("start", 0) or 0),
end=int(img_info.get("end", 0) or 0),
)
proto_chunk.images.append(proto_image)
return proto_chunk
for e in engines_data
]
return ListEnginesResponse(engines=engines)
def main():
# Print effective env/config at startup
config.print_config()
# Create server
server = grpc.server(
futures.ThreadPoolExecutor(max_workers=CONFIG.grpc_max_workers),
options=[
@@ -286,24 +236,18 @@ def main():
],
)
# Register services
docreader_pb2_grpc.add_DocReaderServicer_to_server(DocReaderServicer(), server)
# Register health check service
health_servicer = HealthServicer()
health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server)
# Set listen address
server.add_insecure_port(f"[::]:{CONFIG.grpc_port}")
# Start service
server.start()
logger.info(f"Server started on port {CONFIG.grpc_port}")
logger.info("Server started on port %d", CONFIG.grpc_port)
logger.info("Server is ready to accept connections")
try:
# Wait for service termination
server.wait_for_termination()
except KeyboardInterrupt:
logger.info("Received termination signal, shutting down server")
+8 -18
View File
@@ -1,27 +1,17 @@
from dataclasses import dataclass, field
from dataclasses import dataclass
@dataclass
class ChunkingConfig:
"""
Configuration for text chunking process.
Controls how documents are split into smaller pieces for processing.
"""Legacy config kept for backward compatibility.
After the lightweight refactoring, chunking is done in Go.
This class is only kept so existing parser constructors don't break.
"""
# Maximum size of each chunk in tokens/chars
chunk_size: int = 512
# Number of tokens/chars to overlap between chunks
chunk_overlap: int = 50
# Text separators in order of priority
separators: list[str] = field(default_factory=lambda: ["\n\n", "\n", "。"])
# Whether to enable multimodal processing (text + images)
separators: list[str] | None = None
enable_multimodal: bool = False
# Preferred field name going forward
storage_config: dict[str, str] = field(default_factory=dict)
# VLM configuration for image captioning
vlm_config: dict[str, str] = field(default_factory=dict)
storage_config: dict[str, str] | None = None
vlm_config: dict[str, str] | None = None
+11 -12
View File
@@ -13,7 +13,6 @@ The parsers extract content from documents and can split them into
meaningful chunks for further processing and indexing.
"""
from .csv_parser import CSVParser
from .doc_parser import DocParser
from .docx2_parser import Docx2Parser
from .excel_parser import ExcelParser
@@ -21,19 +20,19 @@ from .image_parser import ImageParser
from .markdown_parser import MarkdownParser
from .parser import Parser
from .pdf_parser import PDFParser
from .text_parser import TextParser
from .registry import ParserEngineRegistry, registry
from .web_parser import WebParser
# Export public classes and modules
__all__ = [
"Docx2Parser", # Parser for .docx files (modern Word documents)
"DocParser", # Parser for .doc files (legacy Word documents)
"PDFParser", # Parser for PDF documents
"MarkdownParser", # Parser for Markdown text files
"TextParser", # Parser for plain text files
"ImageParser", # Parser for images with text content
"WebParser", # Parser for web pages
"Parser", # Main parser factory that selects the appropriate parser
"CSVParser", # Parser for CSV files
"ExcelParser", # Parser for Excel files
"Docx2Parser",
"DocParser",
"PDFParser",
"MarkdownParser",
"ImageParser",
"WebParser",
"Parser",
"ExcelParser",
"ParserEngineRegistry",
"registry",
]
File diff suppressed because it is too large Load Diff
-387
View File
@@ -1,387 +0,0 @@
import json
import logging
import time
from dataclasses import dataclass, field
from typing import Dict, List, Optional, Union
import ollama
import requests
from docreader.config import CONFIG
logger = logging.getLogger(__name__)
@dataclass
class ImageUrl:
"""Image URL data structure for caption requests."""
url: Optional[str] = None
detail: Optional[str] = None
@dataclass
class Content:
"""Content data structure that can contain text or image URL."""
type: Optional[str] = None
text: Optional[str] = None
image_url: Optional[ImageUrl] = None
@dataclass
class SystemMessage:
"""System message for VLM model requests."""
role: Optional[str] = None
content: Optional[str] = None
@dataclass
class UserMessage:
"""User message for VLM model requests, can contain multiple content items."""
role: Optional[str] = None
content: List[Content] = field(default_factory=list)
@dataclass
class CompletionRequest:
"""Request structure for VLM model completion API."""
model: str
temperature: float
top_p: float
messages: List[Union[SystemMessage, UserMessage]]
user: str
@dataclass
class Model:
"""Model identifier structure."""
id: str
@dataclass
class ModelsResp:
"""Response structure for available models API."""
data: List[Model] = field(default_factory=list)
@dataclass
class Message:
"""Message structure in API response."""
role: Optional[str] = None
content: Optional[str] = None
tool_calls: Optional[str] = None
@dataclass
class Choice:
"""Choice structure in API response."""
message: Optional[Message] = None
@dataclass
class Usage:
"""Token usage information in API response."""
prompt_tokens: Optional[int] = 0
total_tokens: Optional[int] = 0
completion_tokens: Optional[int] = 0
@dataclass
class CaptionChatResp:
"""Response structure for caption chat API."""
id: Optional[str] = None
created: Optional[int] = None
model: Optional[Model] = None
object: Optional[str] = None
choices: List[Choice] = field(default_factory=list)
usage: Optional[Usage] = None
@staticmethod
def from_json(json_data: dict) -> "CaptionChatResp":
"""
Parse API response JSON into a CaptionChatResp object.
Args:
json_data: The JSON response from the API
Returns:
A parsed CaptionChatResp object
"""
logger.info("Parsing CaptionChatResp from JSON")
# Manually parse nested fields with safe field extraction
choices = []
for choice in json_data.get("choices", []):
message_data = choice.get("message", {})
message = Message(
role=message_data.get("role"),
content=message_data.get("content"),
tool_calls=message_data.get("tool_calls"),
)
choices.append(Choice(message=message))
# Handle usage with safe field extraction
usage_data = json_data.get("usage", {})
usage = None
if usage_data:
usage = Usage(
prompt_tokens=usage_data.get("prompt_tokens", 0),
total_tokens=usage_data.get("total_tokens", 0),
completion_tokens=usage_data.get("completion_tokens", 0),
)
logger.info(
f"Parsed {len(choices)} choices and usage data: {usage is not None}"
)
return CaptionChatResp(
id=json_data.get("id"),
created=json_data.get("created"),
model=json_data.get("model"),
object=json_data.get("object"),
choices=choices,
usage=usage,
)
def choice_data(self) -> str:
"""
Extract the content from the first choice in the response.
Returns:
The content string from the first choice, or empty string if no choices
"""
if (
not self.choices
or not self.choices[0]
or not self.choices[0].message
or not self.choices[0].message.content
):
logger.warning("No choices available in response")
return ""
logger.info("Retrieving content from first choice")
return self.choices[0].message.content
class Caption:
"""
Service for generating captions for images using a Vision Language Model.
Uses an external API to process images and return textual descriptions.
"""
def __init__(self, vlm_config: Optional[Dict[str, str]] = None):
"""
Initialize the Caption service with configuration
from parameters or environment variables.
"""
logger.info("Initializing Caption service")
# Default prompt for image captioning in Chinese: "Briefly describe the main content of the image"
self.prompt = """简单凝炼的描述图片的主要内容"""
# API request timeout in seconds
self.timeout = 30
# Use provided VLM config if available,
# otherwise fall back to environment variables
if vlm_config and vlm_config.get("base_url") and vlm_config.get("model_name"):
# Build completion URL from provided base URL
self.completion_url = vlm_config.get("base_url", "") + "/chat/completions"
self.model = vlm_config.get("model_name", "")
self.api_key = vlm_config.get("api_key", "")
# Interface type: "ollama" or "openai" (default)
self.interface_type = vlm_config.get("interface_type", "openai").lower()
else:
# Fall back to environment variables if config not provided
self.completion_url = CONFIG.vlm_model_base_url + "/chat/completions"
self.model = CONFIG.vlm_model_name
self.api_key = CONFIG.vlm_model_api_key
self.interface_type = CONFIG.vlm_interface_type
# Validate interface type - must be either "ollama" or "openai"
if self.interface_type not in ["ollama", "openai"]:
logger.warning(
f"Unknown interface type: {self.interface_type}, defaulting to openai"
)
self.interface_type = "openai"
logger.info(
f"Configured with model: {self.model}, "
f"endpoint: {self.completion_url}, interface: {self.interface_type}"
)
def _call_caption_api(self, image_data: str) -> Optional[CaptionChatResp]:
"""
Call the Caption API to generate a description for the given image.
Args:
image_data: URL of the image or base64 encoded image data
Returns:
CaptionChatResp object if successful, None otherwise
"""
logger.info("Calling Caption API for image captioning")
logger.info(f"Processing image data: {image_data[:50]}...")
# Route to appropriate API based on interface type
if self.interface_type == "ollama":
return self._call_ollama_api(image_data)
else:
return self._call_openai_api(image_data)
def _call_ollama_api(self, image_base64: str) -> Optional[CaptionChatResp]:
"""Call Ollama API for image captioning using base64 encoded image data."""
# Extract host URL by removing the chat completions endpoint
# Handle both "/v1/chat/completions" and "/chat/completions" patterns
host = self.completion_url.replace("/v1/chat/completions", "").replace("/chat/completions", "")
# Initialize Ollama client with host and timeout
client = ollama.Client(
host=host,
timeout=self.timeout,
)
try:
logger.info(f"Calling Ollama API with model: {self.model}")
# Call Ollama API with base64 encoded image
# Prompt: "Briefly describe the main content of the image"
response = client.generate(
model=self.model,
prompt="简单凝炼的描述图片的主要内容",
images=[image_base64], # Pass base64 encoded image data
options={
"temperature": 0.1
}, # Low temperature for more deterministic output
stream=False,
)
# Construct response object in standard format
caption_resp = CaptionChatResp(
id="ollama_response",
created=int(time.time()),
model=Model(id=self.model),
object="chat.completion",
choices=[
Choice(message=Message(role="assistant", content=response.response))
],
)
logger.info("Successfully received response from Ollama API")
return caption_resp
except Exception as e:
logger.error(f"Error calling Ollama API: {e}")
return None
def _call_openai_api(self, image_base64: str) -> Optional[CaptionChatResp]:
"""Call OpenAI-compatible API for image captioning."""
logger.info(f"Calling OpenAI-compatible API with model: {self.model}")
# Construct user message with text prompt and base64 encoded image
user_msg = UserMessage(
role="user",
content=[
Content(type="text", text=self.prompt),
Content(
type="image_url",
image_url=ImageUrl(
url="data:image/png;base64," + image_base64, detail="auto"
),
),
],
)
# Build completion request with model parameters
gpt_req = CompletionRequest(
model=self.model,
temperature=0.3, # Moderate randomness for balanced output
top_p=0.8, # Nucleus sampling parameter
messages=[user_msg],
user="abc",
)
# Set up HTTP headers for the API request
headers = {
"Content-Type": "application/json",
"Accept": "text/event-stream",
"Cache-Control": "no-cache",
"Connection": "keep-alive",
}
# Add authorization header if API key is provided
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
try:
logger.info(
f"Sending request to OpenAI-compatible API with model: {self.model}"
)
# Send POST request to the API endpoint
response = requests.post(
self.completion_url,
data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4),
headers=headers,
timeout=self.timeout,
)
# Check for successful response
if response.status_code != 200:
logger.error(
f"OpenAI API returned non-200 status code: {response.status_code}"
)
response.raise_for_status()
logger.info(f"Received from OpenAI with status: {response.status_code}")
logger.info("Converting response to CaptionChatResp object")
# Parse JSON response into structured object
caption_resp = CaptionChatResp.from_json(response.json())
if caption_resp.usage:
logger.info(
f"API usage: prompt_tokens={caption_resp.usage.prompt_tokens}, "
f"completion_tokens={caption_resp.usage.completion_tokens}"
)
return caption_resp
except requests.exceptions.Timeout:
logger.error("Timeout while calling OpenAI-compatible API after 30 seconds")
return None
except requests.exceptions.RequestException as e:
logger.error(f"Request error calling OpenAI-compatible API: {e}")
return None
except Exception as e:
logger.error(f"Unexpected error calling OpenAI-compatible API: {e}")
return None
def get_caption(self, image_data: str) -> str:
"""
Get a caption for the provided image data.
Args:
image_data: URL of the image or base64 encoded image data
Returns:
Caption text as string, or empty string if captioning failed
"""
logger.info("Getting caption for image")
if not image_data or self.completion_url is None:
logger.error("Image data is not set")
return ""
caption_resp = self._call_caption_api(image_data)
if caption_resp:
caption = caption_resp.choice_data()
caption_length = len(caption)
logger.info(f"Successfully generated caption of length {caption_length}")
logger.info(
f"Caption: {caption[:50]}..."
if caption_length > 50
else f"Caption: {caption}"
)
return caption
logger.warning("Failed to get caption from Caption API")
return ""
-101
View File
@@ -1,101 +0,0 @@
"""
CSV Parser Module
This module provides a parser for CSV (Comma-Separated Values) files.
It converts CSV data into a Document with structured chunks, where each row
becomes a separate chunk with key-value pairs.
"""
import logging
from io import BytesIO
from typing import List
import pandas as pd
from docreader.models.document import Chunk, Document
from docreader.parser.base_parser import BaseParser
logger = logging.getLogger(__name__)
class CSVParser(BaseParser):
"""
Parser for CSV files that converts tabular data into structured text.
This parser reads CSV content and transforms each row into a formatted string
with column-value pairs. Each row is stored as a separate Chunk in the Document,
allowing for granular access to individual records.
The output format for each row is:
"column1: value1, column2: value2, column3: value3\n"
Usage:
parser = CSVParser()
with open("data.csv", "rb") as f:
document = parser.parse_into_text(f.read())
"""
def parse_into_text(self, content: bytes) -> Document:
"""Parse CSV content into a Document with structured chunks.
Each row in the CSV is converted into a formatted string and stored as
a separate Chunk. The chunks maintain sequential order and track their
position in the overall document.
Args:
content: Raw bytes content of the CSV file
Returns:
Document: A Document object containing:
- content: Full text with all rows concatenated
- chunks: List of Chunk objects, one per CSV row
Note:
Bad lines in the CSV are automatically skipped using pandas'
on_bad_lines="skip" parameter.
"""
chunks: List[Chunk] = []
text: List[str] = []
start, end = 0, 0
# Read CSV content into a pandas DataFrame, skipping malformed lines
df = pd.read_csv(BytesIO(content), on_bad_lines="skip")
# Process each row in the DataFrame
for i, (idx, row) in enumerate(df.iterrows()):
# Format row as "column: value" pairs separated by commas
content_row = (
",".join(
f"{col.strip()}: {str(row[col]).strip()}" for col in df.columns
)
+ "\n"
)
# Update end position for this chunk
end += len(content_row)
text.append(content_row)
# Create a chunk for this row with position tracking
chunks.append(Chunk(content=content_row, seq=i, start=start, end=end))
# Update start position for next chunk
start = end
return Document(
content="".join(text),
chunks=chunks,
)
if __name__ == "__main__":
# Example usage: Parse a CSV file and display its content
logging.basicConfig(level=logging.DEBUG)
your_file = "/path/to/your/file.csv"
parser = CSVParser()
with open(your_file, "rb") as f:
content = f.read()
document = parser.parse_into_text(content)
# Display full document content
logger.error(document.content)
# Display individual chunks (rows)
for chunk in document.chunks:
logger.error(chunk.content)
+26 -3
View File
@@ -89,11 +89,33 @@ class DocxParser(BaseParser):
logger.info(f"Setting max_workers to {max_workers} for document processing")
try:
inline_images: Dict[str, str] = {}
def _inline_upload(local_path: str) -> str:
"""Read temp image file, base64-encode, and return a ref path.
The Go-side ImageResolver (or main.py _resolve_images) handles
actual storage upload from Document.images.
"""
import base64
import uuid as _uuid
try:
with open(local_path, "rb") as f:
raw = f.read()
ext = os.path.splitext(local_path)[1].lower() or ".png"
ref = f"images/{_uuid.uuid4().hex}{ext}"
inline_images[ref] = base64.b64encode(raw).decode()
return ref
except Exception as exc:
logger.warning("Failed to read temp image %s: %s", local_path, exc)
return ""
logger.info(f"Starting Docx processing with max_pages={self.max_pages}")
docx_processor = Docx(
max_image_size=self.max_image_size,
enable_multimodal=self.enable_multimodal,
upload_file=self.storage.upload_file,
max_image_size=1920,
enable_multimodal=True,
upload_file=_inline_upload,
)
all_lines, tables = docx_processor(
binary=content,
@@ -153,6 +175,7 @@ class DocxParser(BaseParser):
f"generated {len(text)} characters of text"
)
image_parts.update(inline_images)
return DocumentModel(content=text, images=image_parts)
except Exception as e:
logger.error(f"Error parsing DOCX document: {str(e)}")
+9 -29
View File
@@ -5,44 +5,24 @@ import os
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
# Set up logger for this module
logger = logging.getLogger(__name__)
class ImageParser(BaseParser):
"""
Parser for image files with OCR capability.
Extracts text from images and generates captions.
"""Parser for standalone image files.
This parser handles image processing by:
1. Uploading the image to storage
2. Generating a descriptive caption
3. Performing OCR to extract text content
4. Returning a combined result with both text and image reference
Returns the image as a markdown reference with the raw image data
in Document.images so that the Go-side ImageResolver (or main.py's
_resolve_images) can handle storage upload.
"""
def parse_into_text(self, content: bytes) -> Document:
"""
Parse image content into markdown text
:param content: bytes content of the image
:return: Document object
"""
logger.info(f"Parsing image content, size: {len(content)} bytes")
logger.info("Parsing image file=%s, size=%d bytes", self.file_name, len(content))
# Get file extension
ext = os.path.splitext(self.file_name)[1].lower()
ext = os.path.splitext(self.file_name)[1].lower() or ".png"
ref_path = f"images/{self.file_name}"
# Upload image to storage
image_url = self.storage.upload_bytes(content, file_ext=ext)
logger.info(f"Successfully uploaded image, URL: {image_url[:50]}...")
text = f"![{self.file_name}]({ref_path})"
images = {ref_path: base64.b64encode(content).decode()}
if not image_url:
logger.warning(f"Failed to upload image: {self.file_name}")
return Document(content=f"{self.file_name}", images={})
# Generate markdown text
text = f"![{self.file_name}]({image_url})"
images = {image_url: base64.b64encode(content).decode()}
# Create image object and add to map
return Document(content=text, images=images)
+7 -46
View File
@@ -348,23 +348,11 @@ class MarkdownImageUtil:
class MarkdownImageBase64(BaseParser):
"""Parser for extracting and uploading base64 images from Markdown.
"""Parser for extracting base64 images from Markdown.
This parser:
1. Extracts base64-encoded images from Markdown content
2. Uploads the decoded images to storage
3. Replaces base64 data with uploaded URLs
4. Returns a Document with clean Markdown and image mappings
Requires:
- self.storage: Storage backend for uploading images
Example:
>>> parser = MarkdownImageBase64(storage=my_storage)
>>> content = b"![logo](data:image/png;base64,iVBORw0...)"
>>> doc = parser.parse_into_text(content)
>>> print(doc.content)
![logo](https://storage.com/uuid.png)
Extracts base64-encoded images, replaces them with path references,
and returns the raw image data in Document.images for the Go-side
ImageResolver (or main.py _resolve_images) to handle storage.
"""
def __init__(self, **kwargs):
@@ -372,41 +360,14 @@ class MarkdownImageBase64(BaseParser):
self.image_helper = MarkdownImageUtil()
def parse_into_text(self, content: bytes) -> Document:
"""Parse Markdown and process base64 images.
Args:
content: Raw Markdown content as bytes
Returns:
Document with:
- content: Markdown with base64 replaced by URLs
- images: Dict mapping URLs to base64 strings
"""
# Convert byte content to string using universal decoding method
text = endecode.decode_bytes(content)
# Extract base64 images and replace with temporary paths
text, img_b64 = self.image_helper.extract_base64(text, path_prefix="images")
# Final image mapping: URL -> base64 string (for Document)
images: Dict[str, str] = {}
# Temporary mapping: temp_path -> uploaded_URL (for replacement)
image_replace: Dict[str, str] = {}
for ipath, raw_bytes in img_b64.items():
images[ipath] = base64.b64encode(raw_bytes).decode()
logger.debug(f"Uploading {len(img_b64)} images from markdown")
# Upload each extracted image to storage
for ipath, b64_bytes in img_b64.items():
# Get file extension for proper MIME type
ext = os.path.splitext(ipath)[1].lower()
# Upload binary data and get back URL
image_url = self.storage.upload_bytes(b64_bytes, ext)
# Map temp path to uploaded URL for replacement
image_replace[ipath] = image_url
# Store base64 string in final images dict
images[image_url] = base64.b64encode(b64_bytes).decode()
# Replace temporary paths with actual uploaded URLs
text = self.image_helper.replace_path(text, image_replace)
logger.debug("Extracted %d base64 images from markdown", len(images))
return Document(content=text, images=images)
-329
View File
@@ -1,329 +0,0 @@
import logging
import re
import time
from typing import Dict, Optional
import markdownify
import requests
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.chain_parser import PipelineParser
from docreader.parser.markdown_parser import MarkdownImageUtil, MarkdownTableFormatter
from docreader.utils import endecode
logger = logging.getLogger(__name__)
class StdMinerUParser(BaseParser):
"""
Standard MinerU Parser for document parsing.
This parser uses MinerU API to parse documents (especially PDFs) into markdown format,
with support for tables, formulas, and images extraction.
"""
def __init__(
self,
enable_markdownify: bool = True,
mineru_endpoint: Optional[str] = None, # Added: 支持传入自定义 endpoint
**kwargs,
):
"""
Initialize MinerU parser.
Args:
enable_markdownify: Whether to convert HTML tables to markdown format
mineru_endpoint: MinerU API endpoint URL
**kwargs: Additional arguments passed to BaseParser
"""
super().__init__(**kwargs)
# Get MinerU endpoint from environment variable or parameter
# Modified: 优先使用传入的参数,否则使用 Config
base_url = mineru_endpoint if mineru_endpoint else CONFIG.mineru_endpoint
self.minerU = base_url.rstrip("/") if base_url else ""
self.enable_markdownify = enable_markdownify
# Helper for processing markdown images
self.image_helper = MarkdownImageUtil()
# Pattern to match base64 encoded images
self.base64_pattern = re.compile(r"data:image/(\w+);base64,(.*)")
# Check if MinerU API is available
self.enable = self.ping()
def ping(self, timeout: int = 5) -> bool:
"""
Check if MinerU API is available.
Args:
timeout: Request timeout in seconds
Returns:
True if API is available, False otherwise
"""
try:
response = requests.get(
self.minerU + "/docs", timeout=timeout, allow_redirects=True
)
response.raise_for_status()
return True
except Exception:
return False
def parse_into_text(self, content: bytes) -> Document:
"""
Parse document content into text using MinerU API.
Args:
content: Raw document content in bytes
Returns:
Document object containing parsed text and images
"""
if not self.enable:
logger.debug("MinerU API is not enabled")
return Document()
logger.info(f"Parsing scanned PDF via MinerU API (size: {len(content)} bytes)")
md_content: str = ""
images_b64: Dict[str, str] = {}
try:
# Call MinerU API to parse document
response = requests.post(
url=self.minerU + "/file_parse",
data={
"return_md": True, # Return markdown content
"return_images": True, # Return extracted images
"lang_list": ["ch", "en"], # Support Chinese and English
"table_enable": True, # Enable table parsing
"formula_enable": True, # Enable formula parsing
"parse_method": "auto", # Auto detect parsing method
"start_page_id": 0, # Start from first page
"end_page_id": 99999, # Parse all pages
"backend": "pipeline", # Use pipeline backend
"response_format_zip": False, # Return JSON instead of ZIP
"return_middle_json": False, # Don't return intermediate JSON
"return_model_output": False, # Don't return model output
"return_content_list": False, # Don't return content list
},
files={"files": content},
timeout=1000,
)
response.raise_for_status()
result = response.json()["results"]["files"]
md_content = result["md_content"]
images_b64 = result.get("images", {})
except Exception as e:
logger.error(f"MinerU parsing failed: {e}", exc_info=True)
return Document()
# Convert HTML tables in markdown to markdown table format
if self.enable_markdownify:
logger.debug("Converting HTML to Markdown")
md_content = markdownify.markdownify(md_content)
images = {}
image_replace = {}
# Filter images that are actually used in markdown content
# Some images in images_b64 may not be referenced in md_content
# (e.g., images embedded in tables), so we need to filter them
for ipath, b64_str in images_b64.items():
# Skip images that are not referenced in markdown content
if f"images/{ipath}" not in md_content:
logger.debug(f"Image {ipath} not used in markdown")
continue
# Parse base64 image data
match = self.base64_pattern.match(b64_str)
if match:
# Extract image format (e.g., png, jpg)
file_ext = match.group(1)
# Extract base64 encoded data
b64_str = match.group(2)
# Decode base64 string to bytes
image_bytes = endecode.encode_image(b64_str, errors="ignore")
if not image_bytes:
logger.error("Failed to decode base64 image skip it")
continue
# Upload image to storage and get URL
image_url = self.storage.upload_bytes(
image_bytes, file_ext=f".{file_ext}"
)
# Store image mapping for later use
images[image_url] = b64_str
# Prepare replacement mapping for markdown content
image_replace[f"images/{ipath}"] = image_url
logger.info(f"Replaced {len(image_replace)} images in markdown")
# Replace image paths in markdown with uploaded URLs
text = self.image_helper.replace_path(md_content, image_replace)
logger.info(
f"Successfully parsed PDF, text: {len(text)}, images: {len(images)}"
)
return Document(content=text, images=images)
# Added: 新增 MinerUCloudParser 类,支持异步任务提交
class MinerUCloudParser(StdMinerUParser):
"""
MinerU Parser for REMOTE/CLOUD API (Asynchronous).
Uses the /submit -> /status -> /result workflow.
"""
SUBMIT_TIMEOUT = 30
POLL_INTERVAL = 2
MAX_WAIT_TIME = 600
def parse_into_text(self, content: bytes) -> Document:
"""
Parse document content using Cloud MinerU API (Async/Polling).
"""
if not self.enable:
return Document()
logger.info(f"Parsing PDF via Cloud MinerU API (size: {len(content)} bytes)")
try:
# --- Step 1: Submit Task ---
submit_url = f"{self.minerU}/submit"
logger.info(f"Submitting task to {submit_url}")
response = requests.post(
url=submit_url,
files={"files": content},
data={
"enable_formula": "true",
"enable_table": "true",
"layout_model": "doclayout_yolo",
"backend": "pipeline",
},
timeout=self.SUBMIT_TIMEOUT,
)
response.raise_for_status()
# Robust task_id extraction
resp_data = response.json()
task_id = resp_data.get("task_id") or resp_data.get("data", {}).get("task_id")
if not task_id:
raise ValueError(f"No task_id in response: {resp_data}")
logger.info(f"Task submitted, ID: {task_id}, waiting for completion...")
# --- Step 2: Poll Status ---
start_time = time.time()
while True:
if time.time() - start_time > self.MAX_WAIT_TIME:
raise TimeoutError(f"Task {task_id} timed out after {self.MAX_WAIT_TIME}s")
try:
status_resp = requests.get(
f"{self.minerU}/status/{task_id}",
timeout=10
)
status_resp.raise_for_status()
status_data = status_resp.json()
except requests.RequestException as e:
logger.warning(f"Status check failed for {task_id}: {e}. Retrying...")
time.sleep(self.POLL_INTERVAL)
continue
state = status_data.get("status") or status_data.get("state")
if state in ["done", "success"]:
break
elif state == "failed":
error_msg = status_data.get("error") or "Unknown error"
raise RuntimeError(f"Task {task_id} failed: {error_msg}")
else:
time.sleep(self.POLL_INTERVAL)
# --- Step 3: Get Result ---
result_resp = requests.get(
f"{self.minerU}/result/{task_id}",
timeout=30
)
result_resp.raise_for_status()
result_json = result_resp.json()
# Normalize result data
result_data = result_json.get("result", result_json)
md_content = result_data.get("md_content", "")
images_b64 = result_data.get("images", {})
# 使用父类的方法处理图片和Markdown转换 (复用现有逻辑)
# Convert HTML tables
if self.enable_markdownify:
md_content = markdownify.markdownify(md_content)
images = {}
image_replace = {}
for ipath, b64_str in images_b64.items():
if f"images/{ipath}" not in md_content:
continue
match = self.base64_pattern.match(b64_str)
if match:
file_ext = match.group(1)
b64_str_clean = match.group(2)
image_bytes = endecode.encode_image(b64_str_clean, errors="ignore")
if not image_bytes: continue
if self.storage:
image_url = self.storage.upload_bytes(image_bytes, file_ext=f".{file_ext}")
images[image_url] = b64_str_clean
image_replace[f"images/{ipath}"] = image_url
if image_replace:
md_content = self.image_helper.replace_path(md_content, image_replace)
return Document(content=md_content, images=images)
except Exception as e:
logger.error(f"Cloud MinerU parsing failed: {e}", exc_info=True)
return Document()
class MinerUParser(PipelineParser):
"""
MinerU Parser with pipeline processing.
This parser combines StdMinerUParser for document parsing and
MarkdownTableFormatter for table formatting in a pipeline.
"""
_parser_cls = (StdMinerUParser, MarkdownTableFormatter)
if __name__ == "__main__":
import os
# Example usage for testing
logging.basicConfig(level=logging.DEBUG)
# Configure your file path and MinerU endpoint
your_file = "/path/to/your/file.pdf"
# Added: 修改为 Localhost 方便测试
test_endpoint = "http://localhost:9987"
os.environ["MINERU_ENDPOINT"] = test_endpoint
# Create parser instance
# Modified: 传入 endpoint
parser = MinerUParser(mineru_endpoint=test_endpoint)
# Parse PDF file
if os.path.exists(your_file):
with open(your_file, "rb") as f:
content = f.read()
document = parser.parse_into_text(content)
logger.error(document.content)
else:
print(f"File not found: {your_file}")
+42 -138
View File
@@ -1,178 +1,82 @@
import logging
from typing import Dict, Type
from typing import Any, Optional
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.models.read_config import ChunkingConfig
from docreader.parser.base_parser import BaseParser
from docreader.parser.csv_parser import CSVParser
from docreader.parser.doc_parser import DocParser
from docreader.parser.docx2_parser import Docx2Parser
from docreader.parser.excel_parser import ExcelParser
from docreader.parser.image_parser import ImageParser
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.pdf_parser import PDFParser
from docreader.parser.text_parser import TextParser
from docreader.parser.registry import registry
from docreader.parser.web_parser import WebParser
logger = logging.getLogger(__name__)
class Parser:
"""
Document parser facade that integrates all specialized parsers.
Provides a unified interface for parsing various document types.
"""Document parser facade (lightweight version).
Converts files/URLs to markdown + image references.
No chunking, no storage, no OCR, no VLM.
"""
def __init__(self):
# Initialize all parser types - maps file extensions to their corresponding parser classes
self.parsers: Dict[str, Type[BaseParser]] = {
# Document formats
"docx": Docx2Parser,
"doc": DocParser,
"pdf": PDFParser,
"md": MarkdownParser,
"txt": TextParser,
# Image formats - all use the same ImageParser
"jpg": ImageParser,
"jpeg": ImageParser,
"png": ImageParser,
"gif": ImageParser,
"bmp": ImageParser,
"tiff": ImageParser,
"webp": ImageParser,
# Alternative markdown extension
"markdown": MarkdownParser,
# Spreadsheet formats
"csv": CSVParser,
"xlsx": ExcelParser,
"xls": ExcelParser,
}
self.registry = registry
logger.info(
"Parser initialized with %d parsers: %s",
len(self.parsers),
", ".join(self.parsers.keys()),
"Parser initialized with engines: %s",
", ".join(self.registry.get_engine_names()),
)
def get_parser(self, file_type: str) -> Type[BaseParser]:
"""
Get parser class for the specified file type.
Args:
file_type: The file extension or type identifier
Returns:
Parser class for the file type, or None if unsupported
"""
# Look up parser by file type (case-insensitive)
parser = self.parsers.get(file_type.lower())
if not parser:
# Raise error if file type is not supported
raise ValueError(f"Unsupported file type: {file_type}")
return parser
def parse_file(
self,
file_name: str,
file_type: str,
content: bytes,
config: ChunkingConfig,
parser_engine: Optional[str] = None,
engine_overrides: Optional[dict[str, Any]] = None,
) -> Document:
"""
Parse file content using appropriate parser based on file type.
Args:
file_name: Name of the file being parsed
file_type: Type/extension of the file
content: Raw file content as bytes
config: Configuration for chunking process
Returns:
ParseResult containing chunks and metadata, or None if parsing failed
"""
logger.info(f"Parsing file: {file_name} with type: {file_type}")
"""Parse file content to markdown."""
engine = parser_engine or ""
overrides = engine_overrides or {}
logger.info(
f"Chunking config: size={config.chunk_size}, "
f"overlap={config.chunk_overlap}, "
f"multimodal={config.enable_multimodal}"
"Parsing file: %s, type: %s, engine: %s",
file_name,
file_type,
engine or "builtin",
)
# Get appropriate parser class for the file type
cls = self.get_parser(file_type)
# Create parser instance with configuration
logger.info(f"Creating parser instance for {file_type} file")
cls = self.registry.get_parser_class(engine, file_type)
logger.info(
"Creating %s parser instance for %s file",
cls.__name__,
file_type,
)
parser = cls(
file_name=file_name,
file_type=file_type,
chunk_size=config.chunk_size, # Size of each text chunk
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
chunking_config=config, # Pass the entire chunking config for advanced options
max_concurrent_tasks=CONFIG.image_max_concurrent,
ocr_backend=CONFIG.ocr_backend,
**overrides,
)
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
# Execute the parsing process
logger.info("Starting to parse file content, size: %d bytes", len(content))
result = parser.parse(content)
# Validate parsing results and log warnings if needed
if not result.content:
logger.warning(f"Parser returned empty content for file: {file_name}")
elif not result.chunks:
logger.warning(f"Parser returned empty chunks for file: {file_name}")
elif result.chunks[0]:
# Log first chunk size for debugging
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
logger.info(f"Parsed file {file_name}, with {len(result.chunks)} chunks")
logger.warning("Parser returned empty content for file: %s", file_name)
logger.info(
"Parsed file %s, content length=%d", file_name, len(result.content)
)
return result
def parse_url(self, url: str, title: str, config: ChunkingConfig) -> Document:
"""
Parse content from a URL using the WebParser.
Args:
url: URL to parse
title: Title of the webpage (for metadata)
config: Configuration for chunking process
Returns:
ParseResult containing chunks and metadata, or None if parsing failed
"""
logger.info(f"Parsing URL: {url}, title: {title}")
logger.info(
f"Chunking config: size={config.chunk_size}, "
f"overlap={config.chunk_overlap}, multimodal={config.enable_multimodal}"
)
# Create web parser instance with configuration
logger.info("Creating WebParser instance")
parser = WebParser(
title=title, # Webpage title for metadata
chunk_size=config.chunk_size, # Size of each text chunk
chunk_overlap=config.chunk_overlap, # Overlap between consecutive chunks
separators=config.separators, # Text separators for chunking
enable_multimodal=config.enable_multimodal, # Enable image/multimodal processing
max_image_size=1920, # Limit image size to 1920px for performance
chunking_config=config, # Pass the entire chunking config
max_concurrent_tasks=CONFIG.image_max_concurrent,
ocr_backend=CONFIG.ocr_backend,
)
def parse_url(
self,
url: str,
title: str,
parser_engine: Optional[str] = None,
engine_overrides: Optional[dict[str, Any]] = None,
) -> Document:
"""Parse content from a URL to markdown."""
logger.info("Parsing URL: %s, title: %s", url, title)
parser = WebParser(title=title)
logger.info("Starting to parse URL content")
# Parse URL content (encode URL string to bytes as required by parser interface)
result = parser.parse(url.encode())
# Validate parsing results and log warnings if needed
if not result.content:
logger.warning(f"Parser returned empty content for url: {url}")
elif not result.chunks:
logger.warning(f"Parser returned empty chunks for url: {url}")
elif result.chunks[0]:
# Log first chunk size for debugging
logger.info(f"First chunk content length: {len(result.chunks[0].content)}")
logger.info(f"Parsed url {url}, with {len(result.chunks)} chunks")
logger.warning("Parser returned empty content for url: %s", url)
logger.info("Parsed url %s, content length=%d", url, len(result.content))
return result
+1 -2
View File
@@ -1,6 +1,5 @@
from docreader.parser.chain_parser import FirstParser
from docreader.parser.markitdown_parser import MarkitdownParser
from docreader.parser.mineru_parser import MinerUParser
class PDFParser(FirstParser):
@@ -13,4 +12,4 @@ class PDFParser(FirstParser):
The first successful parser result will be returned.
"""
# Parser classes to try in order (chain of responsibility pattern)
_parser_cls = (MinerUParser, MarkitdownParser)
_parser_cls = (MarkitdownParser,)
+160
View File
@@ -0,0 +1,160 @@
import logging
from typing import Any, Callable, Dict, List, Optional, Tuple, Type
from docreader.parser.base_parser import BaseParser
from docreader.parser.doc_parser import DocParser
from docreader.parser.docx2_parser import Docx2Parser
from docreader.parser.excel_parser import ExcelParser
from docreader.parser.image_parser import ImageParser
from docreader.parser.markdown_parser import MarkdownParser
from docreader.parser.markitdown_parser import MarkitdownParser
from docreader.parser.pdf_parser import PDFParser
logger = logging.getLogger(__name__)
BUILTIN_ENGINE = "builtin"
class ParserEngineRegistry:
"""Registry for parser engines.
Each engine maps file extensions to parser classes.
When a requested engine doesn't support a file type, the registry
falls back to the builtin engine automatically.
"""
def __init__(self):
self._engines: Dict[str, Dict[str, Type[BaseParser]]] = {}
self._descriptions: Dict[str, str] = {}
self._check_available: Dict[str, Callable[..., Tuple[bool, str]]] = {}
self._unavailable_hint: Dict[str, str] = {}
def register(
self,
name: str,
file_types: Dict[str, Type[BaseParser]],
description: str = "",
check_available: Callable[..., Tuple[bool, str]] | None = None,
unavailable_hint: str = "",
):
self._engines[name] = file_types
self._descriptions[name] = description
if check_available is not None:
self._check_available[name] = check_available
self._unavailable_hint[name] = unavailable_hint
logger.info(
"Registered parser engine '%s' with file types: %s",
name,
", ".join(file_types.keys()),
)
def get_parser_class(self, engine: str, file_type: str) -> Type[BaseParser]:
"""Resolve parser class for the given engine and file type.
Falls back to builtin engine when the requested engine doesn't
support the file type.
"""
ft = file_type.lower()
if engine and engine in self._engines:
cls = self._engines[engine].get(ft)
if cls:
logger.info("Using engine '%s' for file type '%s'", engine, ft)
return cls
logger.info(
"Engine '%s' does not support '%s', falling back to builtin",
engine,
ft,
)
builtin = self._engines.get(BUILTIN_ENGINE, {})
cls = builtin.get(ft)
if cls:
return cls
raise ValueError(f"Unsupported file type: {file_type}")
def list_engines(self, overrides: Optional[Dict[str, str]] = None) -> List[Dict]:
"""Return metadata for all registered engines, including availability.
Args:
overrides: tenant-level config overrides (e.g. mineru_endpoint, mineru_api_key)
forwarded to each engine's check_available function.
"""
result = []
for name, parsers in self._engines.items():
available = True
unavailable_reason = ""
check = self._check_available.get(name)
if check is not None:
try:
available, unavailable_reason = check(overrides)
except Exception as e:
available = False
unavailable_reason = str(e) or self._unavailable_hint.get(name, "")
if not available and not unavailable_reason:
unavailable_reason = self._unavailable_hint.get(name, "不可用")
result.append(
{
"name": name,
"description": self._descriptions.get(name, ""),
"file_types": sorted(parsers.keys()),
"available": available,
"unavailable_reason": unavailable_reason,
}
)
return result
def get_engine_names(self) -> List[str]:
return list(self._engines.keys())
def _build_default_registry() -> ParserEngineRegistry:
"""Create and populate the default registry with all known engines."""
reg = ParserEngineRegistry()
_image_types = {
ext: ImageParser for ext in ("jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp")
}
reg.register(
BUILTIN_ENGINE,
{
"docx": Docx2Parser,
"doc": DocParser,
"pdf": PDFParser,
"md": MarkdownParser,
"markdown": MarkdownParser,
"xlsx": ExcelParser,
"xls": ExcelParser,
**_image_types,
},
description="内置解析引擎",
)
reg.register(
"markitdown",
{
"md": MarkitdownParser,
"markdown": MarkitdownParser,
"pdf": MarkitdownParser,
"docx": MarkitdownParser,
"doc": MarkitdownParser,
"pptx": MarkitdownParser,
"ppt": MarkitdownParser,
"xlsx": MarkitdownParser,
"xls": MarkitdownParser,
"csv": MarkitdownParser,
},
description="MarkItDown 解析引擎(微软 MarkItDown 库)",
)
# NOTE: Engine listing is managed by Go-side engine registry
# (docparser.ListAllEngines). The Python list_engines method is kept for
# backward compatibility with the gRPC ListEngines RPC but the Go app
# no longer calls it. MinerU engines are handled natively by Go.
return reg
registry = _build_default_registry()
+98 -222
View File
@@ -5,96 +5,70 @@ import os
import traceback
import uuid
from abc import ABC, abstractmethod
from typing import Dict
from typing import Dict, Optional
from minio import Minio
from qcloud_cos import CosConfig, CosS3Client
from docreader.config import CONFIG
from docreader.utils import endecode
logger = logging.getLogger(__name__)
def _cfg(storage_config: Optional[Dict], key: str, *env_keys: str, default: str = "") -> str:
"""Read a value from storage_config dict, falling back to env vars."""
if storage_config:
v = storage_config.get(key, "")
if v:
return str(v)
for ek in env_keys:
v = os.environ.get(ek, "")
if v:
return v
return default
class Storage(ABC):
"""Abstract base class for object storage operations"""
@abstractmethod
def upload_file(self, file_path: str) -> str:
"""Upload file to object storage
Args:
file_path: File path
Returns:
File URL
"""
pass
@abstractmethod
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to object storage
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
pass
class CosStorage(Storage):
"""Tencent Cloud COS storage implementation"""
def __init__(self, storage_config=None):
"""Initialize COS storage
Args:
storage_config: Storage configuration
"""
def __init__(self, storage_config: Optional[Dict] = None):
self.storage_config = storage_config
self.client, self.bucket_name, self.region, self.prefix = (
self._init_cos_client()
)
def _init_cos_client(self):
"""Initialize Tencent Cloud COS client"""
try:
# Use provided COS config if available,
# otherwise fall back to environment variables
if self.storage_config and self.storage_config.get("access_key_id") != "":
cos_config = self.storage_config
secret_id = cos_config.get("access_key_id")
secret_key = cos_config.get("secret_access_key")
region = cos_config.get("region")
bucket_name = cos_config.get("bucket_name")
appid = cos_config.get("app_id")
prefix = cos_config.get("path_prefix", "")
else:
# Get COS configuration from environment variables
secret_id = CONFIG.cos_secret_id
secret_key = CONFIG.cos_secret_key
region = CONFIG.cos_region
bucket_name = CONFIG.cos_bucket_name
appid = CONFIG.cos_app_id
prefix = CONFIG.cos_path_prefix
enable_old_domain = CONFIG.cos_enable_old_domain
sc = self.storage_config
secret_id = _cfg(sc, "access_key_id", "COS_SECRET_ID")
secret_key = _cfg(sc, "secret_access_key", "COS_SECRET_KEY")
region = _cfg(sc, "region", "COS_REGION")
bucket_name = _cfg(sc, "bucket_name", "COS_BUCKET_NAME")
appid = _cfg(sc, "app_id", "COS_APP_ID")
prefix = _cfg(sc, "path_prefix", "COS_PATH_PREFIX")
enable_old_domain = os.environ.get("COS_ENABLE_OLD_DOMAIN", "").lower() in ("1", "true", "yes")
if not all([secret_id, secret_key, region, bucket_name, appid]):
logger.error(
"Incomplete COS configuration, missing environment variables"
f"secret_id: {secret_id}, secret_key: {secret_key}, "
f"region: {region}, bucket_name: {bucket_name}, appid: {appid}"
"Incomplete COS configuration: "
"secret_id=%s, region=%s, bucket=%s, appid=%s",
bool(secret_id), region, bucket_name, appid,
)
return None, None, None, None
# Initialize COS configuration
logger.info(
f"Initializing COS client with region: {region}, bucket: {bucket_name}"
)
logger.info("Initializing COS client: region=%s, bucket=%s", region, bucket_name)
config = CosConfig(
Appid=appid,
Region=region,
@@ -102,93 +76,50 @@ class CosStorage(Storage):
SecretKey=secret_key,
EnableOldDomain=enable_old_domain,
)
# Create client
client = CosS3Client(config)
return client, bucket_name, region, prefix
except Exception as e:
logger.error(f"Failed to initialize COS client: {str(e)}")
logger.error("Failed to initialize COS client: %s", e)
return None, None, None, None
def _get_download_url(self, bucket_name, region, object_key):
"""Generate COS object URL
Args:
bucket_name: Bucket name
region: Region
object_key: Object key
Returns:
File URL
"""
return f"https://{bucket_name}.cos.{region}.myqcloud.com/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to Tencent Cloud COS
Args:
file_path: File path
Returns:
File URL
"""
logger.info(f"Uploading file to COS: {file_path}")
try:
if not self.client:
return ""
# Generate object key, use UUID to avoid conflicts
file_ext = os.path.splitext(file_path)[1]
object_key = f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
logger.info(f"Generated object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to COS")
self.client.upload_file(
Bucket=self.bucket_name,
LocalFilePath=file_path,
Key=object_key,
)
# Get file URL
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
logger.info(f"Successfully uploaded file to COS: {file_url}")
logger.info("COS upload_file ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload file to COS: {str(e)}")
logger.error("COS upload_file failed: %s", e)
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to Tencent Cloud COS
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
try:
logger.info(f"Uploading bytes content to COS, size: {len(content)} bytes")
if not self.client:
return ""
object_key = (
f"{self.prefix}/images/{uuid.uuid4().hex}{file_ext}"
if self.prefix
else f"images/{uuid.uuid4().hex}{file_ext}"
)
logger.info(f"Generated object key: {object_key}")
self.client.put_object(
Bucket=self.bucket_name, Body=content, Key=object_key
)
file_url = self._get_download_url(self.bucket_name, self.region, object_key)
logger.info(f"Successfully uploaded bytes to COS: {file_url}")
logger.info("COS upload_bytes ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload bytes to COS: {str(e)}")
logger.error("COS upload_bytes failed: %s", e)
traceback.print_exc()
return ""
@@ -196,67 +127,34 @@ class CosStorage(Storage):
class MinioStorage(Storage):
"""MinIO storage implementation"""
def __init__(self, storage_config=None):
"""Initialize MinIO storage
Args:
storage_config: Storage configuration
"""
def __init__(self, storage_config: Optional[Dict] = None):
self.storage_config = storage_config
self.client, self.bucket_name, self.use_ssl, self.endpoint, self.path_prefix = (
self._init_minio_client()
)
def _init_minio_client(self):
"""Initialize MinIO client from environment variables or injected config.
If storage_config contains valid configuration, prefer those values
to override environment variables.
"""
try:
# Get configuration from storage_config with environment variables as fallback
# Each field can independently fall back to environment variables
access_key = (
self.storage_config.get("access_key_id")
if self.storage_config and self.storage_config.get("access_key_id")
else CONFIG.minio_access_key_id
)
secret_key = (
self.storage_config.get("secret_access_key")
if self.storage_config and self.storage_config.get("secret_access_key")
else CONFIG.minio_secret_access_key
)
bucket_name = (
self.storage_config.get("bucket_name")
if self.storage_config and self.storage_config.get("bucket_name")
else CONFIG.minio_bucket_name
)
path_prefix_raw = (
self.storage_config.get("path_prefix")
if self.storage_config and self.storage_config.get("path_prefix")
else CONFIG.minio_path_prefix
)
sc = self.storage_config
access_key = _cfg(sc, "access_key_id", "MINIO_ACCESS_KEY_ID")
secret_key = _cfg(sc, "secret_access_key", "MINIO_SECRET_ACCESS_KEY")
bucket_name = _cfg(sc, "bucket_name", "MINIO_BUCKET_NAME")
path_prefix_raw = _cfg(sc, "path_prefix", "MINIO_PATH_PREFIX")
path_prefix = path_prefix_raw.strip().strip("/") if path_prefix_raw else ""
endpoint = CONFIG.minio_endpoint
use_ssl = CONFIG.minio_use_ssl
endpoint = _cfg(sc, "endpoint", "MINIO_ENDPOINT")
use_ssl = os.environ.get("MINIO_USE_SSL", "").lower() in ("1", "true", "yes")
if not all([endpoint, access_key, secret_key, bucket_name]):
logger.error(
"Incomplete MinIO configuration, missing environment variables"
)
logger.error("Incomplete MinIO configuration")
return None, None, None, None, None
# Initialize client
client = Minio(
endpoint, access_key=access_key, secret_key=secret_key, secure=use_ssl
)
# Ensure bucket exists
found = client.bucket_exists(bucket_name)
if not found:
client.make_bucket(bucket_name)
# Set public read policy for the bucket
policy = (
"{"
'"Version":"2012-10-17",'
@@ -273,51 +171,26 @@ class MinioStorage(Storage):
return client, bucket_name, use_ssl, endpoint, path_prefix
except Exception as e:
logger.error(f"Failed to initialize MinIO client: {str(e)}")
logger.error("Failed to initialize MinIO client: %s", e)
return None, None, None, None, None
def _get_download_url(self, object_key: str):
"""Construct a public URL for MinIO object.
If MINIO_PUBLIC_ENDPOINT is provided, use it; otherwise fallback to endpoint.
"""
# 1. Use public endpoint if provided
endpoint = CONFIG.minio_public_endpoint
if endpoint:
return f"{endpoint}/{self.bucket_name}/{object_key}"
# 2. Use SSL if enabled
if self.use_ssl:
return f"https://{self.endpoint}/{self.bucket_name}/{object_key}"
# 3. Use HTTP default
return f"http://{self.endpoint}/{self.bucket_name}/{object_key}"
public_endpoint = os.environ.get("MINIO_PUBLIC_ENDPOINT", "")
if public_endpoint:
return f"{public_endpoint}/{self.bucket_name}/{object_key}"
scheme = "https" if self.use_ssl else "http"
return f"{scheme}://{self.endpoint}/{self.bucket_name}/{object_key}"
def upload_file(self, file_path: str) -> str:
"""Upload file to MinIO
Args:
file_path: File path
Returns:
File URL
"""
logger.info(f"Uploading file to MinIO: {file_path}")
try:
if not self.client:
return ""
# Generate object key, use UUID to avoid conflicts
file_name = os.path.basename(file_path)
object_key = (
f"{self.path_prefix}/images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
if self.path_prefix
else f"images/{uuid.uuid4().hex}{os.path.splitext(file_name)[1]}"
)
logger.info(f"Generated MinIO object key: {object_key}")
# Upload file
logger.info("Attempting to upload file to MinIO")
with open(file_path, "rb") as file_data:
file_size = os.path.getsize(file_path)
self.client.put_object(
@@ -327,38 +200,22 @@ class MinioStorage(Storage):
length=file_size,
content_type="application/octet-stream",
)
# Get file URL
file_url = self._get_download_url(object_key)
logger.info(f"Successfully uploaded file to MinIO: {file_url}")
logger.info("MinIO upload_file ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload file to MinIO: {str(e)}")
logger.error("MinIO upload_file failed: %s", e)
return ""
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
"""Upload bytes to MinIO
Args:
content: Byte content to upload
file_ext: File extension
Returns:
File URL
"""
try:
logger.info(f"Uploading bytes content to MinIO, size: {len(content)} bytes")
if not self.client:
return ""
object_key = (
f"{self.path_prefix}/images/{uuid.uuid4().hex}{file_ext}"
if self.path_prefix
else f"images/{uuid.uuid4().hex}{file_ext}"
)
logger.info(f"Generated MinIO object key: {object_key}")
self.client.put_object(
self.bucket_name or "",
object_key,
@@ -367,52 +224,67 @@ class MinioStorage(Storage):
content_type="application/octet-stream",
)
file_url = self._get_download_url(object_key)
logger.info(f"Successfully uploaded bytes to MinIO: {file_url}")
logger.info("MinIO upload_bytes ok: %s", file_url)
return file_url
except Exception as e:
logger.error(f"Failed to upload bytes to MinIO: {str(e)}")
logger.error("MinIO upload_bytes failed: %s", e)
traceback.print_exc()
return ""
class LocalStorage(Storage):
"""Local file system storage implementation"""
"""Local file system storage implementation.
def __init__(self, storage_config: Dict[str, str] = {}):
self.storage_config = storage_config
base_dir = storage_config.get("base_dir", CONFIG.local_storage_base_dir)
self.image_dir = os.path.join(base_dir, "images")
Saves files under base_dir and returns web-accessible URL paths
(e.g. /files/images/uuid.jpg) so that the Go app can serve them.
"""
def __init__(self, storage_config: Optional[Dict] = None):
sc = storage_config or {}
self.base_dir = (
sc.get("base_dir")
or os.environ.get("LOCAL_STORAGE_BASE_DIR", "/data/files")
)
path_prefix = (sc.get("path_prefix") or "").strip().strip("/")
if path_prefix:
self.image_dir = os.path.join(self.base_dir, path_prefix, "images")
else:
self.image_dir = os.path.join(self.base_dir, "images")
self.url_prefix = (
sc.get("url_prefix")
or os.environ.get("LOCAL_STORAGE_URL_PREFIX", "/files")
)
os.makedirs(self.image_dir, exist_ok=True)
def _to_url(self, fpath: str) -> str:
if self.url_prefix:
rel = os.path.relpath(fpath, self.base_dir)
return f"{self.url_prefix}/{rel}"
return fpath
def upload_file(self, file_path: str) -> str:
logger.info(f"Uploading file to local storage: {file_path}")
return file_path
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
logger.info(f"Uploading file to local storage: {len(content)} bytes")
fname = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
with open(fname, "wb") as f:
fpath = os.path.join(self.image_dir, f"{uuid.uuid4()}{file_ext}")
with open(fpath, "wb") as f:
f.write(content)
return fname
url = self._to_url(fpath)
logger.info("Local storage saved: %s -> %s", fpath, url)
return url
class Base64Storage(Storage):
def upload_file(self, file_path: str) -> str:
logger.info(f"Uploading file to base64 storage: {file_path}")
return file_path
def upload_bytes(self, content: bytes, file_ext: str = ".png") -> str:
logger.info(f"Uploading file to base64 storage: {len(content)} bytes")
file_ext = file_ext.lstrip(".")
return f"data:image/{file_ext};base64,{endecode.decode_image(content)}"
class DummyStorage(Storage):
"""Dummy storage implementation.
It is used in tests or in environments where object storage is disabled.
All upload methods return empty string.
"""
"""Dummy storage — all uploads return empty string."""
def upload_file(self, file_path: str) -> str:
return ""
@@ -421,26 +293,30 @@ class DummyStorage(Storage):
return ""
def create_storage(storage_config: Dict[str, str] | None = None) -> Storage:
"""Create a storage instance based on configuration or environment variables
def create_storage(storage_config: Optional[Dict[str, str]] = None) -> Storage:
"""Create a storage instance based on storage_config dict.
Args:
storage_config: Storage configuration dictionary
Returns:
Storage instance
The ``provider`` key in storage_config determines the backend:
minio, cos, local, base64.
Falls back to STORAGE_TYPE env var, then ``local``.
"""
storage_type = CONFIG.storage_type
storage_type = ""
if storage_config:
storage_type = str(storage_config.get("provider", storage_type)).lower()
logger.info(f"Creating {storage_type} storage instance")
provider = str(storage_config.get("provider", "")).lower().strip()
if provider and provider not in ("unspecified", "storage_provider_unspecified"):
storage_type = provider
if not storage_type:
storage_type = os.environ.get("STORAGE_TYPE", "local").lower().strip()
logger.info("Creating %s storage instance", storage_type)
if storage_type == "minio":
return MinioStorage(storage_config)
elif storage_type == "cos":
return CosStorage(storage_config)
elif storage_type == "local":
return LocalStorage(storage_config or {})
return LocalStorage(storage_config)
elif storage_type == "base64":
return Base64Storage()
return DummyStorage()
-55
View File
@@ -1,55 +0,0 @@
import logging
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.utils import endecode
logger = logging.getLogger(__name__)
class TextParser(BaseParser):
"""
Text document parser for processing plain text files.
This parser handles text extraction and chunking from plain text documents.
"""
def parse_into_text(self, content: bytes) -> Document:
"""
Parse text document content by decoding bytes to string.
This is a straightforward parser that simply converts the binary content
to text using appropriate character encoding.
Args:
content: Raw document content as bytes
Returns:
Parsed text content as string
"""
logger.info(f"Parsing text document, content size: {len(content)} bytes")
text = endecode.decode_bytes(content)
logger.info(
f"Successfully parsed text document, extracted {len(text)} characters"
)
return Document(content=text)
if __name__ == "__main__":
logger = logging.getLogger(__name__)
# Sample text for testing
text = """## 标题1
![alt text](image.png)
## 标题2
![alt text](image2.png)
## 标题3
![alt text](image3.png)"""
logger.info(f"Test text content: {text}")
# Define separators for text splitting
seperators = ["\n\n", "\n", "。"]
parser = TextParser(separators=seperators)
logger.info("Splitting text into units")
units = parser._split_into_units(text)
logger.info(f"Split text into {len(units)} units")
logger.info(f"Units: {units}")
File diff suppressed because it is too large Load Diff
+42 -72
View File
@@ -4,86 +4,56 @@ package docreader;
option go_package = "github.com/Tencent/WeKnora/internal/docreader/proto";
// 文档读取服务
service DocReader {
// 从文件读取文档
rpc ReadFromFile(ReadFromFileRequest) returns (ReadResponse) {}
// 从URL读取文档
rpc ReadFromURL(ReadFromURLRequest) returns (ReadResponse) {}
}
// 对象存储提供方
enum StorageProvider {
STORAGE_PROVIDER_UNSPECIFIED = 0;
COS = 1; // 腾讯云 COS
MINIO = 2; // MinIO/S3 兼容
}
// 通用对象存储配置,兼容 COS 与 MinIO
message StorageConfig {
StorageProvider provider = 1; // 存储提供方
string region = 2; // 区域(COS 使用)
string bucket_name = 3; // 桶名
string access_key_id = 4; // 访问密钥 ID(MinIO/S3 使用)
string secret_access_key = 5; // 访问密钥 Secret(MinIO/S3 使用)
string app_id = 6; // 应用 ID(COS 使用)
string path_prefix = 7; // 路径前缀
}
// VLM 配置
message VLMConfig {
string model_name = 1; // VLM Model Name
string base_url = 2; // VLM Base URL
string api_key = 3; // VLM API Key
string interface_type = 4; // VLM Interface Type: "ollama" or "openai"
rpc Read(ReadRequest) returns (ReadResponse) {}
rpc ListEngines(ListEnginesRequest) returns (ListEnginesResponse) {}
}
message ReadConfig {
int32 chunk_size = 1; // 分块大小
int32 chunk_overlap = 2; // 分块重叠
repeated string separators = 3; // 分隔符
bool enable_multimodal = 4; // 多模态处理
StorageConfig storage_config = 5; // 对象存储配置(通用)
VLMConfig vlm_config = 6; // VLM 配置
string parser_engine = 1;
map<string, string> parser_engine_overrides = 2;
map<string, string> image_storage = 3; // storage provider config (provider, endpoint, bucket, ...)
}
// 从文件读取文档请求
message ReadFromFileRequest {
bytes file_content = 1; // 文件内容
string file_name = 2; // 文件名
string file_type = 3; // 文件类型
ReadConfig read_config = 4;
string request_id = 5;
// Unified read request: set file_content for file mode, url for URL mode.
message ReadRequest {
bytes file_content = 1;
string file_name = 2;
string file_type = 3;
string url = 4;
string title = 5;
ReadConfig config = 6;
string request_id = 7;
}
// 从URL读取文档请求
message ReadFromURLRequest {
string url = 1; // 文档URL
string title = 2; // 标题
ReadConfig read_config = 3;
string request_id = 4;
message ImageRef {
string filename = 1;
string original_ref = 2;
string mime_type = 3;
string storage_key = 4; // download URL from shared storage
bytes image_data = 5; // inline bytes fallback
}
// 图片信息
message Image {
string url = 1; // 图片URL
string caption = 2; // 图片描述
string ocr_text = 3; // OCR提取的文本内容
string original_url = 4; // 原始图片URL
int32 start = 5; // 图片在文本中的开始位置
int32 end = 6; // 图片在文本中的结束位置
}
message Chunk {
string content = 1; // 块内容
int32 seq = 2; // 块在文档中的次序
int32 start = 3; // 块在文档中的起始位置
int32 end = 4; // 块在文档中的结束位置
repeated Image images = 5; // 块中包含的图片信息
}
// 从URL读取文档响应
message ReadResponse {
repeated Chunk chunks = 1; // 文档分块
string error = 2; // 错误信息
}
string markdown_content = 1;
repeated ImageRef image_refs = 2;
string image_dir_path = 3;
map<string, string> metadata = 4;
string error = 5;
}
message ListEnginesRequest {
map<string, string> config_overrides = 1;
}
message ParserEngineInfo {
string name = 1;
string description = 2;
repeated string file_types = 3;
bool available = 4;
string unavailable_reason = 5;
}
message ListEnginesResponse {
repeated ParserEngineInfo engines = 1;
}
+68 -38
View File
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.5.1
// - protoc v5.29.3
// - protoc v6.33.4
// source: docreader.proto
package proto
@@ -19,20 +19,18 @@ import (
const _ = grpc.SupportPackageIsVersion9
const (
DocReader_ReadFromFile_FullMethodName = "/docreader.DocReader/ReadFromFile"
DocReader_ReadFromURL_FullMethodName = "/docreader.DocReader/ReadFromURL"
DocReader_Read_FullMethodName = "/docreader.DocReader/Read"
DocReader_ListEngines_FullMethodName = "/docreader.DocReader/ListEngines"
DocReader_ConvertToPDF_FullMethodName = "/docreader.DocReader/ConvertToPDF"
)
// DocReaderClient is the client API for DocReader service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
//
// 文档读取服务
type DocReaderClient interface {
// 从文件读取文档
ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error)
// 从URL读取文档
ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error)
Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error)
ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error)
ConvertToPDF(ctx context.Context, in *ConvertToPDFRequest, opts ...grpc.CallOption) (*ConvertToPDFResponse, error)
}
type docReaderClient struct {
@@ -43,20 +41,30 @@ func NewDocReaderClient(cc grpc.ClientConnInterface) DocReaderClient {
return &docReaderClient{cc}
}
func (c *docReaderClient) ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
func (c *docReaderClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ReadResponse)
err := c.cc.Invoke(ctx, DocReader_ReadFromFile_FullMethodName, in, out, cOpts...)
err := c.cc.Invoke(ctx, DocReader_Read_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *docReaderClient) ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
func (c *docReaderClient) ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ReadResponse)
err := c.cc.Invoke(ctx, DocReader_ReadFromURL_FullMethodName, in, out, cOpts...)
out := new(ListEnginesResponse)
err := c.cc.Invoke(ctx, DocReader_ListEngines_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *docReaderClient) ConvertToPDF(ctx context.Context, in *ConvertToPDFRequest, opts ...grpc.CallOption) (*ConvertToPDFResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ConvertToPDFResponse)
err := c.cc.Invoke(ctx, DocReader_ConvertToPDF_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
@@ -66,13 +74,10 @@ func (c *docReaderClient) ReadFromURL(ctx context.Context, in *ReadFromURLReques
// DocReaderServer is the server API for DocReader service.
// All implementations must embed UnimplementedDocReaderServer
// for forward compatibility.
//
// 文档读取服务
type DocReaderServer interface {
// 从文件读取文档
ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error)
// 从URL读取文档
ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error)
Read(context.Context, *ReadRequest) (*ReadResponse, error)
ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error)
ConvertToPDF(context.Context, *ConvertToPDFRequest) (*ConvertToPDFResponse, error)
mustEmbedUnimplementedDocReaderServer()
}
@@ -83,11 +88,14 @@ type DocReaderServer interface {
// pointer dereference when methods are called.
type UnimplementedDocReaderServer struct{}
func (UnimplementedDocReaderServer) ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReadFromFile not implemented")
func (UnimplementedDocReaderServer) Read(context.Context, *ReadRequest) (*ReadResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Read not implemented")
}
func (UnimplementedDocReaderServer) ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReadFromURL not implemented")
func (UnimplementedDocReaderServer) ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListEngines not implemented")
}
func (UnimplementedDocReaderServer) ConvertToPDF(context.Context, *ConvertToPDFRequest) (*ConvertToPDFResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ConvertToPDF not implemented")
}
func (UnimplementedDocReaderServer) mustEmbedUnimplementedDocReaderServer() {}
func (UnimplementedDocReaderServer) testEmbeddedByValue() {}
@@ -110,38 +118,56 @@ func RegisterDocReaderServer(s grpc.ServiceRegistrar, srv DocReaderServer) {
s.RegisterService(&DocReader_ServiceDesc, srv)
}
func _DocReader_ReadFromFile_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReadFromFileRequest)
func _DocReader_Read_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReadRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(DocReaderServer).ReadFromFile(ctx, in)
return srv.(DocReaderServer).Read(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: DocReader_ReadFromFile_FullMethodName,
FullMethod: DocReader_Read_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(DocReaderServer).ReadFromFile(ctx, req.(*ReadFromFileRequest))
return srv.(DocReaderServer).Read(ctx, req.(*ReadRequest))
}
return interceptor(ctx, in, info, handler)
}
func _DocReader_ReadFromURL_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReadFromURLRequest)
func _DocReader_ListEngines_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListEnginesRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(DocReaderServer).ReadFromURL(ctx, in)
return srv.(DocReaderServer).ListEngines(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: DocReader_ReadFromURL_FullMethodName,
FullMethod: DocReader_ListEngines_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(DocReaderServer).ReadFromURL(ctx, req.(*ReadFromURLRequest))
return srv.(DocReaderServer).ListEngines(ctx, req.(*ListEnginesRequest))
}
return interceptor(ctx, in, info, handler)
}
func _DocReader_ConvertToPDF_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ConvertToPDFRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(DocReaderServer).ConvertToPDF(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: DocReader_ConvertToPDF_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(DocReaderServer).ConvertToPDF(ctx, req.(*ConvertToPDFRequest))
}
return interceptor(ctx, in, info, handler)
}
@@ -154,12 +180,16 @@ var DocReader_ServiceDesc = grpc.ServiceDesc{
HandlerType: (*DocReaderServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "ReadFromFile",
Handler: _DocReader_ReadFromFile_Handler,
MethodName: "Read",
Handler: _DocReader_Read_Handler,
},
{
MethodName: "ReadFromURL",
Handler: _DocReader_ReadFromURL_Handler,
MethodName: "ListEngines",
Handler: _DocReader_ListEngines_Handler,
},
{
MethodName: "ConvertToPDF",
Handler: _DocReader_ConvertToPDF_Handler,
},
},
Streams: []grpc.StreamDesc{},
+37 -21
View File
@@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xb9\x01\n\rStorageConfig\x12,\n\x08provider\x18\x01 \x01(\x0e\x32\x1a.docreader.StorageProvider\x12\x0e\n\x06region\x18\x02 \x01(\t\x12\x13\n\x0b\x62ucket_name\x18\x03 \x01(\t\x12\x15\n\raccess_key_id\x18\x04 \x01(\t\x12\x19\n\x11secret_access_key\x18\x05 \x01(\t\x12\x0e\n\x06\x61pp_id\x18\x06 \x01(\t\x12\x13\n\x0bpath_prefix\x18\x07 \x01(\t\"Z\n\tVLMConfig\x12\x12\n\nmodel_name\x18\x01 \x01(\t\x12\x10\n\x08\x62\x61se_url\x18\x02 \x01(\t\x12\x0f\n\x07\x61pi_key\x18\x03 \x01(\t\x12\x16\n\x0einterface_type\x18\x04 \x01(\t\"\xc2\x01\n\nReadConfig\x12\x12\n\nchunk_size\x18\x01 \x01(\x05\x12\x15\n\rchunk_overlap\x18\x02 \x01(\x05\x12\x12\n\nseparators\x18\x03 \x03(\t\x12\x19\n\x11\x65nable_multimodal\x18\x04 \x01(\x08\x12\x30\n\x0estorage_config\x18\x05 \x01(\x0b\x32\x18.docreader.StorageConfig\x12(\n\nvlm_config\x18\x06 \x01(\x0b\x32\x14.docreader.VLMConfig\"\x91\x01\n\x13ReadFromFileRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12*\n\x0bread_config\x18\x04 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x05 \x01(\t\"p\n\x12ReadFromURLRequest\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\r\n\x05title\x18\x02 \x01(\t\x12*\n\x0bread_config\x18\x03 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x04 \x01(\t\"i\n\x05Image\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0f\n\x07\x63\x61ption\x18\x02 \x01(\t\x12\x10\n\x08ocr_text\x18\x03 \x01(\t\x12\x14\n\x0coriginal_url\x18\x04 \x01(\t\x12\r\n\x05start\x18\x05 \x01(\x05\x12\x0b\n\x03\x65nd\x18\x06 \x01(\x05\"c\n\x05\x43hunk\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\t\x12\x0b\n\x03seq\x18\x02 \x01(\x05\x12\r\n\x05start\x18\x03 \x01(\x05\x12\x0b\n\x03\x65nd\x18\x04 \x01(\x05\x12 \n\x06images\x18\x05 \x03(\x0b\x32\x10.docreader.Image\"?\n\x0cReadResponse\x12 \n\x06\x63hunks\x18\x01 \x03(\x0b\x32\x10.docreader.Chunk\x12\r\n\x05\x65rror\x18\x02 \x01(\t*G\n\x0fStorageProvider\x12 \n\x1cSTORAGE_PROVIDER_UNSPECIFIED\x10\x00\x12\x07\n\x03\x43OS\x10\x01\x12\t\n\x05MINIO\x10\x02\x32\x9f\x01\n\tDocReader\x12I\n\x0cReadFromFile\x12\x1e.docreader.ReadFromFileRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n\x0bReadFromURL\x12\x1d.docreader.ReadFromURLRequest\x1a\x17.docreader.ReadResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xa9\x02\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x12>\n\rimage_storage\x18\x03 \x03(\x0b\x32\'.docreader.ReadConfig.ImageStorageEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11ImageStorageEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"\xe2\x01\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo\"Q\n\x13\x43onvertToPDFRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\":\n\x14\x43onvertToPDFResponse\x12\x13\n\x0bpdf_content\x18\x01 \x01(\x0c\x12\r\n\x05\x65rror\x18\x02 \x01(\t2\xe9\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x12Q\n\x0c\x43onvertToPDF\x12\x1e.docreader.ConvertToPDFRequest\x1a\x1f.docreader.ConvertToPDFResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -32,24 +32,40 @@ _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'docreader_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals['DESCRIPTOR']._loaded_options = None
_globals['DESCRIPTOR']._serialized_options = b'Z3github.com/Tencent/WeKnora/internal/docreader/proto'
_globals['_STORAGEPROVIDER']._serialized_start=1042
_globals['_STORAGEPROVIDER']._serialized_end=1113
_globals['_STORAGECONFIG']._serialized_start=31
_globals['_STORAGECONFIG']._serialized_end=216
_globals['_VLMCONFIG']._serialized_start=218
_globals['_VLMCONFIG']._serialized_end=308
_globals['_READCONFIG']._serialized_start=311
_globals['_READCONFIG']._serialized_end=505
_globals['_READFROMFILEREQUEST']._serialized_start=508
_globals['_READFROMFILEREQUEST']._serialized_end=653
_globals['_READFROMURLREQUEST']._serialized_start=655
_globals['_READFROMURLREQUEST']._serialized_end=767
_globals['_IMAGE']._serialized_start=769
_globals['_IMAGE']._serialized_end=874
_globals['_CHUNK']._serialized_start=876
_globals['_CHUNK']._serialized_end=975
_globals['_READRESPONSE']._serialized_start=977
_globals['_READRESPONSE']._serialized_end=1040
_globals['_DOCREADER']._serialized_start=1116
_globals['_DOCREADER']._serialized_end=1275
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._loaded_options = None
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_options = b'8\001'
_globals['_READCONFIG_IMAGESTORAGEENTRY']._loaded_options = None
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_options = b'8\001'
_globals['_READRESPONSE_METADATAENTRY']._loaded_options = None
_globals['_READRESPONSE_METADATAENTRY']._serialized_options = b'8\001'
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._loaded_options = None
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_options = b'8\001'
_globals['_READCONFIG']._serialized_start=31
_globals['_READCONFIG']._serialized_end=328
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_start=215
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_end=275
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_start=277
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_end=328
_globals['_READREQUEST']._serialized_start=331
_globals['_READREQUEST']._serialized_end=491
_globals['_IMAGEREF']._serialized_start=493
_globals['_IMAGEREF']._serialized_end=603
_globals['_READRESPONSE']._serialized_start=606
_globals['_READRESPONSE']._serialized_end=832
_globals['_READRESPONSE_METADATAENTRY']._serialized_start=785
_globals['_READRESPONSE_METADATAENTRY']._serialized_end=832
_globals['_LISTENGINESREQUEST']._serialized_start=835
_globals['_LISTENGINESREQUEST']._serialized_end=989
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=935
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=989
_globals['_PARSERENGINEINFO']._serialized_start=991
_globals['_PARSERENGINEINFO']._serialized_end=1111
_globals['_LISTENGINESRESPONSE']._serialized_start=1113
_globals['_LISTENGINESRESPONSE']._serialized_end=1180
_globals['_CONVERTTOPDFREQUEST']._serialized_start=1182
_globals['_CONVERTTOPDFREQUEST']._serialized_end=1263
_globals['_CONVERTTOPDFRESPONSE']._serialized_start=1265
_globals['_CONVERTTOPDFRESPONSE']._serialized_end=1323
_globals['_DOCREADER']._serialized_start=1326
_globals['_DOCREADER']._serialized_end=1559
# @@protoc_insertion_point(module_scope)
+110 -102
View File
@@ -1,5 +1,4 @@
from google.protobuf.internal import containers as _containers
from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from collections.abc import Iterable as _Iterable, Mapping as _Mapping
@@ -7,121 +6,130 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
DESCRIPTOR: _descriptor.FileDescriptor
class StorageProvider(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
__slots__ = ()
STORAGE_PROVIDER_UNSPECIFIED: _ClassVar[StorageProvider]
COS: _ClassVar[StorageProvider]
MINIO: _ClassVar[StorageProvider]
STORAGE_PROVIDER_UNSPECIFIED: StorageProvider
COS: StorageProvider
MINIO: StorageProvider
class StorageConfig(_message.Message):
__slots__ = ("provider", "region", "bucket_name", "access_key_id", "secret_access_key", "app_id", "path_prefix")
PROVIDER_FIELD_NUMBER: _ClassVar[int]
REGION_FIELD_NUMBER: _ClassVar[int]
BUCKET_NAME_FIELD_NUMBER: _ClassVar[int]
ACCESS_KEY_ID_FIELD_NUMBER: _ClassVar[int]
SECRET_ACCESS_KEY_FIELD_NUMBER: _ClassVar[int]
APP_ID_FIELD_NUMBER: _ClassVar[int]
PATH_PREFIX_FIELD_NUMBER: _ClassVar[int]
provider: StorageProvider
region: str
bucket_name: str
access_key_id: str
secret_access_key: str
app_id: str
path_prefix: str
def __init__(self, provider: _Optional[_Union[StorageProvider, str]] = ..., region: _Optional[str] = ..., bucket_name: _Optional[str] = ..., access_key_id: _Optional[str] = ..., secret_access_key: _Optional[str] = ..., app_id: _Optional[str] = ..., path_prefix: _Optional[str] = ...) -> None: ...
class VLMConfig(_message.Message):
__slots__ = ("model_name", "base_url", "api_key", "interface_type")
MODEL_NAME_FIELD_NUMBER: _ClassVar[int]
BASE_URL_FIELD_NUMBER: _ClassVar[int]
API_KEY_FIELD_NUMBER: _ClassVar[int]
INTERFACE_TYPE_FIELD_NUMBER: _ClassVar[int]
model_name: str
base_url: str
api_key: str
interface_type: str
def __init__(self, model_name: _Optional[str] = ..., base_url: _Optional[str] = ..., api_key: _Optional[str] = ..., interface_type: _Optional[str] = ...) -> None: ...
class ReadConfig(_message.Message):
__slots__ = ("chunk_size", "chunk_overlap", "separators", "enable_multimodal", "storage_config", "vlm_config")
CHUNK_SIZE_FIELD_NUMBER: _ClassVar[int]
CHUNK_OVERLAP_FIELD_NUMBER: _ClassVar[int]
SEPARATORS_FIELD_NUMBER: _ClassVar[int]
ENABLE_MULTIMODAL_FIELD_NUMBER: _ClassVar[int]
STORAGE_CONFIG_FIELD_NUMBER: _ClassVar[int]
VLM_CONFIG_FIELD_NUMBER: _ClassVar[int]
chunk_size: int
chunk_overlap: int
separators: _containers.RepeatedScalarFieldContainer[str]
enable_multimodal: bool
storage_config: StorageConfig
vlm_config: VLMConfig
def __init__(self, chunk_size: _Optional[int] = ..., chunk_overlap: _Optional[int] = ..., separators: _Optional[_Iterable[str]] = ..., enable_multimodal: bool = ..., storage_config: _Optional[_Union[StorageConfig, _Mapping]] = ..., vlm_config: _Optional[_Union[VLMConfig, _Mapping]] = ...) -> None: ...
__slots__ = ("parser_engine", "parser_engine_overrides", "image_storage")
class ParserEngineOverridesEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
VALUE_FIELD_NUMBER: _ClassVar[int]
key: str
value: str
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
class ImageStorageEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
VALUE_FIELD_NUMBER: _ClassVar[int]
key: str
value: str
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
PARSER_ENGINE_FIELD_NUMBER: _ClassVar[int]
PARSER_ENGINE_OVERRIDES_FIELD_NUMBER: _ClassVar[int]
IMAGE_STORAGE_FIELD_NUMBER: _ClassVar[int]
parser_engine: str
parser_engine_overrides: _containers.ScalarMap[str, str]
image_storage: _containers.ScalarMap[str, str]
def __init__(self, parser_engine: _Optional[str] = ..., parser_engine_overrides: _Optional[_Mapping[str, str]] = ..., image_storage: _Optional[_Mapping[str, str]] = ...) -> None: ...
class ReadFromFileRequest(_message.Message):
__slots__ = ("file_content", "file_name", "file_type", "read_config", "request_id")
class ReadRequest(_message.Message):
__slots__ = ("file_content", "file_name", "file_type", "url", "title", "config", "request_id")
FILE_CONTENT_FIELD_NUMBER: _ClassVar[int]
FILE_NAME_FIELD_NUMBER: _ClassVar[int]
FILE_TYPE_FIELD_NUMBER: _ClassVar[int]
READ_CONFIG_FIELD_NUMBER: _ClassVar[int]
URL_FIELD_NUMBER: _ClassVar[int]
TITLE_FIELD_NUMBER: _ClassVar[int]
CONFIG_FIELD_NUMBER: _ClassVar[int]
REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
file_content: bytes
file_name: str
file_type: str
read_config: ReadConfig
request_id: str
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ..., read_config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
class ReadFromURLRequest(_message.Message):
__slots__ = ("url", "title", "read_config", "request_id")
URL_FIELD_NUMBER: _ClassVar[int]
TITLE_FIELD_NUMBER: _ClassVar[int]
READ_CONFIG_FIELD_NUMBER: _ClassVar[int]
REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
url: str
title: str
read_config: ReadConfig
config: ReadConfig
request_id: str
def __init__(self, url: _Optional[str] = ..., title: _Optional[str] = ..., read_config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ..., url: _Optional[str] = ..., title: _Optional[str] = ..., config: _Optional[_Union[ReadConfig, _Mapping]] = ..., request_id: _Optional[str] = ...) -> None: ...
class Image(_message.Message):
__slots__ = ("url", "caption", "ocr_text", "original_url", "start", "end")
URL_FIELD_NUMBER: _ClassVar[int]
CAPTION_FIELD_NUMBER: _ClassVar[int]
OCR_TEXT_FIELD_NUMBER: _ClassVar[int]
ORIGINAL_URL_FIELD_NUMBER: _ClassVar[int]
START_FIELD_NUMBER: _ClassVar[int]
END_FIELD_NUMBER: _ClassVar[int]
url: str
caption: str
ocr_text: str
original_url: str
start: int
end: int
def __init__(self, url: _Optional[str] = ..., caption: _Optional[str] = ..., ocr_text: _Optional[str] = ..., original_url: _Optional[str] = ..., start: _Optional[int] = ..., end: _Optional[int] = ...) -> None: ...
class Chunk(_message.Message):
__slots__ = ("content", "seq", "start", "end", "images")
CONTENT_FIELD_NUMBER: _ClassVar[int]
SEQ_FIELD_NUMBER: _ClassVar[int]
START_FIELD_NUMBER: _ClassVar[int]
END_FIELD_NUMBER: _ClassVar[int]
IMAGES_FIELD_NUMBER: _ClassVar[int]
content: str
seq: int
start: int
end: int
images: _containers.RepeatedCompositeFieldContainer[Image]
def __init__(self, content: _Optional[str] = ..., seq: _Optional[int] = ..., start: _Optional[int] = ..., end: _Optional[int] = ..., images: _Optional[_Iterable[_Union[Image, _Mapping]]] = ...) -> None: ...
class ImageRef(_message.Message):
__slots__ = ("filename", "original_ref", "mime_type", "storage_key", "image_data")
FILENAME_FIELD_NUMBER: _ClassVar[int]
ORIGINAL_REF_FIELD_NUMBER: _ClassVar[int]
MIME_TYPE_FIELD_NUMBER: _ClassVar[int]
STORAGE_KEY_FIELD_NUMBER: _ClassVar[int]
IMAGE_DATA_FIELD_NUMBER: _ClassVar[int]
filename: str
original_ref: str
mime_type: str
storage_key: str
image_data: bytes
def __init__(self, filename: _Optional[str] = ..., original_ref: _Optional[str] = ..., mime_type: _Optional[str] = ..., storage_key: _Optional[str] = ..., image_data: _Optional[bytes] = ...) -> None: ...
class ReadResponse(_message.Message):
__slots__ = ("chunks", "error")
CHUNKS_FIELD_NUMBER: _ClassVar[int]
__slots__ = ("markdown_content", "image_refs", "image_dir_path", "metadata", "error")
class MetadataEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
VALUE_FIELD_NUMBER: _ClassVar[int]
key: str
value: str
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
MARKDOWN_CONTENT_FIELD_NUMBER: _ClassVar[int]
IMAGE_REFS_FIELD_NUMBER: _ClassVar[int]
IMAGE_DIR_PATH_FIELD_NUMBER: _ClassVar[int]
METADATA_FIELD_NUMBER: _ClassVar[int]
ERROR_FIELD_NUMBER: _ClassVar[int]
chunks: _containers.RepeatedCompositeFieldContainer[Chunk]
markdown_content: str
image_refs: _containers.RepeatedCompositeFieldContainer[ImageRef]
image_dir_path: str
metadata: _containers.ScalarMap[str, str]
error: str
def __init__(self, chunks: _Optional[_Iterable[_Union[Chunk, _Mapping]]] = ..., error: _Optional[str] = ...) -> None: ...
def __init__(self, markdown_content: _Optional[str] = ..., image_refs: _Optional[_Iterable[_Union[ImageRef, _Mapping]]] = ..., image_dir_path: _Optional[str] = ..., metadata: _Optional[_Mapping[str, str]] = ..., error: _Optional[str] = ...) -> None: ...
class ListEnginesRequest(_message.Message):
__slots__ = ("config_overrides",)
class ConfigOverridesEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
VALUE_FIELD_NUMBER: _ClassVar[int]
key: str
value: str
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
CONFIG_OVERRIDES_FIELD_NUMBER: _ClassVar[int]
config_overrides: _containers.ScalarMap[str, str]
def __init__(self, config_overrides: _Optional[_Mapping[str, str]] = ...) -> None: ...
class ParserEngineInfo(_message.Message):
__slots__ = ("name", "description", "file_types", "available", "unavailable_reason")
NAME_FIELD_NUMBER: _ClassVar[int]
DESCRIPTION_FIELD_NUMBER: _ClassVar[int]
FILE_TYPES_FIELD_NUMBER: _ClassVar[int]
AVAILABLE_FIELD_NUMBER: _ClassVar[int]
UNAVAILABLE_REASON_FIELD_NUMBER: _ClassVar[int]
name: str
description: str
file_types: _containers.RepeatedScalarFieldContainer[str]
available: bool
unavailable_reason: str
def __init__(self, name: _Optional[str] = ..., description: _Optional[str] = ..., file_types: _Optional[_Iterable[str]] = ..., available: bool = ..., unavailable_reason: _Optional[str] = ...) -> None: ...
class ListEnginesResponse(_message.Message):
__slots__ = ("engines",)
ENGINES_FIELD_NUMBER: _ClassVar[int]
engines: _containers.RepeatedCompositeFieldContainer[ParserEngineInfo]
def __init__(self, engines: _Optional[_Iterable[_Union[ParserEngineInfo, _Mapping]]] = ...) -> None: ...
class ConvertToPDFRequest(_message.Message):
__slots__ = ("file_content", "file_name", "file_type")
FILE_CONTENT_FIELD_NUMBER: _ClassVar[int]
FILE_NAME_FIELD_NUMBER: _ClassVar[int]
FILE_TYPE_FIELD_NUMBER: _ClassVar[int]
file_content: bytes
file_name: str
file_type: str
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ...) -> None: ...
class ConvertToPDFResponse(_message.Message):
__slots__ = ("pdf_content", "error")
PDF_CONTENT_FIELD_NUMBER: _ClassVar[int]
ERROR_FIELD_NUMBER: _ClassVar[int]
pdf_content: bytes
error: str
def __init__(self, pdf_content: _Optional[bytes] = ..., error: _Optional[str] = ...) -> None: ...
+73 -35
View File
@@ -3,9 +3,9 @@
import grpc
import warnings
import docreader_pb2 as docreader__pb2
from docreader.proto import docreader_pb2 as docreader__pb2
GRPC_GENERATED_VERSION = '1.76.0'
GRPC_GENERATED_VERSION = '1.78.0'
GRPC_VERSION = grpc.__version__
_version_not_supported = False
@@ -26,8 +26,7 @@ if _version_not_supported:
class DocReaderStub(object):
"""文档读取服务
"""
"""Missing associated documentation comment in .proto file."""
def __init__(self, channel):
"""Constructor.
@@ -35,32 +34,40 @@ class DocReaderStub(object):
Args:
channel: A grpc.Channel.
"""
self.ReadFromFile = channel.unary_unary(
'/docreader.DocReader/ReadFromFile',
request_serializer=docreader__pb2.ReadFromFileRequest.SerializeToString,
self.Read = channel.unary_unary(
'/docreader.DocReader/Read',
request_serializer=docreader__pb2.ReadRequest.SerializeToString,
response_deserializer=docreader__pb2.ReadResponse.FromString,
_registered_method=True)
self.ReadFromURL = channel.unary_unary(
'/docreader.DocReader/ReadFromURL',
request_serializer=docreader__pb2.ReadFromURLRequest.SerializeToString,
response_deserializer=docreader__pb2.ReadResponse.FromString,
self.ListEngines = channel.unary_unary(
'/docreader.DocReader/ListEngines',
request_serializer=docreader__pb2.ListEnginesRequest.SerializeToString,
response_deserializer=docreader__pb2.ListEnginesResponse.FromString,
_registered_method=True)
self.ConvertToPDF = channel.unary_unary(
'/docreader.DocReader/ConvertToPDF',
request_serializer=docreader__pb2.ConvertToPDFRequest.SerializeToString,
response_deserializer=docreader__pb2.ConvertToPDFResponse.FromString,
_registered_method=True)
class DocReaderServicer(object):
"""文档读取服务
"""
"""Missing associated documentation comment in .proto file."""
def ReadFromFile(self, request, context):
"""从文件读取文档
"""
def Read(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def ReadFromURL(self, request, context):
"""从URL读取文档
"""
def ListEngines(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def ConvertToPDF(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
@@ -68,15 +75,20 @@ class DocReaderServicer(object):
def add_DocReaderServicer_to_server(servicer, server):
rpc_method_handlers = {
'ReadFromFile': grpc.unary_unary_rpc_method_handler(
servicer.ReadFromFile,
request_deserializer=docreader__pb2.ReadFromFileRequest.FromString,
'Read': grpc.unary_unary_rpc_method_handler(
servicer.Read,
request_deserializer=docreader__pb2.ReadRequest.FromString,
response_serializer=docreader__pb2.ReadResponse.SerializeToString,
),
'ReadFromURL': grpc.unary_unary_rpc_method_handler(
servicer.ReadFromURL,
request_deserializer=docreader__pb2.ReadFromURLRequest.FromString,
response_serializer=docreader__pb2.ReadResponse.SerializeToString,
'ListEngines': grpc.unary_unary_rpc_method_handler(
servicer.ListEngines,
request_deserializer=docreader__pb2.ListEnginesRequest.FromString,
response_serializer=docreader__pb2.ListEnginesResponse.SerializeToString,
),
'ConvertToPDF': grpc.unary_unary_rpc_method_handler(
servicer.ConvertToPDF,
request_deserializer=docreader__pb2.ConvertToPDFRequest.FromString,
response_serializer=docreader__pb2.ConvertToPDFResponse.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
@@ -87,11 +99,10 @@ def add_DocReaderServicer_to_server(servicer, server):
# This class is part of an EXPERIMENTAL API.
class DocReader(object):
"""文档读取服务
"""
"""Missing associated documentation comment in .proto file."""
@staticmethod
def ReadFromFile(request,
def Read(request,
target,
options=(),
channel_credentials=None,
@@ -104,8 +115,8 @@ class DocReader(object):
return grpc.experimental.unary_unary(
request,
target,
'/docreader.DocReader/ReadFromFile',
docreader__pb2.ReadFromFileRequest.SerializeToString,
'/docreader.DocReader/Read',
docreader__pb2.ReadRequest.SerializeToString,
docreader__pb2.ReadResponse.FromString,
options,
channel_credentials,
@@ -118,7 +129,7 @@ class DocReader(object):
_registered_method=True)
@staticmethod
def ReadFromURL(request,
def ListEngines(request,
target,
options=(),
channel_credentials=None,
@@ -131,9 +142,36 @@ class DocReader(object):
return grpc.experimental.unary_unary(
request,
target,
'/docreader.DocReader/ReadFromURL',
docreader__pb2.ReadFromURLRequest.SerializeToString,
docreader__pb2.ReadResponse.FromString,
'/docreader.DocReader/ListEngines',
docreader__pb2.ListEnginesRequest.SerializeToString,
docreader__pb2.ListEnginesResponse.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True)
@staticmethod
def ConvertToPDF(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(
request,
target,
'/docreader.DocReader/ConvertToPDF',
docreader__pb2.ConvertToPDFRequest.SerializeToString,
docreader__pb2.ConvertToPDFResponse.FromString,
options,
channel_credentials,
insecure,
+11 -7
View File
@@ -1,5 +1,5 @@
#!/bin/bash
set -x
set -ex
# 设置目录
PROTO_DIR="docreader/proto"
@@ -13,12 +13,16 @@ python3 -m grpc_tools.protoc -I${PROTO_DIR} \
--grpc_python_out=${PYTHON_OUT} \
${PROTO_DIR}/docreader.proto
# 生成Go代码
protoc -I${PROTO_DIR} --go_out=${GO_OUT} \
--go_opt=paths=source_relative \
--go-grpc_out=${GO_OUT} \
--go-grpc_opt=paths=source_relative \
${PROTO_DIR}/docreader.proto
# 生成Go代码(仅在 protoc-gen-go 可用时执行)
if command -v protoc-gen-go &> /dev/null; then
protoc -I${PROTO_DIR} --go_out=${GO_OUT} \
--go_opt=paths=source_relative \
--go-grpc_out=${GO_OUT} \
--go-grpc_opt=paths=source_relative \
${PROTO_DIR}/docreader.proto
else
echo "protoc-gen-go not found, skipping Go code generation"
fi
# 修复Python导入问题(MacOS兼容版本)
if [ "$(uname)" == "Darwin" ]; then