refactor: simplify image handling in _resolve_images function

- Removed the image storage handling from the `_resolve_images` function, which now only decodes images and returns them as inline bytes.
- Updated the function's docstring to clarify that image persistence is managed by the Go App, and the return value for `image_dir_path` is always empty.
- Adjusted related gRPC and protobuf files to remove the `ConvertToPDF` method and its associated types, streamlining the API.
- Enhanced security measures in the frontend by implementing a placeholder for provider images and ensuring proper hydration of protected file images.

These changes improve the clarity and efficiency of image processing and API interactions within the application.
This commit is contained in:
wizardchen
2026-03-03 21:30:27 +08:00
committed by lyingbug
parent 894d2c060a
commit 4192cfd072
26 changed files with 835 additions and 897 deletions
+8 -53
View File
@@ -52,16 +52,16 @@ init_logging_request_id()
def _resolve_images(images: dict, request_id: str, storage_map: dict | None = None) -> tuple[str, list]:
"""Resolve document images with priority: shared storage > temp dir > inline bytes.
"""Resolve document images into inline bytes for the Go App to persist.
``images`` is a dict of {relative_path: raw_data} where raw_data is
base64-encoded string or raw bytes.
``storage_map`` is a dict from the request's config.image_storage map.
When it contains a valid "provider" key, images are uploaded to shared object
storage and storage_key is set (no inline bytes, saving gRPC message size).
The Go App is solely responsible for persisting images to the configured
storage backend (local/minio/cos/tos). This function only decodes images
and returns them as inline bytes via ImageRef.
Returns (image_dir_path, list[ImageRef]).
Returns ("", list[ImageRef]). image_dir_path is always empty.
"""
import base64
@@ -73,28 +73,6 @@ def _resolve_images(images: dict, request_id: str, storage_map: dict | None = No
".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
}
storage_client = None
use_storage = False
provider = (storage_map or {}).get("provider", "")
if provider:
try:
from docreader.parser.storage import create_storage
storage_client = create_storage(storage_map)
use_storage = True
logger.info("Using shared storage (%s) for image upload", provider)
except Exception as e:
logger.warning("Failed to init shared storage, falling back to inline bytes: %s", e)
base_dir = CONFIG.image_output_dir
output_dir = os.path.join(base_dir, request_id, "images")
wrote_to_dir = False
if not use_storage:
try:
os.makedirs(output_dir, exist_ok=True)
wrote_to_dir = True
except OSError:
logger.warning("Cannot write to image output dir %s", output_dir)
refs = []
for ref_path, b64data in images.items():
try:
@@ -106,26 +84,6 @@ def _resolve_images(images: dict, request_id: str, storage_map: dict | None = No
ext = os.path.splitext(fname)[1].lower()
mime = mime_map.get(ext, "application/octet-stream")
if use_storage and storage_client:
storage_key = storage_client.upload_bytes(img_bytes, ext)
if storage_key:
refs.append(ImageRef(
filename=fname,
original_ref=ref_path,
mime_type=mime,
storage_key=storage_key,
))
continue
logger.warning("Storage upload failed for %s, falling back to inline bytes", fname)
if wrote_to_dir:
dest = os.path.join(output_dir, fname)
try:
with open(dest, "wb") as f:
f.write(img_bytes)
except OSError as e:
logger.warning("Failed to write image %s: %s", dest, e)
refs.append(ImageRef(
filename=fname,
original_ref=ref_path,
@@ -133,10 +91,8 @@ def _resolve_images(images: dict, request_id: str, storage_map: dict | None = No
image_data=img_bytes,
))
image_dir = os.path.join(base_dir, request_id) if wrote_to_dir else ""
mode = "storage" if use_storage else ("dir+inline" if wrote_to_dir else "inline")
logger.info("Resolved %d images (mode=%s)", len(refs), mode)
return image_dir, refs
logger.info("Resolved %d images (mode=inline)", len(refs))
return "", refs
class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
@@ -154,7 +110,6 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
cfg = request.config
parser_engine = cfg.parser_engine if cfg else ""
engine_overrides = dict(cfg.parser_engine_overrides) if cfg else {}
storage_map = dict(cfg.image_storage) if cfg and cfg.image_storage else None
if is_url:
logger.info("Read(URL): url=%s", request.url)
@@ -189,7 +144,7 @@ class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
_c = to_valid_utf8_text
image_dir, image_refs = _resolve_images(
result.images, request_id, storage_map=storage_map
result.images, request_id
)
response = ReadResponse(
+32 -171
View File
@@ -25,7 +25,6 @@ type ReadConfig struct {
state protoimpl.MessageState `protogen:"open.v1"`
ParserEngine string `protobuf:"bytes,1,opt,name=parser_engine,json=parserEngine,proto3" json:"parser_engine,omitempty"`
ParserEngineOverrides map[string]string `protobuf:"bytes,2,rep,name=parser_engine_overrides,json=parserEngineOverrides,proto3" json:"parser_engine_overrides,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"`
ImageStorage map[string]string `protobuf:"bytes,3,rep,name=image_storage,json=imageStorage,proto3" json:"image_storage,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // storage provider config (provider, endpoint, bucket, ...)
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -74,13 +73,6 @@ func (x *ReadConfig) GetParserEngineOverrides() map[string]string {
return nil
}
func (x *ReadConfig) GetImageStorage() map[string]string {
if x != nil {
return x.ImageStorage
}
return nil
}
// Unified read request: set file_content for file mode, url for URL mode.
type ReadRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -490,134 +482,18 @@ func (x *ListEnginesResponse) GetEngines() []*ParserEngineInfo {
return nil
}
type ConvertToPDFRequest struct {
state protoimpl.MessageState `protogen:"open.v1"`
FileContent []byte `protobuf:"bytes,1,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"`
FileName string `protobuf:"bytes,2,opt,name=file_name,json=fileName,proto3" json:"file_name,omitempty"`
FileType string `protobuf:"bytes,3,opt,name=file_type,json=fileType,proto3" json:"file_type,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ConvertToPDFRequest) Reset() {
*x = ConvertToPDFRequest{}
mi := &file_docreader_proto_msgTypes[7]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ConvertToPDFRequest) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ConvertToPDFRequest) ProtoMessage() {}
func (x *ConvertToPDFRequest) ProtoReflect() protoreflect.Message {
mi := &file_docreader_proto_msgTypes[7]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ConvertToPDFRequest.ProtoReflect.Descriptor instead.
func (*ConvertToPDFRequest) Descriptor() ([]byte, []int) {
return file_docreader_proto_rawDescGZIP(), []int{7}
}
func (x *ConvertToPDFRequest) GetFileContent() []byte {
if x != nil {
return x.FileContent
}
return nil
}
func (x *ConvertToPDFRequest) GetFileName() string {
if x != nil {
return x.FileName
}
return ""
}
func (x *ConvertToPDFRequest) GetFileType() string {
if x != nil {
return x.FileType
}
return ""
}
type ConvertToPDFResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
PdfContent []byte `protobuf:"bytes,1,opt,name=pdf_content,json=pdfContent,proto3" json:"pdf_content,omitempty"`
Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
func (x *ConvertToPDFResponse) Reset() {
*x = ConvertToPDFResponse{}
mi := &file_docreader_proto_msgTypes[8]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
func (x *ConvertToPDFResponse) String() string {
return protoimpl.X.MessageStringOf(x)
}
func (*ConvertToPDFResponse) ProtoMessage() {}
func (x *ConvertToPDFResponse) ProtoReflect() protoreflect.Message {
mi := &file_docreader_proto_msgTypes[8]
if x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
ms.StoreMessageInfo(mi)
}
return ms
}
return mi.MessageOf(x)
}
// Deprecated: Use ConvertToPDFResponse.ProtoReflect.Descriptor instead.
func (*ConvertToPDFResponse) Descriptor() ([]byte, []int) {
return file_docreader_proto_rawDescGZIP(), []int{8}
}
func (x *ConvertToPDFResponse) GetPdfContent() []byte {
if x != nil {
return x.PdfContent
}
return nil
}
func (x *ConvertToPDFResponse) GetError() string {
if x != nil {
return x.Error
}
return ""
}
var File_docreader_proto protoreflect.FileDescriptor
const file_docreader_proto_rawDesc = "" +
"\n" +
"\x0fdocreader.proto\x12\tdocreader\"\xf4\x02\n" +
"\x0fdocreader.proto\x12\tdocreader\"\xeb\x01\n" +
"\n" +
"ReadConfig\x12#\n" +
"\rparser_engine\x18\x01 \x01(\tR\fparserEngine\x12h\n" +
"\x17parser_engine_overrides\x18\x02 \x03(\v20.docreader.ReadConfig.ParserEngineOverridesEntryR\x15parserEngineOverrides\x12L\n" +
"\rimage_storage\x18\x03 \x03(\v2'.docreader.ReadConfig.ImageStorageEntryR\fimageStorage\x1aH\n" +
"\x17parser_engine_overrides\x18\x02 \x03(\v20.docreader.ReadConfig.ParserEngineOverridesEntryR\x15parserEngineOverrides\x1aH\n" +
"\x1aParserEngineOverridesEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\x1a?\n" +
"\x11ImageStorageEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xe0\x01\n" +
"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01J\x04\b\x03\x10\x04\"\xe0\x01\n" +
"\vReadRequest\x12!\n" +
"\ffile_content\x18\x01 \x01(\fR\vfileContent\x12\x1b\n" +
"\tfile_name\x18\x02 \x01(\tR\bfileName\x12\x1b\n" +
@@ -658,19 +534,10 @@ const file_docreader_proto_rawDesc = "" +
"\tavailable\x18\x04 \x01(\bR\tavailable\x12-\n" +
"\x12unavailable_reason\x18\x05 \x01(\tR\x11unavailableReason\"L\n" +
"\x13ListEnginesResponse\x125\n" +
"\aengines\x18\x01 \x03(\v2\x1b.docreader.ParserEngineInfoR\aengines\"r\n" +
"\x13ConvertToPDFRequest\x12!\n" +
"\ffile_content\x18\x01 \x01(\fR\vfileContent\x12\x1b\n" +
"\tfile_name\x18\x02 \x01(\tR\bfileName\x12\x1b\n" +
"\tfile_type\x18\x03 \x01(\tR\bfileType\"M\n" +
"\x14ConvertToPDFResponse\x12\x1f\n" +
"\vpdf_content\x18\x01 \x01(\fR\n" +
"pdfContent\x12\x14\n" +
"\x05error\x18\x02 \x01(\tR\x05error2\xe9\x01\n" +
"\aengines\x18\x01 \x03(\v2\x1b.docreader.ParserEngineInfoR\aengines2\x96\x01\n" +
"\tDocReader\x129\n" +
"\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12N\n" +
"\vListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x12Q\n" +
"\fConvertToPDF\x12\x1e.docreader.ConvertToPDFRequest\x1a\x1f.docreader.ConvertToPDFResponse\"\x00B5Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3"
"\vListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00B5Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3"
var (
file_docreader_proto_rawDescOnce sync.Once
@@ -684,41 +551,35 @@ func file_docreader_proto_rawDescGZIP() []byte {
return file_docreader_proto_rawDescData
}
var file_docreader_proto_msgTypes = make([]protoimpl.MessageInfo, 13)
var file_docreader_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
var file_docreader_proto_goTypes = []any{
(*ReadConfig)(nil), // 0: docreader.ReadConfig
(*ReadRequest)(nil), // 1: docreader.ReadRequest
(*ImageRef)(nil), // 2: docreader.ImageRef
(*ReadResponse)(nil), // 3: docreader.ReadResponse
(*ListEnginesRequest)(nil), // 4: docreader.ListEnginesRequest
(*ParserEngineInfo)(nil), // 5: docreader.ParserEngineInfo
(*ListEnginesResponse)(nil), // 6: docreader.ListEnginesResponse
(*ConvertToPDFRequest)(nil), // 7: docreader.ConvertToPDFRequest
(*ConvertToPDFResponse)(nil), // 8: docreader.ConvertToPDFResponse
nil, // 9: docreader.ReadConfig.ParserEngineOverridesEntry
nil, // 10: docreader.ReadConfig.ImageStorageEntry
nil, // 11: docreader.ReadResponse.MetadataEntry
nil, // 12: docreader.ListEnginesRequest.ConfigOverridesEntry
(*ReadConfig)(nil), // 0: docreader.ReadConfig
(*ReadRequest)(nil), // 1: docreader.ReadRequest
(*ImageRef)(nil), // 2: docreader.ImageRef
(*ReadResponse)(nil), // 3: docreader.ReadResponse
(*ListEnginesRequest)(nil), // 4: docreader.ListEnginesRequest
(*ParserEngineInfo)(nil), // 5: docreader.ParserEngineInfo
(*ListEnginesResponse)(nil), // 6: docreader.ListEnginesResponse
nil, // 7: docreader.ReadConfig.ParserEngineOverridesEntry
nil, // 8: docreader.ReadResponse.MetadataEntry
nil, // 9: docreader.ListEnginesRequest.ConfigOverridesEntry
}
var file_docreader_proto_depIdxs = []int32{
9, // 0: docreader.ReadConfig.parser_engine_overrides:type_name -> docreader.ReadConfig.ParserEngineOverridesEntry
10, // 1: docreader.ReadConfig.image_storage:type_name -> docreader.ReadConfig.ImageStorageEntry
0, // 2: docreader.ReadRequest.config:type_name -> docreader.ReadConfig
2, // 3: docreader.ReadResponse.image_refs:type_name -> docreader.ImageRef
11, // 4: docreader.ReadResponse.metadata:type_name -> docreader.ReadResponse.MetadataEntry
12, // 5: docreader.ListEnginesRequest.config_overrides:type_name -> docreader.ListEnginesRequest.ConfigOverridesEntry
5, // 6: docreader.ListEnginesResponse.engines:type_name -> docreader.ParserEngineInfo
1, // 7: docreader.DocReader.Read:input_type -> docreader.ReadRequest
4, // 8: docreader.DocReader.ListEngines:input_type -> docreader.ListEnginesRequest
7, // 9: docreader.DocReader.ConvertToPDF:input_type -> docreader.ConvertToPDFRequest
3, // 10: docreader.DocReader.Read:output_type -> docreader.ReadResponse
6, // 11: docreader.DocReader.ListEngines:output_type -> docreader.ListEnginesResponse
8, // 12: docreader.DocReader.ConvertToPDF:output_type -> docreader.ConvertToPDFResponse
10, // [10:13] is the sub-list for method output_type
7, // [7:10] is the sub-list for method input_type
7, // [7:7] is the sub-list for extension type_name
7, // [7:7] is the sub-list for extension extendee
0, // [0:7] is the sub-list for field type_name
7, // 0: docreader.ReadConfig.parser_engine_overrides:type_name -> docreader.ReadConfig.ParserEngineOverridesEntry
0, // 1: docreader.ReadRequest.config:type_name -> docreader.ReadConfig
2, // 2: docreader.ReadResponse.image_refs:type_name -> docreader.ImageRef
8, // 3: docreader.ReadResponse.metadata:type_name -> docreader.ReadResponse.MetadataEntry
9, // 4: docreader.ListEnginesRequest.config_overrides:type_name -> docreader.ListEnginesRequest.ConfigOverridesEntry
5, // 5: docreader.ListEnginesResponse.engines:type_name -> docreader.ParserEngineInfo
1, // 6: docreader.DocReader.Read:input_type -> docreader.ReadRequest
4, // 7: docreader.DocReader.ListEngines:input_type -> docreader.ListEnginesRequest
3, // 8: docreader.DocReader.Read:output_type -> docreader.ReadResponse
6, // 9: docreader.DocReader.ListEngines:output_type -> docreader.ListEnginesResponse
8, // [8:10] is the sub-list for method output_type
6, // [6:8] is the sub-list for method input_type
6, // [6:6] is the sub-list for extension type_name
6, // [6:6] is the sub-list for extension extendee
0, // [0:6] is the sub-list for field type_name
}
func init() { file_docreader_proto_init() }
@@ -732,7 +593,7 @@ func file_docreader_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc)),
NumEnums: 0,
NumMessages: 13,
NumMessages: 10,
NumExtensions: 0,
NumServices: 1,
},
+3 -1
View File
@@ -12,7 +12,9 @@ service DocReader {
message ReadConfig {
string parser_engine = 1;
map<string, string> parser_engine_overrides = 2;
map<string, string> image_storage = 3; // storage provider config (provider, endpoint, bucket, ...)
// image_storage removed: image persistence is now handled entirely by the Go App.
// Field number 3 is reserved for backward compatibility.
reserved 3;
}
// Unified read request: set file_content for file mode, url for URL mode.
+2 -40
View File
@@ -19,9 +19,8 @@ import (
const _ = grpc.SupportPackageIsVersion9
const (
DocReader_Read_FullMethodName = "/docreader.DocReader/Read"
DocReader_ListEngines_FullMethodName = "/docreader.DocReader/ListEngines"
DocReader_ConvertToPDF_FullMethodName = "/docreader.DocReader/ConvertToPDF"
DocReader_Read_FullMethodName = "/docreader.DocReader/Read"
DocReader_ListEngines_FullMethodName = "/docreader.DocReader/ListEngines"
)
// DocReaderClient is the client API for DocReader service.
@@ -30,7 +29,6 @@ const (
type DocReaderClient interface {
Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error)
ListEngines(ctx context.Context, in *ListEnginesRequest, opts ...grpc.CallOption) (*ListEnginesResponse, error)
ConvertToPDF(ctx context.Context, in *ConvertToPDFRequest, opts ...grpc.CallOption) (*ConvertToPDFResponse, error)
}
type docReaderClient struct {
@@ -61,23 +59,12 @@ func (c *docReaderClient) ListEngines(ctx context.Context, in *ListEnginesReques
return out, nil
}
func (c *docReaderClient) ConvertToPDF(ctx context.Context, in *ConvertToPDFRequest, opts ...grpc.CallOption) (*ConvertToPDFResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ConvertToPDFResponse)
err := c.cc.Invoke(ctx, DocReader_ConvertToPDF_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// DocReaderServer is the server API for DocReader service.
// All implementations must embed UnimplementedDocReaderServer
// for forward compatibility.
type DocReaderServer interface {
Read(context.Context, *ReadRequest) (*ReadResponse, error)
ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error)
ConvertToPDF(context.Context, *ConvertToPDFRequest) (*ConvertToPDFResponse, error)
mustEmbedUnimplementedDocReaderServer()
}
@@ -94,9 +81,6 @@ func (UnimplementedDocReaderServer) Read(context.Context, *ReadRequest) (*ReadRe
func (UnimplementedDocReaderServer) ListEngines(context.Context, *ListEnginesRequest) (*ListEnginesResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListEngines not implemented")
}
func (UnimplementedDocReaderServer) ConvertToPDF(context.Context, *ConvertToPDFRequest) (*ConvertToPDFResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ConvertToPDF not implemented")
}
func (UnimplementedDocReaderServer) mustEmbedUnimplementedDocReaderServer() {}
func (UnimplementedDocReaderServer) testEmbeddedByValue() {}
@@ -154,24 +138,6 @@ func _DocReader_ListEngines_Handler(srv interface{}, ctx context.Context, dec fu
return interceptor(ctx, in, info, handler)
}
func _DocReader_ConvertToPDF_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ConvertToPDFRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(DocReaderServer).ConvertToPDF(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: DocReader_ConvertToPDF_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(DocReaderServer).ConvertToPDF(ctx, req.(*ConvertToPDFRequest))
}
return interceptor(ctx, in, info, handler)
}
// DocReader_ServiceDesc is the grpc.ServiceDesc for DocReader service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
@@ -187,10 +153,6 @@ var DocReader_ServiceDesc = grpc.ServiceDesc{
MethodName: "ListEngines",
Handler: _DocReader_ListEngines_Handler,
},
{
MethodName: "ConvertToPDF",
Handler: _DocReader_ConvertToPDF_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "docreader.proto",
+22 -30
View File
@@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xa9\x02\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x12>\n\rimage_storage\x18\x03 \x03(\x0b\x32\'.docreader.ReadConfig.ImageStorageEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11ImageStorageEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"\xe2\x01\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo\"Q\n\x13\x43onvertToPDFRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\":\n\x14\x43onvertToPDFResponse\x12\x13\n\x0bpdf_content\x18\x01 \x01(\x0c\x12\r\n\x05\x65rror\x18\x02 \x01(\t2\xe9\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x12Q\n\x0c\x43onvertToPDF\x12\x1e.docreader.ConvertToPDFRequest\x1a\x1f.docreader.ConvertToPDFResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0f\x64ocreader.proto\x12\tdocreader\"\xba\x01\n\nReadConfig\x12\x15\n\rparser_engine\x18\x01 \x01(\t\x12Q\n\x17parser_engine_overrides\x18\x02 \x03(\x0b\x32\x30.docreader.ReadConfig.ParserEngineOverridesEntry\x1a<\n\x1aParserEngineOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01J\x04\x08\x03\x10\x04\"\xa0\x01\n\x0bReadRequest\x12\x14\n\x0c\x66ile_content\x18\x01 \x01(\x0c\x12\x11\n\tfile_name\x18\x02 \x01(\t\x12\x11\n\tfile_type\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\r\n\x05title\x18\x05 \x01(\t\x12%\n\x06\x63onfig\x18\x06 \x01(\x0b\x32\x15.docreader.ReadConfig\x12\x12\n\nrequest_id\x18\x07 \x01(\t\"n\n\x08ImageRef\x12\x10\n\x08\x66ilename\x18\x01 \x01(\t\x12\x14\n\x0coriginal_ref\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0bstorage_key\x18\x04 \x01(\t\x12\x12\n\nimage_data\x18\x05 \x01(\x0c\"\xe2\x01\n\x0cReadResponse\x12\x18\n\x10markdown_content\x18\x01 \x01(\t\x12\'\n\nimage_refs\x18\x02 \x03(\x0b\x32\x13.docreader.ImageRef\x12\x16\n\x0eimage_dir_path\x18\x03 \x01(\t\x12\x37\n\x08metadata\x18\x04 \x03(\x0b\x32%.docreader.ReadResponse.MetadataEntry\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9a\x01\n\x12ListEnginesRequest\x12L\n\x10\x63onfig_overrides\x18\x01 \x03(\x0b\x32\x32.docreader.ListEnginesRequest.ConfigOverridesEntry\x1a\x36\n\x14\x43onfigOverridesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"x\n\x10ParserEngineInfo\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x12\n\nfile_types\x18\x03 \x03(\t\x12\x11\n\tavailable\x18\x04 \x01(\x08\x12\x1a\n\x12unavailable_reason\x18\x05 \x01(\t\"C\n\x13ListEnginesResponse\x12,\n\x07\x65ngines\x18\x01 \x03(\x0b\x32\x1b.docreader.ParserEngineInfo2\x96\x01\n\tDocReader\x12\x39\n\x04Read\x12\x16.docreader.ReadRequest\x1a\x17.docreader.ReadResponse\"\x00\x12N\n\x0bListEngines\x12\x1d.docreader.ListEnginesRequest\x1a\x1e.docreader.ListEnginesResponse\"\x00\x42\x35Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -34,38 +34,30 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals['DESCRIPTOR']._serialized_options = b'Z3github.com/Tencent/WeKnora/internal/docreader/proto'
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._loaded_options = None
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_options = b'8\001'
_globals['_READCONFIG_IMAGESTORAGEENTRY']._loaded_options = None
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_options = b'8\001'
_globals['_READRESPONSE_METADATAENTRY']._loaded_options = None
_globals['_READRESPONSE_METADATAENTRY']._serialized_options = b'8\001'
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._loaded_options = None
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_options = b'8\001'
_globals['_READCONFIG']._serialized_start=31
_globals['_READCONFIG']._serialized_end=328
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_start=215
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_end=275
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_start=277
_globals['_READCONFIG_IMAGESTORAGEENTRY']._serialized_end=328
_globals['_READREQUEST']._serialized_start=331
_globals['_READREQUEST']._serialized_end=491
_globals['_IMAGEREF']._serialized_start=493
_globals['_IMAGEREF']._serialized_end=603
_globals['_READRESPONSE']._serialized_start=606
_globals['_READRESPONSE']._serialized_end=832
_globals['_READRESPONSE_METADATAENTRY']._serialized_start=785
_globals['_READRESPONSE_METADATAENTRY']._serialized_end=832
_globals['_LISTENGINESREQUEST']._serialized_start=835
_globals['_LISTENGINESREQUEST']._serialized_end=989
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=935
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=989
_globals['_PARSERENGINEINFO']._serialized_start=991
_globals['_PARSERENGINEINFO']._serialized_end=1111
_globals['_LISTENGINESRESPONSE']._serialized_start=1113
_globals['_LISTENGINESRESPONSE']._serialized_end=1180
_globals['_CONVERTTOPDFREQUEST']._serialized_start=1182
_globals['_CONVERTTOPDFREQUEST']._serialized_end=1263
_globals['_CONVERTTOPDFRESPONSE']._serialized_start=1265
_globals['_CONVERTTOPDFRESPONSE']._serialized_end=1323
_globals['_DOCREADER']._serialized_start=1326
_globals['_DOCREADER']._serialized_end=1559
_globals['_READCONFIG']._serialized_end=217
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_start=151
_globals['_READCONFIG_PARSERENGINEOVERRIDESENTRY']._serialized_end=211
_globals['_READREQUEST']._serialized_start=220
_globals['_READREQUEST']._serialized_end=380
_globals['_IMAGEREF']._serialized_start=382
_globals['_IMAGEREF']._serialized_end=492
_globals['_READRESPONSE']._serialized_start=495
_globals['_READRESPONSE']._serialized_end=721
_globals['_READRESPONSE_METADATAENTRY']._serialized_start=674
_globals['_READRESPONSE_METADATAENTRY']._serialized_end=721
_globals['_LISTENGINESREQUEST']._serialized_start=724
_globals['_LISTENGINESREQUEST']._serialized_end=878
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_start=824
_globals['_LISTENGINESREQUEST_CONFIGOVERRIDESENTRY']._serialized_end=878
_globals['_PARSERENGINEINFO']._serialized_start=880
_globals['_PARSERENGINEINFO']._serialized_end=1000
_globals['_LISTENGINESRESPONSE']._serialized_start=1002
_globals['_LISTENGINESRESPONSE']._serialized_end=1069
_globals['_DOCREADER']._serialized_start=1072
_globals['_DOCREADER']._serialized_end=1222
# @@protoc_insertion_point(module_scope)
+2 -29
View File
@@ -7,7 +7,7 @@ from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
DESCRIPTOR: _descriptor.FileDescriptor
class ReadConfig(_message.Message):
__slots__ = ("parser_engine", "parser_engine_overrides", "image_storage")
__slots__ = ("parser_engine", "parser_engine_overrides")
class ParserEngineOverridesEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
@@ -15,20 +15,11 @@ class ReadConfig(_message.Message):
key: str
value: str
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
class ImageStorageEntry(_message.Message):
__slots__ = ("key", "value")
KEY_FIELD_NUMBER: _ClassVar[int]
VALUE_FIELD_NUMBER: _ClassVar[int]
key: str
value: str
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
PARSER_ENGINE_FIELD_NUMBER: _ClassVar[int]
PARSER_ENGINE_OVERRIDES_FIELD_NUMBER: _ClassVar[int]
IMAGE_STORAGE_FIELD_NUMBER: _ClassVar[int]
parser_engine: str
parser_engine_overrides: _containers.ScalarMap[str, str]
image_storage: _containers.ScalarMap[str, str]
def __init__(self, parser_engine: _Optional[str] = ..., parser_engine_overrides: _Optional[_Mapping[str, str]] = ..., image_storage: _Optional[_Mapping[str, str]] = ...) -> None: ...
def __init__(self, parser_engine: _Optional[str] = ..., parser_engine_overrides: _Optional[_Mapping[str, str]] = ...) -> None: ...
class ReadRequest(_message.Message):
__slots__ = ("file_content", "file_name", "file_type", "url", "title", "config", "request_id")
@@ -115,21 +106,3 @@ class ListEnginesResponse(_message.Message):
ENGINES_FIELD_NUMBER: _ClassVar[int]
engines: _containers.RepeatedCompositeFieldContainer[ParserEngineInfo]
def __init__(self, engines: _Optional[_Iterable[_Union[ParserEngineInfo, _Mapping]]] = ...) -> None: ...
class ConvertToPDFRequest(_message.Message):
__slots__ = ("file_content", "file_name", "file_type")
FILE_CONTENT_FIELD_NUMBER: _ClassVar[int]
FILE_NAME_FIELD_NUMBER: _ClassVar[int]
FILE_TYPE_FIELD_NUMBER: _ClassVar[int]
file_content: bytes
file_name: str
file_type: str
def __init__(self, file_content: _Optional[bytes] = ..., file_name: _Optional[str] = ..., file_type: _Optional[str] = ...) -> None: ...
class ConvertToPDFResponse(_message.Message):
__slots__ = ("pdf_content", "error")
PDF_CONTENT_FIELD_NUMBER: _ClassVar[int]
ERROR_FIELD_NUMBER: _ClassVar[int]
pdf_content: bytes
error: str
def __init__(self, pdf_content: _Optional[bytes] = ..., error: _Optional[str] = ...) -> None: ...
-43
View File
@@ -44,11 +44,6 @@ class DocReaderStub(object):
request_serializer=docreader__pb2.ListEnginesRequest.SerializeToString,
response_deserializer=docreader__pb2.ListEnginesResponse.FromString,
_registered_method=True)
self.ConvertToPDF = channel.unary_unary(
'/docreader.DocReader/ConvertToPDF',
request_serializer=docreader__pb2.ConvertToPDFRequest.SerializeToString,
response_deserializer=docreader__pb2.ConvertToPDFResponse.FromString,
_registered_method=True)
class DocReaderServicer(object):
@@ -66,12 +61,6 @@ class DocReaderServicer(object):
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def ConvertToPDF(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_DocReaderServicer_to_server(servicer, server):
rpc_method_handlers = {
@@ -85,11 +74,6 @@ def add_DocReaderServicer_to_server(servicer, server):
request_deserializer=docreader__pb2.ListEnginesRequest.FromString,
response_serializer=docreader__pb2.ListEnginesResponse.SerializeToString,
),
'ConvertToPDF': grpc.unary_unary_rpc_method_handler(
servicer.ConvertToPDF,
request_deserializer=docreader__pb2.ConvertToPDFRequest.FromString,
response_serializer=docreader__pb2.ConvertToPDFResponse.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'docreader.DocReader', rpc_method_handlers)
@@ -154,30 +138,3 @@ class DocReader(object):
timeout,
metadata,
_registered_method=True)
@staticmethod
def ConvertToPDF(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(
request,
target,
'/docreader.DocReader/ConvertToPDF',
docreader__pb2.ConvertToPDFRequest.SerializeToString,
docreader__pb2.ConvertToPDFResponse.FromString,
options,
channel_credentials,
insecure,
call_credentials,
compression,
wait_for_ready,
timeout,
metadata,
_registered_method=True)