refactor(parser): reorganize Markdown parser and enhance gRPC document reading
- Moved the _SEPARATOR_CELL regex definition to a more appropriate location in the Markdown parser. - Implemented a fallback mechanism in the gRPC document reader to handle cases where the ReadStream RPC is unimplemented, ensuring compatibility with older versions. - Added a readUnary method to maintain backward compatibility with the legacy unary Read RPC. - Improved cancellation handling in the MinerUCloud and PaddleOCR-VL readers to prevent excessive API calls during context cancellation.
This commit is contained in:
@@ -14,7 +14,9 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/resolver"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
func getMaxMessageSize() int {
|
||||
@@ -129,6 +131,27 @@ func (p *GRPCDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (
|
||||
// Use the streaming RPC so documents with many page images (large scanned
|
||||
// PDFs) are not capped by the unary message-size limit. The meta frame
|
||||
// arrives first, followed by one frame per image.
|
||||
result, err := p.readStream(ctx, client, protoReq)
|
||||
if err != nil {
|
||||
// An older docreader build may not implement ReadStream. Fall back to
|
||||
// the unary Read RPC so a version-skewed deployment still parses
|
||||
// documents (small/medium docs only — the unary path remains capped by
|
||||
// the gRPC message-size limit, which is exactly what streaming avoids).
|
||||
if status.Code(err) == codes.Unimplemented {
|
||||
logger.Warnf(ctx, "docreader ReadStream unimplemented, falling back to unary Read: %v", err)
|
||||
return p.readUnary(ctx, client, protoReq)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// readStream consumes the server-streaming ReadStream RPC: one meta frame
|
||||
// followed by one frame per image. Errors are returned verbatim so the caller
|
||||
// can inspect the gRPC status code (e.g. Unimplemented) for fallback.
|
||||
func (p *GRPCDocumentReader) readStream(
|
||||
ctx context.Context, client proto.DocReaderClient, protoReq *proto.ReadRequest,
|
||||
) (*types.ReadResult, error) {
|
||||
stream, err := client.ReadStream(ctx, protoReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gRPC ReadStream failed: %w", err)
|
||||
@@ -174,6 +197,37 @@ func (p *GRPCDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// readUnary calls the legacy unary Read RPC. Used only as a compatibility
|
||||
// fallback when the connected docreader does not implement ReadStream.
|
||||
func (p *GRPCDocumentReader) readUnary(
|
||||
ctx context.Context, client proto.DocReaderClient, protoReq *proto.ReadRequest,
|
||||
) (*types.ReadResult, error) {
|
||||
resp, err := client.Read(ctx, protoReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gRPC Read failed: %w", err)
|
||||
}
|
||||
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: resp.GetMarkdownContent(),
|
||||
ImageDirPath: resp.GetImageDirPath(),
|
||||
Metadata: resp.GetMetadata(),
|
||||
Error: resp.GetError(),
|
||||
}
|
||||
if refs := resp.GetImageRefs(); len(refs) > 0 {
|
||||
result.ImageRefs = make([]types.ImageRef, 0, len(refs))
|
||||
for _, img := range refs {
|
||||
result.ImageRefs = append(result.ImageRefs, types.ImageRef{
|
||||
Filename: img.GetFilename(),
|
||||
OriginalRef: img.GetOriginalRef(),
|
||||
MimeType: img.GetMimeType(),
|
||||
StorageKey: img.GetStorageKey(),
|
||||
ImageData: img.GetImageData(),
|
||||
})
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *GRPCDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) {
|
||||
p.mu.RLock()
|
||||
client := p.client
|
||||
|
||||
Reference in New Issue
Block a user