240 lines
6.1 KiB
Go
240 lines
6.1 KiB
Go
package docparser
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/docreader/proto"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/credentials"
|
|
"google.golang.org/grpc/credentials/insecure"
|
|
"google.golang.org/grpc/resolver"
|
|
)
|
|
|
|
func getMaxMessageSize() int {
|
|
if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" {
|
|
if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 {
|
|
return size * 1024 * 1024
|
|
}
|
|
}
|
|
return 50 * 1024 * 1024
|
|
}
|
|
|
|
// GRPCDocumentReader implements DocumentReader over gRPC.
|
|
type GRPCDocumentReader struct {
|
|
mu sync.RWMutex
|
|
conn *grpc.ClientConn
|
|
client proto.DocReaderClient
|
|
addr string
|
|
}
|
|
|
|
func NewGRPCDocumentReader(addr string) (*GRPCDocumentReader, error) {
|
|
p := &GRPCDocumentReader{}
|
|
if addr != "" {
|
|
if err := p.connect(addr); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return p, nil
|
|
}
|
|
|
|
func (p *GRPCDocumentReader) connect(addr string) error {
|
|
|
|
maxMsgSize := getMaxMessageSize()
|
|
opts := []grpc.DialOption{
|
|
grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`),
|
|
grpc.WithDefaultCallOptions(
|
|
grpc.MaxCallRecvMsgSize(maxMsgSize),
|
|
grpc.MaxCallSendMsgSize(maxMsgSize),
|
|
),
|
|
}
|
|
|
|
if os.Getenv("GRPC_TLS_ENABLED") == "true" {
|
|
creds, err := buildTLSCredentials()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to build TLS credentials: %w", err)
|
|
}
|
|
opts = append(opts, grpc.WithTransportCredentials(creds))
|
|
logger.Infof(context.Background(), "TLS enabled for docreader gRPC client")
|
|
} else {
|
|
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
|
}
|
|
|
|
if authToken := os.Getenv("GRPC_AUTH_TOKEN"); authToken != "" {
|
|
opts = append(opts, grpc.WithPerRPCCredentials(&tokenAuth{token: authToken}))
|
|
logger.Infof(context.Background(), "Token authentication enabled for docreader gRPC client")
|
|
}
|
|
|
|
resolver.SetDefaultScheme("dns")
|
|
|
|
start := time.Now()
|
|
conn, err := grpc.Dial("dns:///"+addr, opts...)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to connect to docreader: %w", err)
|
|
}
|
|
logger.Infof(context.Background(), "Connected to docreader in %v", time.Since(start))
|
|
|
|
p.conn = conn
|
|
p.client = proto.NewDocReaderClient(conn)
|
|
p.addr = addr
|
|
return nil
|
|
}
|
|
|
|
func (p *GRPCDocumentReader) Reconnect(addr string) error {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
|
|
if p.conn != nil {
|
|
_ = p.conn.Close()
|
|
p.conn = nil
|
|
p.client = nil
|
|
p.addr = ""
|
|
}
|
|
return p.connect(addr)
|
|
}
|
|
|
|
func (p *GRPCDocumentReader) IsConnected() bool {
|
|
p.mu.RLock()
|
|
defer p.mu.RUnlock()
|
|
return p.conn != nil
|
|
}
|
|
|
|
func (p *GRPCDocumentReader) Close() error {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
if p.conn != nil {
|
|
return p.conn.Close()
|
|
}
|
|
return nil
|
|
}
|
|
|
|
var errNotConnected = fmt.Errorf("docreader service not connected")
|
|
|
|
func (p *GRPCDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
|
p.mu.RLock()
|
|
client := p.client
|
|
p.mu.RUnlock()
|
|
if client == nil {
|
|
return nil, errNotConnected
|
|
}
|
|
|
|
protoReq := &proto.ReadRequest{
|
|
FileContent: req.FileContent,
|
|
FileName: req.FileName,
|
|
FileType: req.FileType,
|
|
Url: req.URL,
|
|
Title: req.Title,
|
|
RequestId: req.RequestID,
|
|
Config: &proto.ReadConfig{
|
|
ParserEngine: req.ParserEngine,
|
|
ParserEngineOverrides: req.ParserEngineOverrides,
|
|
},
|
|
}
|
|
|
|
resp, err := client.Read(ctx, protoReq)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("gRPC Read failed: %w", err)
|
|
}
|
|
return fromProtoReadResponse(resp), nil
|
|
}
|
|
|
|
func (p *GRPCDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) {
|
|
p.mu.RLock()
|
|
client := p.client
|
|
p.mu.RUnlock()
|
|
if client == nil {
|
|
return nil, errNotConnected
|
|
}
|
|
|
|
resp, err := client.ListEngines(ctx, &proto.ListEnginesRequest{ConfigOverrides: overrides})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("gRPC ListEngines failed: %w", err)
|
|
}
|
|
|
|
result := make([]types.ParserEngineInfo, 0, len(resp.GetEngines()))
|
|
for _, e := range resp.GetEngines() {
|
|
result = append(result, types.ParserEngineInfo{
|
|
Name: e.GetName(),
|
|
Description: e.GetDescription(),
|
|
FileTypes: e.GetFileTypes(),
|
|
Available: e.GetAvailable(),
|
|
UnavailableReason: e.GetUnavailableReason(),
|
|
})
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func fromProtoReadResponse(resp *proto.ReadResponse) *types.ReadResult {
|
|
result := &types.ReadResult{
|
|
MarkdownContent: resp.GetMarkdownContent(),
|
|
ImageDirPath: resp.GetImageDirPath(),
|
|
Metadata: resp.GetMetadata(),
|
|
Error: resp.GetError(),
|
|
}
|
|
|
|
for _, ref := range resp.GetImageRefs() {
|
|
result.ImageRefs = append(result.ImageRefs, types.ImageRef{
|
|
Filename: ref.GetFilename(),
|
|
OriginalRef: ref.GetOriginalRef(),
|
|
MimeType: ref.GetMimeType(),
|
|
StorageKey: ref.GetStorageKey(),
|
|
ImageData: ref.GetImageData(),
|
|
})
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
func buildTLSCredentials() (credentials.TransportCredentials, error) {
|
|
tlsConfig := &tls.Config{
|
|
MinVersion: tls.VersionTLS12,
|
|
}
|
|
|
|
if caFile := os.Getenv("GRPC_TLS_CA"); caFile != "" {
|
|
caCert, err := os.ReadFile(caFile)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read CA certificate: %w", err)
|
|
}
|
|
certPool := x509.NewCertPool()
|
|
if !certPool.AppendCertsFromPEM(caCert) {
|
|
return nil, fmt.Errorf("failed to parse CA certificate")
|
|
}
|
|
tlsConfig.RootCAs = certPool
|
|
}
|
|
|
|
if certFile := os.Getenv("GRPC_TLS_CERT"); certFile != "" {
|
|
if keyFile := os.Getenv("GRPC_TLS_KEY"); keyFile != "" {
|
|
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to load client certificate: %w", err)
|
|
}
|
|
tlsConfig.Certificates = []tls.Certificate{cert}
|
|
logger.Infof(context.Background(), "mTLS enabled (client certificate loaded)")
|
|
}
|
|
}
|
|
|
|
return credentials.NewTLS(tlsConfig), nil
|
|
}
|
|
|
|
type tokenAuth struct {
|
|
token string
|
|
}
|
|
|
|
func (t *tokenAuth) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
|
|
return map[string]string{
|
|
"authorization": "Bearer " + t.token,
|
|
}, nil
|
|
}
|
|
|
|
func (t *tokenAuth) RequireTransportSecurity() bool {
|
|
return false
|
|
}
|