Files
Enterprise-Knowledge-Base/internal/infrastructure/docparser/grpc_parser.go
T
2026-05-16 21:31:17 +08:00

240 lines
6.1 KiB
Go

package docparser
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"os"
"strconv"
"sync"
"time"
"github.com/Tencent/WeKnora/docreader/proto"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/resolver"
)
func getMaxMessageSize() int {
if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" {
if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 {
return size * 1024 * 1024
}
}
return 50 * 1024 * 1024
}
// GRPCDocumentReader implements DocumentReader over gRPC.
type GRPCDocumentReader struct {
mu sync.RWMutex
conn *grpc.ClientConn
client proto.DocReaderClient
addr string
}
func NewGRPCDocumentReader(addr string) (*GRPCDocumentReader, error) {
p := &GRPCDocumentReader{}
if addr != "" {
if err := p.connect(addr); err != nil {
return nil, err
}
}
return p, nil
}
func (p *GRPCDocumentReader) connect(addr string) error {
maxMsgSize := getMaxMessageSize()
opts := []grpc.DialOption{
grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`),
grpc.WithDefaultCallOptions(
grpc.MaxCallRecvMsgSize(maxMsgSize),
grpc.MaxCallSendMsgSize(maxMsgSize),
),
}
if os.Getenv("GRPC_TLS_ENABLED") == "true" {
creds, err := buildTLSCredentials()
if err != nil {
return fmt.Errorf("failed to build TLS credentials: %w", err)
}
opts = append(opts, grpc.WithTransportCredentials(creds))
logger.Infof(context.Background(), "TLS enabled for docreader gRPC client")
} else {
opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials()))
}
if authToken := os.Getenv("GRPC_AUTH_TOKEN"); authToken != "" {
opts = append(opts, grpc.WithPerRPCCredentials(&tokenAuth{token: authToken}))
logger.Infof(context.Background(), "Token authentication enabled for docreader gRPC client")
}
resolver.SetDefaultScheme("dns")
start := time.Now()
conn, err := grpc.Dial("dns:///"+addr, opts...)
if err != nil {
return fmt.Errorf("failed to connect to docreader: %w", err)
}
logger.Infof(context.Background(), "Connected to docreader in %v", time.Since(start))
p.conn = conn
p.client = proto.NewDocReaderClient(conn)
p.addr = addr
return nil
}
func (p *GRPCDocumentReader) Reconnect(addr string) error {
p.mu.Lock()
defer p.mu.Unlock()
if p.conn != nil {
_ = p.conn.Close()
p.conn = nil
p.client = nil
p.addr = ""
}
return p.connect(addr)
}
func (p *GRPCDocumentReader) IsConnected() bool {
p.mu.RLock()
defer p.mu.RUnlock()
return p.conn != nil
}
func (p *GRPCDocumentReader) Close() error {
p.mu.Lock()
defer p.mu.Unlock()
if p.conn != nil {
return p.conn.Close()
}
return nil
}
var errNotConnected = fmt.Errorf("docreader service not connected")
func (p *GRPCDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
p.mu.RLock()
client := p.client
p.mu.RUnlock()
if client == nil {
return nil, errNotConnected
}
protoReq := &proto.ReadRequest{
FileContent: req.FileContent,
FileName: req.FileName,
FileType: req.FileType,
Url: req.URL,
Title: req.Title,
RequestId: req.RequestID,
Config: &proto.ReadConfig{
ParserEngine: req.ParserEngine,
ParserEngineOverrides: req.ParserEngineOverrides,
},
}
resp, err := client.Read(ctx, protoReq)
if err != nil {
return nil, fmt.Errorf("gRPC Read failed: %w", err)
}
return fromProtoReadResponse(resp), nil
}
func (p *GRPCDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) {
p.mu.RLock()
client := p.client
p.mu.RUnlock()
if client == nil {
return nil, errNotConnected
}
resp, err := client.ListEngines(ctx, &proto.ListEnginesRequest{ConfigOverrides: overrides})
if err != nil {
return nil, fmt.Errorf("gRPC ListEngines failed: %w", err)
}
result := make([]types.ParserEngineInfo, 0, len(resp.GetEngines()))
for _, e := range resp.GetEngines() {
result = append(result, types.ParserEngineInfo{
Name: e.GetName(),
Description: e.GetDescription(),
FileTypes: e.GetFileTypes(),
Available: e.GetAvailable(),
UnavailableReason: e.GetUnavailableReason(),
})
}
return result, nil
}
func fromProtoReadResponse(resp *proto.ReadResponse) *types.ReadResult {
result := &types.ReadResult{
MarkdownContent: resp.GetMarkdownContent(),
ImageDirPath: resp.GetImageDirPath(),
Metadata: resp.GetMetadata(),
Error: resp.GetError(),
}
for _, ref := range resp.GetImageRefs() {
result.ImageRefs = append(result.ImageRefs, types.ImageRef{
Filename: ref.GetFilename(),
OriginalRef: ref.GetOriginalRef(),
MimeType: ref.GetMimeType(),
StorageKey: ref.GetStorageKey(),
ImageData: ref.GetImageData(),
})
}
return result
}
func buildTLSCredentials() (credentials.TransportCredentials, error) {
tlsConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
}
if caFile := os.Getenv("GRPC_TLS_CA"); caFile != "" {
caCert, err := os.ReadFile(caFile)
if err != nil {
return nil, fmt.Errorf("failed to read CA certificate: %w", err)
}
certPool := x509.NewCertPool()
if !certPool.AppendCertsFromPEM(caCert) {
return nil, fmt.Errorf("failed to parse CA certificate")
}
tlsConfig.RootCAs = certPool
}
if certFile := os.Getenv("GRPC_TLS_CERT"); certFile != "" {
if keyFile := os.Getenv("GRPC_TLS_KEY"); keyFile != "" {
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
return nil, fmt.Errorf("failed to load client certificate: %w", err)
}
tlsConfig.Certificates = []tls.Certificate{cert}
logger.Infof(context.Background(), "mTLS enabled (client certificate loaded)")
}
}
return credentials.NewTLS(tlsConfig), nil
}
type tokenAuth struct {
token string
}
func (t *tokenAuth) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) {
return map[string]string{
"authorization": "Bearer " + t.token,
}, nil
}
func (t *tokenAuth) RequireTransportSecurity() bool {
return false
}