package docparser import ( "context" "crypto/tls" "crypto/x509" "fmt" "os" "strconv" "sync" "time" "github.com/Tencent/WeKnora/docreader/proto" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/resolver" ) func getMaxMessageSize() int { if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" { if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 { return size * 1024 * 1024 } } return 50 * 1024 * 1024 } // GRPCDocumentReader implements DocumentReader over gRPC. type GRPCDocumentReader struct { mu sync.RWMutex conn *grpc.ClientConn client proto.DocReaderClient addr string } func NewGRPCDocumentReader(addr string) (*GRPCDocumentReader, error) { p := &GRPCDocumentReader{} if addr != "" { if err := p.connect(addr); err != nil { return nil, err } } return p, nil } func (p *GRPCDocumentReader) connect(addr string) error { maxMsgSize := getMaxMessageSize() opts := []grpc.DialOption{ grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`), grpc.WithDefaultCallOptions( grpc.MaxCallRecvMsgSize(maxMsgSize), grpc.MaxCallSendMsgSize(maxMsgSize), ), } if os.Getenv("GRPC_TLS_ENABLED") == "true" { creds, err := buildTLSCredentials() if err != nil { return fmt.Errorf("failed to build TLS credentials: %w", err) } opts = append(opts, grpc.WithTransportCredentials(creds)) logger.Infof(context.Background(), "TLS enabled for docreader gRPC client") } else { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } if authToken := os.Getenv("GRPC_AUTH_TOKEN"); authToken != "" { opts = append(opts, grpc.WithPerRPCCredentials(&tokenAuth{token: authToken})) logger.Infof(context.Background(), "Token authentication enabled for docreader gRPC client") } resolver.SetDefaultScheme("dns") start := time.Now() conn, err := grpc.Dial("dns:///"+addr, opts...) if err != nil { return fmt.Errorf("failed to connect to docreader: %w", err) } logger.Infof(context.Background(), "Connected to docreader in %v", time.Since(start)) p.conn = conn p.client = proto.NewDocReaderClient(conn) p.addr = addr return nil } func (p *GRPCDocumentReader) Reconnect(addr string) error { p.mu.Lock() defer p.mu.Unlock() if p.conn != nil { _ = p.conn.Close() p.conn = nil p.client = nil p.addr = "" } return p.connect(addr) } func (p *GRPCDocumentReader) IsConnected() bool { p.mu.RLock() defer p.mu.RUnlock() return p.conn != nil } func (p *GRPCDocumentReader) Close() error { p.mu.Lock() defer p.mu.Unlock() if p.conn != nil { return p.conn.Close() } return nil } var errNotConnected = fmt.Errorf("docreader service not connected") func (p *GRPCDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) { p.mu.RLock() client := p.client p.mu.RUnlock() if client == nil { return nil, errNotConnected } protoReq := &proto.ReadRequest{ FileContent: req.FileContent, FileName: req.FileName, FileType: req.FileType, Url: req.URL, Title: req.Title, RequestId: req.RequestID, Config: &proto.ReadConfig{ ParserEngine: req.ParserEngine, ParserEngineOverrides: req.ParserEngineOverrides, }, } resp, err := client.Read(ctx, protoReq) if err != nil { return nil, fmt.Errorf("gRPC Read failed: %w", err) } return fromProtoReadResponse(resp), nil } func (p *GRPCDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) { p.mu.RLock() client := p.client p.mu.RUnlock() if client == nil { return nil, errNotConnected } resp, err := client.ListEngines(ctx, &proto.ListEnginesRequest{ConfigOverrides: overrides}) if err != nil { return nil, fmt.Errorf("gRPC ListEngines failed: %w", err) } result := make([]types.ParserEngineInfo, 0, len(resp.GetEngines())) for _, e := range resp.GetEngines() { result = append(result, types.ParserEngineInfo{ Name: e.GetName(), Description: e.GetDescription(), FileTypes: e.GetFileTypes(), Available: e.GetAvailable(), UnavailableReason: e.GetUnavailableReason(), }) } return result, nil } func fromProtoReadResponse(resp *proto.ReadResponse) *types.ReadResult { result := &types.ReadResult{ MarkdownContent: resp.GetMarkdownContent(), ImageDirPath: resp.GetImageDirPath(), Metadata: resp.GetMetadata(), Error: resp.GetError(), } for _, ref := range resp.GetImageRefs() { result.ImageRefs = append(result.ImageRefs, types.ImageRef{ Filename: ref.GetFilename(), OriginalRef: ref.GetOriginalRef(), MimeType: ref.GetMimeType(), StorageKey: ref.GetStorageKey(), ImageData: ref.GetImageData(), }) } return result } func buildTLSCredentials() (credentials.TransportCredentials, error) { tlsConfig := &tls.Config{ MinVersion: tls.VersionTLS12, } if caFile := os.Getenv("GRPC_TLS_CA"); caFile != "" { caCert, err := os.ReadFile(caFile) if err != nil { return nil, fmt.Errorf("failed to read CA certificate: %w", err) } certPool := x509.NewCertPool() if !certPool.AppendCertsFromPEM(caCert) { return nil, fmt.Errorf("failed to parse CA certificate") } tlsConfig.RootCAs = certPool } if certFile := os.Getenv("GRPC_TLS_CERT"); certFile != "" { if keyFile := os.Getenv("GRPC_TLS_KEY"); keyFile != "" { cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { return nil, fmt.Errorf("failed to load client certificate: %w", err) } tlsConfig.Certificates = []tls.Certificate{cert} logger.Infof(context.Background(), "mTLS enabled (client certificate loaded)") } } return credentials.NewTLS(tlsConfig), nil } type tokenAuth struct { token string } func (t *tokenAuth) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { return map[string]string{ "authorization": "Bearer " + t.token, }, nil } func (t *tokenAuth) RequireTransportSecurity() bool { return false }