init commit
This commit is contained in:
@@ -0,0 +1,115 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/services/docreader/src/proto"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/resolver"
|
||||
)
|
||||
|
||||
const (
|
||||
maxMessageSize = 50 * 1024 * 1024 // 50MB
|
||||
)
|
||||
|
||||
var (
|
||||
// Logger is the default logger used by the client
|
||||
Logger = log.New(os.Stdout, "[DocReader] ", log.LstdFlags|log.Lmicroseconds)
|
||||
)
|
||||
|
||||
// ImageInfo 表示一个图片的信息
|
||||
type ImageInfo struct {
|
||||
URL string // 图片URL(COS)
|
||||
Caption string // 图片描述
|
||||
OCRText string // OCR提取的文本
|
||||
OriginalURL string // 原始图片URL
|
||||
Start int // 图片在文本中的开始位置
|
||||
End int // 图片在文本中的结束位置
|
||||
}
|
||||
|
||||
// Client represents a DocReader service client
|
||||
type Client struct {
|
||||
conn *grpc.ClientConn
|
||||
proto.DocReaderClient
|
||||
debug bool
|
||||
}
|
||||
|
||||
// NewClient creates a new DocReader client with the specified address
|
||||
func NewClient(addr string) (*Client, error) {
|
||||
Logger.Printf("INFO: Creating new DocReader client connecting to %s", addr)
|
||||
|
||||
// 设置消息大小限制
|
||||
opts := []grpc.DialOption{
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`),
|
||||
grpc.WithDefaultCallOptions(
|
||||
grpc.MaxCallRecvMsgSize(maxMessageSize),
|
||||
grpc.MaxCallSendMsgSize(maxMessageSize),
|
||||
),
|
||||
}
|
||||
resolver.SetDefaultScheme("dns")
|
||||
|
||||
startTime := time.Now()
|
||||
conn, err := grpc.Dial("dns:///"+addr, opts...)
|
||||
if err != nil {
|
||||
Logger.Printf("ERROR: Failed to connect to DocReader service: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
Logger.Printf("INFO: Successfully connected to DocReader service in %v", time.Since(startTime))
|
||||
|
||||
return &Client{
|
||||
conn: conn,
|
||||
DocReaderClient: proto.NewDocReaderClient(conn),
|
||||
debug: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Close closes the client connection
|
||||
func (c *Client) Close() error {
|
||||
Logger.Printf("INFO: Closing DocReader client connection")
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// SetDebug enables or disables debug logging
|
||||
func (c *Client) SetDebug(debug bool) {
|
||||
c.debug = debug
|
||||
Logger.Printf("INFO: Debug logging set to %v", debug)
|
||||
}
|
||||
|
||||
// Log logs a message with the appropriate level
|
||||
func (c *Client) Log(level string, format string, args ...interface{}) {
|
||||
if level == "DEBUG" && !c.debug {
|
||||
return
|
||||
}
|
||||
Logger.Printf("%s: %s", level, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// GetImagesFromChunk 从一个Chunk中提取所有图片信息
|
||||
func GetImagesFromChunk(chunk *proto.Chunk) []ImageInfo {
|
||||
if chunk == nil || len(chunk.Images) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
images := make([]ImageInfo, 0, len(chunk.Images))
|
||||
for _, img := range chunk.Images {
|
||||
images = append(images, ImageInfo{
|
||||
URL: img.Url,
|
||||
Caption: img.Caption,
|
||||
OCRText: img.OcrText,
|
||||
OriginalURL: img.OriginalUrl,
|
||||
Start: int(img.Start),
|
||||
End: int(img.End),
|
||||
})
|
||||
}
|
||||
|
||||
return images
|
||||
}
|
||||
|
||||
// HasImagesInChunk 判断一个Chunk是否包含图片
|
||||
func HasImagesInChunk(chunk *proto.Chunk) bool {
|
||||
return chunk != nil && len(chunk.Images) > 0
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/services/docreader/src/proto"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// 配置测试日志
|
||||
log.SetOutput(os.Stdout)
|
||||
log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.Lshortfile)
|
||||
log.Println("INFO: Initializing DocReader client tests")
|
||||
}
|
||||
|
||||
func TestReadFromURL(t *testing.T) {
|
||||
log.Println("INFO: Starting TestReadFromURL")
|
||||
|
||||
// 创建测试客户端
|
||||
log.Println("INFO: Creating test client")
|
||||
client, err := NewClient("localhost:50051")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to create client: %v", err)
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// 启用调试日志
|
||||
client.SetDebug(true)
|
||||
|
||||
// 测试 ReadFromURL 方法
|
||||
log.Println("INFO: Sending ReadFromURL request to server")
|
||||
startTime := time.Now()
|
||||
resp, err := client.ReadFromURL(
|
||||
context.Background(),
|
||||
&proto.ReadFromURLRequest{
|
||||
Url: "https://example.com",
|
||||
Title: "test",
|
||||
ReadConfig: &proto.ReadConfig{
|
||||
ChunkSize: 512,
|
||||
ChunkOverlap: 50,
|
||||
Separators: []string{"\n\n", "\n", "。"},
|
||||
EnableMultimodal: true,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
requestDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: ReadFromURL failed: %v", err)
|
||||
t.Fatalf("ReadFromURL failed: %v", err)
|
||||
}
|
||||
log.Printf("INFO: ReadFromURL completed in %v", requestDuration)
|
||||
|
||||
// 验证结果
|
||||
chunkCount := len(resp.Chunks)
|
||||
log.Printf("INFO: Received %d chunks from URL parsing", chunkCount)
|
||||
if chunkCount == 0 {
|
||||
log.Println("WARN: Expected non-empty content but received none")
|
||||
t.Error("Expected non-empty content")
|
||||
}
|
||||
|
||||
// 打印结果
|
||||
for i, chunk := range resp.Chunks {
|
||||
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
|
||||
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
|
||||
} else if i == 2 && chunkCount > 4 {
|
||||
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("INFO: TestReadFromURL completed successfully")
|
||||
}
|
||||
|
||||
func TestReadFromFileWithChunking(t *testing.T) {
|
||||
log.Println("INFO: Starting TestReadFromFileWithChunking")
|
||||
|
||||
// 创建测试客户端
|
||||
log.Println("INFO: Creating test client")
|
||||
client, err := NewClient("localhost:50051")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to create client: %v", err)
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// 启用调试日志
|
||||
client.SetDebug(true)
|
||||
|
||||
// 读取测试文件
|
||||
log.Println("INFO: Reading test file")
|
||||
fileContent, err := os.ReadFile("../testdata/test.md")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to read test file: %v", err)
|
||||
t.Fatalf("Failed to read test file: %v", err)
|
||||
}
|
||||
log.Printf("INFO: Read test file, size: %d bytes", len(fileContent))
|
||||
|
||||
// 测试 ReadFromFile 方法,带分块参数
|
||||
log.Println("INFO: Sending ReadFromFile request to server")
|
||||
startTime := time.Now()
|
||||
resp, err := client.ReadFromFile(
|
||||
context.Background(),
|
||||
&proto.ReadFromFileRequest{
|
||||
FileContent: fileContent,
|
||||
FileName: "test.md",
|
||||
FileType: "md",
|
||||
ReadConfig: &proto.ReadConfig{
|
||||
ChunkSize: 200,
|
||||
ChunkOverlap: 50,
|
||||
Separators: []string{"\n\n", "\n", "。"},
|
||||
EnableMultimodal: true,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
requestDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: ReadFromFile failed: %v", err)
|
||||
t.Fatalf("ReadFromFile failed: %v", err)
|
||||
}
|
||||
log.Printf("INFO: ReadFromFile completed in %v", requestDuration)
|
||||
|
||||
// 验证结果
|
||||
chunkCount := len(resp.Chunks)
|
||||
log.Printf("INFO: Received %d chunks from file parsing", chunkCount)
|
||||
if chunkCount == 0 {
|
||||
log.Println("WARN: Expected non-empty content but received none")
|
||||
t.Error("Expected non-empty content")
|
||||
}
|
||||
|
||||
// 打印结果
|
||||
for i, chunk := range resp.Chunks {
|
||||
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
|
||||
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
|
||||
} else if i == 2 && chunkCount > 4 {
|
||||
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("INFO: TestReadFromFileWithChunking completed successfully")
|
||||
}
|
||||
|
||||
// 截断字符串以供日志打印
|
||||
func truncateString(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
Reference in New Issue
Block a user