feat: introduce WeKnora Lite edition with lightweight configuration and deployment

- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
wizardchen
2026-02-28 13:22:07 +08:00
committed by lyingbug
parent 5241dbc39e
commit 397689d2f3
140 changed files with 13365 additions and 5413 deletions
+19 -38
View File
@@ -13,42 +13,35 @@ import (
"google.golang.org/grpc/resolver"
)
// getMaxMessageSize returns the maximum gRPC message size in bytes.
// Default is 50MB, can be configured via MAX_FILE_SIZE_MB environment variable.
func getMaxMessageSize() int {
if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" {
if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 {
return size * 1024 * 1024
}
}
return 50 * 1024 * 1024 // default 50MB
return 50 * 1024 * 1024
}
// Logger is the default logger used by the client
var Logger = log.New(os.Stdout, "[DocReader] ", log.LstdFlags|log.Lmicroseconds)
// ImageInfo 表示一个图片的信息
type ImageInfo struct {
URL string // 图片URL(COS)
Caption string // 图片描述
OCRText string // OCR提取的文本
OriginalURL string // 原始图片URL
Start int // 图片在文本中的开始位置
End int // 图片在文本中的结束位置
// ImageRefInfo represents an image reference from a converted document.
type ImageRefInfo struct {
Filename string
OriginalRef string
MimeType string
StorageKey string
}
// Client represents a DocReader service client
// Client represents a DocReader service client.
type Client struct {
conn *grpc.ClientConn
proto.DocReaderClient
debug bool
}
// NewClient creates a new DocReader client with the specified address
func NewClient(addr string) (*Client, error) {
Logger.Printf("INFO: Creating new DocReader client connecting to %s", addr)
// 设置消息大小限制 (configurable via GRPC_MAX_MESSAGE_SIZE_MB)
maxMsgSize := getMaxMessageSize()
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecure.NewCredentials()),
@@ -75,19 +68,15 @@ func NewClient(addr string) (*Client, error) {
}, nil
}
// Close closes the client connection
func (c *Client) Close() error {
Logger.Printf("INFO: Closing DocReader client connection")
return c.conn.Close()
}
// SetDebug enables or disables debug logging
func (c *Client) SetDebug(debug bool) {
c.debug = debug
Logger.Printf("INFO: Debug logging set to %v", debug)
}
// Log logs a message with the appropriate level
func (c *Client) Log(level string, format string, args ...interface{}) {
if level == "DEBUG" && !c.debug {
return
@@ -95,28 +84,20 @@ func (c *Client) Log(level string, format string, args ...interface{}) {
Logger.Printf("%s: %s", level, fmt.Sprintf(format, args...))
}
// GetImagesFromChunk 从一个Chunk中提取所有图片信息
func GetImagesFromChunk(chunk *proto.Chunk) []ImageInfo {
if chunk == nil || len(chunk.Images) == 0 {
// GetImageRefsFromResponse extracts image references from a ReadResponse.
func GetImageRefsFromResponse(resp *proto.ReadResponse) []ImageRefInfo {
if resp == nil || len(resp.ImageRefs) == 0 {
return nil
}
images := make([]ImageInfo, 0, len(chunk.Images))
for _, img := range chunk.Images {
images = append(images, ImageInfo{
URL: img.Url,
Caption: img.Caption,
OCRText: img.OcrText,
OriginalURL: img.OriginalUrl,
Start: int(img.Start),
End: int(img.End),
refs := make([]ImageRefInfo, 0, len(resp.ImageRefs))
for _, ref := range resp.ImageRefs {
refs = append(refs, ImageRefInfo{
Filename: ref.Filename,
OriginalRef: ref.OriginalRef,
MimeType: ref.MimeType,
StorageKey: ref.StorageKey,
})
}
return images
}
// HasImagesInChunk 判断一个Chunk是否包含图片
func HasImagesInChunk(chunk *proto.Chunk) bool {
return chunk != nil && len(chunk.Images) > 0
return refs
}
+22 -91
View File
@@ -11,144 +11,75 @@ import (
)
func init() {
// 配置测试日志
log.SetOutput(os.Stdout)
log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.Lshortfile)
log.Println("INFO: Initializing DocReader client tests")
}
func TestReadFromURL(t *testing.T) {
log.Println("INFO: Starting TestReadFromURL")
// 创建测试客户端
log.Println("INFO: Creating test client")
func TestReadURL(t *testing.T) {
client, err := NewClient("localhost:50051")
if err != nil {
log.Printf("ERROR: Failed to create client: %v", err)
t.Fatalf("Failed to create client: %v", err)
}
defer client.Close()
// 启用调试日志
client.SetDebug(true)
// 测试 ReadFromURL 方法
log.Println("INFO: Sending ReadFromURL request to server")
startTime := time.Now()
resp, err := client.ReadFromURL(
resp, err := client.Read(
context.Background(),
&proto.ReadFromURLRequest{
&proto.ReadRequest{
Url: "https://example.com",
Title: "test",
ReadConfig: &proto.ReadConfig{
ChunkSize: 512,
ChunkOverlap: 50,
Separators: []string{"\n\n", "\n", "。"},
EnableMultimodal: true,
},
},
)
log.Printf("INFO: Read(URL) completed in %v", time.Since(startTime))
requestDuration := time.Since(startTime)
if err != nil {
log.Printf("ERROR: ReadFromURL failed: %v", err)
t.Fatalf("ReadFromURL failed: %v", err)
t.Fatalf("Read failed: %v", err)
}
log.Printf("INFO: ReadFromURL completed in %v", requestDuration)
// 验证结果
chunkCount := len(resp.Chunks)
log.Printf("INFO: Received %d chunks from URL parsing", chunkCount)
if chunkCount == 0 {
log.Println("WARN: Expected non-empty content but received none")
t.Error("Expected non-empty content")
if resp.Error != "" {
t.Fatalf("Read returned error: %s", resp.Error)
}
// 打印结果
for i, chunk := range resp.Chunks {
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
} else if i == 2 && chunkCount > 4 {
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
}
if resp.MarkdownContent == "" {
t.Error("Expected non-empty markdown content")
}
log.Println("INFO: TestReadFromURL completed successfully")
log.Printf("INFO: content_len=%d, images=%d", len(resp.MarkdownContent), len(resp.ImageRefs))
}
func TestReadFromFileWithChunking(t *testing.T) {
log.Println("INFO: Starting TestReadFromFileWithChunking")
// 创建测试客户端
log.Println("INFO: Creating test client")
func TestReadFile(t *testing.T) {
client, err := NewClient("localhost:50051")
if err != nil {
log.Printf("ERROR: Failed to create client: %v", err)
t.Fatalf("Failed to create client: %v", err)
}
defer client.Close()
// 启用调试日志
client.SetDebug(true)
// 读取测试文件
log.Println("INFO: Reading test file")
fileContent, err := os.ReadFile("../testdata/test.md")
if err != nil {
log.Printf("ERROR: Failed to read test file: %v", err)
t.Fatalf("Failed to read test file: %v", err)
}
log.Printf("INFO: Read test file, size: %d bytes", len(fileContent))
// 测试 ReadFromFile 方法,带分块参数
log.Println("INFO: Sending ReadFromFile request to server")
startTime := time.Now()
resp, err := client.ReadFromFile(
resp, err := client.Read(
context.Background(),
&proto.ReadFromFileRequest{
&proto.ReadRequest{
FileContent: fileContent,
FileName: "test.md",
FileType: "md",
ReadConfig: &proto.ReadConfig{
ChunkSize: 200,
ChunkOverlap: 50,
Separators: []string{"\n\n", "\n", "。"},
EnableMultimodal: true,
},
},
)
log.Printf("INFO: Read(file) completed in %v", time.Since(startTime))
requestDuration := time.Since(startTime)
if err != nil {
log.Printf("ERROR: ReadFromFile failed: %v", err)
t.Fatalf("ReadFromFile failed: %v", err)
t.Fatalf("Read failed: %v", err)
}
log.Printf("INFO: ReadFromFile completed in %v", requestDuration)
// 验证结果
chunkCount := len(resp.Chunks)
log.Printf("INFO: Received %d chunks from file parsing", chunkCount)
if chunkCount == 0 {
log.Println("WARN: Expected non-empty content but received none")
t.Error("Expected non-empty content")
if resp.Error != "" {
t.Fatalf("Read returned error: %s", resp.Error)
}
if resp.MarkdownContent == "" {
t.Error("Expected non-empty markdown content")
}
// 打印结果
for i, chunk := range resp.Chunks {
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
} else if i == 2 && chunkCount > 4 {
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
}
}
log.Println("INFO: TestReadFromFileWithChunking completed successfully")
}
// 截断字符串以供日志打印
func truncateString(s string, maxLen int) string {
if len(s) <= maxLen {
return s
}
return s[:maxLen] + "..."
imageRefs := GetImageRefsFromResponse(resp)
log.Printf("INFO: content_len=%d, images=%d", len(resp.MarkdownContent), len(imageRefs))
}