init commit
This commit is contained in:
Vendored
BIN
Binary file not shown.
@@ -0,0 +1,115 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/services/docreader/src/proto"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/resolver"
|
||||
)
|
||||
|
||||
const (
|
||||
maxMessageSize = 50 * 1024 * 1024 // 50MB
|
||||
)
|
||||
|
||||
var (
|
||||
// Logger is the default logger used by the client
|
||||
Logger = log.New(os.Stdout, "[DocReader] ", log.LstdFlags|log.Lmicroseconds)
|
||||
)
|
||||
|
||||
// ImageInfo 表示一个图片的信息
|
||||
type ImageInfo struct {
|
||||
URL string // 图片URL(COS)
|
||||
Caption string // 图片描述
|
||||
OCRText string // OCR提取的文本
|
||||
OriginalURL string // 原始图片URL
|
||||
Start int // 图片在文本中的开始位置
|
||||
End int // 图片在文本中的结束位置
|
||||
}
|
||||
|
||||
// Client represents a DocReader service client
|
||||
type Client struct {
|
||||
conn *grpc.ClientConn
|
||||
proto.DocReaderClient
|
||||
debug bool
|
||||
}
|
||||
|
||||
// NewClient creates a new DocReader client with the specified address
|
||||
func NewClient(addr string) (*Client, error) {
|
||||
Logger.Printf("INFO: Creating new DocReader client connecting to %s", addr)
|
||||
|
||||
// 设置消息大小限制
|
||||
opts := []grpc.DialOption{
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`),
|
||||
grpc.WithDefaultCallOptions(
|
||||
grpc.MaxCallRecvMsgSize(maxMessageSize),
|
||||
grpc.MaxCallSendMsgSize(maxMessageSize),
|
||||
),
|
||||
}
|
||||
resolver.SetDefaultScheme("dns")
|
||||
|
||||
startTime := time.Now()
|
||||
conn, err := grpc.Dial("dns:///"+addr, opts...)
|
||||
if err != nil {
|
||||
Logger.Printf("ERROR: Failed to connect to DocReader service: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
Logger.Printf("INFO: Successfully connected to DocReader service in %v", time.Since(startTime))
|
||||
|
||||
return &Client{
|
||||
conn: conn,
|
||||
DocReaderClient: proto.NewDocReaderClient(conn),
|
||||
debug: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Close closes the client connection
|
||||
func (c *Client) Close() error {
|
||||
Logger.Printf("INFO: Closing DocReader client connection")
|
||||
return c.conn.Close()
|
||||
}
|
||||
|
||||
// SetDebug enables or disables debug logging
|
||||
func (c *Client) SetDebug(debug bool) {
|
||||
c.debug = debug
|
||||
Logger.Printf("INFO: Debug logging set to %v", debug)
|
||||
}
|
||||
|
||||
// Log logs a message with the appropriate level
|
||||
func (c *Client) Log(level string, format string, args ...interface{}) {
|
||||
if level == "DEBUG" && !c.debug {
|
||||
return
|
||||
}
|
||||
Logger.Printf("%s: %s", level, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
// GetImagesFromChunk 从一个Chunk中提取所有图片信息
|
||||
func GetImagesFromChunk(chunk *proto.Chunk) []ImageInfo {
|
||||
if chunk == nil || len(chunk.Images) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
images := make([]ImageInfo, 0, len(chunk.Images))
|
||||
for _, img := range chunk.Images {
|
||||
images = append(images, ImageInfo{
|
||||
URL: img.Url,
|
||||
Caption: img.Caption,
|
||||
OCRText: img.OcrText,
|
||||
OriginalURL: img.OriginalUrl,
|
||||
Start: int(img.Start),
|
||||
End: int(img.End),
|
||||
})
|
||||
}
|
||||
|
||||
return images
|
||||
}
|
||||
|
||||
// HasImagesInChunk 判断一个Chunk是否包含图片
|
||||
func HasImagesInChunk(chunk *proto.Chunk) bool {
|
||||
return chunk != nil && len(chunk.Images) > 0
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/services/docreader/src/proto"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// 配置测试日志
|
||||
log.SetOutput(os.Stdout)
|
||||
log.SetFlags(log.LstdFlags | log.Lmicroseconds | log.Lshortfile)
|
||||
log.Println("INFO: Initializing DocReader client tests")
|
||||
}
|
||||
|
||||
func TestReadFromURL(t *testing.T) {
|
||||
log.Println("INFO: Starting TestReadFromURL")
|
||||
|
||||
// 创建测试客户端
|
||||
log.Println("INFO: Creating test client")
|
||||
client, err := NewClient("localhost:50051")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to create client: %v", err)
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// 启用调试日志
|
||||
client.SetDebug(true)
|
||||
|
||||
// 测试 ReadFromURL 方法
|
||||
log.Println("INFO: Sending ReadFromURL request to server")
|
||||
startTime := time.Now()
|
||||
resp, err := client.ReadFromURL(
|
||||
context.Background(),
|
||||
&proto.ReadFromURLRequest{
|
||||
Url: "https://example.com",
|
||||
Title: "test",
|
||||
ReadConfig: &proto.ReadConfig{
|
||||
ChunkSize: 512,
|
||||
ChunkOverlap: 50,
|
||||
Separators: []string{"\n\n", "\n", "。"},
|
||||
EnableMultimodal: true,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
requestDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: ReadFromURL failed: %v", err)
|
||||
t.Fatalf("ReadFromURL failed: %v", err)
|
||||
}
|
||||
log.Printf("INFO: ReadFromURL completed in %v", requestDuration)
|
||||
|
||||
// 验证结果
|
||||
chunkCount := len(resp.Chunks)
|
||||
log.Printf("INFO: Received %d chunks from URL parsing", chunkCount)
|
||||
if chunkCount == 0 {
|
||||
log.Println("WARN: Expected non-empty content but received none")
|
||||
t.Error("Expected non-empty content")
|
||||
}
|
||||
|
||||
// 打印结果
|
||||
for i, chunk := range resp.Chunks {
|
||||
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
|
||||
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
|
||||
} else if i == 2 && chunkCount > 4 {
|
||||
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("INFO: TestReadFromURL completed successfully")
|
||||
}
|
||||
|
||||
func TestReadFromFileWithChunking(t *testing.T) {
|
||||
log.Println("INFO: Starting TestReadFromFileWithChunking")
|
||||
|
||||
// 创建测试客户端
|
||||
log.Println("INFO: Creating test client")
|
||||
client, err := NewClient("localhost:50051")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to create client: %v", err)
|
||||
t.Fatalf("Failed to create client: %v", err)
|
||||
}
|
||||
defer client.Close()
|
||||
|
||||
// 启用调试日志
|
||||
client.SetDebug(true)
|
||||
|
||||
// 读取测试文件
|
||||
log.Println("INFO: Reading test file")
|
||||
fileContent, err := os.ReadFile("../testdata/test.md")
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to read test file: %v", err)
|
||||
t.Fatalf("Failed to read test file: %v", err)
|
||||
}
|
||||
log.Printf("INFO: Read test file, size: %d bytes", len(fileContent))
|
||||
|
||||
// 测试 ReadFromFile 方法,带分块参数
|
||||
log.Println("INFO: Sending ReadFromFile request to server")
|
||||
startTime := time.Now()
|
||||
resp, err := client.ReadFromFile(
|
||||
context.Background(),
|
||||
&proto.ReadFromFileRequest{
|
||||
FileContent: fileContent,
|
||||
FileName: "test.md",
|
||||
FileType: "md",
|
||||
ReadConfig: &proto.ReadConfig{
|
||||
ChunkSize: 200,
|
||||
ChunkOverlap: 50,
|
||||
Separators: []string{"\n\n", "\n", "。"},
|
||||
EnableMultimodal: true,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
requestDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: ReadFromFile failed: %v", err)
|
||||
t.Fatalf("ReadFromFile failed: %v", err)
|
||||
}
|
||||
log.Printf("INFO: ReadFromFile completed in %v", requestDuration)
|
||||
|
||||
// 验证结果
|
||||
chunkCount := len(resp.Chunks)
|
||||
log.Printf("INFO: Received %d chunks from file parsing", chunkCount)
|
||||
if chunkCount == 0 {
|
||||
log.Println("WARN: Expected non-empty content but received none")
|
||||
t.Error("Expected non-empty content")
|
||||
}
|
||||
|
||||
// 打印结果
|
||||
for i, chunk := range resp.Chunks {
|
||||
if i < 2 || i >= chunkCount-2 { // 只打印前两个和后两个块
|
||||
log.Printf("DEBUG: Chunk %d: %s", chunk.Seq, truncateString(chunk.Content, 50))
|
||||
} else if i == 2 && chunkCount > 4 {
|
||||
log.Printf("DEBUG: ... %d more chunks ...", chunkCount-4)
|
||||
}
|
||||
}
|
||||
|
||||
log.Println("INFO: TestReadFromFileWithChunking completed successfully")
|
||||
}
|
||||
|
||||
// 截断字符串以供日志打印
|
||||
func truncateString(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen] + "..."
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
import sys
|
||||
import os
|
||||
import logging
|
||||
from paddleocr import PaddleOCR
|
||||
|
||||
# 添加当前目录到Python路径
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
if current_dir not in sys.path:
|
||||
sys.path.append(current_dir)
|
||||
|
||||
# 导入ImageParser
|
||||
from parser.image_parser import ImageParser
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def init_ocr_model():
|
||||
PaddleOCR()
|
||||
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
Parser module for WeKnora document processing system.
|
||||
|
||||
This module provides document parsers for various file formats including:
|
||||
- Microsoft Word documents (.doc, .docx)
|
||||
- PDF documents
|
||||
- Markdown files
|
||||
- Plain text files
|
||||
- Images with text content
|
||||
- Web pages
|
||||
|
||||
The parsers extract content from documents and can split them into
|
||||
meaningful chunks for further processing and indexing.
|
||||
"""
|
||||
|
||||
from .base_parser import BaseParser, ParseResult
|
||||
from .docx_parser import DocxParser
|
||||
from .doc_parser import DocParser
|
||||
from .pdf_parser import PDFParser
|
||||
from .markdown_parser import MarkdownParser
|
||||
from .text_parser import TextParser
|
||||
from .image_parser import ImageParser
|
||||
from .web_parser import WebParser
|
||||
from .parser import Parser, ChunkingConfig
|
||||
from .ocr_engine import OCREngine
|
||||
|
||||
# Export public classes and modules
|
||||
__all__ = [
|
||||
"BaseParser", # Base parser class that all format parsers inherit from
|
||||
"DocxParser", # Parser for .docx files (modern Word documents)
|
||||
"DocParser", # Parser for .doc files (legacy Word documents)
|
||||
"PDFParser", # Parser for PDF documents
|
||||
"MarkdownParser", # Parser for Markdown text files
|
||||
"TextParser", # Parser for plain text files
|
||||
"ImageParser", # Parser for images with text content
|
||||
"WebParser", # Parser for web pages
|
||||
"Parser", # Main parser factory that selects the appropriate parser
|
||||
"ChunkingConfig", # Configuration for text chunking behavior
|
||||
"ParseResult", # Standard result format returned by all parsers
|
||||
"OCREngine", # OCR engine for extracting text from images
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,271 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional, Union
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImageUrl:
|
||||
"""Image URL data structure for caption requests."""
|
||||
|
||||
url: Optional[str] = None
|
||||
detail: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Content:
|
||||
"""Content data structure that can contain text or image URL."""
|
||||
|
||||
type: Optional[str] = None
|
||||
text: Optional[str] = None
|
||||
image_url: Optional[ImageUrl] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemMessage:
|
||||
"""System message for VLM model requests."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class UserMessage:
|
||||
"""User message for VLM model requests, can contain multiple content items."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: List[Content] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompletionRequest:
|
||||
"""Request structure for VLM model completion API."""
|
||||
|
||||
model: str
|
||||
temperature: float
|
||||
top_p: float
|
||||
messages: List[Union[SystemMessage, UserMessage]]
|
||||
user: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Model:
|
||||
"""Model identifier structure."""
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelsResp:
|
||||
"""Response structure for available models API."""
|
||||
|
||||
data: List[Model] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Message:
|
||||
"""Message structure in API response."""
|
||||
|
||||
role: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
tool_calls: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Choice:
|
||||
"""Choice structure in API response."""
|
||||
|
||||
message: Optional[Message] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Usage:
|
||||
"""Token usage information in API response."""
|
||||
|
||||
prompt_tokens: Optional[int] = 0
|
||||
total_tokens: Optional[int] = 0
|
||||
completion_tokens: Optional[int] = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class CaptionChatResp:
|
||||
"""Response structure for caption chat API."""
|
||||
|
||||
id: Optional[str] = None
|
||||
created: Optional[int] = None
|
||||
model: Optional[Model] = None
|
||||
object: Optional[str] = None
|
||||
choices: List[Choice] = field(default_factory=list)
|
||||
usage: Optional[Usage] = None
|
||||
|
||||
@staticmethod
|
||||
def from_json(json_data: dict) -> "CaptionChatResp":
|
||||
"""
|
||||
Parse API response JSON into a CaptionChatResp object.
|
||||
|
||||
Args:
|
||||
json_data: The JSON response from the API
|
||||
|
||||
Returns:
|
||||
A parsed CaptionChatResp object
|
||||
"""
|
||||
logger.info("Parsing CaptionChatResp from JSON")
|
||||
# Manually parse nested fields
|
||||
choices = [
|
||||
Choice(message=Message(**choice["message"]))
|
||||
for choice in json_data.get("choices", [])
|
||||
]
|
||||
usage = Usage(**json_data["usage"]) if "usage" in json_data else None
|
||||
logger.info(
|
||||
f"Parsed {len(choices)} choices and usage data: {usage is not None}"
|
||||
)
|
||||
return CaptionChatResp(
|
||||
id=json_data.get("id"),
|
||||
created=json_data.get("created"),
|
||||
model=json_data.get("model"),
|
||||
object=json_data.get("object"),
|
||||
choices=choices,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
def choice_data(self) -> str:
|
||||
"""
|
||||
Extract the content from the first choice in the response.
|
||||
|
||||
Returns:
|
||||
The content string from the first choice, or empty string if no choices
|
||||
"""
|
||||
if self.choices:
|
||||
logger.info("Retrieving content from first choice")
|
||||
return self.choices[0].message.content
|
||||
logger.warning("No choices available in response")
|
||||
return ""
|
||||
|
||||
|
||||
class Caption:
|
||||
"""
|
||||
Service for generating captions for images using a Vision Language Model.
|
||||
Uses an external API to process images and return textual descriptions.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the Caption service with configuration from environment variables."""
|
||||
logger.info("Initializing Caption service")
|
||||
self.prompt = """简单凝炼的描述图片的主要内容"""
|
||||
if os.getenv("VLM_MODEL_BASE_URL") == "" or os.getenv("VLM_MODEL_NAME") == "":
|
||||
logger.error("VLM_MODEL_BASE_URL or VLM_MODEL_NAME is not set")
|
||||
return
|
||||
self.completion_url = os.getenv("VLM_MODEL_BASE_URL") + "/v1/chat/completions"
|
||||
self.model = os.getenv("VLM_MODEL_NAME")
|
||||
logger.info(
|
||||
f"Service configured with model: {self.model}, endpoint: {self.completion_url}"
|
||||
)
|
||||
|
||||
def _call_caption_api(self, image_url: str) -> Optional[CaptionChatResp]:
|
||||
"""
|
||||
Call the Caption API to generate a description for the given image.
|
||||
|
||||
Args:
|
||||
image_url: URL of the image to be captioned
|
||||
|
||||
Returns:
|
||||
CaptionChatResp object if successful, None otherwise
|
||||
"""
|
||||
logger.info(f"Calling Caption API for image captioning")
|
||||
logger.info(f"Processing image from URL: {image_url}")
|
||||
|
||||
user_msg = UserMessage(
|
||||
role="user",
|
||||
content=[
|
||||
Content(type="text", text=self.prompt),
|
||||
Content(type="image_url", image_url=ImageUrl(url=image_url)),
|
||||
],
|
||||
)
|
||||
|
||||
gpt_req = CompletionRequest(
|
||||
model=self.model,
|
||||
temperature=0.3,
|
||||
top_p=0.8,
|
||||
messages=[user_msg],
|
||||
user="abc",
|
||||
)
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "text/event-stream",
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
|
||||
try:
|
||||
logger.info(f"Sending request to Caption API with model: {self.model}")
|
||||
response = requests.post(
|
||||
self.completion_url,
|
||||
data=json.dumps(gpt_req, default=lambda o: o.__dict__, indent=4),
|
||||
headers=headers,
|
||||
timeout=30,
|
||||
)
|
||||
if response.status_code != 200:
|
||||
logger.error(
|
||||
f"Caption API returned non-200 status code: {response.status_code}"
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
logger.info(
|
||||
f"Successfully received response from Caption API with status: {response.status_code}"
|
||||
)
|
||||
logger.info(f"Converting response to CaptionChatResp object")
|
||||
caption_resp = CaptionChatResp.from_json(response.json())
|
||||
|
||||
if caption_resp.usage:
|
||||
logger.info(
|
||||
f"API usage: prompt_tokens={caption_resp.usage.prompt_tokens}, "
|
||||
f"completion_tokens={caption_resp.usage.completion_tokens}"
|
||||
)
|
||||
|
||||
return caption_resp
|
||||
except requests.exceptions.Timeout:
|
||||
logger.error(f"Timeout while calling Caption API after 30 seconds")
|
||||
return None
|
||||
except requests.exceptions.RequestException as e:
|
||||
logger.error(f"Request error calling Caption API: {e}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error calling Caption API: {str(e)}")
|
||||
logger.info(
|
||||
f"Request details: model={self.model}, URL={self.completion_url}"
|
||||
)
|
||||
return None
|
||||
|
||||
def get_caption(self, image_url: str) -> str:
|
||||
"""
|
||||
Get a caption for the provided image URL.
|
||||
|
||||
Args:
|
||||
image_url: URL of the image to be captioned
|
||||
|
||||
Returns:
|
||||
Caption text as string, or empty string if captioning failed
|
||||
"""
|
||||
logger.info("Getting caption for image")
|
||||
if not image_url or self.completion_url is None:
|
||||
logger.error("Image URL is not set")
|
||||
return ""
|
||||
caption_resp = self._call_caption_api(image_url)
|
||||
if caption_resp:
|
||||
caption = caption_resp.choice_data()
|
||||
caption_length = len(caption)
|
||||
logger.info(f"Successfully generated caption of length {caption_length}")
|
||||
logger.info(
|
||||
f"Caption: {caption[:50]}..."
|
||||
if caption_length > 50
|
||||
else f"Caption: {caption}"
|
||||
)
|
||||
return caption
|
||||
logger.warning("Failed to get caption from Caption API")
|
||||
return ""
|
||||
@@ -0,0 +1,314 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import tempfile
|
||||
import os
|
||||
import subprocess
|
||||
import shutil
|
||||
from io import BytesIO
|
||||
from typing import Optional, List, Tuple
|
||||
import textract
|
||||
from PIL import Image
|
||||
import zipfile
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from .base_parser import BaseParser
|
||||
from .docx_parser import DocxParser, Docx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DocParser(BaseParser):
|
||||
"""DOC document parser"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> str:
|
||||
"""Parse DOC document
|
||||
|
||||
Args:
|
||||
content: DOC document content
|
||||
|
||||
Returns:
|
||||
Parse result
|
||||
"""
|
||||
logger.info(f"Parsing DOC document, content size: {len(content)} bytes")
|
||||
|
||||
# Save byte content as a temporary file
|
||||
with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as temp_file:
|
||||
temp_file_path = temp_file.name
|
||||
temp_file.write(content)
|
||||
temp_file.flush()
|
||||
logger.info(f"Saved DOC content to temporary file: {temp_file_path}")
|
||||
|
||||
try:
|
||||
# First try to convert to docx format to extract images
|
||||
if self.enable_multimodal:
|
||||
logger.info("Multimodal enabled, attempting to extract images from DOC")
|
||||
docx_content = self._convert_doc_to_docx(temp_file_path)
|
||||
|
||||
if docx_content:
|
||||
logger.info("Successfully converted DOC to DOCX, using DocxParser")
|
||||
# Use existing DocxParser to parse the converted docx
|
||||
docx_parser = DocxParser(
|
||||
file_name=self.file_name,
|
||||
file_type="docx",
|
||||
enable_multimodal=self.enable_multimodal,
|
||||
chunk_size=self.chunk_size,
|
||||
chunk_overlap=self.chunk_overlap,
|
||||
separators=self.separators,
|
||||
)
|
||||
text = docx_parser.parse_into_text(docx_content)
|
||||
logger.info(f"Extracted {len(text)} characters using DocxParser")
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file: {temp_file_path}")
|
||||
|
||||
return text
|
||||
else:
|
||||
logger.warning(
|
||||
"Failed to convert DOC to DOCX, falling back to text-only extraction"
|
||||
)
|
||||
|
||||
# If image extraction is not needed or conversion failed, try using antiword to extract text
|
||||
try:
|
||||
logger.info("Attempting to parse DOC file with antiword")
|
||||
# Check if antiword is installed
|
||||
antiword_path = self._find_antiword_path()
|
||||
|
||||
if antiword_path:
|
||||
# Use antiword to extract text directly
|
||||
logger.info(f"Using antiword at {antiword_path} to extract text")
|
||||
process = subprocess.Popen(
|
||||
[antiword_path, temp_file_path],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = process.communicate()
|
||||
|
||||
if process.returncode == 0:
|
||||
text = stdout.decode("utf-8", errors="ignore")
|
||||
logger.info(
|
||||
f"Successfully extracted {len(text)} characters using antiword"
|
||||
)
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file: {temp_file_path}")
|
||||
|
||||
return text
|
||||
else:
|
||||
logger.warning(
|
||||
f"antiword extraction failed: {stderr.decode('utf-8', errors='ignore')}"
|
||||
)
|
||||
else:
|
||||
logger.warning("antiword not found, falling back to textract")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Error using antiword: {str(e)}, falling back to textract"
|
||||
)
|
||||
|
||||
# If antiword fails, try using textract
|
||||
logger.info("Parsing DOC file with textract")
|
||||
text = textract.process(temp_file_path, method="antiword").decode("utf-8")
|
||||
logger.info(
|
||||
f"Successfully extracted {len(text)} characters of text from DOC document using textract"
|
||||
)
|
||||
|
||||
# Clean up temporary file
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file: {temp_file_path}")
|
||||
|
||||
return text
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing DOC document: {str(e)}")
|
||||
# Ensure temporary file is cleaned up
|
||||
if os.path.exists(temp_file_path):
|
||||
os.unlink(temp_file_path)
|
||||
logger.info(f"Deleted temporary file after error: {temp_file_path}")
|
||||
return ""
|
||||
|
||||
def _convert_doc_to_docx(self, doc_path: str) -> Optional[bytes]:
|
||||
"""Convert DOC file to DOCX format
|
||||
|
||||
Uses LibreOffice/OpenOffice for conversion
|
||||
|
||||
Args:
|
||||
doc_path: DOC file path
|
||||
|
||||
Returns:
|
||||
Byte stream of DOCX file content, or None if conversion fails
|
||||
"""
|
||||
logger.info(f"Converting DOC to DOCX: {doc_path}")
|
||||
|
||||
# Create a temporary directory to store the converted file
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
docx_path = os.path.join(temp_dir, "converted.docx")
|
||||
|
||||
try:
|
||||
# Check if LibreOffice or OpenOffice is installed
|
||||
soffice_path = self._find_soffice_path()
|
||||
if not soffice_path:
|
||||
logger.error(
|
||||
"LibreOffice/OpenOffice not found, cannot convert DOC to DOCX"
|
||||
)
|
||||
return None
|
||||
|
||||
# Execute conversion command
|
||||
logger.info(f"Using {soffice_path} to convert DOC to DOCX")
|
||||
cmd = [
|
||||
soffice_path,
|
||||
"--headless",
|
||||
"--convert-to",
|
||||
"docx",
|
||||
"--outdir",
|
||||
temp_dir,
|
||||
doc_path,
|
||||
]
|
||||
|
||||
logger.info(f"Running command: {' '.join(cmd)}")
|
||||
process = subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
|
||||
)
|
||||
stdout, stderr = process.communicate()
|
||||
|
||||
if process.returncode != 0:
|
||||
logger.error(
|
||||
f"Error converting DOC to DOCX: {stderr.decode('utf-8', errors='ignore')}"
|
||||
)
|
||||
return None
|
||||
|
||||
# Find the converted file
|
||||
for file in os.listdir(temp_dir):
|
||||
if file.endswith(".docx"):
|
||||
converted_file = os.path.join(temp_dir, file)
|
||||
logger.info(f"Found converted file: {converted_file}")
|
||||
|
||||
# Read the converted file content
|
||||
with open(converted_file, "rb") as f:
|
||||
docx_content = f.read()
|
||||
|
||||
logger.info(
|
||||
f"Successfully read converted DOCX file, size: {len(docx_content)} bytes"
|
||||
)
|
||||
return docx_content
|
||||
|
||||
logger.error("No DOCX file found after conversion")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during DOC to DOCX conversion: {str(e)}")
|
||||
return None
|
||||
finally:
|
||||
# Clean up temporary directory
|
||||
try:
|
||||
shutil.rmtree(temp_dir)
|
||||
logger.info(f"Cleaned up temporary directory: {temp_dir}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clean up temporary directory: {str(e)}")
|
||||
|
||||
def _find_soffice_path(self) -> Optional[str]:
|
||||
"""Find LibreOffice/OpenOffice executable path
|
||||
|
||||
Returns:
|
||||
Executable path, or None if not found
|
||||
"""
|
||||
# Common LibreOffice/OpenOffice executable paths
|
||||
possible_paths = [
|
||||
# Linux
|
||||
"/usr/bin/soffice",
|
||||
"/usr/lib/libreoffice/program/soffice",
|
||||
"/opt/libreoffice25.2/program/soffice",
|
||||
# macOS
|
||||
"/Applications/LibreOffice.app/Contents/MacOS/soffice",
|
||||
# Windows
|
||||
"C:\\Program Files\\LibreOffice\\program\\soffice.exe",
|
||||
"C:\\Program Files (x86)\\LibreOffice\\program\\soffice.exe",
|
||||
]
|
||||
|
||||
# Check if path is set in environment variable
|
||||
if os.environ.get("LIBREOFFICE_PATH"):
|
||||
possible_paths.insert(0, os.environ.get("LIBREOFFICE_PATH"))
|
||||
|
||||
for path in possible_paths:
|
||||
if os.path.exists(path):
|
||||
logger.info(f"Found LibreOffice/OpenOffice at: {path}")
|
||||
return path
|
||||
|
||||
# Try to find in PATH
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["which", "soffice"], capture_output=True, text=True
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
path = result.stdout.strip()
|
||||
logger.info(f"Found LibreOffice/OpenOffice in PATH: {path}")
|
||||
return path
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.warning("LibreOffice/OpenOffice not found")
|
||||
return None
|
||||
|
||||
def _find_antiword_path(self) -> Optional[str]:
|
||||
"""Find antiword executable path
|
||||
|
||||
Returns:
|
||||
Executable path, or None if not found
|
||||
"""
|
||||
# Common antiword executable paths
|
||||
possible_paths = [
|
||||
# Linux/macOS
|
||||
"/usr/bin/antiword",
|
||||
"/usr/local/bin/antiword",
|
||||
# Windows
|
||||
"C:\\Program Files\\Antiword\\antiword.exe",
|
||||
"C:\\Program Files (x86)\\Antiword\\antiword.exe",
|
||||
]
|
||||
|
||||
# Check if path is set in environment variable
|
||||
if os.environ.get("ANTIWORD_PATH"):
|
||||
possible_paths.insert(0, os.environ.get("ANTIWORD_PATH"))
|
||||
|
||||
for path in possible_paths:
|
||||
if os.path.exists(path):
|
||||
logger.info(f"Found antiword at: {path}")
|
||||
return path
|
||||
|
||||
# Try to find in PATH
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["which", "antiword"], capture_output=True, text=True
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
path = result.stdout.strip()
|
||||
logger.info(f"Found antiword in PATH: {path}")
|
||||
return path
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.warning("antiword not found")
|
||||
return None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
logger.info("Running DocParser in standalone mode")
|
||||
|
||||
file_name = "/path/to/your/test.doc"
|
||||
logger.info(f"Processing file: {file_name}")
|
||||
|
||||
doc_parser = DocParser(
|
||||
file_name, enable_multimodal=True, chunk_size=512, chunk_overlap=60
|
||||
)
|
||||
logger.info("Parser initialized, starting processing")
|
||||
|
||||
with open(file_name, "rb") as f:
|
||||
content = f.read()
|
||||
|
||||
text = doc_parser.parse_into_text(content)
|
||||
logger.info(f"Processing complete, extracted text length: {len(text)}")
|
||||
logger.info(f"Sample text: {text[:200]}...")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,58 @@
|
||||
import logging
|
||||
import os
|
||||
import asyncio
|
||||
from PIL import Image
|
||||
import io
|
||||
from .base_parser import BaseParser, ParseResult
|
||||
import numpy as np
|
||||
|
||||
# Set up logger for this module
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ImageParser(BaseParser):
|
||||
"""
|
||||
Parser for image files with OCR capability.
|
||||
Extracts text from images and generates captions.
|
||||
|
||||
This parser handles image processing by:
|
||||
1. Uploading the image to storage
|
||||
2. Generating a descriptive caption
|
||||
3. Performing OCR to extract text content
|
||||
4. Returning a combined result with both text and image reference
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> str:
|
||||
"""
|
||||
Parse image content, only upload the image and return Markdown reference, no OCR or caption processing.
|
||||
|
||||
Args:
|
||||
content: Raw image data (bytes)
|
||||
|
||||
Returns:
|
||||
String containing Markdown image reference
|
||||
"""
|
||||
logger.info(f"Parsing image content, size: {len(content)} bytes")
|
||||
try:
|
||||
# Upload image to storage service
|
||||
logger.info("Uploading image to storage")
|
||||
_, ext = os.path.splitext(self.file_name)
|
||||
image_url = self.upload_bytes(content, file_ext=ext)
|
||||
if not image_url:
|
||||
logger.error("Failed to upload image to storage")
|
||||
return ""
|
||||
logger.info(
|
||||
f"Successfully uploaded image, URL: {image_url[:50]}..."
|
||||
if len(image_url) > 50
|
||||
else f"Successfully uploaded image, URL: {image_url}"
|
||||
)
|
||||
|
||||
# Directly generate Markdown image reference, no OCR or caption processing
|
||||
markdown_text = f""
|
||||
logger.info("Generated Markdown image reference without OCR or caption processing")
|
||||
|
||||
return markdown_text
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing image: {str(e)}")
|
||||
return ""
|
||||
@@ -0,0 +1,33 @@
|
||||
import asyncio
|
||||
import re
|
||||
import logging
|
||||
import numpy as np
|
||||
import os # Import os module to get environment variables
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from .base_parser import BaseParser
|
||||
|
||||
# Get logger object
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MarkdownParser(BaseParser):
|
||||
"""Markdown document parser"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> str:
|
||||
"""Parse Markdown document, only extract text content, do not process images
|
||||
|
||||
Args:
|
||||
content: Markdown document content
|
||||
|
||||
Returns:
|
||||
Parsed text result
|
||||
"""
|
||||
logger.info(f"Parsing Markdown document, content size: {len(content)} bytes")
|
||||
|
||||
# Convert byte content to string using universal decoding method
|
||||
text = self.decode_bytes(content)
|
||||
logger.info(f"Decoded Markdown content, text length: {len(text)} characters")
|
||||
|
||||
logger.info(f"Markdown parsing complete, extracted {len(text)} characters of text")
|
||||
return text
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
import os
|
||||
import logging
|
||||
import base64
|
||||
from typing import Optional, Union, Dict, Any
|
||||
from abc import ABC, abstractmethod
|
||||
from PIL import Image
|
||||
import io
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
class OCRBackend(ABC):
|
||||
"""Base class for OCR backends"""
|
||||
|
||||
@abstractmethod
|
||||
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
|
||||
"""Extract text from an image
|
||||
|
||||
Args:
|
||||
image: Image file path, bytes, or PIL Image object
|
||||
|
||||
Returns:
|
||||
Extracted text
|
||||
"""
|
||||
pass
|
||||
|
||||
class PaddleOCRBackend(OCRBackend):
|
||||
"""PaddleOCR backend implementation"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
"""Initialize PaddleOCR backend"""
|
||||
self.ocr = None
|
||||
try:
|
||||
from paddleocr import PaddleOCR
|
||||
# Default OCR configuration
|
||||
ocr_config = {
|
||||
"use_doc_orientation_classify": False, # Do not use document image orientation classification
|
||||
"use_doc_unwarping": False, # Do not use document unwarping
|
||||
"use_textline_orientation": False, # Do not use textline orientation classification
|
||||
"text_recognition_model_name": "PP-OCRv5_server_rec",
|
||||
"text_detection_model_name": "PP-OCRv5_server_det",
|
||||
"text_recognition_model_dir": "/root/.paddlex/official_models/PP-OCRv5_server_rec_infer",
|
||||
"text_detection_model_dir": "/root/.paddlex/official_models/PP-OCRv5_server_det_infer",
|
||||
"text_det_limit_type": "min", # Limit by short side
|
||||
"text_det_limit_side_len": 736, # Limit side length to 736
|
||||
"text_det_thresh": 0.3, # Text detection pixel threshold
|
||||
"text_det_box_thresh": 0.6, # Text detection box threshold
|
||||
"text_det_unclip_ratio": 1.5, # Text detection expansion ratio
|
||||
"text_rec_score_thresh": 0.0, # Text recognition confidence threshold
|
||||
"ocr_version": "PP-OCRv5", # Switch to PP-OCRv4 here to compare
|
||||
"lang": "ch",
|
||||
}
|
||||
|
||||
self.ocr = PaddleOCR(**ocr_config)
|
||||
logger.info("PaddleOCR engine initialized successfully")
|
||||
except ImportError:
|
||||
logger.error("Failed to import paddleocr. Please install it with 'pip install paddleocr'")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize PaddleOCR: {str(e)}")
|
||||
|
||||
def predict(self, image):
|
||||
"""Perform OCR recognition on the image
|
||||
|
||||
Args:
|
||||
image: Image object (PIL.Image or numpy array)
|
||||
|
||||
Returns:
|
||||
Extracted text string
|
||||
"""
|
||||
try:
|
||||
# Ensure image is in RGB format
|
||||
if hasattr(image, "convert") and image.mode != "RGBA":
|
||||
img_for_ocr = image.convert("RGBA")
|
||||
logger.info(f"Converted image from {image.mode} to RGB format")
|
||||
else:
|
||||
img_for_ocr = image
|
||||
|
||||
# Convert to numpy array if not already
|
||||
if hasattr(img_for_ocr, "convert"):
|
||||
image_array = np.array(img_for_ocr)
|
||||
else:
|
||||
image_array = img_for_ocr
|
||||
|
||||
ocr_result = self.ocr.predict(image_array)
|
||||
logger.info(f"ocr_result: {ocr_result}")
|
||||
|
||||
# Extract text
|
||||
if ocr_result and any(ocr_result):
|
||||
ocr_text = ""
|
||||
for image_result in ocr_result:
|
||||
ocr_text = ocr_text + " ".join(image_result["rec_texts"])
|
||||
text_length = len(ocr_text)
|
||||
if text_length > 0:
|
||||
logger.info(f"OCR extracted {text_length} characters")
|
||||
logger.info(
|
||||
f"OCR text sample: {ocr_text[:100]}..."
|
||||
if text_length > 100
|
||||
else f"OCR text: {ocr_text}"
|
||||
)
|
||||
return ocr_text
|
||||
else:
|
||||
logger.warning("OCR returned empty result")
|
||||
else:
|
||||
logger.warning("OCR did not return any result")
|
||||
return ""
|
||||
except Exception as e:
|
||||
logger.error(f"OCR recognition error: {str(e)}")
|
||||
return ""
|
||||
class NanonetsOCRBackend(OCRBackend):
|
||||
"""Nanonets OCR backend implementation using OpenAI API format"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
"""Initialize Nanonets OCR backend
|
||||
|
||||
Args:
|
||||
api_key: API key for OpenAI API
|
||||
base_url: Base URL for OpenAI API
|
||||
model: Model name
|
||||
"""
|
||||
try:
|
||||
from openai import OpenAI
|
||||
self.api_key = kwargs.get("api_key", "123")
|
||||
self.base_url = kwargs.get("base_url", "http://localhost:8000/v1")
|
||||
self.model = kwargs.get("model", "nanonets/Nanonets-OCR-s")
|
||||
self.temperature = kwargs.get("temperature", 0.0)
|
||||
self.max_tokens = kwargs.get("max_tokens", 15000)
|
||||
|
||||
self.client = OpenAI(api_key=self.api_key, base_url=self.base_url)
|
||||
self.prompt = """
|
||||
## 任务说明
|
||||
|
||||
请从上传的文档中提取文字内容,严格按自然阅读顺序(从上到下,从左到右)输出,并遵循以下格式规范。
|
||||
|
||||
### 1. **文本处理**
|
||||
|
||||
* 按正常阅读顺序提取文字,语句流畅自然。
|
||||
|
||||
### 2. **表格**
|
||||
|
||||
* 所有表格统一转换为 **Markdown 表格格式**。
|
||||
* 内容保持清晰、对齐整齐,便于阅读。
|
||||
|
||||
### 3. **公式**
|
||||
|
||||
* 所有公式转换为 **LaTeX 格式**,使用 `$$公式$$` 包裹。
|
||||
|
||||
### 4. **图片**
|
||||
|
||||
* 忽略图片信息
|
||||
|
||||
### 5. **链接**
|
||||
|
||||
* 不要猜测或补全不确定的链接地址。
|
||||
"""
|
||||
logger.info(f"Nanonets OCR engine initialized with model: {self.model}")
|
||||
except ImportError:
|
||||
logger.error("Failed to import openai. Please install it with 'pip install openai'")
|
||||
self.client = None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize Nanonets OCR: {str(e)}")
|
||||
self.client = None
|
||||
|
||||
def _encode_image(self, image: Union[str, bytes, Image.Image]) -> str:
|
||||
"""Encode image to base64
|
||||
|
||||
Args:
|
||||
image: Image file path, bytes, or PIL Image object
|
||||
|
||||
Returns:
|
||||
Base64 encoded image
|
||||
"""
|
||||
try:
|
||||
if isinstance(image, str):
|
||||
# It's a file path
|
||||
with open(image, "rb") as image_file:
|
||||
return base64.b64encode(image_file.read()).decode("utf-8")
|
||||
elif isinstance(image, bytes):
|
||||
# It's bytes data
|
||||
return base64.b64encode(image).decode("utf-8")
|
||||
elif isinstance(image, Image.Image):
|
||||
# It's a PIL Image
|
||||
buffer = io.BytesIO()
|
||||
image.save(buffer, format="PNG")
|
||||
return base64.b64encode(buffer.getvalue()).decode("utf-8")
|
||||
else:
|
||||
logger.error(f"Unsupported image type: {type(image)}")
|
||||
return ""
|
||||
except Exception as e:
|
||||
logger.error(f"Error encoding image: {str(e)}")
|
||||
return ""
|
||||
|
||||
def predict(self, image: Union[str, bytes, Image.Image]) -> str:
|
||||
"""Extract text from an image using Nanonets OCR
|
||||
|
||||
Args:
|
||||
image: Image file path, bytes, or PIL Image object
|
||||
|
||||
Returns:
|
||||
Extracted text
|
||||
"""
|
||||
if self.client is None:
|
||||
logger.error("Nanonets OCR client not initialized")
|
||||
return ""
|
||||
|
||||
try:
|
||||
# Encode image to base64
|
||||
img_base64 = self._encode_image(image)
|
||||
if not img_base64:
|
||||
return ""
|
||||
|
||||
# Call Nanonets OCR API
|
||||
logger.info(f"Calling Nanonets OCR API with model: {self.model}")
|
||||
response = self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/png;base64,{img_base64}"},
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": self.prompt,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
temperature=self.temperature,
|
||||
max_tokens=self.max_tokens
|
||||
)
|
||||
|
||||
return response.choices[0].message.content
|
||||
except Exception as e:
|
||||
logger.error(f"Nanonets OCR prediction error: {str(e)}")
|
||||
return ""
|
||||
|
||||
class OCREngine:
|
||||
"""OCR Engine factory class"""
|
||||
|
||||
_instance = None
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls, backend_type="paddle", **kwargs) -> Optional[OCRBackend]:
|
||||
"""Get OCR engine instance
|
||||
|
||||
Args:
|
||||
backend_type: OCR backend type, one of: "paddle", "nanonets"
|
||||
**kwargs: Additional arguments for the backend
|
||||
|
||||
Returns:
|
||||
OCR engine instance or None if initialization fails
|
||||
"""
|
||||
if cls._instance is None:
|
||||
logger.info(f"Initializing OCR engine with backend: {backend_type}")
|
||||
|
||||
if backend_type.lower() == "paddle":
|
||||
cls._instance = PaddleOCRBackend(**kwargs)
|
||||
elif backend_type.lower() == "nanonets":
|
||||
cls._instance = NanonetsOCRBackend(**kwargs)
|
||||
else:
|
||||
logger.error(f"Unknown OCR backend type: {backend_type}")
|
||||
return None
|
||||
|
||||
return cls._instance
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Any, Optional, Type
|
||||
|
||||
from .base_parser import BaseParser, ParseResult
|
||||
from .docx_parser import DocxParser
|
||||
from .doc_parser import DocParser
|
||||
from .pdf_parser import PDFParser
|
||||
from .markdown_parser import MarkdownParser
|
||||
from .text_parser import TextParser
|
||||
from .image_parser import ImageParser
|
||||
from .web_parser import WebParser
|
||||
import traceback
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkingConfig:
|
||||
"""
|
||||
Configuration for text chunking process.
|
||||
Controls how documents are split into smaller pieces for processing.
|
||||
"""
|
||||
|
||||
chunk_size: int = 512 # Maximum size of each chunk in tokens/chars
|
||||
chunk_overlap: int = 50 # Number of tokens/chars to overlap between chunks
|
||||
separators: list = field(
|
||||
default_factory=lambda: ["\n\n", "\n", "。"]
|
||||
) # Text separators in order of priority
|
||||
enable_multimodal: bool = (
|
||||
False # Whether to enable multimodal processing (text + images)
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Chunk:
|
||||
"""
|
||||
Represents a single text chunk with associated metadata.
|
||||
Basic unit for document processing and embedding.
|
||||
"""
|
||||
|
||||
content: str # Text content of the chunk
|
||||
metadata: Dict[str, Any] = None # Associated metadata (source, page number, etc.)
|
||||
|
||||
|
||||
class Parser:
|
||||
"""
|
||||
Document parser facade that integrates all specialized parsers.
|
||||
Provides a unified interface for parsing various document types.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
logger.info("Initializing document parser")
|
||||
# Initialize all parser types
|
||||
self.parsers: Dict[str, Type[BaseParser]] = {
|
||||
"docx": DocxParser,
|
||||
"doc": DocParser,
|
||||
"pdf": PDFParser,
|
||||
"md": MarkdownParser,
|
||||
"txt": TextParser,
|
||||
"jpg": ImageParser,
|
||||
"jpeg": ImageParser,
|
||||
"png": ImageParser,
|
||||
"gif": ImageParser,
|
||||
"bmp": ImageParser,
|
||||
"tiff": ImageParser,
|
||||
"webp": ImageParser,
|
||||
"markdown": MarkdownParser,
|
||||
}
|
||||
logger.info(
|
||||
"Parser initialized with %d parsers: %s",
|
||||
len(self.parsers),
|
||||
", ".join(self.parsers.keys()),
|
||||
)
|
||||
|
||||
|
||||
def get_parser(self, file_type: str) -> Optional[Type[BaseParser]]:
|
||||
"""
|
||||
Get parser class for the specified file type.
|
||||
|
||||
Args:
|
||||
file_type: The file extension or type identifier
|
||||
|
||||
Returns:
|
||||
Parser class for the file type, or None if unsupported
|
||||
"""
|
||||
file_type = file_type.lower()
|
||||
parser = self.parsers.get(file_type)
|
||||
if parser:
|
||||
logger.info(f"Found parser for file type: {file_type}")
|
||||
else:
|
||||
logger.warning(f"No parser found for file type: {file_type}")
|
||||
return parser
|
||||
|
||||
def parse_file(
|
||||
self,
|
||||
file_name: str,
|
||||
file_type: str,
|
||||
content: bytes,
|
||||
config: ChunkingConfig,
|
||||
enable_multimodal: bool = False,
|
||||
) -> Optional[ParseResult]:
|
||||
"""
|
||||
Parse file content using appropriate parser based on file type.
|
||||
|
||||
Args:
|
||||
file_name: Name of the file being parsed
|
||||
file_type: Type/extension of the file
|
||||
content: Raw file content as bytes
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing file: {file_name} with type: {file_type}")
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, "
|
||||
f"multimodal={config.enable_multimodal}"
|
||||
)
|
||||
|
||||
parser_instance = None
|
||||
|
||||
try:
|
||||
# Get appropriate parser for file type
|
||||
cls = self.get_parser(file_type)
|
||||
if cls is None:
|
||||
logger.error(f"Unsupported file type: {file_type}")
|
||||
return None
|
||||
|
||||
# Parse file content
|
||||
logger.info(f"Creating parser instance for {file_type} file")
|
||||
parser_instance = cls(
|
||||
file_name=file_name,
|
||||
file_type=file_type,
|
||||
chunk_size=config.chunk_size,
|
||||
chunk_overlap=config.chunk_overlap,
|
||||
separators=config.separators,
|
||||
enable_multimodal=config.enable_multimodal,
|
||||
max_image_size=1920, # Limit image size to 1920px
|
||||
max_concurrent_tasks=5, # Limit concurrent tasks to 5
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse file content, size: {len(content)} bytes")
|
||||
result = parser_instance.parse(content)
|
||||
|
||||
if result:
|
||||
logger.info(
|
||||
f"Successfully parsed file {file_name}, generated {len(result.chunks)} chunks"
|
||||
)
|
||||
if result.chunks and len(result.chunks) > 0:
|
||||
logger.info(
|
||||
f"First chunk content length: {len(result.chunks[0].content)}"
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Parser returned empty chunks for file: {file_name}")
|
||||
else:
|
||||
logger.warning(f"Parser returned None result for file: {file_name}")
|
||||
|
||||
# Return parse results
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing file {file_name}: {str(e)}")
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
return None
|
||||
|
||||
def parse_url(
|
||||
self, url: str, title: str, config: ChunkingConfig
|
||||
) -> Optional[ParseResult]:
|
||||
"""
|
||||
Parse content from a URL using the WebParser.
|
||||
|
||||
Args:
|
||||
url: URL to parse
|
||||
title: Title of the webpage (for metadata)
|
||||
config: Configuration for chunking process
|
||||
|
||||
Returns:
|
||||
ParseResult containing chunks and metadata, or None if parsing failed
|
||||
"""
|
||||
logger.info(f"Parsing URL: {url}, title: {title}")
|
||||
logger.info(
|
||||
f"Chunking config: size={config.chunk_size}, overlap={config.chunk_overlap}, "
|
||||
f"multimodal={config.enable_multimodal}"
|
||||
)
|
||||
|
||||
parser_instance = None
|
||||
|
||||
try:
|
||||
# Create web parser instance
|
||||
logger.info("Creating WebParser instance")
|
||||
parser_instance = WebParser(
|
||||
title=title,
|
||||
chunk_size=config.chunk_size,
|
||||
chunk_overlap=config.chunk_overlap,
|
||||
separators=config.separators,
|
||||
enable_multimodal=config.enable_multimodal,
|
||||
max_image_size=1920, # Limit image size
|
||||
max_concurrent_tasks=5, # Limit concurrent tasks
|
||||
)
|
||||
|
||||
logger.info(f"Starting to parse URL content")
|
||||
result = parser_instance.parse(url)
|
||||
|
||||
if result:
|
||||
logger.info(
|
||||
f"Successfully parsed URL, generated {len(result.chunks)} chunks"
|
||||
)
|
||||
logger.info(
|
||||
f"First chunk content length: {len(result.chunks[0].content) if result.chunks else 0}"
|
||||
)
|
||||
else:
|
||||
logger.warning(f"Parser returned empty result for URL: {url}")
|
||||
|
||||
# Return parse results
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing URL {url}: {str(e)}")
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import logging
|
||||
import os
|
||||
import io
|
||||
from typing import Any, List, Iterator, Optional, Mapping, Tuple, Dict
|
||||
|
||||
from pypdf import PdfReader
|
||||
from .base_parser import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class PDFParser(BaseParser):
|
||||
"""
|
||||
PDF Document Parser
|
||||
|
||||
This parser handles PDF documents by extracting text content.
|
||||
It uses the pypdf library for simple text extraction.
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> str:
|
||||
"""
|
||||
Parse PDF document content into text
|
||||
|
||||
This method processes a PDF document by extracting text content.
|
||||
|
||||
Args:
|
||||
content: PDF document content as bytes
|
||||
|
||||
Returns:
|
||||
Extracted text content
|
||||
"""
|
||||
logger.info(f"Parsing PDF document, content size: {len(content)} bytes")
|
||||
|
||||
try:
|
||||
# Use io.BytesIO to read content from bytes
|
||||
pdf_file = io.BytesIO(content)
|
||||
|
||||
# Create PdfReader object
|
||||
pdf_reader = PdfReader(pdf_file)
|
||||
num_pages = len(pdf_reader.pages)
|
||||
logger.info(f"PDF has {num_pages} pages")
|
||||
|
||||
# Extract text from all pages
|
||||
all_text = []
|
||||
for page_num, page in enumerate(pdf_reader.pages):
|
||||
try:
|
||||
page_text = page.extract_text()
|
||||
if page_text:
|
||||
all_text.append(page_text)
|
||||
logger.info(f"Successfully extracted text from page {page_num+1}/{num_pages}")
|
||||
else:
|
||||
logger.warning(f"No text extracted from page {page_num+1}/{num_pages}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error extracting text from page {page_num+1}: {str(e)}")
|
||||
|
||||
# Combine all extracted text
|
||||
result = "\n\n".join(all_text)
|
||||
logger.info(f"PDF parsing complete, extracted {len(result)} characters of text")
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to parse PDF document: {str(e)}")
|
||||
return ""
|
||||
@@ -0,0 +1,57 @@
|
||||
import logging
|
||||
from .base_parser import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TextParser(BaseParser):
|
||||
"""
|
||||
Text document parser for processing plain text files.
|
||||
This parser handles text extraction and chunking from plain text documents.
|
||||
"""
|
||||
|
||||
def parse_into_text(self, content: bytes) -> str:
|
||||
"""
|
||||
Parse text document content by decoding bytes to string.
|
||||
|
||||
This is a straightforward parser that simply converts the binary content
|
||||
to text using appropriate character encoding.
|
||||
|
||||
Args:
|
||||
content: Raw document content as bytes
|
||||
|
||||
Returns:
|
||||
Parsed text content as string
|
||||
"""
|
||||
logger.info(f"Parsing text document, content size: {len(content)} bytes")
|
||||
text = self.decode_bytes(content)
|
||||
logger.info(
|
||||
f"Successfully parsed text document, extracted {len(text)} characters"
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
logger.info("Running TextParser in standalone mode")
|
||||
|
||||
# Sample text for testing
|
||||
text = """## 标题1
|
||||

|
||||
## 标题2
|
||||

|
||||
## 标题3
|
||||
"""
|
||||
logger.info(f"Test text content: {text}")
|
||||
|
||||
# Define separators for text splitting
|
||||
seperators = ["\n\n", "\n", "。"]
|
||||
parser = TextParser(separators=seperators)
|
||||
logger.info("Splitting text into units")
|
||||
units = parser._split_into_units(text)
|
||||
logger.info(f"Split text into {len(units)} units")
|
||||
logger.info(f"Units: {units}")
|
||||
@@ -0,0 +1,133 @@
|
||||
from typing import Any, Optional
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
from bs4 import BeautifulSoup
|
||||
from .base_parser import BaseParser, ParseResult
|
||||
import logging
|
||||
import asyncio
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WebParser(BaseParser):
|
||||
"""Web page parser"""
|
||||
|
||||
def __init__(self, title: str, **kwargs):
|
||||
self.title = title
|
||||
super().__init__(file_name=title, **kwargs)
|
||||
logger.info(f"Initialized WebParser with title: {title}")
|
||||
|
||||
async def scrape(self, url: str) -> Any:
|
||||
logger.info(f"Starting web page scraping for URL: {url}")
|
||||
try:
|
||||
async with async_playwright() as p:
|
||||
args = [
|
||||
# "--single-process",
|
||||
]
|
||||
logger.info("Launching WebKit browser")
|
||||
browser = await p.webkit.launch(args=args)
|
||||
page = await browser.new_page()
|
||||
|
||||
logger.info(f"Navigating to URL: {url}")
|
||||
try:
|
||||
await page.goto(url, timeout=30000)
|
||||
logger.info("Initial page load complete")
|
||||
except Exception as e:
|
||||
logger.error(f"Error navigating to URL: {str(e)}")
|
||||
await browser.close()
|
||||
return BeautifulSoup(
|
||||
"", "html.parser"
|
||||
) # Return empty soup on navigation error
|
||||
|
||||
try:
|
||||
logger.info("Waiting for network idle")
|
||||
await page.reload(wait_until="networkidle")
|
||||
logger.info("Page reload complete")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Network idle wait failed: {str(e)}, proceeding with current page content"
|
||||
)
|
||||
|
||||
logger.info("Retrieving page HTML content")
|
||||
content = await page.content()
|
||||
logger.info(f"Retrieved {len(content)} bytes of HTML content")
|
||||
|
||||
await browser.close()
|
||||
logger.info("Browser closed")
|
||||
|
||||
# Parse HTML content with BeautifulSoup
|
||||
logger.info("Parsing HTML with BeautifulSoup")
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
logger.info("Successfully parsed HTML content")
|
||||
return soup
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to scrape web page: {str(e)}")
|
||||
# Return empty BeautifulSoup object on error
|
||||
return BeautifulSoup("", "html.parser")
|
||||
|
||||
def parse_into_text(self, content: bytes) -> str:
|
||||
"""Parse web page
|
||||
|
||||
Args:
|
||||
content: Web page content
|
||||
|
||||
Returns:
|
||||
Parse result
|
||||
"""
|
||||
logger.info("Starting web page parsing")
|
||||
|
||||
# Call async method synchronously
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
|
||||
try:
|
||||
# Run async method
|
||||
# Handle content possibly being a string
|
||||
if isinstance(content, bytes):
|
||||
url = self.decode_bytes(content)
|
||||
logger.info(f"Decoded URL from bytes: {url}")
|
||||
else:
|
||||
url = content
|
||||
logger.info(f"Using content as URL directly: {url}")
|
||||
|
||||
logger.info(f"Scraping web page: {url}")
|
||||
soup = loop.run_until_complete(self.scrape(url))
|
||||
|
||||
# Extract page text
|
||||
logger.info("Extracting text from web page")
|
||||
text = soup.get_text("\n")
|
||||
logger.info(f"Extracted {len(text)} characters of text from URL: {url}")
|
||||
|
||||
# Get title, usually in <title> or <h1> tag
|
||||
if self.title != "":
|
||||
title = self.title
|
||||
logger.info(f"Using provided title: {title}")
|
||||
else:
|
||||
title = soup.title.string if soup.title else None
|
||||
logger.info(f"Found title tag: {title}")
|
||||
|
||||
if not title: # If <title> tag does not exist or is empty, try <h1> tag
|
||||
h1_tag = soup.find("h1")
|
||||
if h1_tag:
|
||||
title = h1_tag.get_text()
|
||||
logger.info(f"Using h1 tag as title: {title}")
|
||||
else:
|
||||
title = "Untitled Web Page"
|
||||
logger.info("No title found, using default")
|
||||
|
||||
logger.info(f"Web page title: {title}")
|
||||
result = title + "\n\n" + text
|
||||
logger.info(
|
||||
f"Web page parsing complete, total content: {len(result)} characters"
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing web page: {str(e)}")
|
||||
return f"Error parsing web page: {str(e)}"
|
||||
|
||||
finally:
|
||||
# Close event loop
|
||||
logger.info("Closing event loop")
|
||||
loop.close()
|
||||
@@ -0,0 +1,560 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.36.6
|
||||
// protoc v5.29.3
|
||||
// source: docreader.proto
|
||||
|
||||
package proto
|
||||
|
||||
import (
|
||||
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
|
||||
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
|
||||
reflect "reflect"
|
||||
sync "sync"
|
||||
unsafe "unsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
// Verify that this generated code is sufficiently up-to-date.
|
||||
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
|
||||
// Verify that runtime/protoimpl is sufficiently up-to-date.
|
||||
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
|
||||
)
|
||||
|
||||
type ReadConfig struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
ChunkSize int32 `protobuf:"varint,1,opt,name=chunk_size,json=chunkSize,proto3" json:"chunk_size,omitempty"` // 分块大小
|
||||
ChunkOverlap int32 `protobuf:"varint,2,opt,name=chunk_overlap,json=chunkOverlap,proto3" json:"chunk_overlap,omitempty"` // 分块重叠
|
||||
Separators []string `protobuf:"bytes,3,rep,name=separators,proto3" json:"separators,omitempty"` // 分隔符
|
||||
EnableMultimodal bool `protobuf:"varint,4,opt,name=enable_multimodal,json=enableMultimodal,proto3" json:"enable_multimodal,omitempty"` // 多模态处理
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *ReadConfig) Reset() {
|
||||
*x = ReadConfig{}
|
||||
mi := &file_docreader_proto_msgTypes[0]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *ReadConfig) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*ReadConfig) ProtoMessage() {}
|
||||
|
||||
func (x *ReadConfig) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_docreader_proto_msgTypes[0]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use ReadConfig.ProtoReflect.Descriptor instead.
|
||||
func (*ReadConfig) Descriptor() ([]byte, []int) {
|
||||
return file_docreader_proto_rawDescGZIP(), []int{0}
|
||||
}
|
||||
|
||||
func (x *ReadConfig) GetChunkSize() int32 {
|
||||
if x != nil {
|
||||
return x.ChunkSize
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *ReadConfig) GetChunkOverlap() int32 {
|
||||
if x != nil {
|
||||
return x.ChunkOverlap
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *ReadConfig) GetSeparators() []string {
|
||||
if x != nil {
|
||||
return x.Separators
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ReadConfig) GetEnableMultimodal() bool {
|
||||
if x != nil {
|
||||
return x.EnableMultimodal
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// 从文件读取文档请求
|
||||
type ReadFromFileRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
FileContent []byte `protobuf:"bytes,1,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"` // 文件内容
|
||||
FileName string `protobuf:"bytes,2,opt,name=file_name,json=fileName,proto3" json:"file_name,omitempty"` // 文件名
|
||||
FileType string `protobuf:"bytes,3,opt,name=file_type,json=fileType,proto3" json:"file_type,omitempty"` // 文件类型
|
||||
ReadConfig *ReadConfig `protobuf:"bytes,4,opt,name=read_config,json=readConfig,proto3" json:"read_config,omitempty"`
|
||||
RequestId string `protobuf:"bytes,5,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *ReadFromFileRequest) Reset() {
|
||||
*x = ReadFromFileRequest{}
|
||||
mi := &file_docreader_proto_msgTypes[1]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *ReadFromFileRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*ReadFromFileRequest) ProtoMessage() {}
|
||||
|
||||
func (x *ReadFromFileRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_docreader_proto_msgTypes[1]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use ReadFromFileRequest.ProtoReflect.Descriptor instead.
|
||||
func (*ReadFromFileRequest) Descriptor() ([]byte, []int) {
|
||||
return file_docreader_proto_rawDescGZIP(), []int{1}
|
||||
}
|
||||
|
||||
func (x *ReadFromFileRequest) GetFileContent() []byte {
|
||||
if x != nil {
|
||||
return x.FileContent
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ReadFromFileRequest) GetFileName() string {
|
||||
if x != nil {
|
||||
return x.FileName
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *ReadFromFileRequest) GetFileType() string {
|
||||
if x != nil {
|
||||
return x.FileType
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *ReadFromFileRequest) GetReadConfig() *ReadConfig {
|
||||
if x != nil {
|
||||
return x.ReadConfig
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ReadFromFileRequest) GetRequestId() string {
|
||||
if x != nil {
|
||||
return x.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// 从URL读取文档请求
|
||||
type ReadFromURLRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // 文档URL
|
||||
Title string `protobuf:"bytes,2,opt,name=title,proto3" json:"title,omitempty"` // 标题
|
||||
ReadConfig *ReadConfig `protobuf:"bytes,3,opt,name=read_config,json=readConfig,proto3" json:"read_config,omitempty"`
|
||||
RequestId string `protobuf:"bytes,4,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *ReadFromURLRequest) Reset() {
|
||||
*x = ReadFromURLRequest{}
|
||||
mi := &file_docreader_proto_msgTypes[2]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *ReadFromURLRequest) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*ReadFromURLRequest) ProtoMessage() {}
|
||||
|
||||
func (x *ReadFromURLRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_docreader_proto_msgTypes[2]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use ReadFromURLRequest.ProtoReflect.Descriptor instead.
|
||||
func (*ReadFromURLRequest) Descriptor() ([]byte, []int) {
|
||||
return file_docreader_proto_rawDescGZIP(), []int{2}
|
||||
}
|
||||
|
||||
func (x *ReadFromURLRequest) GetUrl() string {
|
||||
if x != nil {
|
||||
return x.Url
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *ReadFromURLRequest) GetTitle() string {
|
||||
if x != nil {
|
||||
return x.Title
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *ReadFromURLRequest) GetReadConfig() *ReadConfig {
|
||||
if x != nil {
|
||||
return x.ReadConfig
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ReadFromURLRequest) GetRequestId() string {
|
||||
if x != nil {
|
||||
return x.RequestId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// 图片信息
|
||||
type Image struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // 图片URL
|
||||
Caption string `protobuf:"bytes,2,opt,name=caption,proto3" json:"caption,omitempty"` // 图片描述
|
||||
OcrText string `protobuf:"bytes,3,opt,name=ocr_text,json=ocrText,proto3" json:"ocr_text,omitempty"` // OCR提取的文本内容
|
||||
OriginalUrl string `protobuf:"bytes,4,opt,name=original_url,json=originalUrl,proto3" json:"original_url,omitempty"` // 原始图片URL
|
||||
Start int32 `protobuf:"varint,5,opt,name=start,proto3" json:"start,omitempty"` // 图片在文本中的开始位置
|
||||
End int32 `protobuf:"varint,6,opt,name=end,proto3" json:"end,omitempty"` // 图片在文本中的结束位置
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *Image) Reset() {
|
||||
*x = Image{}
|
||||
mi := &file_docreader_proto_msgTypes[3]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *Image) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*Image) ProtoMessage() {}
|
||||
|
||||
func (x *Image) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_docreader_proto_msgTypes[3]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use Image.ProtoReflect.Descriptor instead.
|
||||
func (*Image) Descriptor() ([]byte, []int) {
|
||||
return file_docreader_proto_rawDescGZIP(), []int{3}
|
||||
}
|
||||
|
||||
func (x *Image) GetUrl() string {
|
||||
if x != nil {
|
||||
return x.Url
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *Image) GetCaption() string {
|
||||
if x != nil {
|
||||
return x.Caption
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *Image) GetOcrText() string {
|
||||
if x != nil {
|
||||
return x.OcrText
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *Image) GetOriginalUrl() string {
|
||||
if x != nil {
|
||||
return x.OriginalUrl
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *Image) GetStart() int32 {
|
||||
if x != nil {
|
||||
return x.Start
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *Image) GetEnd() int32 {
|
||||
if x != nil {
|
||||
return x.End
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type Chunk struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Content string `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` // 块内容
|
||||
Seq int32 `protobuf:"varint,2,opt,name=seq,proto3" json:"seq,omitempty"` // 块在文档中的次序
|
||||
Start int32 `protobuf:"varint,3,opt,name=start,proto3" json:"start,omitempty"` // 块在文档中的起始位置
|
||||
End int32 `protobuf:"varint,4,opt,name=end,proto3" json:"end,omitempty"` // 块在文档中的结束位置
|
||||
Images []*Image `protobuf:"bytes,5,rep,name=images,proto3" json:"images,omitempty"` // 块中包含的图片信息
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *Chunk) Reset() {
|
||||
*x = Chunk{}
|
||||
mi := &file_docreader_proto_msgTypes[4]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *Chunk) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*Chunk) ProtoMessage() {}
|
||||
|
||||
func (x *Chunk) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_docreader_proto_msgTypes[4]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use Chunk.ProtoReflect.Descriptor instead.
|
||||
func (*Chunk) Descriptor() ([]byte, []int) {
|
||||
return file_docreader_proto_rawDescGZIP(), []int{4}
|
||||
}
|
||||
|
||||
func (x *Chunk) GetContent() string {
|
||||
if x != nil {
|
||||
return x.Content
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *Chunk) GetSeq() int32 {
|
||||
if x != nil {
|
||||
return x.Seq
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *Chunk) GetStart() int32 {
|
||||
if x != nil {
|
||||
return x.Start
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *Chunk) GetEnd() int32 {
|
||||
if x != nil {
|
||||
return x.End
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *Chunk) GetImages() []*Image {
|
||||
if x != nil {
|
||||
return x.Images
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// 从URL读取文档响应
|
||||
type ReadResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Chunks []*Chunk `protobuf:"bytes,1,rep,name=chunks,proto3" json:"chunks,omitempty"` // 文档分块
|
||||
Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` // 错误信息
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *ReadResponse) Reset() {
|
||||
*x = ReadResponse{}
|
||||
mi := &file_docreader_proto_msgTypes[5]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *ReadResponse) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*ReadResponse) ProtoMessage() {}
|
||||
|
||||
func (x *ReadResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_docreader_proto_msgTypes[5]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use ReadResponse.ProtoReflect.Descriptor instead.
|
||||
func (*ReadResponse) Descriptor() ([]byte, []int) {
|
||||
return file_docreader_proto_rawDescGZIP(), []int{5}
|
||||
}
|
||||
|
||||
func (x *ReadResponse) GetChunks() []*Chunk {
|
||||
if x != nil {
|
||||
return x.Chunks
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ReadResponse) GetError() string {
|
||||
if x != nil {
|
||||
return x.Error
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var File_docreader_proto protoreflect.FileDescriptor
|
||||
|
||||
const file_docreader_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"\x0fdocreader.proto\x12\tdocreader\"\x9d\x01\n" +
|
||||
"\n" +
|
||||
"ReadConfig\x12\x1d\n" +
|
||||
"\n" +
|
||||
"chunk_size\x18\x01 \x01(\x05R\tchunkSize\x12#\n" +
|
||||
"\rchunk_overlap\x18\x02 \x01(\x05R\fchunkOverlap\x12\x1e\n" +
|
||||
"\n" +
|
||||
"separators\x18\x03 \x03(\tR\n" +
|
||||
"separators\x12+\n" +
|
||||
"\x11enable_multimodal\x18\x04 \x01(\bR\x10enableMultimodal\"\xc9\x01\n" +
|
||||
"\x13ReadFromFileRequest\x12!\n" +
|
||||
"\ffile_content\x18\x01 \x01(\fR\vfileContent\x12\x1b\n" +
|
||||
"\tfile_name\x18\x02 \x01(\tR\bfileName\x12\x1b\n" +
|
||||
"\tfile_type\x18\x03 \x01(\tR\bfileType\x126\n" +
|
||||
"\vread_config\x18\x04 \x01(\v2\x15.docreader.ReadConfigR\n" +
|
||||
"readConfig\x12\x1d\n" +
|
||||
"\n" +
|
||||
"request_id\x18\x05 \x01(\tR\trequestId\"\x93\x01\n" +
|
||||
"\x12ReadFromURLRequest\x12\x10\n" +
|
||||
"\x03url\x18\x01 \x01(\tR\x03url\x12\x14\n" +
|
||||
"\x05title\x18\x02 \x01(\tR\x05title\x126\n" +
|
||||
"\vread_config\x18\x03 \x01(\v2\x15.docreader.ReadConfigR\n" +
|
||||
"readConfig\x12\x1d\n" +
|
||||
"\n" +
|
||||
"request_id\x18\x04 \x01(\tR\trequestId\"\x99\x01\n" +
|
||||
"\x05Image\x12\x10\n" +
|
||||
"\x03url\x18\x01 \x01(\tR\x03url\x12\x18\n" +
|
||||
"\acaption\x18\x02 \x01(\tR\acaption\x12\x19\n" +
|
||||
"\bocr_text\x18\x03 \x01(\tR\aocrText\x12!\n" +
|
||||
"\foriginal_url\x18\x04 \x01(\tR\voriginalUrl\x12\x14\n" +
|
||||
"\x05start\x18\x05 \x01(\x05R\x05start\x12\x10\n" +
|
||||
"\x03end\x18\x06 \x01(\x05R\x03end\"\x85\x01\n" +
|
||||
"\x05Chunk\x12\x18\n" +
|
||||
"\acontent\x18\x01 \x01(\tR\acontent\x12\x10\n" +
|
||||
"\x03seq\x18\x02 \x01(\x05R\x03seq\x12\x14\n" +
|
||||
"\x05start\x18\x03 \x01(\x05R\x05start\x12\x10\n" +
|
||||
"\x03end\x18\x04 \x01(\x05R\x03end\x12(\n" +
|
||||
"\x06images\x18\x05 \x03(\v2\x10.docreader.ImageR\x06images\"N\n" +
|
||||
"\fReadResponse\x12(\n" +
|
||||
"\x06chunks\x18\x01 \x03(\v2\x10.docreader.ChunkR\x06chunks\x12\x14\n" +
|
||||
"\x05error\x18\x02 \x01(\tR\x05error2\x9f\x01\n" +
|
||||
"\tDocReader\x12I\n" +
|
||||
"\fReadFromFile\x12\x1e.docreader.ReadFromFileRequest\x1a\x17.docreader.ReadResponse\"\x00\x12G\n" +
|
||||
"\vReadFromURL\x12\x1d.docreader.ReadFromURLRequest\x1a\x17.docreader.ReadResponse\"\x00B5Z3github.com/Tencent/WeKnora/internal/docreader/protob\x06proto3"
|
||||
|
||||
var (
|
||||
file_docreader_proto_rawDescOnce sync.Once
|
||||
file_docreader_proto_rawDescData []byte
|
||||
)
|
||||
|
||||
func file_docreader_proto_rawDescGZIP() []byte {
|
||||
file_docreader_proto_rawDescOnce.Do(func() {
|
||||
file_docreader_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc)))
|
||||
})
|
||||
return file_docreader_proto_rawDescData
|
||||
}
|
||||
|
||||
var file_docreader_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
|
||||
var file_docreader_proto_goTypes = []any{
|
||||
(*ReadConfig)(nil), // 0: docreader.ReadConfig
|
||||
(*ReadFromFileRequest)(nil), // 1: docreader.ReadFromFileRequest
|
||||
(*ReadFromURLRequest)(nil), // 2: docreader.ReadFromURLRequest
|
||||
(*Image)(nil), // 3: docreader.Image
|
||||
(*Chunk)(nil), // 4: docreader.Chunk
|
||||
(*ReadResponse)(nil), // 5: docreader.ReadResponse
|
||||
}
|
||||
var file_docreader_proto_depIdxs = []int32{
|
||||
0, // 0: docreader.ReadFromFileRequest.read_config:type_name -> docreader.ReadConfig
|
||||
0, // 1: docreader.ReadFromURLRequest.read_config:type_name -> docreader.ReadConfig
|
||||
3, // 2: docreader.Chunk.images:type_name -> docreader.Image
|
||||
4, // 3: docreader.ReadResponse.chunks:type_name -> docreader.Chunk
|
||||
1, // 4: docreader.DocReader.ReadFromFile:input_type -> docreader.ReadFromFileRequest
|
||||
2, // 5: docreader.DocReader.ReadFromURL:input_type -> docreader.ReadFromURLRequest
|
||||
5, // 6: docreader.DocReader.ReadFromFile:output_type -> docreader.ReadResponse
|
||||
5, // 7: docreader.DocReader.ReadFromURL:output_type -> docreader.ReadResponse
|
||||
6, // [6:8] is the sub-list for method output_type
|
||||
4, // [4:6] is the sub-list for method input_type
|
||||
4, // [4:4] is the sub-list for extension type_name
|
||||
4, // [4:4] is the sub-list for extension extendee
|
||||
0, // [0:4] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_docreader_proto_init() }
|
||||
func file_docreader_proto_init() {
|
||||
if File_docreader_proto != nil {
|
||||
return
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
File: protoimpl.DescBuilder{
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_docreader_proto_rawDesc), len(file_docreader_proto_rawDesc)),
|
||||
NumEnums: 0,
|
||||
NumMessages: 6,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
GoTypes: file_docreader_proto_goTypes,
|
||||
DependencyIndexes: file_docreader_proto_depIdxs,
|
||||
MessageInfos: file_docreader_proto_msgTypes,
|
||||
}.Build()
|
||||
File_docreader_proto = out.File
|
||||
file_docreader_proto_goTypes = nil
|
||||
file_docreader_proto_depIdxs = nil
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package docreader;
|
||||
|
||||
option go_package = "github.com/Tencent/WeKnora/internal/docreader/proto";
|
||||
|
||||
// 文档读取服务
|
||||
service DocReader {
|
||||
// 从文件读取文档
|
||||
rpc ReadFromFile(ReadFromFileRequest) returns (ReadResponse) {}
|
||||
// 从URL读取文档
|
||||
rpc ReadFromURL(ReadFromURLRequest) returns (ReadResponse) {}
|
||||
}
|
||||
|
||||
message ReadConfig {
|
||||
int32 chunk_size = 1; // 分块大小
|
||||
int32 chunk_overlap = 2; // 分块重叠
|
||||
repeated string separators = 3; // 分隔符
|
||||
bool enable_multimodal = 4; // 多模态处理
|
||||
}
|
||||
|
||||
// 从文件读取文档请求
|
||||
message ReadFromFileRequest {
|
||||
bytes file_content = 1; // 文件内容
|
||||
string file_name = 2; // 文件名
|
||||
string file_type = 3; // 文件类型
|
||||
ReadConfig read_config = 4;
|
||||
string request_id = 5;
|
||||
}
|
||||
|
||||
// 从URL读取文档请求
|
||||
message ReadFromURLRequest {
|
||||
string url = 1; // 文档URL
|
||||
string title = 2; // 标题
|
||||
ReadConfig read_config = 3;
|
||||
string request_id = 4;
|
||||
}
|
||||
|
||||
// 图片信息
|
||||
message Image {
|
||||
string url = 1; // 图片URL
|
||||
string caption = 2; // 图片描述
|
||||
string ocr_text = 3; // OCR提取的文本内容
|
||||
string original_url = 4; // 原始图片URL
|
||||
int32 start = 5; // 图片在文本中的开始位置
|
||||
int32 end = 6; // 图片在文本中的结束位置
|
||||
}
|
||||
|
||||
message Chunk {
|
||||
string content = 1; // 块内容
|
||||
int32 seq = 2; // 块在文档中的次序
|
||||
int32 start = 3; // 块在文档中的起始位置
|
||||
int32 end = 4; // 块在文档中的结束位置
|
||||
repeated Image images = 5; // 块中包含的图片信息
|
||||
}
|
||||
|
||||
// 从URL读取文档响应
|
||||
message ReadResponse {
|
||||
repeated Chunk chunks = 1; // 文档分块
|
||||
string error = 2; // 错误信息
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
|
||||
// versions:
|
||||
// - protoc-gen-go-grpc v1.5.1
|
||||
// - protoc v5.29.3
|
||||
// source: docreader.proto
|
||||
|
||||
package proto
|
||||
|
||||
import (
|
||||
context "context"
|
||||
grpc "google.golang.org/grpc"
|
||||
codes "google.golang.org/grpc/codes"
|
||||
status "google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// This is a compile-time assertion to ensure that this generated file
|
||||
// is compatible with the grpc package it is being compiled against.
|
||||
// Requires gRPC-Go v1.64.0 or later.
|
||||
const _ = grpc.SupportPackageIsVersion9
|
||||
|
||||
const (
|
||||
DocReader_ReadFromFile_FullMethodName = "/docreader.DocReader/ReadFromFile"
|
||||
DocReader_ReadFromURL_FullMethodName = "/docreader.DocReader/ReadFromURL"
|
||||
)
|
||||
|
||||
// DocReaderClient is the client API for DocReader service.
|
||||
//
|
||||
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
|
||||
//
|
||||
// 文档读取服务
|
||||
type DocReaderClient interface {
|
||||
// 从文件读取文档
|
||||
ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error)
|
||||
// 从URL读取文档
|
||||
ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error)
|
||||
}
|
||||
|
||||
type docReaderClient struct {
|
||||
cc grpc.ClientConnInterface
|
||||
}
|
||||
|
||||
func NewDocReaderClient(cc grpc.ClientConnInterface) DocReaderClient {
|
||||
return &docReaderClient{cc}
|
||||
}
|
||||
|
||||
func (c *docReaderClient) ReadFromFile(ctx context.Context, in *ReadFromFileRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ReadResponse)
|
||||
err := c.cc.Invoke(ctx, DocReader_ReadFromFile_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *docReaderClient) ReadFromURL(ctx context.Context, in *ReadFromURLRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ReadResponse)
|
||||
err := c.cc.Invoke(ctx, DocReader_ReadFromURL_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DocReaderServer is the server API for DocReader service.
|
||||
// All implementations must embed UnimplementedDocReaderServer
|
||||
// for forward compatibility.
|
||||
//
|
||||
// 文档读取服务
|
||||
type DocReaderServer interface {
|
||||
// 从文件读取文档
|
||||
ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error)
|
||||
// 从URL读取文档
|
||||
ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error)
|
||||
mustEmbedUnimplementedDocReaderServer()
|
||||
}
|
||||
|
||||
// UnimplementedDocReaderServer must be embedded to have
|
||||
// forward compatible implementations.
|
||||
//
|
||||
// NOTE: this should be embedded by value instead of pointer to avoid a nil
|
||||
// pointer dereference when methods are called.
|
||||
type UnimplementedDocReaderServer struct{}
|
||||
|
||||
func (UnimplementedDocReaderServer) ReadFromFile(context.Context, *ReadFromFileRequest) (*ReadResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ReadFromFile not implemented")
|
||||
}
|
||||
func (UnimplementedDocReaderServer) ReadFromURL(context.Context, *ReadFromURLRequest) (*ReadResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ReadFromURL not implemented")
|
||||
}
|
||||
func (UnimplementedDocReaderServer) mustEmbedUnimplementedDocReaderServer() {}
|
||||
func (UnimplementedDocReaderServer) testEmbeddedByValue() {}
|
||||
|
||||
// UnsafeDocReaderServer may be embedded to opt out of forward compatibility for this service.
|
||||
// Use of this interface is not recommended, as added methods to DocReaderServer will
|
||||
// result in compilation errors.
|
||||
type UnsafeDocReaderServer interface {
|
||||
mustEmbedUnimplementedDocReaderServer()
|
||||
}
|
||||
|
||||
func RegisterDocReaderServer(s grpc.ServiceRegistrar, srv DocReaderServer) {
|
||||
// If the following call pancis, it indicates UnimplementedDocReaderServer was
|
||||
// embedded by pointer and is nil. This will cause panics if an
|
||||
// unimplemented method is ever invoked, so we test this at initialization
|
||||
// time to prevent it from happening at runtime later due to I/O.
|
||||
if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
|
||||
t.testEmbeddedByValue()
|
||||
}
|
||||
s.RegisterService(&DocReader_ServiceDesc, srv)
|
||||
}
|
||||
|
||||
func _DocReader_ReadFromFile_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ReadFromFileRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(DocReaderServer).ReadFromFile(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: DocReader_ReadFromFile_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(DocReaderServer).ReadFromFile(ctx, req.(*ReadFromFileRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _DocReader_ReadFromURL_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ReadFromURLRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(DocReaderServer).ReadFromURL(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: DocReader_ReadFromURL_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(DocReaderServer).ReadFromURL(ctx, req.(*ReadFromURLRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
// DocReader_ServiceDesc is the grpc.ServiceDesc for DocReader service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
var DocReader_ServiceDesc = grpc.ServiceDesc{
|
||||
ServiceName: "docreader.DocReader",
|
||||
HandlerType: (*DocReaderServer)(nil),
|
||||
Methods: []grpc.MethodDesc{
|
||||
{
|
||||
MethodName: "ReadFromFile",
|
||||
Handler: _DocReader_ReadFromFile_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ReadFromURL",
|
||||
Handler: _DocReader_ReadFromURL_Handler,
|
||||
},
|
||||
},
|
||||
Streams: []grpc.StreamDesc{},
|
||||
Metadata: "docreader.proto",
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
|
||||
"""Client and server classes corresponding to protobuf-defined services."""
|
||||
import grpc
|
||||
|
||||
from . import docreader_pb2 as docreader__pb2
|
||||
|
||||
|
||||
class DocReaderStub(object):
|
||||
"""文档读取服务
|
||||
"""
|
||||
|
||||
def __init__(self, channel):
|
||||
"""Constructor.
|
||||
|
||||
Args:
|
||||
channel: A grpc.Channel.
|
||||
"""
|
||||
self.ReadFromFile = channel.unary_unary(
|
||||
'/docreader.DocReader/ReadFromFile',
|
||||
request_serializer=docreader__pb2.ReadFromFileRequest.SerializeToString,
|
||||
response_deserializer=docreader__pb2.ReadResponse.FromString,
|
||||
)
|
||||
self.ReadFromURL = channel.unary_unary(
|
||||
'/docreader.DocReader/ReadFromURL',
|
||||
request_serializer=docreader__pb2.ReadFromURLRequest.SerializeToString,
|
||||
response_deserializer=docreader__pb2.ReadResponse.FromString,
|
||||
)
|
||||
|
||||
|
||||
class DocReaderServicer(object):
|
||||
"""文档读取服务
|
||||
"""
|
||||
|
||||
def ReadFromFile(self, request, context):
|
||||
"""从文件读取文档
|
||||
"""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def ReadFromURL(self, request, context):
|
||||
"""从URL读取文档
|
||||
"""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
|
||||
def add_DocReaderServicer_to_server(servicer, server):
|
||||
rpc_method_handlers = {
|
||||
'ReadFromFile': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ReadFromFile,
|
||||
request_deserializer=docreader__pb2.ReadFromFileRequest.FromString,
|
||||
response_serializer=docreader__pb2.ReadResponse.SerializeToString,
|
||||
),
|
||||
'ReadFromURL': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.ReadFromURL,
|
||||
request_deserializer=docreader__pb2.ReadFromURLRequest.FromString,
|
||||
response_serializer=docreader__pb2.ReadResponse.SerializeToString,
|
||||
),
|
||||
}
|
||||
generic_handler = grpc.method_handlers_generic_handler(
|
||||
'docreader.DocReader', rpc_method_handlers)
|
||||
server.add_generic_rpc_handlers((generic_handler,))
|
||||
|
||||
|
||||
# This class is part of an EXPERIMENTAL API.
|
||||
class DocReader(object):
|
||||
"""文档读取服务
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def ReadFromFile(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(request, target, '/docreader.DocReader/ReadFromFile',
|
||||
docreader__pb2.ReadFromFileRequest.SerializeToString,
|
||||
docreader__pb2.ReadResponse.FromString,
|
||||
options, channel_credentials,
|
||||
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||
|
||||
@staticmethod
|
||||
def ReadFromURL(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(request, target, '/docreader.DocReader/ReadFromURL',
|
||||
docreader__pb2.ReadFromURLRequest.SerializeToString,
|
||||
docreader__pb2.ReadResponse.FromString,
|
||||
options, channel_credentials,
|
||||
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
||||
@@ -0,0 +1,312 @@
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from concurrent import futures
|
||||
import traceback
|
||||
import grpc
|
||||
import uuid
|
||||
|
||||
# Add parent directory to Python path
|
||||
current_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
parent_dir = os.path.dirname(current_dir)
|
||||
if parent_dir not in sys.path:
|
||||
sys.path.insert(0, parent_dir)
|
||||
|
||||
from proto.docreader_pb2 import ReadResponse, Chunk, Image
|
||||
from proto import docreader_pb2_grpc
|
||||
from parser import Parser, ChunkingConfig, OCREngine
|
||||
from utils.request import request_id_context, init_logging_request_id
|
||||
|
||||
# Ensure no existing handlers
|
||||
for handler in logging.root.handlers[:]:
|
||||
logging.root.removeHandler(handler)
|
||||
|
||||
# Configure logging - use stdout
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
logging.root.addHandler(handler)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.info("Initializing server logging")
|
||||
|
||||
# Initialize request ID logging
|
||||
init_logging_request_id()
|
||||
|
||||
# Set max message size to 50MB
|
||||
MAX_MESSAGE_LENGTH = 50 * 1024 * 1024
|
||||
|
||||
|
||||
parser = Parser()
|
||||
|
||||
class DocReaderServicer(docreader_pb2_grpc.DocReaderServicer):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.parser = Parser()
|
||||
|
||||
def ReadFromFile(self, request, context):
|
||||
# Get or generate request ID
|
||||
request_id = (
|
||||
request.request_id
|
||||
if hasattr(request, "request_id") and request.request_id
|
||||
else str(uuid.uuid4())
|
||||
)
|
||||
|
||||
# Use request ID context
|
||||
with request_id_context(request_id):
|
||||
try:
|
||||
# Get file type
|
||||
file_type = (
|
||||
request.file_type or os.path.splitext(request.file_name)[1][1:]
|
||||
)
|
||||
logger.info(
|
||||
f"Received ReadFromFile request for file: {request.file_name}, type: {file_type}"
|
||||
)
|
||||
logger.info(f"File content size: {len(request.file_content)} bytes")
|
||||
|
||||
# Create chunking config
|
||||
chunk_size = request.read_config.chunk_size or 512
|
||||
chunk_overlap = request.read_config.chunk_overlap or 50
|
||||
separators = request.read_config.separators or ["\n\n", "\n", "。"]
|
||||
enable_multimodal = request.read_config.enable_multimodal or False
|
||||
|
||||
logger.info(
|
||||
f"Using chunking config: size={chunk_size}, overlap={chunk_overlap}, "
|
||||
f"multimodal={enable_multimodal}"
|
||||
)
|
||||
|
||||
chunking_config = ChunkingConfig(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
separators=separators,
|
||||
enable_multimodal=enable_multimodal,
|
||||
)
|
||||
|
||||
# Parse file
|
||||
logger.info(f"Starting file parsing process")
|
||||
result = self.parser.parse_file(
|
||||
request.file_name, file_type, request.file_content, chunking_config
|
||||
)
|
||||
|
||||
if not result:
|
||||
error_msg = "Failed to parse file"
|
||||
logger.error(error_msg)
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(error_msg)
|
||||
return ReadResponse()
|
||||
|
||||
# Convert to protobuf message
|
||||
logger.info(
|
||||
f"Successfully parsed file {request.file_name}, returning {len(result.chunks)} chunks"
|
||||
)
|
||||
|
||||
# Build response, including image info
|
||||
response = ReadResponse(
|
||||
chunks=[self._convert_chunk_to_proto(chunk) for chunk in result.chunks]
|
||||
)
|
||||
logger.info(f"Response size: {response.ByteSize()} bytes")
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error reading file: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(str(e))
|
||||
return ReadResponse(error=str(e))
|
||||
|
||||
def ReadFromURL(self, request, context):
|
||||
# Get or generate request ID
|
||||
request_id = (
|
||||
request.request_id
|
||||
if hasattr(request, "request_id") and request.request_id
|
||||
else str(uuid.uuid4())
|
||||
)
|
||||
|
||||
# Use request ID context
|
||||
with request_id_context(request_id):
|
||||
try:
|
||||
logger.info(f"Received ReadFromURL request for URL: {request.url}")
|
||||
|
||||
# Create chunking config
|
||||
chunk_size = request.read_config.chunk_size or 512
|
||||
chunk_overlap = request.read_config.chunk_overlap or 50
|
||||
separators = request.read_config.separators or ["\n\n", "\n", "。"]
|
||||
enable_multimodal = request.read_config.enable_multimodal or False
|
||||
|
||||
logger.info(
|
||||
f"Using chunking config: size={chunk_size}, overlap={chunk_overlap}, "
|
||||
f"multimodal={enable_multimodal}"
|
||||
)
|
||||
|
||||
chunking_config = ChunkingConfig(
|
||||
chunk_size=chunk_size,
|
||||
chunk_overlap=chunk_overlap,
|
||||
separators=separators,
|
||||
enable_multimodal=enable_multimodal,
|
||||
)
|
||||
|
||||
# Parse URL
|
||||
logger.info(f"Starting URL parsing process")
|
||||
result = self.parser.parse_url(request.url, request.title, chunking_config)
|
||||
if not result:
|
||||
error_msg = "Failed to parse URL"
|
||||
logger.error(error_msg)
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(error_msg)
|
||||
return ReadResponse(error=error_msg)
|
||||
|
||||
# Convert to protobuf message, including image info
|
||||
logger.info(
|
||||
f"Successfully parsed URL {request.url}, returning {len(result.chunks)} chunks"
|
||||
)
|
||||
|
||||
response = ReadResponse(
|
||||
chunks=[self._convert_chunk_to_proto(chunk) for chunk in result.chunks]
|
||||
)
|
||||
logger.info(f"Response size: {response.ByteSize()} bytes")
|
||||
return response
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Error reading URL: {str(e)}"
|
||||
logger.error(error_msg)
|
||||
logger.info(f"Detailed traceback: {traceback.format_exc()}")
|
||||
context.set_code(grpc.StatusCode.INTERNAL)
|
||||
context.set_details(str(e))
|
||||
return ReadResponse(error=str(e))
|
||||
|
||||
def _convert_chunk_to_proto(self, chunk):
|
||||
"""Convert internal Chunk object to protobuf Chunk message"""
|
||||
proto_chunk = Chunk(
|
||||
content=chunk.content,
|
||||
seq=chunk.seq,
|
||||
start=chunk.start,
|
||||
end=chunk.end,
|
||||
)
|
||||
|
||||
# If chunk has images attribute and is not empty, add image info
|
||||
if hasattr(chunk, "images") and chunk.images:
|
||||
logger.info(f"Adding {len(chunk.images)} images to chunk {chunk.seq}")
|
||||
for img_info in chunk.images:
|
||||
proto_image = Image(
|
||||
url=img_info.get("cos_url", ""),
|
||||
caption=img_info.get("caption", ""),
|
||||
ocr_text=img_info.get("ocr_text", ""),
|
||||
original_url=img_info.get("original_url", ""),
|
||||
start=img_info.get("start", 0),
|
||||
end=img_info.get("end", 0)
|
||||
)
|
||||
proto_chunk.images.append(proto_image)
|
||||
|
||||
return proto_chunk
|
||||
|
||||
def init_ocr_engine(ocr_backend, ocr_config):
|
||||
"""Initialize OCR engine"""
|
||||
try:
|
||||
logger.info(f"Initializing OCR engine with backend: {ocr_backend}")
|
||||
ocr_engine = OCREngine.get_instance(backend_type=ocr_backend, **ocr_config)
|
||||
if ocr_engine:
|
||||
logger.info("OCR engine initialized successfully")
|
||||
return True
|
||||
else:
|
||||
logger.error("OCR engine initialization failed")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"Error initializing OCR engine: {str(e)}")
|
||||
return False
|
||||
|
||||
def serve():
|
||||
init_ocr_engine(os.getenv("OCR_BACKEND", "paddle"), {
|
||||
"OCR_API_BASE_URL": os.getenv("OCR_API_BASE_URL", ""),
|
||||
})
|
||||
# Set max number of worker threads and processes
|
||||
max_workers = int(os.environ.get("GRPC_MAX_WORKERS", "4"))
|
||||
worker_processes = int(os.environ.get("GRPC_WORKER_PROCESSES", str(os.cpu_count() or 1)))
|
||||
logger.info(f"Starting DocReader service, max worker threads per process: {max_workers}, "
|
||||
f"processes: {worker_processes}")
|
||||
|
||||
# Get port number
|
||||
port = os.environ.get("GRPC_PORT", "50051")
|
||||
|
||||
# Multi-process mode
|
||||
if worker_processes > 1:
|
||||
import multiprocessing
|
||||
processes = []
|
||||
|
||||
def run_server():
|
||||
# Create server
|
||||
server = grpc.server(
|
||||
futures.ThreadPoolExecutor(max_workers=max_workers),
|
||||
options=[
|
||||
('grpc.max_send_message_length', MAX_MESSAGE_LENGTH),
|
||||
('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH),
|
||||
],
|
||||
)
|
||||
|
||||
# Register service
|
||||
docreader_pb2_grpc.add_DocReaderServicer_to_server(DocReaderServicer(), server)
|
||||
|
||||
# Set listen address
|
||||
server.add_insecure_port(f"[::]:{port}")
|
||||
|
||||
# Start service
|
||||
server.start()
|
||||
|
||||
logger.info(f"Worker process {os.getpid()} started on port {port}")
|
||||
|
||||
try:
|
||||
# Wait for service termination
|
||||
server.wait_for_termination()
|
||||
except KeyboardInterrupt:
|
||||
logger.info(f"Worker process {os.getpid()} received termination signal")
|
||||
server.stop(0)
|
||||
|
||||
# Start specified number of worker processes
|
||||
for i in range(worker_processes):
|
||||
process = multiprocessing.Process(target=run_server)
|
||||
processes.append(process)
|
||||
process.start()
|
||||
logger.info(f"Started worker process {process.pid} ({i+1}/{worker_processes})")
|
||||
|
||||
# Wait for all processes to complete
|
||||
try:
|
||||
for process in processes:
|
||||
process.join()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Master process received termination signal")
|
||||
for process in processes:
|
||||
if process.is_alive():
|
||||
logger.info(f"Terminating worker process {process.pid}")
|
||||
process.terminate()
|
||||
|
||||
# Single-process mode
|
||||
else:
|
||||
# Create server
|
||||
server = grpc.server(
|
||||
futures.ThreadPoolExecutor(max_workers=max_workers),
|
||||
options=[
|
||||
('grpc.max_send_message_length', MAX_MESSAGE_LENGTH),
|
||||
('grpc.max_receive_message_length', MAX_MESSAGE_LENGTH),
|
||||
],
|
||||
)
|
||||
|
||||
# Register service
|
||||
docreader_pb2_grpc.add_DocReaderServicer_to_server(DocReaderServicer(), server)
|
||||
|
||||
# Set listen address
|
||||
server.add_insecure_port(f"[::]:{port}")
|
||||
|
||||
# Start service
|
||||
server.start()
|
||||
|
||||
logger.info(f"Server started on port {port} (single process mode)")
|
||||
logger.info("Server is ready to accept connections")
|
||||
|
||||
try:
|
||||
# Wait for service termination
|
||||
server.wait_for_termination()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Received termination signal, shutting down server")
|
||||
server.stop(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
serve()
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.8 KiB |
+56
@@ -0,0 +1,56 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>测试 HTML 文档</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>测试 HTML 文档</h1>
|
||||
|
||||
<p>这是一个测试 HTML 文档,用于测试 HTML 解析功能。</p>
|
||||
|
||||
<h2>包含图片</h2>
|
||||
<img src="https://example.com/image.jpg" alt="测试图片">
|
||||
|
||||
<h2>包含链接</h2>
|
||||
<p>这是一个<a href="https://example.com">测试链接</a>。</p>
|
||||
|
||||
<h2>包含代码块</h2>
|
||||
<pre><code>
|
||||
def hello_world():
|
||||
print("Hello, World!")
|
||||
</code></pre>
|
||||
|
||||
<h2>包含表格</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>表头1</th>
|
||||
<th>表头2</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>内容1</td>
|
||||
<td>内容2</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>内容3</td>
|
||||
<td>内容4</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<h2>测试分块功能</h2>
|
||||
<p>这部分内容用于测试分块功能,确保 HTML 结构在分块时保持完整。</p>
|
||||
<ul>
|
||||
<li>第一块内容</li>
|
||||
<li>第二块内容</li>
|
||||
<li>第三块内容</li>
|
||||
</ul>
|
||||
|
||||
<h2>测试重叠功能</h2>
|
||||
<p>这部分内容可能会在分块时与前后块重叠,以确保上下文的连续性。</p>
|
||||
</body>
|
||||
</html>
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
# 测试 Markdown 文档
|
||||
|
||||
这是一个测试 Markdown 文档,用于测试 Markdown 解析功能。
|
||||
|
||||
## 包含图片
|
||||
|
||||

|
||||
|
||||
## 包含链接
|
||||
|
||||
这是一个[测试链接](https://example.com)。
|
||||
|
||||
## 包含代码块
|
||||
|
||||
```python
|
||||
def hello_world():
|
||||
print("Hello, World!")
|
||||
```
|
||||
|
||||
## 包含表格
|
||||
|
||||
| 表头1 | 表头2 |
|
||||
|-------|-------|
|
||||
| 内容1 | 内容2 |
|
||||
| 内容3 | 内容4 |
|
||||
|
||||
## 测试分块功能
|
||||
|
||||
这部分内容用于测试分块功能,确保 Markdown 结构在分块时保持完整。
|
||||
|
||||
- 第一块内容
|
||||
- 第二块内容
|
||||
- 第三块内容
|
||||
|
||||
## 测试重叠功能
|
||||
|
||||
这部分内容可能会在分块时与前后块重叠,以确保上下文的连续性。
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
这是一个测试文档
|
||||
包含多行内容
|
||||
用于测试文档解析功能
|
||||
|
||||
这个文档包含以下内容:
|
||||
1. 基本文本内容
|
||||
2. 多行段落
|
||||
3. 列表项
|
||||
|
||||
测试分块功能:
|
||||
- 第一块内容
|
||||
- 第二块内容
|
||||
- 第三块内容
|
||||
|
||||
测试重叠功能:
|
||||
这部分内容可能会在分块时与前后块重叠,以确保上下文的连续性。
|
||||
@@ -0,0 +1,19 @@
|
||||
这是一个测试文档
|
||||
包含多行内容
|
||||
用于测试文档解析功能
|
||||
|
||||
这个文档包含以下内容:
|
||||
1. 基本文本内容
|
||||
2. 多行段落
|
||||
3. 列表项
|
||||
|
||||
测试分块功能:
|
||||
- 第一块内容
|
||||
- 第二块内容
|
||||
- 第三块内容
|
||||
|
||||
测试重叠功能:
|
||||
这部分内容可能会在分块时与前后块重叠,以确保上下文的连续性。
|
||||
|
||||
|
||||
test
|
||||
@@ -0,0 +1,83 @@
|
||||
#
|
||||
# Copyright 2024 The InfiniFlow Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
|
||||
# 配置日志
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def singleton(cls, *args, **kw):
|
||||
instances = {}
|
||||
|
||||
def _singleton():
|
||||
key = str(cls) + str(os.getpid())
|
||||
if key not in instances:
|
||||
logger.info(f"Creating new singleton instance with key: {key}")
|
||||
instances[key] = cls(*args, **kw)
|
||||
else:
|
||||
logger.info(f"Returning existing singleton instance with key: {key}")
|
||||
return instances[key]
|
||||
|
||||
return _singleton
|
||||
|
||||
|
||||
def rmSpace(txt):
|
||||
logger.info(f"Removing spaces from text of length: {len(txt)}")
|
||||
txt = re.sub(r"([^a-z0-9.,\)>]) +([^ ])", r"\1\2", txt, flags=re.IGNORECASE)
|
||||
return re.sub(r"([^ ]) +([^a-z0-9.,\(<])", r"\1\2", txt, flags=re.IGNORECASE)
|
||||
|
||||
|
||||
def findMaxDt(fnm):
|
||||
m = "1970-01-01 00:00:00"
|
||||
logger.info(f"Finding maximum date in file: {fnm}")
|
||||
try:
|
||||
with open(fnm, "r") as f:
|
||||
while True:
|
||||
l = f.readline()
|
||||
if not l:
|
||||
break
|
||||
l = l.strip("\n")
|
||||
if l == "nan":
|
||||
continue
|
||||
if l > m:
|
||||
m = l
|
||||
logger.info(f"Maximum date found: {m}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading file {fnm} for max date: {str(e)}")
|
||||
return m
|
||||
|
||||
|
||||
def findMaxTm(fnm):
|
||||
m = 0
|
||||
logger.info(f"Finding maximum time in file: {fnm}")
|
||||
try:
|
||||
with open(fnm, "r") as f:
|
||||
while True:
|
||||
l = f.readline()
|
||||
if not l:
|
||||
break
|
||||
l = l.strip("\n")
|
||||
if l == "nan":
|
||||
continue
|
||||
if int(l) > m:
|
||||
m = int(l)
|
||||
logger.info(f"Maximum time found: {m}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error reading file {fnm} for max time: {str(e)}")
|
||||
return m
|
||||
@@ -0,0 +1,149 @@
|
||||
from contextvars import ContextVar
|
||||
import logging
|
||||
import uuid
|
||||
import contextlib
|
||||
import time
|
||||
from typing import Optional
|
||||
from logging import LogRecord
|
||||
|
||||
# 配置日志
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 定义上下文变量
|
||||
request_id_var = ContextVar("request_id", default=None)
|
||||
_request_start_time_ctx = ContextVar("request_start_time", default=None)
|
||||
|
||||
|
||||
def set_request_id(request_id: str) -> None:
|
||||
"""设置当前上下文的请求ID"""
|
||||
request_id_var.set(request_id)
|
||||
|
||||
|
||||
def get_request_id() -> Optional[str]:
|
||||
"""获取当前上下文的请求ID"""
|
||||
return request_id_var.get()
|
||||
|
||||
|
||||
class MillisecondFormatter(logging.Formatter):
|
||||
"""自定义日志格式化器,只显示毫秒级时间戳(3位数字)而不是微秒(6位)"""
|
||||
|
||||
def formatTime(self, record, datefmt=None):
|
||||
"""重写formatTime方法,将微秒格式化为毫秒"""
|
||||
# 先获取标准的格式化时间
|
||||
result = super().formatTime(record, datefmt)
|
||||
|
||||
# 如果使用了包含.%f的格式,则将微秒(6位)截断为毫秒(3位)
|
||||
if datefmt and ".%f" in datefmt:
|
||||
# 格式化的时间字符串应该在最后有6位微秒数
|
||||
parts = result.split('.')
|
||||
if len(parts) > 1 and len(parts[1]) >= 6:
|
||||
# 只保留前3位作为毫秒
|
||||
millis = parts[1][:3]
|
||||
result = f"{parts[0]}.{millis}"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def init_logging_request_id():
|
||||
"""
|
||||
Initialize logging to include request ID in log messages.
|
||||
Add the custom filter to all existing handlers
|
||||
"""
|
||||
logger.info("Initializing request ID logging")
|
||||
root_logger = logging.getLogger()
|
||||
|
||||
# 添加自定义过滤器到所有处理器
|
||||
for handler in root_logger.handlers:
|
||||
# 添加请求ID过滤器
|
||||
handler.addFilter(RequestIdFilter())
|
||||
|
||||
# 更新格式化器以包含请求ID,调整格式使其更紧凑整齐
|
||||
formatter = logging.Formatter(
|
||||
fmt="%(asctime)s.%(msecs)03d [%(request_id)s] %(levelname)-5s %(name)-20s | %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
|
||||
logger.info(
|
||||
f"Updated {len(root_logger.handlers)} handlers with request ID formatting"
|
||||
)
|
||||
|
||||
# 如果没有处理器,添加一个标准输出处理器
|
||||
if not root_logger.handlers:
|
||||
handler = logging.StreamHandler()
|
||||
formatter = logging.Formatter(
|
||||
fmt="%(asctime)s.%(msecs)03d [%(request_id)s] %(levelname)-5s %(name)-20s | %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
handler.addFilter(RequestIdFilter())
|
||||
root_logger.addHandler(handler)
|
||||
logger.info("Added new StreamHandler with request ID formatting")
|
||||
|
||||
|
||||
class RequestIdFilter(logging.Filter):
|
||||
"""Filter that adds request ID to log messages"""
|
||||
|
||||
def filter(self, record: LogRecord) -> bool:
|
||||
request_id = request_id_var.get()
|
||||
if request_id is not None:
|
||||
# 为日志记录添加请求ID属性,使用短格式
|
||||
if len(request_id) > 8:
|
||||
# 截取ID的前8个字符,确保显示整齐
|
||||
short_id = request_id[:8]
|
||||
if "-" in request_id:
|
||||
# 尝试保留格式,例如 test-req-1-XXX
|
||||
parts = request_id.split("-")
|
||||
if len(parts) >= 3:
|
||||
# 如果格式是 xxx-xxx-n-randompart
|
||||
short_id = f"{parts[0]}-{parts[1]}-{parts[2]}"
|
||||
record.request_id = short_id
|
||||
else:
|
||||
record.request_id = request_id
|
||||
|
||||
# 添加执行时间属性
|
||||
start_time = _request_start_time_ctx.get()
|
||||
if start_time is not None:
|
||||
elapsed_ms = int((time.time() - start_time) * 1000)
|
||||
record.elapsed_ms = elapsed_ms
|
||||
# 添加执行时间到消息中
|
||||
if not hasattr(record, "message_with_elapsed"):
|
||||
record.message_with_elapsed = True
|
||||
record.msg = f"{record.msg} (elapsed: {elapsed_ms}ms)"
|
||||
else:
|
||||
# 如果没有请求ID,使用占位符
|
||||
record.request_id = "no-req-id"
|
||||
|
||||
return True
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def request_id_context(request_id: str = None):
|
||||
"""Context manager that sets a request ID for the current context
|
||||
|
||||
Args:
|
||||
request_id: 要使用的请求ID,如果为None则自动生成
|
||||
|
||||
Example:
|
||||
with request_id_context("req-123"):
|
||||
# 在这个代码块中的所有日志都会包含请求ID req-123
|
||||
logging.info("Processing request")
|
||||
"""
|
||||
# Generate or use provided request ID
|
||||
req_id = request_id or str(uuid.uuid4())
|
||||
|
||||
# Set start time and request ID
|
||||
start_time = time.time()
|
||||
req_token = request_id_var.set(req_id)
|
||||
time_token = _request_start_time_ctx.set(start_time)
|
||||
|
||||
logger.info(f"Starting new request with ID: {req_id}")
|
||||
|
||||
try:
|
||||
yield request_id_var.get()
|
||||
finally:
|
||||
# Log completion and reset context vars
|
||||
elapsed_ms = int((time.time() - start_time) * 1000)
|
||||
logger.info(f"Request {req_id} completed in {elapsed_ms}ms")
|
||||
request_id_var.reset(req_token)
|
||||
_request_start_time_ctx.reset(time_token)
|
||||
Reference in New Issue
Block a user