feat: introduce WeKnora Lite edition with lightweight configuration and deployment
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template. - Updated `.env.example` to remove deprecated variables and include new Docreader settings. - Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service. - Introduced a Makefile target for building and running the Lite version, along with packaging capabilities. - Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support. - Implemented a new service file for managing the Lite version as a system service. This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
This commit is contained in:
@@ -0,0 +1,181 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// simpleFormats lists file extensions that Go can handle without the Python service.
|
||||
var simpleFormats = map[string]bool{
|
||||
"md": true, "markdown": true,
|
||||
"txt": true, "text": true,
|
||||
"csv": true,
|
||||
}
|
||||
|
||||
var imageFormats = map[string]bool{
|
||||
"jpg": true, "jpeg": true, "png": true, "gif": true,
|
||||
"bmp": true, "tiff": true, "webp": true,
|
||||
}
|
||||
|
||||
func init() {
|
||||
for k := range imageFormats {
|
||||
simpleFormats[k] = true
|
||||
}
|
||||
}
|
||||
|
||||
// IsSimpleFormat returns true if the file type can be handled by the Go SimpleFormatReader.
|
||||
func IsSimpleFormat(fileType string) bool {
|
||||
return simpleFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
|
||||
}
|
||||
|
||||
// SimpleFormatReader handles simple file formats and images directly in Go,
|
||||
// bypassing the Python docreader service.
|
||||
type SimpleFormatReader struct{}
|
||||
|
||||
// Read reads simple format files and returns markdown.
|
||||
func (b *SimpleFormatReader) Read(_ context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
||||
ft := strings.ToLower(strings.TrimPrefix(req.FileType, "."))
|
||||
if ft == "" {
|
||||
ft = strings.TrimPrefix(strings.ToLower(filepath.Ext(req.FileName)), ".")
|
||||
}
|
||||
|
||||
switch {
|
||||
case ft == "md" || ft == "markdown":
|
||||
return &types.ReadResult{MarkdownContent: string(req.FileContent)}, nil
|
||||
case ft == "txt" || ft == "text":
|
||||
return &types.ReadResult{MarkdownContent: string(req.FileContent)}, nil
|
||||
case ft == "csv":
|
||||
md, err := csvToMarkdown(req.FileContent)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("csv conversion failed: %w", err)
|
||||
}
|
||||
return &types.ReadResult{MarkdownContent: md}, nil
|
||||
case imageFormats[ft]:
|
||||
return imageToResult(req.FileName, req.FileContent), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported simple format: %s", ft)
|
||||
}
|
||||
}
|
||||
|
||||
// imageToResult wraps a standalone image as a markdown image reference with
|
||||
// the raw bytes in ImageRefs, matching Python ImageParser behaviour.
|
||||
func imageToResult(fileName string, data []byte) *types.ReadResult {
|
||||
if fileName == "" {
|
||||
fileName = "image.png"
|
||||
}
|
||||
refPath := "images/" + fileName
|
||||
mime := http.DetectContentType(data)
|
||||
|
||||
return &types.ReadResult{
|
||||
MarkdownContent: fmt.Sprintf("", fileName, refPath),
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: fileName,
|
||||
OriginalRef: refPath,
|
||||
MimeType: mime,
|
||||
ImageData: data,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// IsImageFormat returns true if the file type is a recognized image format.
|
||||
func IsImageFormat(fileType string) bool {
|
||||
return imageFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
|
||||
}
|
||||
|
||||
// ensureOriginalImageRef checks whether the input file is an image and, if the
|
||||
// returned markdown does not already contain a markdown image reference for it,
|
||||
// prepends one and appends the raw bytes to imageRefs. This guarantees that
|
||||
// when MinerU OCRs a standalone image, the downstream chunks still carry the
|
||||
// original image link for retrieval display.
|
||||
func ensureOriginalImageRef(req *types.ReadRequest, mdContent string, imageRefs []types.ImageRef) (string, []types.ImageRef) {
|
||||
ft := strings.ToLower(strings.TrimPrefix(req.FileType, "."))
|
||||
if ft == "" {
|
||||
ft = strings.TrimPrefix(strings.ToLower(filepath.Ext(req.FileName)), ".")
|
||||
}
|
||||
if !imageFormats[ft] {
|
||||
return mdContent, imageRefs
|
||||
}
|
||||
if len(req.FileContent) == 0 {
|
||||
return mdContent, imageRefs
|
||||
}
|
||||
|
||||
fileName := req.FileName
|
||||
if fileName == "" {
|
||||
fileName = "image." + ft
|
||||
}
|
||||
refPath := "images/" + fileName
|
||||
|
||||
if strings.Contains(mdContent, refPath) {
|
||||
return mdContent, imageRefs
|
||||
}
|
||||
|
||||
imgLine := fmt.Sprintf("", fileName, refPath)
|
||||
if strings.TrimSpace(mdContent) == "" {
|
||||
mdContent = imgLine
|
||||
} else {
|
||||
mdContent = imgLine + "\n\n" + mdContent
|
||||
}
|
||||
|
||||
mime := http.DetectContentType(req.FileContent)
|
||||
imageRefs = append(imageRefs, types.ImageRef{
|
||||
Filename: fileName,
|
||||
OriginalRef: refPath,
|
||||
MimeType: mime,
|
||||
ImageData: req.FileContent,
|
||||
})
|
||||
|
||||
return mdContent, imageRefs
|
||||
}
|
||||
|
||||
func csvToMarkdown(data []byte) (string, error) {
|
||||
reader := csv.NewReader(strings.NewReader(string(data)))
|
||||
reader.LazyQuotes = true
|
||||
reader.TrimLeadingSpace = true
|
||||
|
||||
records, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
|
||||
// Header row
|
||||
header := records[0]
|
||||
sb.WriteString("| ")
|
||||
sb.WriteString(strings.Join(header, " | "))
|
||||
sb.WriteString(" |\n")
|
||||
|
||||
// Separator
|
||||
sb.WriteString("|")
|
||||
for range header {
|
||||
sb.WriteString(" --- |")
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
|
||||
// Data rows
|
||||
for _, row := range records[1:] {
|
||||
sb.WriteString("| ")
|
||||
// Pad row if shorter than header
|
||||
cells := make([]string, len(header))
|
||||
for i := range cells {
|
||||
if i < len(row) {
|
||||
cells[i] = row[i]
|
||||
}
|
||||
}
|
||||
sb.WriteString(strings.Join(cells, " | "))
|
||||
sb.WriteString(" |\n")
|
||||
}
|
||||
|
||||
return sb.String(), nil
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// EngineRegistration is the interface every locally registered parser engine
|
||||
// must implement. Remote-only engines (e.g. markitdown) are discovered via
|
||||
// the docreader ListEngines RPC and do not need a local registration.
|
||||
type EngineRegistration interface {
|
||||
Name() string
|
||||
Description() string
|
||||
FileTypes(docreaderConnected bool) []string
|
||||
CheckAvailable(docreaderConnected bool, overrides map[string]string) (available bool, reason string)
|
||||
}
|
||||
|
||||
// localEngines holds all locally registered parser engines.
|
||||
var localEngines []EngineRegistration
|
||||
|
||||
// RegisterEngine adds an engine to the local registry. Called in init().
|
||||
func RegisterEngine(e EngineRegistration) {
|
||||
localEngines = append(localEngines, e)
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterEngine(&simpleEngine{})
|
||||
RegisterEngine(&mineruEngine{})
|
||||
RegisterEngine(&mineruCloudEngine{})
|
||||
}
|
||||
|
||||
// SimpleEngineName is the engine name for Go-native simple format handling.
|
||||
const SimpleEngineName = "simple"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// simple — Go handles md/txt/csv natively, no external service needed.
|
||||
// Distinct from docreader's "builtin" which uses Python libraries for
|
||||
// complex formats (docx, pdf, etc.).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type simpleEngine struct{}
|
||||
|
||||
func (e *simpleEngine) Name() string { return SimpleEngineName }
|
||||
func (e *simpleEngine) Description() string { return "简单格式 & 图片解析(无需外部服务)" }
|
||||
func (e *simpleEngine) FileTypes(_ bool) []string {
|
||||
return []string{"md", "markdown", "txt", "csv", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"}
|
||||
}
|
||||
func (e *simpleEngine) CheckAvailable(_ bool, _ map[string]string) (bool, string) {
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// mineru — Go-native, calls self-hosted MinerU API directly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type mineruEngine struct{}
|
||||
|
||||
func (e *mineruEngine) Name() string { return "mineru" }
|
||||
func (e *mineruEngine) Description() string { return "MinerU 自部署服务" }
|
||||
func (e *mineruEngine) FileTypes(_ bool) []string {
|
||||
return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff", "doc", "docx", "ppt", "pptx"}
|
||||
}
|
||||
func (e *mineruEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) {
|
||||
endpoint := strings.TrimSpace(overrides["mineru_endpoint"])
|
||||
if endpoint == "" {
|
||||
return false, "未配置 MinerU 服务"
|
||||
}
|
||||
return PingMinerU(endpoint)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// mineru_cloud — Go-native, calls MinerU Cloud API directly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type mineruCloudEngine struct{}
|
||||
|
||||
func (e *mineruCloudEngine) Name() string { return "mineru_cloud" }
|
||||
func (e *mineruCloudEngine) Description() string { return "MinerU Cloud API" }
|
||||
func (e *mineruCloudEngine) FileTypes(_ bool) []string {
|
||||
return []string{"pdf", "jpg", "jpeg", "png", "bmp", "tiff", "doc", "docx", "ppt", "pptx"}
|
||||
}
|
||||
func (e *mineruCloudEngine) CheckAvailable(_ bool, overrides map[string]string) (bool, string) {
|
||||
apiKey := strings.TrimSpace(overrides["mineru_api_key"])
|
||||
if apiKey == "" {
|
||||
return false, "未配置 MinerU API Key"
|
||||
}
|
||||
return PingMinerUCloud(apiKey, overrides["mineru_api_base_url"])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ListAllEngines — merge local + remote
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// ListAllEngines returns the merged engine list: locally registered engines
|
||||
// plus engines discovered from the remote docreader via ListEngines RPC.
|
||||
//
|
||||
// Merge rules:
|
||||
// - Local engines are always included, with Go-side availability checks.
|
||||
// - For a remote engine whose name matches a local one, the remote's
|
||||
// file_types and description take precedence (the remote service is
|
||||
// authoritative for its own capabilities).
|
||||
// - Remote engines not present locally are appended as-is, enabling
|
||||
// auto-discovery of newly added docreader engines without Go changes.
|
||||
func ListAllEngines(docreaderConnected bool, overrides map[string]string, remoteEngines []types.ParserEngineInfo) []types.ParserEngineInfo {
|
||||
remoteMap := make(map[string]types.ParserEngineInfo, len(remoteEngines))
|
||||
for _, re := range remoteEngines {
|
||||
remoteMap[re.Name] = re
|
||||
}
|
||||
|
||||
seen := make(map[string]bool, len(localEngines))
|
||||
result := make([]types.ParserEngineInfo, 0, len(localEngines)+len(remoteEngines))
|
||||
|
||||
for _, e := range localEngines {
|
||||
name := e.Name()
|
||||
seen[name] = true
|
||||
|
||||
fileTypes := e.FileTypes(docreaderConnected)
|
||||
description := e.Description()
|
||||
|
||||
if re, ok := remoteMap[name]; ok {
|
||||
if len(re.FileTypes) > 0 {
|
||||
fileTypes = re.FileTypes
|
||||
}
|
||||
if re.Description != "" {
|
||||
description = re.Description
|
||||
}
|
||||
}
|
||||
|
||||
available, reason := e.CheckAvailable(docreaderConnected, overrides)
|
||||
result = append(result, types.ParserEngineInfo{
|
||||
Name: name,
|
||||
Description: description,
|
||||
FileTypes: fileTypes,
|
||||
Available: available,
|
||||
UnavailableReason: reason,
|
||||
})
|
||||
}
|
||||
|
||||
for _, re := range remoteEngines {
|
||||
if seen[re.Name] {
|
||||
continue
|
||||
}
|
||||
result = append(result, re)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/docreader/proto"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/resolver"
|
||||
)
|
||||
|
||||
var logger = log.New(os.Stdout, "[DocParser] ", log.LstdFlags|log.Lmicroseconds)
|
||||
|
||||
func getMaxMessageSize() int {
|
||||
if sizeStr := os.Getenv("MAX_FILE_SIZE_MB"); sizeStr != "" {
|
||||
if size, err := strconv.Atoi(sizeStr); err == nil && size > 0 {
|
||||
return size * 1024 * 1024
|
||||
}
|
||||
}
|
||||
return 50 * 1024 * 1024
|
||||
}
|
||||
|
||||
// GRPCDocumentReader implements DocumentReader over gRPC.
|
||||
type GRPCDocumentReader struct {
|
||||
mu sync.RWMutex
|
||||
conn *grpc.ClientConn
|
||||
client proto.DocReaderClient
|
||||
addr string
|
||||
}
|
||||
|
||||
func NewGRPCDocumentReader(addr string) (*GRPCDocumentReader, error) {
|
||||
p := &GRPCDocumentReader{}
|
||||
if addr != "" {
|
||||
if err := p.connect(addr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (p *GRPCDocumentReader) connect(addr string) error {
|
||||
logger.Printf("INFO: Connecting to docreader at %s", addr)
|
||||
|
||||
maxMsgSize := getMaxMessageSize()
|
||||
opts := []grpc.DialOption{
|
||||
grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
grpc.WithDefaultServiceConfig(`{"loadBalancingPolicy":"round_robin"}`),
|
||||
grpc.WithDefaultCallOptions(
|
||||
grpc.MaxCallRecvMsgSize(maxMsgSize),
|
||||
grpc.MaxCallSendMsgSize(maxMsgSize),
|
||||
),
|
||||
}
|
||||
resolver.SetDefaultScheme("dns")
|
||||
|
||||
start := time.Now()
|
||||
conn, err := grpc.Dial("dns:///"+addr, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to docreader: %w", err)
|
||||
}
|
||||
logger.Printf("INFO: Connected to docreader in %v", time.Since(start))
|
||||
|
||||
p.conn = conn
|
||||
p.client = proto.NewDocReaderClient(conn)
|
||||
p.addr = addr
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *GRPCDocumentReader) Reconnect(addr string) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if p.conn != nil {
|
||||
_ = p.conn.Close()
|
||||
p.conn = nil
|
||||
p.client = nil
|
||||
p.addr = ""
|
||||
}
|
||||
return p.connect(addr)
|
||||
}
|
||||
|
||||
func (p *GRPCDocumentReader) IsConnected() bool {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.conn != nil
|
||||
}
|
||||
|
||||
func (p *GRPCDocumentReader) Close() error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.conn != nil {
|
||||
return p.conn.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var errNotConnected = fmt.Errorf("docreader service not connected")
|
||||
|
||||
func (p *GRPCDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
||||
p.mu.RLock()
|
||||
client := p.client
|
||||
p.mu.RUnlock()
|
||||
if client == nil {
|
||||
return nil, errNotConnected
|
||||
}
|
||||
|
||||
protoReq := &proto.ReadRequest{
|
||||
FileContent: req.FileContent,
|
||||
FileName: req.FileName,
|
||||
FileType: req.FileType,
|
||||
Url: req.URL,
|
||||
Title: req.Title,
|
||||
RequestId: req.RequestID,
|
||||
Config: &proto.ReadConfig{
|
||||
ParserEngine: req.ParserEngine,
|
||||
ParserEngineOverrides: req.ParserEngineOverrides,
|
||||
ImageStorage: req.ImageStorage,
|
||||
},
|
||||
}
|
||||
|
||||
resp, err := client.Read(ctx, protoReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gRPC Read failed: %w", err)
|
||||
}
|
||||
return fromProtoReadResponse(resp), nil
|
||||
}
|
||||
|
||||
func (p *GRPCDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) {
|
||||
p.mu.RLock()
|
||||
client := p.client
|
||||
p.mu.RUnlock()
|
||||
if client == nil {
|
||||
return nil, errNotConnected
|
||||
}
|
||||
|
||||
resp, err := client.ListEngines(ctx, &proto.ListEnginesRequest{ConfigOverrides: overrides})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("gRPC ListEngines failed: %w", err)
|
||||
}
|
||||
|
||||
result := make([]types.ParserEngineInfo, 0, len(resp.GetEngines()))
|
||||
for _, e := range resp.GetEngines() {
|
||||
result = append(result, types.ParserEngineInfo{
|
||||
Name: e.GetName(),
|
||||
Description: e.GetDescription(),
|
||||
FileTypes: e.GetFileTypes(),
|
||||
Available: e.GetAvailable(),
|
||||
UnavailableReason: e.GetUnavailableReason(),
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func fromProtoReadResponse(resp *proto.ReadResponse) *types.ReadResult {
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: resp.GetMarkdownContent(),
|
||||
ImageDirPath: resp.GetImageDirPath(),
|
||||
Metadata: resp.GetMetadata(),
|
||||
Error: resp.GetError(),
|
||||
}
|
||||
|
||||
for _, ref := range resp.GetImageRefs() {
|
||||
result.ImageRefs = append(result.ImageRefs, types.ImageRef{
|
||||
Filename: ref.GetFilename(),
|
||||
OriginalRef: ref.GetOriginalRef(),
|
||||
MimeType: ref.GetMimeType(),
|
||||
StorageKey: ref.GetStorageKey(),
|
||||
ImageData: ref.GetImageData(),
|
||||
})
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
const (
|
||||
PathRead = "/read"
|
||||
PathListEngines = "/list-engines"
|
||||
)
|
||||
|
||||
// --- JSON DTOs ---
|
||||
|
||||
type httpReadConfig struct {
|
||||
ParserEngine string `json:"parser_engine,omitempty"`
|
||||
ParserEngineOverrides map[string]string `json:"parser_engine_overrides,omitempty"`
|
||||
ImageStorage map[string]string `json:"image_storage,omitempty"`
|
||||
}
|
||||
|
||||
type httpReadRequest struct {
|
||||
FileContent string `json:"file_content,omitempty"` // base64
|
||||
FileName string `json:"file_name,omitempty"`
|
||||
FileType string `json:"file_type,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Config *httpReadConfig `json:"config,omitempty"`
|
||||
RequestID string `json:"request_id,omitempty"`
|
||||
}
|
||||
|
||||
type httpImageRef struct {
|
||||
Filename string `json:"filename"`
|
||||
OriginalRef string `json:"original_ref"`
|
||||
MimeType string `json:"mime_type"`
|
||||
StorageKey string `json:"storage_key,omitempty"`
|
||||
ImageData []byte `json:"image_data,omitempty"`
|
||||
}
|
||||
|
||||
type httpReadResponse struct {
|
||||
MarkdownContent string `json:"markdown_content"`
|
||||
ImageRefs []httpImageRef `json:"image_refs,omitempty"`
|
||||
ImageDirPath string `json:"image_dir_path,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// HTTPDocumentReader implements DocumentReader over HTTP/JSON.
|
||||
type HTTPDocumentReader struct {
|
||||
mu sync.RWMutex
|
||||
baseURL string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewHTTPDocumentReader(baseURL string) (*HTTPDocumentReader, error) {
|
||||
p := &HTTPDocumentReader{
|
||||
baseURL: strings.TrimSuffix(baseURL, "/"),
|
||||
client: &http.Client{
|
||||
Timeout: 5 * time.Minute,
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
MaxIdleConnsPerHost: 5,
|
||||
},
|
||||
},
|
||||
}
|
||||
if p.baseURL != "" {
|
||||
logger.Printf("INFO: HTTP docreader base URL: %s", p.baseURL)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (p *HTTPDocumentReader) base() string {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.baseURL
|
||||
}
|
||||
|
||||
func (p *HTTPDocumentReader) Reconnect(addr string) error {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.baseURL = strings.TrimSuffix(addr, "/")
|
||||
logger.Printf("INFO: HTTP docreader base URL set to %s", p.baseURL)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *HTTPDocumentReader) IsConnected() bool {
|
||||
p.mu.RLock()
|
||||
defer p.mu.RUnlock()
|
||||
return p.baseURL != ""
|
||||
}
|
||||
|
||||
func (p *HTTPDocumentReader) Close() error { return nil }
|
||||
|
||||
type httpListEnginesRequest struct {
|
||||
ConfigOverrides map[string]string `json:"config_overrides,omitempty"`
|
||||
}
|
||||
|
||||
type httpParserEngineInfo struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
FileTypes []string `json:"file_types"`
|
||||
Available bool `json:"available"`
|
||||
UnavailableReason string `json:"unavailable_reason,omitempty"`
|
||||
}
|
||||
|
||||
type httpListEnginesResponse struct {
|
||||
Engines []httpParserEngineInfo `json:"engines"`
|
||||
}
|
||||
|
||||
func (p *HTTPDocumentReader) ListEngines(ctx context.Context, overrides map[string]string) ([]types.ParserEngineInfo, error) {
|
||||
base := p.base()
|
||||
if base == "" {
|
||||
return nil, errNotConnected
|
||||
}
|
||||
|
||||
body := httpListEnginesRequest{ConfigOverrides: overrides}
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http marshal list-engines request: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, base+PathListEngines, bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http new request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := p.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http list-engines failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respBytes, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("http list-engines status %d: %s", resp.StatusCode, string(respBytes))
|
||||
}
|
||||
|
||||
var out httpListEnginesResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("http decode list-engines response: %w", err)
|
||||
}
|
||||
|
||||
result := make([]types.ParserEngineInfo, 0, len(out.Engines))
|
||||
for _, e := range out.Engines {
|
||||
result = append(result, types.ParserEngineInfo{
|
||||
Name: e.Name,
|
||||
Description: e.Description,
|
||||
FileTypes: e.FileTypes,
|
||||
Available: e.Available,
|
||||
UnavailableReason: e.UnavailableReason,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func fromHTTPReadResponse(resp *httpReadResponse) *types.ReadResult {
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: resp.MarkdownContent,
|
||||
ImageDirPath: resp.ImageDirPath,
|
||||
Metadata: resp.Metadata,
|
||||
Error: resp.Error,
|
||||
}
|
||||
for _, ref := range resp.ImageRefs {
|
||||
result.ImageRefs = append(result.ImageRefs, types.ImageRef{
|
||||
Filename: ref.Filename,
|
||||
OriginalRef: ref.OriginalRef,
|
||||
MimeType: ref.MimeType,
|
||||
StorageKey: ref.StorageKey,
|
||||
ImageData: ref.ImageData,
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (p *HTTPDocumentReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
||||
base := p.base()
|
||||
if base == "" {
|
||||
return nil, errNotConnected
|
||||
}
|
||||
|
||||
body := httpReadRequest{
|
||||
FileName: req.FileName,
|
||||
FileType: req.FileType,
|
||||
URL: req.URL,
|
||||
Title: req.Title,
|
||||
RequestID: req.RequestID,
|
||||
Config: &httpReadConfig{
|
||||
ParserEngine: req.ParserEngine,
|
||||
ParserEngineOverrides: req.ParserEngineOverrides,
|
||||
ImageStorage: req.ImageStorage,
|
||||
},
|
||||
}
|
||||
if len(req.FileContent) > 0 {
|
||||
body.FileContent = base64.StdEncoding.EncodeToString(req.FileContent)
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http marshal read request: %w", err)
|
||||
}
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, base+PathRead, bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http new request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
httpReq.ContentLength = int64(len(jsonBody))
|
||||
|
||||
resp, err := p.client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("http read failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||
return nil, fmt.Errorf("http read status %d: %s", resp.StatusCode, string(bodyBytes))
|
||||
}
|
||||
var out httpReadResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return nil, fmt.Errorf("http decode read response: %w", err)
|
||||
}
|
||||
return fromHTTPReadResponse(&out), nil
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var httpImageClient = &http.Client{Timeout: 60 * time.Second}
|
||||
|
||||
// StoredImage describes an image that has been saved to local storage.
|
||||
type StoredImage struct {
|
||||
OriginalRef string // reference in the original markdown
|
||||
LocalPath string // absolute path on disk (/data/files/images/xxx.png)
|
||||
ServingURL string // URL for frontend access (/files/images/xxx.png)
|
||||
MimeType string
|
||||
}
|
||||
|
||||
// ImageResolver reads images from a DocReader ReadResult (temp dir or shared storage)
|
||||
// and saves them to the local file-serving directory, replacing markdown references.
|
||||
type ImageResolver struct {
|
||||
// LocalStorageDir is where images are persisted (e.g. /data/files)
|
||||
LocalStorageDir string
|
||||
// URLPrefix is the HTTP path prefix for serving (e.g. /files)
|
||||
URLPrefix string
|
||||
}
|
||||
|
||||
// NewImageResolver creates a resolver with defaults from environment.
|
||||
func NewImageResolver(localStorageDir, urlPrefix string) *ImageResolver {
|
||||
if localStorageDir == "" {
|
||||
localStorageDir = "/data/files"
|
||||
}
|
||||
if urlPrefix == "" {
|
||||
urlPrefix = "/files"
|
||||
}
|
||||
return &ImageResolver{
|
||||
LocalStorageDir: localStorageDir,
|
||||
URLPrefix: urlPrefix,
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveAndStore reads images from the convert result and persists them.
|
||||
// It returns the updated markdown with replaced image refs and a list of stored images.
|
||||
func (r *ImageResolver) ResolveAndStore(
|
||||
result *types.ReadResult,
|
||||
) (updatedMarkdown string, images []StoredImage, err error) {
|
||||
markdown := result.MarkdownContent
|
||||
if len(result.ImageRefs) == 0 && result.ImageDirPath == "" {
|
||||
return markdown, nil, nil
|
||||
}
|
||||
|
||||
imageDir := filepath.Join(r.LocalStorageDir, "images")
|
||||
if err := os.MkdirAll(imageDir, 0o755); err != nil {
|
||||
return markdown, nil, fmt.Errorf("create image dir: %w", err)
|
||||
}
|
||||
|
||||
// Build a map of original_ref -> image ref for fast lookup
|
||||
refMap := make(map[string]types.ImageRef)
|
||||
for _, ref := range result.ImageRefs {
|
||||
refMap[ref.OriginalRef] = ref
|
||||
}
|
||||
|
||||
// Process each image reference found in the markdown
|
||||
imgPattern := regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
|
||||
matches := imgPattern.FindAllStringSubmatchIndex(markdown, -1)
|
||||
|
||||
// Process in reverse order to preserve positions when replacing
|
||||
for i := len(matches) - 1; i >= 0; i-- {
|
||||
m := matches[i]
|
||||
refPath := markdown[m[4]:m[5]] // group 2: the URL/path
|
||||
|
||||
// Skip already-resolved URLs
|
||||
if strings.HasPrefix(refPath, "http://") || strings.HasPrefix(refPath, "https://") || strings.HasPrefix(refPath, "/files/") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Try to find the image data
|
||||
imgBytes, mimeType, err := r.readImageBytes(result, refPath, refMap)
|
||||
if err != nil || imgBytes == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Determine extension
|
||||
ext := extFromMime(mimeType)
|
||||
if ext == "" {
|
||||
ext = filepath.Ext(refPath)
|
||||
}
|
||||
if ext == "" {
|
||||
ext = ".png"
|
||||
}
|
||||
|
||||
// Save to local storage
|
||||
newName := uuid.New().String() + ext
|
||||
destPath := filepath.Join(imageDir, newName)
|
||||
if writeErr := os.WriteFile(destPath, imgBytes, 0o644); writeErr != nil {
|
||||
logger.Printf("WARN: failed to save image %s: %v", refPath, writeErr)
|
||||
continue
|
||||
}
|
||||
|
||||
servingURL := r.URLPrefix + "/images/" + newName
|
||||
|
||||
images = append(images, StoredImage{
|
||||
OriginalRef: refPath,
|
||||
LocalPath: destPath,
|
||||
ServingURL: servingURL,
|
||||
MimeType: mimeType,
|
||||
})
|
||||
|
||||
// Replace in markdown
|
||||
markdown = markdown[:m[4]] + servingURL + markdown[m[5]:]
|
||||
}
|
||||
|
||||
return markdown, images, nil
|
||||
}
|
||||
|
||||
// readImageBytes resolves image data with cascading fallback:
|
||||
// - Mode A: shared temp directory (same-machine / Docker volume)
|
||||
// - Mode C: inline bytes carried in gRPC response (universal, cross-machine)
|
||||
// - Mode B: shared object storage via storage_key (future)
|
||||
func (r *ImageResolver) readImageBytes(
|
||||
result *types.ReadResult,
|
||||
refPath string,
|
||||
refMap map[string]types.ImageRef,
|
||||
) ([]byte, string, error) {
|
||||
ref, found := refMap[refPath]
|
||||
|
||||
// Mode A: read from shared temp directory
|
||||
if result.ImageDirPath != "" {
|
||||
candidates := []string{
|
||||
filepath.Join(result.ImageDirPath, "images", filepath.Base(refPath)),
|
||||
filepath.Join(result.ImageDirPath, refPath),
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
data, err := os.ReadFile(candidate)
|
||||
if err == nil {
|
||||
mime := "application/octet-stream"
|
||||
if found {
|
||||
mime = ref.MimeType
|
||||
}
|
||||
if mime == "" {
|
||||
mime = mimeFromExt(filepath.Ext(candidate))
|
||||
}
|
||||
return data, mime, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Mode C: inline image bytes from gRPC/HTTP response
|
||||
if found && len(ref.ImageData) > 0 {
|
||||
mime := ref.MimeType
|
||||
if mime == "" {
|
||||
mime = mimeFromExt(filepath.Ext(ref.Filename))
|
||||
}
|
||||
return ref.ImageData, mime, nil
|
||||
}
|
||||
|
||||
// Mode B: shared object storage — storage_key is a download URL returned by storage.upload_bytes()
|
||||
if found && ref.StorageKey != "" {
|
||||
data, err := downloadImage(ref.StorageKey)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("download image from storage %s: %w", ref.StorageKey, err)
|
||||
}
|
||||
mime := ref.MimeType
|
||||
if mime == "" {
|
||||
mime = mimeFromExt(filepath.Ext(ref.Filename))
|
||||
}
|
||||
return data, mime, nil
|
||||
}
|
||||
|
||||
return nil, "", fmt.Errorf("image not found: %s", refPath)
|
||||
}
|
||||
|
||||
func downloadImage(url string) ([]byte, error) {
|
||||
resp, err := httpImageClient.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("HTTP GET: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d for %s", resp.StatusCode, url)
|
||||
}
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func extFromMime(mime string) string {
|
||||
switch mime {
|
||||
case "image/png":
|
||||
return ".png"
|
||||
case "image/jpeg":
|
||||
return ".jpg"
|
||||
case "image/gif":
|
||||
return ".gif"
|
||||
case "image/webp":
|
||||
return ".webp"
|
||||
case "image/bmp":
|
||||
return ".bmp"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func mimeFromExt(ext string) string {
|
||||
switch strings.ToLower(ext) {
|
||||
case ".png":
|
||||
return "image/png"
|
||||
case ".jpg", ".jpeg":
|
||||
return "image/jpeg"
|
||||
case ".gif":
|
||||
return "image/gif"
|
||||
case ".webp":
|
||||
return "image/webp"
|
||||
case ".bmp":
|
||||
return "image/bmp"
|
||||
default:
|
||||
return "application/octet-stream"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,589 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultPollInterval = 3 * time.Second
|
||||
defaultCloudTimeout = 600 * time.Second
|
||||
defaultBaseURL = "https://mineru.net/api/v4"
|
||||
)
|
||||
|
||||
// MinerUCloudReader calls the MinerU Cloud API (mineru.net) to read/convert documents.
|
||||
// Flow: POST /file-urls/batch → PUT file → poll GET /extract-results/batch/{batch_id}.
|
||||
type MinerUCloudReader struct {
|
||||
apiKey string
|
||||
baseURL string
|
||||
model string
|
||||
formulaEnable bool
|
||||
tableEnable bool
|
||||
ocrEnable bool
|
||||
language string
|
||||
}
|
||||
|
||||
// NewMinerUCloudReader creates a reader from ParserEngineOverrides.
|
||||
func NewMinerUCloudReader(overrides map[string]string) *MinerUCloudReader {
|
||||
return &MinerUCloudReader{
|
||||
apiKey: strings.TrimSpace(overrides["mineru_api_key"]),
|
||||
baseURL: strings.TrimRight(stringOr(overrides["mineru_api_base_url"], defaultBaseURL), "/"),
|
||||
model: stringOr(overrides["mineru_cloud_model"], "pipeline"),
|
||||
formulaEnable: parseBoolOr(overrides["mineru_cloud_enable_formula"], true),
|
||||
tableEnable: parseBoolOr(overrides["mineru_cloud_enable_table"], true),
|
||||
ocrEnable: parseBoolOr(overrides["mineru_cloud_enable_ocr"], true),
|
||||
language: stringOr(overrides["mineru_cloud_language"], "ch"),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *MinerUCloudReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
||||
if c.apiKey == "" {
|
||||
return &types.ReadResult{Error: "MinerU Cloud API key is not configured"}, nil
|
||||
}
|
||||
|
||||
content := req.FileContent
|
||||
if len(content) == 0 {
|
||||
return &types.ReadResult{Error: "no file content provided"}, nil
|
||||
}
|
||||
|
||||
logger.Printf("INFO: [MinerUCloud] Parsing file=%s size=%d via %s", req.FileName, len(content), c.baseURL)
|
||||
|
||||
ext := filepath.Ext(req.FileName)
|
||||
if ext == "" && req.FileType != "" {
|
||||
ext = "." + req.FileType
|
||||
}
|
||||
if ext == "" {
|
||||
ext = ".pdf"
|
||||
}
|
||||
fileName := strings.TrimSuffix(req.FileName, ext) + ext
|
||||
if fileName == ext {
|
||||
fileName = "document" + ext
|
||||
}
|
||||
|
||||
batchID, uploadURL, err := c.applyUploadURLs(ctx, fileName, ext)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("MinerU Cloud apply upload URLs: %w", err)
|
||||
}
|
||||
|
||||
if err := c.uploadFile(ctx, uploadURL, content); err != nil {
|
||||
return nil, fmt.Errorf("MinerU Cloud file upload: %w", err)
|
||||
}
|
||||
|
||||
mdContent, imageRefs, err := c.pollBatchResult(ctx, batchID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("MinerU Cloud poll: %w", err)
|
||||
}
|
||||
|
||||
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)
|
||||
|
||||
return &types.ReadResult{
|
||||
MarkdownContent: mdContent,
|
||||
ImageRefs: imageRefs,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// --- batch upload API ---
|
||||
|
||||
type batchApplyResponse struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
BatchID string `json:"batch_id"`
|
||||
FileURLs []string `json:"file_urls"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (c *MinerUCloudReader) applyUploadURLs(ctx context.Context, fileName, ext string) (string, string, error) {
|
||||
modelVersion := c.model
|
||||
if strings.ToLower(ext) == ".html" {
|
||||
modelVersion = "MinerU-HTML"
|
||||
}
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"files": []map[string]string{{"name": fileName, "data_id": uuid.New().String()}},
|
||||
"model_version": modelVersion,
|
||||
"is_ocr": c.ocrEnable,
|
||||
"enable_formula": c.formulaEnable,
|
||||
"enable_table": c.tableEnable,
|
||||
"language": c.language,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("marshal payload: %w", err)
|
||||
}
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/file-urls/batch", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("HTTP request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return "", "", fmt.Errorf("API status %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
var result batchApplyResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return "", "", fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
if result.Code != 0 {
|
||||
return "", "", fmt.Errorf("API error: %s", result.Msg)
|
||||
}
|
||||
if len(result.Data.FileURLs) == 0 {
|
||||
return "", "", fmt.Errorf("API returned no file_urls")
|
||||
}
|
||||
|
||||
logger.Printf("INFO: [MinerUCloud] batch apply ok: batch_id=%s, urls=%d", result.Data.BatchID, len(result.Data.FileURLs))
|
||||
return result.Data.BatchID, result.Data.FileURLs[0], nil
|
||||
}
|
||||
|
||||
func (c *MinerUCloudReader) uploadFile(ctx context.Context, uploadURL string, content []byte) error {
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPut, uploadURL, bytes.NewReader(content))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create PUT request: %w", err)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 120 * time.Second}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return fmt.Errorf("PUT upload: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("PUT upload status %d", resp.StatusCode)
|
||||
}
|
||||
logger.Printf("INFO: [MinerUCloud] file uploaded, status=%d", resp.StatusCode)
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- polling ---
|
||||
|
||||
type batchPollResponse struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
ExtractResult json.RawMessage `json:"extract_result"` // can be object or array
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type extractResultItem struct {
|
||||
State string `json:"state"`
|
||||
FileName string `json:"file_name"`
|
||||
Markdown string `json:"markdown"`
|
||||
Content string `json:"content"`
|
||||
Text string `json:"text"`
|
||||
ErrMsg string `json:"err_msg"`
|
||||
Progress struct {
|
||||
ExtractedPages int `json:"extracted_pages"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
} `json:"extract_progress"`
|
||||
FullZipURL string `json:"full_zip_url"`
|
||||
}
|
||||
|
||||
func (c *MinerUCloudReader) pollBatchResult(ctx context.Context, batchID string) (string, []types.ImageRef, error) {
|
||||
deadline := time.Now().Add(defaultCloudTimeout)
|
||||
pollCount := 0
|
||||
headers := map[string]string{
|
||||
"Authorization": "Bearer " + c.apiKey,
|
||||
}
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
pollCount++
|
||||
|
||||
items, err := c.fetchBatchStatus(ctx, batchID, headers)
|
||||
if err != nil {
|
||||
logger.Printf("WARN: [MinerUCloud] poll #%d failed: %v", pollCount, err)
|
||||
sleepCtx(ctx, defaultPollInterval)
|
||||
continue
|
||||
}
|
||||
|
||||
if len(items) == 0 {
|
||||
if pollCount <= 3 || pollCount%10 == 0 {
|
||||
logger.Printf("INFO: [MinerUCloud] poll #%d: extract_result empty, retrying", pollCount)
|
||||
}
|
||||
sleepCtx(ctx, defaultPollInterval)
|
||||
continue
|
||||
}
|
||||
|
||||
item := items[0]
|
||||
state := strings.ToLower(item.State)
|
||||
|
||||
if pollCount == 1 || pollCount%10 == 0 || state == "done" || state == "failed" {
|
||||
logger.Printf("INFO: [MinerUCloud] poll #%d: file=%s state=%s pages=%d/%d",
|
||||
pollCount, item.FileName, state, item.Progress.ExtractedPages, item.Progress.TotalPages)
|
||||
}
|
||||
|
||||
if state == "failed" {
|
||||
return "", nil, fmt.Errorf("MinerU Cloud task failed: %s", item.ErrMsg)
|
||||
}
|
||||
|
||||
if state == "done" {
|
||||
return c.extractDoneResult(ctx, &item)
|
||||
}
|
||||
|
||||
sleepCtx(ctx, defaultPollInterval)
|
||||
}
|
||||
|
||||
return "", nil, fmt.Errorf("MinerU Cloud task timed out after %d polls", pollCount)
|
||||
}
|
||||
|
||||
func (c *MinerUCloudReader) fetchBatchStatus(ctx context.Context, batchID string, headers map[string]string) ([]extractResultItem, error) {
|
||||
url := fmt.Sprintf("%s/extract-results/batch/%s", c.baseURL, batchID)
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range headers {
|
||||
httpReq.Header.Set(k, v)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read poll response body: %w", err)
|
||||
}
|
||||
|
||||
var pollResp batchPollResponse
|
||||
if err := json.Unmarshal(respBody, &pollResp); err != nil {
|
||||
return nil, fmt.Errorf("decode poll response: %w", err)
|
||||
}
|
||||
if pollResp.Code != 0 {
|
||||
return nil, fmt.Errorf("poll error code=%d msg=%s", pollResp.Code, pollResp.Msg)
|
||||
}
|
||||
|
||||
if len(pollResp.Data.ExtractResult) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Dump the raw extract_result JSON for debugging
|
||||
rawExtract := string(pollResp.Data.ExtractResult)
|
||||
if len(rawExtract) > 4000 {
|
||||
logger.Printf("DEBUG: [MinerUCloud] Raw extract_result (truncated to 4000 chars): %s ...", rawExtract[:4000])
|
||||
} else {
|
||||
logger.Printf("DEBUG: [MinerUCloud] Raw extract_result: %s", rawExtract)
|
||||
}
|
||||
|
||||
// Pretty-print the structure to reveal all available fields
|
||||
var rawObj interface{}
|
||||
if err := json.Unmarshal(pollResp.Data.ExtractResult, &rawObj); err == nil {
|
||||
logCloudResponseStructure(rawObj, "extract_result")
|
||||
}
|
||||
|
||||
// The extract_result can be either a single object or an array
|
||||
var items []extractResultItem
|
||||
if pollResp.Data.ExtractResult[0] == '[' {
|
||||
if err := json.Unmarshal(pollResp.Data.ExtractResult, &items); err != nil {
|
||||
return nil, fmt.Errorf("decode extract_result array: %w", err)
|
||||
}
|
||||
} else {
|
||||
var single extractResultItem
|
||||
if err := json.Unmarshal(pollResp.Data.ExtractResult, &single); err != nil {
|
||||
return nil, fmt.Errorf("decode extract_result object: %w", err)
|
||||
}
|
||||
items = []extractResultItem{single}
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// extractDoneResult extracts markdown and images from a completed batch item.
|
||||
// Prefers inline markdown/content fields; falls back to downloading full_zip_url.
|
||||
func (c *MinerUCloudReader) extractDoneResult(_ context.Context, item *extractResultItem) (string, []types.ImageRef, error) {
|
||||
text := firstNonEmpty(item.Markdown, item.Content, item.Text)
|
||||
if text != "" {
|
||||
logger.Printf("INFO: [MinerUCloud] parsed (inline), length=%d", len(text))
|
||||
return text, nil, nil
|
||||
}
|
||||
|
||||
if item.FullZipURL == "" {
|
||||
return "", nil, fmt.Errorf("MinerU Cloud state=done but no markdown/content or full_zip_url")
|
||||
}
|
||||
|
||||
md, imageRefs, err := downloadAndExtractZip(item.FullZipURL)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("extract zip: %w", err)
|
||||
}
|
||||
|
||||
logger.Printf("INFO: [MinerUCloud] parsed (zip), markdown=%d chars, images=%d", len(md), len(imageRefs))
|
||||
return md, imageRefs, nil
|
||||
}
|
||||
|
||||
// --- ZIP handling ---
|
||||
|
||||
var imgRefPattern = regexp.MustCompile(`!\[[^\]]*\]\(([^)]+)\)`)
|
||||
|
||||
func downloadAndExtractZip(zipURL string) (string, []types.ImageRef, error) {
|
||||
client := &http.Client{Timeout: 120 * time.Second}
|
||||
resp, err := client.Get(zipURL)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("download zip: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", nil, fmt.Errorf("download zip status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
zipData, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("read zip body: %w", err)
|
||||
}
|
||||
|
||||
zr, err := zip.NewReader(bytes.NewReader(zipData), int64(len(zipData)))
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("open zip: %w", err)
|
||||
}
|
||||
|
||||
// Find .md files
|
||||
var mdFiles []string
|
||||
entries := make(map[string]*zip.File)
|
||||
for _, f := range zr.File {
|
||||
entries[f.Name] = f
|
||||
if strings.HasSuffix(f.Name, ".md") {
|
||||
mdFiles = append(mdFiles, f.Name)
|
||||
}
|
||||
}
|
||||
if len(mdFiles) == 0 {
|
||||
return "", nil, fmt.Errorf("no .md file found in zip")
|
||||
}
|
||||
sort.Slice(mdFiles, func(i, j int) bool {
|
||||
di, dj := strings.Count(mdFiles[i], "/"), strings.Count(mdFiles[j], "/")
|
||||
if di != dj {
|
||||
return di < dj
|
||||
}
|
||||
return mdFiles[i] < mdFiles[j]
|
||||
})
|
||||
|
||||
mdText, err := readZipEntry(entries[mdFiles[0]])
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("read md file: %w", err)
|
||||
}
|
||||
|
||||
mdDir := filepath.Dir(mdFiles[0])
|
||||
|
||||
// Extract referenced images
|
||||
var imageRefs []types.ImageRef
|
||||
seen := map[string]bool{}
|
||||
for _, match := range imgRefPattern.FindAllStringSubmatch(mdText, -1) {
|
||||
imgPath := match[1]
|
||||
if strings.HasPrefix(imgPath, "http://") || strings.HasPrefix(imgPath, "https://") || strings.HasPrefix(imgPath, "data:") {
|
||||
continue
|
||||
}
|
||||
if seen[imgPath] {
|
||||
continue
|
||||
}
|
||||
seen[imgPath] = true
|
||||
|
||||
resolved := resolveInZip(imgPath, mdDir, entries)
|
||||
if resolved == nil {
|
||||
logger.Printf("WARN: [MinerUCloud] image not found in zip: %s", imgPath)
|
||||
continue
|
||||
}
|
||||
|
||||
imgData, err := readZipEntryBytes(resolved)
|
||||
if err != nil {
|
||||
logger.Printf("WARN: [MinerUCloud] failed to read zip image %s: %v", imgPath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
ext := strings.ToLower(filepath.Ext(resolved.Name))
|
||||
if ext == "" {
|
||||
ext = ".png"
|
||||
}
|
||||
mimeType := mime.TypeByExtension(ext)
|
||||
if mimeType == "" {
|
||||
mimeType = "image/png"
|
||||
}
|
||||
|
||||
imageRefs = append(imageRefs, types.ImageRef{
|
||||
Filename: filepath.Base(resolved.Name),
|
||||
OriginalRef: imgPath,
|
||||
MimeType: mimeType,
|
||||
ImageData: imgData,
|
||||
})
|
||||
}
|
||||
|
||||
return mdText, imageRefs, nil
|
||||
}
|
||||
|
||||
func resolveInZip(imgPath, mdDir string, entries map[string]*zip.File) *zip.File {
|
||||
normalized := strings.ReplaceAll(imgPath, "\\", "/")
|
||||
if f, ok := entries[normalized]; ok {
|
||||
return f
|
||||
}
|
||||
if mdDir != "" && mdDir != "." {
|
||||
joined := mdDir + "/" + normalized
|
||||
if f, ok := entries[joined]; ok {
|
||||
return f
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readZipEntry(f *zip.File) (string, error) {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer rc.Close()
|
||||
data, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(data), nil
|
||||
}
|
||||
|
||||
func readZipEntryBytes(f *zip.File) ([]byte, error) {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rc.Close()
|
||||
return io.ReadAll(rc)
|
||||
}
|
||||
|
||||
// logCloudResponseStructure recursively logs the structure of the Cloud API response,
|
||||
// truncating large string values to reveal all available fields (e.g. caption, bbox, etc.).
|
||||
func logCloudResponseStructure(obj interface{}, prefix string) {
|
||||
switch v := obj.(type) {
|
||||
case map[string]interface{}:
|
||||
keys := make([]string, 0, len(v))
|
||||
for k := range v {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = {object with %d keys: %s}", prefix, len(v), strings.Join(keys, ", "))
|
||||
for _, key := range keys {
|
||||
val := v[key]
|
||||
path := prefix + "." + key
|
||||
switch inner := val.(type) {
|
||||
case map[string]interface{}:
|
||||
logCloudResponseStructure(inner, path)
|
||||
case []interface{}:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = [array with %d items]", path, len(inner))
|
||||
if len(inner) > 0 {
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s[0] type=%T", path, inner[0])
|
||||
if len(inner) <= 3 {
|
||||
for i, item := range inner {
|
||||
logCloudResponseStructure(item, fmt.Sprintf("%s[%d]", path, i))
|
||||
}
|
||||
} else {
|
||||
logCloudResponseStructure(inner[0], path+"[0]")
|
||||
logger.Printf("DEBUG: [MinerUCloud] ... and %d more items in %s", len(inner)-1, path)
|
||||
}
|
||||
}
|
||||
case string:
|
||||
if len(inner) > 200 {
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = string(%d chars): %.200s...", path, len(inner), inner)
|
||||
} else {
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = %q", path, inner)
|
||||
}
|
||||
case float64:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = %v (number)", path, inner)
|
||||
case bool:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = %v (bool)", path, inner)
|
||||
case nil:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = null", path)
|
||||
default:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = %v (%T)", path, val, val)
|
||||
}
|
||||
}
|
||||
case []interface{}:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = [array with %d items]", prefix, len(v))
|
||||
if len(v) > 0 {
|
||||
if len(v) <= 3 {
|
||||
for i, item := range v {
|
||||
logCloudResponseStructure(item, fmt.Sprintf("%s[%d]", prefix, i))
|
||||
}
|
||||
} else {
|
||||
logCloudResponseStructure(v[0], prefix+"[0]")
|
||||
logger.Printf("DEBUG: [MinerUCloud] ... and %d more items in %s", len(v)-1, prefix)
|
||||
}
|
||||
}
|
||||
default:
|
||||
logger.Printf("DEBUG: [MinerUCloud] %s = %v (%T)", prefix, v, v)
|
||||
}
|
||||
}
|
||||
|
||||
// PingMinerUCloud checks if the MinerU Cloud API is reachable with the given API key.
|
||||
func PingMinerUCloud(apiKey, baseURL string) (bool, string) {
|
||||
apiKey = strings.TrimSpace(apiKey)
|
||||
baseURL = strings.TrimRight(strings.TrimSpace(baseURL), "/")
|
||||
if apiKey == "" {
|
||||
return false, "未配置 MinerU Cloud API Key"
|
||||
}
|
||||
if baseURL == "" {
|
||||
baseURL = defaultBaseURL
|
||||
}
|
||||
|
||||
// Light probe: request an empty batch to verify credentials
|
||||
payload := []byte(`{"files":[],"model_version":"pipeline"}`)
|
||||
req, err := http.NewRequest(http.MethodPost, baseURL+"/file-urls/batch", bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return false, fmt.Sprintf("构建请求失败: %v", err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false, fmt.Sprintf("MinerU Cloud 不可达: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
||||
return false, "MinerU Cloud API Key 无效"
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func sleepCtx(ctx context.Context, d time.Duration) {
|
||||
t := time.NewTimer(d)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,329 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
const mineruTimeout = 1000 * time.Second // large docs can take a while
|
||||
|
||||
var b64DataURIPattern = regexp.MustCompile(`^data:image/(\w+);base64,(.+)$`)
|
||||
|
||||
// MinerUReader calls a self-hosted MinerU API to read/convert documents.
|
||||
type MinerUReader struct {
|
||||
endpoint string
|
||||
backend string // "pipeline", "vlm-*", "hybrid-*"
|
||||
formulaEnable bool
|
||||
tableEnable bool
|
||||
ocrEnable bool
|
||||
language string
|
||||
}
|
||||
|
||||
// NewMinerUReader creates a reader from ParserEngineOverrides.
|
||||
func NewMinerUReader(overrides map[string]string) *MinerUReader {
|
||||
c := &MinerUReader{
|
||||
endpoint: strings.TrimRight(overrides["mineru_endpoint"], "/"),
|
||||
backend: stringOr(overrides["mineru_model"], "pipeline"),
|
||||
formulaEnable: parseBoolOr(overrides["mineru_enable_formula"], true),
|
||||
tableEnable: parseBoolOr(overrides["mineru_enable_table"], true),
|
||||
ocrEnable: parseBoolOr(overrides["mineru_enable_ocr"], true),
|
||||
language: stringOr(overrides["mineru_language"], "ch"),
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
||||
if c.endpoint == "" {
|
||||
return &types.ReadResult{Error: "MinerU endpoint is not configured"}, nil
|
||||
}
|
||||
|
||||
content := req.FileContent
|
||||
if len(content) == 0 {
|
||||
return &types.ReadResult{Error: "no file content provided"}, nil
|
||||
}
|
||||
|
||||
logger.Printf("INFO: [MinerU] Parsing file=%s size=%d via %s", req.FileName, len(content), c.endpoint)
|
||||
|
||||
mdContent, imagesB64, err := c.callFileParse(ctx, content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("MinerU file_parse: %w", err)
|
||||
}
|
||||
|
||||
// HTML -> Markdown conversion (equivalent to Python markdownify)
|
||||
mdContent = htmlToMarkdown(mdContent)
|
||||
|
||||
// Process images: decode base64, build ImageRef list, replace refs in markdown
|
||||
imageRefs, mdContent := c.processImages(mdContent, imagesB64)
|
||||
|
||||
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)
|
||||
|
||||
logger.Printf("INFO: [MinerU] Parsed successfully, markdown=%d chars, images=%d", len(mdContent), len(imageRefs))
|
||||
|
||||
return &types.ReadResult{
|
||||
MarkdownContent: mdContent,
|
||||
ImageRefs: imageRefs,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// mineruFileParseResponse mirrors the relevant fields from the MinerU API response.
|
||||
type mineruFileParseResponse struct {
|
||||
Results struct {
|
||||
Files struct {
|
||||
MDContent string `json:"md_content"`
|
||||
Images map[string]string `json:"images"` // path -> "data:image/png;base64,..." or raw base64
|
||||
} `json:"files"`
|
||||
} `json:"results"`
|
||||
}
|
||||
|
||||
func (c *MinerUReader) callFileParse(ctx context.Context, content []byte) (string, map[string]string, error) {
|
||||
var body bytes.Buffer
|
||||
writer := multipart.NewWriter(&body)
|
||||
|
||||
// Form fields
|
||||
fields := map[string]string{
|
||||
"return_md": "true",
|
||||
"return_images": "true",
|
||||
"table_enable": fmt.Sprintf("%v", c.tableEnable),
|
||||
"formula_enable": fmt.Sprintf("%v", c.formulaEnable),
|
||||
"parse_method": "ocr",
|
||||
"start_page_id": "0",
|
||||
"end_page_id": "99999",
|
||||
"backend": c.backend,
|
||||
"response_format_zip": "false",
|
||||
"return_middle_json": "false",
|
||||
"return_model_output": "false",
|
||||
"return_content_list": "true",
|
||||
}
|
||||
if !c.ocrEnable {
|
||||
fields["parse_method"] = "txt"
|
||||
}
|
||||
if c.language != "" {
|
||||
fields["lang_list"] = c.language
|
||||
}
|
||||
for k, v := range fields {
|
||||
_ = writer.WriteField(k, v)
|
||||
}
|
||||
|
||||
// File part
|
||||
part, err := writer.CreateFormFile("files", "document")
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("create form file: %w", err)
|
||||
}
|
||||
if _, err := part.Write(content); err != nil {
|
||||
return "", nil, fmt.Errorf("write file content: %w", err)
|
||||
}
|
||||
writer.Close()
|
||||
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint+"/file_parse", &body)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
client := &http.Client{Timeout: mineruTimeout}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("HTTP request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
respBody, _ := io.ReadAll(resp.Body)
|
||||
return "", nil, fmt.Errorf("MinerU API status %d: %s", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("read response body: %w", err)
|
||||
}
|
||||
|
||||
// Dump raw response for debugging (truncate if too large)
|
||||
rawStr := string(respBody)
|
||||
if len(rawStr) > 4000 {
|
||||
logger.Printf("DEBUG: [MinerU] Raw response (truncated to 4000 chars): %s ...", rawStr[:4000])
|
||||
} else {
|
||||
logger.Printf("DEBUG: [MinerU] Raw response: %s", rawStr)
|
||||
}
|
||||
|
||||
// Also pretty-print the top-level structure (without large base64 blobs)
|
||||
var rawMap map[string]interface{}
|
||||
if err := json.Unmarshal(respBody, &rawMap); err == nil {
|
||||
c.logResponseStructure(rawMap, "")
|
||||
}
|
||||
|
||||
var result mineruFileParseResponse
|
||||
if err := json.Unmarshal(respBody, &result); err != nil {
|
||||
return "", nil, fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
|
||||
return result.Results.Files.MDContent, result.Results.Files.Images, nil
|
||||
}
|
||||
|
||||
// processImages decodes base64 images from MinerU response and returns ImageRef list.
|
||||
// It also replaces image references in the markdown content.
|
||||
func (c *MinerUReader) processImages(mdContent string, imagesB64 map[string]string) ([]types.ImageRef, string) {
|
||||
var refs []types.ImageRef
|
||||
|
||||
for ipath, b64Str := range imagesB64 {
|
||||
originalRef := "images/" + ipath
|
||||
if !strings.Contains(mdContent, originalRef) {
|
||||
continue
|
||||
}
|
||||
|
||||
var imgBytes []byte
|
||||
var ext string
|
||||
|
||||
if m := b64DataURIPattern.FindStringSubmatch(b64Str); len(m) == 3 {
|
||||
ext = m[1]
|
||||
decoded, err := base64.StdEncoding.DecodeString(m[2])
|
||||
if err != nil {
|
||||
logger.Printf("WARN: [MinerU] Failed to decode base64 image %s: %v", ipath, err)
|
||||
continue
|
||||
}
|
||||
imgBytes = decoded
|
||||
} else {
|
||||
// raw base64 without data URI prefix
|
||||
decoded, err := base64.StdEncoding.DecodeString(b64Str)
|
||||
if err != nil {
|
||||
logger.Printf("WARN: [MinerU] Failed to decode raw base64 image %s: %v", ipath, err)
|
||||
continue
|
||||
}
|
||||
imgBytes = decoded
|
||||
ext = strings.TrimPrefix(filepath.Ext(ipath), ".")
|
||||
if ext == "" {
|
||||
ext = "png"
|
||||
}
|
||||
}
|
||||
|
||||
mimeType := mime.TypeByExtension("." + ext)
|
||||
if mimeType == "" {
|
||||
mimeType = "image/png"
|
||||
}
|
||||
|
||||
refs = append(refs, types.ImageRef{
|
||||
Filename: ipath,
|
||||
OriginalRef: originalRef,
|
||||
MimeType: mimeType,
|
||||
ImageData: imgBytes,
|
||||
})
|
||||
}
|
||||
|
||||
return refs, mdContent
|
||||
}
|
||||
|
||||
// logResponseStructure recursively logs the structure of the API response,
|
||||
// truncating large string values to reveal all available fields.
|
||||
func (c *MinerUReader) logResponseStructure(obj interface{}, prefix string) {
|
||||
switch v := obj.(type) {
|
||||
case map[string]interface{}:
|
||||
for key, val := range v {
|
||||
path := prefix + "." + key
|
||||
if path[0] == '.' {
|
||||
path = path[1:]
|
||||
}
|
||||
switch inner := val.(type) {
|
||||
case map[string]interface{}:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = {object with %d keys: %s}", path, len(inner), mapKeys(inner))
|
||||
c.logResponseStructure(inner, path)
|
||||
case []interface{}:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = [array with %d items]", path, len(inner))
|
||||
if len(inner) > 0 {
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s[0] type=%T", path, inner[0])
|
||||
if len(inner) <= 3 {
|
||||
for i, item := range inner {
|
||||
c.logResponseStructure(item, fmt.Sprintf("%s[%d]", path, i))
|
||||
}
|
||||
} else {
|
||||
c.logResponseStructure(inner[0], path+"[0]")
|
||||
logger.Printf("DEBUG: [MinerU] ... and %d more items in %s", len(inner)-1, path)
|
||||
}
|
||||
}
|
||||
case string:
|
||||
if len(inner) > 200 {
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = string(%d chars): %.200s...", path, len(inner), inner)
|
||||
} else {
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = %q", path, inner)
|
||||
}
|
||||
case float64:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = %v (number)", path, inner)
|
||||
case bool:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = %v (bool)", path, inner)
|
||||
case nil:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = null", path)
|
||||
default:
|
||||
logger.Printf("DEBUG: [MinerU] Response field: %s = %v (%T)", path, inner, inner)
|
||||
}
|
||||
}
|
||||
default:
|
||||
logger.Printf("DEBUG: [MinerU] Value at %s: %v (%T)", prefix, v, v)
|
||||
}
|
||||
}
|
||||
|
||||
func mapKeys(m map[string]interface{}) string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return strings.Join(keys, ", ")
|
||||
}
|
||||
|
||||
// PingMinerU checks if the self-hosted MinerU service is reachable.
|
||||
func PingMinerU(endpoint string) (bool, string) {
|
||||
endpoint = strings.TrimRight(endpoint, "/")
|
||||
if endpoint == "" {
|
||||
return false, "未配置 MinerU 端点"
|
||||
}
|
||||
client := &http.Client{Timeout: 5 * time.Second}
|
||||
resp, err := client.Get(endpoint + "/docs")
|
||||
if err != nil {
|
||||
return false, fmt.Sprintf("MinerU 服务不可达: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return false, fmt.Sprintf("MinerU 服务返回状态 %d", resp.StatusCode)
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// htmlToMarkdown converts HTML content to markdown.
|
||||
// Falls back to the original content if conversion fails.
|
||||
func htmlToMarkdown(content string) string {
|
||||
md, err := htmltomd.ConvertString(content)
|
||||
if err != nil {
|
||||
logger.Printf("WARN: [MinerU] html-to-markdown conversion failed, using raw content: %v", err)
|
||||
return content
|
||||
}
|
||||
return md
|
||||
}
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
func stringOr(val, fallback string) string {
|
||||
val = strings.TrimSpace(val)
|
||||
if val == "" {
|
||||
return fallback
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
func parseBoolOr(val string, fallback bool) bool {
|
||||
val = strings.ToLower(strings.TrimSpace(val))
|
||||
if val == "" {
|
||||
return fallback
|
||||
}
|
||||
return val == "true" || val == "1" || val == "yes"
|
||||
}
|
||||
Reference in New Issue
Block a user