Files
Enterprise-Knowledge-Base/internal/infrastructure/docparser/mineru_converter.go
T
wizardchen 397689d2f3 feat: introduce WeKnora Lite edition with lightweight configuration and deployment
- Added a new `.env.lite.example` file for the Lite version, providing a minimal configuration template.
- Updated `.env.example` to remove deprecated variables and include new Docreader settings.
- Enhanced Docker configurations to support the Lite version, including a new Dockerfile for the Docreader service.
- Introduced a Makefile target for building and running the Lite version, along with packaging capabilities.
- Created GitHub workflows for building and releasing Lite binaries, including Homebrew formula support.
- Implemented a new service file for managing the Lite version as a system service.

This update enables a streamlined, single-binary deployment of WeKnora, reducing external dependencies and simplifying setup.
2026-03-02 21:21:49 +08:00

330 lines
9.8 KiB
Go

package docparser
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"mime"
"mime/multipart"
"net/http"
"path/filepath"
"regexp"
"strings"
"time"
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/Tencent/WeKnora/internal/types"
)
const mineruTimeout = 1000 * time.Second // large docs can take a while
var b64DataURIPattern = regexp.MustCompile(`^data:image/(\w+);base64,(.+)$`)
// MinerUReader calls a self-hosted MinerU API to read/convert documents.
type MinerUReader struct {
endpoint string
backend string // "pipeline", "vlm-*", "hybrid-*"
formulaEnable bool
tableEnable bool
ocrEnable bool
language string
}
// NewMinerUReader creates a reader from ParserEngineOverrides.
func NewMinerUReader(overrides map[string]string) *MinerUReader {
c := &MinerUReader{
endpoint: strings.TrimRight(overrides["mineru_endpoint"], "/"),
backend: stringOr(overrides["mineru_model"], "pipeline"),
formulaEnable: parseBoolOr(overrides["mineru_enable_formula"], true),
tableEnable: parseBoolOr(overrides["mineru_enable_table"], true),
ocrEnable: parseBoolOr(overrides["mineru_enable_ocr"], true),
language: stringOr(overrides["mineru_language"], "ch"),
}
return c
}
func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
if c.endpoint == "" {
return &types.ReadResult{Error: "MinerU endpoint is not configured"}, nil
}
content := req.FileContent
if len(content) == 0 {
return &types.ReadResult{Error: "no file content provided"}, nil
}
logger.Printf("INFO: [MinerU] Parsing file=%s size=%d via %s", req.FileName, len(content), c.endpoint)
mdContent, imagesB64, err := c.callFileParse(ctx, content)
if err != nil {
return nil, fmt.Errorf("MinerU file_parse: %w", err)
}
// HTML -> Markdown conversion (equivalent to Python markdownify)
mdContent = htmlToMarkdown(mdContent)
// Process images: decode base64, build ImageRef list, replace refs in markdown
imageRefs, mdContent := c.processImages(mdContent, imagesB64)
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)
logger.Printf("INFO: [MinerU] Parsed successfully, markdown=%d chars, images=%d", len(mdContent), len(imageRefs))
return &types.ReadResult{
MarkdownContent: mdContent,
ImageRefs: imageRefs,
}, nil
}
// mineruFileParseResponse mirrors the relevant fields from the MinerU API response.
type mineruFileParseResponse struct {
Results struct {
Files struct {
MDContent string `json:"md_content"`
Images map[string]string `json:"images"` // path -> "data:image/png;base64,..." or raw base64
} `json:"files"`
} `json:"results"`
}
func (c *MinerUReader) callFileParse(ctx context.Context, content []byte) (string, map[string]string, error) {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
// Form fields
fields := map[string]string{
"return_md": "true",
"return_images": "true",
"table_enable": fmt.Sprintf("%v", c.tableEnable),
"formula_enable": fmt.Sprintf("%v", c.formulaEnable),
"parse_method": "ocr",
"start_page_id": "0",
"end_page_id": "99999",
"backend": c.backend,
"response_format_zip": "false",
"return_middle_json": "false",
"return_model_output": "false",
"return_content_list": "true",
}
if !c.ocrEnable {
fields["parse_method"] = "txt"
}
if c.language != "" {
fields["lang_list"] = c.language
}
for k, v := range fields {
_ = writer.WriteField(k, v)
}
// File part
part, err := writer.CreateFormFile("files", "document")
if err != nil {
return "", nil, fmt.Errorf("create form file: %w", err)
}
if _, err := part.Write(content); err != nil {
return "", nil, fmt.Errorf("write file content: %w", err)
}
writer.Close()
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint+"/file_parse", &body)
if err != nil {
return "", nil, fmt.Errorf("create request: %w", err)
}
httpReq.Header.Set("Content-Type", writer.FormDataContentType())
client := &http.Client{Timeout: mineruTimeout}
resp, err := client.Do(httpReq)
if err != nil {
return "", nil, fmt.Errorf("HTTP request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
respBody, _ := io.ReadAll(resp.Body)
return "", nil, fmt.Errorf("MinerU API status %d: %s", resp.StatusCode, string(respBody))
}
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return "", nil, fmt.Errorf("read response body: %w", err)
}
// Dump raw response for debugging (truncate if too large)
rawStr := string(respBody)
if len(rawStr) > 4000 {
logger.Printf("DEBUG: [MinerU] Raw response (truncated to 4000 chars): %s ...", rawStr[:4000])
} else {
logger.Printf("DEBUG: [MinerU] Raw response: %s", rawStr)
}
// Also pretty-print the top-level structure (without large base64 blobs)
var rawMap map[string]interface{}
if err := json.Unmarshal(respBody, &rawMap); err == nil {
c.logResponseStructure(rawMap, "")
}
var result mineruFileParseResponse
if err := json.Unmarshal(respBody, &result); err != nil {
return "", nil, fmt.Errorf("decode response: %w", err)
}
return result.Results.Files.MDContent, result.Results.Files.Images, nil
}
// processImages decodes base64 images from MinerU response and returns ImageRef list.
// It also replaces image references in the markdown content.
func (c *MinerUReader) processImages(mdContent string, imagesB64 map[string]string) ([]types.ImageRef, string) {
var refs []types.ImageRef
for ipath, b64Str := range imagesB64 {
originalRef := "images/" + ipath
if !strings.Contains(mdContent, originalRef) {
continue
}
var imgBytes []byte
var ext string
if m := b64DataURIPattern.FindStringSubmatch(b64Str); len(m) == 3 {
ext = m[1]
decoded, err := base64.StdEncoding.DecodeString(m[2])
if err != nil {
logger.Printf("WARN: [MinerU] Failed to decode base64 image %s: %v", ipath, err)
continue
}
imgBytes = decoded
} else {
// raw base64 without data URI prefix
decoded, err := base64.StdEncoding.DecodeString(b64Str)
if err != nil {
logger.Printf("WARN: [MinerU] Failed to decode raw base64 image %s: %v", ipath, err)
continue
}
imgBytes = decoded
ext = strings.TrimPrefix(filepath.Ext(ipath), ".")
if ext == "" {
ext = "png"
}
}
mimeType := mime.TypeByExtension("." + ext)
if mimeType == "" {
mimeType = "image/png"
}
refs = append(refs, types.ImageRef{
Filename: ipath,
OriginalRef: originalRef,
MimeType: mimeType,
ImageData: imgBytes,
})
}
return refs, mdContent
}
// logResponseStructure recursively logs the structure of the API response,
// truncating large string values to reveal all available fields.
func (c *MinerUReader) logResponseStructure(obj interface{}, prefix string) {
switch v := obj.(type) {
case map[string]interface{}:
for key, val := range v {
path := prefix + "." + key
if path[0] == '.' {
path = path[1:]
}
switch inner := val.(type) {
case map[string]interface{}:
logger.Printf("DEBUG: [MinerU] Response field: %s = {object with %d keys: %s}", path, len(inner), mapKeys(inner))
c.logResponseStructure(inner, path)
case []interface{}:
logger.Printf("DEBUG: [MinerU] Response field: %s = [array with %d items]", path, len(inner))
if len(inner) > 0 {
logger.Printf("DEBUG: [MinerU] Response field: %s[0] type=%T", path, inner[0])
if len(inner) <= 3 {
for i, item := range inner {
c.logResponseStructure(item, fmt.Sprintf("%s[%d]", path, i))
}
} else {
c.logResponseStructure(inner[0], path+"[0]")
logger.Printf("DEBUG: [MinerU] ... and %d more items in %s", len(inner)-1, path)
}
}
case string:
if len(inner) > 200 {
logger.Printf("DEBUG: [MinerU] Response field: %s = string(%d chars): %.200s...", path, len(inner), inner)
} else {
logger.Printf("DEBUG: [MinerU] Response field: %s = %q", path, inner)
}
case float64:
logger.Printf("DEBUG: [MinerU] Response field: %s = %v (number)", path, inner)
case bool:
logger.Printf("DEBUG: [MinerU] Response field: %s = %v (bool)", path, inner)
case nil:
logger.Printf("DEBUG: [MinerU] Response field: %s = null", path)
default:
logger.Printf("DEBUG: [MinerU] Response field: %s = %v (%T)", path, inner, inner)
}
}
default:
logger.Printf("DEBUG: [MinerU] Value at %s: %v (%T)", prefix, v, v)
}
}
func mapKeys(m map[string]interface{}) string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, k)
}
return strings.Join(keys, ", ")
}
// PingMinerU checks if the self-hosted MinerU service is reachable.
func PingMinerU(endpoint string) (bool, string) {
endpoint = strings.TrimRight(endpoint, "/")
if endpoint == "" {
return false, "未配置 MinerU 端点"
}
client := &http.Client{Timeout: 5 * time.Second}
resp, err := client.Get(endpoint + "/docs")
if err != nil {
return false, fmt.Sprintf("MinerU 服务不可达: %v", err)
}
resp.Body.Close()
if resp.StatusCode >= 400 {
return false, fmt.Sprintf("MinerU 服务返回状态 %d", resp.StatusCode)
}
return true, ""
}
// htmlToMarkdown converts HTML content to markdown.
// Falls back to the original content if conversion fails.
func htmlToMarkdown(content string) string {
md, err := htmltomd.ConvertString(content)
if err != nil {
logger.Printf("WARN: [MinerU] html-to-markdown conversion failed, using raw content: %v", err)
return content
}
return md
}
// --- helpers ---
func stringOr(val, fallback string) string {
val = strings.TrimSpace(val)
if val == "" {
return fallback
}
return val
}
func parseBoolOr(val string, fallback bool) bool {
val = strings.ToLower(strings.TrimSpace(val))
if val == "" {
return fallback
}
return val == "true" || val == "1" || val == "yes"
}