Files
Enterprise-Knowledge-Base/internal/infrastructure/docparser/mineru_converter.go
T
MidnightSun 6210f44fb6 fix(docparser): preserve MinerU markdown and persist relative images
MinerU already returns markdown with embedded HTML blocks, but the current\nreader runs the whole document back through html-to-markdown. That\nsecond conversion escapes valid headings and image syntax, so chunk\nprofiling sees plain text instead of markdown structure and relative\nimage references stop matching the storage pipeline.\n\nKeep MinerU output in its original markdown form and only apply narrow\ncompatibility normalization for the specific over-escaped patterns we\nactually need to recover. The converter now matches image refs by the\npaths that are really present in markdown or embedded HTML instead of\nassuming a single images/<name> form.\n\nExtend ImageResolver so relative HTML <img src=...> references share the\nsame storage rewrite path as markdown images, deduplicate repeated saves,\nand keep the frontend sanitizer compatible with MinerU's details/summary\nblocks. Add focused docparser tests that cover escaped markdown repair,\nvariant image path matching, and relative HTML image persistence.
2026-05-21 11:39:57 +08:00

360 lines
11 KiB
Go

package docparser
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"mime"
"mime/multipart"
"net/http"
"net/url"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
const mineruTimeout = 1000 * time.Second // large docs can take a while
var b64DataURIPattern = regexp.MustCompile(`^data:image/(\w+);base64,(.+)$`)
// MinerUReader calls a self-hosted MinerU API to read/convert documents.
type MinerUReader struct {
endpoint string
backend string // "pipeline", "vlm-*", "hybrid-*"
vlmServerURL string // vLLM server URL for vlm-http-client / hybrid-http-client
formulaEnable bool
tableEnable bool
ocrEnable bool
language string
}
// NewMinerUReader creates a reader from ParserEngineOverrides.
func NewMinerUReader(overrides map[string]string) *MinerUReader {
c := &MinerUReader{
endpoint: strings.TrimRight(overrides["mineru_endpoint"], "/"),
backend: stringOr(overrides["mineru_model"], "pipeline"),
vlmServerURL: overrides["mineru_vlm_server_url"],
formulaEnable: parseBoolOr(overrides["mineru_enable_formula"], true),
tableEnable: parseBoolOr(overrides["mineru_enable_table"], true),
ocrEnable: parseBoolOr(overrides["mineru_enable_ocr"], true),
language: stringOr(overrides["mineru_language"], "ch"),
}
return c
}
func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
if c.endpoint == "" {
return &types.ReadResult{Error: "MinerU endpoint is not configured"}, nil
}
content := req.FileContent
if len(content) == 0 {
return &types.ReadResult{Error: "no file content provided"}, nil
}
logger.Infof(context.Background(), "[MinerU] Parsing file=%s size=%d via %s", req.FileName, len(content), c.endpoint)
mdContent, imagesB64, err := c.callFileParse(ctx, content)
if err != nil {
return nil, fmt.Errorf("MinerU file_parse: %w", err)
}
// MinerU already returns markdown with embedded HTML blocks (e.g. <table>, <details>).
// Re-running the whole document through html-to-markdown corrupts valid markdown
// by escaping headings and image syntax. Only apply narrow compatibility fixes.
mdContent = normalizeMinerUMarkdown(mdContent)
// Process images: decode base64, build ImageRef list, replace refs in markdown
imageRefs, mdContent := c.processImages(mdContent, imagesB64)
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)
logger.Infof(context.Background(), "[MinerU] Parsed successfully, markdown=%d chars, images=%d", len(mdContent), len(imageRefs))
return &types.ReadResult{
MarkdownContent: mdContent,
ImageRefs: imageRefs,
}, nil
}
// mineruFileParseResponse mirrors the relevant fields from the MinerU API response.
type mineruFileParseResponse struct {
Results struct {
Document struct {
MDContent string `json:"md_content"`
Images map[string]string `json:"images"` // path -> "data:image/png;base64,..." or raw base64
} `json:"document"`
Files struct {
MDContent string `json:"md_content"`
Images map[string]string `json:"images"` // path -> "data:image/png;base64,..." or raw base64
} `json:"files"`
} `json:"results"`
}
func (c *MinerUReader) callFileParse(ctx context.Context, content []byte) (string, map[string]string, error) {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
// Form fields
fields := map[string]string{
"return_md": "true",
"return_images": "true",
"table_enable": fmt.Sprintf("%v", c.tableEnable),
"formula_enable": fmt.Sprintf("%v", c.formulaEnable),
"parse_method": "ocr",
"start_page_id": "0",
"end_page_id": "99999",
"backend": c.backend,
"response_format_zip": "false",
"return_middle_json": "false",
"return_model_output": "false",
"return_content_list": "true",
}
if !c.ocrEnable {
fields["parse_method"] = "txt"
}
if c.language != "" {
fields["lang_list"] = c.language
}
if c.vlmServerURL != "" && (strings.HasPrefix(c.backend, "vlm-http-client") || strings.HasPrefix(c.backend, "hybrid-http-client")) {
fields["server_url"] = c.vlmServerURL
}
for k, v := range fields {
_ = writer.WriteField(k, v)
}
// File part
part, err := writer.CreateFormFile("files", "document")
if err != nil {
return "", nil, fmt.Errorf("create form file: %w", err)
}
if _, err := part.Write(content); err != nil {
return "", nil, fmt.Errorf("write file content: %w", err)
}
writer.Close()
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint+"/file_parse", &body)
if err != nil {
return "", nil, fmt.Errorf("create request: %w", err)
}
httpReq.Header.Set("Content-Type", writer.FormDataContentType())
client := &http.Client{Timeout: mineruTimeout}
resp, err := client.Do(httpReq)
if err != nil {
return "", nil, fmt.Errorf("HTTP request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
respBody, _ := io.ReadAll(resp.Body)
return "", nil, fmt.Errorf("MinerU API status %d: %s", resp.StatusCode, string(respBody))
}
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return "", nil, fmt.Errorf("read response body: %w", err)
}
// Dump raw response for debugging (truncate if too large)
rawStr := string(respBody)
if len(rawStr) > 4000 {
logger.Infof(context.Background(), "[MinerU] Raw response (truncated to 4000 chars): %s ...", rawStr[:4000])
} else {
logger.Infof(context.Background(), "[MinerU] Raw response: %s", rawStr)
}
// Also pretty-print the top-level structure (without large base64 blobs)
var rawMap map[string]interface{}
if err := json.Unmarshal(respBody, &rawMap); err == nil {
c.logMinerUResponseStructure(rawMap, "")
}
var result mineruFileParseResponse
if err := json.Unmarshal(respBody, &result); err != nil {
return "", nil, fmt.Errorf("decode response: %w", err)
}
// MinerU response schema differs by version/deployment:
// - older/self-hosted variants: results.document.*
// - some variants: results.files.*
// Prefer document when available, then fallback to files.
if result.Results.Document.MDContent != "" || len(result.Results.Document.Images) > 0 {
logger.Infof(context.Background(), "[MinerU] Using response path: results.document")
return result.Results.Document.MDContent, result.Results.Document.Images, nil
}
if result.Results.Files.MDContent != "" || len(result.Results.Files.Images) > 0 {
logger.Infof(context.Background(), "[MinerU] Using response path: results.files")
return result.Results.Files.MDContent, result.Results.Files.Images, nil
}
logger.Errorf(context.Background(), "[MinerU] Response has no markdown/images under results.document or results.files")
return "", nil, nil
}
// processImages decodes base64 images from MinerU response and returns ImageRef list.
// It also replaces image references in the markdown content.
func (c *MinerUReader) processImages(mdContent string, imagesB64 map[string]string) ([]types.ImageRef, string) {
var refs []types.ImageRef
for ipath, b64Str := range imagesB64 {
matchedRefs := mineruImageOriginalRefs(mdContent, ipath)
if len(matchedRefs) == 0 {
continue
}
var imgBytes []byte
var ext string
if m := b64DataURIPattern.FindStringSubmatch(b64Str); len(m) == 3 {
ext = m[1]
decoded, err := base64.StdEncoding.DecodeString(m[2])
if err != nil {
logger.Errorf(context.Background(), "[MinerU] Failed to decode base64 image %s: %v", ipath, err)
continue
}
imgBytes = decoded
} else {
// raw base64 without data URI prefix
decoded, err := base64.StdEncoding.DecodeString(b64Str)
if err != nil {
logger.Errorf(context.Background(), "[MinerU] Failed to decode raw base64 image %s: %v", ipath, err)
continue
}
imgBytes = decoded
ext = strings.TrimPrefix(filepath.Ext(ipath), ".")
if ext == "" {
ext = "png"
}
}
mimeType := mime.TypeByExtension("." + ext)
if mimeType == "" {
mimeType = "image/png"
}
for _, originalRef := range matchedRefs {
refs = append(refs, types.ImageRef{
Filename: ipath,
OriginalRef: originalRef,
MimeType: mimeType,
ImageData: imgBytes,
})
}
}
return refs, mdContent
}
// logMinerUResponseStructure logs the structure of the MinerU API response.
func (c *MinerUReader) logMinerUResponseStructure(obj interface{}, prefix string) {
logResponseStructure("MinerU", obj, prefix)
}
// PingMinerU checks if the self-hosted MinerU service is reachable.
func PingMinerU(endpoint string) (bool, string) {
endpoint = strings.TrimRight(endpoint, "/")
if endpoint == "" {
return false, "未配置 MinerU 端点"
}
client := &http.Client{Timeout: 5 * time.Second}
resp, err := client.Get(endpoint + "/docs")
if err != nil {
return false, fmt.Sprintf("MinerU 服务不可达: %v", err)
}
resp.Body.Close()
if resp.StatusCode >= 400 {
return false, fmt.Sprintf("MinerU 服务返回状态 %d", resp.StatusCode)
}
return true, ""
}
// escapedImageSyntaxPattern matches markdown image references whose `[` was
// over-escaped to `\[` by html-to-markdown. The URL group mirrors the
// downstream image-extraction regex so escapes are only stripped for actual
// image references.
var escapedImageSyntaxPattern = regexp.MustCompile(`!\\\[(.*?)\\?\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`)
// escapedHeadingPattern restores markdown headings that were over-escaped to
// \# Heading. We only touch line-leading heading markers to avoid rewriting
// body text that intentionally contains escaped # characters.
var escapedHeadingPattern = regexp.MustCompile(`(?m)^\\(#{1,6})(\s+)`)
// unescapeMarkdownImageSyntax restores `![alt](url)` from html-to-markdown's
// over-escaped `!\[alt\](url)` form. Without this, the downstream image regex
// in ImageResolver fails to match and images are never persisted.
func unescapeMarkdownImageSyntax(content string) string {
return escapedImageSyntaxPattern.ReplaceAllString(content, "![$1]($2)")
}
func normalizeEscapedMarkdownHeadings(content string) string {
return escapedHeadingPattern.ReplaceAllString(content, `$1$2`)
}
func normalizeMinerUMarkdown(content string) string {
content = unescapeMarkdownImageSyntax(content)
content = normalizeEscapedMarkdownHeadings(content)
return content
}
func mineruImageOriginalRefs(mdContent, imagePath string) []string {
normalizedTarget := normalizeMinerUImagePath(imagePath)
if normalizedTarget == "" {
return nil
}
referenced := extractImageRefsFromContent(mdContent)
seen := make(map[string]struct{}, len(referenced))
var matched []string
for _, refPath := range referenced {
if normalizeMinerUImagePath(refPath) != normalizedTarget {
continue
}
if _, ok := seen[refPath]; ok {
continue
}
matched = append(matched, refPath)
seen[refPath] = struct{}{}
}
return matched
}
func extractImageRefsFromContent(content string) []string {
var refs []string
for _, match := range imgMarkdownPattern.FindAllStringSubmatch(content, -1) {
if len(match) >= 3 {
refs = append(refs, match[2])
}
}
for _, match := range imgHTMLRelativeSrc.FindAllStringSubmatch(content, -1) {
if len(match) >= 3 {
refs = append(refs, match[2])
}
}
return refs
}
func normalizeMinerUImagePath(p string) string {
p = strings.TrimSpace(p)
if p == "" {
return ""
}
if decoded, err := url.PathUnescape(p); err == nil && decoded != "" {
p = decoded
}
p = strings.TrimPrefix(p, "./")
p = strings.TrimPrefix(p, "/")
p = strings.TrimPrefix(p, "images/")
return p
}