7099a7e181
Stop returning bot credentials from Viewer-accessible IM channel list endpoints and validate self-hosted MinerU URLs at config save and fetch time.
398 lines
13 KiB
Go
398 lines
13 KiB
Go
package docparser
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"mime"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"net/url"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/utils"
|
|
)
|
|
|
|
const mineruTimeout = 1000 * time.Second // large docs can take a while
|
|
|
|
var b64DataURIPattern = regexp.MustCompile(`^data:image/(\w+);base64,(.+)$`)
|
|
|
|
// MinerUReader calls a self-hosted MinerU API to read/convert documents.
|
|
type MinerUReader struct {
|
|
endpoint string
|
|
backend string // "pipeline", "vlm-*", "hybrid-*"
|
|
vlmServerURL string // vLLM server URL for vlm-http-client / hybrid-http-client
|
|
formulaEnable bool
|
|
tableEnable bool
|
|
ocrEnable bool
|
|
language string
|
|
}
|
|
|
|
// NewMinerUReader creates a reader from ParserEngineOverrides.
|
|
func NewMinerUReader(overrides map[string]string) *MinerUReader {
|
|
c := &MinerUReader{
|
|
endpoint: strings.TrimRight(overrides["mineru_endpoint"], "/"),
|
|
backend: stringOr(overrides["mineru_model"], "pipeline"),
|
|
vlmServerURL: overrides["mineru_vlm_server_url"],
|
|
formulaEnable: parseBoolOr(overrides["mineru_enable_formula"], true),
|
|
tableEnable: parseBoolOr(overrides["mineru_enable_table"], true),
|
|
ocrEnable: parseBoolOr(overrides["mineru_enable_ocr"], true),
|
|
language: stringOr(overrides["mineru_language"], "ch"),
|
|
}
|
|
return c
|
|
}
|
|
|
|
func (c *MinerUReader) Read(ctx context.Context, req *types.ReadRequest) (*types.ReadResult, error) {
|
|
if c.endpoint == "" {
|
|
return &types.ReadResult{Error: "MinerU endpoint is not configured"}, nil
|
|
}
|
|
if err := validateMinerUOutboundURL(c.endpoint); err != nil {
|
|
return &types.ReadResult{Error: err.Error()}, nil
|
|
}
|
|
if c.vlmServerURL != "" {
|
|
if err := validateMinerUOutboundURL(c.vlmServerURL); err != nil {
|
|
return &types.ReadResult{Error: err.Error()}, nil
|
|
}
|
|
}
|
|
|
|
content := req.FileContent
|
|
if len(content) == 0 {
|
|
return &types.ReadResult{Error: "no file content provided"}, nil
|
|
}
|
|
|
|
logger.Infof(context.Background(), "[MinerU] Parsing file=%s size=%d via %s", req.FileName, len(content), c.endpoint)
|
|
|
|
mdContent, imagesB64, err := c.callFileParse(ctx, content)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("MinerU file_parse: %w", err)
|
|
}
|
|
|
|
// MinerU already returns markdown with embedded HTML blocks (e.g. <table>, <details>).
|
|
// Re-running the whole document through html-to-markdown corrupts valid markdown
|
|
// by escaping headings and image syntax. Only apply narrow compatibility fixes.
|
|
mdContent = normalizeMinerUMarkdown(mdContent)
|
|
|
|
// Process images: decode base64, build ImageRef list, replace refs in markdown
|
|
imageRefs, mdContent := c.processImages(mdContent, imagesB64)
|
|
|
|
mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)
|
|
|
|
logger.Infof(context.Background(), "[MinerU] Parsed successfully, markdown=%d chars, images=%d", len(mdContent), len(imageRefs))
|
|
|
|
return &types.ReadResult{
|
|
MarkdownContent: mdContent,
|
|
ImageRefs: imageRefs,
|
|
}, nil
|
|
}
|
|
|
|
// mineruFileParseResponse mirrors the relevant fields from the MinerU API response.
|
|
type mineruFileParseResponse struct {
|
|
Results struct {
|
|
Document struct {
|
|
MDContent string `json:"md_content"`
|
|
Images map[string]string `json:"images"` // path -> "data:image/png;base64,..." or raw base64
|
|
} `json:"document"`
|
|
Files struct {
|
|
MDContent string `json:"md_content"`
|
|
Images map[string]string `json:"images"` // path -> "data:image/png;base64,..." or raw base64
|
|
} `json:"files"`
|
|
} `json:"results"`
|
|
}
|
|
|
|
func (c *MinerUReader) callFileParse(ctx context.Context, content []byte) (string, map[string]string, error) {
|
|
var body bytes.Buffer
|
|
writer := multipart.NewWriter(&body)
|
|
|
|
// Form fields
|
|
fields := map[string]string{
|
|
"return_md": "true",
|
|
"return_images": "true",
|
|
"table_enable": fmt.Sprintf("%v", c.tableEnable),
|
|
"formula_enable": fmt.Sprintf("%v", c.formulaEnable),
|
|
"parse_method": "ocr",
|
|
"start_page_id": "0",
|
|
"end_page_id": "99999",
|
|
"backend": c.backend,
|
|
"response_format_zip": "false",
|
|
"return_middle_json": "false",
|
|
"return_model_output": "false",
|
|
"return_content_list": "true",
|
|
}
|
|
if !c.ocrEnable {
|
|
fields["parse_method"] = "txt"
|
|
}
|
|
if c.language != "" {
|
|
fields["lang_list"] = c.language
|
|
}
|
|
if c.vlmServerURL != "" && (strings.HasPrefix(c.backend, "vlm-http-client") || strings.HasPrefix(c.backend, "hybrid-http-client")) {
|
|
fields["server_url"] = c.vlmServerURL
|
|
}
|
|
for k, v := range fields {
|
|
_ = writer.WriteField(k, v)
|
|
}
|
|
|
|
// File part
|
|
part, err := writer.CreateFormFile("files", "document")
|
|
if err != nil {
|
|
return "", nil, fmt.Errorf("create form file: %w", err)
|
|
}
|
|
if _, err := part.Write(content); err != nil {
|
|
return "", nil, fmt.Errorf("write file content: %w", err)
|
|
}
|
|
writer.Close()
|
|
|
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.endpoint+"/file_parse", &body)
|
|
if err != nil {
|
|
return "", nil, fmt.Errorf("create request: %w", err)
|
|
}
|
|
httpReq.Header.Set("Content-Type", writer.FormDataContentType())
|
|
|
|
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
|
|
Timeout: mineruTimeout,
|
|
MaxRedirects: 5,
|
|
})
|
|
resp, err := client.Do(httpReq)
|
|
if err != nil {
|
|
return "", nil, fmt.Errorf("HTTP request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
respBody, _ := io.ReadAll(resp.Body)
|
|
return "", nil, fmt.Errorf("MinerU API status %d: %s", resp.StatusCode, string(respBody))
|
|
}
|
|
|
|
respBody, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return "", nil, fmt.Errorf("read response body: %w", err)
|
|
}
|
|
|
|
// Dump raw response for debugging (truncate if too large)
|
|
rawStr := string(respBody)
|
|
if len(rawStr) > 4000 {
|
|
logger.Infof(context.Background(), "[MinerU] Raw response (truncated to 4000 chars): %s ...", rawStr[:4000])
|
|
} else {
|
|
logger.Infof(context.Background(), "[MinerU] Raw response: %s", rawStr)
|
|
}
|
|
|
|
// Also pretty-print the top-level structure (without large base64 blobs)
|
|
var rawMap map[string]interface{}
|
|
if err := json.Unmarshal(respBody, &rawMap); err == nil {
|
|
c.logMinerUResponseStructure(rawMap, "")
|
|
}
|
|
|
|
var result mineruFileParseResponse
|
|
if err := json.Unmarshal(respBody, &result); err != nil {
|
|
return "", nil, fmt.Errorf("decode response: %w", err)
|
|
}
|
|
|
|
// MinerU response schema differs by version/deployment:
|
|
// - older/self-hosted variants: results.document.*
|
|
// - some variants: results.files.*
|
|
// Prefer document when available, then fallback to files.
|
|
if result.Results.Document.MDContent != "" || len(result.Results.Document.Images) > 0 {
|
|
logger.Infof(context.Background(), "[MinerU] Using response path: results.document")
|
|
return result.Results.Document.MDContent, result.Results.Document.Images, nil
|
|
}
|
|
if result.Results.Files.MDContent != "" || len(result.Results.Files.Images) > 0 {
|
|
logger.Infof(context.Background(), "[MinerU] Using response path: results.files")
|
|
return result.Results.Files.MDContent, result.Results.Files.Images, nil
|
|
}
|
|
|
|
logger.Errorf(context.Background(), "[MinerU] Response has no markdown/images under results.document or results.files")
|
|
return "", nil, nil
|
|
}
|
|
|
|
// processImages decodes base64 images from MinerU response and returns ImageRef list.
|
|
// It also replaces image references in the markdown content.
|
|
func (c *MinerUReader) processImages(mdContent string, imagesB64 map[string]string) ([]types.ImageRef, string) {
|
|
var refs []types.ImageRef
|
|
|
|
for ipath, b64Str := range imagesB64 {
|
|
matchedRefs := mineruImageOriginalRefs(mdContent, ipath)
|
|
if len(matchedRefs) == 0 {
|
|
continue
|
|
}
|
|
|
|
var imgBytes []byte
|
|
var ext string
|
|
|
|
if m := b64DataURIPattern.FindStringSubmatch(b64Str); len(m) == 3 {
|
|
ext = m[1]
|
|
decoded, err := base64.StdEncoding.DecodeString(m[2])
|
|
if err != nil {
|
|
logger.Errorf(context.Background(), "[MinerU] Failed to decode base64 image %s: %v", ipath, err)
|
|
continue
|
|
}
|
|
imgBytes = decoded
|
|
} else {
|
|
// raw base64 without data URI prefix
|
|
decoded, err := base64.StdEncoding.DecodeString(b64Str)
|
|
if err != nil {
|
|
logger.Errorf(context.Background(), "[MinerU] Failed to decode raw base64 image %s: %v", ipath, err)
|
|
continue
|
|
}
|
|
imgBytes = decoded
|
|
ext = strings.TrimPrefix(filepath.Ext(ipath), ".")
|
|
if ext == "" {
|
|
ext = "png"
|
|
}
|
|
}
|
|
|
|
mimeType := mime.TypeByExtension("." + ext)
|
|
if mimeType == "" {
|
|
mimeType = "image/png"
|
|
}
|
|
|
|
for _, originalRef := range matchedRefs {
|
|
refs = append(refs, types.ImageRef{
|
|
Filename: ipath,
|
|
OriginalRef: originalRef,
|
|
MimeType: mimeType,
|
|
ImageData: imgBytes,
|
|
})
|
|
}
|
|
}
|
|
|
|
return refs, mdContent
|
|
}
|
|
|
|
// logMinerUResponseStructure logs the structure of the MinerU API response.
|
|
func (c *MinerUReader) logMinerUResponseStructure(obj interface{}, prefix string) {
|
|
logResponseStructure("MinerU", obj, prefix)
|
|
}
|
|
|
|
// validateMinerUOutboundURL rejects MinerU endpoints that would reach private
|
|
// or otherwise restricted hosts when parsed or probed from the app server.
|
|
func validateMinerUOutboundURL(rawURL string) error {
|
|
rawURL = strings.TrimSpace(rawURL)
|
|
if rawURL == "" {
|
|
return nil
|
|
}
|
|
if err := utils.ValidateURLForSSRF(rawURL); err != nil {
|
|
return fmt.Errorf("MinerU URL blocked by SSRF check: %v", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// PingMinerU checks if the self-hosted MinerU service is reachable.
|
|
func PingMinerU(endpoint string) (bool, string) {
|
|
endpoint = strings.TrimRight(endpoint, "/")
|
|
if endpoint == "" {
|
|
return false, "未配置 MinerU 端点"
|
|
}
|
|
if err := validateMinerUOutboundURL(endpoint); err != nil {
|
|
return false, err.Error()
|
|
}
|
|
client := utils.NewSSRFSafeHTTPClient(utils.SSRFSafeHTTPClientConfig{
|
|
Timeout: 5 * time.Second,
|
|
MaxRedirects: 5,
|
|
})
|
|
resp, err := client.Get(endpoint + "/docs")
|
|
if err != nil {
|
|
return false, fmt.Sprintf("MinerU 服务不可达: %v", err)
|
|
}
|
|
resp.Body.Close()
|
|
if resp.StatusCode >= 400 {
|
|
return false, fmt.Sprintf("MinerU 服务返回状态 %d", resp.StatusCode)
|
|
}
|
|
return true, ""
|
|
}
|
|
|
|
// escapedImageSyntaxPattern matches markdown image references whose `[` was
|
|
// over-escaped to `\[` by html-to-markdown. The URL group mirrors the
|
|
// downstream image-extraction regex so escapes are only stripped for actual
|
|
// image references.
|
|
var escapedImageSyntaxPattern = regexp.MustCompile(`!\\\[(.*?)\\?\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`)
|
|
|
|
// escapedHeadingPattern restores markdown headings that were over-escaped to
|
|
// \# Heading. We only touch line-leading heading markers to avoid rewriting
|
|
// body text that intentionally contains escaped # characters.
|
|
var escapedHeadingPattern = regexp.MustCompile(`(?m)^\\(#{1,6})(\s+)`)
|
|
|
|
// unescapeMarkdownImageSyntax restores `` from html-to-markdown's
|
|
// over-escaped `!\[alt\](url)` form. Without this, the downstream image regex
|
|
// in ImageResolver fails to match and images are never persisted.
|
|
func unescapeMarkdownImageSyntax(content string) string {
|
|
return escapedImageSyntaxPattern.ReplaceAllString(content, "")
|
|
}
|
|
|
|
func normalizeEscapedMarkdownHeadings(content string) string {
|
|
return escapedHeadingPattern.ReplaceAllString(content, `$1$2`)
|
|
}
|
|
|
|
func normalizeMinerUMarkdown(content string) string {
|
|
content = unescapeMarkdownImageSyntax(content)
|
|
content = normalizeEscapedMarkdownHeadings(content)
|
|
return content
|
|
}
|
|
|
|
func mineruImageOriginalRefs(mdContent, imagePath string) []string {
|
|
normalizedTarget := normalizeMinerUImagePath(imagePath)
|
|
if normalizedTarget == "" {
|
|
return nil
|
|
}
|
|
|
|
referenced := extractImageRefsFromContent(mdContent)
|
|
seen := make(map[string]struct{}, len(referenced))
|
|
var matched []string
|
|
for _, refPath := range referenced {
|
|
if normalizeMinerUImagePath(refPath) != normalizedTarget {
|
|
continue
|
|
}
|
|
if _, ok := seen[refPath]; ok {
|
|
continue
|
|
}
|
|
matched = append(matched, refPath)
|
|
seen[refPath] = struct{}{}
|
|
}
|
|
|
|
return matched
|
|
}
|
|
|
|
// imgMarkdownPatternAllowSpaces matches markdown image syntax while allowing
|
|
// spaces in the URL group, so that paths like "images/第 1 页.jpg" produced by
|
|
// MinerU on Chinese documents are still detected as image references.
|
|
var imgMarkdownPatternAllowSpaces = regexp.MustCompile(
|
|
`!\[(.*?)\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`,
|
|
)
|
|
|
|
func extractImageRefsFromContent(content string) []string {
|
|
var refs []string
|
|
|
|
for _, match := range imgMarkdownPatternAllowSpaces.FindAllStringSubmatch(content, -1) {
|
|
if len(match) >= 3 {
|
|
refs = append(refs, strings.TrimSpace(match[2]))
|
|
}
|
|
}
|
|
for _, match := range imgHTMLRelativeSrc.FindAllStringSubmatch(content, -1) {
|
|
if len(match) >= 3 {
|
|
refs = append(refs, match[2])
|
|
}
|
|
}
|
|
|
|
return refs
|
|
}
|
|
|
|
func normalizeMinerUImagePath(p string) string {
|
|
p = strings.TrimSpace(p)
|
|
if p == "" {
|
|
return ""
|
|
}
|
|
if decoded, err := url.PathUnescape(p); err == nil && decoded != "" {
|
|
p = decoded
|
|
}
|
|
p = strings.TrimPrefix(p, "./")
|
|
p = strings.TrimPrefix(p, "/")
|
|
p = strings.TrimPrefix(p, "images/")
|
|
return p
|
|
}
|