feat(url): 支持docreader不上传替换白名单url的图片

This commit is contained in:
Li Xianggang
2026-05-18 17:25:22 +08:00
committed by lyingbug
parent b3dbf86bf5
commit c19d3543c8
4 changed files with 70 additions and 12 deletions
@@ -13,6 +13,8 @@ import (
"log"
"mime"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
@@ -195,6 +197,35 @@ func isProviderScheme(p string) bool {
return false
}
// isWhitelistedImageHost checks if the image URL's host is in the whitelist.
// Whitelisted hosts are trusted (e.g. internal MinerU service) — images are
// still downloaded for validation and OCR/caption analysis, but not uploaded
// to object storage. The markdown keeps the original URL.
// Configure via IMAGE_HOST_KEEP_URL env var (comma-separated hosts).
func isWhitelistedImageHost(rawURL string) bool {
whitelist := strings.TrimSpace(os.Getenv("IMAGE_HOST_KEEP_URL"))
if whitelist == "" {
return false
}
u, err := url.Parse(rawURL)
if err != nil || u.Host == "" {
return false
}
host := strings.ToLower(u.Host)
hostname := strings.ToLower(u.Hostname())
for _, h := range strings.Split(whitelist, ",") {
h = strings.ToLower(strings.TrimSpace(h))
if h == "" {
continue
}
// Exact host match (includes port) or hostname match (any port)
if host == h || hostname == h {
return true
}
}
return false
}
// ---------------------------------------------------------------------------
// Helper functions for base64 image handling
// ---------------------------------------------------------------------------
@@ -670,10 +701,18 @@ func (r *ImageResolver) ResolveRemoteImages(
continue
}
// --- SSRF check (centralised entry-point with whitelist support) ---
if err := secutils.ValidateURLForSSRF(imgURL); err != nil {
log.Printf("WARN: remote image blocked by SSRF check (%v): %s", err, imgURL)
continue
// For whitelisted hosts: download to validate (mime type, icon check),
// create StoredImage for downstream OCR/caption analysis, but do NOT
// upload to storage and keep the original URL in markdown.
// The multimodal service will download from the original URL later.
whitelisted := isWhitelistedImageHost(imgURL)
// --- SSRF check (skip for whitelisted) ---
if !whitelisted {
if err := secutils.ValidateURLForSSRF(imgURL); err != nil {
log.Printf("WARN: remote image blocked by SSRF check (%v): %s", err, imgURL)
continue
}
}
// --- Download ---
@@ -697,12 +736,20 @@ func (r *ImageResolver) ResolveRemoteImages(
ext = ".png" // safe default
}
// --- Upload to storage ---
fileName := uuid.New().String() + ext
servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false)
if saveErr != nil {
log.Printf("WARN: failed to save remote image %s: %v", imgURL, saveErr)
continue
var servingURL string
if whitelisted {
// Keep the original URL — ImageMultimodalService will download it
// directly for OCR/caption analysis.
servingURL = imgURL
} else {
// --- Upload to storage ---
fileName := uuid.New().String() + ext
var saveErr error
servingURL, saveErr = fileSvc.SaveBytes(ctx, data, tenantID, fileName, false)
if saveErr != nil {
log.Printf("WARN: failed to save remote image %s: %v", imgURL, saveErr)
continue
}
}
images = append(images, StoredImage{
@@ -711,8 +758,10 @@ func (r *ImageResolver) ResolveRemoteImages(
MimeType: mimeType,
})
// Replace URL in markdown.
markdown = markdown[:m[4]] + servingURL + markdown[m[5]:]
if !whitelisted {
// Replace URL in markdown.
markdown = markdown[:m[4]] + servingURL + markdown[m[5]:]
}
processed++
}