feat(image): enhance image resolution handling for data URIs and remote images

- Added functionality to resolve embedded data:image;base64 images in markdown, storing them and replacing references with stable URLs.
- Updated the knowledge service to process data URIs before handling remote images, ensuring all images are resolved consistently.
- Improved logging for resolved images, providing better insights during manual processing.
- Added unit tests to validate the new data URI resolution functionality and ensure reliability.
This commit is contained in:
wizardchen
2026-03-20 18:53:16 +08:00
committed by lyingbug
parent 9650188a35
commit 0d4ceb6c96
3 changed files with 187 additions and 4 deletions
+9 -3
View File
@@ -6803,11 +6803,17 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
return
}
// Resolve remote images: download http(s) images, upload to storage, replace URLs.
// This runs before chunking so that chunks contain stable provider:// URLs.
// Resolve embedded data:base64 images and remote http(s) images → storage, replace URLs.
// Runs before chunking so chunks contain stable provider:// URLs.
var resolvedImages []docparser.StoredImage
if s.imageResolver != nil {
fileSvc := s.resolveFileService(ctx, kb)
afterDataURI, fromDataURI, _ := s.imageResolver.ResolveDataURIImages(ctx, clean, fileSvc, knowledge.TenantID)
if len(fromDataURI) > 0 {
logger.Infof(ctx, "Resolved %d data-URI images for manual knowledge %s", len(fromDataURI), knowledge.ID)
clean = afterDataURI
resolvedImages = append(resolvedImages, fromDataURI...)
}
updatedContent, storedImages, resolveErr := s.imageResolver.ResolveRemoteImages(ctx, clean, fileSvc, knowledge.TenantID)
if resolveErr != nil {
logger.Warnf(ctx, "Remote image resolution partially failed: %v", resolveErr)
@@ -6815,7 +6821,7 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
if len(storedImages) > 0 {
logger.Infof(ctx, "Resolved %d remote images for manual knowledge %s", len(storedImages), knowledge.ID)
clean = updatedContent
resolvedImages = storedImages
resolvedImages = append(resolvedImages, storedImages...)
}
}
@@ -3,6 +3,7 @@ package docparser
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"image"
_ "image/gif"
@@ -78,8 +79,12 @@ func (r *ImageResolver) ResolveAndStore(
tenantID uint64,
) (updatedMarkdown string, images []StoredImage, err error) {
markdown := UnwrapLinkedImages(result.MarkdownContent)
md2, imgDataURIs, _ := r.ResolveDataURIImages(ctx, markdown, fileSvc, tenantID)
markdown = md2
images = append(images, imgDataURIs...)
if len(result.ImageRefs) == 0 {
return markdown, nil, nil
return markdown, images, nil
}
// Build a map of original_ref -> image ref for fast lookup
@@ -160,6 +165,8 @@ func extFromMime(mime string) string {
return ".webp"
case "image/bmp":
return ".bmp"
case "image/svg+xml":
return ".svg"
default:
return ""
}
@@ -210,6 +217,103 @@ func UnwrapLinkedImages(markdown string) string {
// like https://example.com/item_(abc)/123 are captured in full.
var imgMarkdownPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
// imgMarkdownDataURI matches markdown images whose URL is a data:image/*;base64,...
// payload. (?i) applies to the whole parenthesized data URI.
var imgMarkdownDataURI = regexp.MustCompile(
`!\[([^\]]*)\]\((?i:(data:image/[^;]+;base64,\s*[^)]+))\)`,
)
// parseImageDataURI splits a data URI into image MIME type and base64 payload.
func parseImageDataURI(dataURI string) (mimeType string, b64Payload string, ok bool) {
const sep = ";base64,"
idx := strings.Index(strings.ToLower(dataURI), sep)
if idx < 0 {
return "", "", false
}
meta := strings.TrimSpace(dataURI[:idx])
const prefix = "data:image/"
if len(meta) < len(prefix) || !strings.EqualFold(meta[:len(prefix)], prefix) {
return "", "", false
}
sub := strings.TrimSpace(meta[len(prefix):])
mimeType = "image/" + strings.ToLower(sub)
b64Payload = strings.TrimSpace(dataURI[idx+len(sep):])
if b64Payload == "" {
return "", "", false
}
return mimeType, b64Payload, true
}
// ResolveDataURIImages finds embedded data:image/*;base64 images in markdown,
// decodes them, stores via fileSvc, and replaces each reference with the returned
// provider URL (same limits as remote images: count and decoded size).
func (r *ImageResolver) ResolveDataURIImages(
ctx context.Context,
markdown string,
fileSvc interfaces.FileService,
tenantID uint64,
) (updatedMarkdown string, images []StoredImage, err error) {
markdown = UnwrapLinkedImages(markdown)
matches := imgMarkdownDataURI.FindAllStringSubmatchIndex(markdown, -1)
if len(matches) == 0 {
return markdown, nil, nil
}
processed := 0
for i := len(matches) - 1; i >= 0; i-- {
if processed >= maxRemoteImages {
break
}
m := matches[i]
if len(m) < 6 {
continue
}
dataURI := markdown[m[4]:m[5]]
mimeType, payload, ok := parseImageDataURI(dataURI)
if !ok {
continue
}
payload = strings.ReplaceAll(payload, "\n", "")
payload = strings.ReplaceAll(payload, "\r", "")
payload = strings.ReplaceAll(payload, "\t", "")
payload = strings.ReplaceAll(payload, " ", "")
if payload == "" {
continue
}
data, decErr := base64.StdEncoding.DecodeString(payload)
if decErr != nil {
log.Printf("WARN: data URI base64 decode failed: %v", decErr)
continue
}
if len(data) > maxRemoteImageSize {
log.Printf("WARN: data URI image exceeds size limit (%d bytes)", maxRemoteImageSize)
continue
}
if isIconImage(data) {
markdown = markdown[:m[0]] + markdown[m[1]:]
continue
}
ext := extFromMime(mimeType)
if ext == "" {
ext = ".png"
}
fileName := uuid.New().String() + ext
servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false)
if saveErr != nil {
log.Printf("WARN: failed to save data URI image: %v", saveErr)
continue
}
images = append(images, StoredImage{
OriginalRef: dataURI,
ServingURL: servingURL,
MimeType: mimeType,
})
markdown = markdown[:m[4]] + servingURL + markdown[m[5]:]
processed++
}
return markdown, images, nil
}
// ResolveRemoteImages scans a Markdown string for image references whose URL
// is http:// or https://, downloads each one through an SSRF-safe HTTP client,
// uploads the bytes via fileSvc, and replaces the original URL with the
@@ -2,10 +2,18 @@ package docparser
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"image"
"image/color"
"image/png"
"io"
"mime/multipart"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// createTestPNG generates a minimal PNG image with the given dimensions.
@@ -67,3 +75,68 @@ func TestIsIconImage(t *testing.T) {
})
}
}
// captureSaveBytes implements interfaces.FileService for tests; only SaveBytes records data.
type captureSaveBytes struct {
saved [][]byte
urls []string
}
func (c *captureSaveBytes) CheckConnectivity(context.Context) error { return nil }
func (c *captureSaveBytes) SaveFile(context.Context, *multipart.FileHeader, uint64, string) (string, error) {
return "", fmt.Errorf("not implemented")
}
func (c *captureSaveBytes) SaveBytes(_ context.Context, data []byte, _ uint64, fileName string, _ bool) (string, error) {
c.saved = append(c.saved, append([]byte(nil), data...))
u := "local://test/" + fileName
c.urls = append(c.urls, u)
return u, nil
}
func (c *captureSaveBytes) GetFile(context.Context, string) (io.ReadCloser, error) {
return nil, fmt.Errorf("not implemented")
}
func (c *captureSaveBytes) GetFileURL(context.Context, string) (string, error) { return "", nil }
func (c *captureSaveBytes) DeleteFile(context.Context, string) error { return nil }
var _ interfaces.FileService = (*captureSaveBytes)(nil)
func TestResolveDataURIImages(t *testing.T) {
png := createTestPNG(200, 150)
b64 := base64.StdEncoding.EncodeToString(png)
md := "pre ![](data:image/png;base64," + b64 + ") post"
svc := &captureSaveBytes{}
r := NewImageResolver()
out, imgs, err := r.ResolveDataURIImages(context.Background(), md, svc, 1)
if err != nil {
t.Fatal(err)
}
if len(imgs) != 1 {
t.Fatalf("got %d images", len(imgs))
}
if len(svc.saved) != 1 || !bytes.Equal(svc.saved[0], png) {
t.Fatal("SaveBytes payload mismatch")
}
if !strings.Contains(out, "local://test/") || strings.Contains(out, "data:image") {
t.Fatalf("markdown: %s", out)
}
}
func TestResolveDataURIImages_CaseInsensitive(t *testing.T) {
png := createTestPNG(200, 150)
b64 := base64.StdEncoding.EncodeToString(png)
md := "![](DATA:image/png;BASE64," + b64 + ")"
svc := &captureSaveBytes{}
r := NewImageResolver()
out, imgs, err := r.ResolveDataURIImages(context.Background(), md, svc, 1)
if err != nil {
t.Fatal(err)
}
if len(imgs) != 1 || !strings.Contains(out, "local://test/") {
t.Fatalf("imgs=%d out=%q", len(imgs), out)
}
}