feat(image): enhance image resolution handling for data URIs and remote images
- Added functionality to resolve embedded data:image;base64 images in markdown, storing them and replacing references with stable URLs. - Updated the knowledge service to process data URIs before handling remote images, ensuring all images are resolved consistently. - Improved logging for resolved images, providing better insights during manual processing. - Added unit tests to validate the new data URI resolution functionality and ensure reliability.
This commit is contained in:
@@ -6803,11 +6803,17 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
return
|
||||
}
|
||||
|
||||
// Resolve remote images: download http(s) images, upload to storage, replace URLs.
|
||||
// This runs before chunking so that chunks contain stable provider:// URLs.
|
||||
// Resolve embedded data:base64 images and remote http(s) images → storage, replace URLs.
|
||||
// Runs before chunking so chunks contain stable provider:// URLs.
|
||||
var resolvedImages []docparser.StoredImage
|
||||
if s.imageResolver != nil {
|
||||
fileSvc := s.resolveFileService(ctx, kb)
|
||||
afterDataURI, fromDataURI, _ := s.imageResolver.ResolveDataURIImages(ctx, clean, fileSvc, knowledge.TenantID)
|
||||
if len(fromDataURI) > 0 {
|
||||
logger.Infof(ctx, "Resolved %d data-URI images for manual knowledge %s", len(fromDataURI), knowledge.ID)
|
||||
clean = afterDataURI
|
||||
resolvedImages = append(resolvedImages, fromDataURI...)
|
||||
}
|
||||
updatedContent, storedImages, resolveErr := s.imageResolver.ResolveRemoteImages(ctx, clean, fileSvc, knowledge.TenantID)
|
||||
if resolveErr != nil {
|
||||
logger.Warnf(ctx, "Remote image resolution partially failed: %v", resolveErr)
|
||||
@@ -6815,7 +6821,7 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
|
||||
if len(storedImages) > 0 {
|
||||
logger.Infof(ctx, "Resolved %d remote images for manual knowledge %s", len(storedImages), knowledge.ID)
|
||||
clean = updatedContent
|
||||
resolvedImages = storedImages
|
||||
resolvedImages = append(resolvedImages, storedImages...)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package docparser
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
@@ -78,8 +79,12 @@ func (r *ImageResolver) ResolveAndStore(
|
||||
tenantID uint64,
|
||||
) (updatedMarkdown string, images []StoredImage, err error) {
|
||||
markdown := UnwrapLinkedImages(result.MarkdownContent)
|
||||
md2, imgDataURIs, _ := r.ResolveDataURIImages(ctx, markdown, fileSvc, tenantID)
|
||||
markdown = md2
|
||||
images = append(images, imgDataURIs...)
|
||||
|
||||
if len(result.ImageRefs) == 0 {
|
||||
return markdown, nil, nil
|
||||
return markdown, images, nil
|
||||
}
|
||||
|
||||
// Build a map of original_ref -> image ref for fast lookup
|
||||
@@ -160,6 +165,8 @@ func extFromMime(mime string) string {
|
||||
return ".webp"
|
||||
case "image/bmp":
|
||||
return ".bmp"
|
||||
case "image/svg+xml":
|
||||
return ".svg"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
@@ -210,6 +217,103 @@ func UnwrapLinkedImages(markdown string) string {
|
||||
// like https://example.com/item_(abc)/123 are captured in full.
|
||||
var imgMarkdownPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
|
||||
|
||||
// imgMarkdownDataURI matches markdown images whose URL is a data:image/*;base64,...
|
||||
// payload. (?i) applies to the whole parenthesized data URI.
|
||||
var imgMarkdownDataURI = regexp.MustCompile(
|
||||
`!\[([^\]]*)\]\((?i:(data:image/[^;]+;base64,\s*[^)]+))\)`,
|
||||
)
|
||||
|
||||
// parseImageDataURI splits a data URI into image MIME type and base64 payload.
|
||||
func parseImageDataURI(dataURI string) (mimeType string, b64Payload string, ok bool) {
|
||||
const sep = ";base64,"
|
||||
idx := strings.Index(strings.ToLower(dataURI), sep)
|
||||
if idx < 0 {
|
||||
return "", "", false
|
||||
}
|
||||
meta := strings.TrimSpace(dataURI[:idx])
|
||||
const prefix = "data:image/"
|
||||
if len(meta) < len(prefix) || !strings.EqualFold(meta[:len(prefix)], prefix) {
|
||||
return "", "", false
|
||||
}
|
||||
sub := strings.TrimSpace(meta[len(prefix):])
|
||||
mimeType = "image/" + strings.ToLower(sub)
|
||||
b64Payload = strings.TrimSpace(dataURI[idx+len(sep):])
|
||||
if b64Payload == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return mimeType, b64Payload, true
|
||||
}
|
||||
|
||||
// ResolveDataURIImages finds embedded data:image/*;base64 images in markdown,
|
||||
// decodes them, stores via fileSvc, and replaces each reference with the returned
|
||||
// provider URL (same limits as remote images: count and decoded size).
|
||||
func (r *ImageResolver) ResolveDataURIImages(
|
||||
ctx context.Context,
|
||||
markdown string,
|
||||
fileSvc interfaces.FileService,
|
||||
tenantID uint64,
|
||||
) (updatedMarkdown string, images []StoredImage, err error) {
|
||||
markdown = UnwrapLinkedImages(markdown)
|
||||
matches := imgMarkdownDataURI.FindAllStringSubmatchIndex(markdown, -1)
|
||||
if len(matches) == 0 {
|
||||
return markdown, nil, nil
|
||||
}
|
||||
|
||||
processed := 0
|
||||
for i := len(matches) - 1; i >= 0; i-- {
|
||||
if processed >= maxRemoteImages {
|
||||
break
|
||||
}
|
||||
m := matches[i]
|
||||
if len(m) < 6 {
|
||||
continue
|
||||
}
|
||||
dataURI := markdown[m[4]:m[5]]
|
||||
mimeType, payload, ok := parseImageDataURI(dataURI)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
payload = strings.ReplaceAll(payload, "\n", "")
|
||||
payload = strings.ReplaceAll(payload, "\r", "")
|
||||
payload = strings.ReplaceAll(payload, "\t", "")
|
||||
payload = strings.ReplaceAll(payload, " ", "")
|
||||
if payload == "" {
|
||||
continue
|
||||
}
|
||||
data, decErr := base64.StdEncoding.DecodeString(payload)
|
||||
if decErr != nil {
|
||||
log.Printf("WARN: data URI base64 decode failed: %v", decErr)
|
||||
continue
|
||||
}
|
||||
if len(data) > maxRemoteImageSize {
|
||||
log.Printf("WARN: data URI image exceeds size limit (%d bytes)", maxRemoteImageSize)
|
||||
continue
|
||||
}
|
||||
if isIconImage(data) {
|
||||
markdown = markdown[:m[0]] + markdown[m[1]:]
|
||||
continue
|
||||
}
|
||||
ext := extFromMime(mimeType)
|
||||
if ext == "" {
|
||||
ext = ".png"
|
||||
}
|
||||
fileName := uuid.New().String() + ext
|
||||
servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false)
|
||||
if saveErr != nil {
|
||||
log.Printf("WARN: failed to save data URI image: %v", saveErr)
|
||||
continue
|
||||
}
|
||||
images = append(images, StoredImage{
|
||||
OriginalRef: dataURI,
|
||||
ServingURL: servingURL,
|
||||
MimeType: mimeType,
|
||||
})
|
||||
markdown = markdown[:m[4]] + servingURL + markdown[m[5]:]
|
||||
processed++
|
||||
}
|
||||
return markdown, images, nil
|
||||
}
|
||||
|
||||
// ResolveRemoteImages scans a Markdown string for image references whose URL
|
||||
// is http:// or https://, downloads each one through an SSRF-safe HTTP client,
|
||||
// uploads the bytes via fileSvc, and replaces the original URL with the
|
||||
|
||||
@@ -2,10 +2,18 @@ package docparser
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// createTestPNG generates a minimal PNG image with the given dimensions.
|
||||
@@ -67,3 +75,68 @@ func TestIsIconImage(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// captureSaveBytes implements interfaces.FileService for tests; only SaveBytes records data.
|
||||
type captureSaveBytes struct {
|
||||
saved [][]byte
|
||||
urls []string
|
||||
}
|
||||
|
||||
func (c *captureSaveBytes) CheckConnectivity(context.Context) error { return nil }
|
||||
|
||||
func (c *captureSaveBytes) SaveFile(context.Context, *multipart.FileHeader, uint64, string) (string, error) {
|
||||
return "", fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (c *captureSaveBytes) SaveBytes(_ context.Context, data []byte, _ uint64, fileName string, _ bool) (string, error) {
|
||||
c.saved = append(c.saved, append([]byte(nil), data...))
|
||||
u := "local://test/" + fileName
|
||||
c.urls = append(c.urls, u)
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (c *captureSaveBytes) GetFile(context.Context, string) (io.ReadCloser, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (c *captureSaveBytes) GetFileURL(context.Context, string) (string, error) { return "", nil }
|
||||
|
||||
func (c *captureSaveBytes) DeleteFile(context.Context, string) error { return nil }
|
||||
|
||||
var _ interfaces.FileService = (*captureSaveBytes)(nil)
|
||||
|
||||
func TestResolveDataURIImages(t *testing.T) {
|
||||
png := createTestPNG(200, 150)
|
||||
b64 := base64.StdEncoding.EncodeToString(png)
|
||||
md := "pre  post"
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
out, imgs, err := r.ResolveDataURIImages(context.Background(), md, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(imgs) != 1 {
|
||||
t.Fatalf("got %d images", len(imgs))
|
||||
}
|
||||
if len(svc.saved) != 1 || !bytes.Equal(svc.saved[0], png) {
|
||||
t.Fatal("SaveBytes payload mismatch")
|
||||
}
|
||||
if !strings.Contains(out, "local://test/") || strings.Contains(out, "data:image") {
|
||||
t.Fatalf("markdown: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveDataURIImages_CaseInsensitive(t *testing.T) {
|
||||
png := createTestPNG(200, 150)
|
||||
b64 := base64.StdEncoding.EncodeToString(png)
|
||||
md := ""
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
out, imgs, err := r.ResolveDataURIImages(context.Background(), md, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(imgs) != 1 || !strings.Contains(out, "local://test/") {
|
||||
t.Fatalf("imgs=%d out=%q", len(imgs), out)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user