From 1c9503b0630b055ef9057f86c93607e32dc7a3a9 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Tue, 24 Mar 2026 15:52:12 +0800 Subject: [PATCH] feat(docparser): enhance image resolution capabilities in markdown and HTML - Updated regex patterns in MarkdownImageUtil to support alt text containing brackets and handle MIME types with hyphens. - Implemented new functions in ImageResolver for resolving HTML tags with data URIs and bare base64 content, improving image handling in markdown. - Added comprehensive tests for various image scenarios, ensuring robust handling of data URIs and base64 images. --- docreader/parser/markdown_parser.py | 10 +- .../docparser/image_resolver.go | 321 +++++++++++++++++- .../docparser/image_resolver_test.go | 130 +++++++ 3 files changed, 448 insertions(+), 13 deletions(-) diff --git a/docreader/parser/markdown_parser.py b/docreader/parser/markdown_parser.py index 93e0497a..b36258a8 100644 --- a/docreader/parser/markdown_parser.py +++ b/docreader/parser/markdown_parser.py @@ -177,13 +177,15 @@ class MarkdownImageUtil: def __init__(self): # Pattern to match base64 embedded images # Captures: (1) alt text, (2) image format, (3) base64 data + # Alt text uses .*? (non-greedy) to allow literal ] (e.g. Windows paths). + # MIME subtype uses [^;]+ to handle types with hyphens like x-emf. self.b64_pattern = re.compile( - r"!\[([^\]]*)\]\(data:image/(\w+)\+?\w*;base64,([^\)]+)\)" + r"!\[(.*?)\]\(data:image/([^;]+);base64,([^\)]+)\)" ) - # Pattern to match regular image syntax - self.image_pattern = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)") + # Pattern to match regular image syntax (alt text allows ]) + self.image_pattern = re.compile(r"!\[(.*?)\]\(([^)]+)\)") # Pattern for replacing image paths - self.replace_pattern = re.compile(r"!\[([^\]]*)\]\(([^)]+)\)") + self.replace_pattern = re.compile(r"!\[(.*?)\]\(([^)]+)\)") def extract_image( self, diff --git a/internal/infrastructure/docparser/image_resolver.go b/internal/infrastructure/docparser/image_resolver.go index f6acf8f6..aa8f86b4 100644 --- a/internal/infrastructure/docparser/image_resolver.go +++ b/internal/infrastructure/docparser/image_resolver.go @@ -83,6 +83,14 @@ func (r *ImageResolver) ResolveAndStore( markdown = md2 images = append(images, imgDataURIs...) + md3, imgHTML, _ := r.ResolveHTMLDataURIImages(ctx, markdown, fileSvc, tenantID) + markdown = md3 + images = append(images, imgHTML...) + + md4, imgBare, _ := r.ResolveBareBase64Content(ctx, markdown, fileSvc, tenantID) + markdown = md4 + images = append(images, imgBare...) + if len(result.ImageRefs) == 0 { return markdown, images, nil } @@ -96,7 +104,7 @@ func (r *ImageResolver) ResolveAndStore( // Process each image reference found in the markdown. // The URL group supports one level of balanced parentheses so that URLs // like https://example.com/item_(abc)/123 are captured in full. - imgPattern := regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`) + imgPattern := regexp.MustCompile(`!\[(.*?)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`) matches := imgPattern.FindAllStringSubmatchIndex(markdown, -1) // Process in reverse order to preserve positions when replacing @@ -174,7 +182,7 @@ func extFromMime(mime string) string { // isProviderScheme checks if the path uses a provider:// scheme (local://, minio://, cos://, tos://). func isProviderScheme(p string) bool { - for _, prefix := range []string{"local://", "minio://", "cos://", "tos://"} { + for _, prefix := range []string{"local://", "minio://", "cos://", "tos://", "s3://"} { if strings.HasPrefix(p, prefix) { return true } @@ -182,6 +190,301 @@ func isProviderScheme(p string) bool { return false } +// --------------------------------------------------------------------------- +// Helper functions for base64 image handling +// --------------------------------------------------------------------------- + +// cleanBase64Payload removes whitespace characters from a base64 payload string. +func cleanBase64Payload(payload string) string { + payload = strings.ReplaceAll(payload, "\n", "") + payload = strings.ReplaceAll(payload, "\r", "") + payload = strings.ReplaceAll(payload, "\t", "") + payload = strings.ReplaceAll(payload, " ", "") + return payload +} + +// decodeBase64Flexible tries standard, raw, URL-safe, and raw-URL-safe base64 decodings. +func decodeBase64Flexible(payload string) ([]byte, error) { + if data, err := base64.StdEncoding.DecodeString(payload); err == nil { + return data, nil + } + if data, err := base64.RawStdEncoding.DecodeString(payload); err == nil { + return data, nil + } + if data, err := base64.URLEncoding.DecodeString(payload); err == nil { + return data, nil + } + return base64.RawURLEncoding.DecodeString(payload) +} + +// sniffImageMime detects the MIME type by examining the magic bytes of image data. +func sniffImageMime(data []byte) string { + if len(data) < 4 { + return "" + } + if data[0] == 0x89 && data[1] == 'P' && data[2] == 'N' && data[3] == 'G' { + return "image/png" + } + if data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF { + return "image/jpeg" + } + if data[0] == 'G' && data[1] == 'I' && data[2] == 'F' { + return "image/gif" + } + if len(data) >= 12 && + data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' && + data[8] == 'W' && data[9] == 'E' && data[10] == 'B' && data[11] == 'P' { + return "image/webp" + } + if data[0] == 'B' && data[1] == 'M' { + return "image/bmp" + } + return "" +} + +// --------------------------------------------------------------------------- +// HTML tag data URI resolution +// --------------------------------------------------------------------------- + +// imgHTMLDataURI matches HTML tags with inline data:image/*;base64,... in the src attribute. +var imgHTMLDataURI = regexp.MustCompile( + `(?i)]*?src\s*=\s*["'](data:image/[^;]+;base64,[^"']+)["'][^>]*?/?\s*>`, +) + +// ResolveHTMLDataURIImages finds tags in markdown, +// decodes the images, stores them via fileSvc, and replaces each tag with a markdown +// image reference using the storage URL. +func (r *ImageResolver) ResolveHTMLDataURIImages( + ctx context.Context, + markdown string, + fileSvc interfaces.FileService, + tenantID uint64, +) (updatedMarkdown string, images []StoredImage, err error) { + matches := imgHTMLDataURI.FindAllStringSubmatchIndex(markdown, -1) + if len(matches) == 0 { + return markdown, nil, nil + } + + processed := 0 + for i := len(matches) - 1; i >= 0; i-- { + if processed >= maxRemoteImages { + break + } + m := matches[i] + dataURI := markdown[m[2]:m[3]] + mimeType, payload, ok := parseImageDataURI(dataURI) + if !ok { + continue + } + payload = cleanBase64Payload(payload) + if payload == "" { + continue + } + data, decErr := decodeBase64Flexible(payload) + if decErr != nil { + log.Printf("WARN: HTML img data URI base64 decode failed: %v", decErr) + continue + } + if len(data) > maxRemoteImageSize { + continue + } + if isIconImage(data) { + markdown = markdown[:m[0]] + markdown[m[1]:] + continue + } + ext := extFromMime(mimeType) + if ext == "" { + ext = ".png" + } + fileName := uuid.New().String() + ext + servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false) + if saveErr != nil { + log.Printf("WARN: failed to save HTML img data URI image: %v", saveErr) + continue + } + images = append(images, StoredImage{ + OriginalRef: "html-img-data-uri", + ServingURL: servingURL, + MimeType: mimeType, + }) + markdown = markdown[:m[0]] + fmt.Sprintf("![image](%s)", servingURL) + markdown[m[1]:] + processed++ + } + return markdown, images, nil +} + +// --------------------------------------------------------------------------- +// Bare base64/data URI resolution (catch-all) +// --------------------------------------------------------------------------- + +// bareDataURIPattern matches standalone data:image/*;base64,... strings. +var bareDataURIPattern = regexp.MustCompile( + `(?i)data:image/([^;\s]+);base64,([A-Za-z0-9+/=]{100,})`, +) + +// bareBase64CommaPrefixed matches base64,DATA patterns (partial data URIs missing the mime prefix). +var bareBase64CommaPrefixed = regexp.MustCompile( + `base64,([A-Za-z0-9+/=]{200,})`, +) + +// ResolveBareBase64Content finds remaining bare data URIs and base64 image content +// in the markdown text, decodes and stores them, and replaces with image references. +// This acts as a catch-all after the standard markdown and HTML resolvers. +func (r *ImageResolver) ResolveBareBase64Content( + ctx context.Context, + markdown string, + fileSvc interfaces.FileService, + tenantID uint64, +) (updatedMarkdown string, images []StoredImage, err error) { + md, imgs1 := r.resolveBareDataURIs(ctx, markdown, fileSvc, tenantID) + markdown = md + images = append(images, imgs1...) + + md2, imgs2 := r.resolveBareBase64Prefix(ctx, markdown, fileSvc, tenantID) + markdown = md2 + images = append(images, imgs2...) + + return markdown, images, nil +} + +func (r *ImageResolver) resolveBareDataURIs( + ctx context.Context, + markdown string, + fileSvc interfaces.FileService, + tenantID uint64, +) (string, []StoredImage) { + matches := bareDataURIPattern.FindAllStringSubmatchIndex(markdown, -1) + if len(matches) == 0 { + return markdown, nil + } + + var images []StoredImage + processed := 0 + for i := len(matches) - 1; i >= 0; i-- { + if processed >= maxRemoteImages { + break + } + m := matches[i] + // Check context: skip HTML src attributes, but handle broken markdown refs + insideWrapper := false + if m[0] > 0 { + prev := markdown[m[0]-1] + if prev == '"' || prev == '\'' { + continue // inside HTML attribute — already handled by ResolveHTMLDataURIImages + } + if prev == '(' { + insideWrapper = true // likely inside a broken ![...](...) ref + } + } + mimeSubtype := strings.ToLower(markdown[m[2]:m[3]]) + payload := markdown[m[4]:m[5]] + mimeType := "image/" + mimeSubtype + + payload = cleanBase64Payload(payload) + if payload == "" { + continue + } + data, decErr := decodeBase64Flexible(payload) + if decErr != nil { + log.Printf("WARN: bare data URI base64 decode failed: %v", decErr) + continue + } + if len(data) > maxRemoteImageSize { + continue + } + if isIconImage(data) { + markdown = markdown[:m[0]] + markdown[m[1]:] + continue + } + ext := extFromMime(mimeType) + if ext == "" { + ext = ".png" + } + fileName := uuid.New().String() + ext + servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false) + if saveErr != nil { + log.Printf("WARN: failed to save bare data URI image: %v", saveErr) + continue + } + images = append(images, StoredImage{ + OriginalRef: "bare-data-uri", + ServingURL: servingURL, + MimeType: mimeType, + }) + if insideWrapper { + // Inside a broken markdown ref like ![weird]alt](data:...) — replace data URI only + markdown = markdown[:m[0]] + servingURL + markdown[m[1]:] + } else { + markdown = markdown[:m[0]] + fmt.Sprintf("![image](%s)", servingURL) + markdown[m[1]:] + } + processed++ + } + return markdown, images +} + +func (r *ImageResolver) resolveBareBase64Prefix( + ctx context.Context, + markdown string, + fileSvc interfaces.FileService, + tenantID uint64, +) (string, []StoredImage) { + matches := bareBase64CommaPrefixed.FindAllStringSubmatchIndex(markdown, -1) + if len(matches) == 0 { + return markdown, nil + } + + var images []StoredImage + processed := 0 + for i := len(matches) - 1; i >= 0; i-- { + if processed >= maxRemoteImages { + break + } + m := matches[i] + // Skip if preceded by ';' — this is part of a data URI handled above + if m[0] > 0 && markdown[m[0]-1] == ';' { + continue + } + payload := markdown[m[2]:m[3]] + payload = cleanBase64Payload(payload) + if payload == "" { + continue + } + data, decErr := decodeBase64Flexible(payload) + if decErr != nil { + continue + } + if len(data) > maxRemoteImageSize { + continue + } + mimeType := sniffImageMime(data) + if mimeType == "" { + continue + } + if isIconImage(data) { + markdown = markdown[:m[0]] + markdown[m[1]:] + continue + } + ext := extFromMime(mimeType) + if ext == "" { + ext = ".png" + } + fileName := uuid.New().String() + ext + servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false) + if saveErr != nil { + log.Printf("WARN: failed to save bare base64 image: %v", saveErr) + continue + } + images = append(images, StoredImage{ + OriginalRef: "bare-base64", + ServingURL: servingURL, + MimeType: mimeType, + }) + markdown = markdown[:m[0]] + fmt.Sprintf("![image](%s)", servingURL) + markdown[m[1]:] + processed++ + } + return markdown, images +} + // --------------------------------------------------------------------------- // Remote image resolution (for manual / web-clipped markdown content) // --------------------------------------------------------------------------- @@ -213,14 +516,17 @@ func UnwrapLinkedImages(markdown string) string { } // imgMarkdownPattern matches Markdown image syntax: ![alt](url). +// The alt-text group uses .*? (non-greedy) to allow literal ] in alt text. // The URL group supports one level of balanced parentheses so that URLs // like https://example.com/item_(abc)/123 are captured in full. -var imgMarkdownPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`) +var imgMarkdownPattern = regexp.MustCompile(`!\[(.*?)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`) // imgMarkdownDataURI matches markdown images whose URL is a data:image/*;base64,... // payload. (?i) applies to the whole parenthesized data URI. +// The alt-text group uses .*? (non-greedy) to allow literal ] inside alt text +// (e.g. file paths like ![C:\img]name.png](data:...)). var imgMarkdownDataURI = regexp.MustCompile( - `!\[([^\]]*)\]\((?i:(data:image/[^;]+;base64,\s*[^)]+))\)`, + `!\[(.*?)\]\((?i:(data:image/[^;]+;base64,\s*[^)]+))\)`, ) // parseImageDataURI splits a data URI into image MIME type and base64 payload. @@ -273,14 +579,11 @@ func (r *ImageResolver) ResolveDataURIImages( if !ok { continue } - payload = strings.ReplaceAll(payload, "\n", "") - payload = strings.ReplaceAll(payload, "\r", "") - payload = strings.ReplaceAll(payload, "\t", "") - payload = strings.ReplaceAll(payload, " ", "") + payload = cleanBase64Payload(payload) if payload == "" { continue } - data, decErr := base64.StdEncoding.DecodeString(payload) + data, decErr := decodeBase64Flexible(payload) if decErr != nil { log.Printf("WARN: data URI base64 decode failed: %v", decErr) continue diff --git a/internal/infrastructure/docparser/image_resolver_test.go b/internal/infrastructure/docparser/image_resolver_test.go index cef6069a..8ac889d1 100644 --- a/internal/infrastructure/docparser/image_resolver_test.go +++ b/internal/infrastructure/docparser/image_resolver_test.go @@ -13,6 +13,7 @@ import ( "strings" "testing" + "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/types/interfaces" ) @@ -140,3 +141,132 @@ func TestResolveDataURIImages_CaseInsensitive(t *testing.T) { t.Fatalf("imgs=%d out=%q", len(imgs), out) } } + +func TestResolveDataURIImages_AltTextWithBracket(t *testing.T) { + png := createTestPNG(200, 150) + b64 := base64.StdEncoding.EncodeToString(png) + md := `pre ![C:\Users\Frank\RichOle\YAJ]7D)SW_W6.png](data:image/png;base64,` + b64 + `) post` + svc := &captureSaveBytes{} + r := NewImageResolver() + out, imgs, err := r.ResolveDataURIImages(context.Background(), md, svc, 1) + if err != nil { + t.Fatal(err) + } + if len(imgs) != 1 { + t.Fatalf("expected 1 image but got %d; alt text with ] was not handled", len(imgs)) + } + if strings.Contains(out, "base64") { + t.Fatalf("base64 content still present in output: %s", out[:min(200, len(out))]) + } +} + +func TestResolveDataURIImages_XEmfMimeType(t *testing.T) { + png := createTestPNG(200, 150) + b64 := base64.StdEncoding.EncodeToString(png) + md := `![](data:image/x-emf;base64,` + b64 + `)` + svc := &captureSaveBytes{} + r := NewImageResolver() + out, imgs, err := r.ResolveDataURIImages(context.Background(), md, svc, 1) + if err != nil { + t.Fatal(err) + } + if len(imgs) != 1 { + t.Fatalf("expected 1 image for x-emf mime type but got %d", len(imgs)) + } + if strings.Contains(out, "data:image") { + t.Fatalf("data URI still present for x-emf: %s", out[:min(200, len(out))]) + } +} + +func TestResolveHTMLDataURIImages(t *testing.T) { + png := createTestPNG(200, 150) + b64 := base64.StdEncoding.EncodeToString(png) + md := `text test more` + svc := &captureSaveBytes{} + r := NewImageResolver() + out, imgs, err := r.ResolveHTMLDataURIImages(context.Background(), md, svc, 1) + if err != nil { + t.Fatal(err) + } + if len(imgs) != 1 { + t.Fatalf("expected 1 image from HTML img tag but got %d", len(imgs)) + } + if strings.Contains(out, "