fix(docparser): handle nested linked images and url parens

This commit is contained in:
wizardchen
2026-03-17 23:25:54 +08:00
committed by lyingbug
parent 3450ed5e41
commit f3a3c72a4f
3 changed files with 47 additions and 8 deletions
@@ -402,9 +402,17 @@ func applyMMR(
var (
// reMarkdownImage matches ![alt](url) — the entire construct is noise.
reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`)
// URL group supports one level of balanced parentheses.
reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)
// reLinkedImage matches [![alt](img_url)](link_url) — unwrap to ![alt](img_url)
// so that the subsequent reMarkdownImage pass can remove the image.
reLinkedImage = regexp.MustCompile(
`\[!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)\]` +
`\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`,
)
// reMarkdownLink matches [text](url) — we keep the text, drop the URL.
reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`)
// URL group supports one level of balanced parentheses.
reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)
// reRawURL matches standalone http(s) URLs.
reRawURL = regexp.MustCompile(`https?://[^\s)\]>]+`)
// reCodeBlock matches fenced code blocks (```...```).
@@ -442,6 +450,9 @@ func cleanPassageForRerank(text string) string {
text = reLatexBlock.ReplaceAllString(text, "")
// 3. Remove HTML tags
text = reHTMLTag.ReplaceAllString(text, "")
// 3.5. Unwrap nested [![alt](img_url)](link_url) → ![alt](img_url)
// so that the next step removes the full construct cleanly.
text = reLinkedImage.ReplaceAllString(text, "![$1]($2)")
// 4. Remove markdown image references entirely
text = reMarkdownImage.ReplaceAllString(text, "")
// 5. Convert markdown links to just their display text
+6 -1
View File
@@ -7,6 +7,8 @@ import (
"regexp"
"strings"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
)
// Chunk represents a piece of split text with position tracking.
@@ -451,9 +453,12 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Paren
}
// ExtractImageRefs extracts markdown image references from text.
var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
// The URL group supports one level of balanced parentheses so that URLs
// like https://example.com/item_(abc)/123 are captured in full.
var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
func ExtractImageRefs(text string) []ImageRef {
text = docparser.UnwrapLinkedImages(text)
matches := imageRefPattern.FindAllStringSubmatchIndex(text, -1)
var refs []ImageRef
for _, m := range matches {
@@ -77,7 +77,7 @@ func (r *ImageResolver) ResolveAndStore(
fileSvc interfaces.FileService,
tenantID uint64,
) (updatedMarkdown string, images []StoredImage, err error) {
markdown := result.MarkdownContent
markdown := UnwrapLinkedImages(result.MarkdownContent)
if len(result.ImageRefs) == 0 {
return markdown, nil, nil
}
@@ -88,8 +88,10 @@ func (r *ImageResolver) ResolveAndStore(
refMap[ref.OriginalRef] = ref
}
// Process each image reference found in the markdown
imgPattern := regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
// Process each image reference found in the markdown.
// The URL group supports one level of balanced parentheses so that URLs
// like https://example.com/item_(abc)/123 are captured in full.
imgPattern := regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
matches := imgPattern.FindAllStringSubmatchIndex(markdown, -1)
// Process in reverse order to preserve positions when replacing
@@ -186,8 +188,27 @@ const (
remoteImageFetchTimeout = 15 * time.Second
)
// imgMarkdownPattern matches Markdown image syntax: ![alt](url)
var imgMarkdownPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)\s]+)\)`)
// reLinkedImage matches the nested [![alt](img_url)](link_url) pattern where
// an image is wrapped inside a Markdown link. We unwrap it to just ![alt](img_url)
// so that downstream image-processing regexes only have to handle the flat form.
// The URL groups support one level of balanced parentheses.
var reLinkedImage = regexp.MustCompile(
`\[!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)\]` + // [![alt](img_url)]
`\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`, // (link_url) — captured but discarded
)
// UnwrapLinkedImages replaces all [![alt](img_url)](link_url) occurrences in
// the markdown with just ![alt](img_url), stripping the outer link wrapper.
// This should be called before any image-extraction regex so that only the
// flat ![alt](url) form needs to be handled.
func UnwrapLinkedImages(markdown string) string {
return reLinkedImage.ReplaceAllString(markdown, "![$1]($2)")
}
// imgMarkdownPattern matches Markdown image syntax: ![alt](url).
// The URL group supports one level of balanced parentheses so that URLs
// like https://example.com/item_(abc)/123 are captured in full.
var imgMarkdownPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
// ResolveRemoteImages scans a Markdown string for image references whose URL
// is http:// or https://, downloads each one through an SSRF-safe HTTP client,
@@ -204,6 +225,8 @@ func (r *ImageResolver) ResolveRemoteImages(
fileSvc interfaces.FileService,
tenantID uint64,
) (updatedMarkdown string, images []StoredImage, err error) {
markdown = UnwrapLinkedImages(markdown)
matches := imgMarkdownPattern.FindAllStringSubmatchIndex(markdown, -1)
if len(matches) == 0 {
return markdown, nil, nil