diff --git a/internal/application/service/chat_pipline/rerank.go b/internal/application/service/chat_pipline/rerank.go index 94560316..aa2f8ef0 100644 --- a/internal/application/service/chat_pipline/rerank.go +++ b/internal/application/service/chat_pipline/rerank.go @@ -402,9 +402,17 @@ func applyMMR( var ( // reMarkdownImage matches ![alt](url) — the entire construct is noise. - reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`) + // URL group supports one level of balanced parentheses. + reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`) + // reLinkedImage matches [![alt](img_url)](link_url) — unwrap to ![alt](img_url) + // so that the subsequent reMarkdownImage pass can remove the image. + reLinkedImage = regexp.MustCompile( + `\[!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)\]` + + `\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`, + ) // reMarkdownLink matches [text](url) — we keep the text, drop the URL. - reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`) + // URL group supports one level of balanced parentheses. + reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`) // reRawURL matches standalone http(s) URLs. reRawURL = regexp.MustCompile(`https?://[^\s)\]>]+`) // reCodeBlock matches fenced code blocks (```...```). @@ -442,6 +450,9 @@ func cleanPassageForRerank(text string) string { text = reLatexBlock.ReplaceAllString(text, "") // 3. Remove HTML tags text = reHTMLTag.ReplaceAllString(text, "") + // 3.5. Unwrap nested [![alt](img_url)](link_url) → ![alt](img_url) + // so that the next step removes the full construct cleanly. + text = reLinkedImage.ReplaceAllString(text, "![$1]($2)") // 4. Remove markdown image references entirely text = reMarkdownImage.ReplaceAllString(text, "") // 5. Convert markdown links to just their display text diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index fefae05e..4518e7b0 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -7,6 +7,8 @@ import ( "regexp" "strings" "unicode/utf8" + + "github.com/Tencent/WeKnora/internal/infrastructure/docparser" ) // Chunk represents a piece of split text with position tracking. @@ -451,9 +453,12 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Paren } // ExtractImageRefs extracts markdown image references from text. -var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`) +// The URL group supports one level of balanced parentheses so that URLs +// like https://example.com/item_(abc)/123 are captured in full. +var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`) func ExtractImageRefs(text string) []ImageRef { + text = docparser.UnwrapLinkedImages(text) matches := imageRefPattern.FindAllStringSubmatchIndex(text, -1) var refs []ImageRef for _, m := range matches { diff --git a/internal/infrastructure/docparser/image_resolver.go b/internal/infrastructure/docparser/image_resolver.go index 5c1eb1ae..f58e6338 100644 --- a/internal/infrastructure/docparser/image_resolver.go +++ b/internal/infrastructure/docparser/image_resolver.go @@ -77,7 +77,7 @@ func (r *ImageResolver) ResolveAndStore( fileSvc interfaces.FileService, tenantID uint64, ) (updatedMarkdown string, images []StoredImage, err error) { - markdown := result.MarkdownContent + markdown := UnwrapLinkedImages(result.MarkdownContent) if len(result.ImageRefs) == 0 { return markdown, nil, nil } @@ -88,8 +88,10 @@ func (r *ImageResolver) ResolveAndStore( refMap[ref.OriginalRef] = ref } - // Process each image reference found in the markdown - imgPattern := regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`) + // Process each image reference found in the markdown. + // The URL group supports one level of balanced parentheses so that URLs + // like https://example.com/item_(abc)/123 are captured in full. + imgPattern := regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`) matches := imgPattern.FindAllStringSubmatchIndex(markdown, -1) // Process in reverse order to preserve positions when replacing @@ -186,8 +188,27 @@ const ( remoteImageFetchTimeout = 15 * time.Second ) -// imgMarkdownPattern matches Markdown image syntax: ![alt](url) -var imgMarkdownPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)\s]+)\)`) +// reLinkedImage matches the nested [![alt](img_url)](link_url) pattern where +// an image is wrapped inside a Markdown link. We unwrap it to just ![alt](img_url) +// so that downstream image-processing regexes only have to handle the flat form. +// The URL groups support one level of balanced parentheses. +var reLinkedImage = regexp.MustCompile( + `\[!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)\]` + // [![alt](img_url)] + `\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`, // (link_url) — captured but discarded +) + +// UnwrapLinkedImages replaces all [![alt](img_url)](link_url) occurrences in +// the markdown with just ![alt](img_url), stripping the outer link wrapper. +// This should be called before any image-extraction regex so that only the +// flat ![alt](url) form needs to be handled. +func UnwrapLinkedImages(markdown string) string { + return reLinkedImage.ReplaceAllString(markdown, "![$1]($2)") +} + +// imgMarkdownPattern matches Markdown image syntax: ![alt](url). +// The URL group supports one level of balanced parentheses so that URLs +// like https://example.com/item_(abc)/123 are captured in full. +var imgMarkdownPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`) // ResolveRemoteImages scans a Markdown string for image references whose URL // is http:// or https://, downloads each one through an SSRF-safe HTTP client, @@ -204,6 +225,8 @@ func (r *ImageResolver) ResolveRemoteImages( fileSvc interfaces.FileService, tenantID uint64, ) (updatedMarkdown string, images []StoredImage, err error) { + markdown = UnwrapLinkedImages(markdown) + matches := imgMarkdownPattern.FindAllStringSubmatchIndex(markdown, -1) if len(matches) == 0 { return markdown, nil, nil