fix(docparser): handle nested linked images and url parens
This commit is contained in:
@@ -402,9 +402,17 @@ func applyMMR(
|
||||
|
||||
var (
|
||||
// reMarkdownImage matches  — the entire construct is noise.
|
||||
reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`)
|
||||
// URL group supports one level of balanced parentheses.
|
||||
reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)
|
||||
// reLinkedImage matches [](link_url) — unwrap to 
|
||||
// so that the subsequent reMarkdownImage pass can remove the image.
|
||||
reLinkedImage = regexp.MustCompile(
|
||||
`\[!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)\]` +
|
||||
`\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`,
|
||||
)
|
||||
// reMarkdownLink matches [text](url) — we keep the text, drop the URL.
|
||||
reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`)
|
||||
// URL group supports one level of balanced parentheses.
|
||||
reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)
|
||||
// reRawURL matches standalone http(s) URLs.
|
||||
reRawURL = regexp.MustCompile(`https?://[^\s)\]>]+`)
|
||||
// reCodeBlock matches fenced code blocks (```...```).
|
||||
@@ -442,6 +450,9 @@ func cleanPassageForRerank(text string) string {
|
||||
text = reLatexBlock.ReplaceAllString(text, "")
|
||||
// 3. Remove HTML tags
|
||||
text = reHTMLTag.ReplaceAllString(text, "")
|
||||
// 3.5. Unwrap nested [](link_url) → 
|
||||
// so that the next step removes the full construct cleanly.
|
||||
text = reLinkedImage.ReplaceAllString(text, "")
|
||||
// 4. Remove markdown image references entirely
|
||||
text = reMarkdownImage.ReplaceAllString(text, "")
|
||||
// 5. Convert markdown links to just their display text
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
)
|
||||
|
||||
// Chunk represents a piece of split text with position tracking.
|
||||
@@ -451,9 +453,12 @@ func SplitTextParentChild(text string, parentCfg, childCfg SplitterConfig) Paren
|
||||
}
|
||||
|
||||
// ExtractImageRefs extracts markdown image references from text.
|
||||
var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
|
||||
// The URL group supports one level of balanced parentheses so that URLs
|
||||
// like https://example.com/item_(abc)/123 are captured in full.
|
||||
var imageRefPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
|
||||
|
||||
func ExtractImageRefs(text string) []ImageRef {
|
||||
text = docparser.UnwrapLinkedImages(text)
|
||||
matches := imageRefPattern.FindAllStringSubmatchIndex(text, -1)
|
||||
var refs []ImageRef
|
||||
for _, m := range matches {
|
||||
|
||||
@@ -77,7 +77,7 @@ func (r *ImageResolver) ResolveAndStore(
|
||||
fileSvc interfaces.FileService,
|
||||
tenantID uint64,
|
||||
) (updatedMarkdown string, images []StoredImage, err error) {
|
||||
markdown := result.MarkdownContent
|
||||
markdown := UnwrapLinkedImages(result.MarkdownContent)
|
||||
if len(result.ImageRefs) == 0 {
|
||||
return markdown, nil, nil
|
||||
}
|
||||
@@ -88,8 +88,10 @@ func (r *ImageResolver) ResolveAndStore(
|
||||
refMap[ref.OriginalRef] = ref
|
||||
}
|
||||
|
||||
// Process each image reference found in the markdown
|
||||
imgPattern := regexp.MustCompile(`!\[([^\]]*)\]\(([^)]+)\)`)
|
||||
// Process each image reference found in the markdown.
|
||||
// The URL group supports one level of balanced parentheses so that URLs
|
||||
// like https://example.com/item_(abc)/123 are captured in full.
|
||||
imgPattern := regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
|
||||
matches := imgPattern.FindAllStringSubmatchIndex(markdown, -1)
|
||||
|
||||
// Process in reverse order to preserve positions when replacing
|
||||
@@ -186,8 +188,27 @@ const (
|
||||
remoteImageFetchTimeout = 15 * time.Second
|
||||
)
|
||||
|
||||
// imgMarkdownPattern matches Markdown image syntax: 
|
||||
var imgMarkdownPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^)\s]+)\)`)
|
||||
// reLinkedImage matches the nested [](link_url) pattern where
|
||||
// an image is wrapped inside a Markdown link. We unwrap it to just 
|
||||
// so that downstream image-processing regexes only have to handle the flat form.
|
||||
// The URL groups support one level of balanced parentheses.
|
||||
var reLinkedImage = regexp.MustCompile(
|
||||
`\[!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)\]` + // []
|
||||
`\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`, // (link_url) — captured but discarded
|
||||
)
|
||||
|
||||
// UnwrapLinkedImages replaces all [](link_url) occurrences in
|
||||
// the markdown with just , stripping the outer link wrapper.
|
||||
// This should be called before any image-extraction regex so that only the
|
||||
// flat  form needs to be handled.
|
||||
func UnwrapLinkedImages(markdown string) string {
|
||||
return reLinkedImage.ReplaceAllString(markdown, "")
|
||||
}
|
||||
|
||||
// imgMarkdownPattern matches Markdown image syntax: .
|
||||
// The URL group supports one level of balanced parentheses so that URLs
|
||||
// like https://example.com/item_(abc)/123 are captured in full.
|
||||
var imgMarkdownPattern = regexp.MustCompile(`!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)
|
||||
|
||||
// ResolveRemoteImages scans a Markdown string for image references whose URL
|
||||
// is http:// or https://, downloads each one through an SSRF-safe HTTP client,
|
||||
@@ -204,6 +225,8 @@ func (r *ImageResolver) ResolveRemoteImages(
|
||||
fileSvc interfaces.FileService,
|
||||
tenantID uint64,
|
||||
) (updatedMarkdown string, images []StoredImage, err error) {
|
||||
markdown = UnwrapLinkedImages(markdown)
|
||||
|
||||
matches := imgMarkdownPattern.FindAllStringSubmatchIndex(markdown, -1)
|
||||
if len(matches) == 0 {
|
||||
return markdown, nil, nil
|
||||
|
||||
Reference in New Issue
Block a user