fix(docparser): handle nested linked images and url parens
This commit is contained in:
@@ -402,9 +402,17 @@ func applyMMR(
|
||||
|
||||
var (
|
||||
// reMarkdownImage matches  — the entire construct is noise.
|
||||
reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`)
|
||||
// URL group supports one level of balanced parentheses.
|
||||
reMarkdownImage = regexp.MustCompile(`!\[[^\]]*\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)
|
||||
// reLinkedImage matches [](link_url) — unwrap to 
|
||||
// so that the subsequent reMarkdownImage pass can remove the image.
|
||||
reLinkedImage = regexp.MustCompile(
|
||||
`\[!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)\]` +
|
||||
`\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`,
|
||||
)
|
||||
// reMarkdownLink matches [text](url) — we keep the text, drop the URL.
|
||||
reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^)]+\)`)
|
||||
// URL group supports one level of balanced parentheses.
|
||||
reMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`)
|
||||
// reRawURL matches standalone http(s) URLs.
|
||||
reRawURL = regexp.MustCompile(`https?://[^\s)\]>]+`)
|
||||
// reCodeBlock matches fenced code blocks (```...```).
|
||||
@@ -442,6 +450,9 @@ func cleanPassageForRerank(text string) string {
|
||||
text = reLatexBlock.ReplaceAllString(text, "")
|
||||
// 3. Remove HTML tags
|
||||
text = reHTMLTag.ReplaceAllString(text, "")
|
||||
// 3.5. Unwrap nested [](link_url) → 
|
||||
// so that the next step removes the full construct cleanly.
|
||||
text = reLinkedImage.ReplaceAllString(text, "")
|
||||
// 4. Remove markdown image references entirely
|
||||
text = reMarkdownImage.ReplaceAllString(text, "")
|
||||
// 5. Convert markdown links to just their display text
|
||||
|
||||
Reference in New Issue
Block a user