fix: preserve MHTML image resolution and Markdown structure (#1743)
* fix(docreader): preserve MHTML blocks and resolve titled images * fix(docreader): harden MHTML markdown image resolution * fix(docreader): preserve markdown hard break spaces --------- Co-authored-by: FFFFFFpy <bushiyan@outlook.com>
This commit is contained in:
@@ -8,7 +8,7 @@ import email
|
|||||||
import html
|
import html
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from urllib.parse import unquote, urljoin
|
from urllib.parse import unquote, urljoin, urlparse
|
||||||
import uuid
|
import uuid
|
||||||
from typing import Dict
|
from typing import Dict
|
||||||
|
|
||||||
@@ -70,8 +70,7 @@ class MHTMLParser(BaseParser):
|
|||||||
elif content_type.startswith("image/") and self.extract_images:
|
elif content_type.startswith("image/") and self.extract_images:
|
||||||
image_data = part.get_payload(decode=True)
|
image_data = part.get_payload(decode=True)
|
||||||
if image_data:
|
if image_data:
|
||||||
ext = self._image_extension(content_type)
|
image_path = self._image_path_for_part(part, content_type, images)
|
||||||
image_path = f"images/{uuid.uuid4().hex}{ext}"
|
|
||||||
images[image_path] = base64.b64encode(image_data).decode("utf-8")
|
images[image_path] = base64.b64encode(image_data).decode("utf-8")
|
||||||
self._add_image_aliases(image_aliases, part, image_path)
|
self._add_image_aliases(image_aliases, part, image_path)
|
||||||
|
|
||||||
@@ -154,6 +153,45 @@ class MHTMLParser(BaseParser):
|
|||||||
"image/x-icon": ".ico",
|
"image/x-icon": ".ico",
|
||||||
}.get(content_type, ".png")
|
}.get(content_type, ".png")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _image_path_for_part(
|
||||||
|
cls, part, content_type: str, images: Dict[str, str]
|
||||||
|
) -> str:
|
||||||
|
"""Choose a stable image path when the MHTML part exposes a filename."""
|
||||||
|
ext = cls._image_extension(content_type)
|
||||||
|
location = (part.get("Content-Location", "") or "").strip()
|
||||||
|
filename = cls._filename_from_content_location(location)
|
||||||
|
if not filename:
|
||||||
|
return f"images/{uuid.uuid4().hex}{ext}"
|
||||||
|
|
||||||
|
stem, location_ext = os.path.splitext(filename)
|
||||||
|
if not location_ext:
|
||||||
|
filename = f"{filename}{ext}"
|
||||||
|
image_path = f"images/{filename}"
|
||||||
|
if image_path not in images:
|
||||||
|
return image_path
|
||||||
|
|
||||||
|
suffix = 2
|
||||||
|
stem, location_ext = os.path.splitext(filename)
|
||||||
|
while True:
|
||||||
|
candidate = f"images/{stem}_{suffix}{location_ext}"
|
||||||
|
if candidate not in images:
|
||||||
|
return candidate
|
||||||
|
suffix += 1
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _filename_from_content_location(location: str) -> str:
|
||||||
|
decoded = unquote(html.unescape(location.strip()))
|
||||||
|
if not decoded or decoded.lower().startswith("cid:"):
|
||||||
|
return ""
|
||||||
|
path = urlparse(decoded).path or decoded
|
||||||
|
filename = os.path.basename(path)
|
||||||
|
if not filename or filename in {".", ".."}:
|
||||||
|
return ""
|
||||||
|
if "/" in filename or "\\" in filename:
|
||||||
|
return ""
|
||||||
|
return filename
|
||||||
|
|
||||||
def _html_to_markdown(
|
def _html_to_markdown(
|
||||||
self,
|
self,
|
||||||
html_content: str,
|
html_content: str,
|
||||||
@@ -171,9 +209,7 @@ class MHTMLParser(BaseParser):
|
|||||||
self._rewrite_image_sources(soup, image_aliases, base_location)
|
self._rewrite_image_sources(soup, image_aliases, base_location)
|
||||||
text_fallback = soup.get_text(separator="\n", strip=True)
|
text_fallback = soup.get_text(separator="\n", strip=True)
|
||||||
markdown_text = md(str(soup), heading_style="ATX")
|
markdown_text = md(str(soup), heading_style="ATX")
|
||||||
result = "\n".join(
|
result = self._normalize_markdown(markdown_text)
|
||||||
line.strip() for line in markdown_text.split("\n") if line.strip()
|
|
||||||
)
|
|
||||||
if not result and text_fallback:
|
if not result and text_fallback:
|
||||||
logger.warning("Markdown empty, falling back to text extraction")
|
logger.warning("Markdown empty, falling back to text extraction")
|
||||||
return text_fallback
|
return text_fallback
|
||||||
@@ -187,6 +223,71 @@ class MHTMLParser(BaseParser):
|
|||||||
logger.error("HTML to Markdown conversion failed: %s", e)
|
logger.error("HTML to Markdown conversion failed: %s", e)
|
||||||
return f"```html\n{html_content}\n```"
|
return f"```html\n{html_content}\n```"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _normalize_markdown(markdown_text: str) -> str:
|
||||||
|
text = markdown_text.replace("\r\n", "\n").replace("\r", "\n")
|
||||||
|
output: list[str] = []
|
||||||
|
pending_blank = False
|
||||||
|
fence_char: str | None = None
|
||||||
|
fence_len = 0
|
||||||
|
|
||||||
|
for line in text.split("\n"):
|
||||||
|
if fence_char is not None:
|
||||||
|
output.append(line)
|
||||||
|
if MHTMLParser._is_closing_fence(line, fence_char, fence_len):
|
||||||
|
fence_char = None
|
||||||
|
fence_len = 0
|
||||||
|
continue
|
||||||
|
|
||||||
|
opening = MHTMLParser._opening_fence(line)
|
||||||
|
if opening is not None:
|
||||||
|
if pending_blank and output:
|
||||||
|
output.append("")
|
||||||
|
pending_blank = False
|
||||||
|
output.append(line)
|
||||||
|
fence_char, fence_len = opening
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not line.strip(" \t"):
|
||||||
|
pending_blank = True
|
||||||
|
continue
|
||||||
|
|
||||||
|
if pending_blank and output:
|
||||||
|
output.append("")
|
||||||
|
pending_blank = False
|
||||||
|
|
||||||
|
trailing_spaces = len(line) - len(line.rstrip(" "))
|
||||||
|
if trailing_spaces >= 2:
|
||||||
|
line = line.rstrip(" \t") + " "
|
||||||
|
else:
|
||||||
|
line = line.rstrip(" \t")
|
||||||
|
output.append(line)
|
||||||
|
|
||||||
|
return "\n".join(output).strip("\n")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _opening_fence(line: str) -> tuple[str, int] | None:
|
||||||
|
stripped = line.lstrip(" ")
|
||||||
|
if len(line) - len(stripped) > 3 or not stripped:
|
||||||
|
return None
|
||||||
|
fence_char = stripped[0]
|
||||||
|
if fence_char not in {"`", "~"}:
|
||||||
|
return None
|
||||||
|
fence_len = len(stripped) - len(stripped.lstrip(fence_char))
|
||||||
|
if fence_len < 3:
|
||||||
|
return None
|
||||||
|
return fence_char, fence_len
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_closing_fence(line: str, fence_char: str, fence_len: int) -> bool:
|
||||||
|
stripped = line.lstrip(" ")
|
||||||
|
if len(line) - len(stripped) > 3:
|
||||||
|
return False
|
||||||
|
closing_len = len(stripped) - len(stripped.lstrip(fence_char))
|
||||||
|
if closing_len < fence_len:
|
||||||
|
return False
|
||||||
|
return not stripped[closing_len:].strip(" \t")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _strip_internal_links(soup: BeautifulSoup) -> None:
|
def _strip_internal_links(soup: BeautifulSoup) -> None:
|
||||||
"""Unwrap links that don't point to an external resource."""
|
"""Unwrap links that don't point to an external resource."""
|
||||||
|
|||||||
@@ -1,9 +1,14 @@
|
|||||||
|
import base64
|
||||||
|
import json
|
||||||
import unittest
|
import unittest
|
||||||
from email.message import EmailMessage
|
from email.message import EmailMessage
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from docreader.parser.mhtml_parser import MHTMLParser
|
from docreader.parser.mhtml_parser import MHTMLParser
|
||||||
from docreader.parser.registry import registry
|
from docreader.parser.registry import registry
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
def _minimal_mhtml_bytes() -> bytes:
|
def _minimal_mhtml_bytes() -> bytes:
|
||||||
root = EmailMessage()
|
root = EmailMessage()
|
||||||
@@ -46,6 +51,39 @@ def _minimal_mhtml_bytes() -> bytes:
|
|||||||
return root.as_bytes()
|
return root.as_bytes()
|
||||||
|
|
||||||
|
|
||||||
|
def _mhtml_with_table_image_and_caption() -> bytes:
|
||||||
|
root = EmailMessage()
|
||||||
|
root["Subject"] = "MHTML with table image"
|
||||||
|
root.make_related()
|
||||||
|
|
||||||
|
main = EmailMessage()
|
||||||
|
main.set_content(
|
||||||
|
"<html><body>"
|
||||||
|
"<table>"
|
||||||
|
"<tr><th>体验方向</th><th>代表内容</th></tr>"
|
||||||
|
"<tr><td>赛季制建立</td><td>BP、Rank</td></tr>"
|
||||||
|
"</table>"
|
||||||
|
'<img src="cid:test-image" alt="图片" title="图片">'
|
||||||
|
"<p>高机动性身法与独特枪械反馈</p>"
|
||||||
|
"</body></html>",
|
||||||
|
subtype="html",
|
||||||
|
)
|
||||||
|
main["Content-Location"] = "https://example.com/article"
|
||||||
|
root.attach(main)
|
||||||
|
|
||||||
|
image = EmailMessage()
|
||||||
|
image.set_content(
|
||||||
|
b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff,\x00\x00"
|
||||||
|
b"\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;",
|
||||||
|
maintype="image",
|
||||||
|
subtype="gif",
|
||||||
|
)
|
||||||
|
image["Content-ID"] = "<test-image>"
|
||||||
|
root.attach(image)
|
||||||
|
|
||||||
|
return root.as_bytes()
|
||||||
|
|
||||||
|
|
||||||
class MHTMLParserTest(unittest.TestCase):
|
class MHTMLParserTest(unittest.TestCase):
|
||||||
def test_parse_selects_main_html_and_filters_noise(self):
|
def test_parse_selects_main_html_and_filters_noise(self):
|
||||||
document = MHTMLParser(
|
document = MHTMLParser(
|
||||||
@@ -84,6 +122,115 @@ class MHTMLParserTest(unittest.TestCase):
|
|||||||
self.assertNotIn("cid:tiny-image", with_images.content)
|
self.assertNotIn("cid:tiny-image", with_images.content)
|
||||||
self.assertEqual(without_images.images, {})
|
self.assertEqual(without_images.images, {})
|
||||||
|
|
||||||
|
def test_table_image_and_caption_keep_markdown_block_boundaries(self):
|
||||||
|
document = MHTMLParser(
|
||||||
|
file_name="article.mhtml", file_type="mhtml"
|
||||||
|
).parse_into_text(_mhtml_with_table_image_and_caption())
|
||||||
|
|
||||||
|
self.assertEqual(len(document.images), 1)
|
||||||
|
image_ref = next(iter(document.images))
|
||||||
|
self.assertIn(f'', document.content)
|
||||||
|
self.assertIn(
|
||||||
|
f"| 赛季制建立 | BP、Rank |\n\n",
|
||||||
|
document.content,
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
f'\n\n高机动性身法与独特枪械反馈',
|
||||||
|
document.content,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_html_to_markdown_preserves_indentation_and_code_block_blanks(self):
|
||||||
|
markdown = MHTMLParser(
|
||||||
|
file_name="article.mhtml", file_type="mhtml"
|
||||||
|
)._html_to_markdown(
|
||||||
|
"<ul><li>parent<ul><li>child</li></ul></li></ul>"
|
||||||
|
"<blockquote><p>quoted</p></blockquote>"
|
||||||
|
"<pre><code>line1\n\n indented\n</code></pre>"
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIn("* parent\n + child", markdown)
|
||||||
|
self.assertIn("\n\n> quoted\n\n", markdown)
|
||||||
|
self.assertIn("```\nline1\n\n indented\n```", markdown)
|
||||||
|
|
||||||
|
def test_html_to_markdown_preserves_nested_list_indentation(self):
|
||||||
|
markdown = MHTMLParser(
|
||||||
|
file_name="article.mhtml", file_type="mhtml"
|
||||||
|
)._html_to_markdown("<ul><li>parent<ul><li>child</li></ul></li></ul>")
|
||||||
|
|
||||||
|
self.assertIn("* parent\n + child", markdown)
|
||||||
|
|
||||||
|
def test_html_to_markdown_preserves_blockquote_boundaries(self):
|
||||||
|
markdown = MHTMLParser(
|
||||||
|
file_name="article.mhtml", file_type="mhtml"
|
||||||
|
)._html_to_markdown("<p>before</p><blockquote><p>quoted</p></blockquote><p>after</p>")
|
||||||
|
|
||||||
|
self.assertIn("before\n\n> quoted\n\nafter", markdown)
|
||||||
|
|
||||||
|
def test_html_to_markdown_preserves_fenced_code_blank_lines(self):
|
||||||
|
markdown = MHTMLParser(
|
||||||
|
file_name="article.mhtml", file_type="mhtml"
|
||||||
|
)._html_to_markdown("<pre><code>line1\n\n\nline2\n</code></pre>")
|
||||||
|
|
||||||
|
self.assertIn("```\nline1\n\n\nline2\n```", markdown)
|
||||||
|
|
||||||
|
def test_html_to_markdown_collapses_excess_blank_lines_outside_code(self):
|
||||||
|
markdown = MHTMLParser._normalize_markdown("alpha\n\n \n\t\n\nbeta")
|
||||||
|
|
||||||
|
self.assertEqual(markdown, "alpha\n\nbeta")
|
||||||
|
self.assertNotIn("\n\n\n", markdown)
|
||||||
|
|
||||||
|
def test_html_to_markdown_preserves_hard_break_spaces(self):
|
||||||
|
markdown = MHTMLParser(
|
||||||
|
file_name="article.mhtml", file_type="mhtml"
|
||||||
|
)._html_to_markdown("<p>alpha<br>beta</p>")
|
||||||
|
|
||||||
|
self.assertEqual(markdown, "alpha \nbeta")
|
||||||
|
|
||||||
|
def test_normalize_markdown_preserves_two_space_hard_break(self):
|
||||||
|
markdown = MHTMLParser._normalize_markdown("alpha \nbeta")
|
||||||
|
|
||||||
|
self.assertEqual(markdown, "alpha \nbeta")
|
||||||
|
|
||||||
|
def test_html_to_markdown_normalizes_crlf(self):
|
||||||
|
markdown = MHTMLParser._normalize_markdown("alpha\r\n\r\nbeta\rgamma")
|
||||||
|
|
||||||
|
self.assertEqual(markdown, "alpha\n\nbeta\ngamma")
|
||||||
|
|
||||||
|
def test_html_to_markdown_does_not_strip_leading_indentation_at_document_start(self):
|
||||||
|
markdown = MHTMLParser._normalize_markdown(" indented start\n")
|
||||||
|
|
||||||
|
self.assertEqual(markdown, " indented start")
|
||||||
|
|
||||||
|
def test_mhtml_shared_contract_fixture(self):
|
||||||
|
fixture = REPO_ROOT / "testdata" / "mhtml" / "titled-image.mhtml"
|
||||||
|
contract_path = REPO_ROOT / "testdata" / "mhtml" / "titled-image-contract.json"
|
||||||
|
contract = json.loads(contract_path.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
document = MHTMLParser(
|
||||||
|
file_name="titled-image.mhtml", file_type="mhtml"
|
||||||
|
).parse_into_text(fixture.read_bytes())
|
||||||
|
|
||||||
|
self.assertEqual(document.content, contract["markdown_content"])
|
||||||
|
self.assertEqual(len(document.images), 1)
|
||||||
|
image_contract = contract["images"][0]
|
||||||
|
self.assertIn(image_contract["original_ref"], document.images)
|
||||||
|
self.assertEqual(
|
||||||
|
base64.b64decode(document.images[image_contract["original_ref"]]),
|
||||||
|
base64.b64decode(image_contract["image_data_base64"]),
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
'| 赛季制建立 | BP、Rank |\n\n.gif "阶段 1) 图片")',
|
||||||
|
document.content,
|
||||||
|
)
|
||||||
|
self.assertIn(
|
||||||
|
'.gif "阶段 1) 图片")\n\n高机动性身法与独特枪械反馈',
|
||||||
|
document.content,
|
||||||
|
)
|
||||||
|
self.assertNotIn("\n\n\n", document.content.split("```", 1)[0])
|
||||||
|
self.assertIn("```\nline1\n\n\nline2\n```", document.content)
|
||||||
|
self.assertIn("* parent\n + child", document.content)
|
||||||
|
self.assertIn("alpha \nbeta", document.content)
|
||||||
|
|
||||||
def test_registry_resolves_mhtml(self):
|
def test_registry_resolves_mhtml(self):
|
||||||
self.assertIs(registry.get_parser_class("", "mhtml"), MHTMLParser)
|
self.assertIs(registry.get_parser_class("", "mhtml"), MHTMLParser)
|
||||||
|
|
||||||
|
|||||||
@@ -104,18 +104,16 @@ func (r *ImageResolver) ResolveAndStore(
|
|||||||
}
|
}
|
||||||
savedRefs := make(map[string]StoredImage)
|
savedRefs := make(map[string]StoredImage)
|
||||||
|
|
||||||
// Process each image reference found in the markdown.
|
matches := scanMarkdownImageTargets(markdown)
|
||||||
// The URL group supports one level of balanced parentheses so that URLs
|
|
||||||
// like https://example.com/item_(abc)/123 are captured in full.
|
|
||||||
// Allow spaces in URLs (exclude only parens and newlines) to handle
|
|
||||||
// filenames with spaces, e.g. "images/第 1 页.jpg".
|
|
||||||
imgPattern := regexp.MustCompile(`!\[(.*?)\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`)
|
|
||||||
matches := imgPattern.FindAllStringSubmatchIndex(markdown, -1)
|
|
||||||
|
|
||||||
// Process in reverse order to preserve positions when replacing
|
// Process in reverse order to preserve positions when replacing
|
||||||
for i := len(matches) - 1; i >= 0; i-- {
|
for i := len(matches) - 1; i >= 0; i-- {
|
||||||
m := matches[i]
|
match := matches[i]
|
||||||
refPath := markdown[m[4]:m[5]] // group 2: the URL/path
|
rawTarget := markdown[match.TargetStart:match.TargetEnd]
|
||||||
|
refPath, pathStart, pathEnd, ok := splitMarkdownImageTarget(rawTarget, refMap)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
// Skip already-resolved URLs (http/https, unified /files/, or provider:// scheme)
|
// Skip already-resolved URLs (http/https, unified /files/, or provider:// scheme)
|
||||||
if strings.HasPrefix(refPath, "http://") || strings.HasPrefix(refPath, "https://") ||
|
if strings.HasPrefix(refPath, "http://") || strings.HasPrefix(refPath, "https://") ||
|
||||||
@@ -131,7 +129,9 @@ func (r *ImageResolver) ResolveAndStore(
|
|||||||
images = appendStoredImage(images, stored)
|
images = appendStoredImage(images, stored)
|
||||||
|
|
||||||
// Replace in markdown
|
// Replace in markdown
|
||||||
markdown = markdown[:m[4]] + stored.ServingURL + markdown[m[5]:]
|
absolutePathStart := match.TargetStart + pathStart
|
||||||
|
absolutePathEnd := match.TargetStart + pathEnd
|
||||||
|
markdown = markdown[:absolutePathStart] + stored.ServingURL + markdown[absolutePathEnd:]
|
||||||
}
|
}
|
||||||
|
|
||||||
md5, imgRelativeHTML, _ := r.ResolveRelativeHTMLImages(ctx, markdown, fileSvc, tenantID, refMap, savedRefs)
|
md5, imgRelativeHTML, _ := r.ResolveRelativeHTMLImages(ctx, markdown, fileSvc, tenantID, refMap, savedRefs)
|
||||||
|
|||||||
@@ -4,12 +4,15 @@ import (
|
|||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"image"
|
"image"
|
||||||
"image/color"
|
"image/color"
|
||||||
"image/png"
|
"image/png"
|
||||||
"io"
|
"io"
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -274,3 +277,324 @@ func TestResolveAndStore_MultipleFormats(t *testing.T) {
|
|||||||
t.Fatalf("data URI still in output after ResolveAndStore")
|
t.Fatalf("data URI still in output after ResolveAndStore")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolveAndStoreMarkdownImageWithTitle(t *testing.T) {
|
||||||
|
png := createTestPNG(200, 150)
|
||||||
|
result := &types.ReadResult{
|
||||||
|
MarkdownContent: ``,
|
||||||
|
ImageRefs: []types.ImageRef{
|
||||||
|
{
|
||||||
|
Filename: "test.png",
|
||||||
|
OriginalRef: "images/test.png",
|
||||||
|
MimeType: "image/png",
|
||||||
|
ImageData: png,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := &captureSaveBytes{}
|
||||||
|
r := NewImageResolver()
|
||||||
|
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(imgs) != 1 {
|
||||||
|
t.Fatalf("expected 1 image but got %d", len(imgs))
|
||||||
|
}
|
||||||
|
if len(svc.saved) != 1 {
|
||||||
|
t.Fatalf("expected SaveBytes to be called once but got %d", len(svc.saved))
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, ` || !strings.Contains(out, ` "图片")`) {
|
||||||
|
t.Fatalf("markdown image title was not preserved around stored URL: %s", out)
|
||||||
|
}
|
||||||
|
if strings.Contains(out, "images/test.png") {
|
||||||
|
t.Fatalf("original image path was not replaced: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveAndStoreMarkdownImageWithSingleQuotedTitle(t *testing.T) {
|
||||||
|
png := createTestPNG(200, 150)
|
||||||
|
result := &types.ReadResult{
|
||||||
|
MarkdownContent: ``,
|
||||||
|
ImageRefs: []types.ImageRef{
|
||||||
|
{
|
||||||
|
Filename: "test.png",
|
||||||
|
OriginalRef: "images/test.png",
|
||||||
|
MimeType: "image/png",
|
||||||
|
ImageData: png,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := &captureSaveBytes{}
|
||||||
|
r := NewImageResolver()
|
||||||
|
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(imgs) != 1 || len(svc.saved) != 1 {
|
||||||
|
t.Fatalf("expected one stored image, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, ` || !strings.Contains(out, ` '图片')`) {
|
||||||
|
t.Fatalf("markdown image title was not preserved around stored URL: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveAndStoreMarkdownImageWithSpacedFilename(t *testing.T) {
|
||||||
|
png := createTestPNG(200, 150)
|
||||||
|
result := &types.ReadResult{
|
||||||
|
MarkdownContent: ``,
|
||||||
|
ImageRefs: []types.ImageRef{
|
||||||
|
{
|
||||||
|
Filename: "第 1 页.png",
|
||||||
|
OriginalRef: "images/第 1 页.png",
|
||||||
|
MimeType: "image/png",
|
||||||
|
ImageData: png,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := &captureSaveBytes{}
|
||||||
|
r := NewImageResolver()
|
||||||
|
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(imgs) != 1 || len(svc.saved) != 1 {
|
||||||
|
t.Fatalf("expected one stored image, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, ` {
|
||||||
|
t.Fatalf("spaced filename path was not replaced: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveAndStoreMarkdownImageTitleContainingRightParen(t *testing.T) {
|
||||||
|
png := createTestPNG(200, 150)
|
||||||
|
result := &types.ReadResult{
|
||||||
|
MarkdownContent: ` 图片")`,
|
||||||
|
ImageRefs: []types.ImageRef{
|
||||||
|
{
|
||||||
|
Filename: "test.png",
|
||||||
|
OriginalRef: "images/test.png",
|
||||||
|
MimeType: "image/png",
|
||||||
|
ImageData: png,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := &captureSaveBytes{}
|
||||||
|
r := NewImageResolver()
|
||||||
|
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(imgs) != 1 || len(svc.saved) != 1 {
|
||||||
|
t.Fatalf("expected one stored image, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, ` || !strings.Contains(out, ` "阶段 1) 图片")`) {
|
||||||
|
t.Fatalf("right-paren title was not preserved around stored URL: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveAndStoreMarkdownImageWithMultilineTitle(t *testing.T) {
|
||||||
|
png := createTestPNG(200, 150)
|
||||||
|
result := &types.ReadResult{
|
||||||
|
MarkdownContent: "",
|
||||||
|
ImageRefs: []types.ImageRef{
|
||||||
|
{
|
||||||
|
Filename: "test.png",
|
||||||
|
OriginalRef: "images/test.png",
|
||||||
|
MimeType: "image/png",
|
||||||
|
ImageData: png,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := &captureSaveBytes{}
|
||||||
|
r := NewImageResolver()
|
||||||
|
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(imgs) != 1 || len(svc.saved) != 1 {
|
||||||
|
t.Fatalf("expected one stored image, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, " || !strings.Contains(out, "\n \"图片说明\")") {
|
||||||
|
t.Fatalf("multiline title was not preserved around stored URL: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveAndStoreMarkdownImageWithAngleDestination(t *testing.T) {
|
||||||
|
png := createTestPNG(200, 150)
|
||||||
|
result := &types.ReadResult{
|
||||||
|
MarkdownContent: `.png> "阶段 1) 图片")`,
|
||||||
|
ImageRefs: []types.ImageRef{
|
||||||
|
{
|
||||||
|
Filename: "第 1 页 (测试).png",
|
||||||
|
OriginalRef: "images/第 1 页 (测试).png",
|
||||||
|
MimeType: "image/png",
|
||||||
|
ImageData: png,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := &captureSaveBytes{}
|
||||||
|
r := NewImageResolver()
|
||||||
|
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(imgs) != 1 || len(svc.saved) != 1 {
|
||||||
|
t.Fatalf("expected one stored image, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, ` || !strings.Contains(out, `> "阶段 1) 图片")`) {
|
||||||
|
t.Fatalf("angle destination wrapper or title was not preserved: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveAndStoreDuplicateReferencesStoreOnce(t *testing.T) {
|
||||||
|
png := createTestPNG(200, 150)
|
||||||
|
result := &types.ReadResult{
|
||||||
|
MarkdownContent: strings.Join([]string{
|
||||||
|
``,
|
||||||
|
``,
|
||||||
|
}, "\n\n"),
|
||||||
|
ImageRefs: []types.ImageRef{
|
||||||
|
{
|
||||||
|
Filename: "test.png",
|
||||||
|
OriginalRef: "images/test.png",
|
||||||
|
MimeType: "image/png",
|
||||||
|
ImageData: png,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := &captureSaveBytes{}
|
||||||
|
r := NewImageResolver()
|
||||||
|
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(svc.saved) != 1 {
|
||||||
|
t.Fatalf("expected duplicate references to save once but got %d", len(svc.saved))
|
||||||
|
}
|
||||||
|
if len(imgs) != 1 {
|
||||||
|
t.Fatalf("expected one stored image record but got %d", len(imgs))
|
||||||
|
}
|
||||||
|
if strings.Count(out, "local://test/") != 2 {
|
||||||
|
t.Fatalf("expected both markdown references to be replaced: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, `"第一处"`) || !strings.Contains(out, `"第二处"`) {
|
||||||
|
t.Fatalf("duplicate reference titles were not preserved: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveAndStoreUnknownReferenceUnchanged(t *testing.T) {
|
||||||
|
png := createTestPNG(200, 150)
|
||||||
|
result := &types.ReadResult{
|
||||||
|
MarkdownContent: ``,
|
||||||
|
ImageRefs: []types.ImageRef{
|
||||||
|
{
|
||||||
|
Filename: "test.png",
|
||||||
|
OriginalRef: "images/test.png",
|
||||||
|
MimeType: "image/png",
|
||||||
|
ImageData: png,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := &captureSaveBytes{}
|
||||||
|
r := NewImageResolver()
|
||||||
|
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(svc.saved) != 0 || len(imgs) != 0 {
|
||||||
|
t.Fatalf("unknown reference should not be saved, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||||
|
}
|
||||||
|
if out != result.MarkdownContent {
|
||||||
|
t.Fatalf("unknown reference changed: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveAndStoreSharedMHTMLContract(t *testing.T) {
|
||||||
|
type contractImage struct {
|
||||||
|
Filename string `json:"filename"`
|
||||||
|
OriginalRef string `json:"original_ref"`
|
||||||
|
MimeType string `json:"mime_type"`
|
||||||
|
ImageDataBase64 string `json:"image_data_base64"`
|
||||||
|
}
|
||||||
|
var contract struct {
|
||||||
|
MarkdownContent string `json:"markdown_content"`
|
||||||
|
Images []contractImage `json:"images"`
|
||||||
|
}
|
||||||
|
|
||||||
|
contractPath := filepath.Join("..", "..", "..", "testdata", "mhtml", "titled-image-contract.json")
|
||||||
|
raw, err := os.ReadFile(contractPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &contract); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(contract.Images) != 1 {
|
||||||
|
t.Fatalf("expected one contract image but got %d", len(contract.Images))
|
||||||
|
}
|
||||||
|
|
||||||
|
imageContract := contract.Images[0]
|
||||||
|
imageData, err := base64.StdEncoding.DecodeString(imageContract.ImageDataBase64)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
result := &types.ReadResult{
|
||||||
|
MarkdownContent: contract.MarkdownContent,
|
||||||
|
ImageRefs: []types.ImageRef{
|
||||||
|
{
|
||||||
|
Filename: imageContract.Filename,
|
||||||
|
OriginalRef: imageContract.OriginalRef,
|
||||||
|
MimeType: imageContract.MimeType,
|
||||||
|
ImageData: imageData,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := &captureSaveBytes{}
|
||||||
|
r := NewImageResolver()
|
||||||
|
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(svc.saved) != 1 {
|
||||||
|
t.Fatalf("expected SaveBytes once but got %d", len(svc.saved))
|
||||||
|
}
|
||||||
|
if !bytes.Equal(svc.saved[0], imageData) {
|
||||||
|
t.Fatal("stored image bytes do not match contract fixture")
|
||||||
|
}
|
||||||
|
if len(imgs) != 1 {
|
||||||
|
t.Fatalf("expected one stored image record but got %d", len(imgs))
|
||||||
|
}
|
||||||
|
if imgs[0].OriginalRef != imageContract.OriginalRef || imgs[0].MimeType != imageContract.MimeType {
|
||||||
|
t.Fatalf("unexpected stored image metadata: %#v", imgs[0])
|
||||||
|
}
|
||||||
|
if strings.Contains(out, imageContract.OriginalRef) {
|
||||||
|
t.Fatalf("original image path remained in output: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, ` {
|
||||||
|
t.Fatalf("stored URL missing from markdown: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, `"阶段 1) 图片"`) {
|
||||||
|
t.Fatalf("image title was not preserved: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "| 赛季制建立 | BP、Rank |\n\n {
|
||||||
|
t.Fatalf("table-to-image block boundary changed: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, `"阶段 1) 图片")`+"\n\n高机动性身法与独特枪械反馈") {
|
||||||
|
t.Fatalf("image-to-body block boundary changed: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "alpha \nbeta") {
|
||||||
|
t.Fatalf("markdown hard break changed: %s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "```\nline1\n\n\nline2\n```") {
|
||||||
|
t.Fatalf("fenced code blank lines changed: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,262 @@
|
|||||||
|
package docparser
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/Tencent/WeKnora/internal/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
type markdownImageTargetSpan struct {
|
||||||
|
TargetStart int
|
||||||
|
TargetEnd int
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanMarkdownImageTargets(markdown string) []markdownImageTargetSpan {
|
||||||
|
var spans []markdownImageTargetSpan
|
||||||
|
for i := 0; i+1 < len(markdown); i++ {
|
||||||
|
if markdown[i] != '!' || markdown[i+1] != '[' || isEscaped(markdown, i) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
altEnd := findMarkdownImageAltEnd(markdown, i+2)
|
||||||
|
if altEnd == -1 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
targetStart := altEnd + 2
|
||||||
|
targetEnd, ok := findMarkdownImageTargetEnd(markdown, targetStart)
|
||||||
|
if !ok {
|
||||||
|
i = altEnd
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
spans = append(spans, markdownImageTargetSpan{
|
||||||
|
TargetStart: targetStart,
|
||||||
|
TargetEnd: targetEnd,
|
||||||
|
})
|
||||||
|
i = targetEnd
|
||||||
|
}
|
||||||
|
return spans
|
||||||
|
}
|
||||||
|
|
||||||
|
func findMarkdownImageAltEnd(markdown string, start int) int {
|
||||||
|
for i := start; i+1 < len(markdown); i++ {
|
||||||
|
if markdown[i] == ']' && markdown[i+1] == '(' && !isEscaped(markdown, i) {
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
|
||||||
|
func findMarkdownImageTargetEnd(markdown string, start int) (int, bool) {
|
||||||
|
parenDepth := 1
|
||||||
|
inAngleDestination := false
|
||||||
|
seenNonSpace := false
|
||||||
|
var inQuote byte
|
||||||
|
|
||||||
|
for i := start; i < len(markdown); i++ {
|
||||||
|
ch := markdown[i]
|
||||||
|
if ch == '\\' {
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !seenNonSpace && !isMarkdownSpace(ch) {
|
||||||
|
seenNonSpace = true
|
||||||
|
if ch == '<' {
|
||||||
|
inAngleDestination = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if inAngleDestination {
|
||||||
|
if ch == '>' {
|
||||||
|
inAngleDestination = false
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if inQuote != 0 {
|
||||||
|
if ch == inQuote {
|
||||||
|
inQuote = 0
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ch == '"' || ch == '\'') && i > start && isMarkdownSpace(markdown[i-1]) {
|
||||||
|
inQuote = ch
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch ch {
|
||||||
|
case '(':
|
||||||
|
parenDepth++
|
||||||
|
case ')':
|
||||||
|
parenDepth--
|
||||||
|
if parenDepth == 0 {
|
||||||
|
return i, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitMarkdownImageTarget(
|
||||||
|
raw string,
|
||||||
|
refMap map[string]types.ImageRef,
|
||||||
|
) (path string, pathStart int, pathEnd int, ok bool) {
|
||||||
|
start, end := trimMarkdownSpaceBounds(raw, 0, len(raw))
|
||||||
|
if start >= end {
|
||||||
|
return "", 0, 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
trimmed := raw[start:end]
|
||||||
|
if _, found := refMap[trimmed]; found {
|
||||||
|
return trimmed, start, end, true
|
||||||
|
}
|
||||||
|
|
||||||
|
if raw[start] == '<' {
|
||||||
|
return splitAngleMarkdownImageTarget(raw, start, end, refMap)
|
||||||
|
}
|
||||||
|
|
||||||
|
titleStart, found := parseMarkdownImageTitleSuffix(trimmed)
|
||||||
|
if !found {
|
||||||
|
return "", 0, 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
pathEnd = start + titleStart
|
||||||
|
for pathEnd > start && isMarkdownSpace(raw[pathEnd-1]) {
|
||||||
|
pathEnd--
|
||||||
|
}
|
||||||
|
if pathEnd == start {
|
||||||
|
return "", 0, 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
path = raw[start:pathEnd]
|
||||||
|
if _, found := refMap[path]; !found {
|
||||||
|
return "", 0, 0, false
|
||||||
|
}
|
||||||
|
return path, start, pathEnd, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitAngleMarkdownImageTarget(
|
||||||
|
raw string,
|
||||||
|
start int,
|
||||||
|
end int,
|
||||||
|
refMap map[string]types.ImageRef,
|
||||||
|
) (path string, pathStart int, pathEnd int, ok bool) {
|
||||||
|
close := -1
|
||||||
|
for i := start + 1; i < end; i++ {
|
||||||
|
if raw[i] == '>' && !isEscaped(raw, i) {
|
||||||
|
close = i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if close == -1 {
|
||||||
|
return "", 0, 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
path = raw[start+1 : close]
|
||||||
|
if _, found := refMap[path]; !found {
|
||||||
|
return "", 0, 0, false
|
||||||
|
}
|
||||||
|
if !isEmptyOrMarkdownImageTitleSuffix(raw[close+1 : end]) {
|
||||||
|
return "", 0, 0, false
|
||||||
|
}
|
||||||
|
return path, start + 1, close, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseMarkdownImageTitleSuffix(raw string) (titleStart int, ok bool) {
|
||||||
|
_, end := trimMarkdownSpaceBounds(raw, 0, len(raw))
|
||||||
|
if end == 0 {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
switch raw[end-1] {
|
||||||
|
case '"', '\'':
|
||||||
|
quote := raw[end-1]
|
||||||
|
for i := end - 2; i >= 0; i-- {
|
||||||
|
if raw[i] != quote || isEscaped(raw, i) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if i == 0 || !isMarkdownSpace(raw[i-1]) {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
if markdownTitleHasBlankLine(raw[i+1 : end-1]) {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return i, true
|
||||||
|
}
|
||||||
|
case ')':
|
||||||
|
depth := 0
|
||||||
|
for i := end - 2; i >= 0; i-- {
|
||||||
|
if isEscaped(raw, i) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch raw[i] {
|
||||||
|
case ')':
|
||||||
|
depth++
|
||||||
|
case '(':
|
||||||
|
if depth == 0 {
|
||||||
|
if i == 0 || !isMarkdownSpace(raw[i-1]) {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
if markdownTitleHasBlankLine(raw[i+1 : end-1]) {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
return i, true
|
||||||
|
}
|
||||||
|
depth--
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func isEmptyOrMarkdownImageTitleSuffix(raw string) bool {
|
||||||
|
start, end := trimMarkdownSpaceBounds(raw, 0, len(raw))
|
||||||
|
if start == end {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
titleStart, ok := parseMarkdownImageTitleSuffix(raw[:end])
|
||||||
|
return ok && isAllMarkdownSpace(raw[:titleStart])
|
||||||
|
}
|
||||||
|
|
||||||
|
func markdownTitleHasBlankLine(title string) bool {
|
||||||
|
for _, line := range strings.Split(title, "\n") {
|
||||||
|
if strings.Trim(line, " \t") == "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func isAllMarkdownSpace(raw string) bool {
|
||||||
|
for i := 0; i < len(raw); i++ {
|
||||||
|
if !isMarkdownSpace(raw[i]) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func trimMarkdownSpaceBounds(raw string, start int, end int) (int, int) {
|
||||||
|
for start < end && isMarkdownSpace(raw[start]) {
|
||||||
|
start++
|
||||||
|
}
|
||||||
|
for end > start && isMarkdownSpace(raw[end-1]) {
|
||||||
|
end--
|
||||||
|
}
|
||||||
|
return start, end
|
||||||
|
}
|
||||||
|
|
||||||
|
func isMarkdownSpace(b byte) bool {
|
||||||
|
return b == ' ' || b == '\t' || b == '\n' || b == '\r'
|
||||||
|
}
|
||||||
|
|
||||||
|
func isEscaped(s string, pos int) bool {
|
||||||
|
backslashes := 0
|
||||||
|
for i := pos - 1; i >= 0 && s[i] == '\\'; i-- {
|
||||||
|
backslashes++
|
||||||
|
}
|
||||||
|
return backslashes%2 == 1
|
||||||
|
}
|
||||||
@@ -0,0 +1,203 @@
|
|||||||
|
package docparser
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/Tencent/WeKnora/internal/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestScanMarkdownImageTargets(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
input string
|
||||||
|
targets []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "without title",
|
||||||
|
input: `before  after`,
|
||||||
|
targets: []string{`images/a.png`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "double quoted title",
|
||||||
|
input: ``,
|
||||||
|
targets: []string{`images/a.png "图"`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "single quoted title",
|
||||||
|
input: ``,
|
||||||
|
targets: []string{`images/a.png '图'`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "parenthesized title",
|
||||||
|
input: `)`,
|
||||||
|
targets: []string{`images/a.png (图)`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "title containing right paren",
|
||||||
|
input: ` 结果")`,
|
||||||
|
targets: []string{`images/a.png "阶段 1) 结果"`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "title containing both parens",
|
||||||
|
input: ` 结果')`,
|
||||||
|
targets: []string{`images/a.png '阶段 (1) 结果'`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "escaped quote in title",
|
||||||
|
input: ``,
|
||||||
|
targets: []string{`images/a.png "阶段 \"1\""`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiline title",
|
||||||
|
input: "",
|
||||||
|
targets: []string{"images/a.png\n \"多行 title\""},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "spaced path",
|
||||||
|
input: ``,
|
||||||
|
targets: []string{`images/第 1 页.png`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path containing balanced parens",
|
||||||
|
input: `.png "title")`,
|
||||||
|
targets: []string{`images/a_(1).png "title"`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "angle destination",
|
||||||
|
input: ``,
|
||||||
|
targets: []string{`<images/a b.png> "title"`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "malformed image is skipped",
|
||||||
|
input: ` after `,
|
||||||
|
targets: []string{`images/b.png`},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "escaped image marker is skipped",
|
||||||
|
input: `\ `,
|
||||||
|
targets: []string{`images/b.png`},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
spans := scanMarkdownImageTargets(tt.input)
|
||||||
|
if len(spans) != len(tt.targets) {
|
||||||
|
t.Fatalf("got %d spans, want %d: %#v", len(spans), len(tt.targets), spans)
|
||||||
|
}
|
||||||
|
for i, span := range spans {
|
||||||
|
got := tt.input[span.TargetStart:span.TargetEnd]
|
||||||
|
if got != tt.targets[i] {
|
||||||
|
t.Fatalf("target %d = %q, want %q", i, got, tt.targets[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSplitMarkdownImageTarget(t *testing.T) {
|
||||||
|
refMap := map[string]types.ImageRef{
|
||||||
|
"images/a.png": {OriginalRef: "images/a.png"},
|
||||||
|
"images/第 1 页.png": {OriginalRef: "images/第 1 页.png"},
|
||||||
|
"images/a_(1).png": {OriginalRef: "images/a_(1).png"},
|
||||||
|
"images/第 1 页 (测试).gif": {OriginalRef: "images/第 1 页 (测试).gif"},
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
raw string
|
||||||
|
wantPath string
|
||||||
|
wantText string
|
||||||
|
wantOK bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "without title",
|
||||||
|
raw: `images/a.png`,
|
||||||
|
wantPath: `images/a.png`,
|
||||||
|
wantText: `local://stored`,
|
||||||
|
wantOK: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "double quoted title",
|
||||||
|
raw: `images/a.png "图"`,
|
||||||
|
wantPath: `images/a.png`,
|
||||||
|
wantText: `local://stored "图"`,
|
||||||
|
wantOK: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "single quoted title",
|
||||||
|
raw: `images/a.png '图'`,
|
||||||
|
wantPath: `images/a.png`,
|
||||||
|
wantText: `local://stored '图'`,
|
||||||
|
wantOK: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "parenthesized title",
|
||||||
|
raw: `images/a.png (阶段 (1) 结果)`,
|
||||||
|
wantPath: `images/a.png`,
|
||||||
|
wantText: `local://stored (阶段 (1) 结果)`,
|
||||||
|
wantOK: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multiline title",
|
||||||
|
raw: "images/a.png\n \"多行 title\"",
|
||||||
|
wantPath: `images/a.png`,
|
||||||
|
wantText: "local://stored\n \"多行 title\"",
|
||||||
|
wantOK: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path with spaces wins without title",
|
||||||
|
raw: `images/第 1 页.png`,
|
||||||
|
wantPath: `images/第 1 页.png`,
|
||||||
|
wantText: `local://stored`,
|
||||||
|
wantOK: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "path with balanced parens",
|
||||||
|
raw: `images/a_(1).png "title"`,
|
||||||
|
wantPath: `images/a_(1).png`,
|
||||||
|
wantText: `local://stored "title"`,
|
||||||
|
wantOK: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "angle destination replaces only inner path",
|
||||||
|
raw: `<images/第 1 页 (测试).gif> "阶段 1) 图片"`,
|
||||||
|
wantPath: `images/第 1 页 (测试).gif`,
|
||||||
|
wantText: `<local://stored> "阶段 1) 图片"`,
|
||||||
|
wantOK: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown reference",
|
||||||
|
raw: `images/missing.png "title"`,
|
||||||
|
wantOK: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank line in title is invalid",
|
||||||
|
raw: "images/a.png \"line1\n\nline2\"",
|
||||||
|
wantOK: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
path, pathStart, pathEnd, ok := splitMarkdownImageTarget(tt.raw, refMap)
|
||||||
|
if ok != tt.wantOK {
|
||||||
|
t.Fatalf("ok = %v, want %v", ok, tt.wantOK)
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if path != tt.wantPath {
|
||||||
|
t.Fatalf("path = %q, want %q", path, tt.wantPath)
|
||||||
|
}
|
||||||
|
replaced := tt.raw[:pathStart] + "local://stored" + tt.raw[pathEnd:]
|
||||||
|
if replaced != tt.wantText {
|
||||||
|
t.Fatalf("replacement = %q, want %q", replaced, tt.wantText)
|
||||||
|
}
|
||||||
|
if strings.Contains(replaced, tt.wantPath) {
|
||||||
|
t.Fatalf("replacement still contains original path: %q", replaced)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+11
@@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"markdown_content": "| 体验方向 | 代表内容 |\n| --- | --- |\n| 赛季制建立 | BP、Rank |\n\n.gif \"阶段 1) 图片\")\n\n高机动性身法与独特枪械反馈\n\nalpha \nbeta\n\n* parent\n + child\n\n> quoted\n\n```\nline1\n\n\nline2\n```",
|
||||||
|
"images": [
|
||||||
|
{
|
||||||
|
"filename": "第 1 页 (测试).gif",
|
||||||
|
"original_ref": "images/第 1 页 (测试).gif",
|
||||||
|
"mime_type": "image/gif",
|
||||||
|
"image_data_base64": "R0lGODdhQABAAIEAAFCM3AAAAAAAAAAAACwAAAAAQABAAEAIaQABCBxIsKDBgwgTKlzIsKHDhxAjSpxIsaLFixgzatzIsaPHjyBDihxJsqTJkyhTqlzJsqXLlzBjypxJs6bNmzhz6tzJs6fPn0CDCh1KtKjRo0iTKl3KtKnTp1CjSp1KtarVq1izagUQEAA7"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Vendored
+48
@@ -0,0 +1,48 @@
|
|||||||
|
MIME-Version: 1.0
|
||||||
|
Subject: Titled Image Contract
|
||||||
|
Content-Type: multipart/related; type="text/html"; boundary="----=_NextPart_Contract"
|
||||||
|
|
||||||
|
------=_NextPart_Contract
|
||||||
|
Content-Type: text/html; charset="utf-8"
|
||||||
|
Content-Transfer-Encoding: 8bit
|
||||||
|
Content-Location: https://example.com/titled-image
|
||||||
|
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<table>
|
||||||
|
<tr><th>体验方向</th><th>代表内容</th></tr>
|
||||||
|
<tr><td>赛季制建立</td><td>BP、Rank</td></tr>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<p><img src="images/%E7%AC%AC%201%20%E9%A1%B5%20%28%E6%B5%8B%E8%AF%95%29.gif" alt="图片" title="阶段 1) 图片"></p>
|
||||||
|
|
||||||
|
<p>高机动性身法与独特枪械反馈</p>
|
||||||
|
|
||||||
|
<p>alpha<br>beta</p>
|
||||||
|
|
||||||
|
<p><br><br><br></p>
|
||||||
|
|
||||||
|
<ul>
|
||||||
|
<li>parent
|
||||||
|
<ul><li>child</li></ul>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
<blockquote><p>quoted</p></blockquote>
|
||||||
|
|
||||||
|
<pre><code>line1
|
||||||
|
|
||||||
|
|
||||||
|
line2
|
||||||
|
</code></pre>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
|
||||||
|
------=_NextPart_Contract
|
||||||
|
Content-Type: image/gif
|
||||||
|
Content-Transfer-Encoding: base64
|
||||||
|
Content-Location: images/%E7%AC%AC%201%20%E9%A1%B5%20%28%E6%B5%8B%E8%AF%95%29.gif
|
||||||
|
|
||||||
|
R0lGODdhQABAAIEAAFCM3AAAAAAAAAAAACwAAAAAQABAAEAIaQABCBxIsKDBgwgTKlzIsKHDhxAjSpxIsaLFixgzatzIsaPHjyBDihxJsqTJkyhTqlzJsqXLlzBjypxJs6bNmzhz6tzJs6fPn0CDCh1KtKjRo0iTKl3KtKnTp1CjSp1KtarVq1izagUQEAA7
|
||||||
|
|
||||||
|
------=_NextPart_Contract--
|
||||||
Reference in New Issue
Block a user