fix: preserve MHTML image resolution and Markdown structure (#1743)
* fix(docreader): preserve MHTML blocks and resolve titled images * fix(docreader): harden MHTML markdown image resolution * fix(docreader): preserve markdown hard break spaces --------- Co-authored-by: FFFFFFpy <bushiyan@outlook.com>
This commit is contained in:
@@ -104,18 +104,16 @@ func (r *ImageResolver) ResolveAndStore(
|
||||
}
|
||||
savedRefs := make(map[string]StoredImage)
|
||||
|
||||
// Process each image reference found in the markdown.
|
||||
// The URL group supports one level of balanced parentheses so that URLs
|
||||
// like https://example.com/item_(abc)/123 are captured in full.
|
||||
// Allow spaces in URLs (exclude only parens and newlines) to handle
|
||||
// filenames with spaces, e.g. "images/第 1 页.jpg".
|
||||
imgPattern := regexp.MustCompile(`!\[(.*?)\]\(([^()\n]*(?:\([^)]*\)[^()\n]*)*)\)`)
|
||||
matches := imgPattern.FindAllStringSubmatchIndex(markdown, -1)
|
||||
matches := scanMarkdownImageTargets(markdown)
|
||||
|
||||
// Process in reverse order to preserve positions when replacing
|
||||
for i := len(matches) - 1; i >= 0; i-- {
|
||||
m := matches[i]
|
||||
refPath := markdown[m[4]:m[5]] // group 2: the URL/path
|
||||
match := matches[i]
|
||||
rawTarget := markdown[match.TargetStart:match.TargetEnd]
|
||||
refPath, pathStart, pathEnd, ok := splitMarkdownImageTarget(rawTarget, refMap)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip already-resolved URLs (http/https, unified /files/, or provider:// scheme)
|
||||
if strings.HasPrefix(refPath, "http://") || strings.HasPrefix(refPath, "https://") ||
|
||||
@@ -131,7 +129,9 @@ func (r *ImageResolver) ResolveAndStore(
|
||||
images = appendStoredImage(images, stored)
|
||||
|
||||
// Replace in markdown
|
||||
markdown = markdown[:m[4]] + stored.ServingURL + markdown[m[5]:]
|
||||
absolutePathStart := match.TargetStart + pathStart
|
||||
absolutePathEnd := match.TargetStart + pathEnd
|
||||
markdown = markdown[:absolutePathStart] + stored.ServingURL + markdown[absolutePathEnd:]
|
||||
}
|
||||
|
||||
md5, imgRelativeHTML, _ := r.ResolveRelativeHTMLImages(ctx, markdown, fileSvc, tenantID, refMap, savedRefs)
|
||||
|
||||
@@ -4,12 +4,15 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -274,3 +277,324 @@ func TestResolveAndStore_MultipleFormats(t *testing.T) {
|
||||
t.Fatalf("data URI still in output after ResolveAndStore")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAndStoreMarkdownImageWithTitle(t *testing.T) {
|
||||
png := createTestPNG(200, 150)
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: ``,
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: "test.png",
|
||||
OriginalRef: "images/test.png",
|
||||
MimeType: "image/png",
|
||||
ImageData: png,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(imgs) != 1 {
|
||||
t.Fatalf("expected 1 image but got %d", len(imgs))
|
||||
}
|
||||
if len(svc.saved) != 1 {
|
||||
t.Fatalf("expected SaveBytes to be called once but got %d", len(svc.saved))
|
||||
}
|
||||
if !strings.Contains(out, ` || !strings.Contains(out, ` "图片")`) {
|
||||
t.Fatalf("markdown image title was not preserved around stored URL: %s", out)
|
||||
}
|
||||
if strings.Contains(out, "images/test.png") {
|
||||
t.Fatalf("original image path was not replaced: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAndStoreMarkdownImageWithSingleQuotedTitle(t *testing.T) {
|
||||
png := createTestPNG(200, 150)
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: ``,
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: "test.png",
|
||||
OriginalRef: "images/test.png",
|
||||
MimeType: "image/png",
|
||||
ImageData: png,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(imgs) != 1 || len(svc.saved) != 1 {
|
||||
t.Fatalf("expected one stored image, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||
}
|
||||
if !strings.Contains(out, ` || !strings.Contains(out, ` '图片')`) {
|
||||
t.Fatalf("markdown image title was not preserved around stored URL: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAndStoreMarkdownImageWithSpacedFilename(t *testing.T) {
|
||||
png := createTestPNG(200, 150)
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: ``,
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: "第 1 页.png",
|
||||
OriginalRef: "images/第 1 页.png",
|
||||
MimeType: "image/png",
|
||||
ImageData: png,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(imgs) != 1 || len(svc.saved) != 1 {
|
||||
t.Fatalf("expected one stored image, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||
}
|
||||
if !strings.Contains(out, ` {
|
||||
t.Fatalf("spaced filename path was not replaced: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAndStoreMarkdownImageTitleContainingRightParen(t *testing.T) {
|
||||
png := createTestPNG(200, 150)
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: ` 图片")`,
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: "test.png",
|
||||
OriginalRef: "images/test.png",
|
||||
MimeType: "image/png",
|
||||
ImageData: png,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(imgs) != 1 || len(svc.saved) != 1 {
|
||||
t.Fatalf("expected one stored image, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||
}
|
||||
if !strings.Contains(out, ` || !strings.Contains(out, ` "阶段 1) 图片")`) {
|
||||
t.Fatalf("right-paren title was not preserved around stored URL: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAndStoreMarkdownImageWithMultilineTitle(t *testing.T) {
|
||||
png := createTestPNG(200, 150)
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: "",
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: "test.png",
|
||||
OriginalRef: "images/test.png",
|
||||
MimeType: "image/png",
|
||||
ImageData: png,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(imgs) != 1 || len(svc.saved) != 1 {
|
||||
t.Fatalf("expected one stored image, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||
}
|
||||
if !strings.Contains(out, " || !strings.Contains(out, "\n \"图片说明\")") {
|
||||
t.Fatalf("multiline title was not preserved around stored URL: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAndStoreMarkdownImageWithAngleDestination(t *testing.T) {
|
||||
png := createTestPNG(200, 150)
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: `.png> "阶段 1) 图片")`,
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: "第 1 页 (测试).png",
|
||||
OriginalRef: "images/第 1 页 (测试).png",
|
||||
MimeType: "image/png",
|
||||
ImageData: png,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(imgs) != 1 || len(svc.saved) != 1 {
|
||||
t.Fatalf("expected one stored image, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||
}
|
||||
if !strings.Contains(out, ` || !strings.Contains(out, `> "阶段 1) 图片")`) {
|
||||
t.Fatalf("angle destination wrapper or title was not preserved: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAndStoreDuplicateReferencesStoreOnce(t *testing.T) {
|
||||
png := createTestPNG(200, 150)
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: strings.Join([]string{
|
||||
``,
|
||||
``,
|
||||
}, "\n\n"),
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: "test.png",
|
||||
OriginalRef: "images/test.png",
|
||||
MimeType: "image/png",
|
||||
ImageData: png,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(svc.saved) != 1 {
|
||||
t.Fatalf("expected duplicate references to save once but got %d", len(svc.saved))
|
||||
}
|
||||
if len(imgs) != 1 {
|
||||
t.Fatalf("expected one stored image record but got %d", len(imgs))
|
||||
}
|
||||
if strings.Count(out, "local://test/") != 2 {
|
||||
t.Fatalf("expected both markdown references to be replaced: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, `"第一处"`) || !strings.Contains(out, `"第二处"`) {
|
||||
t.Fatalf("duplicate reference titles were not preserved: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAndStoreUnknownReferenceUnchanged(t *testing.T) {
|
||||
png := createTestPNG(200, 150)
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: ``,
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: "test.png",
|
||||
OriginalRef: "images/test.png",
|
||||
MimeType: "image/png",
|
||||
ImageData: png,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(svc.saved) != 0 || len(imgs) != 0 {
|
||||
t.Fatalf("unknown reference should not be saved, got imgs=%d saved=%d", len(imgs), len(svc.saved))
|
||||
}
|
||||
if out != result.MarkdownContent {
|
||||
t.Fatalf("unknown reference changed: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAndStoreSharedMHTMLContract(t *testing.T) {
|
||||
type contractImage struct {
|
||||
Filename string `json:"filename"`
|
||||
OriginalRef string `json:"original_ref"`
|
||||
MimeType string `json:"mime_type"`
|
||||
ImageDataBase64 string `json:"image_data_base64"`
|
||||
}
|
||||
var contract struct {
|
||||
MarkdownContent string `json:"markdown_content"`
|
||||
Images []contractImage `json:"images"`
|
||||
}
|
||||
|
||||
contractPath := filepath.Join("..", "..", "..", "testdata", "mhtml", "titled-image-contract.json")
|
||||
raw, err := os.ReadFile(contractPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(raw, &contract); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(contract.Images) != 1 {
|
||||
t.Fatalf("expected one contract image but got %d", len(contract.Images))
|
||||
}
|
||||
|
||||
imageContract := contract.Images[0]
|
||||
imageData, err := base64.StdEncoding.DecodeString(imageContract.ImageDataBase64)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := &types.ReadResult{
|
||||
MarkdownContent: contract.MarkdownContent,
|
||||
ImageRefs: []types.ImageRef{
|
||||
{
|
||||
Filename: imageContract.Filename,
|
||||
OriginalRef: imageContract.OriginalRef,
|
||||
MimeType: imageContract.MimeType,
|
||||
ImageData: imageData,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
svc := &captureSaveBytes{}
|
||||
r := NewImageResolver()
|
||||
out, imgs, err := r.ResolveAndStore(context.Background(), result, svc, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(svc.saved) != 1 {
|
||||
t.Fatalf("expected SaveBytes once but got %d", len(svc.saved))
|
||||
}
|
||||
if !bytes.Equal(svc.saved[0], imageData) {
|
||||
t.Fatal("stored image bytes do not match contract fixture")
|
||||
}
|
||||
if len(imgs) != 1 {
|
||||
t.Fatalf("expected one stored image record but got %d", len(imgs))
|
||||
}
|
||||
if imgs[0].OriginalRef != imageContract.OriginalRef || imgs[0].MimeType != imageContract.MimeType {
|
||||
t.Fatalf("unexpected stored image metadata: %#v", imgs[0])
|
||||
}
|
||||
if strings.Contains(out, imageContract.OriginalRef) {
|
||||
t.Fatalf("original image path remained in output: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, ` {
|
||||
t.Fatalf("stored URL missing from markdown: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, `"阶段 1) 图片"`) {
|
||||
t.Fatalf("image title was not preserved: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, "| 赛季制建立 | BP、Rank |\n\n {
|
||||
t.Fatalf("table-to-image block boundary changed: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, `"阶段 1) 图片")`+"\n\n高机动性身法与独特枪械反馈") {
|
||||
t.Fatalf("image-to-body block boundary changed: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, "alpha \nbeta") {
|
||||
t.Fatalf("markdown hard break changed: %s", out)
|
||||
}
|
||||
if !strings.Contains(out, "```\nline1\n\n\nline2\n```") {
|
||||
t.Fatalf("fenced code blank lines changed: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
type markdownImageTargetSpan struct {
|
||||
TargetStart int
|
||||
TargetEnd int
|
||||
}
|
||||
|
||||
func scanMarkdownImageTargets(markdown string) []markdownImageTargetSpan {
|
||||
var spans []markdownImageTargetSpan
|
||||
for i := 0; i+1 < len(markdown); i++ {
|
||||
if markdown[i] != '!' || markdown[i+1] != '[' || isEscaped(markdown, i) {
|
||||
continue
|
||||
}
|
||||
|
||||
altEnd := findMarkdownImageAltEnd(markdown, i+2)
|
||||
if altEnd == -1 {
|
||||
continue
|
||||
}
|
||||
|
||||
targetStart := altEnd + 2
|
||||
targetEnd, ok := findMarkdownImageTargetEnd(markdown, targetStart)
|
||||
if !ok {
|
||||
i = altEnd
|
||||
continue
|
||||
}
|
||||
spans = append(spans, markdownImageTargetSpan{
|
||||
TargetStart: targetStart,
|
||||
TargetEnd: targetEnd,
|
||||
})
|
||||
i = targetEnd
|
||||
}
|
||||
return spans
|
||||
}
|
||||
|
||||
func findMarkdownImageAltEnd(markdown string, start int) int {
|
||||
for i := start; i+1 < len(markdown); i++ {
|
||||
if markdown[i] == ']' && markdown[i+1] == '(' && !isEscaped(markdown, i) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func findMarkdownImageTargetEnd(markdown string, start int) (int, bool) {
|
||||
parenDepth := 1
|
||||
inAngleDestination := false
|
||||
seenNonSpace := false
|
||||
var inQuote byte
|
||||
|
||||
for i := start; i < len(markdown); i++ {
|
||||
ch := markdown[i]
|
||||
if ch == '\\' {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
if !seenNonSpace && !isMarkdownSpace(ch) {
|
||||
seenNonSpace = true
|
||||
if ch == '<' {
|
||||
inAngleDestination = true
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if inAngleDestination {
|
||||
if ch == '>' {
|
||||
inAngleDestination = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if inQuote != 0 {
|
||||
if ch == inQuote {
|
||||
inQuote = 0
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (ch == '"' || ch == '\'') && i > start && isMarkdownSpace(markdown[i-1]) {
|
||||
inQuote = ch
|
||||
continue
|
||||
}
|
||||
|
||||
switch ch {
|
||||
case '(':
|
||||
parenDepth++
|
||||
case ')':
|
||||
parenDepth--
|
||||
if parenDepth == 0 {
|
||||
return i, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func splitMarkdownImageTarget(
|
||||
raw string,
|
||||
refMap map[string]types.ImageRef,
|
||||
) (path string, pathStart int, pathEnd int, ok bool) {
|
||||
start, end := trimMarkdownSpaceBounds(raw, 0, len(raw))
|
||||
if start >= end {
|
||||
return "", 0, 0, false
|
||||
}
|
||||
|
||||
trimmed := raw[start:end]
|
||||
if _, found := refMap[trimmed]; found {
|
||||
return trimmed, start, end, true
|
||||
}
|
||||
|
||||
if raw[start] == '<' {
|
||||
return splitAngleMarkdownImageTarget(raw, start, end, refMap)
|
||||
}
|
||||
|
||||
titleStart, found := parseMarkdownImageTitleSuffix(trimmed)
|
||||
if !found {
|
||||
return "", 0, 0, false
|
||||
}
|
||||
|
||||
pathEnd = start + titleStart
|
||||
for pathEnd > start && isMarkdownSpace(raw[pathEnd-1]) {
|
||||
pathEnd--
|
||||
}
|
||||
if pathEnd == start {
|
||||
return "", 0, 0, false
|
||||
}
|
||||
|
||||
path = raw[start:pathEnd]
|
||||
if _, found := refMap[path]; !found {
|
||||
return "", 0, 0, false
|
||||
}
|
||||
return path, start, pathEnd, true
|
||||
}
|
||||
|
||||
func splitAngleMarkdownImageTarget(
|
||||
raw string,
|
||||
start int,
|
||||
end int,
|
||||
refMap map[string]types.ImageRef,
|
||||
) (path string, pathStart int, pathEnd int, ok bool) {
|
||||
close := -1
|
||||
for i := start + 1; i < end; i++ {
|
||||
if raw[i] == '>' && !isEscaped(raw, i) {
|
||||
close = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if close == -1 {
|
||||
return "", 0, 0, false
|
||||
}
|
||||
|
||||
path = raw[start+1 : close]
|
||||
if _, found := refMap[path]; !found {
|
||||
return "", 0, 0, false
|
||||
}
|
||||
if !isEmptyOrMarkdownImageTitleSuffix(raw[close+1 : end]) {
|
||||
return "", 0, 0, false
|
||||
}
|
||||
return path, start + 1, close, true
|
||||
}
|
||||
|
||||
func parseMarkdownImageTitleSuffix(raw string) (titleStart int, ok bool) {
|
||||
_, end := trimMarkdownSpaceBounds(raw, 0, len(raw))
|
||||
if end == 0 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
switch raw[end-1] {
|
||||
case '"', '\'':
|
||||
quote := raw[end-1]
|
||||
for i := end - 2; i >= 0; i-- {
|
||||
if raw[i] != quote || isEscaped(raw, i) {
|
||||
continue
|
||||
}
|
||||
if i == 0 || !isMarkdownSpace(raw[i-1]) {
|
||||
return 0, false
|
||||
}
|
||||
if markdownTitleHasBlankLine(raw[i+1 : end-1]) {
|
||||
return 0, false
|
||||
}
|
||||
return i, true
|
||||
}
|
||||
case ')':
|
||||
depth := 0
|
||||
for i := end - 2; i >= 0; i-- {
|
||||
if isEscaped(raw, i) {
|
||||
continue
|
||||
}
|
||||
switch raw[i] {
|
||||
case ')':
|
||||
depth++
|
||||
case '(':
|
||||
if depth == 0 {
|
||||
if i == 0 || !isMarkdownSpace(raw[i-1]) {
|
||||
return 0, false
|
||||
}
|
||||
if markdownTitleHasBlankLine(raw[i+1 : end-1]) {
|
||||
return 0, false
|
||||
}
|
||||
return i, true
|
||||
}
|
||||
depth--
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func isEmptyOrMarkdownImageTitleSuffix(raw string) bool {
|
||||
start, end := trimMarkdownSpaceBounds(raw, 0, len(raw))
|
||||
if start == end {
|
||||
return true
|
||||
}
|
||||
titleStart, ok := parseMarkdownImageTitleSuffix(raw[:end])
|
||||
return ok && isAllMarkdownSpace(raw[:titleStart])
|
||||
}
|
||||
|
||||
func markdownTitleHasBlankLine(title string) bool {
|
||||
for _, line := range strings.Split(title, "\n") {
|
||||
if strings.Trim(line, " \t") == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isAllMarkdownSpace(raw string) bool {
|
||||
for i := 0; i < len(raw); i++ {
|
||||
if !isMarkdownSpace(raw[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func trimMarkdownSpaceBounds(raw string, start int, end int) (int, int) {
|
||||
for start < end && isMarkdownSpace(raw[start]) {
|
||||
start++
|
||||
}
|
||||
for end > start && isMarkdownSpace(raw[end-1]) {
|
||||
end--
|
||||
}
|
||||
return start, end
|
||||
}
|
||||
|
||||
func isMarkdownSpace(b byte) bool {
|
||||
return b == ' ' || b == '\t' || b == '\n' || b == '\r'
|
||||
}
|
||||
|
||||
func isEscaped(s string, pos int) bool {
|
||||
backslashes := 0
|
||||
for i := pos - 1; i >= 0 && s[i] == '\\'; i-- {
|
||||
backslashes++
|
||||
}
|
||||
return backslashes%2 == 1
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package docparser
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestScanMarkdownImageTargets(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
targets []string
|
||||
}{
|
||||
{
|
||||
name: "without title",
|
||||
input: `before  after`,
|
||||
targets: []string{`images/a.png`},
|
||||
},
|
||||
{
|
||||
name: "double quoted title",
|
||||
input: ``,
|
||||
targets: []string{`images/a.png "图"`},
|
||||
},
|
||||
{
|
||||
name: "single quoted title",
|
||||
input: ``,
|
||||
targets: []string{`images/a.png '图'`},
|
||||
},
|
||||
{
|
||||
name: "parenthesized title",
|
||||
input: `)`,
|
||||
targets: []string{`images/a.png (图)`},
|
||||
},
|
||||
{
|
||||
name: "title containing right paren",
|
||||
input: ` 结果")`,
|
||||
targets: []string{`images/a.png "阶段 1) 结果"`},
|
||||
},
|
||||
{
|
||||
name: "title containing both parens",
|
||||
input: ` 结果')`,
|
||||
targets: []string{`images/a.png '阶段 (1) 结果'`},
|
||||
},
|
||||
{
|
||||
name: "escaped quote in title",
|
||||
input: ``,
|
||||
targets: []string{`images/a.png "阶段 \"1\""`},
|
||||
},
|
||||
{
|
||||
name: "multiline title",
|
||||
input: "",
|
||||
targets: []string{"images/a.png\n \"多行 title\""},
|
||||
},
|
||||
{
|
||||
name: "spaced path",
|
||||
input: ``,
|
||||
targets: []string{`images/第 1 页.png`},
|
||||
},
|
||||
{
|
||||
name: "path containing balanced parens",
|
||||
input: `.png "title")`,
|
||||
targets: []string{`images/a_(1).png "title"`},
|
||||
},
|
||||
{
|
||||
name: "angle destination",
|
||||
input: ``,
|
||||
targets: []string{`<images/a b.png> "title"`},
|
||||
},
|
||||
{
|
||||
name: "malformed image is skipped",
|
||||
input: ` after `,
|
||||
targets: []string{`images/b.png`},
|
||||
},
|
||||
{
|
||||
name: "escaped image marker is skipped",
|
||||
input: `\ `,
|
||||
targets: []string{`images/b.png`},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
spans := scanMarkdownImageTargets(tt.input)
|
||||
if len(spans) != len(tt.targets) {
|
||||
t.Fatalf("got %d spans, want %d: %#v", len(spans), len(tt.targets), spans)
|
||||
}
|
||||
for i, span := range spans {
|
||||
got := tt.input[span.TargetStart:span.TargetEnd]
|
||||
if got != tt.targets[i] {
|
||||
t.Fatalf("target %d = %q, want %q", i, got, tt.targets[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitMarkdownImageTarget(t *testing.T) {
|
||||
refMap := map[string]types.ImageRef{
|
||||
"images/a.png": {OriginalRef: "images/a.png"},
|
||||
"images/第 1 页.png": {OriginalRef: "images/第 1 页.png"},
|
||||
"images/a_(1).png": {OriginalRef: "images/a_(1).png"},
|
||||
"images/第 1 页 (测试).gif": {OriginalRef: "images/第 1 页 (测试).gif"},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
wantPath string
|
||||
wantText string
|
||||
wantOK bool
|
||||
}{
|
||||
{
|
||||
name: "without title",
|
||||
raw: `images/a.png`,
|
||||
wantPath: `images/a.png`,
|
||||
wantText: `local://stored`,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "double quoted title",
|
||||
raw: `images/a.png "图"`,
|
||||
wantPath: `images/a.png`,
|
||||
wantText: `local://stored "图"`,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "single quoted title",
|
||||
raw: `images/a.png '图'`,
|
||||
wantPath: `images/a.png`,
|
||||
wantText: `local://stored '图'`,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "parenthesized title",
|
||||
raw: `images/a.png (阶段 (1) 结果)`,
|
||||
wantPath: `images/a.png`,
|
||||
wantText: `local://stored (阶段 (1) 结果)`,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "multiline title",
|
||||
raw: "images/a.png\n \"多行 title\"",
|
||||
wantPath: `images/a.png`,
|
||||
wantText: "local://stored\n \"多行 title\"",
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "path with spaces wins without title",
|
||||
raw: `images/第 1 页.png`,
|
||||
wantPath: `images/第 1 页.png`,
|
||||
wantText: `local://stored`,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "path with balanced parens",
|
||||
raw: `images/a_(1).png "title"`,
|
||||
wantPath: `images/a_(1).png`,
|
||||
wantText: `local://stored "title"`,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "angle destination replaces only inner path",
|
||||
raw: `<images/第 1 页 (测试).gif> "阶段 1) 图片"`,
|
||||
wantPath: `images/第 1 页 (测试).gif`,
|
||||
wantText: `<local://stored> "阶段 1) 图片"`,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "unknown reference",
|
||||
raw: `images/missing.png "title"`,
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "blank line in title is invalid",
|
||||
raw: "images/a.png \"line1\n\nline2\"",
|
||||
wantOK: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
path, pathStart, pathEnd, ok := splitMarkdownImageTarget(tt.raw, refMap)
|
||||
if ok != tt.wantOK {
|
||||
t.Fatalf("ok = %v, want %v", ok, tt.wantOK)
|
||||
}
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if path != tt.wantPath {
|
||||
t.Fatalf("path = %q, want %q", path, tt.wantPath)
|
||||
}
|
||||
replaced := tt.raw[:pathStart] + "local://stored" + tt.raw[pathEnd:]
|
||||
if replaced != tt.wantText {
|
||||
t.Fatalf("replacement = %q, want %q", replaced, tt.wantText)
|
||||
}
|
||||
if strings.Contains(replaced, tt.wantPath) {
|
||||
t.Fatalf("replacement still contains original path: %q", replaced)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user