Files
dobotai-platform/internal/outboundimage/rewriter.go

213 lines
7.2 KiB
Go

package outboundimage
import (
"context"
"net"
"net/url"
"path"
"regexp"
"strings"
)
var (
resourceMarkdownImageRE = regexp.MustCompile(`!\[[^\]]*\]\((resource://[0-9A-Za-z_-]+)(?:\s+(?:"[^"\r\n]*"|'[^'\r\n]*'|\([^\)\r\n]*\)))?\)`)
resourceHTMLDoubleImageRE = regexp.MustCompile(`(?i)<img\b[^>]*\bsrc\s*=\s*"(resource://[0-9A-Za-z_-]+)"[^>]*>`)
resourceHTMLSingleImageRE = regexp.MustCompile(`(?i)<img\b[^>]*\bsrc\s*=\s*'(resource://[0-9A-Za-z_-]+)'[^>]*>`)
resourceHTMLUnsafeImageRE = regexp.MustCompile(`(?i)<img\b[^>]*resource://[0-9A-Za-z_-]+[^>]*>`)
markdownImageRE = regexp.MustCompile(`!\[[^\]]*\]\(([^\s)]+)(?:\s+(?:"[^"\r\n]*"|'[^'\r\n]*'|\([^\)\r\n]*\)))?\)`)
incompleteMarkdownImageRE = regexp.MustCompile(`!\[[^\]\r\n]*\]\([^\)\r\n]*$`)
htmlImageRE = regexp.MustCompile(`(?i)<img\b[^>]*>`)
incompleteHTMLImageRE = regexp.MustCompile(`(?i)<img\b[^>]*$`)
htmlDoubleSrcRE = regexp.MustCompile(`(?i)\bsrc\s*=\s*"([^"]+)"`)
htmlSingleSrcRE = regexp.MustCompile(`(?i)\bsrc\s*=\s*'([^']+)'`)
htmlDoubleAltRE = regexp.MustCompile(`(?i)\balt\s*=\s*"([^"]*)"`)
htmlSingleAltRE = regexp.MustCompile(`(?i)\balt\s*=\s*'([^']*)'`)
resourceReferenceRE = regexp.MustCompile(`resource://[0-9A-Za-z_-]+`)
dataImageReferenceRE = regexp.MustCompile(`(?i)data:image[^\s]*`)
)
type Fetcher interface {
Fetch(ctx context.Context, reference string) ([]byte, string, error)
}
type Publisher interface {
Publish(ctx context.Context, data []byte, contentType string) (string, error)
}
type Rewriter struct {
fetcher Fetcher
publisher Publisher
maxImages int
}
func NewRewriter(fetcher Fetcher, publisher Publisher, maxImages int) *Rewriter {
return &Rewriter{fetcher: fetcher, publisher: publisher, maxImages: maxImages}
}
func (r *Rewriter) Rewrite(ctx context.Context, markdown string) string {
if r == nil || r.fetcher == nil || r.publisher == nil || r.maxImages <= 0 {
markdown = normalizePublicHTMLImages(stripResourceImages(markdown))
markdown = resourceReferenceRE.ReplaceAllString(resourceHTMLUnsafeImageRE.ReplaceAllString(markdown, ""), "")
return sanitizePublicMarkdownImages(markdown)
}
urls := map[string]string{}
attempted := 0
resolve := func(reference string) string {
if publishedURL, ok := urls[reference]; ok {
return publishedURL
}
if attempted >= r.maxImages {
urls[reference] = ""
return ""
}
attempted++
data, contentType, err := r.fetcher.Fetch(ctx, reference)
if err != nil {
urls[reference] = ""
return ""
}
publishedURL, err := r.publisher.Publish(ctx, data, contentType)
if err != nil || !strings.HasPrefix(publishedURL, "https://") {
urls[reference] = ""
return ""
}
urls[reference] = publishedURL
return publishedURL
}
markdown = resourceMarkdownImageRE.ReplaceAllStringFunc(markdown, func(image string) string {
match := resourceMarkdownImageRE.FindStringSubmatch(image)
if len(match) != 2 {
return ""
}
reference := match[1]
publishedURL := resolve(reference)
if publishedURL == "" {
return ""
}
return strings.Replace(image, reference, publishedURL, 1)
})
markdown = rewriteHTMLImages(markdown, resourceHTMLDoubleImageRE, resolve)
markdown = rewriteHTMLImages(markdown, resourceHTMLSingleImageRE, resolve)
markdown = normalizePublicHTMLImages(markdown)
markdown = resourceReferenceRE.ReplaceAllString(resourceHTMLUnsafeImageRE.ReplaceAllString(markdown, ""), "")
return sanitizePublicMarkdownImages(markdown)
}
func sanitizePublicMarkdownImages(markdown string) string {
markdown = markdownImageRE.ReplaceAllStringFunc(markdown, func(image string) string {
match := markdownImageRE.FindStringSubmatch(image)
if len(match) != 2 || !isSafePublicImageURL(match[1]) {
return ""
}
if isKnownDocumentURL(match[1]) {
return strings.TrimPrefix(image, "!")
}
return image
})
markdown = incompleteMarkdownImageRE.ReplaceAllString(markdown, "")
markdown = incompleteHTMLImageRE.ReplaceAllString(markdown, "")
return dataImageReferenceRE.ReplaceAllString(markdown, "")
}
func isKnownDocumentURL(raw string) bool {
u, err := url.Parse(strings.TrimSpace(raw))
if err != nil {
return false
}
switch strings.ToLower(path.Ext(u.Path)) {
case ".pdf", ".doc", ".docx", ".xls", ".xlsx", ".ppt", ".pptx",
".csv", ".txt", ".zip", ".rar", ".7z":
return true
default:
return false
}
}
func rewriteHTMLImages(content string, pattern *regexp.Regexp, resolve func(string) string) string {
return pattern.ReplaceAllStringFunc(content, func(image string) string {
match := pattern.FindStringSubmatch(image)
if len(match) != 2 {
return ""
}
publishedURL := resolve(match[1])
if publishedURL == "" {
return ""
}
alt := "图片"
for _, pattern := range []*regexp.Regexp{htmlDoubleAltRE, htmlSingleAltRE} {
if altMatch := pattern.FindStringSubmatch(image); len(altMatch) == 2 && strings.TrimSpace(altMatch[1]) != "" {
alt = strings.ReplaceAll(strings.TrimSpace(altMatch[1]), "]", `\]`)
break
}
}
return "![" + alt + "](" + publishedURL + ")"
})
}
func normalizePublicHTMLImages(content string) string {
return htmlImageRE.ReplaceAllStringFunc(content, func(image string) string {
source := ""
for _, pattern := range []*regexp.Regexp{htmlDoubleSrcRE, htmlSingleSrcRE} {
if match := pattern.FindStringSubmatch(image); len(match) == 2 {
source = strings.TrimSpace(match[1])
break
}
}
if !isSafePublicImageURL(source) {
return ""
}
return "![" + imageAlt(image) + "](" + source + ")"
})
}
func imageAlt(image string) string {
for _, pattern := range []*regexp.Regexp{htmlDoubleAltRE, htmlSingleAltRE} {
if match := pattern.FindStringSubmatch(image); len(match) == 2 && strings.TrimSpace(match[1]) != "" {
return strings.ReplaceAll(strings.TrimSpace(match[1]), "]", `\]`)
}
}
return "图片"
}
func isSafePublicImageURL(raw string) bool {
u, err := url.Parse(strings.TrimSpace(raw))
if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil {
return false
}
hostname := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".")
if hostname == "" || hostname == "localhost" || strings.HasSuffix(hostname, ".localhost") || strings.HasSuffix(hostname, ".local") || strings.HasSuffix(hostname, ".internal") {
return false
}
if port := u.Port(); port != "" && port != "443" {
return false
}
if ip := net.ParseIP(hostname); ip != nil {
if !ip.IsGlobalUnicast() || ip.IsPrivate() {
return false
}
} else if !strings.Contains(hostname, ".") {
return false
}
for key := range u.Query() {
normalized := strings.ToLower(key)
if strings.Contains(normalized, "signature") || normalized == "ossaccesskeyid" || strings.Contains(normalized, "credential") {
return false
}
}
return true
}
func stripResourceImages(content string) string {
content = resourceMarkdownImageRE.ReplaceAllString(content, "")
content = resourceHTMLDoubleImageRE.ReplaceAllString(content, "")
content = resourceHTMLSingleImageRE.ReplaceAllString(content, "")
return resourceHTMLUnsafeImageRE.ReplaceAllString(content, "")
}
func (r *Rewriter) SanitizePartial(content string) string {
content = resourceReferenceRE.ReplaceAllString(stripResourceImages(content), "")
content = normalizePublicHTMLImages(content)
return sanitizePublicMarkdownImages(content)
}