feat(dingtalk): enable secure multimodal image QA
This commit is contained in:
+121
-14
@@ -1,29 +1,136 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var imageDataURLPatternForLog = regexp.MustCompile(`data:image\/[a-zA-Z0-9.+-]+;base64,[A-Za-z0-9+/=]+`)
|
||||
|
||||
const (
|
||||
defaultMaxLogChars = 12000
|
||||
defaultMaxDataURLPreview = 96
|
||||
defaultMaxLogChars = 12000
|
||||
dataURLRedaction = "[image-data-redacted]"
|
||||
)
|
||||
|
||||
// CompactImageDataURLForLog shortens large image data URLs for log output.
|
||||
// CompactImageDataURLForLog removes every data URI before log or dump output.
|
||||
// It deliberately handles non-canonical forms because debug paths may receive
|
||||
// JSON-escaped or percent-encoded requests in addition to ordinary data: URLs.
|
||||
func CompactImageDataURLForLog(raw string) string {
|
||||
masked := imageDataURLPatternForLog.ReplaceAllStringFunc(raw, func(match string) string {
|
||||
if len(match) <= defaultMaxDataURLPreview {
|
||||
return match
|
||||
}
|
||||
hidden := len(match) - defaultMaxDataURLPreview
|
||||
return match[:defaultMaxDataURLPreview] + "...<omitted " + strconv.Itoa(hidden) + " chars>"
|
||||
})
|
||||
|
||||
masked := RedactDataURLsForLog(raw)
|
||||
if len(masked) <= defaultMaxLogChars {
|
||||
return masked
|
||||
}
|
||||
return masked[:defaultMaxLogChars] + "... (truncated, total " + strconv.Itoa(len(masked)) + " chars)"
|
||||
}
|
||||
|
||||
// RedactDataURLsForLog removes data URIs without truncating surrounding text.
|
||||
func RedactDataURLsForLog(raw string) string {
|
||||
return redactDataURLs(raw)
|
||||
}
|
||||
|
||||
func redactDataURLs(raw string) string {
|
||||
var out strings.Builder
|
||||
out.Grow(len(raw))
|
||||
for i := 0; i < len(raw); {
|
||||
prefixLen := dataURLPrefixLen(raw[i:])
|
||||
if prefixLen == 0 {
|
||||
out.WriteByte(raw[i])
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
out.WriteString(dataURLRedaction)
|
||||
i += prefixLen
|
||||
for i < len(raw) && !isDataURLTerminator(raw, i) {
|
||||
i++
|
||||
}
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// dataURLPrefixLen recognizes "data:" when each ASCII character is either
|
||||
// literal, percent encoded, or represented as one-or-more JSON escape layers.
|
||||
func dataURLPrefixLen(value string) int {
|
||||
pos := 0
|
||||
for i := 0; i < len("data:"); i++ {
|
||||
consumed := encodedASCIILen(value[pos:], "data:"[i])
|
||||
if consumed == 0 {
|
||||
return 0
|
||||
}
|
||||
pos += consumed
|
||||
}
|
||||
return pos
|
||||
}
|
||||
|
||||
func encodedASCIILen(value string, want byte) int {
|
||||
if len(value) > 0 && strings.EqualFold(value[:1], string(want)) {
|
||||
return 1
|
||||
}
|
||||
if len(value) >= 3 && value[0] == '%' {
|
||||
pos := 0
|
||||
for len(value) >= pos+3 && strings.EqualFold(value[pos:pos+3], "%25") {
|
||||
pos += 3
|
||||
}
|
||||
if pos == 0 {
|
||||
if decoded, ok := decodeHexByte(value[1], value[2]); ok && decoded == want {
|
||||
return 3
|
||||
}
|
||||
} else if len(value) >= pos+2 {
|
||||
if decoded, ok := decodeHexByte(value[pos], value[pos+1]); ok && decoded == want {
|
||||
return pos + 2
|
||||
}
|
||||
}
|
||||
}
|
||||
backslashes := 0
|
||||
for backslashes < len(value) && value[backslashes] == '\\' {
|
||||
backslashes++
|
||||
}
|
||||
if backslashes > 0 && len(value) >= backslashes+5 && (value[backslashes] == 'u' || value[backslashes] == 'U') {
|
||||
hex := value[backslashes+1 : backslashes+5]
|
||||
if strings.EqualFold(hex[:2], "00") {
|
||||
if decoded, ok := decodeHexByte(hex[2], hex[3]); ok && decoded == want {
|
||||
return backslashes + 5
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func decodeHexByte(high, low byte) (byte, bool) {
|
||||
hex := func(ch byte) (byte, bool) {
|
||||
switch {
|
||||
case ch >= '0' && ch <= '9':
|
||||
return ch - '0', true
|
||||
case ch >= 'a' && ch <= 'f':
|
||||
return ch - 'a' + 10, true
|
||||
case ch >= 'A' && ch <= 'F':
|
||||
return ch - 'A' + 10, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
hi, ok := hex(high)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
lo, ok := hex(low)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
return hi<<4 | lo, true
|
||||
}
|
||||
|
||||
// Data characters may legally include punctuation such as parentheses and
|
||||
// backslashes. Stop only at whitespace or an unescaped serialized-string quote.
|
||||
func isDataURLTerminator(raw string, pos int) bool {
|
||||
switch raw[pos] {
|
||||
case ' ', '\t', '\r', '\n':
|
||||
return true
|
||||
case '"':
|
||||
backslashes := 0
|
||||
for i := pos - 1; i >= 0 && raw[i] == '\\'; i-- {
|
||||
backslashes++
|
||||
}
|
||||
return backslashes%2 == 0
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCompactImageDataURLForLogRedactsEscapedAndLegalPayloadForms(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
secret string
|
||||
}{
|
||||
{"encoded leading d", `\u0064ata:image/png;base64,ESCAPEDLEADING`, "ESCAPEDLEADING"},
|
||||
{"fully percent encoded prefix", `%64%61%74%61%3Aimage/png;base64,PERCENTPREFIX`, "PERCENTPREFIX"},
|
||||
{"escaped payload sequence", `data:image/png;base64,BEFORE\u002bAFTER`, "AFTER"},
|
||||
{"parenthesis is legal payload", `data:text/plain,BEFORE(AFTER)SUFFIX next`, "SUFFIX"},
|
||||
{"apostrophe is legal payload", `data:image/png;name=foo'bar;base64,APOSTROPHESECRET next`, "APOSTROPHESECRET"},
|
||||
{"double percent encoded prefix", `%2564%2561%2574%2561%253Aimage/png;base64,DOUBLEPERCENT`, "DOUBLEPERCENT"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := CompactImageDataURLForLog(tt.raw)
|
||||
if strings.Contains(got, tt.secret) || !strings.Contains(got, dataURLRedaction) {
|
||||
t.Fatalf("CompactImageDataURLForLog(%q) = %q", tt.raw, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCompactImageDataURLForLogFullyRedactsImages(t *testing.T) {
|
||||
cases := []string{
|
||||
"data:image/png;base64,YWJj",
|
||||
"DATA:image/png;BASE64," + strings.Repeat("QUJD", 100),
|
||||
`{"image":"data:image\\/png;base64,TOPSECRET"}`,
|
||||
"data:image/png,%89PNG%0D%0A",
|
||||
"data:text/plain,NONIMAGESECRET",
|
||||
"data%3Aimage%2Fpng%3Bbase64%2CENCODEDSECRET",
|
||||
`data\u003aimage/png;base64,ESCAPEDSECRET`,
|
||||
}
|
||||
for _, raw := range cases {
|
||||
got := CompactImageDataURLForLog(raw)
|
||||
if !strings.Contains(got, "[image-data-redacted]") {
|
||||
t.Fatalf("CompactImageDataURLForLog(%q) = %q", raw, got)
|
||||
}
|
||||
for _, secret := range []string{"YWJj", "QUJD", "TOPSECRET", "%89PNG", "NONIMAGESECRET", "ENCODEDSECRET", "ESCAPEDSECRET"} {
|
||||
if strings.Contains(got, secret) {
|
||||
t.Fatalf("sanitized output retained %q: %q", secret, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+37
-13
@@ -226,6 +226,14 @@ var restrictedHostSuffixes = []string{
|
||||
var restrictedIPv4Ranges = []*net.IPNet{
|
||||
// 100.64.0.0/10 - Carrier-grade NAT (RFC 6598)
|
||||
mustParseCIDR("100.64.0.0/10"),
|
||||
// 192.31.196.0/24 - AS112-v4 special-purpose service
|
||||
mustParseCIDR("192.31.196.0/24"),
|
||||
// 192.52.193.0/24 - AMT special-purpose service
|
||||
mustParseCIDR("192.52.193.0/24"),
|
||||
// 192.88.99.0/24 - Deprecated 6to4 relay anycast
|
||||
mustParseCIDR("192.88.99.0/24"),
|
||||
// 192.175.48.0/24 - Direct Delegation AS112 service
|
||||
mustParseCIDR("192.175.48.0/24"),
|
||||
// 198.18.0.0/15 - Network device benchmark testing (RFC 2544)
|
||||
mustParseCIDR("198.18.0.0/15"),
|
||||
// 198.51.100.0/24 - TEST-NET-2 for documentation (RFC 5737)
|
||||
@@ -250,6 +258,22 @@ var restrictedIPv4Ranges = []*net.IPNet{
|
||||
mustParseCIDR("172.20.0.0/16"),
|
||||
}
|
||||
|
||||
// restrictedIPv6Ranges contains special-purpose address space that must not be
|
||||
// used as an outbound media-fetch destination. These ranges are not ordinary
|
||||
// public unicast destinations and some can translate or encapsulate IPv4.
|
||||
var restrictedIPv6Ranges = []*net.IPNet{
|
||||
mustParseCIDR("64:ff9b::/96"),
|
||||
mustParseCIDR("64:ff9b:1::/48"),
|
||||
mustParseCIDR("100::/64"),
|
||||
mustParseCIDR("100:0:0:1::/64"),
|
||||
mustParseCIDR("2001::/23"),
|
||||
mustParseCIDR("2001:db8::/32"),
|
||||
mustParseCIDR("2002::/16"),
|
||||
mustParseCIDR("2620:4f:8000::/48"),
|
||||
mustParseCIDR("3fff::/20"),
|
||||
mustParseCIDR("5f00::/16"),
|
||||
}
|
||||
|
||||
// mustParseCIDR parses a CIDR string and panics on error
|
||||
func mustParseCIDR(s string) *net.IPNet {
|
||||
_, ipNet, err := net.ParseCIDR(s)
|
||||
@@ -289,6 +313,19 @@ func isRestrictedIP(ip net.IP) (bool, string) {
|
||||
|
||||
// Check IPv6-specific restrictions
|
||||
if ip.To4() == nil && len(ip) == 16 {
|
||||
// Preserve explicit diagnostics for tunneling prefixes before the broader
|
||||
// IANA special-purpose range table.
|
||||
if ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x00 && ip[3] == 0x00 {
|
||||
return true, "Teredo tunneling address"
|
||||
}
|
||||
if ip[0] == 0x20 && ip[1] == 0x02 {
|
||||
return true, "6to4 special-purpose address"
|
||||
}
|
||||
for _, cidr := range restrictedIPv6Ranges {
|
||||
if cidr.Contains(ip) {
|
||||
return true, fmt.Sprintf("restricted range %s", cidr.String())
|
||||
}
|
||||
}
|
||||
// Site-local (deprecated but still blocked): fec0::/10
|
||||
if ip[0] == 0xfe && (ip[1]&0xc0) == 0xc0 {
|
||||
return true, "site-local IPv6 address"
|
||||
@@ -304,19 +341,6 @@ func isRestrictedIP(ip net.IP) (bool, string) {
|
||||
return true, fmt.Sprintf("IPv4-mapped %s", reason)
|
||||
}
|
||||
}
|
||||
// Teredo tunneling addresses: 2001:0000::/32
|
||||
// Embed arbitrary IPv4 in the payload; can reach internal hosts via relay.
|
||||
if ip[0] == 0x20 && ip[1] == 0x01 && ip[2] == 0x00 && ip[3] == 0x00 {
|
||||
return true, "Teredo tunneling address"
|
||||
}
|
||||
// 6to4 addresses: 2002::/16
|
||||
// Bits 16-47 carry an IPv4 address; block when embedded IPv4 is restricted.
|
||||
if ip[0] == 0x20 && ip[1] == 0x02 {
|
||||
embeddedIP := net.IP(ip[2:6])
|
||||
if restricted, reason := isRestrictedIP(embeddedIP); restricted {
|
||||
return true, fmt.Sprintf("6to4 embedded %s", reason)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false, ""
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
package utils
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIsPublicIPRejectsSpecialPurposeIPv6(t *testing.T) {
|
||||
for _, raw := range []string{
|
||||
"192.31.196.1",
|
||||
"192.52.193.1",
|
||||
"192.88.99.1",
|
||||
"192.175.48.1",
|
||||
"100::1",
|
||||
"100:0:0:1::1",
|
||||
"2001:1::1",
|
||||
"2001:2::1",
|
||||
"2001:10::1",
|
||||
"2001:20::1",
|
||||
"64:ff9b:1::1",
|
||||
"2002:0808:0808::1",
|
||||
"2620:4f:8000::1",
|
||||
} {
|
||||
if IsPublicIP(parseIPForTest(t, raw)) {
|
||||
t.Errorf("IsPublicIP(%s) = true; want false for special-purpose IPv6", raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -266,7 +266,7 @@ func TestIsRestrictedIP_IPv6(t *testing.T) {
|
||||
{"site-local", "fec0::1", true, "site-local"},
|
||||
{"Teredo", "2001:0000:4136:e378:8000:63bf:3fff:fdd2", true, "Teredo"},
|
||||
{"6to4 private", "2002:c0a8:0101::1", true, "6to4"},
|
||||
{"6to4 public", "2002:0808:0808::1", false, ""},
|
||||
{"6to4 public endpoint still special-purpose", "2002:0808:0808::1", true, "6to4"},
|
||||
{"public IPv6", "2001:4860:4860::8888", false, ""},
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user