feat(dingtalk): support image questions with vision models

This commit is contained in:
姚章浩
2026-07-15 13:54:34 +08:00
parent 9645de64a1
commit 1b0b0fb8ed
7 changed files with 498 additions and 25 deletions
+5
View File
@@ -4,6 +4,7 @@ import (
"context"
"io"
"github.com/Tencent/WeKnora/internal/types"
"github.com/gin-gonic/gin"
)
@@ -64,6 +65,10 @@ type IncomingMessage struct {
FileName string
// FileSize is the file size in bytes (for file messages, optional).
FileSize int64
// ImageURLs carries transient model-ready image inputs (usually data URIs). Never log these values.
ImageURLs []string
// ImageReferences carries durable, non-secret source references persisted on the user message.
ImageReferences types.MessageImages
// ThreadID is the platform-specific thread identifier.
// - Slack: thread_ts (top-level message uses its own timestamp)
// - Mattermost: root_id, or post_id if top-level
+151 -19
View File
@@ -9,7 +9,11 @@ import (
"encoding/json"
"fmt"
"io"
"mime"
"net"
"net/http"
"net/url"
"path/filepath"
"strconv"
"strings"
"sync"
@@ -33,15 +37,18 @@ const dingtalkConvTypeGroup = "2"
// Compile-time checks.
var (
_ im.Adapter = (*Adapter)(nil)
_ im.StreamSender = (*Adapter)(nil)
_ im.Adapter = (*Adapter)(nil)
_ im.StreamSender = (*Adapter)(nil)
_ im.FileDownloader = (*Adapter)(nil)
)
// Adapter implements im.Adapter for DingTalk.
type Adapter struct {
clientID string
clientSecret string
cardTemplateID string // optional: enables AI card streaming when set
clientID string
clientSecret string
cardTemplateID string // optional: enables AI card streaming when set
messageFileDownloadURL string
allowInsecureDownload bool // tests only; production requires HTTPS
// accessToken cache
tokenMu sync.RWMutex
@@ -114,19 +121,20 @@ func (a *Adapter) VerifyCallback(c *gin.Context) error {
// DingTalk callback message structure.
type callbackMessage struct {
ConversationID string `json:"conversationId"`
ConversationType string `json:"conversationType"`
MsgID string `json:"msgId"`
Msgtype string `json:"msgtype"`
Text *textContent `json:"text"`
SenderNick string `json:"senderNick"`
SenderStaffId string `json:"senderStaffId"`
SenderID string `json:"senderId"`
SessionWebhook string `json:"sessionWebhook"`
RobotCode string `json:"robotCode"`
AtUsers []atUser `json:"atUsers"`
IsInAtList bool `json:"isInAtList"`
ChatbotCorpId string `json:"chatbotCorpId"`
ConversationID string `json:"conversationId"`
ConversationType string `json:"conversationType"`
MsgID string `json:"msgId"`
Msgtype string `json:"msgtype"`
Text *textContent `json:"text"`
Content json.RawMessage `json:"content"`
SenderNick string `json:"senderNick"`
SenderStaffId string `json:"senderStaffId"`
SenderID string `json:"senderId"`
SessionWebhook string `json:"sessionWebhook"`
RobotCode string `json:"robotCode"`
AtUsers []atUser `json:"atUsers"`
IsInAtList bool `json:"isInAtList"`
ChatbotCorpId string `json:"chatbotCorpId"`
}
type textContent struct {
@@ -170,6 +178,20 @@ func parseCallbackMessage(msg *callbackMessage) *im.IncomingMessage {
if msg.Text != nil {
content = strings.TrimSpace(msg.Text.Content)
}
messageType := im.MessageTypeText
fileKey := ""
fileName := ""
if strings.EqualFold(msg.Msgtype, "picture") {
var picture struct {
DownloadCode string `json:"downloadCode"`
}
if json.Unmarshal(msg.Content, &picture) == nil && strings.TrimSpace(picture.DownloadCode) != "" {
messageType = im.MessageTypeImage
fileKey = strings.TrimSpace(picture.DownloadCode)
fileName = msg.MsgID + ".png"
content = ""
}
}
return &im.IncomingMessage{
Platform: im.PlatformDingtalk,
@@ -178,14 +200,124 @@ func parseCallbackMessage(msg *callbackMessage) *im.IncomingMessage {
ChatID: chatID,
ChatType: chatType,
MessageID: msg.MsgID,
MessageType: im.MessageTypeText,
MessageType: messageType,
Content: content,
FileKey: fileKey,
FileName: fileName,
Extra: map[string]string{
"session_webhook": msg.SessionWebhook,
},
}
}
const dingtalkMessageFileDownloadURL = "https://api.dingtalk.com/v1.0/robot/messageFiles/download"
func (a *Adapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) (io.ReadCloser, string, error) {
if msg == nil || strings.TrimSpace(msg.FileKey) == "" {
return nil, "", fmt.Errorf("download code is required")
}
token, err := a.getAccessToken(ctx)
if err != nil {
return nil, "", fmt.Errorf("get access token: %w", err)
}
body, err := json.Marshal(map[string]string{
"downloadCode": msg.FileKey,
"robotCode": a.clientID,
})
if err != nil {
return nil, "", fmt.Errorf("marshal message file request: %w", err)
}
apiURL := a.messageFileDownloadURL
if apiURL == "" {
apiURL = dingtalkMessageFileDownloadURL
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, bytes.NewReader(body))
if err != nil {
return nil, "", fmt.Errorf("create message file request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("x-acs-dingtalk-access-token", token)
resp, err := httpClient.Do(req)
if err != nil {
return nil, "", fmt.Errorf("request message file URL: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
return nil, "", fmt.Errorf("dingtalk message file API returned %d", resp.StatusCode)
}
var result struct {
DownloadURL string `json:"downloadUrl"`
}
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&result); err != nil {
return nil, "", fmt.Errorf("decode message file response: %w", err)
}
if err := validateDingTalkDownloadURL(result.DownloadURL, a.allowInsecureDownload); err != nil {
return nil, "", err
}
downloadReq, err := http.NewRequestWithContext(ctx, http.MethodGet, result.DownloadURL, nil)
if err != nil {
return nil, "", fmt.Errorf("create image download request: %w", err)
}
downloadClient := *httpClient
downloadClient.CheckRedirect = func(req *http.Request, _ []*http.Request) error {
return validateDingTalkDownloadURL(req.URL.String(), a.allowInsecureDownload)
}
downloadResp, err := downloadClient.Do(downloadReq)
if err != nil {
return nil, "", fmt.Errorf("download dingtalk image: %w", err)
}
if downloadResp.StatusCode < http.StatusOK || downloadResp.StatusCode >= http.StatusMultipleChoices {
downloadResp.Body.Close()
return nil, "", fmt.Errorf("dingtalk image download returned %d", downloadResp.StatusCode)
}
contentType := strings.ToLower(strings.TrimSpace(strings.Split(downloadResp.Header.Get("Content-Type"), ";")[0]))
if msg.MessageType == im.MessageTypeImage && !strings.HasPrefix(contentType, "image/") {
downloadResp.Body.Close()
return nil, "", fmt.Errorf("dingtalk picture response is not an image")
}
return downloadResp.Body, dingtalkDownloadedFileName(downloadResp, msg, contentType), nil
}
func validateDingTalkDownloadURL(rawURL string, allowInsecure bool) error {
u, err := url.Parse(strings.TrimSpace(rawURL))
if err != nil || u.Hostname() == "" || u.User != nil {
return fmt.Errorf("invalid dingtalk download URL")
}
if allowInsecure {
if u.Scheme != "http" && u.Scheme != "https" {
return fmt.Errorf("invalid dingtalk download URL scheme")
}
return nil
}
if u.Scheme != "https" {
return fmt.Errorf("dingtalk download URL must use HTTPS")
}
if ip := net.ParseIP(u.Hostname()); ip != nil && (ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsUnspecified()) {
return fmt.Errorf("dingtalk download URL points to a private address")
}
return nil
}
func dingtalkDownloadedFileName(resp *http.Response, msg *im.IncomingMessage, contentType string) string {
if _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")); err == nil {
if name := filepath.Base(strings.TrimSpace(params["filename"])); name != "" && name != "." && name != ".." {
return name
}
}
if msg != nil && strings.TrimSpace(msg.FileName) != "" {
return filepath.Base(msg.FileName)
}
ext := ".bin"
if extensions, _ := mime.ExtensionsByType(contentType); len(extensions) > 0 {
ext = extensions[0]
}
if msg != nil && msg.MessageID != "" {
return msg.MessageID + ext
}
return "dingtalk-file" + ext
}
// ── Send reply ──
func (a *Adapter) SendReply(ctx context.Context, incoming *im.IncomingMessage, reply *im.ReplyMessage) error {
+93
View File
@@ -0,0 +1,93 @@
package dingtalk
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/Tencent/WeKnora/internal/im"
)
func TestParseCallbackMessagePicture(t *testing.T) {
msg := &callbackMessage{
ConversationID: "cid-group",
ConversationType: dingtalkConvTypeGroup,
MsgID: "msg-picture-1",
Msgtype: "picture",
Content: []byte(`{"downloadCode":"download-code-1"}`),
SenderStaffId: "staff-1",
SenderNick: "Alice",
SessionWebhook: "https://example.invalid/session",
}
got := parseCallbackMessage(msg)
if got.MessageType != im.MessageTypeImage {
t.Fatalf("MessageType = %q, want image", got.MessageType)
}
if got.FileKey != "download-code-1" {
t.Fatalf("FileKey = %q, want download code", got.FileKey)
}
if got.FileName != "msg-picture-1.png" {
t.Fatalf("FileName = %q", got.FileName)
}
if got.Content != "" {
t.Fatalf("Content = %q, want empty", got.Content)
}
}
func TestAdapterDownloadFileUsesMessageFilesAPI(t *testing.T) {
var sawToken bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1.0/robot/messageFiles/download":
sawToken = r.Header.Get("x-acs-dingtalk-access-token") == "cached-access-token"
body, _ := io.ReadAll(r.Body)
if !strings.Contains(string(body), `"downloadCode":"download-code-1"`) || !strings.Contains(string(body), `"robotCode":"robot-code-1"`) {
t.Errorf("unexpected request body: %s", body)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"downloadUrl":"` + serverURLForRequest(r) + `/media/picture"}`))
case "/media/picture":
w.Header().Set("Content-Type", "image/png")
w.Header().Set("Content-Disposition", `attachment; filename="question.png"`)
_, _ = w.Write([]byte("PNGDATA"))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
adapter := NewAdapter("robot-code-1", "secret", "")
adapter.token = "cached-access-token"
adapter.tokenExpAt = time.Now().Add(time.Hour)
adapter.messageFileDownloadURL = server.URL + "/v1.0/robot/messageFiles/download"
adapter.allowInsecureDownload = true
reader, fileName, err := adapter.DownloadFile(context.Background(), &im.IncomingMessage{
MessageType: im.MessageTypeImage,
MessageID: "msg-picture-1",
FileKey: "download-code-1",
})
if err != nil {
t.Fatalf("DownloadFile: %v", err)
}
defer reader.Close()
body, _ := io.ReadAll(reader)
if string(body) != "PNGDATA" {
t.Fatalf("body = %q", body)
}
if fileName != "question.png" {
t.Fatalf("fileName = %q", fileName)
}
if !sawToken {
t.Fatal("missing DingTalk access token header")
}
}
func serverURLForRequest(r *http.Request) string {
return "http://" + r.Host
}
+37 -2
View File
@@ -2,6 +2,7 @@ package dingtalk
import (
"context"
"encoding/json"
"strings"
"github.com/open-dingtalk/dingtalk-stream-sdk-go/chatbot"
@@ -71,7 +72,11 @@ func (c *LongConnClient) onChatBotMessage(ctx context.Context, data *chatbot.Bot
userID = data.SenderId
}
content := strings.TrimSpace(data.Text.Content)
messageType, content, fileKey := parseDingTalkStreamPayload(data.Msgtype, data.Content, data.Text.Content)
fileName := ""
if messageType == im.MessageTypeImage {
fileName = data.MsgId + ".png"
}
incoming := &im.IncomingMessage{
Platform: im.PlatformDingtalk,
@@ -80,8 +85,10 @@ func (c *LongConnClient) onChatBotMessage(ctx context.Context, data *chatbot.Bot
ChatID: chatID,
ChatType: chatType,
MessageID: data.MsgId,
MessageType: im.MessageTypeText,
MessageType: messageType,
Content: content,
FileKey: fileKey,
FileName: fileName,
Extra: map[string]string{
"session_webhook": data.SessionWebhook,
},
@@ -93,3 +100,31 @@ func (c *LongConnClient) onChatBotMessage(ctx context.Context, data *chatbot.Bot
return nil, nil
}
func parseDingTalkStreamPayload(msgType string, rawContent interface{}, text string) (im.MessageType, string, string) {
if !strings.EqualFold(strings.TrimSpace(msgType), "picture") {
return im.MessageTypeText, strings.TrimSpace(text), ""
}
var payload []byte
switch value := rawContent.(type) {
case string:
payload = []byte(value)
case []byte:
payload = value
case json.RawMessage:
payload = value
default:
var err error
payload, err = json.Marshal(rawContent)
if err != nil {
return im.MessageTypeImage, "", ""
}
}
var picture struct {
DownloadCode string `json:"downloadCode"`
}
if json.Unmarshal(payload, &picture) != nil {
return im.MessageTypeImage, "", ""
}
return im.MessageTypeImage, "", strings.TrimSpace(picture.DownloadCode)
}
+37
View File
@@ -0,0 +1,37 @@
package dingtalk
import (
"testing"
"github.com/Tencent/WeKnora/internal/im"
)
func TestParseDingTalkStreamPayloadPicture(t *testing.T) {
messageType, content, fileKey := parseDingTalkStreamPayload("picture", map[string]interface{}{
"downloadCode": "stream-download-code",
}, "ignored text")
if messageType != im.MessageTypeImage {
t.Fatalf("MessageType = %q, want image", messageType)
}
if content != "" {
t.Fatalf("content = %q, want empty", content)
}
if fileKey != "stream-download-code" {
t.Fatalf("fileKey = %q", fileKey)
}
}
func TestParseDingTalkStreamPayloadText(t *testing.T) {
messageType, content, fileKey := parseDingTalkStreamPayload("text", nil, " hello ")
if messageType != im.MessageTypeText || content != "hello" || fileKey != "" {
t.Fatalf("got type=%q content=%q fileKey=%q", messageType, content, fileKey)
}
}
func TestParseDingTalkStreamPayloadPictureJSONString(t *testing.T) {
messageType, _, fileKey := parseDingTalkStreamPayload("picture", `{"downloadCode":"string-code"}`, "")
if messageType != im.MessageTypeImage || fileKey != "string-code" {
t.Fatalf("got type=%q fileKey=%q", messageType, fileKey)
}
}
+83
View File
@@ -0,0 +1,83 @@
package im
import (
"context"
"io"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/types"
)
type staticImageDownloader struct {
body string
fileName string
}
func (d staticImageDownloader) DownloadFile(context.Context, *IncomingMessage) (io.ReadCloser, string, error) {
return io.NopCloser(strings.NewReader(d.body)), d.fileName, nil
}
func TestPrepareIMImageForQAAndBuildRequest(t *testing.T) {
png := string([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}) + strings.Repeat("x", 600)
msg := &IncomingMessage{
Platform: PlatformDingtalk,
MessageType: MessageTypeImage,
MessageID: "picture-message-1",
FileKey: "temporary-download-code",
}
if err := prepareIMImageForQA(context.Background(), msg, staticImageDownloader{body: png, fileName: "question.png"}); err != nil {
t.Fatalf("prepareIMImageForQA: %v", err)
}
if len(msg.ImageURLs) != 1 || !strings.HasPrefix(msg.ImageURLs[0], "data:image/png;base64,") {
t.Fatalf("ImageURLs = %#v", msg.ImageURLs)
}
if len(msg.ImageReferences) != 1 {
t.Fatalf("ImageReferences = %#v", msg.ImageReferences)
}
if got := msg.ImageReferences[0]; got.URL != "dingtalk://message/picture-message-1" || got.Caption != "question.png" {
t.Fatalf("image reference = %#v", got)
}
if msg.Content == "" {
t.Fatal("image-only message should receive a default visual question")
}
req := buildIMQARequest(&types.Session{ID: "session-1"}, msg.Content, "assistant-1", "user-1", nil, nil, nil, msg.ImageURLs)
if len(req.ImageURLs) != 1 || req.ImageURLs[0] != msg.ImageURLs[0] {
t.Fatalf("request ImageURLs = %#v", req.ImageURLs)
}
audit := newIMMessageAuditContext(msg, "channel-1", nil)
userMessage := createIMUserMessagePayload("session-1", msg.Content, "request-1", audit)
assistantMessage := createIMAssistantMessagePayload("session-1", "request-1", audit)
if len(userMessage.Images) != 1 || userMessage.Images[0].URL != "dingtalk://message/picture-message-1" {
t.Fatalf("user message Images = %#v", userMessage.Images)
}
if len(assistantMessage.Images) != 0 {
t.Fatalf("assistant message should not duplicate input image: %#v", assistantMessage.Images)
}
}
func TestPrepareIMImageForQARejectsInvalidOrOversizedPayload(t *testing.T) {
tests := []struct {
name string
body string
want string
}{
{name: "not an image", body: strings.Repeat("plain text", 80), want: "unsupported image content type"},
{name: "oversized", body: strings.Repeat("x", maxIMImageBytes+1), want: "exceeds 10 MiB limit"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
msg := &IncomingMessage{Platform: PlatformDingtalk, MessageType: MessageTypeImage, MessageID: "image-1"}
err := prepareIMImageForQA(context.Background(), msg, staticImageDownloader{body: tt.body, fileName: "payload.bin"})
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("error = %v, want containing %q", err, tt.want)
}
if len(msg.ImageURLs) != 0 {
t.Fatal("rejected image must not reach the model")
}
})
}
}
+92 -4
View File
@@ -3,11 +3,14 @@ package im
import (
"bytes"
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"os"
"regexp"
"strings"
@@ -483,12 +486,17 @@ func buildIMQARequest(
customAgent *types.CustomAgent,
kbIDs []string,
quote *QuotedMessage,
imageURLs ...[]string,
) *types.QARequest {
// WebSearchEnabled: the web handler passes this per-request from the
// frontend toggle; for IM channels the user has no per-message toggle,
// so we derive it from the agent config (the single source of truth).
webSearchEnabled := customAgent != nil && customAgent.Config.WebSearchEnabled
quotedContext := formatQuotedContext(quote)
var images []string
if len(imageURLs) > 0 {
images = append([]string(nil), imageURLs[0]...)
}
return &types.QARequest{
Session: session,
Query: query,
@@ -498,9 +506,62 @@ func buildIMQARequest(
UserMessageID: userMessageID,
WebSearchEnabled: webSearchEnabled,
QuotedContext: quotedContext,
ImageURLs: images,
}
}
const maxIMImageBytes = 10 << 20
func prepareIMImageForQA(ctx context.Context, msg *IncomingMessage, downloader FileDownloader) error {
if msg == nil || downloader == nil {
return fmt.Errorf("image message and downloader are required")
}
reader, fileName, err := downloader.DownloadFile(ctx, msg)
if err != nil {
return fmt.Errorf("download image: %w", err)
}
defer reader.Close()
data, err := io.ReadAll(io.LimitReader(reader, maxIMImageBytes+1))
if err != nil {
return fmt.Errorf("read image: %w", err)
}
if len(data) == 0 {
return fmt.Errorf("downloaded image is empty")
}
if len(data) > maxIMImageBytes {
return fmt.Errorf("image exceeds %d MiB limit", maxIMImageBytes>>20)
}
contentType := strings.ToLower(http.DetectContentType(data))
switch contentType {
case "image/jpeg", "image/png", "image/gif", "image/webp":
default:
return fmt.Errorf("unsupported image content type %q", contentType)
}
msg.FileName = fileName
msg.ImageURLs = []string{"data:" + contentType + ";base64," + base64.StdEncoding.EncodeToString(data)}
referenceID := strings.TrimSpace(msg.MessageID)
if referenceID == "" {
referenceID = "unknown"
}
msg.ImageReferences = types.MessageImages{{
URL: string(msg.Platform) + "://message/" + url.PathEscape(referenceID),
Caption: fileName,
}}
if strings.TrimSpace(msg.Content) == "" {
msg.Content = "请识别这张图片的内容,并回答图片中的问题。"
}
return nil
}
func auditImageURLs(audit *imMessageAuditContext) []string {
if audit == nil {
return nil
}
return append([]string(nil), audit.imageURLs...)
}
func buildIMLastRequestState(agentID string, customAgent *types.CustomAgent, kbIDs []string) *types.SessionLastRequestState {
state := &types.SessionLastRequestState{
AgentID: agentID,
@@ -529,6 +590,8 @@ type imMessageAuditContext struct {
chatID string
threadID string
knowledgeBaseIDs types.MessageKnowledgeBaseIDs
imageReferences types.MessageImages
imageURLs []string
}
func newIMMessageAuditContext(msg *IncomingMessage, channelID string, kbIDs []string) *imMessageAuditContext {
@@ -544,6 +607,8 @@ func newIMMessageAuditContext(msg *IncomingMessage, channelID string, kbIDs []st
audit.userID = msg.UserID
audit.chatID = msg.ChatID
audit.threadID = msg.ThreadID
audit.imageReferences = append(types.MessageImages(nil), msg.ImageReferences...)
audit.imageURLs = append([]string(nil), msg.ImageURLs...)
return audit
}
@@ -571,6 +636,9 @@ func createIMUserMessagePayload(sessionID, content, requestID string, audit *imM
Channel: "im",
}
applyIMMessageAudit(message, audit)
if audit != nil {
message.Images = append(types.MessageImages(nil), audit.imageReferences...)
}
return message
}
@@ -1293,7 +1361,7 @@ func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, chann
// ── File/Image message shortcut ──
// If the message is a file or image and the channel has a knowledge_base_id configured,
// handle it separately without entering the QA pipeline.
if (msg.MessageType == MessageTypeFile || msg.MessageType == MessageTypeImage) && channel.KnowledgeBaseID != "" {
if msg.MessageType == MessageTypeFile && channel.KnowledgeBaseID != "" {
return s.handleFileMessage(ctx, msg, adapter, channel)
}
@@ -1301,7 +1369,7 @@ func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, chann
// If the message is an image/file/video but has no text content, the QA pipeline
// cannot do anything useful (no vision support in IM yet). Sending an empty query
// to KB retrieval would return irrelevant results and cause hallucination.
if msg.Content == "" && (msg.MessageType == MessageTypeImage || msg.MessageType == MessageTypeFile) {
if msg.Content == "" && msg.MessageType == MessageTypeFile {
logger.Infof(ctx, "[IM] Skipping QA for non-text message without content: type=%s", msg.MessageType)
if err := adapter.SendReply(ctx, msg, &ReplyMessage{
Content: "当前渠道未配置文件知识库,无法处理图片/文件消息。请在渠道设置中配置文件知识库后再发送,或直接用文字描述您的问题。",
@@ -1337,6 +1405,26 @@ func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, chann
}
}
if msg.MessageType == MessageTypeImage {
if customAgent == nil || !customAgent.Config.ImageUploadEnabled {
if channel.KnowledgeBaseID != "" {
return s.handleFileMessage(ctx, msg, adapter, channel)
}
return adapter.SendReply(ctx, msg, &ReplyMessage{
Content: "当前机器人未启用图片识别,请在智能体设置中开启图片上传并配置视觉模型。",
IsFinal: true,
})
}
downloader, ok := adapter.(FileDownloader)
if !ok {
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: "当前钉钉渠道暂不支持图片下载。", IsFinal: true})
}
if err := prepareIMImageForQA(sessionCtx, msg, downloader); err != nil {
logger.Warnf(ctx, "[IM] Failed to prepare image for QA: %v", err)
return adapter.SendReply(ctx, msg, &ReplyMessage{Content: "图片下载或格式校验失败,请发送10MB以内的 JPG、PNG、GIF 或 WebP 图片。", IsFinal: true})
}
}
// ── Slash-command dispatch ──
// Commands are handled before the QA pipeline so they respond instantly.
if cmd, args, ok := s.cmdRegistry.Parse(msg.Content); ok {
@@ -2193,7 +2281,7 @@ func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage,
// Run QA async
go func() {
var err error
req := buildIMQARequest(session, msg.Content, assistantMsg.ID, userMsg.ID, customAgent, kbIDs, msg.Quote)
req := buildIMQARequest(session, msg.Content, assistantMsg.ID, userMsg.ID, customAgent, kbIDs, msg.Quote, msg.ImageURLs)
if req.QuotedContext != "" {
logger.Debugf(qaCtx, "[IM] QuotedContext set: length=%d", len(req.QuotedContext))
}
@@ -2428,7 +2516,7 @@ func (s *Service) runQA(ctx context.Context, session *types.Session, query strin
// Run QA async
go func() {
var err error
req := buildIMQARequest(session, query, assistantMsg.ID, userMsg.ID, customAgent, kbIDs, quote)
req := buildIMQARequest(session, query, assistantMsg.ID, userMsg.ID, customAgent, kbIDs, quote, auditImageURLs(audit))
if req.QuotedContext != "" {
logger.Debugf(ctx, "[IM] QuotedContext set: length=%d", len(req.QuotedContext))
}