diff --git a/internal/im/adapter.go b/internal/im/adapter.go index 41ec2234..5e6a4b90 100644 --- a/internal/im/adapter.go +++ b/internal/im/adapter.go @@ -4,6 +4,7 @@ import ( "context" "io" + "github.com/Tencent/WeKnora/internal/types" "github.com/gin-gonic/gin" ) @@ -64,6 +65,10 @@ type IncomingMessage struct { FileName string // FileSize is the file size in bytes (for file messages, optional). FileSize int64 + // ImageURLs carries transient model-ready image inputs (usually data URIs). Never log these values. + ImageURLs []string + // ImageReferences carries durable, non-secret source references persisted on the user message. + ImageReferences types.MessageImages // ThreadID is the platform-specific thread identifier. // - Slack: thread_ts (top-level message uses its own timestamp) // - Mattermost: root_id, or post_id if top-level diff --git a/internal/im/dingtalk/adapter.go b/internal/im/dingtalk/adapter.go index f978dc52..6de3ae88 100644 --- a/internal/im/dingtalk/adapter.go +++ b/internal/im/dingtalk/adapter.go @@ -9,7 +9,11 @@ import ( "encoding/json" "fmt" "io" + "mime" + "net" "net/http" + "net/url" + "path/filepath" "strconv" "strings" "sync" @@ -33,15 +37,18 @@ const dingtalkConvTypeGroup = "2" // Compile-time checks. var ( - _ im.Adapter = (*Adapter)(nil) - _ im.StreamSender = (*Adapter)(nil) + _ im.Adapter = (*Adapter)(nil) + _ im.StreamSender = (*Adapter)(nil) + _ im.FileDownloader = (*Adapter)(nil) ) // Adapter implements im.Adapter for DingTalk. type Adapter struct { - clientID string - clientSecret string - cardTemplateID string // optional: enables AI card streaming when set + clientID string + clientSecret string + cardTemplateID string // optional: enables AI card streaming when set + messageFileDownloadURL string + allowInsecureDownload bool // tests only; production requires HTTPS // accessToken cache tokenMu sync.RWMutex @@ -114,19 +121,20 @@ func (a *Adapter) VerifyCallback(c *gin.Context) error { // DingTalk callback message structure. type callbackMessage struct { - ConversationID string `json:"conversationId"` - ConversationType string `json:"conversationType"` - MsgID string `json:"msgId"` - Msgtype string `json:"msgtype"` - Text *textContent `json:"text"` - SenderNick string `json:"senderNick"` - SenderStaffId string `json:"senderStaffId"` - SenderID string `json:"senderId"` - SessionWebhook string `json:"sessionWebhook"` - RobotCode string `json:"robotCode"` - AtUsers []atUser `json:"atUsers"` - IsInAtList bool `json:"isInAtList"` - ChatbotCorpId string `json:"chatbotCorpId"` + ConversationID string `json:"conversationId"` + ConversationType string `json:"conversationType"` + MsgID string `json:"msgId"` + Msgtype string `json:"msgtype"` + Text *textContent `json:"text"` + Content json.RawMessage `json:"content"` + SenderNick string `json:"senderNick"` + SenderStaffId string `json:"senderStaffId"` + SenderID string `json:"senderId"` + SessionWebhook string `json:"sessionWebhook"` + RobotCode string `json:"robotCode"` + AtUsers []atUser `json:"atUsers"` + IsInAtList bool `json:"isInAtList"` + ChatbotCorpId string `json:"chatbotCorpId"` } type textContent struct { @@ -170,6 +178,20 @@ func parseCallbackMessage(msg *callbackMessage) *im.IncomingMessage { if msg.Text != nil { content = strings.TrimSpace(msg.Text.Content) } + messageType := im.MessageTypeText + fileKey := "" + fileName := "" + if strings.EqualFold(msg.Msgtype, "picture") { + var picture struct { + DownloadCode string `json:"downloadCode"` + } + if json.Unmarshal(msg.Content, &picture) == nil && strings.TrimSpace(picture.DownloadCode) != "" { + messageType = im.MessageTypeImage + fileKey = strings.TrimSpace(picture.DownloadCode) + fileName = msg.MsgID + ".png" + content = "" + } + } return &im.IncomingMessage{ Platform: im.PlatformDingtalk, @@ -178,14 +200,124 @@ func parseCallbackMessage(msg *callbackMessage) *im.IncomingMessage { ChatID: chatID, ChatType: chatType, MessageID: msg.MsgID, - MessageType: im.MessageTypeText, + MessageType: messageType, Content: content, + FileKey: fileKey, + FileName: fileName, Extra: map[string]string{ "session_webhook": msg.SessionWebhook, }, } } +const dingtalkMessageFileDownloadURL = "https://api.dingtalk.com/v1.0/robot/messageFiles/download" + +func (a *Adapter) DownloadFile(ctx context.Context, msg *im.IncomingMessage) (io.ReadCloser, string, error) { + if msg == nil || strings.TrimSpace(msg.FileKey) == "" { + return nil, "", fmt.Errorf("download code is required") + } + token, err := a.getAccessToken(ctx) + if err != nil { + return nil, "", fmt.Errorf("get access token: %w", err) + } + body, err := json.Marshal(map[string]string{ + "downloadCode": msg.FileKey, + "robotCode": a.clientID, + }) + if err != nil { + return nil, "", fmt.Errorf("marshal message file request: %w", err) + } + apiURL := a.messageFileDownloadURL + if apiURL == "" { + apiURL = dingtalkMessageFileDownloadURL + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, bytes.NewReader(body)) + if err != nil { + return nil, "", fmt.Errorf("create message file request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-acs-dingtalk-access-token", token) + resp, err := httpClient.Do(req) + if err != nil { + return nil, "", fmt.Errorf("request message file URL: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return nil, "", fmt.Errorf("dingtalk message file API returned %d", resp.StatusCode) + } + var result struct { + DownloadURL string `json:"downloadUrl"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&result); err != nil { + return nil, "", fmt.Errorf("decode message file response: %w", err) + } + if err := validateDingTalkDownloadURL(result.DownloadURL, a.allowInsecureDownload); err != nil { + return nil, "", err + } + + downloadReq, err := http.NewRequestWithContext(ctx, http.MethodGet, result.DownloadURL, nil) + if err != nil { + return nil, "", fmt.Errorf("create image download request: %w", err) + } + downloadClient := *httpClient + downloadClient.CheckRedirect = func(req *http.Request, _ []*http.Request) error { + return validateDingTalkDownloadURL(req.URL.String(), a.allowInsecureDownload) + } + downloadResp, err := downloadClient.Do(downloadReq) + if err != nil { + return nil, "", fmt.Errorf("download dingtalk image: %w", err) + } + if downloadResp.StatusCode < http.StatusOK || downloadResp.StatusCode >= http.StatusMultipleChoices { + downloadResp.Body.Close() + return nil, "", fmt.Errorf("dingtalk image download returned %d", downloadResp.StatusCode) + } + contentType := strings.ToLower(strings.TrimSpace(strings.Split(downloadResp.Header.Get("Content-Type"), ";")[0])) + if msg.MessageType == im.MessageTypeImage && !strings.HasPrefix(contentType, "image/") { + downloadResp.Body.Close() + return nil, "", fmt.Errorf("dingtalk picture response is not an image") + } + return downloadResp.Body, dingtalkDownloadedFileName(downloadResp, msg, contentType), nil +} + +func validateDingTalkDownloadURL(rawURL string, allowInsecure bool) error { + u, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || u.Hostname() == "" || u.User != nil { + return fmt.Errorf("invalid dingtalk download URL") + } + if allowInsecure { + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("invalid dingtalk download URL scheme") + } + return nil + } + if u.Scheme != "https" { + return fmt.Errorf("dingtalk download URL must use HTTPS") + } + if ip := net.ParseIP(u.Hostname()); ip != nil && (ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsUnspecified()) { + return fmt.Errorf("dingtalk download URL points to a private address") + } + return nil +} + +func dingtalkDownloadedFileName(resp *http.Response, msg *im.IncomingMessage, contentType string) string { + if _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")); err == nil { + if name := filepath.Base(strings.TrimSpace(params["filename"])); name != "" && name != "." && name != ".." { + return name + } + } + if msg != nil && strings.TrimSpace(msg.FileName) != "" { + return filepath.Base(msg.FileName) + } + ext := ".bin" + if extensions, _ := mime.ExtensionsByType(contentType); len(extensions) > 0 { + ext = extensions[0] + } + if msg != nil && msg.MessageID != "" { + return msg.MessageID + ext + } + return "dingtalk-file" + ext +} + // ── Send reply ── func (a *Adapter) SendReply(ctx context.Context, incoming *im.IncomingMessage, reply *im.ReplyMessage) error { diff --git a/internal/im/dingtalk/adapter_test.go b/internal/im/dingtalk/adapter_test.go new file mode 100644 index 00000000..1a5124e5 --- /dev/null +++ b/internal/im/dingtalk/adapter_test.go @@ -0,0 +1,93 @@ +package dingtalk + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/im" +) + +func TestParseCallbackMessagePicture(t *testing.T) { + msg := &callbackMessage{ + ConversationID: "cid-group", + ConversationType: dingtalkConvTypeGroup, + MsgID: "msg-picture-1", + Msgtype: "picture", + Content: []byte(`{"downloadCode":"download-code-1"}`), + SenderStaffId: "staff-1", + SenderNick: "Alice", + SessionWebhook: "https://example.invalid/session", + } + + got := parseCallbackMessage(msg) + if got.MessageType != im.MessageTypeImage { + t.Fatalf("MessageType = %q, want image", got.MessageType) + } + if got.FileKey != "download-code-1" { + t.Fatalf("FileKey = %q, want download code", got.FileKey) + } + if got.FileName != "msg-picture-1.png" { + t.Fatalf("FileName = %q", got.FileName) + } + if got.Content != "" { + t.Fatalf("Content = %q, want empty", got.Content) + } +} + +func TestAdapterDownloadFileUsesMessageFilesAPI(t *testing.T) { + var sawToken bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1.0/robot/messageFiles/download": + sawToken = r.Header.Get("x-acs-dingtalk-access-token") == "cached-access-token" + body, _ := io.ReadAll(r.Body) + if !strings.Contains(string(body), `"downloadCode":"download-code-1"`) || !strings.Contains(string(body), `"robotCode":"robot-code-1"`) { + t.Errorf("unexpected request body: %s", body) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"downloadUrl":"` + serverURLForRequest(r) + `/media/picture"}`)) + case "/media/picture": + w.Header().Set("Content-Type", "image/png") + w.Header().Set("Content-Disposition", `attachment; filename="question.png"`) + _, _ = w.Write([]byte("PNGDATA")) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + adapter := NewAdapter("robot-code-1", "secret", "") + adapter.token = "cached-access-token" + adapter.tokenExpAt = time.Now().Add(time.Hour) + adapter.messageFileDownloadURL = server.URL + "/v1.0/robot/messageFiles/download" + adapter.allowInsecureDownload = true + + reader, fileName, err := adapter.DownloadFile(context.Background(), &im.IncomingMessage{ + MessageType: im.MessageTypeImage, + MessageID: "msg-picture-1", + FileKey: "download-code-1", + }) + if err != nil { + t.Fatalf("DownloadFile: %v", err) + } + defer reader.Close() + body, _ := io.ReadAll(reader) + if string(body) != "PNGDATA" { + t.Fatalf("body = %q", body) + } + if fileName != "question.png" { + t.Fatalf("fileName = %q", fileName) + } + if !sawToken { + t.Fatal("missing DingTalk access token header") + } +} + +func serverURLForRequest(r *http.Request) string { + return "http://" + r.Host +} diff --git a/internal/im/dingtalk/longconn.go b/internal/im/dingtalk/longconn.go index 0931b912..c6e5203b 100644 --- a/internal/im/dingtalk/longconn.go +++ b/internal/im/dingtalk/longconn.go @@ -2,6 +2,7 @@ package dingtalk import ( "context" + "encoding/json" "strings" "github.com/open-dingtalk/dingtalk-stream-sdk-go/chatbot" @@ -71,7 +72,11 @@ func (c *LongConnClient) onChatBotMessage(ctx context.Context, data *chatbot.Bot userID = data.SenderId } - content := strings.TrimSpace(data.Text.Content) + messageType, content, fileKey := parseDingTalkStreamPayload(data.Msgtype, data.Content, data.Text.Content) + fileName := "" + if messageType == im.MessageTypeImage { + fileName = data.MsgId + ".png" + } incoming := &im.IncomingMessage{ Platform: im.PlatformDingtalk, @@ -80,8 +85,10 @@ func (c *LongConnClient) onChatBotMessage(ctx context.Context, data *chatbot.Bot ChatID: chatID, ChatType: chatType, MessageID: data.MsgId, - MessageType: im.MessageTypeText, + MessageType: messageType, Content: content, + FileKey: fileKey, + FileName: fileName, Extra: map[string]string{ "session_webhook": data.SessionWebhook, }, @@ -93,3 +100,31 @@ func (c *LongConnClient) onChatBotMessage(ctx context.Context, data *chatbot.Bot return nil, nil } + +func parseDingTalkStreamPayload(msgType string, rawContent interface{}, text string) (im.MessageType, string, string) { + if !strings.EqualFold(strings.TrimSpace(msgType), "picture") { + return im.MessageTypeText, strings.TrimSpace(text), "" + } + var payload []byte + switch value := rawContent.(type) { + case string: + payload = []byte(value) + case []byte: + payload = value + case json.RawMessage: + payload = value + default: + var err error + payload, err = json.Marshal(rawContent) + if err != nil { + return im.MessageTypeImage, "", "" + } + } + var picture struct { + DownloadCode string `json:"downloadCode"` + } + if json.Unmarshal(payload, &picture) != nil { + return im.MessageTypeImage, "", "" + } + return im.MessageTypeImage, "", strings.TrimSpace(picture.DownloadCode) +} diff --git a/internal/im/dingtalk/longconn_test.go b/internal/im/dingtalk/longconn_test.go new file mode 100644 index 00000000..5579c0e6 --- /dev/null +++ b/internal/im/dingtalk/longconn_test.go @@ -0,0 +1,37 @@ +package dingtalk + +import ( + "testing" + + "github.com/Tencent/WeKnora/internal/im" +) + +func TestParseDingTalkStreamPayloadPicture(t *testing.T) { + messageType, content, fileKey := parseDingTalkStreamPayload("picture", map[string]interface{}{ + "downloadCode": "stream-download-code", + }, "ignored text") + + if messageType != im.MessageTypeImage { + t.Fatalf("MessageType = %q, want image", messageType) + } + if content != "" { + t.Fatalf("content = %q, want empty", content) + } + if fileKey != "stream-download-code" { + t.Fatalf("fileKey = %q", fileKey) + } +} + +func TestParseDingTalkStreamPayloadText(t *testing.T) { + messageType, content, fileKey := parseDingTalkStreamPayload("text", nil, " hello ") + if messageType != im.MessageTypeText || content != "hello" || fileKey != "" { + t.Fatalf("got type=%q content=%q fileKey=%q", messageType, content, fileKey) + } +} + +func TestParseDingTalkStreamPayloadPictureJSONString(t *testing.T) { + messageType, _, fileKey := parseDingTalkStreamPayload("picture", `{"downloadCode":"string-code"}`, "") + if messageType != im.MessageTypeImage || fileKey != "string-code" { + t.Fatalf("got type=%q fileKey=%q", messageType, fileKey) + } +} diff --git a/internal/im/image_qa_test.go b/internal/im/image_qa_test.go new file mode 100644 index 00000000..5c103a72 --- /dev/null +++ b/internal/im/image_qa_test.go @@ -0,0 +1,83 @@ +package im + +import ( + "context" + "io" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/types" +) + +type staticImageDownloader struct { + body string + fileName string +} + +func (d staticImageDownloader) DownloadFile(context.Context, *IncomingMessage) (io.ReadCloser, string, error) { + return io.NopCloser(strings.NewReader(d.body)), d.fileName, nil +} + +func TestPrepareIMImageForQAAndBuildRequest(t *testing.T) { + png := string([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}) + strings.Repeat("x", 600) + msg := &IncomingMessage{ + Platform: PlatformDingtalk, + MessageType: MessageTypeImage, + MessageID: "picture-message-1", + FileKey: "temporary-download-code", + } + + if err := prepareIMImageForQA(context.Background(), msg, staticImageDownloader{body: png, fileName: "question.png"}); err != nil { + t.Fatalf("prepareIMImageForQA: %v", err) + } + if len(msg.ImageURLs) != 1 || !strings.HasPrefix(msg.ImageURLs[0], "data:image/png;base64,") { + t.Fatalf("ImageURLs = %#v", msg.ImageURLs) + } + if len(msg.ImageReferences) != 1 { + t.Fatalf("ImageReferences = %#v", msg.ImageReferences) + } + if got := msg.ImageReferences[0]; got.URL != "dingtalk://message/picture-message-1" || got.Caption != "question.png" { + t.Fatalf("image reference = %#v", got) + } + if msg.Content == "" { + t.Fatal("image-only message should receive a default visual question") + } + + req := buildIMQARequest(&types.Session{ID: "session-1"}, msg.Content, "assistant-1", "user-1", nil, nil, nil, msg.ImageURLs) + if len(req.ImageURLs) != 1 || req.ImageURLs[0] != msg.ImageURLs[0] { + t.Fatalf("request ImageURLs = %#v", req.ImageURLs) + } + + audit := newIMMessageAuditContext(msg, "channel-1", nil) + userMessage := createIMUserMessagePayload("session-1", msg.Content, "request-1", audit) + assistantMessage := createIMAssistantMessagePayload("session-1", "request-1", audit) + if len(userMessage.Images) != 1 || userMessage.Images[0].URL != "dingtalk://message/picture-message-1" { + t.Fatalf("user message Images = %#v", userMessage.Images) + } + if len(assistantMessage.Images) != 0 { + t.Fatalf("assistant message should not duplicate input image: %#v", assistantMessage.Images) + } +} + +func TestPrepareIMImageForQARejectsInvalidOrOversizedPayload(t *testing.T) { + tests := []struct { + name string + body string + want string + }{ + {name: "not an image", body: strings.Repeat("plain text", 80), want: "unsupported image content type"}, + {name: "oversized", body: strings.Repeat("x", maxIMImageBytes+1), want: "exceeds 10 MiB limit"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + msg := &IncomingMessage{Platform: PlatformDingtalk, MessageType: MessageTypeImage, MessageID: "image-1"} + err := prepareIMImageForQA(context.Background(), msg, staticImageDownloader{body: tt.body, fileName: "payload.bin"}) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("error = %v, want containing %q", err, tt.want) + } + if len(msg.ImageURLs) != 0 { + t.Fatal("rejected image must not reach the model") + } + }) + } +} diff --git a/internal/im/service.go b/internal/im/service.go index 24e99401..2f1caf15 100644 --- a/internal/im/service.go +++ b/internal/im/service.go @@ -3,11 +3,14 @@ package im import ( "bytes" "context" + "encoding/base64" "errors" "fmt" "io" "mime/multipart" + "net/http" "net/textproto" + "net/url" "os" "regexp" "strings" @@ -483,12 +486,17 @@ func buildIMQARequest( customAgent *types.CustomAgent, kbIDs []string, quote *QuotedMessage, + imageURLs ...[]string, ) *types.QARequest { // WebSearchEnabled: the web handler passes this per-request from the // frontend toggle; for IM channels the user has no per-message toggle, // so we derive it from the agent config (the single source of truth). webSearchEnabled := customAgent != nil && customAgent.Config.WebSearchEnabled quotedContext := formatQuotedContext(quote) + var images []string + if len(imageURLs) > 0 { + images = append([]string(nil), imageURLs[0]...) + } return &types.QARequest{ Session: session, Query: query, @@ -498,9 +506,62 @@ func buildIMQARequest( UserMessageID: userMessageID, WebSearchEnabled: webSearchEnabled, QuotedContext: quotedContext, + ImageURLs: images, } } +const maxIMImageBytes = 10 << 20 + +func prepareIMImageForQA(ctx context.Context, msg *IncomingMessage, downloader FileDownloader) error { + if msg == nil || downloader == nil { + return fmt.Errorf("image message and downloader are required") + } + reader, fileName, err := downloader.DownloadFile(ctx, msg) + if err != nil { + return fmt.Errorf("download image: %w", err) + } + defer reader.Close() + + data, err := io.ReadAll(io.LimitReader(reader, maxIMImageBytes+1)) + if err != nil { + return fmt.Errorf("read image: %w", err) + } + if len(data) == 0 { + return fmt.Errorf("downloaded image is empty") + } + if len(data) > maxIMImageBytes { + return fmt.Errorf("image exceeds %d MiB limit", maxIMImageBytes>>20) + } + contentType := strings.ToLower(http.DetectContentType(data)) + switch contentType { + case "image/jpeg", "image/png", "image/gif", "image/webp": + default: + return fmt.Errorf("unsupported image content type %q", contentType) + } + + msg.FileName = fileName + msg.ImageURLs = []string{"data:" + contentType + ";base64," + base64.StdEncoding.EncodeToString(data)} + referenceID := strings.TrimSpace(msg.MessageID) + if referenceID == "" { + referenceID = "unknown" + } + msg.ImageReferences = types.MessageImages{{ + URL: string(msg.Platform) + "://message/" + url.PathEscape(referenceID), + Caption: fileName, + }} + if strings.TrimSpace(msg.Content) == "" { + msg.Content = "请识别这张图片的内容,并回答图片中的问题。" + } + return nil +} + +func auditImageURLs(audit *imMessageAuditContext) []string { + if audit == nil { + return nil + } + return append([]string(nil), audit.imageURLs...) +} + func buildIMLastRequestState(agentID string, customAgent *types.CustomAgent, kbIDs []string) *types.SessionLastRequestState { state := &types.SessionLastRequestState{ AgentID: agentID, @@ -529,6 +590,8 @@ type imMessageAuditContext struct { chatID string threadID string knowledgeBaseIDs types.MessageKnowledgeBaseIDs + imageReferences types.MessageImages + imageURLs []string } func newIMMessageAuditContext(msg *IncomingMessage, channelID string, kbIDs []string) *imMessageAuditContext { @@ -544,6 +607,8 @@ func newIMMessageAuditContext(msg *IncomingMessage, channelID string, kbIDs []st audit.userID = msg.UserID audit.chatID = msg.ChatID audit.threadID = msg.ThreadID + audit.imageReferences = append(types.MessageImages(nil), msg.ImageReferences...) + audit.imageURLs = append([]string(nil), msg.ImageURLs...) return audit } @@ -571,6 +636,9 @@ func createIMUserMessagePayload(sessionID, content, requestID string, audit *imM Channel: "im", } applyIMMessageAudit(message, audit) + if audit != nil { + message.Images = append(types.MessageImages(nil), audit.imageReferences...) + } return message } @@ -1293,7 +1361,7 @@ func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, chann // ── File/Image message shortcut ── // If the message is a file or image and the channel has a knowledge_base_id configured, // handle it separately without entering the QA pipeline. - if (msg.MessageType == MessageTypeFile || msg.MessageType == MessageTypeImage) && channel.KnowledgeBaseID != "" { + if msg.MessageType == MessageTypeFile && channel.KnowledgeBaseID != "" { return s.handleFileMessage(ctx, msg, adapter, channel) } @@ -1301,7 +1369,7 @@ func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, chann // If the message is an image/file/video but has no text content, the QA pipeline // cannot do anything useful (no vision support in IM yet). Sending an empty query // to KB retrieval would return irrelevant results and cause hallucination. - if msg.Content == "" && (msg.MessageType == MessageTypeImage || msg.MessageType == MessageTypeFile) { + if msg.Content == "" && msg.MessageType == MessageTypeFile { logger.Infof(ctx, "[IM] Skipping QA for non-text message without content: type=%s", msg.MessageType) if err := adapter.SendReply(ctx, msg, &ReplyMessage{ Content: "当前渠道未配置文件知识库,无法处理图片/文件消息。请在渠道设置中配置文件知识库后再发送,或直接用文字描述您的问题。", @@ -1337,6 +1405,26 @@ func (s *Service) HandleMessage(ctx context.Context, msg *IncomingMessage, chann } } + if msg.MessageType == MessageTypeImage { + if customAgent == nil || !customAgent.Config.ImageUploadEnabled { + if channel.KnowledgeBaseID != "" { + return s.handleFileMessage(ctx, msg, adapter, channel) + } + return adapter.SendReply(ctx, msg, &ReplyMessage{ + Content: "当前机器人未启用图片识别,请在智能体设置中开启图片上传并配置视觉模型。", + IsFinal: true, + }) + } + downloader, ok := adapter.(FileDownloader) + if !ok { + return adapter.SendReply(ctx, msg, &ReplyMessage{Content: "当前钉钉渠道暂不支持图片下载。", IsFinal: true}) + } + if err := prepareIMImageForQA(sessionCtx, msg, downloader); err != nil { + logger.Warnf(ctx, "[IM] Failed to prepare image for QA: %v", err) + return adapter.SendReply(ctx, msg, &ReplyMessage{Content: "图片下载或格式校验失败,请发送10MB以内的 JPG、PNG、GIF 或 WebP 图片。", IsFinal: true}) + } + } + // ── Slash-command dispatch ── // Commands are handled before the QA pipeline so they respond instantly. if cmd, args, ok := s.cmdRegistry.Parse(msg.Content); ok { @@ -2193,7 +2281,7 @@ func (s *Service) handleMessageStream(ctx context.Context, msg *IncomingMessage, // Run QA async go func() { var err error - req := buildIMQARequest(session, msg.Content, assistantMsg.ID, userMsg.ID, customAgent, kbIDs, msg.Quote) + req := buildIMQARequest(session, msg.Content, assistantMsg.ID, userMsg.ID, customAgent, kbIDs, msg.Quote, msg.ImageURLs) if req.QuotedContext != "" { logger.Debugf(qaCtx, "[IM] QuotedContext set: length=%d", len(req.QuotedContext)) } @@ -2428,7 +2516,7 @@ func (s *Service) runQA(ctx context.Context, session *types.Session, query strin // Run QA async go func() { var err error - req := buildIMQARequest(session, query, assistantMsg.ID, userMsg.ID, customAgent, kbIDs, quote) + req := buildIMQARequest(session, query, assistantMsg.ID, userMsg.ID, customAgent, kbIDs, quote, auditImageURLs(audit)) if req.QuotedContext != "" { logger.Debugf(ctx, "[IM] QuotedContext set: length=%d", len(req.QuotedContext)) }