Files
Enterprise-Knowledge-Base/internal/application/service/session_agent_image_route.go
T
2026-07-16 00:17:16 +08:00

64 lines
2.3 KiB
Go

package service
import (
"context"
"encoding/base64"
"errors"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
const maxAgentFallbackImageBytes = 10 << 20
// prepareAgentImageInput routes image input according to the resolved primary
// transport capability. A configured VLM is required only when the actual
// primary Chat transport cannot carry images directly.
func prepareAgentImageInput(
ctx context.Context,
req *types.QARequest,
primarySupportsVision bool,
modelService interfaces.ModelService,
) (string, []string, error) {
if req == nil {
return "", nil, errors.New("QA request is required")
}
if len(req.ImageURLs) == 0 || primarySupportsVision || strings.TrimSpace(req.ImageDescription) != "" {
return routeAgentImageInput(req, primarySupportsVision)
}
if req.CustomAgent == nil || strings.TrimSpace(req.CustomAgent.Config.VLMModelID) == "" {
return "", nil, errors.New("agent model does not support visual input and no VLM is configured")
}
if modelService == nil {
return "", nil, errors.New("agent VLM service is unavailable")
}
images := make([][]byte, 0, len(req.ImageURLs))
for _, raw := range req.ImageURLs {
comma := strings.IndexByte(raw, ',')
if comma <= 0 || !strings.HasPrefix(strings.ToLower(raw[:comma]), "data:image/") ||
!strings.HasSuffix(strings.ToLower(raw[:comma]), ";base64") {
return "", nil, errors.New("agent image payload is invalid")
}
decoded, err := base64.StdEncoding.Strict().DecodeString(raw[comma+1:])
if err != nil || len(decoded) == 0 || len(decoded) > maxAgentFallbackImageBytes {
return "", nil, errors.New("agent image payload is invalid")
}
images = append(images, decoded)
}
model, err := modelService.GetVLMModel(ctx, req.CustomAgent.Config.VLMModelID)
if err != nil || model == nil {
return "", nil, errors.New("agent VLM is unavailable")
}
prompt := fmt.Sprintf("Analyze the uploaded image(s) for the user's question. Return a concise factual description only.\nQuestion: %s", req.Query)
description, err := model.Predict(ctx, images, prompt)
if err != nil || strings.TrimSpace(description) == "" {
return "", nil, errors.New("agent VLM analysis failed")
}
req.ImageDescription = strings.TrimSpace(description)
return routeAgentImageInput(req, false)
}