64 lines
2.3 KiB
Go
64 lines
2.3 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
)
|
|
|
|
const maxAgentFallbackImageBytes = 10 << 20
|
|
|
|
// prepareAgentImageInput routes image input according to the resolved primary
|
|
// transport capability. A configured VLM is required only when the actual
|
|
// primary Chat transport cannot carry images directly.
|
|
func prepareAgentImageInput(
|
|
ctx context.Context,
|
|
req *types.QARequest,
|
|
primarySupportsVision bool,
|
|
modelService interfaces.ModelService,
|
|
) (string, []string, error) {
|
|
if req == nil {
|
|
return "", nil, errors.New("QA request is required")
|
|
}
|
|
if len(req.ImageURLs) == 0 || primarySupportsVision || strings.TrimSpace(req.ImageDescription) != "" {
|
|
return routeAgentImageInput(req, primarySupportsVision)
|
|
}
|
|
if req.CustomAgent == nil || strings.TrimSpace(req.CustomAgent.Config.VLMModelID) == "" {
|
|
return "", nil, errors.New("agent model does not support visual input and no VLM is configured")
|
|
}
|
|
if modelService == nil {
|
|
return "", nil, errors.New("agent VLM service is unavailable")
|
|
}
|
|
|
|
images := make([][]byte, 0, len(req.ImageURLs))
|
|
for _, raw := range req.ImageURLs {
|
|
comma := strings.IndexByte(raw, ',')
|
|
if comma <= 0 || !strings.HasPrefix(strings.ToLower(raw[:comma]), "data:image/") ||
|
|
!strings.HasSuffix(strings.ToLower(raw[:comma]), ";base64") {
|
|
return "", nil, errors.New("agent image payload is invalid")
|
|
}
|
|
decoded, err := base64.StdEncoding.Strict().DecodeString(raw[comma+1:])
|
|
if err != nil || len(decoded) == 0 || len(decoded) > maxAgentFallbackImageBytes {
|
|
return "", nil, errors.New("agent image payload is invalid")
|
|
}
|
|
images = append(images, decoded)
|
|
}
|
|
|
|
model, err := modelService.GetVLMModel(ctx, req.CustomAgent.Config.VLMModelID)
|
|
if err != nil || model == nil {
|
|
return "", nil, errors.New("agent VLM is unavailable")
|
|
}
|
|
prompt := fmt.Sprintf("Analyze the uploaded image(s) for the user's question. Return a concise factual description only.\nQuestion: %s", req.Query)
|
|
description, err := model.Predict(ctx, images, prompt)
|
|
if err != nil || strings.TrimSpace(description) == "" {
|
|
return "", nil, errors.New("agent VLM analysis failed")
|
|
}
|
|
req.ImageDescription = strings.TrimSpace(description)
|
|
return routeAgentImageInput(req, false)
|
|
}
|