wizardchen
991bd29a14
- Added functionality for image uploads in chat, allowing users to attach images for multimodal Q&A. - Enhanced the input field to handle image selection via drag-and-drop and paste, with validation for file types and sizes. - Updated the backend to process images alongside text queries, including support for image analysis. - Introduced new UI components for image previews and management, improving user interaction with uploaded content. - Added localization strings for image upload features and error messages, enhancing accessibility for users in multiple languages. These changes significantly improve the chat experience by enabling users to incorporate images into their queries, facilitating richer interactions and responses.
102 lines
2.6 KiB
Go
102 lines
2.6 KiB
Go
package vlm
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
openai "github.com/sashabaranov/go-openai"
|
|
)
|
|
|
|
const (
|
|
defaultTimeout = 90 * time.Second
|
|
defaultMaxToks = 5000
|
|
defaultTemp = float32(0.1)
|
|
)
|
|
|
|
// RemoteAPIVLM implements VLM via an OpenAI-compatible chat completions API.
|
|
type RemoteAPIVLM struct {
|
|
modelName string
|
|
modelID string
|
|
client *openai.Client
|
|
baseURL string
|
|
}
|
|
|
|
// NewRemoteAPIVLM creates a remote-API backed VLM instance.
|
|
func NewRemoteAPIVLM(config *Config) (*RemoteAPIVLM, error) {
|
|
apiCfg := openai.DefaultConfig(config.APIKey)
|
|
if config.BaseURL != "" {
|
|
apiCfg.BaseURL = config.BaseURL
|
|
}
|
|
apiCfg.HTTPClient = &http.Client{Timeout: defaultTimeout}
|
|
|
|
return &RemoteAPIVLM{
|
|
modelName: config.ModelName,
|
|
modelID: config.ModelID,
|
|
client: openai.NewClientWithConfig(apiCfg),
|
|
baseURL: config.BaseURL,
|
|
}, nil
|
|
}
|
|
|
|
// Predict sends an image with a text prompt to the OpenAI-compatible API.
|
|
func (v *RemoteAPIVLM) Predict(ctx context.Context, imgBytes []byte, prompt string) (string, error) {
|
|
mimeType := detectImageMIME(imgBytes)
|
|
b64 := base64.StdEncoding.EncodeToString(imgBytes)
|
|
dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, b64)
|
|
|
|
req := openai.ChatCompletionRequest{
|
|
Model: v.modelName,
|
|
Messages: []openai.ChatCompletionMessage{
|
|
{
|
|
Role: openai.ChatMessageRoleUser,
|
|
MultiContent: []openai.ChatMessagePart{
|
|
{
|
|
Type: openai.ChatMessagePartTypeImageURL,
|
|
ImageURL: &openai.ChatMessageImageURL{
|
|
URL: dataURI,
|
|
Detail: openai.ImageURLDetailAuto,
|
|
},
|
|
},
|
|
{
|
|
Type: openai.ChatMessagePartTypeText,
|
|
Text: prompt,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
MaxTokens: defaultMaxToks,
|
|
Temperature: defaultTemp,
|
|
}
|
|
|
|
logger.Infof(ctx, "[VLM] Calling OpenAI-compatible API, model=%s, baseURL=%s, imageSize=%d",
|
|
v.modelName, v.baseURL, len(imgBytes))
|
|
|
|
resp, err := v.client.CreateChatCompletion(ctx, req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("OpenAI VLM request: %w", err)
|
|
}
|
|
if len(resp.Choices) == 0 {
|
|
return "", fmt.Errorf("OpenAI VLM returned no choices")
|
|
}
|
|
|
|
content := resp.Choices[0].Message.Content
|
|
logger.Infof(ctx, "[VLM] OpenAI response received, len=%d", len(content))
|
|
return content, nil
|
|
}
|
|
|
|
func (v *RemoteAPIVLM) GetModelName() string { return v.modelName }
|
|
func (v *RemoteAPIVLM) GetModelID() string { return v.modelID }
|
|
|
|
// detectImageMIME returns the MIME type for the given image bytes.
|
|
func detectImageMIME(data []byte) string {
|
|
ct := http.DetectContentType(data)
|
|
if strings.HasPrefix(ct, "image/") {
|
|
return ct
|
|
}
|
|
return "image/png"
|
|
}
|