liweixin
123 lines
3.2 KiB
Go
123 lines
3.2 KiB
Go
package vlm
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/models/provider"
|
|
openai "github.com/sashabaranov/go-openai"
|
|
)
|
|
|
|
const (
|
|
defaultTimeout = 90 * time.Second
|
|
defaultMaxToks = 5000
|
|
defaultTemp = float32(0.1)
|
|
)
|
|
|
|
// RemoteAPIVLM implements VLM via an OpenAI-compatible chat completions API.
|
|
type RemoteAPIVLM struct {
|
|
modelName string
|
|
modelID string
|
|
client *openai.Client
|
|
baseURL string
|
|
}
|
|
|
|
// NewRemoteAPIVLM creates a remote-API backed VLM instance.
|
|
func NewRemoteAPIVLM(config *Config) (*RemoteAPIVLM, error) {
|
|
providerName := provider.ProviderName(config.Provider)
|
|
if providerName == "" {
|
|
providerName = provider.DetectProvider(config.BaseURL)
|
|
}
|
|
|
|
var apiCfg openai.ClientConfig
|
|
if providerName == provider.ProviderAzureOpenAI {
|
|
apiCfg = openai.DefaultAzureConfig(config.APIKey, config.BaseURL)
|
|
apiCfg.AzureModelMapperFunc = func(model string) string {
|
|
return model
|
|
}
|
|
if config.Extra != nil {
|
|
if v, ok := config.Extra["api_version"]; ok {
|
|
if vs, ok := v.(string); ok && vs != "" {
|
|
apiCfg.APIVersion = vs
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
apiCfg = openai.DefaultConfig(config.APIKey)
|
|
if config.BaseURL != "" {
|
|
apiCfg.BaseURL = config.BaseURL
|
|
}
|
|
}
|
|
apiCfg.HTTPClient = &http.Client{Timeout: defaultTimeout}
|
|
|
|
return &RemoteAPIVLM{
|
|
modelName: config.ModelName,
|
|
modelID: config.ModelID,
|
|
client: openai.NewClientWithConfig(apiCfg),
|
|
baseURL: config.BaseURL,
|
|
}, nil
|
|
}
|
|
|
|
// Predict sends an image with a text prompt to the OpenAI-compatible API.
|
|
func (v *RemoteAPIVLM) Predict(ctx context.Context, imgBytes []byte, prompt string) (string, error) {
|
|
mimeType := detectImageMIME(imgBytes)
|
|
b64 := base64.StdEncoding.EncodeToString(imgBytes)
|
|
dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, b64)
|
|
|
|
req := openai.ChatCompletionRequest{
|
|
Model: v.modelName,
|
|
Messages: []openai.ChatCompletionMessage{
|
|
{
|
|
Role: openai.ChatMessageRoleUser,
|
|
MultiContent: []openai.ChatMessagePart{
|
|
{
|
|
Type: openai.ChatMessagePartTypeImageURL,
|
|
ImageURL: &openai.ChatMessageImageURL{
|
|
URL: dataURI,
|
|
Detail: openai.ImageURLDetailAuto,
|
|
},
|
|
},
|
|
{
|
|
Type: openai.ChatMessagePartTypeText,
|
|
Text: prompt,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
MaxTokens: defaultMaxToks,
|
|
Temperature: defaultTemp,
|
|
}
|
|
|
|
logger.Infof(ctx, "[VLM] Calling OpenAI-compatible API, model=%s, baseURL=%s, imageSize=%d",
|
|
v.modelName, v.baseURL, len(imgBytes))
|
|
|
|
resp, err := v.client.CreateChatCompletion(ctx, req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("OpenAI VLM request: %w", err)
|
|
}
|
|
if len(resp.Choices) == 0 {
|
|
return "", fmt.Errorf("OpenAI VLM returned no choices")
|
|
}
|
|
|
|
content := resp.Choices[0].Message.Content
|
|
logger.Infof(ctx, "[VLM] OpenAI response received, len=%d", len(content))
|
|
return content, nil
|
|
}
|
|
|
|
func (v *RemoteAPIVLM) GetModelName() string { return v.modelName }
|
|
func (v *RemoteAPIVLM) GetModelID() string { return v.modelID }
|
|
|
|
// detectImageMIME returns the MIME type for the given image bytes.
|
|
func detectImageMIME(data []byte) string {
|
|
ct := http.DetectContentType(data)
|
|
if strings.HasPrefix(ct, "image/") {
|
|
return ct
|
|
}
|
|
return "image/png"
|
|
}
|