67 lines
1.8 KiB
Go
67 lines
1.8 KiB
Go
package vlm
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
)
|
|
|
|
// VLM defines the interface for Vision Language Model operations.
|
|
type VLM interface {
|
|
// Predict sends an image with a text prompt to the VLM and returns the generated text.
|
|
Predict(ctx context.Context, imgBytes []byte, prompt string) (string, error)
|
|
|
|
GetModelName() string
|
|
GetModelID() string
|
|
}
|
|
|
|
// Config holds the configuration needed to create a VLM instance.
|
|
type Config struct {
|
|
Source types.ModelSource
|
|
BaseURL string
|
|
ModelName string
|
|
APIKey string
|
|
ModelID string
|
|
InterfaceType string // "ollama" or "openai" (default)
|
|
Provider string
|
|
Extra map[string]any
|
|
}
|
|
|
|
// NewVLM creates a VLM instance based on the provided configuration.
|
|
func NewVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
|
|
ifType := strings.ToLower(config.InterfaceType)
|
|
|
|
if ifType == "ollama" || config.Source == types.ModelSourceLocal {
|
|
return NewOllamaVLM(config, ollamaService)
|
|
}
|
|
return NewRemoteAPIVLM(config)
|
|
}
|
|
|
|
// NewVLMFromLegacyConfig creates a VLM from a legacy VLMConfig (inline BaseURL/APIKey/ModelName).
|
|
func NewVLMFromLegacyConfig(vlmCfg types.VLMConfig, ollamaService *ollama.OllamaService) (VLM, error) {
|
|
if !vlmCfg.IsEnabled() {
|
|
return nil, fmt.Errorf("VLM config is not enabled")
|
|
}
|
|
|
|
ifType := vlmCfg.InterfaceType
|
|
if ifType == "" {
|
|
ifType = "openai"
|
|
}
|
|
|
|
source := types.ModelSourceRemote
|
|
if strings.EqualFold(ifType, "ollama") {
|
|
source = types.ModelSourceLocal
|
|
}
|
|
|
|
return NewVLM(&Config{
|
|
Source: source,
|
|
BaseURL: vlmCfg.BaseURL,
|
|
ModelName: vlmCfg.ModelName,
|
|
APIKey: vlmCfg.APIKey,
|
|
InterfaceType: ifType,
|
|
}, ollamaService)
|
|
}
|