Files
Enterprise-Knowledge-Base/internal/models/vlm/vlm.go
T
2026-04-09 19:19:25 +08:00

67 lines
1.8 KiB
Go

package vlm
import (
"context"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/types"
)
// VLM defines the interface for Vision Language Model operations.
type VLM interface {
// Predict sends an image with a text prompt to the VLM and returns the generated text.
Predict(ctx context.Context, imgBytes []byte, prompt string) (string, error)
GetModelName() string
GetModelID() string
}
// Config holds the configuration needed to create a VLM instance.
type Config struct {
Source types.ModelSource
BaseURL string
ModelName string
APIKey string
ModelID string
InterfaceType string // "ollama" or "openai" (default)
Provider string
Extra map[string]any
}
// NewVLM creates a VLM instance based on the provided configuration.
func NewVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
ifType := strings.ToLower(config.InterfaceType)
if ifType == "ollama" || config.Source == types.ModelSourceLocal {
return NewOllamaVLM(config, ollamaService)
}
return NewRemoteAPIVLM(config)
}
// NewVLMFromLegacyConfig creates a VLM from a legacy VLMConfig (inline BaseURL/APIKey/ModelName).
func NewVLMFromLegacyConfig(vlmCfg types.VLMConfig, ollamaService *ollama.OllamaService) (VLM, error) {
if !vlmCfg.IsEnabled() {
return nil, fmt.Errorf("VLM config is not enabled")
}
ifType := vlmCfg.InterfaceType
if ifType == "" {
ifType = "openai"
}
source := types.ModelSourceRemote
if strings.EqualFold(ifType, "ollama") {
source = types.ModelSourceLocal
}
return NewVLM(&Config{
Source: source,
BaseURL: vlmCfg.BaseURL,
ModelName: vlmCfg.ModelName,
APIKey: vlmCfg.APIKey,
InterfaceType: ifType,
}, ollamaService)
}