0fcab3a413
- Changed 'Audio Processing' to 'Audio & Video' in English, Korean, Russian, and Chinese translations to reflect expanded functionality. - Updated ASR configuration titles and descriptions to include video processing capabilities, enhancing user understanding of the feature. - Renamed file type labels from 'Audio Files' to 'Audio & Video' across multiple languages for consistency and clarity. - Enhanced document upload handling in the KnowledgeBase to filter unsupported file types, providing user feedback for filtered images, videos, and audio files. This update improves the internationalization of the application and clarifies the capabilities of the ASR feature.
67 lines
1.8 KiB
Go
67 lines
1.8 KiB
Go
package vlm
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
)
|
|
|
|
// VLM defines the interface for Vision Language Model operations.
|
|
type VLM interface {
|
|
// Predict sends one or more images with a text prompt to the VLM and returns the generated text.
|
|
Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error)
|
|
|
|
GetModelName() string
|
|
GetModelID() string
|
|
}
|
|
|
|
// Config holds the configuration needed to create a VLM instance.
|
|
type Config struct {
|
|
Source types.ModelSource
|
|
BaseURL string
|
|
ModelName string
|
|
APIKey string
|
|
ModelID string
|
|
InterfaceType string // "ollama" or "openai" (default)
|
|
Provider string
|
|
Extra map[string]any
|
|
}
|
|
|
|
// NewVLM creates a VLM instance based on the provided configuration.
|
|
func NewVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
|
|
ifType := strings.ToLower(config.InterfaceType)
|
|
|
|
if ifType == "ollama" || config.Source == types.ModelSourceLocal {
|
|
return NewOllamaVLM(config, ollamaService)
|
|
}
|
|
return NewRemoteAPIVLM(config)
|
|
}
|
|
|
|
// NewVLMFromLegacyConfig creates a VLM from a legacy VLMConfig (inline BaseURL/APIKey/ModelName).
|
|
func NewVLMFromLegacyConfig(vlmCfg types.VLMConfig, ollamaService *ollama.OllamaService) (VLM, error) {
|
|
if !vlmCfg.IsEnabled() {
|
|
return nil, fmt.Errorf("VLM config is not enabled")
|
|
}
|
|
|
|
ifType := vlmCfg.InterfaceType
|
|
if ifType == "" {
|
|
ifType = "openai"
|
|
}
|
|
|
|
source := types.ModelSourceRemote
|
|
if strings.EqualFold(ifType, "ollama") {
|
|
source = types.ModelSourceLocal
|
|
}
|
|
|
|
return NewVLM(&Config{
|
|
Source: source,
|
|
BaseURL: vlmCfg.BaseURL,
|
|
ModelName: vlmCfg.ModelName,
|
|
APIKey: vlmCfg.APIKey,
|
|
InterfaceType: ifType,
|
|
}, ollamaService)
|
|
}
|