Files
Enterprise-Knowledge-Base/internal/models/vlm/vlm.go
T
wizardchen 0fcab3a413 feat(video): update ASR terminology and file type labels for improved clarity
- Changed 'Audio Processing' to 'Audio & Video' in English, Korean, Russian, and Chinese translations to reflect expanded functionality.
- Updated ASR configuration titles and descriptions to include video processing capabilities, enhancing user understanding of the feature.
- Renamed file type labels from 'Audio Files' to 'Audio & Video' across multiple languages for consistency and clarity.
- Enhanced document upload handling in the KnowledgeBase to filter unsupported file types, providing user feedback for filtered images, videos, and audio files.

This update improves the internationalization of the application and clarifies the capabilities of the ASR feature.
2026-04-14 22:59:14 +08:00

67 lines
1.8 KiB
Go

package vlm
import (
"context"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/types"
)
// VLM defines the interface for Vision Language Model operations.
type VLM interface {
// Predict sends one or more images with a text prompt to the VLM and returns the generated text.
Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error)
GetModelName() string
GetModelID() string
}
// Config holds the configuration needed to create a VLM instance.
type Config struct {
Source types.ModelSource
BaseURL string
ModelName string
APIKey string
ModelID string
InterfaceType string // "ollama" or "openai" (default)
Provider string
Extra map[string]any
}
// NewVLM creates a VLM instance based on the provided configuration.
func NewVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
ifType := strings.ToLower(config.InterfaceType)
if ifType == "ollama" || config.Source == types.ModelSourceLocal {
return NewOllamaVLM(config, ollamaService)
}
return NewRemoteAPIVLM(config)
}
// NewVLMFromLegacyConfig creates a VLM from a legacy VLMConfig (inline BaseURL/APIKey/ModelName).
func NewVLMFromLegacyConfig(vlmCfg types.VLMConfig, ollamaService *ollama.OllamaService) (VLM, error) {
if !vlmCfg.IsEnabled() {
return nil, fmt.Errorf("VLM config is not enabled")
}
ifType := vlmCfg.InterfaceType
if ifType == "" {
ifType = "openai"
}
source := types.ModelSourceRemote
if strings.EqualFold(ifType, "ollama") {
source = types.ModelSourceLocal
}
return NewVLM(&Config{
Source: source,
BaseURL: vlmCfg.BaseURL,
ModelName: vlmCfg.ModelName,
APIKey: vlmCfg.APIKey,
InterfaceType: ifType,
}, ollamaService)
}