Files
Enterprise-Knowledge-Base/internal/models/vlm/vlm.go
T
wizardchen 9ba8582f70 feat(weknoracloud): implement WeKnoraCloud VLM integration
- Added WeKnoraCloudVLM struct to handle interactions with the WeKnoraCloud API, including image and text processing capabilities.
- Enhanced model configuration to include AppID and AppSecret for secure API access.
- Updated modelService to resolve WeKnoraCloud credentials and pass them to the VLM configuration.
- Implemented Predict method for sending requests to the WeKnoraCloud API and processing responses.

This update introduces a new integration with WeKnoraCloud, expanding the application's capabilities in handling multimedia content.
2026-04-15 15:43:08 +08:00

79 lines
2.1 KiB
Go

package vlm
import (
"context"
"fmt"
"strings"
"github.com/Tencent/WeKnora/internal/models/provider"
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
"github.com/Tencent/WeKnora/internal/types"
)
// VLM defines the interface for Vision Language Model operations.
type VLM interface {
// Predict sends one or more images with a text prompt to the VLM and returns the generated text.
Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error)
GetModelName() string
GetModelID() string
}
// Config holds the configuration needed to create a VLM instance.
type Config struct {
Source types.ModelSource
BaseURL string
ModelName string
APIKey string
ModelID string
InterfaceType string // "ollama" or "openai" (default)
Provider string
Extra map[string]any
AppID string
AppSecret string
}
// NewVLM creates a VLM instance based on the provided configuration.
func NewVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
ifType := strings.ToLower(config.InterfaceType)
if ifType == "ollama" || config.Source == types.ModelSourceLocal {
return NewOllamaVLM(config, ollamaService)
}
providerName := provider.ProviderName(config.Provider)
if providerName == "" {
providerName = provider.DetectProvider(config.BaseURL)
}
if providerName == provider.ProviderWeKnoraCloud {
return NewWeKnoraCloudVLM(config)
}
return NewRemoteAPIVLM(config)
}
// NewVLMFromLegacyConfig creates a VLM from a legacy VLMConfig (inline BaseURL/APIKey/ModelName).
func NewVLMFromLegacyConfig(vlmCfg types.VLMConfig, ollamaService *ollama.OllamaService) (VLM, error) {
if !vlmCfg.IsEnabled() {
return nil, fmt.Errorf("VLM config is not enabled")
}
ifType := vlmCfg.InterfaceType
if ifType == "" {
ifType = "openai"
}
source := types.ModelSourceRemote
if strings.EqualFold(ifType, "ollama") {
source = types.ModelSourceLocal
}
return NewVLM(&Config{
Source: source,
BaseURL: vlmCfg.BaseURL,
ModelName: vlmCfg.ModelName,
APIKey: vlmCfg.APIKey,
InterfaceType: ifType,
}, ollamaService)
}