53e8e17df4
- Added a new logger for LLM calls, enabling detailed logging of model interactions, including request and response data. - Introduced configuration options for enabling and specifying the log directory for LLM debug logs. - Implemented cleanup for old log files to manage disk space effectively. - Wrapped existing chat implementations to log calls when debug logging is enabled, enhancing traceability for model interactions.
88 lines
2.4 KiB
Go
88 lines
2.4 KiB
Go
package vlm
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/models/provider"
|
|
"github.com/Tencent/WeKnora/internal/models/utils/ollama"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
)
|
|
|
|
// VLM defines the interface for Vision Language Model operations.
|
|
type VLM interface {
|
|
// Predict sends one or more images with a text prompt to the VLM and returns the generated text.
|
|
Predict(ctx context.Context, imgBytes [][]byte, prompt string) (string, error)
|
|
|
|
GetModelName() string
|
|
GetModelID() string
|
|
}
|
|
|
|
// Config holds the configuration needed to create a VLM instance.
|
|
type Config struct {
|
|
Source types.ModelSource
|
|
BaseURL string
|
|
ModelName string
|
|
APIKey string
|
|
ModelID string
|
|
InterfaceType string // "ollama" or "openai" (default)
|
|
Provider string
|
|
Extra map[string]any
|
|
AppID string
|
|
AppSecret string
|
|
}
|
|
|
|
// NewVLM creates a VLM instance based on the provided configuration.
|
|
func NewVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
|
|
v, err := newVLM(config, ollamaService)
|
|
if err != nil || !logger.LLMDebugEnabled() {
|
|
return v, err
|
|
}
|
|
return &debugVLM{inner: v}, nil
|
|
}
|
|
|
|
func newVLM(config *Config, ollamaService *ollama.OllamaService) (VLM, error) {
|
|
ifType := strings.ToLower(config.InterfaceType)
|
|
|
|
if ifType == "ollama" || config.Source == types.ModelSourceLocal {
|
|
return NewOllamaVLM(config, ollamaService)
|
|
}
|
|
|
|
providerName := provider.ProviderName(config.Provider)
|
|
if providerName == "" {
|
|
providerName = provider.DetectProvider(config.BaseURL)
|
|
}
|
|
if providerName == provider.ProviderWeKnoraCloud {
|
|
return NewWeKnoraCloudVLM(config)
|
|
}
|
|
|
|
return NewRemoteAPIVLM(config)
|
|
}
|
|
|
|
// NewVLMFromLegacyConfig creates a VLM from a legacy VLMConfig (inline BaseURL/APIKey/ModelName).
|
|
func NewVLMFromLegacyConfig(vlmCfg types.VLMConfig, ollamaService *ollama.OllamaService) (VLM, error) {
|
|
if !vlmCfg.IsEnabled() {
|
|
return nil, fmt.Errorf("VLM config is not enabled")
|
|
}
|
|
|
|
ifType := vlmCfg.InterfaceType
|
|
if ifType == "" {
|
|
ifType = "openai"
|
|
}
|
|
|
|
source := types.ModelSourceRemote
|
|
if strings.EqualFold(ifType, "ollama") {
|
|
source = types.ModelSourceLocal
|
|
}
|
|
|
|
return NewVLM(&Config{
|
|
Source: source,
|
|
BaseURL: vlmCfg.BaseURL,
|
|
ModelName: vlmCfg.ModelName,
|
|
APIKey: vlmCfg.APIKey,
|
|
InterfaceType: ifType,
|
|
}, ollamaService)
|
|
}
|