Files
Enterprise-Knowledge-Base/internal/application/service/chat_pipline/common.go
T
wizardchen 421c83452c feat: enhance multimodal image processing and session management
- Updated configuration to support three tasks in the assistant's prompt: rewriting questions, classifying intent for knowledge base retrieval, and analyzing attached images.
- Implemented backend functionality to update image descriptions in user messages, ensuring that image context is preserved across conversation history.
- Modified session handling to return complete session data after updates, improving the accuracy of session information provided to users.
- Enhanced the chat pipeline to include detailed image descriptions and manage image captions effectively, enriching user interactions with visual content.

These improvements significantly enhance the application's multimodal capabilities, providing a more robust and user-friendly experience when handling image-based queries.
2026-03-16 02:26:47 +08:00

108 lines
3.8 KiB
Go

package chatpipline
import (
"context"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// pipelineInfo logs pipeline info level entries.
func pipelineInfo(ctx context.Context, stage, action string, fields map[string]interface{}) {
common.PipelineInfo(ctx, stage, action, fields)
}
// pipelineWarn logs pipeline warning level entries.
func pipelineWarn(ctx context.Context, stage, action string, fields map[string]interface{}) {
common.PipelineWarn(ctx, stage, action, fields)
}
// pipelineError logs pipeline error level entries.
func pipelineError(ctx context.Context, stage, action string, fields map[string]interface{}) {
common.PipelineError(ctx, stage, action, fields)
}
// prepareChatModel shared logic to prepare chat model and options
// it gets the chat model and sets up the chat options based on the chat manage.
func prepareChatModel(ctx context.Context, modelService interfaces.ModelService,
chatManage *types.ChatManage,
) (chat.Chat, *chat.ChatOptions, error) {
chatModel, err := modelService.GetChatModel(ctx, chatManage.ChatModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get chat model: %v", err)
return nil, nil, err
}
opt := &chat.ChatOptions{
Temperature: chatManage.SummaryConfig.Temperature,
TopP: chatManage.SummaryConfig.TopP,
Seed: chatManage.SummaryConfig.Seed,
MaxTokens: chatManage.SummaryConfig.MaxTokens,
MaxCompletionTokens: chatManage.SummaryConfig.MaxCompletionTokens,
FrequencyPenalty: chatManage.SummaryConfig.FrequencyPenalty,
PresencePenalty: chatManage.SummaryConfig.PresencePenalty,
Thinking: chatManage.SummaryConfig.Thinking,
}
return chatModel, opt, nil
}
// prepareMessagesWithHistory prepare complete messages including history
func prepareMessagesWithHistory(chatManage *types.ChatManage) []chat.Message {
// Replace placeholders in system prompt
systemPrompt := renderSystemPromptPlaceholders(chatManage.SummaryConfig.Prompt)
chatMessages := []chat.Message{
{Role: "system", Content: systemPrompt},
}
// Add conversation history (already limited by maxRounds in load_history/rewrite plugins)
for _, history := range chatManage.History {
chatMessages = append(chatMessages, chat.Message{Role: "user", Content: history.Query})
chatMessages = append(chatMessages, chat.Message{Role: "assistant", Content: history.Answer})
}
// Add current user message. Only include images when the chat model supports
// vision; non-vision models rely on the text description in UserContent.
userMsg := chat.Message{Role: "user", Content: chatManage.UserContent}
if chatManage.ChatModelSupportsVision && len(chatManage.Images) > 0 {
userMsg.Images = chatManage.Images
}
chatMessages = append(chatMessages, userMsg)
return chatMessages
}
// extractImageCaptions concatenates non-empty Caption fields from stored
// message images. Used when loading history so that previous turns' image
// descriptions are visible to the model.
func extractImageCaptions(images types.MessageImages) string {
var parts []string
for _, img := range images {
if img.Caption != "" {
parts = append(parts, img.Caption)
}
}
return strings.Join(parts, "\n")
}
// renderSystemPromptPlaceholders replaces placeholders in system prompt
// Supported placeholders:
// - {{current_time}} -> current time in RFC3339 format
func renderSystemPromptPlaceholders(prompt string) string {
result := prompt
// Replace {{current_time}} placeholder
if strings.Contains(result, "{{current_time}}") {
currentTime := time.Now().Format(time.RFC3339)
result = strings.ReplaceAll(result, "{{current_time}}", currentTime)
}
return result
}