Files
Enterprise-Knowledge-Base/internal/application/service/chat_pipeline/query_understand.go
T
wizardchen 72dfb9ce75 feat(prompt): enhance context and intent classification in templates
- Updated context template to include runtime metadata such as current time and week, improving contextual awareness in user queries.
- Enhanced rewrite template with critical instructions for intent classification, ensuring that rewritten questions preserve essential entities and keywords.
- Refined intent classification logic to prioritize user intents more effectively, improving the accuracy of responses based on user queries.
- Added examples to clarify expected input and output formats for intent classification, enhancing usability for developers.
2026-03-26 18:57:12 +08:00

455 lines
14 KiB
Go

package chatpipeline
import (
"context"
"encoding/json"
"regexp"
"strings"
"time"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/google/uuid"
)
// PluginQueryUnderstand performs query rewriting and intent classification.
// It uses conversation history and an LLM to optimise the user's original query
// and determine the downstream pipeline behaviour.
type PluginQueryUnderstand struct {
modelService interfaces.ModelService
messageService interfaces.MessageService
config *config.Config
}
var rewriteImageSepPattern = regexp.MustCompile(`(?s)^(.*?)\s*\n?---\n(.*)$`)
type queryUnderstandOutput struct {
RewriteQuery string `json:"rewrite_query"`
Intent types.QueryIntent `json:"intent"`
ImageDescription string `json:"image_description"`
}
// NewPluginQueryUnderstand creates a new query-understanding plugin instance
// and registers it with the event manager.
func NewPluginQueryUnderstand(eventManager *EventManager,
modelService interfaces.ModelService, messageService interfaces.MessageService,
config *config.Config,
) *PluginQueryUnderstand {
res := &PluginQueryUnderstand{
modelService: modelService,
messageService: messageService,
config: config,
}
eventManager.Register(res)
return res
}
// ActivationEvents returns the list of event types this plugin responds to.
func (p *PluginQueryUnderstand) ActivationEvents() []types.EventType {
return []types.EventType{types.QUERY_UNDERSTAND}
}
// OnEvent processes triggered events.
// Handles three input combinations:
// - Text only: standard rewrite + intent classification (uses chat model)
// - Text + images: multimodal rewrite + intent + image description (uses VLM/vision model)
// - Images only: multimodal analysis + intent + image description (uses VLM/vision model)
func (p *PluginQueryUnderstand) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
chatManage.RewriteQuery = chatManage.Query
hasImages := len(chatManage.Images) > 0
needRewrite := chatManage.EnableRewrite
if !needRewrite && !hasImages {
pipelineInfo(ctx, "QueryUnderstand", "skip", map[string]interface{}{
"session_id": chatManage.SessionID,
"reason": "rewrite_disabled_no_images",
})
return next()
}
pipelineInfo(ctx, "QueryUnderstand", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"tenant_id": chatManage.TenantID,
"user_query": chatManage.Query,
"has_images": hasImages,
"enable_rewrite": chatManage.EnableRewrite,
})
// --- Load and prepare conversation history ---
var historyList []*types.History
if len(chatManage.History) > 0 {
historyList = chatManage.History
pipelineInfo(ctx, "QueryUnderstand", "history_reused", map[string]interface{}{
"session_id": chatManage.SessionID,
"rounds": len(historyList),
})
} else {
historyList = p.loadHistory(ctx, chatManage)
}
// --- Select the appropriate model ---
rewriteModel, useImages := p.selectModel(ctx, chatManage, hasImages)
if rewriteModel == nil {
pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{
"session_id": chatManage.SessionID,
})
return next()
}
// --- Build prompts ---
systemContent, userContent := p.buildPrompts(chatManage, historyList)
userMsg := chat.Message{Role: "user", Content: userContent}
if useImages {
userMsg.Images = chatManage.Images
}
maxTokens := 150
if useImages {
maxTokens = 500
}
// --- Emit progress event for image analysis ---
var toolCallID string
if useImages && chatManage.EventBus != nil {
toolCallID = uuid.New().String()
chatManage.EventBus.Emit(ctx, types.Event{
Type: types.EventType(event.EventAgentToolCall),
SessionID: chatManage.SessionID,
Data: event.AgentToolCallData{
ToolCallID: toolCallID,
ToolName: "image_analysis",
},
})
}
// --- Call model ---
thinking := false
vlmStart := time.Now()
response, err := rewriteModel.Chat(ctx, []chat.Message{
{Role: "system", Content: systemContent},
userMsg,
}, &chat.ChatOptions{
Temperature: 0.3,
MaxCompletionTokens: maxTokens,
Thinking: &thinking,
})
if err != nil {
if toolCallID != "" && chatManage.EventBus != nil {
chatManage.EventBus.Emit(ctx, types.Event{
Type: types.EventType(event.EventAgentToolResult),
SessionID: chatManage.SessionID,
Data: event.AgentToolResultData{
ToolCallID: toolCallID,
ToolName: "image_analysis",
Output: "图片分析失败",
Success: false,
Duration: time.Since(vlmStart).Milliseconds(),
},
})
}
pipelineError(ctx, "QueryUnderstand", "model_call", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
return next()
}
// --- Emit completion event for image analysis ---
if toolCallID != "" && chatManage.EventBus != nil {
chatManage.EventBus.Emit(ctx, types.Event{
Type: types.EventType(event.EventAgentToolResult),
SessionID: chatManage.SessionID,
Data: event.AgentToolResultData{
ToolCallID: toolCallID,
ToolName: "image_analysis",
Output: "已分析图片内容",
Success: true,
Duration: time.Since(vlmStart).Milliseconds(),
},
})
}
// --- Parse structured output ---
p.parseOutput(chatManage, response.Content)
// Persist image description asynchronously — this DB write does not affect
// the current pipeline result, so it can run in the background.
if chatManage.ImageDescription != "" && chatManage.UserMessageID != "" {
go p.updateUserMessageImageCaption(context.WithoutCancel(ctx), chatManage)
}
// --- Apply intent-specific system prompt override ---
if !chatManage.NeedsRetrieval() {
if prompt, ok := p.config.Conversation.IntentSystemPrompts[string(chatManage.Intent)]; ok {
chatManage.SystemPromptOverride = prompt
pipelineInfo(ctx, "QueryUnderstand", "prompt_override", map[string]interface{}{
"session_id": chatManage.SessionID,
"intent": chatManage.Intent,
})
}
}
pipelineInfo(ctx, "QueryUnderstand", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"rewrite_query": chatManage.RewriteQuery,
"intent": chatManage.Intent,
"has_image_desc": chatManage.ImageDescription != "",
"has_prompt_override": chatManage.SystemPromptOverride != "",
"original_output": response.Content,
})
return next()
}
// updateUserMessageImageCaption writes the generated ImageDescription back to
// the stored user message so that subsequent turns can see it in history.
func (p *PluginQueryUnderstand) updateUserMessageImageCaption(ctx context.Context, chatManage *types.ChatManage) {
msg, err := p.messageService.GetMessage(ctx, chatManage.SessionID, chatManage.UserMessageID)
if err != nil {
pipelineWarn(ctx, "QueryUnderstand", "get_user_message", map[string]interface{}{
"session_id": chatManage.SessionID,
"user_message_id": chatManage.UserMessageID,
"error": err.Error(),
})
return
}
if len(msg.Images) == 0 {
return
}
msg.Images[0].Caption = chatManage.ImageDescription
if err := p.messageService.UpdateMessageImages(ctx, chatManage.SessionID, chatManage.UserMessageID, msg.Images); err != nil {
pipelineWarn(ctx, "QueryUnderstand", "update_image_caption", map[string]interface{}{
"session_id": chatManage.SessionID,
"user_message_id": chatManage.UserMessageID,
"error": err.Error(),
})
}
}
// loadHistory fetches and processes conversation history for rewrite context.
func (p *PluginQueryUnderstand) loadHistory(ctx context.Context, chatManage *types.ChatManage) []*types.History {
maxRounds := p.config.Conversation.MaxRounds
if chatManage.MaxRounds > 0 {
maxRounds = chatManage.MaxRounds
}
historyList, err := loadAndProcessHistory(ctx, p.messageService, chatManage.SessionID, maxRounds, 20)
if err != nil {
pipelineWarn(ctx, "QueryUnderstand", "history_fetch", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
return nil
}
chatManage.History = historyList
if len(historyList) > 0 {
pipelineInfo(ctx, "QueryUnderstand", "history_ready", map[string]interface{}{
"session_id": chatManage.SessionID,
"history_rounds": len(historyList),
})
}
return historyList
}
// selectModel picks the model for query understanding. When images are present
// it prefers a vision-capable model. Returns (model, useImages).
func (p *PluginQueryUnderstand) selectModel(ctx context.Context, chatManage *types.ChatManage, hasImages bool) (chat.Chat, bool) {
if hasImages {
if chatManage.ChatModelSupportsVision {
m, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
if err == nil {
return m, true
}
pipelineWarn(ctx, "QueryUnderstand", "vision_model_fallback", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
}
if chatManage.VLMModelID != "" {
m, err := p.modelService.GetChatModel(ctx, chatManage.VLMModelID)
if err == nil {
return m, true
}
pipelineWarn(ctx, "QueryUnderstand", "vlm_model_fallback", map[string]interface{}{
"session_id": chatManage.SessionID,
"vlm_model_id": chatManage.VLMModelID,
"error": err.Error(),
})
}
pipelineWarn(ctx, "QueryUnderstand", "no_vision_model", map[string]interface{}{
"session_id": chatManage.SessionID,
})
}
m, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
if err != nil {
pipelineError(ctx, "QueryUnderstand", "get_model", map[string]interface{}{
"session_id": chatManage.SessionID,
"chat_model_id": chatManage.ChatModelID,
"error": err.Error(),
})
return nil, false
}
return m, false
}
// buildPrompts constructs system and user prompts with placeholder replacement.
func (p *PluginQueryUnderstand) buildPrompts(chatManage *types.ChatManage, historyList []*types.History) (string, string) {
userPrompt := p.config.Conversation.RewritePromptUser
if chatManage.RewritePromptUser != "" {
userPrompt = chatManage.RewritePromptUser
}
systemPrompt := p.config.Conversation.RewritePromptSystem
if chatManage.RewritePromptSystem != "" {
systemPrompt = chatManage.RewritePromptSystem
}
conversationText := formatConversationHistory(historyList)
vals := types.PlaceholderValues{
"conversation": conversationText,
"query": chatManage.Query,
"language": chatManage.Language,
}
return types.RenderPromptPlaceholders(systemPrompt, vals),
types.RenderPromptPlaceholders(userPrompt, vals)
}
// parseOutput extracts the rewritten query, intent classification, and optional
// image description from the model's structured JSON output.
//
// Expected format: {"rewrite_query":"...","intent":"kb_search","image_description":"..."}
func (p *PluginQueryUnderstand) parseOutput(chatManage *types.ChatManage, raw string) {
content := strings.TrimSpace(raw)
if content == "" {
return
}
if output, ok := parseStructuredQueryOutput(content); ok {
if rewrite := strings.TrimSpace(output.RewriteQuery); rewrite != "" {
chatManage.RewriteQuery = rewrite
}
chatManage.Intent = output.Intent
chatManage.ImageDescription = strings.TrimSpace(output.ImageDescription)
return
}
// If JSON parsing failed entirely, treat the raw text as the rewritten query
// and default to IntentKBSearch for safety.
if content != "" {
chatManage.RewriteQuery = content
}
}
func parseStructuredQueryOutput(raw string) (queryUnderstandOutput, bool) {
content := strings.TrimSpace(raw)
if content == "" {
return queryUnderstandOutput{}, false
}
if parsed, ok := parseStructuredQueryOutputJSON(content); ok {
return parsed, true
}
// Be tolerant to occasional markdown wrappers or extra prose.
start := strings.Index(content, "{")
end := strings.LastIndex(content, "}")
if start < 0 || end <= start {
return queryUnderstandOutput{}, false
}
candidate := content[start : end+1]
if parsed, ok := parseStructuredQueryOutputJSON(candidate); ok {
return parsed, true
}
return queryUnderstandOutput{}, false
}
func parseStructuredQueryOutputJSON(content string) (queryUnderstandOutput, bool) {
var obj map[string]json.RawMessage
if err := json.Unmarshal([]byte(content), &obj); err != nil {
return queryUnderstandOutput{}, false
}
out := queryUnderstandOutput{
RewriteQuery: strings.TrimSpace(firstStringField(obj,
"rewrite_query", "rewritten_query", "query", "question")),
}
intentStr := strings.TrimSpace(firstStringField(obj, "intent"))
if intentStr != "" {
out.Intent = types.QueryIntent(intentStr)
}
desc := strings.TrimSpace(firstStringField(obj,
"image_description", "image_desc", "image_text", "image_ocr_text", "description"))
ocr := strings.TrimSpace(firstStringField(obj,
"ocr_text", "ocr", "full_ocr", "image_ocr", "ocr_content"))
combined, set := mergeImageDescAndOCR(desc, ocr)
if set {
out.ImageDescription = combined
}
return out, true
}
func firstStringField(obj map[string]json.RawMessage, keys ...string) string {
for _, key := range keys {
raw, ok := obj[key]
if !ok || len(raw) == 0 {
continue
}
var s string
if err := json.Unmarshal(raw, &s); err == nil {
return s
}
}
return ""
}
func mergeImageDescAndOCR(desc, ocr string) (string, bool) {
if desc == "" && ocr == "" {
return "", false
}
if desc == "" {
return ocr, true
}
if ocr == "" {
return desc, true
}
if strings.Contains(desc, ocr) {
return desc, true
}
return desc + "\n\n[OCR]\n" + ocr, true
}
// formatConversationHistory formats conversation history for prompt template.
func formatConversationHistory(historyList []*types.History) string {
if len(historyList) == 0 {
return ""
}
var builder strings.Builder
for _, h := range historyList {
builder.WriteString("------BEGIN------\n")
builder.WriteString("User question: ")
builder.WriteString(h.Query)
builder.WriteString("\nAssistant answer: ")
builder.WriteString(h.Answer)
builder.WriteString("\n------END------\n")
}
return builder.String()
}