feat: Revamp conversation configuration management and enhance localization
- Removed outdated DEVELOPMENT.md, consolidating development instructions into relevant documentation. - Updated conversation configuration to include new parameters such as max rounds, embedding thresholds, and rewrite prompts. - Enhanced UI components to support new conversation settings, including AgentSettings.vue and related views. - Improved localization in English, Russian, and Chinese for new conversation configuration fields and UI elements. - Implemented logic to apply tenant-specific conversation settings, ensuring defaults are respected.
This commit is contained in:
@@ -3,7 +3,6 @@ package chatpipline
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
@@ -34,7 +33,12 @@ func (p *PluginChatCompletion) ActivationEvents() []types.EventType {
|
||||
func (p *PluginChatCompletion) OnEvent(
|
||||
ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
logger.Info(ctx, "Starting chat completion")
|
||||
pipelineInfo(ctx, "Completion", "input", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"user_question": chatManage.UserContent,
|
||||
"history_rounds": len(chatManage.History),
|
||||
"chat_model": chatManage.ChatModelID,
|
||||
})
|
||||
|
||||
// Prepare chat model and options
|
||||
chatModel, opt, err := prepareChatModel(ctx, p.modelService, chatManage)
|
||||
@@ -43,18 +47,30 @@ func (p *PluginChatCompletion) OnEvent(
|
||||
}
|
||||
|
||||
// Prepare messages including conversation history
|
||||
logger.Info(ctx, "Preparing chat messages with history")
|
||||
pipelineInfo(ctx, "Completion", "messages_ready", map[string]interface{}{
|
||||
"message_count": len(chatManage.History) + 2,
|
||||
})
|
||||
chatMessages := prepareMessagesWithHistory(chatManage)
|
||||
|
||||
// Call the chat model to generate response
|
||||
logger.Info(ctx, "Calling chat model")
|
||||
pipelineInfo(ctx, "Completion", "model_call", map[string]interface{}{
|
||||
"chat_model": chatManage.ChatModelID,
|
||||
})
|
||||
chatResponse, err := chatModel.Chat(ctx, chatMessages, opt)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to call chat model: %v", err)
|
||||
pipelineError(ctx, "Completion", "model_call", map[string]interface{}{
|
||||
"chat_model": chatManage.ChatModelID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return ErrModelCall.WithError(err)
|
||||
}
|
||||
|
||||
logger.Info(ctx, "Chat completion successful")
|
||||
pipelineInfo(ctx, "Completion", "output", map[string]interface{}{
|
||||
"answer_preview": truncateForLog(chatResponse.Content),
|
||||
"finish_reason": chatResponse.FinishReason,
|
||||
"completion_tokens": chatResponse.Usage.CompletionTokens,
|
||||
"prompt_tokens": chatResponse.Usage.PromptTokens,
|
||||
})
|
||||
chatManage.ChatResponse = chatResponse
|
||||
return next()
|
||||
}
|
||||
|
||||
@@ -40,7 +40,12 @@ func (p *PluginChatCompletionStream) ActivationEvents() []types.EventType {
|
||||
func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
logger.Info(ctx, "Starting chat completion stream")
|
||||
pipelineInfo(ctx, "Stream", "input", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"user_question": chatManage.UserContent,
|
||||
"history_rounds": len(chatManage.History),
|
||||
"chat_model": chatManage.ChatModelID,
|
||||
})
|
||||
|
||||
// Prepare chat model and options
|
||||
chatModel, opt, err := prepareChatModel(ctx, p.modelService, chatManage)
|
||||
@@ -49,31 +54,49 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
||||
}
|
||||
|
||||
// Prepare base messages without history
|
||||
logger.Info(ctx, "Preparing chat messages")
|
||||
chatMessages := prepareMessagesWithHistory(chatManage)
|
||||
|
||||
chatMessages := prepareMessagesWithHistory(chatManage)
|
||||
pipelineInfo(ctx, "Stream", "messages_ready", map[string]interface{}{
|
||||
"message_count": len(chatMessages),
|
||||
"system_prompt": chatMessages[0].Content,
|
||||
"user_content": chatMessages[len(chatMessages)-1].Content,
|
||||
})
|
||||
// EventBus is required for event-driven streaming
|
||||
if chatManage.EventBus == nil {
|
||||
logger.Error(ctx, "EventBus is required but not available")
|
||||
pipelineError(ctx, "Stream", "eventbus_missing", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
return ErrModelCall.WithError(errors.New("EventBus is required for streaming"))
|
||||
}
|
||||
eventBus := chatManage.EventBus
|
||||
|
||||
logger.Info(ctx, "EventBus detected, enabling event-driven streaming mode")
|
||||
pipelineInfo(ctx, "Stream", "eventbus_ready", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
|
||||
// Initiate streaming chat model call with independent context
|
||||
logger.Info(ctx, "Calling chat stream model")
|
||||
pipelineInfo(ctx, "Stream", "model_call", map[string]interface{}{
|
||||
"chat_model": chatManage.ChatModelID,
|
||||
})
|
||||
responseChan, err := chatModel.ChatStream(ctx, chatMessages, opt)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to call chat stream model: %v", err)
|
||||
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
|
||||
"chat_model": chatManage.ChatModelID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return ErrModelCall.WithError(err)
|
||||
}
|
||||
if responseChan == nil {
|
||||
logger.Error(ctx, "Chat stream returned nil channel")
|
||||
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
|
||||
"chat_model": chatManage.ChatModelID,
|
||||
"error": "nil_channel",
|
||||
})
|
||||
return ErrModelCall.WithError(errors.New("chat stream returned nil channel"))
|
||||
}
|
||||
|
||||
logger.Info(ctx, "Chat stream initiated successfully")
|
||||
pipelineInfo(ctx, "Stream", "model_started", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
|
||||
// Start goroutine to consume channel and emit events directly
|
||||
go func() {
|
||||
@@ -98,24 +121,9 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info(ctx, "Chat stream completed, emitting completion event")
|
||||
|
||||
// Emit completion event when stream finishes
|
||||
// This allows other components to detect stream completion
|
||||
if err := eventBus.Emit(ctx, types.Event{
|
||||
ID: fmt.Sprintf("%s-complete", uuid.New().String()[:8]),
|
||||
Type: types.EventType(event.EventAgentComplete),
|
||||
SessionID: chatManage.SessionID,
|
||||
Data: event.AgentCompleteData{
|
||||
SessionID: chatManage.SessionID,
|
||||
MessageID: chatManage.MessageID,
|
||||
FinalAnswer: finalContent,
|
||||
},
|
||||
}); err != nil {
|
||||
logger.Errorf(ctx, "Failed to emit completion event: %v", err)
|
||||
}
|
||||
|
||||
logger.Info(ctx, "Chat stream completed and completion event emitted")
|
||||
pipelineInfo(ctx, "Stream", "channel_close", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
}()
|
||||
|
||||
return next()
|
||||
|
||||
@@ -2,6 +2,10 @@ package chatpipline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
@@ -9,6 +13,78 @@ import (
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
const (
|
||||
logValueMaxRune = 300
|
||||
defaultStageName = "PIPELINE"
|
||||
defaultActionName = "info"
|
||||
pipelineLogPrefix = "[PIPELINE]"
|
||||
pipelineTruncateEll = "..."
|
||||
)
|
||||
|
||||
func pipelineLog(stage, action string, fields map[string]interface{}) string {
|
||||
if stage == "" {
|
||||
stage = defaultStageName
|
||||
}
|
||||
if action == "" {
|
||||
action = defaultActionName
|
||||
}
|
||||
|
||||
builder := strings.Builder{}
|
||||
builder.Grow(128)
|
||||
builder.WriteString(pipelineLogPrefix)
|
||||
builder.WriteString(" stage=")
|
||||
builder.WriteString(stage)
|
||||
builder.WriteString(" action=")
|
||||
builder.WriteString(action)
|
||||
|
||||
if len(fields) > 0 {
|
||||
keys := make([]string, 0, len(fields))
|
||||
for k := range fields {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
builder.WriteString(" ")
|
||||
builder.WriteString(key)
|
||||
builder.WriteString("=")
|
||||
builder.WriteString(formatLogValue(fields[key]))
|
||||
}
|
||||
}
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
func pipelineInfo(ctx context.Context, stage, action string, fields map[string]interface{}) {
|
||||
logger.GetLogger(ctx).Info(pipelineLog(stage, action, fields))
|
||||
}
|
||||
|
||||
func pipelineWarn(ctx context.Context, stage, action string, fields map[string]interface{}) {
|
||||
logger.GetLogger(ctx).Warn(pipelineLog(stage, action, fields))
|
||||
}
|
||||
|
||||
func pipelineError(ctx context.Context, stage, action string, fields map[string]interface{}) {
|
||||
logger.GetLogger(ctx).Error(pipelineLog(stage, action, fields))
|
||||
}
|
||||
|
||||
func formatLogValue(value interface{}) string {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
return strconv.Quote(truncateForLog(v))
|
||||
case fmt.Stringer:
|
||||
return strconv.Quote(truncateForLog(v.String()))
|
||||
default:
|
||||
return fmt.Sprintf("%v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func truncateForLog(content string) string {
|
||||
content = strings.ReplaceAll(content, "\n", "\\n")
|
||||
runes := []rune(content)
|
||||
if len(runes) <= logValueMaxRune {
|
||||
return content
|
||||
}
|
||||
return string(runes[:logValueMaxRune]) + pipelineTruncateEll
|
||||
}
|
||||
|
||||
// prepareChatModel shared logic to prepare chat model and options
|
||||
func prepareChatModel(ctx context.Context, modelService interfaces.ModelService,
|
||||
chatManage *types.ChatManage,
|
||||
@@ -35,14 +111,6 @@ func prepareChatModel(ctx context.Context, modelService interfaces.ModelService,
|
||||
return chatModel, opt, nil
|
||||
}
|
||||
|
||||
// prepareBaseMessages prepare basic messages (system prompt and current user content)
|
||||
func prepareBaseMessages(chatManage *types.ChatManage) []chat.Message {
|
||||
var chatMessages []chat.Message
|
||||
chatMessages = append(chatMessages, chat.Message{Role: "system", Content: chatManage.SummaryConfig.Prompt})
|
||||
chatMessages = append(chatMessages, chat.Message{Role: "user", Content: chatManage.UserContent})
|
||||
return chatMessages
|
||||
}
|
||||
|
||||
// prepareMessagesWithHistory prepare complete messages including history
|
||||
func prepareMessagesWithHistory(chatManage *types.ChatManage) []chat.Message {
|
||||
chatMessages := []chat.Message{
|
||||
|
||||
@@ -3,7 +3,6 @@ package chatpipline
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
@@ -27,12 +26,20 @@ func (p *PluginFilterTopK) ActivationEvents() []types.EventType {
|
||||
func (p *PluginFilterTopK) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
logger.Info(ctx, "Starting filter top-K process")
|
||||
logger.Infof(ctx, "Filter configuration: top-K = %d", chatManage.RerankTopK)
|
||||
pipelineInfo(ctx, "FilterTopK", "input", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"top_k": chatManage.RerankTopK,
|
||||
"merge_cnt": len(chatManage.MergeResult),
|
||||
"rerank_cnt": len(chatManage.RerankResult),
|
||||
"search_cnt": len(chatManage.SearchResult),
|
||||
})
|
||||
|
||||
filterTopK := func(searchResult []*types.SearchResult, topK int) []*types.SearchResult {
|
||||
if topK > 0 && len(searchResult) > topK {
|
||||
logger.Infof(ctx, "Filtering results: before=%d, after=%d", len(searchResult), topK)
|
||||
pipelineInfo(ctx, "FilterTopK", "filter", map[string]interface{}{
|
||||
"before": len(searchResult),
|
||||
"after": topK,
|
||||
})
|
||||
searchResult = searchResult[:topK]
|
||||
}
|
||||
return searchResult
|
||||
@@ -45,9 +52,15 @@ func (p *PluginFilterTopK) OnEvent(ctx context.Context,
|
||||
} else if len(chatManage.SearchResult) > 0 {
|
||||
chatManage.SearchResult = filterTopK(chatManage.SearchResult, chatManage.RerankTopK)
|
||||
} else {
|
||||
logger.Info(ctx, "No results to filter")
|
||||
pipelineWarn(ctx, "FilterTopK", "skip", map[string]interface{}{
|
||||
"reason": "no_results",
|
||||
})
|
||||
}
|
||||
|
||||
logger.Info(ctx, "Filter top-K process completed")
|
||||
pipelineInfo(ctx, "FilterTopK", "output", map[string]interface{}{
|
||||
"merge_cnt": len(chatManage.MergeResult),
|
||||
"rerank_cnt": len(chatManage.RerankResult),
|
||||
"search_cnt": len(chatManage.SearchResult),
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
@@ -34,6 +33,12 @@ func (p *PluginIntoChatMessage) ActivationEvents() []types.EventType {
|
||||
func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
pipelineInfo(ctx, "IntoChatMessage", "input", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"merge_result_cnt": len(chatManage.MergeResult),
|
||||
"template_len": len(chatManage.SummaryConfig.ContextTemplate),
|
||||
})
|
||||
|
||||
// Extract content from merge results
|
||||
passages := make([]string, len(chatManage.MergeResult))
|
||||
for i, result := range chatManage.MergeResult {
|
||||
@@ -44,6 +49,10 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
// Parse the context template
|
||||
tmpl, err := template.New("searchContent").Parse(chatManage.SummaryConfig.ContextTemplate)
|
||||
if err != nil {
|
||||
pipelineError(ctx, "IntoChatMessage", "parse_template", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return ErrTemplateParse.WithError(err)
|
||||
}
|
||||
|
||||
@@ -54,7 +63,9 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
// 验证用户查询的安全性
|
||||
safeQuery, isValid := secutils.ValidateInput(chatManage.Query)
|
||||
if !isValid {
|
||||
logger.Errorf(ctx, "Invalid user query: %s", chatManage.Query)
|
||||
pipelineWarn(ctx, "IntoChatMessage", "invalid_query", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
return ErrTemplateExecute.WithError(fmt.Errorf("用户查询包含非法内容"))
|
||||
}
|
||||
|
||||
@@ -66,11 +77,19 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
"CurrentWeek": weekdayName[time.Now().Weekday()], // Current weekday in Chinese
|
||||
})
|
||||
if err != nil {
|
||||
pipelineError(ctx, "IntoChatMessage", "render_template", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return ErrTemplateExecute.WithError(err)
|
||||
}
|
||||
|
||||
// Set formatted content back to chat management
|
||||
chatManage.UserContent = userContent.String()
|
||||
pipelineInfo(ctx, "IntoChatMessage", "output", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"user_content_len": len(chatManage.UserContent),
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
@@ -99,7 +118,9 @@ func enrichContentWithImageInfo(ctx context.Context, content string, imageInfoJS
|
||||
var imageInfos []types.ImageInfo
|
||||
err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to parse ImageInfo: %v, using content only", err)
|
||||
pipelineWarn(ctx, "IntoChatMessage", "image_parse_error", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return content
|
||||
}
|
||||
|
||||
@@ -125,7 +146,9 @@ func enrichContentWithImageInfo(ctx context.Context, content string, imageInfoJS
|
||||
// 用于存储已处理的图片URL
|
||||
processedURLs := make(map[string]bool)
|
||||
|
||||
logger.Infof(ctx, "Found %d Markdown image links in content", len(matches))
|
||||
pipelineInfo(ctx, "IntoChatMessage", "image_markdown_links", map[string]interface{}{
|
||||
"match_count": len(matches),
|
||||
})
|
||||
|
||||
// 替换每个图片链接,添加描述和OCR文本
|
||||
for _, match := range matches {
|
||||
@@ -184,8 +207,10 @@ func enrichContentWithImageInfo(ctx context.Context, content string, imageInfoJS
|
||||
content += "附加图片信息:\n" + strings.Join(additionalImageTexts, "\n")
|
||||
}
|
||||
|
||||
logger.Debugf(ctx, "Enhanced content with image info: found %d Markdown images, added %d additional images",
|
||||
len(matches), len(additionalImageTexts))
|
||||
pipelineInfo(ctx, "IntoChatMessage", "image_enrich_summary", map[string]interface{}{
|
||||
"markdown_images": len(matches),
|
||||
"additional_imgs": len(additionalImageTexts),
|
||||
})
|
||||
|
||||
return content
|
||||
}
|
||||
|
||||
@@ -5,16 +5,20 @@ import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// PluginMerge handles merging of search result chunks
|
||||
type PluginMerge struct{}
|
||||
type PluginMerge struct {
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
}
|
||||
|
||||
// NewPluginMerge creates and registers a new PluginMerge instance
|
||||
func NewPluginMerge(eventManager *EventManager) *PluginMerge {
|
||||
res := &PluginMerge{}
|
||||
func NewPluginMerge(eventManager *EventManager, chunkRepo interfaces.ChunkRepository) *PluginMerge {
|
||||
res := &PluginMerge{
|
||||
chunkRepo: chunkRepo,
|
||||
}
|
||||
eventManager.Register(res)
|
||||
return res
|
||||
}
|
||||
@@ -28,19 +32,29 @@ func (p *PluginMerge) ActivationEvents() []types.EventType {
|
||||
func (p *PluginMerge) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
logger.Info(ctx, "Starting chunk merge process")
|
||||
pipelineInfo(ctx, "Merge", "input", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"candidate_cnt": len(chatManage.RerankResult),
|
||||
})
|
||||
|
||||
// Use rerank results if available, fallback to search results
|
||||
searchResult := chatManage.RerankResult
|
||||
if len(searchResult) == 0 {
|
||||
logger.Info(ctx, "No rerank results available, using search results")
|
||||
pipelineWarn(ctx, "Merge", "fallback", map[string]interface{}{
|
||||
"reason": "empty_rerank_result",
|
||||
})
|
||||
searchResult = chatManage.SearchResult
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Processing %d chunks for merging", len(searchResult))
|
||||
pipelineInfo(ctx, "Merge", "candidate_ready", map[string]interface{}{
|
||||
"chunk_cnt": len(searchResult),
|
||||
})
|
||||
|
||||
if len(searchResult) == 0 {
|
||||
logger.Info(ctx, "No chunks available for merging")
|
||||
pipelineWarn(ctx, "Merge", "output", map[string]interface{}{
|
||||
"chunk_cnt": 0,
|
||||
"reason": "no_candidates",
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
@@ -50,12 +64,17 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
|
||||
knowledgeGroup[chunk.KnowledgeID] = append(knowledgeGroup[chunk.KnowledgeID], chunk)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Grouped chunks by knowledge ID, %d knowledge sources", len(knowledgeGroup))
|
||||
pipelineInfo(ctx, "Merge", "group_summary", map[string]interface{}{
|
||||
"knowledge_cnt": len(knowledgeGroup),
|
||||
})
|
||||
|
||||
mergedChunks := []*types.SearchResult{}
|
||||
// Process each knowledge source separately
|
||||
for knowledgeID, chunks := range knowledgeGroup {
|
||||
logger.Infof(ctx, "Processing knowledge ID: %s with %d chunks", knowledgeID, len(chunks))
|
||||
pipelineInfo(ctx, "Merge", "group_process", map[string]interface{}{
|
||||
"knowledge_id": knowledgeID,
|
||||
"chunk_cnt": len(chunks),
|
||||
})
|
||||
|
||||
// Sort chunks by their start position in the original document
|
||||
sort.Slice(chunks, func(i, j int) bool {
|
||||
@@ -92,7 +111,10 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
|
||||
|
||||
// 合并 ImageInfo
|
||||
if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil {
|
||||
logger.Warnf(ctx, "Failed to merge ImageInfo: %v", err)
|
||||
pipelineWarn(ctx, "Merge", "image_merge", map[string]interface{}{
|
||||
"knowledge_id": knowledgeID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
if chunks[i].Score > lastChunk.Score {
|
||||
@@ -100,8 +122,10 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Merged %d chunks into %d chunks for knowledge ID: %s",
|
||||
len(chunks), len(knowledgeMergedChunks), knowledgeID)
|
||||
pipelineInfo(ctx, "Merge", "group_output", map[string]interface{}{
|
||||
"knowledge_id": knowledgeID,
|
||||
"merged_chunks": len(knowledgeMergedChunks),
|
||||
})
|
||||
|
||||
mergedChunks = append(mergedChunks, knowledgeMergedChunks...)
|
||||
}
|
||||
@@ -111,7 +135,11 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
|
||||
return mergedChunks[i].Score > mergedChunks[j].Score
|
||||
})
|
||||
|
||||
logger.Infof(ctx, "Final merged result: %d chunks, sorted by score", len(mergedChunks))
|
||||
pipelineInfo(ctx, "Merge", "output", map[string]interface{}{
|
||||
"merged_total": len(mergedChunks),
|
||||
})
|
||||
|
||||
mergedChunks = p.expandShortContextWithNeighbors(ctx, chatManage, mergedChunks)
|
||||
|
||||
chatManage.MergeResult = mergedChunks
|
||||
return next()
|
||||
@@ -126,7 +154,9 @@ func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *typ
|
||||
|
||||
var sourceImageInfos []types.ImageInfo
|
||||
if err := json.Unmarshal([]byte(source.ImageInfo), &sourceImageInfos); err != nil {
|
||||
logger.Warnf(ctx, "Failed to unmarshal source ImageInfo: %v", err)
|
||||
pipelineWarn(ctx, "Merge", "image_unmarshal_source", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -139,7 +169,9 @@ func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *typ
|
||||
var targetImageInfos []types.ImageInfo
|
||||
if target.ImageInfo != "" {
|
||||
if err := json.Unmarshal([]byte(target.ImageInfo), &targetImageInfos); err != nil {
|
||||
logger.Warnf(ctx, "Failed to unmarshal target ImageInfo: %v", err)
|
||||
pipelineWarn(ctx, "Merge", "image_unmarshal_target", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
// 如果目标解析失败,直接使用源数据
|
||||
target.ImageInfo = source.ImageInfo
|
||||
return nil
|
||||
@@ -164,12 +196,345 @@ func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *typ
|
||||
// 序列化合并后的ImageInfo
|
||||
mergedImageInfoJSON, err := json.Marshal(uniqueImageInfos)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to marshal merged ImageInfo: %v", err)
|
||||
pipelineWarn(ctx, "Merge", "image_marshal", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// 更新目标chunk的ImageInfo
|
||||
target.ImageInfo = string(mergedImageInfoJSON)
|
||||
logger.Infof(ctx, "Successfully merged ImageInfo, total count: %d", len(uniqueImageInfos))
|
||||
pipelineInfo(ctx, "Merge", "image_merged", map[string]interface{}{
|
||||
"image_refs": len(uniqueImageInfos),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *PluginMerge) expandShortContextWithNeighbors(ctx context.Context, chatManage *types.ChatManage, results []*types.SearchResult) []*types.SearchResult {
|
||||
const (
|
||||
minLen = 350
|
||||
maxLen = 850
|
||||
)
|
||||
|
||||
if len(results) == 0 || p.chunkRepo == nil {
|
||||
return results
|
||||
}
|
||||
|
||||
tenantID, _ := ctx.Value(types.TenantIDContextKey).(uint)
|
||||
if tenantID == 0 && chatManage != nil {
|
||||
tenantID = chatManage.TenantID
|
||||
}
|
||||
if tenantID == 0 {
|
||||
pipelineWarn(ctx, "Merge", "expand_skip", map[string]interface{}{
|
||||
"reason": "missing_tenant",
|
||||
})
|
||||
return results
|
||||
}
|
||||
|
||||
type targetInfo struct {
|
||||
result *types.SearchResult
|
||||
}
|
||||
|
||||
targets := make([]targetInfo, 0)
|
||||
baseIDsSet := make(map[string]struct{})
|
||||
|
||||
for _, r := range results {
|
||||
if r == nil || r.ID == "" || r.Content == "" {
|
||||
continue
|
||||
}
|
||||
if r.ChunkType != string(types.ChunkTypeText) {
|
||||
continue
|
||||
}
|
||||
if runeLen(r.Content) >= minLen {
|
||||
continue
|
||||
}
|
||||
targets = append(targets, targetInfo{result: r})
|
||||
baseIDsSet[r.ID] = struct{}{}
|
||||
pipelineInfo(ctx, "Merge", "need_expand", map[string]interface{}{
|
||||
"chunk_id": r.ID,
|
||||
"content": r.Content,
|
||||
"chunk_type": r.ChunkType,
|
||||
"len": runeLen(r.Content),
|
||||
})
|
||||
}
|
||||
|
||||
if len(targets) == 0 {
|
||||
return results
|
||||
}
|
||||
|
||||
baseIDs := make([]string, 0, len(baseIDsSet))
|
||||
for id := range baseIDsSet {
|
||||
baseIDs = append(baseIDs, id)
|
||||
}
|
||||
|
||||
chunkMap := make(map[string]*types.Chunk, len(baseIDs))
|
||||
chunks, err := p.chunkRepo.ListChunksByID(ctx, tenantID, baseIDs)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Merge", "expand_list_base_failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return results
|
||||
}
|
||||
for _, chunk := range chunks {
|
||||
chunkMap[chunk.ID] = chunk
|
||||
}
|
||||
|
||||
neighborIDsSet := make(map[string]struct{})
|
||||
for _, chunk := range chunkMap {
|
||||
if chunk == nil {
|
||||
continue
|
||||
}
|
||||
if chunk.PreChunkID != "" {
|
||||
if _, exists := chunkMap[chunk.PreChunkID]; !exists {
|
||||
neighborIDsSet[chunk.PreChunkID] = struct{}{}
|
||||
}
|
||||
}
|
||||
if chunk.NextChunkID != "" {
|
||||
if _, exists := chunkMap[chunk.NextChunkID]; !exists {
|
||||
neighborIDsSet[chunk.NextChunkID] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(neighborIDsSet) > 0 {
|
||||
neighborIDs := make([]string, 0, len(neighborIDsSet))
|
||||
for id := range neighborIDsSet {
|
||||
neighborIDs = append(neighborIDs, id)
|
||||
}
|
||||
neighbors, err := p.chunkRepo.ListChunksByID(ctx, tenantID, neighborIDs)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Merge", "expand_list_neighbor_failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
} else {
|
||||
for _, chunk := range neighbors {
|
||||
chunkMap[chunk.ID] = chunk
|
||||
pipelineInfo(ctx, "Merge", "expand_list_neighbor_success", map[string]interface{}{
|
||||
"neighbor_chunk_id": chunk.ID,
|
||||
"neighbor_content": chunk.Content,
|
||||
"neighbor_chunk_type": chunk.ChunkType,
|
||||
"neighbor_len": runeLen(chunk.Content),
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
for _, target := range targets {
|
||||
res := target.result
|
||||
p.fetchChunksIfMissing(ctx, tenantID, chunkMap, res.ID)
|
||||
baseChunk := chunkMap[res.ID]
|
||||
if baseChunk == nil || baseChunk.Content == "" || baseChunk.ChunkType != types.ChunkTypeText {
|
||||
continue
|
||||
}
|
||||
|
||||
prevContent := ""
|
||||
nextContent := ""
|
||||
prevIDs := []string{}
|
||||
nextIDs := []string{}
|
||||
|
||||
prevCursor := baseChunk.PreChunkID
|
||||
nextCursor := baseChunk.NextChunkID
|
||||
|
||||
p.fetchChunksIfMissing(ctx, tenantID, chunkMap, prevCursor, nextCursor)
|
||||
|
||||
if prevCursor != "" {
|
||||
if prevChunk := chunkMap[prevCursor]; prevChunk != nil && prevChunk.KnowledgeID == baseChunk.KnowledgeID {
|
||||
prevContent = prevChunk.Content
|
||||
prevIDs = append(prevIDs, prevChunk.ID)
|
||||
prevCursor = prevChunk.PreChunkID
|
||||
} else {
|
||||
prevCursor = ""
|
||||
}
|
||||
}
|
||||
|
||||
if nextCursor != "" {
|
||||
if nextChunk := chunkMap[nextCursor]; nextChunk != nil && nextChunk.KnowledgeID == baseChunk.KnowledgeID {
|
||||
nextContent = nextChunk.Content
|
||||
nextIDs = append(nextIDs, nextChunk.ID)
|
||||
nextCursor = nextChunk.NextChunkID
|
||||
} else {
|
||||
nextCursor = ""
|
||||
}
|
||||
}
|
||||
|
||||
var merged string
|
||||
for {
|
||||
merged = mergeOrderedContent(prevContent, baseChunk.Content, nextContent, maxLen)
|
||||
if merged == "" {
|
||||
break
|
||||
}
|
||||
if runeLen(merged) >= minLen {
|
||||
break
|
||||
}
|
||||
if prevCursor == "" && nextCursor == "" {
|
||||
break
|
||||
}
|
||||
|
||||
expanded := false
|
||||
if prevCursor != "" {
|
||||
p.fetchChunksIfMissing(ctx, tenantID, chunkMap, prevCursor)
|
||||
if prevChunk := chunkMap[prevCursor]; prevChunk != nil && prevChunk.KnowledgeID == baseChunk.KnowledgeID {
|
||||
prevContent = concatNoOverlap(prevChunk.Content, prevContent)
|
||||
prevIDs = append([]string{prevChunk.ID}, prevIDs...)
|
||||
prevCursor = prevChunk.PreChunkID
|
||||
expanded = true
|
||||
} else {
|
||||
prevCursor = ""
|
||||
}
|
||||
}
|
||||
|
||||
merged = mergeOrderedContent(prevContent, baseChunk.Content, nextContent, maxLen)
|
||||
if runeLen(merged) >= minLen {
|
||||
break
|
||||
}
|
||||
|
||||
if nextCursor != "" {
|
||||
p.fetchChunksIfMissing(ctx, tenantID, chunkMap, nextCursor)
|
||||
if nextChunk := chunkMap[nextCursor]; nextChunk != nil && nextChunk.KnowledgeID == baseChunk.KnowledgeID {
|
||||
nextContent = concatNoOverlap(nextContent, nextChunk.Content)
|
||||
nextIDs = append(nextIDs, nextChunk.ID)
|
||||
nextCursor = nextChunk.NextChunkID
|
||||
expanded = true
|
||||
} else {
|
||||
nextCursor = ""
|
||||
}
|
||||
}
|
||||
|
||||
if !expanded {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if merged == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
beforeLen := runeLen(res.Content)
|
||||
res.Content = merged
|
||||
|
||||
for _, id := range prevIDs {
|
||||
if id != "" && !containsID(res.SubChunkID, id) {
|
||||
res.SubChunkID = append(res.SubChunkID, id)
|
||||
}
|
||||
}
|
||||
for _, id := range nextIDs {
|
||||
if id != "" && !containsID(res.SubChunkID, id) {
|
||||
res.SubChunkID = append(res.SubChunkID, id)
|
||||
}
|
||||
}
|
||||
|
||||
if prevContent != "" {
|
||||
res.StartAt = baseChunk.StartAt - runeLen(prevContent)
|
||||
if res.StartAt < 0 {
|
||||
res.StartAt = 0
|
||||
}
|
||||
}
|
||||
res.EndAt = res.StartAt + runeLen(res.Content)
|
||||
|
||||
pipelineInfo(ctx, "Merge", "expand_short_chunk", map[string]interface{}{
|
||||
"chunk_id": res.ID,
|
||||
"prev_ids": prevIDs,
|
||||
"next_ids": nextIDs,
|
||||
"before_len": beforeLen,
|
||||
"after_len": runeLen(res.Content),
|
||||
"base_content": baseChunk.Content,
|
||||
"after_content": res.Content,
|
||||
"chunk_type": res.ChunkType,
|
||||
"remaining_prev": prevCursor,
|
||||
"remaining_next": nextCursor,
|
||||
})
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
func runeLen(s string) int {
|
||||
return len([]rune(s))
|
||||
}
|
||||
|
||||
func mergeOrderedContent(prev, base, next string, maxLen int) string {
|
||||
content := base
|
||||
if prev != "" {
|
||||
content = concatNoOverlap(prev, content)
|
||||
}
|
||||
if next != "" {
|
||||
content = concatNoOverlap(content, next)
|
||||
}
|
||||
runes := []rune(content)
|
||||
if len(runes) > maxLen {
|
||||
return string(runes[:maxLen])
|
||||
}
|
||||
return content
|
||||
}
|
||||
|
||||
// concatNoOverlap concatenates a and b, removing potential overlapping prefix/suffix
|
||||
func concatNoOverlap(a, b string) string {
|
||||
if a == "" {
|
||||
return b
|
||||
}
|
||||
if b == "" {
|
||||
return a
|
||||
}
|
||||
|
||||
ar := []rune(a)
|
||||
br := []rune(b)
|
||||
maxOverlap := minInt(len(ar), len(br))
|
||||
for k := maxOverlap; k > 0; k-- {
|
||||
if string(ar[len(ar)-k:]) == string(br[:k]) {
|
||||
return string(ar) + string(br[k:])
|
||||
}
|
||||
}
|
||||
return string(ar) + string(br)
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func containsID(ids []string, target string) bool {
|
||||
for _, id := range ids {
|
||||
if id == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *PluginMerge) fetchChunksIfMissing(ctx context.Context, tenantID uint, chunkMap map[string]*types.Chunk, chunkIDs ...string) {
|
||||
missing := make([]string, 0, len(chunkIDs))
|
||||
for _, id := range chunkIDs {
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := chunkMap[id]; !exists {
|
||||
missing = append(missing, id)
|
||||
}
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
chunks, err := p.chunkRepo.ListChunksByID(ctx, tenantID, missing)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Merge", "expand_fetch_missing_failed", map[string]interface{}{
|
||||
"missing_cnt": len(missing),
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
found := make(map[string]struct{}, len(chunks))
|
||||
for _, chunk := range chunks {
|
||||
chunkMap[chunk.ID] = chunk
|
||||
found[chunk.ID] = struct{}{}
|
||||
}
|
||||
|
||||
for _, id := range missing {
|
||||
if _, ok := found[id]; !ok {
|
||||
chunkMap[id] = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,23 +1,26 @@
|
||||
package chatpipline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/yanyiwu/gojieba"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/yanyiwu/gojieba"
|
||||
)
|
||||
|
||||
// PluginPreprocess Query preprocessing plugin
|
||||
type PluginPreprocess struct {
|
||||
config *config.Config
|
||||
jieba *gojieba.Jieba
|
||||
stopwords map[string]struct{}
|
||||
config *config.Config
|
||||
jieba *gojieba.Jieba
|
||||
stopwords map[string]struct{}
|
||||
modelService interfaces.ModelService
|
||||
}
|
||||
|
||||
// Regular expressions for text cleaning
|
||||
@@ -28,11 +31,14 @@ var (
|
||||
punctRegex = regexp.MustCompile(`[^\p{L}\p{N}\s]`) // Punctuation marks
|
||||
)
|
||||
|
||||
const maxProcessedTokens = 12
|
||||
|
||||
// NewPluginPreprocess Creates a new query preprocessing plugin
|
||||
func NewPluginPreprocess(
|
||||
eventManager *EventManager,
|
||||
config *config.Config,
|
||||
cleaner interfaces.ResourceCleaner,
|
||||
eventManager *EventManager,
|
||||
config *config.Config,
|
||||
cleaner interfaces.ResourceCleaner,
|
||||
modelService interfaces.ModelService,
|
||||
) *PluginPreprocess {
|
||||
// Use default dictionary for Jieba tokenizer
|
||||
jieba := gojieba.NewJieba()
|
||||
@@ -40,11 +46,12 @@ func NewPluginPreprocess(
|
||||
// Load stopwords from built-in stopword library
|
||||
stopwords := loadStopwords()
|
||||
|
||||
res := &PluginPreprocess{
|
||||
config: config,
|
||||
jieba: jieba,
|
||||
stopwords: stopwords,
|
||||
}
|
||||
res := &PluginPreprocess{
|
||||
config: config,
|
||||
jieba: jieba,
|
||||
stopwords: stopwords,
|
||||
modelService: modelService,
|
||||
}
|
||||
|
||||
// Register resource cleanup function
|
||||
if cleaner != nil {
|
||||
@@ -85,25 +92,66 @@ func (p *PluginPreprocess) ActivationEvents() []types.EventType {
|
||||
|
||||
// OnEvent Process events
|
||||
func (p *PluginPreprocess) OnEvent(ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError) *PluginError {
|
||||
if chatManage.RewriteQuery == "" {
|
||||
return next()
|
||||
rawQuery := strings.TrimSpace(chatManage.RewriteQuery)
|
||||
if rawQuery == "" {
|
||||
return next()
|
||||
}
|
||||
|
||||
pipelineInfo(ctx, "Preprocess", "input", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"rewrite_query": rawQuery,
|
||||
})
|
||||
|
||||
normalized := normalizeWhitespace(rawQuery)
|
||||
sanitized := strings.TrimSpace(p.cleanText(normalized))
|
||||
if sanitized == "" {
|
||||
sanitized = normalized
|
||||
}
|
||||
|
||||
logger.GetLogger(ctx).Infof("Starting query preprocessing, original query: %s", chatManage.RewriteQuery)
|
||||
var (
|
||||
processed = sanitized
|
||||
strategy = "original"
|
||||
tokenPreview string
|
||||
tokenCount int
|
||||
)
|
||||
|
||||
// 1. Basic text cleaning
|
||||
processed := p.cleanText(chatManage.RewriteQuery)
|
||||
switch {
|
||||
case containsChineseCharacters(sanitized):
|
||||
segments := p.segmentText(sanitized)
|
||||
tokens := p.selectMeaningfulTokens(segments)
|
||||
tokenCount = len(tokens)
|
||||
if len(tokens) >= 2 {
|
||||
processed = strings.Join(tokens, " ")
|
||||
strategy = "zh_tokens"
|
||||
tokenPreview = strings.Join(tokens, ",")
|
||||
} else {
|
||||
strategy = "fallback_original"
|
||||
}
|
||||
case containsLatinLetters(sanitized):
|
||||
processed = normalizeLatinQuery(sanitized)
|
||||
if processed != sanitized {
|
||||
strategy = "latin_normalize"
|
||||
}
|
||||
default:
|
||||
strategy = "original"
|
||||
}
|
||||
|
||||
// 2. Tokenization
|
||||
segments := p.segmentText(processed)
|
||||
if strings.TrimSpace(processed) == "" {
|
||||
processed = rawQuery
|
||||
strategy = "fallback_original"
|
||||
}
|
||||
|
||||
// 3. Stopword filtering and reconstruction
|
||||
filteredSegments := p.filterStopwords(segments)
|
||||
chatManage.ProcessedQuery = processed
|
||||
chatManage.QueryIntent = p.detectIntentLLM(ctx, chatManage, sanitized)
|
||||
|
||||
// Update preprocessed query
|
||||
chatManage.ProcessedQuery = strings.Join(filteredSegments, " ")
|
||||
|
||||
logger.GetLogger(ctx).Infof("Query preprocessing complete, processed query: %s", chatManage.ProcessedQuery)
|
||||
pipelineInfo(ctx, "Preprocess", "output", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"processed_query": processed,
|
||||
"strategy": strategy,
|
||||
"token_count": tokenCount,
|
||||
"token_preview": truncateForLog(tokenPreview),
|
||||
"query_intent": chatManage.QueryIntent,
|
||||
})
|
||||
|
||||
return next()
|
||||
}
|
||||
@@ -136,32 +184,137 @@ func (p *PluginPreprocess) segmentText(text string) []string {
|
||||
}
|
||||
|
||||
// filterStopwords Filter stopwords
|
||||
func (p *PluginPreprocess) filterStopwords(segments []string) []string {
|
||||
var filtered []string
|
||||
func (p *PluginPreprocess) selectMeaningfulTokens(segments []string) []string {
|
||||
var tokens []string
|
||||
seen := make(map[string]struct{})
|
||||
|
||||
for _, word := range segments {
|
||||
// If not a stopword and not blank, keep it
|
||||
if _, isStopword := p.stopwords[word]; !isStopword && !isBlank(word) {
|
||||
filtered = append(filtered, word)
|
||||
word = strings.TrimSpace(word)
|
||||
if word == "" {
|
||||
continue
|
||||
}
|
||||
if _, stop := p.stopwords[word]; stop {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[word]; exists {
|
||||
continue
|
||||
}
|
||||
if !isInformativeToken(word) {
|
||||
continue
|
||||
}
|
||||
|
||||
seen[word] = struct{}{}
|
||||
tokens = append(tokens, word)
|
||||
if len(tokens) >= maxProcessedTokens {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// If filtering results in empty list, return original tokenization results
|
||||
if len(filtered) == 0 {
|
||||
return segments
|
||||
}
|
||||
|
||||
return filtered
|
||||
return tokens
|
||||
}
|
||||
|
||||
// isBlank Check if a string is blank
|
||||
func isBlank(str string) bool {
|
||||
for _, r := range str {
|
||||
if !unicode.IsSpace(r) {
|
||||
return false
|
||||
func isInformativeToken(token string) bool {
|
||||
if token == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
runeCount := utf8.RuneCountInString(token)
|
||||
if runeCount == 1 {
|
||||
r, _ := utf8.DecodeRuneInString(token)
|
||||
if unicode.IsDigit(r) {
|
||||
return true
|
||||
}
|
||||
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func containsChineseCharacters(text string) bool {
|
||||
for _, r := range text {
|
||||
if unicode.Is(unicode.Han, r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return true
|
||||
return false
|
||||
}
|
||||
|
||||
func containsLatinLetters(text string) bool {
|
||||
for _, r := range text {
|
||||
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeWhitespace(text string) string {
|
||||
text = strings.TrimSpace(text)
|
||||
return multiSpaceRegex.ReplaceAllString(text, " ")
|
||||
}
|
||||
|
||||
func normalizeLatinQuery(text string) string {
|
||||
text = strings.ToLower(text)
|
||||
text = multiSpaceRegex.ReplaceAllString(text, " ")
|
||||
return strings.TrimSpace(text)
|
||||
}
|
||||
|
||||
type intentResp struct {
|
||||
Intent string `json:"intent"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
|
||||
func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *types.ChatManage, text string) string {
|
||||
if p.modelService == nil || chatManage.ChatModelID == "" {
|
||||
pipelineWarn(ctx, "IntentDetect", "skip", map[string]interface{}{ "reason": "no_model", "session_id": chatManage.SessionID })
|
||||
return "general"
|
||||
}
|
||||
chatModel, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "IntentDetect", "get_model_failed", map[string]interface{}{ "error": err.Error(), "model_id": chatManage.ChatModelID })
|
||||
return "general"
|
||||
}
|
||||
pipelineInfo(ctx, "IntentDetect", "start", map[string]interface{}{ "session_id": chatManage.SessionID, "model_id": chatManage.ChatModelID })
|
||||
sys := "You are a query intent classifier. Classify the user's query into one of: definition, howto, compare, qa, general. Respond ONLY with a JSON object {\"intent\": \"...\", \"confidence\": 0.0 } inside a markdown fenced block."
|
||||
usr := text
|
||||
think := false
|
||||
resp, err := chatModel.Chat(ctx, []chat.Message{
|
||||
{Role: "system", Content: sys},
|
||||
{Role: "user", Content: usr},
|
||||
}, &chat.ChatOptions{Temperature: 0.0, MaxCompletionTokens: 64, Thinking: &think})
|
||||
if err != nil || resp.Content == "" {
|
||||
pipelineWarn(ctx, "IntentDetect", "model_call_failed", map[string]interface{}{ "error": err })
|
||||
return "general"
|
||||
}
|
||||
body := extractJSONBody(resp.Content)
|
||||
var ir intentResp
|
||||
if err := json.Unmarshal([]byte(body), &ir); err != nil {
|
||||
pipelineWarn(ctx, "IntentDetect", "parse_failed", map[string]interface{}{ "body": truncateForLog(body), "error": err.Error() })
|
||||
return "general"
|
||||
}
|
||||
pipelineInfo(ctx, "IntentDetect", "result", map[string]interface{}{ "intent": ir.Intent, "confidence": ir.Confidence })
|
||||
switch strings.ToLower(strings.TrimSpace(ir.Intent)) {
|
||||
case "definition", "howto", "compare", "qa", "general":
|
||||
return strings.ToLower(ir.Intent)
|
||||
default:
|
||||
return "general"
|
||||
}
|
||||
}
|
||||
|
||||
func extractJSONBody(text string) string {
|
||||
t := strings.TrimSpace(text)
|
||||
// Try fenced block first
|
||||
if i := strings.Index(t, "{"); i >= 0 {
|
||||
j := strings.LastIndex(t, "}")
|
||||
if j > i {
|
||||
return t[i : j+1]
|
||||
}
|
||||
}
|
||||
return "{}"
|
||||
}
|
||||
|
||||
// Ensure resources are properly released
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
package chatpipline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/rerank"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/models/rerank"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// PluginRerank implements reranking functionality for chat pipeline
|
||||
@@ -34,28 +33,43 @@ func (p *PluginRerank) ActivationEvents() []types.EventType {
|
||||
|
||||
// OnEvent handles reranking events in the chat pipeline
|
||||
func (p *PluginRerank) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
logger.Info(ctx, "Starting reranking process")
|
||||
logger.Infof(ctx, "Getting rerank model, model ID: %s", chatManage.RerankModelID)
|
||||
pipelineInfo(ctx, "Rerank", "input", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"candidate_cnt": len(chatManage.SearchResult),
|
||||
"rerank_model": chatManage.RerankModelID,
|
||||
"rerank_thresh": chatManage.RerankThreshold,
|
||||
"rewrite_query": chatManage.RewriteQuery,
|
||||
"processed_query": chatManage.ProcessedQuery,
|
||||
})
|
||||
if len(chatManage.SearchResult) == 0 {
|
||||
logger.Infof(ctx, "No search result, skip reranking")
|
||||
pipelineInfo(ctx, "Rerank", "skip", map[string]interface{}{
|
||||
"reason": "empty_search_result",
|
||||
})
|
||||
return next()
|
||||
}
|
||||
if chatManage.RerankModelID == "" {
|
||||
logger.Warn(ctx, "Rerank model ID is empty, skipping reranking")
|
||||
pipelineWarn(ctx, "Rerank", "skip", map[string]interface{}{
|
||||
"reason": "empty_model_id",
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
// Get rerank model from service
|
||||
rerankModel, err := p.modelService.GetRerankModel(ctx, chatManage.RerankModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get rerank model: %v, rerank model ID: %s", err, chatManage.RerankModelID)
|
||||
pipelineError(ctx, "Rerank", "get_model", map[string]interface{}{
|
||||
"model_id": chatManage.RerankModelID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return ErrGetRerankModel.WithError(err)
|
||||
}
|
||||
|
||||
// Prepare passages for reranking
|
||||
logger.Infof(ctx, "Preparing passages for reranking, search result count: %d", len(chatManage.SearchResult))
|
||||
pipelineInfo(ctx, "Rerank", "build_passages", map[string]interface{}{
|
||||
"candidate_cnt": len(chatManage.SearchResult),
|
||||
})
|
||||
var passages []string
|
||||
for _, result := range chatManage.SearchResult {
|
||||
// 合并Content和ImageInfo的文本内容
|
||||
@@ -72,41 +86,79 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
// Update search results with reranked scores
|
||||
logger.Infof(ctx, "Filtered rerank results, original: %d, filtered: %d", len(rerankResp), len(rerankResp))
|
||||
result := []*types.SearchResult{}
|
||||
for _, rr := range rerankResp {
|
||||
chatManage.SearchResult[rr.Index].Score = rr.RelevanceScore
|
||||
result = append(result, chatManage.SearchResult[rr.Index])
|
||||
}
|
||||
chatManage.RerankResult = result
|
||||
pipelineInfo(ctx, "Rerank", "model_response", map[string]interface{}{
|
||||
"result_cnt": len(rerankResp),
|
||||
})
|
||||
for i := range chatManage.SearchResult {
|
||||
chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata)
|
||||
}
|
||||
reranked := make([]*types.SearchResult, 0, len(rerankResp))
|
||||
for _, rr := range rerankResp {
|
||||
sr := chatManage.SearchResult[rr.Index]
|
||||
base := sr.Score
|
||||
sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base)
|
||||
sr.Score = rr.RelevanceScore
|
||||
sr.Score = compositeScore(sr, rr.RelevanceScore, base, chatManage)
|
||||
reranked = append(reranked, sr)
|
||||
}
|
||||
final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7)
|
||||
chatManage.RerankResult = final
|
||||
|
||||
if len(chatManage.RerankResult) == 0 {
|
||||
logger.Warn(ctx, "Reranking produced no results above threshold")
|
||||
return ErrSearchNothing
|
||||
}
|
||||
// Log composite top scores and MMR selection summary
|
||||
topN := min(3, len(reranked))
|
||||
for i := 0; i < topN; i++ {
|
||||
pipelineInfo(ctx, "Rerank", "composite_top", map[string]interface{}{
|
||||
"rank": i + 1,
|
||||
"chunk_id": reranked[i].ID,
|
||||
"base_score": reranked[i].Metadata["base_score"],
|
||||
"final_score": fmt.Sprintf("%.4f", reranked[i].Score),
|
||||
"intent": chatManage.QueryIntent,
|
||||
})
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Reranking process completed successfully, result count: %d", len(chatManage.RerankResult))
|
||||
return next()
|
||||
if len(chatManage.RerankResult) == 0 {
|
||||
pipelineWarn(ctx, "Rerank", "output", map[string]interface{}{
|
||||
"filtered_cnt": 0,
|
||||
})
|
||||
return ErrSearchNothing
|
||||
}
|
||||
|
||||
pipelineInfo(ctx, "Rerank", "output", map[string]interface{}{
|
||||
"filtered_cnt": len(chatManage.RerankResult),
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
// rerank performs the actual reranking operation with given query and passages
|
||||
func (p *PluginRerank) rerank(ctx context.Context,
|
||||
chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string,
|
||||
chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string,
|
||||
) []rerank.RankResult {
|
||||
logger.Infof(ctx, "Executing reranking with query: %s, passage count: %d", query, len(passages))
|
||||
pipelineInfo(ctx, "Rerank", "model_call", map[string]interface{}{
|
||||
"query_variant": query,
|
||||
"passages": len(passages),
|
||||
})
|
||||
rerankResp, err := rerankModel.Rerank(ctx, query, passages)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Reranking failed: %v", err)
|
||||
pipelineError(ctx, "Rerank", "model_call", map[string]interface{}{
|
||||
"query_variant": query,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Log top scores for debugging
|
||||
logger.Infof(ctx, "Reranking completed, filtering results with threshold: %f", chatManage.RerankThreshold)
|
||||
for i := range min(3, len(rerankResp)) {
|
||||
logger.Infof(ctx, "Top %d score of rerankResp: %f, passages: %s, index: %d",
|
||||
i+1, rerankResp[i].RelevanceScore, rerankResp[i].Document.Text, rerankResp[i].Index,
|
||||
)
|
||||
pipelineInfo(ctx, "Rerank", "threshold", map[string]interface{}{
|
||||
"threshold": chatManage.RerankThreshold,
|
||||
})
|
||||
for i := range min(5, len(rerankResp)) {
|
||||
pipelineInfo(ctx, "Rerank", "top_score", map[string]interface{}{
|
||||
"rank": i + 1,
|
||||
"score": rerankResp[i].RelevanceScore,
|
||||
"chunk_id": chatManage.SearchResult[rerankResp[i].Index].ID,
|
||||
"match_type": chatManage.SearchResult[rerankResp[i].Index].MatchType,
|
||||
"chunk_type": chatManage.SearchResult[rerankResp[i].Index].ChunkType,
|
||||
"content": chatManage.SearchResult[rerankResp[i].Index].Content,
|
||||
})
|
||||
}
|
||||
|
||||
// Filter results based on threshold with special handling for history matches
|
||||
@@ -121,7 +173,149 @@ func (p *PluginRerank) rerank(ctx context.Context,
|
||||
rankFilter = append(rankFilter, result)
|
||||
}
|
||||
}
|
||||
return rankFilter
|
||||
return rankFilter
|
||||
}
|
||||
|
||||
func ensureMetadata(m map[string]string) map[string]string {
|
||||
if m == nil {
|
||||
return make(map[string]string)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatManage *types.ChatManage) float64 {
|
||||
sourceWeight := 1.0
|
||||
switch strings.ToLower(sr.KnowledgeSource) {
|
||||
case "web_search":
|
||||
sourceWeight = 0.95
|
||||
default:
|
||||
sourceWeight = 1.0
|
||||
}
|
||||
intentBoost := 1.0
|
||||
if chatManage.QueryIntent != "" {
|
||||
switch chatManage.QueryIntent {
|
||||
case "definition":
|
||||
if sr.ChunkType == string(types.ChunkTypeSummary) {
|
||||
intentBoost = 1.05
|
||||
}
|
||||
case "howto":
|
||||
if sr.EndAt-sr.StartAt > 300 {
|
||||
intentBoost = 1.03
|
||||
}
|
||||
case "compare":
|
||||
intentBoost = 1.0
|
||||
}
|
||||
}
|
||||
positionPrior := 1.0
|
||||
if sr.StartAt >= 0 {
|
||||
positionPrior += clampFloat(1.0-float64(sr.StartAt)/float64(sr.EndAt+1), -0.05, 0.05)
|
||||
}
|
||||
composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight
|
||||
composite *= intentBoost
|
||||
composite *= positionPrior
|
||||
if composite < 0 {
|
||||
composite = 0
|
||||
}
|
||||
if composite > 1 {
|
||||
composite = 1
|
||||
}
|
||||
return composite
|
||||
}
|
||||
|
||||
func applyMMR(ctx context.Context, results []*types.SearchResult, chatManage *types.ChatManage, k int, lambda float64) []*types.SearchResult {
|
||||
if k <= 0 || len(results) == 0 {
|
||||
return nil
|
||||
}
|
||||
pipelineInfo(ctx, "Rerank", "mmr_start", map[string]interface{}{
|
||||
"lambda": lambda,
|
||||
"k": k,
|
||||
"candidates": len(results),
|
||||
})
|
||||
selected := make([]*types.SearchResult, 0, k)
|
||||
candidates := make([]*types.SearchResult, len(results))
|
||||
copy(candidates, results)
|
||||
tokenSets := make([]map[string]struct{}, len(candidates))
|
||||
for i, r := range candidates {
|
||||
tokenSets[i] = tokenizeSimple(getEnrichedPassage(ctx, r))
|
||||
}
|
||||
for len(selected) < k && len(candidates) > 0 {
|
||||
bestIdx := 0
|
||||
bestScore := -1.0
|
||||
for i, r := range candidates {
|
||||
relevance := r.Score
|
||||
redundancy := 0.0
|
||||
for _, s := range selected {
|
||||
redundancy = math.Max(redundancy, jaccard(tokenSets[i], tokenizeSimple(getEnrichedPassage(ctx, s))))
|
||||
}
|
||||
mmr := lambda*relevance - (1.0-lambda)*redundancy
|
||||
if mmr > bestScore {
|
||||
bestScore = mmr
|
||||
bestIdx = i
|
||||
}
|
||||
}
|
||||
selected = append(selected, candidates[bestIdx])
|
||||
candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...)
|
||||
}
|
||||
// Compute average redundancy among selected
|
||||
avgRed := 0.0
|
||||
if len(selected) > 1 {
|
||||
pairs := 0
|
||||
for i := 0; i < len(selected); i++ {
|
||||
for j := i + 1; j < len(selected); j++ {
|
||||
si := tokenizeSimple(getEnrichedPassage(ctx, selected[i]))
|
||||
sj := tokenizeSimple(getEnrichedPassage(ctx, selected[j]))
|
||||
avgRed += jaccard(si, sj)
|
||||
pairs++
|
||||
}
|
||||
}
|
||||
if pairs > 0 {
|
||||
avgRed /= float64(pairs)
|
||||
}
|
||||
}
|
||||
pipelineInfo(ctx, "Rerank", "mmr_done", map[string]interface{}{
|
||||
"selected": len(selected),
|
||||
"avg_redundancy": fmt.Sprintf("%.4f", avgRed),
|
||||
})
|
||||
return selected
|
||||
}
|
||||
|
||||
func tokenizeSimple(text string) map[string]struct{} {
|
||||
text = strings.ToLower(text)
|
||||
fields := strings.Fields(text)
|
||||
set := make(map[string]struct{}, len(fields))
|
||||
for _, f := range fields {
|
||||
if len(f) > 1 {
|
||||
set[f] = struct{}{}
|
||||
}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func jaccard(a, b map[string]struct{}) float64 {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return 0
|
||||
}
|
||||
inter := 0
|
||||
for k := range a {
|
||||
if _, ok := b[k]; ok {
|
||||
inter++
|
||||
}
|
||||
}
|
||||
union := len(a) + len(b) - inter
|
||||
if union == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(inter) / float64(union)
|
||||
}
|
||||
|
||||
func clampFloat(v, minV, maxV float64) float64 {
|
||||
if v < minV {
|
||||
return minV
|
||||
}
|
||||
if v > maxV {
|
||||
return maxV
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// getEnrichedPassage 合并Content和ImageInfo的文本内容
|
||||
@@ -134,7 +328,9 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string
|
||||
var imageInfos []types.ImageInfo
|
||||
err := json.Unmarshal([]byte(result.ImageInfo), &imageInfos)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to parse ImageInfo: %v, using content only", err)
|
||||
pipelineWarn(ctx, "Rerank", "image_info_parse", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return result.Content
|
||||
}
|
||||
|
||||
@@ -164,8 +360,10 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string
|
||||
}
|
||||
combinedText += strings.Join(imageTexts, "\n")
|
||||
|
||||
logger.Debugf(ctx, "Enhanced passage with image info: content length %d, image texts length %d",
|
||||
len(result.Content), len(strings.Join(imageTexts, "\n")))
|
||||
pipelineInfo(ctx, "Rerank", "image_info_merge", map[string]interface{}{
|
||||
"content_len": len(result.Content),
|
||||
"image_len": len(strings.Join(imageTexts, "\n")),
|
||||
})
|
||||
|
||||
return combinedText
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
@@ -58,10 +57,28 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
// Initialize rewritten query as original query
|
||||
chatManage.RewriteQuery = chatManage.Query
|
||||
|
||||
if !chatManage.EnableRewrite {
|
||||
pipelineInfo(ctx, "Rewrite", "skip", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"reason": "rewrite_disabled",
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
pipelineInfo(ctx, "Rewrite", "input", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"tenant_id": chatManage.TenantID,
|
||||
"user_query": chatManage.Query,
|
||||
"enable_rewrite": chatManage.EnableRewrite,
|
||||
})
|
||||
|
||||
// Get conversation history
|
||||
history, err := p.messageService.GetRecentMessagesBySession(ctx, chatManage.SessionID, 20)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get conversation history, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
pipelineWarn(ctx, "Rewrite", "history_fetch", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Convert historical messages to conversation history structure
|
||||
@@ -99,22 +116,52 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
})
|
||||
|
||||
// Limit the number of historical records
|
||||
if len(historyList) > p.config.Conversation.MaxRounds {
|
||||
historyList = historyList[:p.config.Conversation.MaxRounds]
|
||||
maxRounds := p.config.Conversation.MaxRounds
|
||||
if chatManage.MaxRounds > 0 {
|
||||
maxRounds = chatManage.MaxRounds
|
||||
}
|
||||
if len(historyList) > maxRounds {
|
||||
historyList = historyList[:maxRounds]
|
||||
}
|
||||
|
||||
// Reverse to chronological order
|
||||
slices.Reverse(historyList)
|
||||
chatManage.History = historyList
|
||||
|
||||
userTmpl, err := template.New("rewriteContent").Parse(p.config.Conversation.RewritePromptUser)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
if len(historyList) == 0 {
|
||||
pipelineInfo(ctx, "Rewrite", "skip", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"reason": "empty_history",
|
||||
})
|
||||
return next()
|
||||
}
|
||||
systemTmpl, err := template.New("rewriteContent").Parse(p.config.Conversation.RewritePromptSystem)
|
||||
pipelineInfo(ctx, "Rewrite", "history_ready", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"history_rounds": len(historyList),
|
||||
"max_rounds": maxRounds,
|
||||
})
|
||||
|
||||
userPrompt := p.config.Conversation.RewritePromptUser
|
||||
if chatManage.RewritePromptUser != "" {
|
||||
userPrompt = chatManage.RewritePromptUser
|
||||
}
|
||||
userTmpl, err := template.New("rewriteContent").Parse(userPrompt)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
pipelineError(ctx, "Rewrite", "parse_user_template", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return next()
|
||||
}
|
||||
systemPrompt := p.config.Conversation.RewritePromptSystem
|
||||
if chatManage.RewritePromptSystem != "" {
|
||||
systemPrompt = chatManage.RewritePromptSystem
|
||||
}
|
||||
systemTmpl, err := template.New("rewriteContent").Parse(systemPrompt)
|
||||
if err != nil {
|
||||
pipelineError(ctx, "Rewrite", "parse_system_template", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return next()
|
||||
}
|
||||
currentTime := time.Now().Format("2006-01-02 15:04:05")
|
||||
@@ -126,7 +173,10 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
"Conversation": historyList,
|
||||
})
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).Errorf("Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
pipelineError(ctx, "Rewrite", "render_user_template", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return next()
|
||||
}
|
||||
err = systemTmpl.Execute(&systemContent, map[string]interface{}{
|
||||
@@ -136,12 +186,19 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
"Conversation": historyList,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
pipelineError(ctx, "Rewrite", "render_system_template", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return next()
|
||||
}
|
||||
rewriteModel, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to get model, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
pipelineError(ctx, "Rewrite", "get_model", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"chat_model_id": chatManage.ChatModelID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
@@ -162,7 +219,10 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
Thinking: &thinking,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to execute model, session_id: %s, error: %v", chatManage.SessionID, err)
|
||||
pipelineError(ctx, "Rewrite", "model_call", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
@@ -170,7 +230,9 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
// Update rewritten query
|
||||
chatManage.RewriteQuery = response.Content
|
||||
}
|
||||
logger.GetLogger(ctx).Infof("Rewritten query, session_id: %s, rewrite_query: %s",
|
||||
chatManage.SessionID, chatManage.RewriteQuery)
|
||||
pipelineInfo(ctx, "Rewrite", "output", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"rewrite_query": chatManage.RewriteQuery,
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
@@ -1,16 +1,17 @@
|
||||
package chatpipline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// PluginSearch implements search functionality for chat pipeline
|
||||
@@ -53,23 +54,42 @@ func (p *PluginSearch) ActivationEvents() []types.EventType {
|
||||
|
||||
// OnEvent handles search events in the chat pipeline
|
||||
func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
// Get knowledge base IDs list
|
||||
knowledgeBaseIDs := chatManage.KnowledgeBaseIDs
|
||||
if len(knowledgeBaseIDs) == 0 && chatManage.KnowledgeBaseID != "" {
|
||||
// Fall back to single knowledge base
|
||||
knowledgeBaseIDs = []string{chatManage.KnowledgeBaseID}
|
||||
logger.Infof(ctx, "No KnowledgeBaseIDs provided, falling back to single KB: %s", chatManage.KnowledgeBaseID)
|
||||
pipelineInfo(ctx, "Search", "fallback_kb", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"kb_id": chatManage.KnowledgeBaseID,
|
||||
})
|
||||
}
|
||||
|
||||
if len(knowledgeBaseIDs) == 0 {
|
||||
logger.Errorf(ctx, "No knowledge base IDs available for search")
|
||||
pipelineError(ctx, "Search", "kb_not_found", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
return ErrSearch.WithError(nil)
|
||||
}
|
||||
|
||||
pipelineInfo(ctx, "Search", "input", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"rewrite_query": chatManage.RewriteQuery,
|
||||
"processed_query": chatManage.ProcessedQuery,
|
||||
"kb_ids": strings.Join(knowledgeBaseIDs, ","),
|
||||
"tenant_id": chatManage.TenantID,
|
||||
"web_enabled": chatManage.WebSearchEnabled,
|
||||
})
|
||||
|
||||
// Run KB search and web search concurrently
|
||||
logger.Infof(ctx, "Searching across %d knowledge base(s): %v", len(knowledgeBaseIDs), knowledgeBaseIDs)
|
||||
pipelineInfo(ctx, "Search", "plan", map[string]interface{}{
|
||||
"kb_count": len(knowledgeBaseIDs),
|
||||
"embedding_top_k": chatManage.EmbeddingTopK,
|
||||
"vector_threshold": chatManage.VectorThreshold,
|
||||
"keyword_threshold": chatManage.KeywordThreshold,
|
||||
})
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
allResults := make([]*types.SearchResult, 0)
|
||||
@@ -97,30 +117,116 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
}
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
wg.Wait()
|
||||
|
||||
chatManage.SearchResult = allResults
|
||||
chatManage.SearchResult = allResults
|
||||
|
||||
// If recall is low, attempt query expansion with keyword-focused search
|
||||
if len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK/2) {
|
||||
pipelineInfo(ctx, "Search", "recall_low", map[string]interface{}{
|
||||
"current": len(chatManage.SearchResult),
|
||||
"threshold": chatManage.EmbeddingTopK / 2,
|
||||
})
|
||||
expansions := p.expandQueries(ctx, chatManage)
|
||||
if len(expansions) > 0 {
|
||||
pipelineInfo(ctx, "Search", "expansion_start", map[string]interface{}{
|
||||
"variants": len(expansions),
|
||||
})
|
||||
expTopK := max(chatManage.EmbeddingTopK*2, chatManage.RerankTopK*2)
|
||||
expKwTh := chatManage.KeywordThreshold * 0.8
|
||||
// Concurrent expansion retrieval across queries and KBs
|
||||
expResults := make([]*types.SearchResult, 0, expTopK*len(expansions))
|
||||
var muExp sync.Mutex
|
||||
var wgExp sync.WaitGroup
|
||||
jobs := len(expansions) * len(knowledgeBaseIDs)
|
||||
capSem := 16
|
||||
if jobs < capSem {
|
||||
capSem = jobs
|
||||
}
|
||||
if capSem <= 0 {
|
||||
capSem = 1
|
||||
}
|
||||
sem := make(chan struct{}, capSem)
|
||||
pipelineInfo(ctx, "Search", "expansion_concurrency", map[string]interface{}{
|
||||
"jobs": jobs,
|
||||
"cap": capSem,
|
||||
})
|
||||
for _, q := range expansions {
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
wgExp.Add(1)
|
||||
go func(q string, kbID string) {
|
||||
defer wgExp.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
paramsExp := types.SearchParams{
|
||||
QueryText: q,
|
||||
VectorThreshold: chatManage.VectorThreshold,
|
||||
KeywordThreshold: expKwTh,
|
||||
MatchCount: expTopK,
|
||||
DisableVectorMatch: true,
|
||||
DisableKeywordsMatch: false,
|
||||
}
|
||||
res, err := p.knowledgeBaseService.HybridSearch(ctx, kbID, paramsExp)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Search", "expansion_error", map[string]interface{}{
|
||||
"kb_id": kbID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
if len(res) > 0 {
|
||||
pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{
|
||||
"kb_id": kbID,
|
||||
"query": truncateForLog(q),
|
||||
"hits": len(res),
|
||||
})
|
||||
muExp.Lock()
|
||||
expResults = append(expResults, res...)
|
||||
muExp.Unlock()
|
||||
}
|
||||
}(q, kbID)
|
||||
}
|
||||
}
|
||||
wgExp.Wait()
|
||||
if len(expResults) > 0 {
|
||||
pipelineInfo(ctx, "Search", "expansion_done", map[string]interface{}{
|
||||
"added": len(expResults),
|
||||
})
|
||||
chatManage.SearchResult = append(chatManage.SearchResult, expResults...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add relevant results from chat history
|
||||
historyResult := p.getSearchResultFromHistory(chatManage)
|
||||
if historyResult != nil {
|
||||
logger.Infof(ctx, "Add history result, result count: %d", len(historyResult))
|
||||
pipelineInfo(ctx, "Search", "history_hits", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"history_hits": len(historyResult),
|
||||
})
|
||||
chatManage.SearchResult = append(chatManage.SearchResult, historyResult...)
|
||||
}
|
||||
|
||||
// Remove duplicate results
|
||||
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
|
||||
// Remove duplicate results
|
||||
before := len(chatManage.SearchResult)
|
||||
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
|
||||
pipelineInfo(ctx, "Search", "dedup_summary", map[string]interface{}{
|
||||
"before": before,
|
||||
"after": len(chatManage.SearchResult),
|
||||
})
|
||||
|
||||
// Return if we have results
|
||||
if len(chatManage.SearchResult) != 0 {
|
||||
logger.Infof(
|
||||
ctx,
|
||||
"Get search results, count: %d, session_id: %s",
|
||||
len(chatManage.SearchResult), chatManage.SessionID,
|
||||
)
|
||||
pipelineInfo(ctx, "Search", "output", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"result_count": len(chatManage.SearchResult),
|
||||
})
|
||||
return next()
|
||||
}
|
||||
logger.Infof(ctx, "No search result, session_id: %s", chatManage.SessionID)
|
||||
pipelineWarn(ctx, "Search", "output", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"result_count": 0,
|
||||
})
|
||||
return ErrSearchNothing
|
||||
}
|
||||
|
||||
@@ -143,15 +249,52 @@ func (p *PluginSearch) getSearchResultFromHistory(chatManage *types.ChatManage)
|
||||
}
|
||||
|
||||
func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult {
|
||||
seen := make(map[string]bool)
|
||||
var uniqueResults []*types.SearchResult
|
||||
for _, result := range results {
|
||||
if !seen[result.ID] {
|
||||
seen[result.ID] = true
|
||||
uniqueResults = append(uniqueResults, result)
|
||||
}
|
||||
}
|
||||
return uniqueResults
|
||||
seen := make(map[string]bool)
|
||||
contentSig := make(map[string]bool)
|
||||
var uniqueResults []*types.SearchResult
|
||||
for _, r := range results {
|
||||
keys := []string{r.ID}
|
||||
if r.ParentChunkID != "" {
|
||||
keys = append(keys, "parent:"+r.ParentChunkID)
|
||||
}
|
||||
if r.KnowledgeID != "" {
|
||||
keys = append(keys, fmt.Sprintf("kb:%s#%d", r.KnowledgeID, r.ChunkIndex))
|
||||
}
|
||||
dup := false
|
||||
for _, k := range keys {
|
||||
if seen[k] {
|
||||
dup = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if dup {
|
||||
continue
|
||||
}
|
||||
sig := buildContentSignature(r.Content)
|
||||
if sig != "" {
|
||||
if contentSig[sig] {
|
||||
continue
|
||||
}
|
||||
contentSig[sig] = true
|
||||
}
|
||||
for _, k := range keys {
|
||||
seen[k] = true
|
||||
}
|
||||
uniqueResults = append(uniqueResults, r)
|
||||
}
|
||||
return uniqueResults
|
||||
}
|
||||
|
||||
func buildContentSignature(content string) string {
|
||||
c := strings.ToLower(strings.TrimSpace(content))
|
||||
if c == "" {
|
||||
return ""
|
||||
}
|
||||
c = strings.Join(strings.Fields(c), " ")
|
||||
if len(c) > 128 {
|
||||
c = c[:128]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// searchKnowledgeBases performs KB searches for rewrite and processed queries across KB IDs
|
||||
@@ -175,10 +318,19 @@ func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseID
|
||||
defer wg.Done()
|
||||
res, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, baseParams)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to search KB %s: %v", knowledgeBaseID, err)
|
||||
pipelineWarn(ctx, "Search", "kb_search_error", map[string]interface{}{
|
||||
"kb_id": knowledgeBaseID,
|
||||
"query": baseParams.QueryText,
|
||||
"error": err.Error(),
|
||||
"query_ty": "rewrite",
|
||||
})
|
||||
return
|
||||
}
|
||||
logger.Infof(ctx, "KB %s search results count: %d", knowledgeBaseID, len(res))
|
||||
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
|
||||
"kb_id": knowledgeBaseID,
|
||||
"query_ty": "rewrite",
|
||||
"hit_count": len(res),
|
||||
})
|
||||
mu.Lock()
|
||||
results = append(results, res...)
|
||||
mu.Unlock()
|
||||
@@ -191,7 +343,9 @@ func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseID
|
||||
if chatManage.RewriteQuery != chatManage.ProcessedQuery {
|
||||
paramsProcessed := baseParams
|
||||
paramsProcessed.QueryText = strings.TrimSpace(chatManage.ProcessedQuery)
|
||||
logger.Infof(ctx, "Searching with processed query: %s", paramsProcessed.QueryText)
|
||||
pipelineInfo(ctx, "Search", "processed_query_search", map[string]interface{}{
|
||||
"query": paramsProcessed.QueryText,
|
||||
})
|
||||
|
||||
wg = sync.WaitGroup{}
|
||||
for _, kbID := range knowledgeBaseIDs {
|
||||
@@ -200,10 +354,19 @@ func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseID
|
||||
defer wg.Done()
|
||||
res, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, paramsProcessed)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to search KB %s with processed query: %v", knowledgeBaseID, err)
|
||||
pipelineWarn(ctx, "Search", "kb_search_error", map[string]interface{}{
|
||||
"kb_id": knowledgeBaseID,
|
||||
"query": paramsProcessed.QueryText,
|
||||
"error": err.Error(),
|
||||
"query_ty": "processed",
|
||||
})
|
||||
return
|
||||
}
|
||||
logger.Infof(ctx, "KB %s processed query results count: %d", knowledgeBaseID, len(res))
|
||||
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
|
||||
"kb_id": knowledgeBaseID,
|
||||
"query_ty": "processed",
|
||||
"hit_count": len(res),
|
||||
})
|
||||
mu.Lock()
|
||||
results = append(results, res...)
|
||||
mu.Unlock()
|
||||
@@ -212,7 +375,9 @@ func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseID
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Total KB results (rewrite + processed): %d", len(results))
|
||||
pipelineInfo(ctx, "Search", "kb_result_summary", map[string]interface{}{
|
||||
"total_hits": len(results),
|
||||
})
|
||||
return results
|
||||
}
|
||||
|
||||
@@ -223,14 +388,22 @@ func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types
|
||||
}
|
||||
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if tenant == nil || tenant.WebSearchConfig == nil || tenant.WebSearchConfig.Provider == "" {
|
||||
logger.Warnf(ctx, "Web search enabled but no valid configuration found for tenant %d", chatManage.TenantID)
|
||||
pipelineWarn(ctx, "Search", "web_config_missing", map[string]interface{}{
|
||||
"tenant_id": chatManage.TenantID,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Performing web search with provider: %s", tenant.WebSearchConfig.Provider)
|
||||
pipelineInfo(ctx, "Search", "web_request", map[string]interface{}{
|
||||
"tenant_id": chatManage.TenantID,
|
||||
"provider": tenant.WebSearchConfig.Provider,
|
||||
})
|
||||
webResults, err := p.webSearchService.Search(ctx, tenant.WebSearchConfig, chatManage.RewriteQuery)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Web search failed: %v", err)
|
||||
pipelineWarn(ctx, "Search", "web_search_error", map[string]interface{}{
|
||||
"tenant_id": chatManage.TenantID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
// Build questions (rewrite + processed if different)
|
||||
@@ -245,14 +418,18 @@ func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types
|
||||
p.knowledgeBaseService, p.knowledgeService, seen, ids,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "RAG compression failed, falling back to raw: %v", err)
|
||||
pipelineWarn(ctx, "Search", "web_compress_error", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
} else {
|
||||
webResults = compressed
|
||||
// Persist temp KB state back into Redis using SessionService
|
||||
p.sessionService.SaveWebSearchTempKBState(ctx, chatManage.SessionID, kbID, newSeen, newIDs)
|
||||
}
|
||||
res := convertWebSearchResults(webResults)
|
||||
logger.Infof(ctx, "Web search returned %d results", len(res))
|
||||
pipelineInfo(ctx, "Search", "web_hits", map[string]interface{}{
|
||||
"hit_count": len(res),
|
||||
})
|
||||
return res
|
||||
}
|
||||
|
||||
@@ -323,3 +500,79 @@ func convertWebSearchResults(webResults []*types.WebSearchResult) []*types.Searc
|
||||
|
||||
return results
|
||||
}
|
||||
// expandQueries generates paraphrases and synonyms using chat model to improve keyword recall
|
||||
func (p *PluginSearch) expandQueries(ctx context.Context, chatManage *types.ChatManage) []string {
|
||||
if p.modelService == nil || chatManage.ChatModelID == "" {
|
||||
pipelineWarn(ctx, "Search", "expansion_skip", map[string]interface{}{
|
||||
"reason": "no_model",
|
||||
})
|
||||
return nil
|
||||
}
|
||||
model, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
|
||||
if err != nil {
|
||||
pipelineWarn(ctx, "Search", "expansion_get_model_failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
sys := "Generate up to 5 diverse paraphrases or keyword variants for the user query to improve keyword-based search recall. Respond ONLY with a JSON array of strings inside a fenced code block."
|
||||
usr := chatManage.RewriteQuery
|
||||
think := false
|
||||
resp, err := model.Chat(ctx, []chat.Message{{Role: "system", Content: sys}, {Role: "user", Content: usr}}, &chat.ChatOptions{Temperature: 0.2, MaxCompletionTokens: 80, Thinking: &think})
|
||||
if err != nil || resp.Content == "" {
|
||||
pipelineWarn(ctx, "Search", "expansion_model_call_failed", map[string]interface{}{
|
||||
"error": err,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
body := extractJSONBlock(resp.Content)
|
||||
var arr []string
|
||||
if err := json.Unmarshal([]byte(body), &arr); err != nil || len(arr) == 0 {
|
||||
// Fallback: split lines
|
||||
lines := strings.Split(resp.Content, "\n")
|
||||
for _, l := range lines {
|
||||
l = strings.TrimSpace(l)
|
||||
if l != "" {
|
||||
arr = append(arr, l)
|
||||
}
|
||||
}
|
||||
}
|
||||
uniq := make(map[string]struct{})
|
||||
base := []string{chatManage.Query, chatManage.RewriteQuery, chatManage.ProcessedQuery}
|
||||
for _, b := range base {
|
||||
if s := strings.TrimSpace(b); s != "" {
|
||||
uniq[strings.ToLower(s)] = struct{}{}
|
||||
}
|
||||
}
|
||||
expansions := make([]string, 0, len(arr))
|
||||
for _, a := range arr {
|
||||
s := strings.TrimSpace(a)
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
key := strings.ToLower(s)
|
||||
if _, ok := uniq[key]; ok {
|
||||
continue
|
||||
}
|
||||
uniq[key] = struct{}{}
|
||||
expansions = append(expansions, s)
|
||||
if len(expansions) >= 5 {
|
||||
break
|
||||
}
|
||||
}
|
||||
pipelineInfo(ctx, "Search", "expansion_result", map[string]interface{}{
|
||||
"variants": len(expansions),
|
||||
})
|
||||
return expansions
|
||||
}
|
||||
|
||||
func extractJSONBlock(text string) string {
|
||||
t := strings.TrimSpace(text)
|
||||
if i := strings.Index(t, "["); i >= 0 {
|
||||
j := strings.LastIndex(t, "]")
|
||||
if j > i {
|
||||
return t[i : j+1]
|
||||
}
|
||||
}
|
||||
return "[]"
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/event"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -31,11 +30,17 @@ func (p *PluginStreamFilter) ActivationEvents() []types.EventType {
|
||||
func (p *PluginStreamFilter) OnEvent(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
logger.Info(ctx, "Starting stream filter")
|
||||
pipelineInfo(ctx, "StreamFilter", "input", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"has_event_bus": chatManage.EventBus != nil,
|
||||
"no_match_prefix": chatManage.SummaryConfig.NoMatchPrefix,
|
||||
})
|
||||
|
||||
// EventBus is required
|
||||
if chatManage.EventBus == nil {
|
||||
logger.Error(ctx, "EventBus is required but not available")
|
||||
pipelineError(ctx, "StreamFilter", "eventbus_missing", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
return ErrModelCall.WithError(errors.New("EventBus is required for stream filtering"))
|
||||
}
|
||||
eventBus := chatManage.EventBus
|
||||
@@ -44,13 +49,17 @@ func (p *PluginStreamFilter) OnEvent(ctx context.Context,
|
||||
matchNoMatchBuilderPrefix := chatManage.SummaryConfig.NoMatchPrefix != ""
|
||||
|
||||
if matchNoMatchBuilderPrefix {
|
||||
logger.Infof(ctx, "Using no match prefix filter: %s", chatManage.SummaryConfig.NoMatchPrefix)
|
||||
pipelineInfo(ctx, "StreamFilter", "enable_prefix_filter", map[string]interface{}{
|
||||
"prefix": chatManage.SummaryConfig.NoMatchPrefix,
|
||||
})
|
||||
// Create an event interceptor for prefix filtering
|
||||
return p.filterEventsWithPrefix(ctx, chatManage, eventBus, next)
|
||||
}
|
||||
|
||||
// No filtering needed, just pass through
|
||||
logger.Info(ctx, "No prefix filtering required, passing through")
|
||||
pipelineInfo(ctx, "StreamFilter", "passthrough", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
@@ -61,7 +70,9 @@ func (p *PluginStreamFilter) filterEventsWithPrefix(
|
||||
originalEventBus types.EventBusInterface,
|
||||
next func() *PluginError,
|
||||
) *PluginError {
|
||||
logger.Info(ctx, "Setting up event-based stream filtering with NoMatchPrefix")
|
||||
pipelineInfo(ctx, "StreamFilter", "setup_temp_bus", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
|
||||
// Create a temporary EventBus to intercept events
|
||||
tempEventBus := event.NewEventBus()
|
||||
@@ -81,7 +92,9 @@ func (p *PluginStreamFilter) filterEventsWithPrefix(
|
||||
|
||||
// Check if content does NOT match the no-match prefix (meaning it's valid content)
|
||||
if !strings.HasPrefix(chatManage.SummaryConfig.NoMatchPrefix, responseBuilder.String()) {
|
||||
logger.Infof(ctx, "Content does not match no-match prefix, emitting valid content: %s", responseBuilder.String())
|
||||
pipelineInfo(ctx, "StreamFilter", "emit_valid_chunk", map[string]interface{}{
|
||||
"chunk_len": len(responseBuilder.String()),
|
||||
})
|
||||
|
||||
// Emit the accumulated content as valid answer
|
||||
originalEventBus.Emit(ctx, types.Event{
|
||||
@@ -104,7 +117,9 @@ func (p *PluginStreamFilter) filterEventsWithPrefix(
|
||||
|
||||
// After pipeline completes, check if we need fallback
|
||||
if !matchFound && responseBuilder.Len() > 0 {
|
||||
logger.Info(ctx, "Content matches no-match prefix, emitting fallback response")
|
||||
pipelineInfo(ctx, "StreamFilter", "emit_fallback", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
fallbackID := fmt.Sprintf("%s-fallback", uuid.New().String()[:8])
|
||||
originalEventBus.Emit(ctx, types.Event{
|
||||
ID: fallbackID,
|
||||
|
||||
@@ -77,7 +77,7 @@ func (p *PluginTracing) OnEvent(ctx context.Context,
|
||||
|
||||
// Search traces search operations in the chat pipeline
|
||||
func (p *PluginTracing) Search(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
_, span := tracing.ContextWithSpan(ctx, "PluginTracing.Search")
|
||||
defer span.End()
|
||||
@@ -87,18 +87,23 @@ func (p *PluginTracing) Search(ctx context.Context,
|
||||
attribute.Float64("keyword_threshold", chatManage.KeywordThreshold),
|
||||
attribute.Int("match_count", chatManage.EmbeddingTopK),
|
||||
)
|
||||
err := next()
|
||||
searchResultJson, _ := json.Marshal(chatManage.SearchResult)
|
||||
span.SetAttributes(
|
||||
attribute.String("hybrid_search", string(searchResultJson)),
|
||||
attribute.String("processed_query", chatManage.ProcessedQuery),
|
||||
)
|
||||
return err
|
||||
err := next()
|
||||
searchResultJson, _ := json.Marshal(chatManage.SearchResult)
|
||||
unique := make(map[string]struct{})
|
||||
for _, r := range chatManage.SearchResult {
|
||||
unique[r.ID] = struct{}{}
|
||||
}
|
||||
span.SetAttributes(
|
||||
attribute.String("hybrid_search", string(searchResultJson)),
|
||||
attribute.String("processed_query", chatManage.ProcessedQuery),
|
||||
attribute.Int("search_unique_count", len(unique)),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// Rerank traces rerank operations in the chat pipeline
|
||||
func (p *PluginTracing) Rerank(ctx context.Context,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
||||
) *PluginError {
|
||||
_, span := tracing.ContextWithSpan(ctx, "PluginTracing.Rerank")
|
||||
defer span.End()
|
||||
@@ -109,13 +114,14 @@ func (p *PluginTracing) Rerank(ctx context.Context,
|
||||
attribute.Float64("rerank_filter_threshold", chatManage.RerankThreshold),
|
||||
attribute.Int("rerank_filter_topk", chatManage.RerankTopK),
|
||||
)
|
||||
err := next()
|
||||
resultJson, _ := json.Marshal(chatManage.RerankResult)
|
||||
span.SetAttributes(
|
||||
attribute.Int("rerank_resp_count", len(chatManage.RerankResult)),
|
||||
attribute.String("rerank_resp_results", string(resultJson)),
|
||||
)
|
||||
return err
|
||||
err := next()
|
||||
resultJson, _ := json.Marshal(chatManage.RerankResult)
|
||||
span.SetAttributes(
|
||||
attribute.Int("rerank_resp_count", len(chatManage.RerankResult)),
|
||||
attribute.String("rerank_resp_results", string(resultJson)),
|
||||
attribute.String("query_intent", chatManage.QueryIntent),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// Merge traces merge operations in the chat pipeline
|
||||
|
||||
@@ -261,6 +261,7 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
|
||||
VectorThreshold: e.config.Conversation.VectorThreshold,
|
||||
KeywordThreshold: e.config.Conversation.KeywordThreshold,
|
||||
EmbeddingTopK: e.config.Conversation.EmbeddingTopK,
|
||||
MaxRounds: e.config.Conversation.MaxRounds,
|
||||
RerankModelID: rerankModelID,
|
||||
RerankTopK: e.config.Conversation.RerankTopK,
|
||||
RerankThreshold: e.config.Conversation.RerankThreshold,
|
||||
@@ -279,7 +280,9 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
|
||||
Seed: e.config.Conversation.Summary.Seed,
|
||||
MaxCompletionTokens: e.config.Conversation.Summary.MaxCompletionTokens,
|
||||
},
|
||||
FallbackResponse: e.config.Conversation.FallbackResponse,
|
||||
FallbackResponse: e.config.Conversation.FallbackResponse,
|
||||
RewritePromptSystem: e.config.Conversation.RewritePromptSystem,
|
||||
RewritePromptUser: e.config.Conversation.RewritePromptUser,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -372,6 +372,41 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Normalize keyword retriever scores into [0,1] per-engine batch
|
||||
for i := range retrieveResults {
|
||||
rr := retrieveResults[i]
|
||||
if rr.Error != nil || rr.RetrieverType != types.KeywordsRetrieverType || len(rr.Results) == 0 {
|
||||
continue
|
||||
}
|
||||
minS := rr.Results[0].Score
|
||||
maxS := rr.Results[0].Score
|
||||
for _, r := range rr.Results {
|
||||
if r.Score < minS {
|
||||
minS = r.Score
|
||||
}
|
||||
if r.Score > maxS {
|
||||
maxS = r.Score
|
||||
}
|
||||
}
|
||||
if maxS > minS {
|
||||
for _, r := range rr.Results {
|
||||
ns := (r.Score - minS) / (maxS - minS)
|
||||
if ns < 0 {
|
||||
ns = 0
|
||||
} else if ns > 1 {
|
||||
ns = 1
|
||||
}
|
||||
r.Score = ns
|
||||
}
|
||||
logger.Infof(ctx, "Normalized keyword scores for engine %s: min=%f, max=%f", rr.RetrieverEngineType, minS, maxS)
|
||||
} else {
|
||||
for _, r := range rr.Results {
|
||||
r.Score = 1.0
|
||||
}
|
||||
logger.Infof(ctx, "Keyword scores have no variance for engine %s, set to 1.0", rr.RetrieverEngineType)
|
||||
}
|
||||
}
|
||||
|
||||
// Collect all results from different retrievers and deduplicate by chunk ID
|
||||
logger.Infof(ctx, "Processing retrieval results")
|
||||
matchResults := []*types.IndexWithScore{}
|
||||
|
||||
@@ -436,40 +436,100 @@ func (s *sessionService) KnowledgeQA(ctx context.Context, session *types.Session
|
||||
return err
|
||||
}
|
||||
|
||||
rewritePromptSystem := s.cfg.Conversation.RewritePromptSystem
|
||||
rewritePromptUser := s.cfg.Conversation.RewritePromptUser
|
||||
var tenantConv *types.ConversationConfig
|
||||
if tc, err := getTenantConversationConfig(ctx); err == nil {
|
||||
tenantConv = tc
|
||||
} else {
|
||||
logger.Warnf(ctx, "Failed to load tenant conversation config, tenant ID: %d, error: %v", session.TenantID, err)
|
||||
}
|
||||
|
||||
vectorThreshold := session.VectorThreshold
|
||||
keywordThreshold := session.KeywordThreshold
|
||||
embeddingTopK := session.EmbeddingTopK
|
||||
rerankModelID := session.RerankModelID
|
||||
rerankTopK := session.RerankTopK
|
||||
rerankThreshold := session.RerankThreshold
|
||||
maxRounds := session.MaxRounds
|
||||
fallbackResponse := session.FallbackResponse
|
||||
enableRewrite := session.EnableRewrite
|
||||
|
||||
summaryParams := session.SummaryParameters
|
||||
if summaryParams == nil {
|
||||
summaryParams = &types.SummaryConfig{}
|
||||
}
|
||||
summaryConfig := types.SummaryConfig{
|
||||
MaxTokens: summaryParams.MaxTokens,
|
||||
RepeatPenalty: summaryParams.RepeatPenalty,
|
||||
TopK: summaryParams.TopK,
|
||||
TopP: summaryParams.TopP,
|
||||
FrequencyPenalty: summaryParams.FrequencyPenalty,
|
||||
PresencePenalty: summaryParams.PresencePenalty,
|
||||
Prompt: summaryParams.Prompt,
|
||||
ContextTemplate: summaryParams.ContextTemplate,
|
||||
Temperature: summaryParams.Temperature,
|
||||
Seed: summaryParams.Seed,
|
||||
NoMatchPrefix: summaryParams.NoMatchPrefix,
|
||||
MaxCompletionTokens: summaryParams.MaxCompletionTokens,
|
||||
}
|
||||
|
||||
if tenantConv != nil {
|
||||
vectorThreshold = tenantConv.VectorThreshold
|
||||
keywordThreshold = tenantConv.KeywordThreshold
|
||||
embeddingTopK = tenantConv.EmbeddingTopK
|
||||
rerankModelID = tenantConv.RerankModelID
|
||||
rerankTopK = tenantConv.RerankTopK
|
||||
rerankThreshold = tenantConv.RerankThreshold
|
||||
maxRounds = tenantConv.MaxRounds
|
||||
fallbackResponse = tenantConv.FallbackResponse
|
||||
enableRewrite = tenantConv.EnableRewrite
|
||||
|
||||
if tenantConv.MaxTokens != 0 {
|
||||
summaryConfig.MaxTokens = tenantConv.MaxTokens
|
||||
}
|
||||
if tenantConv.Prompt != "" {
|
||||
summaryConfig.Prompt = tenantConv.Prompt
|
||||
}
|
||||
if tenantConv.ContextTemplate != "" {
|
||||
summaryConfig.ContextTemplate = tenantConv.ContextTemplate
|
||||
}
|
||||
if tenantConv.Temperature != 0 {
|
||||
summaryConfig.Temperature = tenantConv.Temperature
|
||||
}
|
||||
if tenantConv.RewritePromptSystem != "" {
|
||||
rewritePromptSystem = tenantConv.RewritePromptSystem
|
||||
}
|
||||
if tenantConv.RewritePromptUser != "" {
|
||||
rewritePromptUser = tenantConv.RewritePromptUser
|
||||
}
|
||||
}
|
||||
|
||||
// Create chat management object with session settings
|
||||
logger.Infof(ctx, "Creating chat manage object, knowledge base IDs: %v, chat model ID: %s", knowledgeBaseIDs, chatModelID)
|
||||
chatManage := &types.ChatManage{
|
||||
Query: query,
|
||||
RewriteQuery: query,
|
||||
SessionID: session.ID,
|
||||
MessageID: assistantMessageID, // NEW: For event emission in pipeline
|
||||
KnowledgeBaseID: knowledgeBaseIDs[0], // For backward compatibility, use first KB ID
|
||||
KnowledgeBaseIDs: knowledgeBaseIDs, // Multi-KB support
|
||||
VectorThreshold: session.VectorThreshold,
|
||||
KeywordThreshold: session.KeywordThreshold,
|
||||
EmbeddingTopK: session.EmbeddingTopK,
|
||||
RerankModelID: session.RerankModelID,
|
||||
RerankTopK: session.RerankTopK,
|
||||
RerankThreshold: session.RerankThreshold,
|
||||
ChatModelID: chatModelID,
|
||||
SummaryConfig: types.SummaryConfig{
|
||||
MaxTokens: session.SummaryParameters.MaxTokens,
|
||||
RepeatPenalty: session.SummaryParameters.RepeatPenalty,
|
||||
TopK: session.SummaryParameters.TopK,
|
||||
TopP: session.SummaryParameters.TopP,
|
||||
FrequencyPenalty: session.SummaryParameters.FrequencyPenalty,
|
||||
PresencePenalty: session.SummaryParameters.PresencePenalty,
|
||||
Prompt: session.SummaryParameters.Prompt,
|
||||
ContextTemplate: session.SummaryParameters.ContextTemplate,
|
||||
Temperature: session.SummaryParameters.Temperature,
|
||||
Seed: session.SummaryParameters.Seed,
|
||||
NoMatchPrefix: session.SummaryParameters.NoMatchPrefix,
|
||||
MaxCompletionTokens: session.SummaryParameters.MaxCompletionTokens,
|
||||
},
|
||||
FallbackResponse: session.FallbackResponse,
|
||||
EventBus: eventBus.AsEventBusInterface(), // NEW: For pipeline to emit events directly
|
||||
WebSearchEnabled: webSearchEnabled,
|
||||
TenantID: session.TenantID,
|
||||
Query: query,
|
||||
RewriteQuery: query,
|
||||
SessionID: session.ID,
|
||||
MessageID: assistantMessageID, // NEW: For event emission in pipeline
|
||||
KnowledgeBaseID: knowledgeBaseIDs[0], // For backward compatibility, use first KB ID
|
||||
KnowledgeBaseIDs: knowledgeBaseIDs, // Multi-KB support
|
||||
VectorThreshold: vectorThreshold,
|
||||
KeywordThreshold: keywordThreshold,
|
||||
EmbeddingTopK: embeddingTopK,
|
||||
RerankModelID: rerankModelID,
|
||||
RerankTopK: rerankTopK,
|
||||
RerankThreshold: rerankThreshold,
|
||||
MaxRounds: maxRounds,
|
||||
ChatModelID: chatModelID,
|
||||
SummaryConfig: summaryConfig,
|
||||
FallbackResponse: fallbackResponse,
|
||||
EventBus: eventBus.AsEventBusInterface(), // NEW: For pipeline to emit events directly
|
||||
WebSearchEnabled: webSearchEnabled,
|
||||
TenantID: session.TenantID,
|
||||
RewritePromptSystem: rewritePromptSystem,
|
||||
RewritePromptUser: rewritePromptUser,
|
||||
EnableRewrite: enableRewrite,
|
||||
}
|
||||
|
||||
// Start knowledge QA event processing
|
||||
@@ -676,6 +736,17 @@ func (s *sessionService) KnowledgeQAByEvent(ctx context.Context,
|
||||
return nil
|
||||
}
|
||||
|
||||
func getTenantConversationConfig(ctx context.Context) (*types.ConversationConfig, error) {
|
||||
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if tenant == nil {
|
||||
return nil, errors.New("tenant is empty")
|
||||
}
|
||||
if tenant.ConversationConfig == nil {
|
||||
return nil, errors.New("tenant has no conversation config")
|
||||
}
|
||||
return tenant.ConversationConfig, nil
|
||||
}
|
||||
|
||||
// SearchKnowledge performs knowledge base search without LLM summarization
|
||||
func (s *sessionService) SearchKnowledge(ctx context.Context,
|
||||
knowledgeBaseID, query string,
|
||||
@@ -685,14 +756,17 @@ func (s *sessionService) SearchKnowledge(ctx context.Context,
|
||||
|
||||
// Create default retrieval parameters
|
||||
chatManage := &types.ChatManage{
|
||||
Query: query,
|
||||
RewriteQuery: query,
|
||||
KnowledgeBaseID: knowledgeBaseID,
|
||||
VectorThreshold: s.cfg.Conversation.VectorThreshold, // Use default configuration
|
||||
KeywordThreshold: s.cfg.Conversation.KeywordThreshold, // Use default configuration
|
||||
EmbeddingTopK: s.cfg.Conversation.EmbeddingTopK, // Use default configuration
|
||||
RerankTopK: s.cfg.Conversation.RerankTopK, // Use default configuration
|
||||
RerankThreshold: s.cfg.Conversation.RerankThreshold, // Use default configuration
|
||||
Query: query,
|
||||
RewriteQuery: query,
|
||||
KnowledgeBaseID: knowledgeBaseID,
|
||||
VectorThreshold: s.cfg.Conversation.VectorThreshold, // Use default configuration
|
||||
KeywordThreshold: s.cfg.Conversation.KeywordThreshold, // Use default configuration
|
||||
EmbeddingTopK: s.cfg.Conversation.EmbeddingTopK, // Use default configuration
|
||||
RerankTopK: s.cfg.Conversation.RerankTopK, // Use default configuration
|
||||
RerankThreshold: s.cfg.Conversation.RerankThreshold, // Use default configuration
|
||||
MaxRounds: s.cfg.Conversation.MaxRounds,
|
||||
RewritePromptSystem: s.cfg.Conversation.RewritePromptSystem,
|
||||
RewritePromptUser: s.cfg.Conversation.RewritePromptUser,
|
||||
}
|
||||
|
||||
// Get default models
|
||||
|
||||
Reference in New Issue
Block a user