feat: Revamp conversation configuration management and enhance localization

- Removed outdated DEVELOPMENT.md, consolidating development instructions into relevant documentation.
- Updated conversation configuration to include new parameters such as max rounds, embedding thresholds, and rewrite prompts.
- Enhanced UI components to support new conversation settings, including AgentSettings.vue and related views.
- Improved localization in English, Russian, and Chinese for new conversation configuration fields and UI elements.
- Implemented logic to apply tenant-specific conversation settings, ensuring defaults are respected.
This commit is contained in:
wizardchen
2025-11-18 23:18:43 +08:00
parent 60240ee337
commit ddf6b4e748
36 changed files with 2941 additions and 1914 deletions
@@ -3,7 +3,6 @@ package chatpipline
import (
"context"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
@@ -34,7 +33,12 @@ func (p *PluginChatCompletion) ActivationEvents() []types.EventType {
func (p *PluginChatCompletion) OnEvent(
ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
logger.Info(ctx, "Starting chat completion")
pipelineInfo(ctx, "Completion", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"user_question": chatManage.UserContent,
"history_rounds": len(chatManage.History),
"chat_model": chatManage.ChatModelID,
})
// Prepare chat model and options
chatModel, opt, err := prepareChatModel(ctx, p.modelService, chatManage)
@@ -43,18 +47,30 @@ func (p *PluginChatCompletion) OnEvent(
}
// Prepare messages including conversation history
logger.Info(ctx, "Preparing chat messages with history")
pipelineInfo(ctx, "Completion", "messages_ready", map[string]interface{}{
"message_count": len(chatManage.History) + 2,
})
chatMessages := prepareMessagesWithHistory(chatManage)
// Call the chat model to generate response
logger.Info(ctx, "Calling chat model")
pipelineInfo(ctx, "Completion", "model_call", map[string]interface{}{
"chat_model": chatManage.ChatModelID,
})
chatResponse, err := chatModel.Chat(ctx, chatMessages, opt)
if err != nil {
logger.Errorf(ctx, "Failed to call chat model: %v", err)
pipelineError(ctx, "Completion", "model_call", map[string]interface{}{
"chat_model": chatManage.ChatModelID,
"error": err.Error(),
})
return ErrModelCall.WithError(err)
}
logger.Info(ctx, "Chat completion successful")
pipelineInfo(ctx, "Completion", "output", map[string]interface{}{
"answer_preview": truncateForLog(chatResponse.Content),
"finish_reason": chatResponse.FinishReason,
"completion_tokens": chatResponse.Usage.CompletionTokens,
"prompt_tokens": chatResponse.Usage.PromptTokens,
})
chatManage.ChatResponse = chatResponse
return next()
}
@@ -40,7 +40,12 @@ func (p *PluginChatCompletionStream) ActivationEvents() []types.EventType {
func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
logger.Info(ctx, "Starting chat completion stream")
pipelineInfo(ctx, "Stream", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"user_question": chatManage.UserContent,
"history_rounds": len(chatManage.History),
"chat_model": chatManage.ChatModelID,
})
// Prepare chat model and options
chatModel, opt, err := prepareChatModel(ctx, p.modelService, chatManage)
@@ -49,31 +54,49 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
}
// Prepare base messages without history
logger.Info(ctx, "Preparing chat messages")
chatMessages := prepareMessagesWithHistory(chatManage)
chatMessages := prepareMessagesWithHistory(chatManage)
pipelineInfo(ctx, "Stream", "messages_ready", map[string]interface{}{
"message_count": len(chatMessages),
"system_prompt": chatMessages[0].Content,
"user_content": chatMessages[len(chatMessages)-1].Content,
})
// EventBus is required for event-driven streaming
if chatManage.EventBus == nil {
logger.Error(ctx, "EventBus is required but not available")
pipelineError(ctx, "Stream", "eventbus_missing", map[string]interface{}{
"session_id": chatManage.SessionID,
})
return ErrModelCall.WithError(errors.New("EventBus is required for streaming"))
}
eventBus := chatManage.EventBus
logger.Info(ctx, "EventBus detected, enabling event-driven streaming mode")
pipelineInfo(ctx, "Stream", "eventbus_ready", map[string]interface{}{
"session_id": chatManage.SessionID,
})
// Initiate streaming chat model call with independent context
logger.Info(ctx, "Calling chat stream model")
pipelineInfo(ctx, "Stream", "model_call", map[string]interface{}{
"chat_model": chatManage.ChatModelID,
})
responseChan, err := chatModel.ChatStream(ctx, chatMessages, opt)
if err != nil {
logger.Errorf(ctx, "Failed to call chat stream model: %v", err)
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
"chat_model": chatManage.ChatModelID,
"error": err.Error(),
})
return ErrModelCall.WithError(err)
}
if responseChan == nil {
logger.Error(ctx, "Chat stream returned nil channel")
pipelineError(ctx, "Stream", "model_call", map[string]interface{}{
"chat_model": chatManage.ChatModelID,
"error": "nil_channel",
})
return ErrModelCall.WithError(errors.New("chat stream returned nil channel"))
}
logger.Info(ctx, "Chat stream initiated successfully")
pipelineInfo(ctx, "Stream", "model_started", map[string]interface{}{
"session_id": chatManage.SessionID,
})
// Start goroutine to consume channel and emit events directly
go func() {
@@ -98,24 +121,9 @@ func (p *PluginChatCompletionStream) OnEvent(ctx context.Context,
}
}
logger.Info(ctx, "Chat stream completed, emitting completion event")
// Emit completion event when stream finishes
// This allows other components to detect stream completion
if err := eventBus.Emit(ctx, types.Event{
ID: fmt.Sprintf("%s-complete", uuid.New().String()[:8]),
Type: types.EventType(event.EventAgentComplete),
SessionID: chatManage.SessionID,
Data: event.AgentCompleteData{
SessionID: chatManage.SessionID,
MessageID: chatManage.MessageID,
FinalAnswer: finalContent,
},
}); err != nil {
logger.Errorf(ctx, "Failed to emit completion event: %v", err)
}
logger.Info(ctx, "Chat stream completed and completion event emitted")
pipelineInfo(ctx, "Stream", "channel_close", map[string]interface{}{
"session_id": chatManage.SessionID,
})
}()
return next()
@@ -2,6 +2,10 @@ package chatpipline
import (
"context"
"fmt"
"sort"
"strconv"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
@@ -9,6 +13,78 @@ import (
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
const (
logValueMaxRune = 300
defaultStageName = "PIPELINE"
defaultActionName = "info"
pipelineLogPrefix = "[PIPELINE]"
pipelineTruncateEll = "..."
)
func pipelineLog(stage, action string, fields map[string]interface{}) string {
if stage == "" {
stage = defaultStageName
}
if action == "" {
action = defaultActionName
}
builder := strings.Builder{}
builder.Grow(128)
builder.WriteString(pipelineLogPrefix)
builder.WriteString(" stage=")
builder.WriteString(stage)
builder.WriteString(" action=")
builder.WriteString(action)
if len(fields) > 0 {
keys := make([]string, 0, len(fields))
for k := range fields {
keys = append(keys, k)
}
sort.Strings(keys)
for _, key := range keys {
builder.WriteString(" ")
builder.WriteString(key)
builder.WriteString("=")
builder.WriteString(formatLogValue(fields[key]))
}
}
return builder.String()
}
func pipelineInfo(ctx context.Context, stage, action string, fields map[string]interface{}) {
logger.GetLogger(ctx).Info(pipelineLog(stage, action, fields))
}
func pipelineWarn(ctx context.Context, stage, action string, fields map[string]interface{}) {
logger.GetLogger(ctx).Warn(pipelineLog(stage, action, fields))
}
func pipelineError(ctx context.Context, stage, action string, fields map[string]interface{}) {
logger.GetLogger(ctx).Error(pipelineLog(stage, action, fields))
}
func formatLogValue(value interface{}) string {
switch v := value.(type) {
case string:
return strconv.Quote(truncateForLog(v))
case fmt.Stringer:
return strconv.Quote(truncateForLog(v.String()))
default:
return fmt.Sprintf("%v", v)
}
}
func truncateForLog(content string) string {
content = strings.ReplaceAll(content, "\n", "\\n")
runes := []rune(content)
if len(runes) <= logValueMaxRune {
return content
}
return string(runes[:logValueMaxRune]) + pipelineTruncateEll
}
// prepareChatModel shared logic to prepare chat model and options
func prepareChatModel(ctx context.Context, modelService interfaces.ModelService,
chatManage *types.ChatManage,
@@ -35,14 +111,6 @@ func prepareChatModel(ctx context.Context, modelService interfaces.ModelService,
return chatModel, opt, nil
}
// prepareBaseMessages prepare basic messages (system prompt and current user content)
func prepareBaseMessages(chatManage *types.ChatManage) []chat.Message {
var chatMessages []chat.Message
chatMessages = append(chatMessages, chat.Message{Role: "system", Content: chatManage.SummaryConfig.Prompt})
chatMessages = append(chatMessages, chat.Message{Role: "user", Content: chatManage.UserContent})
return chatMessages
}
// prepareMessagesWithHistory prepare complete messages including history
func prepareMessagesWithHistory(chatManage *types.ChatManage) []chat.Message {
chatMessages := []chat.Message{
@@ -3,7 +3,6 @@ package chatpipline
import (
"context"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
)
@@ -27,12 +26,20 @@ func (p *PluginFilterTopK) ActivationEvents() []types.EventType {
func (p *PluginFilterTopK) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
logger.Info(ctx, "Starting filter top-K process")
logger.Infof(ctx, "Filter configuration: top-K = %d", chatManage.RerankTopK)
pipelineInfo(ctx, "FilterTopK", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"top_k": chatManage.RerankTopK,
"merge_cnt": len(chatManage.MergeResult),
"rerank_cnt": len(chatManage.RerankResult),
"search_cnt": len(chatManage.SearchResult),
})
filterTopK := func(searchResult []*types.SearchResult, topK int) []*types.SearchResult {
if topK > 0 && len(searchResult) > topK {
logger.Infof(ctx, "Filtering results: before=%d, after=%d", len(searchResult), topK)
pipelineInfo(ctx, "FilterTopK", "filter", map[string]interface{}{
"before": len(searchResult),
"after": topK,
})
searchResult = searchResult[:topK]
}
return searchResult
@@ -45,9 +52,15 @@ func (p *PluginFilterTopK) OnEvent(ctx context.Context,
} else if len(chatManage.SearchResult) > 0 {
chatManage.SearchResult = filterTopK(chatManage.SearchResult, chatManage.RerankTopK)
} else {
logger.Info(ctx, "No results to filter")
pipelineWarn(ctx, "FilterTopK", "skip", map[string]interface{}{
"reason": "no_results",
})
}
logger.Info(ctx, "Filter top-K process completed")
pipelineInfo(ctx, "FilterTopK", "output", map[string]interface{}{
"merge_cnt": len(chatManage.MergeResult),
"rerank_cnt": len(chatManage.RerankResult),
"search_cnt": len(chatManage.SearchResult),
})
return next()
}
@@ -10,7 +10,6 @@ import (
"strings"
"time"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
@@ -34,6 +33,12 @@ func (p *PluginIntoChatMessage) ActivationEvents() []types.EventType {
func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
pipelineInfo(ctx, "IntoChatMessage", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"merge_result_cnt": len(chatManage.MergeResult),
"template_len": len(chatManage.SummaryConfig.ContextTemplate),
})
// Extract content from merge results
passages := make([]string, len(chatManage.MergeResult))
for i, result := range chatManage.MergeResult {
@@ -44,6 +49,10 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
// Parse the context template
tmpl, err := template.New("searchContent").Parse(chatManage.SummaryConfig.ContextTemplate)
if err != nil {
pipelineError(ctx, "IntoChatMessage", "parse_template", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
return ErrTemplateParse.WithError(err)
}
@@ -54,7 +63,9 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
// 验证用户查询的安全性
safeQuery, isValid := secutils.ValidateInput(chatManage.Query)
if !isValid {
logger.Errorf(ctx, "Invalid user query: %s", chatManage.Query)
pipelineWarn(ctx, "IntoChatMessage", "invalid_query", map[string]interface{}{
"session_id": chatManage.SessionID,
})
return ErrTemplateExecute.WithError(fmt.Errorf("用户查询包含非法内容"))
}
@@ -66,11 +77,19 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
"CurrentWeek": weekdayName[time.Now().Weekday()], // Current weekday in Chinese
})
if err != nil {
pipelineError(ctx, "IntoChatMessage", "render_template", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
return ErrTemplateExecute.WithError(err)
}
// Set formatted content back to chat management
chatManage.UserContent = userContent.String()
pipelineInfo(ctx, "IntoChatMessage", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"user_content_len": len(chatManage.UserContent),
})
return next()
}
@@ -99,7 +118,9 @@ func enrichContentWithImageInfo(ctx context.Context, content string, imageInfoJS
var imageInfos []types.ImageInfo
err := json.Unmarshal([]byte(imageInfoJSON), &imageInfos)
if err != nil {
logger.Warnf(ctx, "Failed to parse ImageInfo: %v, using content only", err)
pipelineWarn(ctx, "IntoChatMessage", "image_parse_error", map[string]interface{}{
"error": err.Error(),
})
return content
}
@@ -125,7 +146,9 @@ func enrichContentWithImageInfo(ctx context.Context, content string, imageInfoJS
// 用于存储已处理的图片URL
processedURLs := make(map[string]bool)
logger.Infof(ctx, "Found %d Markdown image links in content", len(matches))
pipelineInfo(ctx, "IntoChatMessage", "image_markdown_links", map[string]interface{}{
"match_count": len(matches),
})
// 替换每个图片链接,添加描述和OCR文本
for _, match := range matches {
@@ -184,8 +207,10 @@ func enrichContentWithImageInfo(ctx context.Context, content string, imageInfoJS
content += "附加图片信息:\n" + strings.Join(additionalImageTexts, "\n")
}
logger.Debugf(ctx, "Enhanced content with image info: found %d Markdown images, added %d additional images",
len(matches), len(additionalImageTexts))
pipelineInfo(ctx, "IntoChatMessage", "image_enrich_summary", map[string]interface{}{
"markdown_images": len(matches),
"additional_imgs": len(additionalImageTexts),
})
return content
}
@@ -5,16 +5,20 @@ import (
"encoding/json"
"sort"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// PluginMerge handles merging of search result chunks
type PluginMerge struct{}
type PluginMerge struct {
chunkRepo interfaces.ChunkRepository
}
// NewPluginMerge creates and registers a new PluginMerge instance
func NewPluginMerge(eventManager *EventManager) *PluginMerge {
res := &PluginMerge{}
func NewPluginMerge(eventManager *EventManager, chunkRepo interfaces.ChunkRepository) *PluginMerge {
res := &PluginMerge{
chunkRepo: chunkRepo,
}
eventManager.Register(res)
return res
}
@@ -28,19 +32,29 @@ func (p *PluginMerge) ActivationEvents() []types.EventType {
func (p *PluginMerge) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
logger.Info(ctx, "Starting chunk merge process")
pipelineInfo(ctx, "Merge", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"candidate_cnt": len(chatManage.RerankResult),
})
// Use rerank results if available, fallback to search results
searchResult := chatManage.RerankResult
if len(searchResult) == 0 {
logger.Info(ctx, "No rerank results available, using search results")
pipelineWarn(ctx, "Merge", "fallback", map[string]interface{}{
"reason": "empty_rerank_result",
})
searchResult = chatManage.SearchResult
}
logger.Infof(ctx, "Processing %d chunks for merging", len(searchResult))
pipelineInfo(ctx, "Merge", "candidate_ready", map[string]interface{}{
"chunk_cnt": len(searchResult),
})
if len(searchResult) == 0 {
logger.Info(ctx, "No chunks available for merging")
pipelineWarn(ctx, "Merge", "output", map[string]interface{}{
"chunk_cnt": 0,
"reason": "no_candidates",
})
return next()
}
@@ -50,12 +64,17 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
knowledgeGroup[chunk.KnowledgeID] = append(knowledgeGroup[chunk.KnowledgeID], chunk)
}
logger.Infof(ctx, "Grouped chunks by knowledge ID, %d knowledge sources", len(knowledgeGroup))
pipelineInfo(ctx, "Merge", "group_summary", map[string]interface{}{
"knowledge_cnt": len(knowledgeGroup),
})
mergedChunks := []*types.SearchResult{}
// Process each knowledge source separately
for knowledgeID, chunks := range knowledgeGroup {
logger.Infof(ctx, "Processing knowledge ID: %s with %d chunks", knowledgeID, len(chunks))
pipelineInfo(ctx, "Merge", "group_process", map[string]interface{}{
"knowledge_id": knowledgeID,
"chunk_cnt": len(chunks),
})
// Sort chunks by their start position in the original document
sort.Slice(chunks, func(i, j int) bool {
@@ -92,7 +111,10 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
// 合并 ImageInfo
if err := mergeImageInfo(ctx, lastChunk, chunks[i]); err != nil {
logger.Warnf(ctx, "Failed to merge ImageInfo: %v", err)
pipelineWarn(ctx, "Merge", "image_merge", map[string]interface{}{
"knowledge_id": knowledgeID,
"error": err.Error(),
})
}
}
if chunks[i].Score > lastChunk.Score {
@@ -100,8 +122,10 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
}
}
logger.Infof(ctx, "Merged %d chunks into %d chunks for knowledge ID: %s",
len(chunks), len(knowledgeMergedChunks), knowledgeID)
pipelineInfo(ctx, "Merge", "group_output", map[string]interface{}{
"knowledge_id": knowledgeID,
"merged_chunks": len(knowledgeMergedChunks),
})
mergedChunks = append(mergedChunks, knowledgeMergedChunks...)
}
@@ -111,7 +135,11 @@ func (p *PluginMerge) OnEvent(ctx context.Context,
return mergedChunks[i].Score > mergedChunks[j].Score
})
logger.Infof(ctx, "Final merged result: %d chunks, sorted by score", len(mergedChunks))
pipelineInfo(ctx, "Merge", "output", map[string]interface{}{
"merged_total": len(mergedChunks),
})
mergedChunks = p.expandShortContextWithNeighbors(ctx, chatManage, mergedChunks)
chatManage.MergeResult = mergedChunks
return next()
@@ -126,7 +154,9 @@ func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *typ
var sourceImageInfos []types.ImageInfo
if err := json.Unmarshal([]byte(source.ImageInfo), &sourceImageInfos); err != nil {
logger.Warnf(ctx, "Failed to unmarshal source ImageInfo: %v", err)
pipelineWarn(ctx, "Merge", "image_unmarshal_source", map[string]interface{}{
"error": err.Error(),
})
return err
}
@@ -139,7 +169,9 @@ func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *typ
var targetImageInfos []types.ImageInfo
if target.ImageInfo != "" {
if err := json.Unmarshal([]byte(target.ImageInfo), &targetImageInfos); err != nil {
logger.Warnf(ctx, "Failed to unmarshal target ImageInfo: %v", err)
pipelineWarn(ctx, "Merge", "image_unmarshal_target", map[string]interface{}{
"error": err.Error(),
})
// 如果目标解析失败,直接使用源数据
target.ImageInfo = source.ImageInfo
return nil
@@ -164,12 +196,345 @@ func mergeImageInfo(ctx context.Context, target *types.SearchResult, source *typ
// 序列化合并后的ImageInfo
mergedImageInfoJSON, err := json.Marshal(uniqueImageInfos)
if err != nil {
logger.Warnf(ctx, "Failed to marshal merged ImageInfo: %v", err)
pipelineWarn(ctx, "Merge", "image_marshal", map[string]interface{}{
"error": err.Error(),
})
return err
}
// 更新目标chunk的ImageInfo
target.ImageInfo = string(mergedImageInfoJSON)
logger.Infof(ctx, "Successfully merged ImageInfo, total count: %d", len(uniqueImageInfos))
pipelineInfo(ctx, "Merge", "image_merged", map[string]interface{}{
"image_refs": len(uniqueImageInfos),
})
return nil
}
func (p *PluginMerge) expandShortContextWithNeighbors(ctx context.Context, chatManage *types.ChatManage, results []*types.SearchResult) []*types.SearchResult {
const (
minLen = 350
maxLen = 850
)
if len(results) == 0 || p.chunkRepo == nil {
return results
}
tenantID, _ := ctx.Value(types.TenantIDContextKey).(uint)
if tenantID == 0 && chatManage != nil {
tenantID = chatManage.TenantID
}
if tenantID == 0 {
pipelineWarn(ctx, "Merge", "expand_skip", map[string]interface{}{
"reason": "missing_tenant",
})
return results
}
type targetInfo struct {
result *types.SearchResult
}
targets := make([]targetInfo, 0)
baseIDsSet := make(map[string]struct{})
for _, r := range results {
if r == nil || r.ID == "" || r.Content == "" {
continue
}
if r.ChunkType != string(types.ChunkTypeText) {
continue
}
if runeLen(r.Content) >= minLen {
continue
}
targets = append(targets, targetInfo{result: r})
baseIDsSet[r.ID] = struct{}{}
pipelineInfo(ctx, "Merge", "need_expand", map[string]interface{}{
"chunk_id": r.ID,
"content": r.Content,
"chunk_type": r.ChunkType,
"len": runeLen(r.Content),
})
}
if len(targets) == 0 {
return results
}
baseIDs := make([]string, 0, len(baseIDsSet))
for id := range baseIDsSet {
baseIDs = append(baseIDs, id)
}
chunkMap := make(map[string]*types.Chunk, len(baseIDs))
chunks, err := p.chunkRepo.ListChunksByID(ctx, tenantID, baseIDs)
if err != nil {
pipelineWarn(ctx, "Merge", "expand_list_base_failed", map[string]interface{}{
"error": err.Error(),
})
return results
}
for _, chunk := range chunks {
chunkMap[chunk.ID] = chunk
}
neighborIDsSet := make(map[string]struct{})
for _, chunk := range chunkMap {
if chunk == nil {
continue
}
if chunk.PreChunkID != "" {
if _, exists := chunkMap[chunk.PreChunkID]; !exists {
neighborIDsSet[chunk.PreChunkID] = struct{}{}
}
}
if chunk.NextChunkID != "" {
if _, exists := chunkMap[chunk.NextChunkID]; !exists {
neighborIDsSet[chunk.NextChunkID] = struct{}{}
}
}
}
if len(neighborIDsSet) > 0 {
neighborIDs := make([]string, 0, len(neighborIDsSet))
for id := range neighborIDsSet {
neighborIDs = append(neighborIDs, id)
}
neighbors, err := p.chunkRepo.ListChunksByID(ctx, tenantID, neighborIDs)
if err != nil {
pipelineWarn(ctx, "Merge", "expand_list_neighbor_failed", map[string]interface{}{
"error": err.Error(),
})
} else {
for _, chunk := range neighbors {
chunkMap[chunk.ID] = chunk
pipelineInfo(ctx, "Merge", "expand_list_neighbor_success", map[string]interface{}{
"neighbor_chunk_id": chunk.ID,
"neighbor_content": chunk.Content,
"neighbor_chunk_type": chunk.ChunkType,
"neighbor_len": runeLen(chunk.Content),
})
}
}
}
for _, target := range targets {
res := target.result
p.fetchChunksIfMissing(ctx, tenantID, chunkMap, res.ID)
baseChunk := chunkMap[res.ID]
if baseChunk == nil || baseChunk.Content == "" || baseChunk.ChunkType != types.ChunkTypeText {
continue
}
prevContent := ""
nextContent := ""
prevIDs := []string{}
nextIDs := []string{}
prevCursor := baseChunk.PreChunkID
nextCursor := baseChunk.NextChunkID
p.fetchChunksIfMissing(ctx, tenantID, chunkMap, prevCursor, nextCursor)
if prevCursor != "" {
if prevChunk := chunkMap[prevCursor]; prevChunk != nil && prevChunk.KnowledgeID == baseChunk.KnowledgeID {
prevContent = prevChunk.Content
prevIDs = append(prevIDs, prevChunk.ID)
prevCursor = prevChunk.PreChunkID
} else {
prevCursor = ""
}
}
if nextCursor != "" {
if nextChunk := chunkMap[nextCursor]; nextChunk != nil && nextChunk.KnowledgeID == baseChunk.KnowledgeID {
nextContent = nextChunk.Content
nextIDs = append(nextIDs, nextChunk.ID)
nextCursor = nextChunk.NextChunkID
} else {
nextCursor = ""
}
}
var merged string
for {
merged = mergeOrderedContent(prevContent, baseChunk.Content, nextContent, maxLen)
if merged == "" {
break
}
if runeLen(merged) >= minLen {
break
}
if prevCursor == "" && nextCursor == "" {
break
}
expanded := false
if prevCursor != "" {
p.fetchChunksIfMissing(ctx, tenantID, chunkMap, prevCursor)
if prevChunk := chunkMap[prevCursor]; prevChunk != nil && prevChunk.KnowledgeID == baseChunk.KnowledgeID {
prevContent = concatNoOverlap(prevChunk.Content, prevContent)
prevIDs = append([]string{prevChunk.ID}, prevIDs...)
prevCursor = prevChunk.PreChunkID
expanded = true
} else {
prevCursor = ""
}
}
merged = mergeOrderedContent(prevContent, baseChunk.Content, nextContent, maxLen)
if runeLen(merged) >= minLen {
break
}
if nextCursor != "" {
p.fetchChunksIfMissing(ctx, tenantID, chunkMap, nextCursor)
if nextChunk := chunkMap[nextCursor]; nextChunk != nil && nextChunk.KnowledgeID == baseChunk.KnowledgeID {
nextContent = concatNoOverlap(nextContent, nextChunk.Content)
nextIDs = append(nextIDs, nextChunk.ID)
nextCursor = nextChunk.NextChunkID
expanded = true
} else {
nextCursor = ""
}
}
if !expanded {
break
}
}
if merged == "" {
continue
}
beforeLen := runeLen(res.Content)
res.Content = merged
for _, id := range prevIDs {
if id != "" && !containsID(res.SubChunkID, id) {
res.SubChunkID = append(res.SubChunkID, id)
}
}
for _, id := range nextIDs {
if id != "" && !containsID(res.SubChunkID, id) {
res.SubChunkID = append(res.SubChunkID, id)
}
}
if prevContent != "" {
res.StartAt = baseChunk.StartAt - runeLen(prevContent)
if res.StartAt < 0 {
res.StartAt = 0
}
}
res.EndAt = res.StartAt + runeLen(res.Content)
pipelineInfo(ctx, "Merge", "expand_short_chunk", map[string]interface{}{
"chunk_id": res.ID,
"prev_ids": prevIDs,
"next_ids": nextIDs,
"before_len": beforeLen,
"after_len": runeLen(res.Content),
"base_content": baseChunk.Content,
"after_content": res.Content,
"chunk_type": res.ChunkType,
"remaining_prev": prevCursor,
"remaining_next": nextCursor,
})
}
return results
}
func runeLen(s string) int {
return len([]rune(s))
}
func mergeOrderedContent(prev, base, next string, maxLen int) string {
content := base
if prev != "" {
content = concatNoOverlap(prev, content)
}
if next != "" {
content = concatNoOverlap(content, next)
}
runes := []rune(content)
if len(runes) > maxLen {
return string(runes[:maxLen])
}
return content
}
// concatNoOverlap concatenates a and b, removing potential overlapping prefix/suffix
func concatNoOverlap(a, b string) string {
if a == "" {
return b
}
if b == "" {
return a
}
ar := []rune(a)
br := []rune(b)
maxOverlap := minInt(len(ar), len(br))
for k := maxOverlap; k > 0; k-- {
if string(ar[len(ar)-k:]) == string(br[:k]) {
return string(ar) + string(br[k:])
}
}
return string(ar) + string(br)
}
func minInt(a, b int) int {
if a < b {
return a
}
return b
}
func containsID(ids []string, target string) bool {
for _, id := range ids {
if id == target {
return true
}
}
return false
}
func (p *PluginMerge) fetchChunksIfMissing(ctx context.Context, tenantID uint, chunkMap map[string]*types.Chunk, chunkIDs ...string) {
missing := make([]string, 0, len(chunkIDs))
for _, id := range chunkIDs {
if id == "" {
continue
}
if _, exists := chunkMap[id]; !exists {
missing = append(missing, id)
}
}
if len(missing) == 0 {
return
}
chunks, err := p.chunkRepo.ListChunksByID(ctx, tenantID, missing)
if err != nil {
pipelineWarn(ctx, "Merge", "expand_fetch_missing_failed", map[string]interface{}{
"missing_cnt": len(missing),
"error": err.Error(),
})
}
found := make(map[string]struct{}, len(chunks))
for _, chunk := range chunks {
chunkMap[chunk.ID] = chunk
found[chunk.ID] = struct{}{}
}
for _, id := range missing {
if _, ok := found[id]; !ok {
chunkMap[id] = nil
}
}
}
@@ -1,23 +1,26 @@
package chatpipline
import (
"context"
"regexp"
"strings"
"unicode"
"context"
"encoding/json"
"regexp"
"strings"
"unicode"
"unicode/utf8"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/yanyiwu/gojieba"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/yanyiwu/gojieba"
)
// PluginPreprocess Query preprocessing plugin
type PluginPreprocess struct {
config *config.Config
jieba *gojieba.Jieba
stopwords map[string]struct{}
config *config.Config
jieba *gojieba.Jieba
stopwords map[string]struct{}
modelService interfaces.ModelService
}
// Regular expressions for text cleaning
@@ -28,11 +31,14 @@ var (
punctRegex = regexp.MustCompile(`[^\p{L}\p{N}\s]`) // Punctuation marks
)
const maxProcessedTokens = 12
// NewPluginPreprocess Creates a new query preprocessing plugin
func NewPluginPreprocess(
eventManager *EventManager,
config *config.Config,
cleaner interfaces.ResourceCleaner,
eventManager *EventManager,
config *config.Config,
cleaner interfaces.ResourceCleaner,
modelService interfaces.ModelService,
) *PluginPreprocess {
// Use default dictionary for Jieba tokenizer
jieba := gojieba.NewJieba()
@@ -40,11 +46,12 @@ func NewPluginPreprocess(
// Load stopwords from built-in stopword library
stopwords := loadStopwords()
res := &PluginPreprocess{
config: config,
jieba: jieba,
stopwords: stopwords,
}
res := &PluginPreprocess{
config: config,
jieba: jieba,
stopwords: stopwords,
modelService: modelService,
}
// Register resource cleanup function
if cleaner != nil {
@@ -85,25 +92,66 @@ func (p *PluginPreprocess) ActivationEvents() []types.EventType {
// OnEvent Process events
func (p *PluginPreprocess) OnEvent(ctx context.Context, eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError) *PluginError {
if chatManage.RewriteQuery == "" {
return next()
rawQuery := strings.TrimSpace(chatManage.RewriteQuery)
if rawQuery == "" {
return next()
}
pipelineInfo(ctx, "Preprocess", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"rewrite_query": rawQuery,
})
normalized := normalizeWhitespace(rawQuery)
sanitized := strings.TrimSpace(p.cleanText(normalized))
if sanitized == "" {
sanitized = normalized
}
logger.GetLogger(ctx).Infof("Starting query preprocessing, original query: %s", chatManage.RewriteQuery)
var (
processed = sanitized
strategy = "original"
tokenPreview string
tokenCount int
)
// 1. Basic text cleaning
processed := p.cleanText(chatManage.RewriteQuery)
switch {
case containsChineseCharacters(sanitized):
segments := p.segmentText(sanitized)
tokens := p.selectMeaningfulTokens(segments)
tokenCount = len(tokens)
if len(tokens) >= 2 {
processed = strings.Join(tokens, " ")
strategy = "zh_tokens"
tokenPreview = strings.Join(tokens, ",")
} else {
strategy = "fallback_original"
}
case containsLatinLetters(sanitized):
processed = normalizeLatinQuery(sanitized)
if processed != sanitized {
strategy = "latin_normalize"
}
default:
strategy = "original"
}
// 2. Tokenization
segments := p.segmentText(processed)
if strings.TrimSpace(processed) == "" {
processed = rawQuery
strategy = "fallback_original"
}
// 3. Stopword filtering and reconstruction
filteredSegments := p.filterStopwords(segments)
chatManage.ProcessedQuery = processed
chatManage.QueryIntent = p.detectIntentLLM(ctx, chatManage, sanitized)
// Update preprocessed query
chatManage.ProcessedQuery = strings.Join(filteredSegments, " ")
logger.GetLogger(ctx).Infof("Query preprocessing complete, processed query: %s", chatManage.ProcessedQuery)
pipelineInfo(ctx, "Preprocess", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"processed_query": processed,
"strategy": strategy,
"token_count": tokenCount,
"token_preview": truncateForLog(tokenPreview),
"query_intent": chatManage.QueryIntent,
})
return next()
}
@@ -136,32 +184,137 @@ func (p *PluginPreprocess) segmentText(text string) []string {
}
// filterStopwords Filter stopwords
func (p *PluginPreprocess) filterStopwords(segments []string) []string {
var filtered []string
func (p *PluginPreprocess) selectMeaningfulTokens(segments []string) []string {
var tokens []string
seen := make(map[string]struct{})
for _, word := range segments {
// If not a stopword and not blank, keep it
if _, isStopword := p.stopwords[word]; !isStopword && !isBlank(word) {
filtered = append(filtered, word)
word = strings.TrimSpace(word)
if word == "" {
continue
}
if _, stop := p.stopwords[word]; stop {
continue
}
if _, exists := seen[word]; exists {
continue
}
if !isInformativeToken(word) {
continue
}
seen[word] = struct{}{}
tokens = append(tokens, word)
if len(tokens) >= maxProcessedTokens {
break
}
}
// If filtering results in empty list, return original tokenization results
if len(filtered) == 0 {
return segments
}
return filtered
return tokens
}
// isBlank Check if a string is blank
func isBlank(str string) bool {
for _, r := range str {
if !unicode.IsSpace(r) {
return false
func isInformativeToken(token string) bool {
if token == "" {
return false
}
runeCount := utf8.RuneCountInString(token)
if runeCount == 1 {
r, _ := utf8.DecodeRuneInString(token)
if unicode.IsDigit(r) {
return true
}
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
return true
}
return false
}
return true
}
func containsChineseCharacters(text string) bool {
for _, r := range text {
if unicode.Is(unicode.Han, r) {
return true
}
}
return true
return false
}
func containsLatinLetters(text string) bool {
for _, r := range text {
if r <= unicode.MaxASCII && unicode.IsLetter(r) {
return true
}
}
return false
}
func normalizeWhitespace(text string) string {
text = strings.TrimSpace(text)
return multiSpaceRegex.ReplaceAllString(text, " ")
}
func normalizeLatinQuery(text string) string {
text = strings.ToLower(text)
text = multiSpaceRegex.ReplaceAllString(text, " ")
return strings.TrimSpace(text)
}
type intentResp struct {
Intent string `json:"intent"`
Confidence float64 `json:"confidence"`
}
func (p *PluginPreprocess) detectIntentLLM(ctx context.Context, chatManage *types.ChatManage, text string) string {
if p.modelService == nil || chatManage.ChatModelID == "" {
pipelineWarn(ctx, "IntentDetect", "skip", map[string]interface{}{ "reason": "no_model", "session_id": chatManage.SessionID })
return "general"
}
chatModel, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
if err != nil {
pipelineWarn(ctx, "IntentDetect", "get_model_failed", map[string]interface{}{ "error": err.Error(), "model_id": chatManage.ChatModelID })
return "general"
}
pipelineInfo(ctx, "IntentDetect", "start", map[string]interface{}{ "session_id": chatManage.SessionID, "model_id": chatManage.ChatModelID })
sys := "You are a query intent classifier. Classify the user's query into one of: definition, howto, compare, qa, general. Respond ONLY with a JSON object {\"intent\": \"...\", \"confidence\": 0.0 } inside a markdown fenced block."
usr := text
think := false
resp, err := chatModel.Chat(ctx, []chat.Message{
{Role: "system", Content: sys},
{Role: "user", Content: usr},
}, &chat.ChatOptions{Temperature: 0.0, MaxCompletionTokens: 64, Thinking: &think})
if err != nil || resp.Content == "" {
pipelineWarn(ctx, "IntentDetect", "model_call_failed", map[string]interface{}{ "error": err })
return "general"
}
body := extractJSONBody(resp.Content)
var ir intentResp
if err := json.Unmarshal([]byte(body), &ir); err != nil {
pipelineWarn(ctx, "IntentDetect", "parse_failed", map[string]interface{}{ "body": truncateForLog(body), "error": err.Error() })
return "general"
}
pipelineInfo(ctx, "IntentDetect", "result", map[string]interface{}{ "intent": ir.Intent, "confidence": ir.Confidence })
switch strings.ToLower(strings.TrimSpace(ir.Intent)) {
case "definition", "howto", "compare", "qa", "general":
return strings.ToLower(ir.Intent)
default:
return "general"
}
}
func extractJSONBody(text string) string {
t := strings.TrimSpace(text)
// Try fenced block first
if i := strings.Index(t, "{"); i >= 0 {
j := strings.LastIndex(t, "}")
if j > i {
return t[i : j+1]
}
}
return "{}"
}
// Ensure resources are properly released
@@ -1,16 +1,15 @@
package chatpipline
import (
"context"
"encoding/json"
"fmt"
"math"
"strings"
"context"
"encoding/json"
"fmt"
"math"
"strings"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// PluginRerank implements reranking functionality for chat pipeline
@@ -34,28 +33,43 @@ func (p *PluginRerank) ActivationEvents() []types.EventType {
// OnEvent handles reranking events in the chat pipeline
func (p *PluginRerank) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
logger.Info(ctx, "Starting reranking process")
logger.Infof(ctx, "Getting rerank model, model ID: %s", chatManage.RerankModelID)
pipelineInfo(ctx, "Rerank", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"candidate_cnt": len(chatManage.SearchResult),
"rerank_model": chatManage.RerankModelID,
"rerank_thresh": chatManage.RerankThreshold,
"rewrite_query": chatManage.RewriteQuery,
"processed_query": chatManage.ProcessedQuery,
})
if len(chatManage.SearchResult) == 0 {
logger.Infof(ctx, "No search result, skip reranking")
pipelineInfo(ctx, "Rerank", "skip", map[string]interface{}{
"reason": "empty_search_result",
})
return next()
}
if chatManage.RerankModelID == "" {
logger.Warn(ctx, "Rerank model ID is empty, skipping reranking")
pipelineWarn(ctx, "Rerank", "skip", map[string]interface{}{
"reason": "empty_model_id",
})
return next()
}
// Get rerank model from service
rerankModel, err := p.modelService.GetRerankModel(ctx, chatManage.RerankModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get rerank model: %v, rerank model ID: %s", err, chatManage.RerankModelID)
pipelineError(ctx, "Rerank", "get_model", map[string]interface{}{
"model_id": chatManage.RerankModelID,
"error": err.Error(),
})
return ErrGetRerankModel.WithError(err)
}
// Prepare passages for reranking
logger.Infof(ctx, "Preparing passages for reranking, search result count: %d", len(chatManage.SearchResult))
pipelineInfo(ctx, "Rerank", "build_passages", map[string]interface{}{
"candidate_cnt": len(chatManage.SearchResult),
})
var passages []string
for _, result := range chatManage.SearchResult {
// 合并Content和ImageInfo的文本内容
@@ -72,41 +86,79 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
}
}
// Update search results with reranked scores
logger.Infof(ctx, "Filtered rerank results, original: %d, filtered: %d", len(rerankResp), len(rerankResp))
result := []*types.SearchResult{}
for _, rr := range rerankResp {
chatManage.SearchResult[rr.Index].Score = rr.RelevanceScore
result = append(result, chatManage.SearchResult[rr.Index])
}
chatManage.RerankResult = result
pipelineInfo(ctx, "Rerank", "model_response", map[string]interface{}{
"result_cnt": len(rerankResp),
})
for i := range chatManage.SearchResult {
chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata)
}
reranked := make([]*types.SearchResult, 0, len(rerankResp))
for _, rr := range rerankResp {
sr := chatManage.SearchResult[rr.Index]
base := sr.Score
sr.Metadata["base_score"] = fmt.Sprintf("%.4f", base)
sr.Score = rr.RelevanceScore
sr.Score = compositeScore(sr, rr.RelevanceScore, base, chatManage)
reranked = append(reranked, sr)
}
final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7)
chatManage.RerankResult = final
if len(chatManage.RerankResult) == 0 {
logger.Warn(ctx, "Reranking produced no results above threshold")
return ErrSearchNothing
}
// Log composite top scores and MMR selection summary
topN := min(3, len(reranked))
for i := 0; i < topN; i++ {
pipelineInfo(ctx, "Rerank", "composite_top", map[string]interface{}{
"rank": i + 1,
"chunk_id": reranked[i].ID,
"base_score": reranked[i].Metadata["base_score"],
"final_score": fmt.Sprintf("%.4f", reranked[i].Score),
"intent": chatManage.QueryIntent,
})
}
logger.Infof(ctx, "Reranking process completed successfully, result count: %d", len(chatManage.RerankResult))
return next()
if len(chatManage.RerankResult) == 0 {
pipelineWarn(ctx, "Rerank", "output", map[string]interface{}{
"filtered_cnt": 0,
})
return ErrSearchNothing
}
pipelineInfo(ctx, "Rerank", "output", map[string]interface{}{
"filtered_cnt": len(chatManage.RerankResult),
})
return next()
}
// rerank performs the actual reranking operation with given query and passages
func (p *PluginRerank) rerank(ctx context.Context,
chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string,
chatManage *types.ChatManage, rerankModel rerank.Reranker, query string, passages []string,
) []rerank.RankResult {
logger.Infof(ctx, "Executing reranking with query: %s, passage count: %d", query, len(passages))
pipelineInfo(ctx, "Rerank", "model_call", map[string]interface{}{
"query_variant": query,
"passages": len(passages),
})
rerankResp, err := rerankModel.Rerank(ctx, query, passages)
if err != nil {
logger.Errorf(ctx, "Reranking failed: %v", err)
pipelineError(ctx, "Rerank", "model_call", map[string]interface{}{
"query_variant": query,
"error": err.Error(),
})
return nil
}
// Log top scores for debugging
logger.Infof(ctx, "Reranking completed, filtering results with threshold: %f", chatManage.RerankThreshold)
for i := range min(3, len(rerankResp)) {
logger.Infof(ctx, "Top %d score of rerankResp: %f, passages: %s, index: %d",
i+1, rerankResp[i].RelevanceScore, rerankResp[i].Document.Text, rerankResp[i].Index,
)
pipelineInfo(ctx, "Rerank", "threshold", map[string]interface{}{
"threshold": chatManage.RerankThreshold,
})
for i := range min(5, len(rerankResp)) {
pipelineInfo(ctx, "Rerank", "top_score", map[string]interface{}{
"rank": i + 1,
"score": rerankResp[i].RelevanceScore,
"chunk_id": chatManage.SearchResult[rerankResp[i].Index].ID,
"match_type": chatManage.SearchResult[rerankResp[i].Index].MatchType,
"chunk_type": chatManage.SearchResult[rerankResp[i].Index].ChunkType,
"content": chatManage.SearchResult[rerankResp[i].Index].Content,
})
}
// Filter results based on threshold with special handling for history matches
@@ -121,7 +173,149 @@ func (p *PluginRerank) rerank(ctx context.Context,
rankFilter = append(rankFilter, result)
}
}
return rankFilter
return rankFilter
}
func ensureMetadata(m map[string]string) map[string]string {
if m == nil {
return make(map[string]string)
}
return m
}
func compositeScore(sr *types.SearchResult, modelScore, baseScore float64, chatManage *types.ChatManage) float64 {
sourceWeight := 1.0
switch strings.ToLower(sr.KnowledgeSource) {
case "web_search":
sourceWeight = 0.95
default:
sourceWeight = 1.0
}
intentBoost := 1.0
if chatManage.QueryIntent != "" {
switch chatManage.QueryIntent {
case "definition":
if sr.ChunkType == string(types.ChunkTypeSummary) {
intentBoost = 1.05
}
case "howto":
if sr.EndAt-sr.StartAt > 300 {
intentBoost = 1.03
}
case "compare":
intentBoost = 1.0
}
}
positionPrior := 1.0
if sr.StartAt >= 0 {
positionPrior += clampFloat(1.0-float64(sr.StartAt)/float64(sr.EndAt+1), -0.05, 0.05)
}
composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight
composite *= intentBoost
composite *= positionPrior
if composite < 0 {
composite = 0
}
if composite > 1 {
composite = 1
}
return composite
}
func applyMMR(ctx context.Context, results []*types.SearchResult, chatManage *types.ChatManage, k int, lambda float64) []*types.SearchResult {
if k <= 0 || len(results) == 0 {
return nil
}
pipelineInfo(ctx, "Rerank", "mmr_start", map[string]interface{}{
"lambda": lambda,
"k": k,
"candidates": len(results),
})
selected := make([]*types.SearchResult, 0, k)
candidates := make([]*types.SearchResult, len(results))
copy(candidates, results)
tokenSets := make([]map[string]struct{}, len(candidates))
for i, r := range candidates {
tokenSets[i] = tokenizeSimple(getEnrichedPassage(ctx, r))
}
for len(selected) < k && len(candidates) > 0 {
bestIdx := 0
bestScore := -1.0
for i, r := range candidates {
relevance := r.Score
redundancy := 0.0
for _, s := range selected {
redundancy = math.Max(redundancy, jaccard(tokenSets[i], tokenizeSimple(getEnrichedPassage(ctx, s))))
}
mmr := lambda*relevance - (1.0-lambda)*redundancy
if mmr > bestScore {
bestScore = mmr
bestIdx = i
}
}
selected = append(selected, candidates[bestIdx])
candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...)
}
// Compute average redundancy among selected
avgRed := 0.0
if len(selected) > 1 {
pairs := 0
for i := 0; i < len(selected); i++ {
for j := i + 1; j < len(selected); j++ {
si := tokenizeSimple(getEnrichedPassage(ctx, selected[i]))
sj := tokenizeSimple(getEnrichedPassage(ctx, selected[j]))
avgRed += jaccard(si, sj)
pairs++
}
}
if pairs > 0 {
avgRed /= float64(pairs)
}
}
pipelineInfo(ctx, "Rerank", "mmr_done", map[string]interface{}{
"selected": len(selected),
"avg_redundancy": fmt.Sprintf("%.4f", avgRed),
})
return selected
}
func tokenizeSimple(text string) map[string]struct{} {
text = strings.ToLower(text)
fields := strings.Fields(text)
set := make(map[string]struct{}, len(fields))
for _, f := range fields {
if len(f) > 1 {
set[f] = struct{}{}
}
}
return set
}
func jaccard(a, b map[string]struct{}) float64 {
if len(a) == 0 && len(b) == 0 {
return 0
}
inter := 0
for k := range a {
if _, ok := b[k]; ok {
inter++
}
}
union := len(a) + len(b) - inter
if union == 0 {
return 0
}
return float64(inter) / float64(union)
}
func clampFloat(v, minV, maxV float64) float64 {
if v < minV {
return minV
}
if v > maxV {
return maxV
}
return v
}
// getEnrichedPassage 合并Content和ImageInfo的文本内容
@@ -134,7 +328,9 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string
var imageInfos []types.ImageInfo
err := json.Unmarshal([]byte(result.ImageInfo), &imageInfos)
if err != nil {
logger.Warnf(ctx, "Failed to parse ImageInfo: %v, using content only", err)
pipelineWarn(ctx, "Rerank", "image_info_parse", map[string]interface{}{
"error": err.Error(),
})
return result.Content
}
@@ -164,8 +360,10 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string
}
combinedText += strings.Join(imageTexts, "\n")
logger.Debugf(ctx, "Enhanced passage with image info: content length %d, image texts length %d",
len(result.Content), len(strings.Join(imageTexts, "\n")))
pipelineInfo(ctx, "Rerank", "image_info_merge", map[string]interface{}{
"content_len": len(result.Content),
"image_len": len(strings.Join(imageTexts, "\n")),
})
return combinedText
}
@@ -12,7 +12,6 @@ import (
"time"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
@@ -58,10 +57,28 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
// Initialize rewritten query as original query
chatManage.RewriteQuery = chatManage.Query
if !chatManage.EnableRewrite {
pipelineInfo(ctx, "Rewrite", "skip", map[string]interface{}{
"session_id": chatManage.SessionID,
"reason": "rewrite_disabled",
})
return next()
}
pipelineInfo(ctx, "Rewrite", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"tenant_id": chatManage.TenantID,
"user_query": chatManage.Query,
"enable_rewrite": chatManage.EnableRewrite,
})
// Get conversation history
history, err := p.messageService.GetRecentMessagesBySession(ctx, chatManage.SessionID, 20)
if err != nil {
logger.Errorf(ctx, "Failed to get conversation history, session_id: %s, error: %v", chatManage.SessionID, err)
pipelineWarn(ctx, "Rewrite", "history_fetch", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
}
// Convert historical messages to conversation history structure
@@ -99,22 +116,52 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
})
// Limit the number of historical records
if len(historyList) > p.config.Conversation.MaxRounds {
historyList = historyList[:p.config.Conversation.MaxRounds]
maxRounds := p.config.Conversation.MaxRounds
if chatManage.MaxRounds > 0 {
maxRounds = chatManage.MaxRounds
}
if len(historyList) > maxRounds {
historyList = historyList[:maxRounds]
}
// Reverse to chronological order
slices.Reverse(historyList)
chatManage.History = historyList
userTmpl, err := template.New("rewriteContent").Parse(p.config.Conversation.RewritePromptUser)
if err != nil {
logger.Errorf(ctx, "Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err)
if len(historyList) == 0 {
pipelineInfo(ctx, "Rewrite", "skip", map[string]interface{}{
"session_id": chatManage.SessionID,
"reason": "empty_history",
})
return next()
}
systemTmpl, err := template.New("rewriteContent").Parse(p.config.Conversation.RewritePromptSystem)
pipelineInfo(ctx, "Rewrite", "history_ready", map[string]interface{}{
"session_id": chatManage.SessionID,
"history_rounds": len(historyList),
"max_rounds": maxRounds,
})
userPrompt := p.config.Conversation.RewritePromptUser
if chatManage.RewritePromptUser != "" {
userPrompt = chatManage.RewritePromptUser
}
userTmpl, err := template.New("rewriteContent").Parse(userPrompt)
if err != nil {
logger.GetLogger(ctx).Errorf("Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err)
pipelineError(ctx, "Rewrite", "parse_user_template", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
return next()
}
systemPrompt := p.config.Conversation.RewritePromptSystem
if chatManage.RewritePromptSystem != "" {
systemPrompt = chatManage.RewritePromptSystem
}
systemTmpl, err := template.New("rewriteContent").Parse(systemPrompt)
if err != nil {
pipelineError(ctx, "Rewrite", "parse_system_template", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
return next()
}
currentTime := time.Now().Format("2006-01-02 15:04:05")
@@ -126,7 +173,10 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
"Conversation": historyList,
})
if err != nil {
logger.GetLogger(ctx).Errorf("Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err)
pipelineError(ctx, "Rewrite", "render_user_template", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
return next()
}
err = systemTmpl.Execute(&systemContent, map[string]interface{}{
@@ -136,12 +186,19 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
"Conversation": historyList,
})
if err != nil {
logger.Errorf(ctx, "Failed to execute template, session_id: %s, error: %v", chatManage.SessionID, err)
pipelineError(ctx, "Rewrite", "render_system_template", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
return next()
}
rewriteModel, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
if err != nil {
logger.Errorf(ctx, "Failed to get model, session_id: %s, error: %v", chatManage.SessionID, err)
pipelineError(ctx, "Rewrite", "get_model", map[string]interface{}{
"session_id": chatManage.SessionID,
"chat_model_id": chatManage.ChatModelID,
"error": err.Error(),
})
return next()
}
@@ -162,7 +219,10 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
Thinking: &thinking,
})
if err != nil {
logger.Errorf(ctx, "Failed to execute model, session_id: %s, error: %v", chatManage.SessionID, err)
pipelineError(ctx, "Rewrite", "model_call", map[string]interface{}{
"session_id": chatManage.SessionID,
"error": err.Error(),
})
return next()
}
@@ -170,7 +230,9 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
// Update rewritten query
chatManage.RewriteQuery = response.Content
}
logger.GetLogger(ctx).Infof("Rewritten query, session_id: %s, rewrite_query: %s",
chatManage.SessionID, chatManage.RewriteQuery)
pipelineInfo(ctx, "Rewrite", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"rewrite_query": chatManage.RewriteQuery,
})
return next()
}
@@ -1,16 +1,17 @@
package chatpipline
import (
"context"
"fmt"
"strings"
"sync"
"time"
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"time"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// PluginSearch implements search functionality for chat pipeline
@@ -53,23 +54,42 @@ func (p *PluginSearch) ActivationEvents() []types.EventType {
// OnEvent handles search events in the chat pipeline
func (p *PluginSearch) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
// Get knowledge base IDs list
knowledgeBaseIDs := chatManage.KnowledgeBaseIDs
if len(knowledgeBaseIDs) == 0 && chatManage.KnowledgeBaseID != "" {
// Fall back to single knowledge base
knowledgeBaseIDs = []string{chatManage.KnowledgeBaseID}
logger.Infof(ctx, "No KnowledgeBaseIDs provided, falling back to single KB: %s", chatManage.KnowledgeBaseID)
pipelineInfo(ctx, "Search", "fallback_kb", map[string]interface{}{
"session_id": chatManage.SessionID,
"kb_id": chatManage.KnowledgeBaseID,
})
}
if len(knowledgeBaseIDs) == 0 {
logger.Errorf(ctx, "No knowledge base IDs available for search")
pipelineError(ctx, "Search", "kb_not_found", map[string]interface{}{
"session_id": chatManage.SessionID,
})
return ErrSearch.WithError(nil)
}
pipelineInfo(ctx, "Search", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"rewrite_query": chatManage.RewriteQuery,
"processed_query": chatManage.ProcessedQuery,
"kb_ids": strings.Join(knowledgeBaseIDs, ","),
"tenant_id": chatManage.TenantID,
"web_enabled": chatManage.WebSearchEnabled,
})
// Run KB search and web search concurrently
logger.Infof(ctx, "Searching across %d knowledge base(s): %v", len(knowledgeBaseIDs), knowledgeBaseIDs)
pipelineInfo(ctx, "Search", "plan", map[string]interface{}{
"kb_count": len(knowledgeBaseIDs),
"embedding_top_k": chatManage.EmbeddingTopK,
"vector_threshold": chatManage.VectorThreshold,
"keyword_threshold": chatManage.KeywordThreshold,
})
var wg sync.WaitGroup
var mu sync.Mutex
allResults := make([]*types.SearchResult, 0)
@@ -97,30 +117,116 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
}
}()
wg.Wait()
wg.Wait()
chatManage.SearchResult = allResults
chatManage.SearchResult = allResults
// If recall is low, attempt query expansion with keyword-focused search
if len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK/2) {
pipelineInfo(ctx, "Search", "recall_low", map[string]interface{}{
"current": len(chatManage.SearchResult),
"threshold": chatManage.EmbeddingTopK / 2,
})
expansions := p.expandQueries(ctx, chatManage)
if len(expansions) > 0 {
pipelineInfo(ctx, "Search", "expansion_start", map[string]interface{}{
"variants": len(expansions),
})
expTopK := max(chatManage.EmbeddingTopK*2, chatManage.RerankTopK*2)
expKwTh := chatManage.KeywordThreshold * 0.8
// Concurrent expansion retrieval across queries and KBs
expResults := make([]*types.SearchResult, 0, expTopK*len(expansions))
var muExp sync.Mutex
var wgExp sync.WaitGroup
jobs := len(expansions) * len(knowledgeBaseIDs)
capSem := 16
if jobs < capSem {
capSem = jobs
}
if capSem <= 0 {
capSem = 1
}
sem := make(chan struct{}, capSem)
pipelineInfo(ctx, "Search", "expansion_concurrency", map[string]interface{}{
"jobs": jobs,
"cap": capSem,
})
for _, q := range expansions {
for _, kbID := range knowledgeBaseIDs {
wgExp.Add(1)
go func(q string, kbID string) {
defer wgExp.Done()
sem <- struct{}{}
defer func() { <-sem }()
paramsExp := types.SearchParams{
QueryText: q,
VectorThreshold: chatManage.VectorThreshold,
KeywordThreshold: expKwTh,
MatchCount: expTopK,
DisableVectorMatch: true,
DisableKeywordsMatch: false,
}
res, err := p.knowledgeBaseService.HybridSearch(ctx, kbID, paramsExp)
if err != nil {
pipelineWarn(ctx, "Search", "expansion_error", map[string]interface{}{
"kb_id": kbID,
"error": err.Error(),
})
return
}
if len(res) > 0 {
pipelineInfo(ctx, "Search", "expansion_hits", map[string]interface{}{
"kb_id": kbID,
"query": truncateForLog(q),
"hits": len(res),
})
muExp.Lock()
expResults = append(expResults, res...)
muExp.Unlock()
}
}(q, kbID)
}
}
wgExp.Wait()
if len(expResults) > 0 {
pipelineInfo(ctx, "Search", "expansion_done", map[string]interface{}{
"added": len(expResults),
})
chatManage.SearchResult = append(chatManage.SearchResult, expResults...)
}
}
}
// Add relevant results from chat history
historyResult := p.getSearchResultFromHistory(chatManage)
if historyResult != nil {
logger.Infof(ctx, "Add history result, result count: %d", len(historyResult))
pipelineInfo(ctx, "Search", "history_hits", map[string]interface{}{
"session_id": chatManage.SessionID,
"history_hits": len(historyResult),
})
chatManage.SearchResult = append(chatManage.SearchResult, historyResult...)
}
// Remove duplicate results
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
// Remove duplicate results
before := len(chatManage.SearchResult)
chatManage.SearchResult = removeDuplicateResults(chatManage.SearchResult)
pipelineInfo(ctx, "Search", "dedup_summary", map[string]interface{}{
"before": before,
"after": len(chatManage.SearchResult),
})
// Return if we have results
if len(chatManage.SearchResult) != 0 {
logger.Infof(
ctx,
"Get search results, count: %d, session_id: %s",
len(chatManage.SearchResult), chatManage.SessionID,
)
pipelineInfo(ctx, "Search", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"result_count": len(chatManage.SearchResult),
})
return next()
}
logger.Infof(ctx, "No search result, session_id: %s", chatManage.SessionID)
pipelineWarn(ctx, "Search", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"result_count": 0,
})
return ErrSearchNothing
}
@@ -143,15 +249,52 @@ func (p *PluginSearch) getSearchResultFromHistory(chatManage *types.ChatManage)
}
func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult {
seen := make(map[string]bool)
var uniqueResults []*types.SearchResult
for _, result := range results {
if !seen[result.ID] {
seen[result.ID] = true
uniqueResults = append(uniqueResults, result)
}
}
return uniqueResults
seen := make(map[string]bool)
contentSig := make(map[string]bool)
var uniqueResults []*types.SearchResult
for _, r := range results {
keys := []string{r.ID}
if r.ParentChunkID != "" {
keys = append(keys, "parent:"+r.ParentChunkID)
}
if r.KnowledgeID != "" {
keys = append(keys, fmt.Sprintf("kb:%s#%d", r.KnowledgeID, r.ChunkIndex))
}
dup := false
for _, k := range keys {
if seen[k] {
dup = true
break
}
}
if dup {
continue
}
sig := buildContentSignature(r.Content)
if sig != "" {
if contentSig[sig] {
continue
}
contentSig[sig] = true
}
for _, k := range keys {
seen[k] = true
}
uniqueResults = append(uniqueResults, r)
}
return uniqueResults
}
func buildContentSignature(content string) string {
c := strings.ToLower(strings.TrimSpace(content))
if c == "" {
return ""
}
c = strings.Join(strings.Fields(c), " ")
if len(c) > 128 {
c = c[:128]
}
return c
}
// searchKnowledgeBases performs KB searches for rewrite and processed queries across KB IDs
@@ -175,10 +318,19 @@ func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseID
defer wg.Done()
res, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, baseParams)
if err != nil {
logger.Errorf(ctx, "Failed to search KB %s: %v", knowledgeBaseID, err)
pipelineWarn(ctx, "Search", "kb_search_error", map[string]interface{}{
"kb_id": knowledgeBaseID,
"query": baseParams.QueryText,
"error": err.Error(),
"query_ty": "rewrite",
})
return
}
logger.Infof(ctx, "KB %s search results count: %d", knowledgeBaseID, len(res))
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
"kb_id": knowledgeBaseID,
"query_ty": "rewrite",
"hit_count": len(res),
})
mu.Lock()
results = append(results, res...)
mu.Unlock()
@@ -191,7 +343,9 @@ func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseID
if chatManage.RewriteQuery != chatManage.ProcessedQuery {
paramsProcessed := baseParams
paramsProcessed.QueryText = strings.TrimSpace(chatManage.ProcessedQuery)
logger.Infof(ctx, "Searching with processed query: %s", paramsProcessed.QueryText)
pipelineInfo(ctx, "Search", "processed_query_search", map[string]interface{}{
"query": paramsProcessed.QueryText,
})
wg = sync.WaitGroup{}
for _, kbID := range knowledgeBaseIDs {
@@ -200,10 +354,19 @@ func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseID
defer wg.Done()
res, err := p.knowledgeBaseService.HybridSearch(ctx, knowledgeBaseID, paramsProcessed)
if err != nil {
logger.Errorf(ctx, "Failed to search KB %s with processed query: %v", knowledgeBaseID, err)
pipelineWarn(ctx, "Search", "kb_search_error", map[string]interface{}{
"kb_id": knowledgeBaseID,
"query": paramsProcessed.QueryText,
"error": err.Error(),
"query_ty": "processed",
})
return
}
logger.Infof(ctx, "KB %s processed query results count: %d", knowledgeBaseID, len(res))
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
"kb_id": knowledgeBaseID,
"query_ty": "processed",
"hit_count": len(res),
})
mu.Lock()
results = append(results, res...)
mu.Unlock()
@@ -212,7 +375,9 @@ func (p *PluginSearch) searchKnowledgeBases(ctx context.Context, knowledgeBaseID
wg.Wait()
}
logger.Infof(ctx, "Total KB results (rewrite + processed): %d", len(results))
pipelineInfo(ctx, "Search", "kb_result_summary", map[string]interface{}{
"total_hits": len(results),
})
return results
}
@@ -223,14 +388,22 @@ func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types
}
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil || tenant.WebSearchConfig == nil || tenant.WebSearchConfig.Provider == "" {
logger.Warnf(ctx, "Web search enabled but no valid configuration found for tenant %d", chatManage.TenantID)
pipelineWarn(ctx, "Search", "web_config_missing", map[string]interface{}{
"tenant_id": chatManage.TenantID,
})
return nil
}
logger.Infof(ctx, "Performing web search with provider: %s", tenant.WebSearchConfig.Provider)
pipelineInfo(ctx, "Search", "web_request", map[string]interface{}{
"tenant_id": chatManage.TenantID,
"provider": tenant.WebSearchConfig.Provider,
})
webResults, err := p.webSearchService.Search(ctx, tenant.WebSearchConfig, chatManage.RewriteQuery)
if err != nil {
logger.Warnf(ctx, "Web search failed: %v", err)
pipelineWarn(ctx, "Search", "web_search_error", map[string]interface{}{
"tenant_id": chatManage.TenantID,
"error": err.Error(),
})
return nil
}
// Build questions (rewrite + processed if different)
@@ -245,14 +418,18 @@ func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types
p.knowledgeBaseService, p.knowledgeService, seen, ids,
)
if err != nil {
logger.Warnf(ctx, "RAG compression failed, falling back to raw: %v", err)
pipelineWarn(ctx, "Search", "web_compress_error", map[string]interface{}{
"error": err.Error(),
})
} else {
webResults = compressed
// Persist temp KB state back into Redis using SessionService
p.sessionService.SaveWebSearchTempKBState(ctx, chatManage.SessionID, kbID, newSeen, newIDs)
}
res := convertWebSearchResults(webResults)
logger.Infof(ctx, "Web search returned %d results", len(res))
pipelineInfo(ctx, "Search", "web_hits", map[string]interface{}{
"hit_count": len(res),
})
return res
}
@@ -323,3 +500,79 @@ func convertWebSearchResults(webResults []*types.WebSearchResult) []*types.Searc
return results
}
// expandQueries generates paraphrases and synonyms using chat model to improve keyword recall
func (p *PluginSearch) expandQueries(ctx context.Context, chatManage *types.ChatManage) []string {
if p.modelService == nil || chatManage.ChatModelID == "" {
pipelineWarn(ctx, "Search", "expansion_skip", map[string]interface{}{
"reason": "no_model",
})
return nil
}
model, err := p.modelService.GetChatModel(ctx, chatManage.ChatModelID)
if err != nil {
pipelineWarn(ctx, "Search", "expansion_get_model_failed", map[string]interface{}{
"error": err.Error(),
})
return nil
}
sys := "Generate up to 5 diverse paraphrases or keyword variants for the user query to improve keyword-based search recall. Respond ONLY with a JSON array of strings inside a fenced code block."
usr := chatManage.RewriteQuery
think := false
resp, err := model.Chat(ctx, []chat.Message{{Role: "system", Content: sys}, {Role: "user", Content: usr}}, &chat.ChatOptions{Temperature: 0.2, MaxCompletionTokens: 80, Thinking: &think})
if err != nil || resp.Content == "" {
pipelineWarn(ctx, "Search", "expansion_model_call_failed", map[string]interface{}{
"error": err,
})
return nil
}
body := extractJSONBlock(resp.Content)
var arr []string
if err := json.Unmarshal([]byte(body), &arr); err != nil || len(arr) == 0 {
// Fallback: split lines
lines := strings.Split(resp.Content, "\n")
for _, l := range lines {
l = strings.TrimSpace(l)
if l != "" {
arr = append(arr, l)
}
}
}
uniq := make(map[string]struct{})
base := []string{chatManage.Query, chatManage.RewriteQuery, chatManage.ProcessedQuery}
for _, b := range base {
if s := strings.TrimSpace(b); s != "" {
uniq[strings.ToLower(s)] = struct{}{}
}
}
expansions := make([]string, 0, len(arr))
for _, a := range arr {
s := strings.TrimSpace(a)
if s == "" {
continue
}
key := strings.ToLower(s)
if _, ok := uniq[key]; ok {
continue
}
uniq[key] = struct{}{}
expansions = append(expansions, s)
if len(expansions) >= 5 {
break
}
}
pipelineInfo(ctx, "Search", "expansion_result", map[string]interface{}{
"variants": len(expansions),
})
return expansions
}
func extractJSONBlock(text string) string {
t := strings.TrimSpace(text)
if i := strings.Index(t, "["); i >= 0 {
j := strings.LastIndex(t, "]")
if j > i {
return t[i : j+1]
}
}
return "[]"
}
@@ -7,7 +7,6 @@ import (
"strings"
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/google/uuid"
)
@@ -31,11 +30,17 @@ func (p *PluginStreamFilter) ActivationEvents() []types.EventType {
func (p *PluginStreamFilter) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
logger.Info(ctx, "Starting stream filter")
pipelineInfo(ctx, "StreamFilter", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"has_event_bus": chatManage.EventBus != nil,
"no_match_prefix": chatManage.SummaryConfig.NoMatchPrefix,
})
// EventBus is required
if chatManage.EventBus == nil {
logger.Error(ctx, "EventBus is required but not available")
pipelineError(ctx, "StreamFilter", "eventbus_missing", map[string]interface{}{
"session_id": chatManage.SessionID,
})
return ErrModelCall.WithError(errors.New("EventBus is required for stream filtering"))
}
eventBus := chatManage.EventBus
@@ -44,13 +49,17 @@ func (p *PluginStreamFilter) OnEvent(ctx context.Context,
matchNoMatchBuilderPrefix := chatManage.SummaryConfig.NoMatchPrefix != ""
if matchNoMatchBuilderPrefix {
logger.Infof(ctx, "Using no match prefix filter: %s", chatManage.SummaryConfig.NoMatchPrefix)
pipelineInfo(ctx, "StreamFilter", "enable_prefix_filter", map[string]interface{}{
"prefix": chatManage.SummaryConfig.NoMatchPrefix,
})
// Create an event interceptor for prefix filtering
return p.filterEventsWithPrefix(ctx, chatManage, eventBus, next)
}
// No filtering needed, just pass through
logger.Info(ctx, "No prefix filtering required, passing through")
pipelineInfo(ctx, "StreamFilter", "passthrough", map[string]interface{}{
"session_id": chatManage.SessionID,
})
return next()
}
@@ -61,7 +70,9 @@ func (p *PluginStreamFilter) filterEventsWithPrefix(
originalEventBus types.EventBusInterface,
next func() *PluginError,
) *PluginError {
logger.Info(ctx, "Setting up event-based stream filtering with NoMatchPrefix")
pipelineInfo(ctx, "StreamFilter", "setup_temp_bus", map[string]interface{}{
"session_id": chatManage.SessionID,
})
// Create a temporary EventBus to intercept events
tempEventBus := event.NewEventBus()
@@ -81,7 +92,9 @@ func (p *PluginStreamFilter) filterEventsWithPrefix(
// Check if content does NOT match the no-match prefix (meaning it's valid content)
if !strings.HasPrefix(chatManage.SummaryConfig.NoMatchPrefix, responseBuilder.String()) {
logger.Infof(ctx, "Content does not match no-match prefix, emitting valid content: %s", responseBuilder.String())
pipelineInfo(ctx, "StreamFilter", "emit_valid_chunk", map[string]interface{}{
"chunk_len": len(responseBuilder.String()),
})
// Emit the accumulated content as valid answer
originalEventBus.Emit(ctx, types.Event{
@@ -104,7 +117,9 @@ func (p *PluginStreamFilter) filterEventsWithPrefix(
// After pipeline completes, check if we need fallback
if !matchFound && responseBuilder.Len() > 0 {
logger.Info(ctx, "Content matches no-match prefix, emitting fallback response")
pipelineInfo(ctx, "StreamFilter", "emit_fallback", map[string]interface{}{
"session_id": chatManage.SessionID,
})
fallbackID := fmt.Sprintf("%s-fallback", uuid.New().String()[:8])
originalEventBus.Emit(ctx, types.Event{
ID: fallbackID,
@@ -77,7 +77,7 @@ func (p *PluginTracing) OnEvent(ctx context.Context,
// Search traces search operations in the chat pipeline
func (p *PluginTracing) Search(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
_, span := tracing.ContextWithSpan(ctx, "PluginTracing.Search")
defer span.End()
@@ -87,18 +87,23 @@ func (p *PluginTracing) Search(ctx context.Context,
attribute.Float64("keyword_threshold", chatManage.KeywordThreshold),
attribute.Int("match_count", chatManage.EmbeddingTopK),
)
err := next()
searchResultJson, _ := json.Marshal(chatManage.SearchResult)
span.SetAttributes(
attribute.String("hybrid_search", string(searchResultJson)),
attribute.String("processed_query", chatManage.ProcessedQuery),
)
return err
err := next()
searchResultJson, _ := json.Marshal(chatManage.SearchResult)
unique := make(map[string]struct{})
for _, r := range chatManage.SearchResult {
unique[r.ID] = struct{}{}
}
span.SetAttributes(
attribute.String("hybrid_search", string(searchResultJson)),
attribute.String("processed_query", chatManage.ProcessedQuery),
attribute.Int("search_unique_count", len(unique)),
)
return err
}
// Rerank traces rerank operations in the chat pipeline
func (p *PluginTracing) Rerank(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
_, span := tracing.ContextWithSpan(ctx, "PluginTracing.Rerank")
defer span.End()
@@ -109,13 +114,14 @@ func (p *PluginTracing) Rerank(ctx context.Context,
attribute.Float64("rerank_filter_threshold", chatManage.RerankThreshold),
attribute.Int("rerank_filter_topk", chatManage.RerankTopK),
)
err := next()
resultJson, _ := json.Marshal(chatManage.RerankResult)
span.SetAttributes(
attribute.Int("rerank_resp_count", len(chatManage.RerankResult)),
attribute.String("rerank_resp_results", string(resultJson)),
)
return err
err := next()
resultJson, _ := json.Marshal(chatManage.RerankResult)
span.SetAttributes(
attribute.Int("rerank_resp_count", len(chatManage.RerankResult)),
attribute.String("rerank_resp_results", string(resultJson)),
attribute.String("query_intent", chatManage.QueryIntent),
)
return err
}
// Merge traces merge operations in the chat pipeline
+4 -1
View File
@@ -261,6 +261,7 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
VectorThreshold: e.config.Conversation.VectorThreshold,
KeywordThreshold: e.config.Conversation.KeywordThreshold,
EmbeddingTopK: e.config.Conversation.EmbeddingTopK,
MaxRounds: e.config.Conversation.MaxRounds,
RerankModelID: rerankModelID,
RerankTopK: e.config.Conversation.RerankTopK,
RerankThreshold: e.config.Conversation.RerankThreshold,
@@ -279,7 +280,9 @@ func (e *EvaluationService) Evaluation(ctx context.Context,
Seed: e.config.Conversation.Summary.Seed,
MaxCompletionTokens: e.config.Conversation.Summary.MaxCompletionTokens,
},
FallbackResponse: e.config.Conversation.FallbackResponse,
FallbackResponse: e.config.Conversation.FallbackResponse,
RewritePromptSystem: e.config.Conversation.RewritePromptSystem,
RewritePromptUser: e.config.Conversation.RewritePromptUser,
},
}
@@ -372,6 +372,41 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
return nil, err
}
// Normalize keyword retriever scores into [0,1] per-engine batch
for i := range retrieveResults {
rr := retrieveResults[i]
if rr.Error != nil || rr.RetrieverType != types.KeywordsRetrieverType || len(rr.Results) == 0 {
continue
}
minS := rr.Results[0].Score
maxS := rr.Results[0].Score
for _, r := range rr.Results {
if r.Score < minS {
minS = r.Score
}
if r.Score > maxS {
maxS = r.Score
}
}
if maxS > minS {
for _, r := range rr.Results {
ns := (r.Score - minS) / (maxS - minS)
if ns < 0 {
ns = 0
} else if ns > 1 {
ns = 1
}
r.Score = ns
}
logger.Infof(ctx, "Normalized keyword scores for engine %s: min=%f, max=%f", rr.RetrieverEngineType, minS, maxS)
} else {
for _, r := range rr.Results {
r.Score = 1.0
}
logger.Infof(ctx, "Keyword scores have no variance for engine %s, set to 1.0", rr.RetrieverEngineType)
}
}
// Collect all results from different retrievers and deduplicate by chunk ID
logger.Infof(ctx, "Processing retrieval results")
matchResults := []*types.IndexWithScore{}
+113 -39
View File
@@ -436,40 +436,100 @@ func (s *sessionService) KnowledgeQA(ctx context.Context, session *types.Session
return err
}
rewritePromptSystem := s.cfg.Conversation.RewritePromptSystem
rewritePromptUser := s.cfg.Conversation.RewritePromptUser
var tenantConv *types.ConversationConfig
if tc, err := getTenantConversationConfig(ctx); err == nil {
tenantConv = tc
} else {
logger.Warnf(ctx, "Failed to load tenant conversation config, tenant ID: %d, error: %v", session.TenantID, err)
}
vectorThreshold := session.VectorThreshold
keywordThreshold := session.KeywordThreshold
embeddingTopK := session.EmbeddingTopK
rerankModelID := session.RerankModelID
rerankTopK := session.RerankTopK
rerankThreshold := session.RerankThreshold
maxRounds := session.MaxRounds
fallbackResponse := session.FallbackResponse
enableRewrite := session.EnableRewrite
summaryParams := session.SummaryParameters
if summaryParams == nil {
summaryParams = &types.SummaryConfig{}
}
summaryConfig := types.SummaryConfig{
MaxTokens: summaryParams.MaxTokens,
RepeatPenalty: summaryParams.RepeatPenalty,
TopK: summaryParams.TopK,
TopP: summaryParams.TopP,
FrequencyPenalty: summaryParams.FrequencyPenalty,
PresencePenalty: summaryParams.PresencePenalty,
Prompt: summaryParams.Prompt,
ContextTemplate: summaryParams.ContextTemplate,
Temperature: summaryParams.Temperature,
Seed: summaryParams.Seed,
NoMatchPrefix: summaryParams.NoMatchPrefix,
MaxCompletionTokens: summaryParams.MaxCompletionTokens,
}
if tenantConv != nil {
vectorThreshold = tenantConv.VectorThreshold
keywordThreshold = tenantConv.KeywordThreshold
embeddingTopK = tenantConv.EmbeddingTopK
rerankModelID = tenantConv.RerankModelID
rerankTopK = tenantConv.RerankTopK
rerankThreshold = tenantConv.RerankThreshold
maxRounds = tenantConv.MaxRounds
fallbackResponse = tenantConv.FallbackResponse
enableRewrite = tenantConv.EnableRewrite
if tenantConv.MaxTokens != 0 {
summaryConfig.MaxTokens = tenantConv.MaxTokens
}
if tenantConv.Prompt != "" {
summaryConfig.Prompt = tenantConv.Prompt
}
if tenantConv.ContextTemplate != "" {
summaryConfig.ContextTemplate = tenantConv.ContextTemplate
}
if tenantConv.Temperature != 0 {
summaryConfig.Temperature = tenantConv.Temperature
}
if tenantConv.RewritePromptSystem != "" {
rewritePromptSystem = tenantConv.RewritePromptSystem
}
if tenantConv.RewritePromptUser != "" {
rewritePromptUser = tenantConv.RewritePromptUser
}
}
// Create chat management object with session settings
logger.Infof(ctx, "Creating chat manage object, knowledge base IDs: %v, chat model ID: %s", knowledgeBaseIDs, chatModelID)
chatManage := &types.ChatManage{
Query: query,
RewriteQuery: query,
SessionID: session.ID,
MessageID: assistantMessageID, // NEW: For event emission in pipeline
KnowledgeBaseID: knowledgeBaseIDs[0], // For backward compatibility, use first KB ID
KnowledgeBaseIDs: knowledgeBaseIDs, // Multi-KB support
VectorThreshold: session.VectorThreshold,
KeywordThreshold: session.KeywordThreshold,
EmbeddingTopK: session.EmbeddingTopK,
RerankModelID: session.RerankModelID,
RerankTopK: session.RerankTopK,
RerankThreshold: session.RerankThreshold,
ChatModelID: chatModelID,
SummaryConfig: types.SummaryConfig{
MaxTokens: session.SummaryParameters.MaxTokens,
RepeatPenalty: session.SummaryParameters.RepeatPenalty,
TopK: session.SummaryParameters.TopK,
TopP: session.SummaryParameters.TopP,
FrequencyPenalty: session.SummaryParameters.FrequencyPenalty,
PresencePenalty: session.SummaryParameters.PresencePenalty,
Prompt: session.SummaryParameters.Prompt,
ContextTemplate: session.SummaryParameters.ContextTemplate,
Temperature: session.SummaryParameters.Temperature,
Seed: session.SummaryParameters.Seed,
NoMatchPrefix: session.SummaryParameters.NoMatchPrefix,
MaxCompletionTokens: session.SummaryParameters.MaxCompletionTokens,
},
FallbackResponse: session.FallbackResponse,
EventBus: eventBus.AsEventBusInterface(), // NEW: For pipeline to emit events directly
WebSearchEnabled: webSearchEnabled,
TenantID: session.TenantID,
Query: query,
RewriteQuery: query,
SessionID: session.ID,
MessageID: assistantMessageID, // NEW: For event emission in pipeline
KnowledgeBaseID: knowledgeBaseIDs[0], // For backward compatibility, use first KB ID
KnowledgeBaseIDs: knowledgeBaseIDs, // Multi-KB support
VectorThreshold: vectorThreshold,
KeywordThreshold: keywordThreshold,
EmbeddingTopK: embeddingTopK,
RerankModelID: rerankModelID,
RerankTopK: rerankTopK,
RerankThreshold: rerankThreshold,
MaxRounds: maxRounds,
ChatModelID: chatModelID,
SummaryConfig: summaryConfig,
FallbackResponse: fallbackResponse,
EventBus: eventBus.AsEventBusInterface(), // NEW: For pipeline to emit events directly
WebSearchEnabled: webSearchEnabled,
TenantID: session.TenantID,
RewritePromptSystem: rewritePromptSystem,
RewritePromptUser: rewritePromptUser,
EnableRewrite: enableRewrite,
}
// Start knowledge QA event processing
@@ -676,6 +736,17 @@ func (s *sessionService) KnowledgeQAByEvent(ctx context.Context,
return nil
}
func getTenantConversationConfig(ctx context.Context) (*types.ConversationConfig, error) {
tenant := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
return nil, errors.New("tenant is empty")
}
if tenant.ConversationConfig == nil {
return nil, errors.New("tenant has no conversation config")
}
return tenant.ConversationConfig, nil
}
// SearchKnowledge performs knowledge base search without LLM summarization
func (s *sessionService) SearchKnowledge(ctx context.Context,
knowledgeBaseID, query string,
@@ -685,14 +756,17 @@ func (s *sessionService) SearchKnowledge(ctx context.Context,
// Create default retrieval parameters
chatManage := &types.ChatManage{
Query: query,
RewriteQuery: query,
KnowledgeBaseID: knowledgeBaseID,
VectorThreshold: s.cfg.Conversation.VectorThreshold, // Use default configuration
KeywordThreshold: s.cfg.Conversation.KeywordThreshold, // Use default configuration
EmbeddingTopK: s.cfg.Conversation.EmbeddingTopK, // Use default configuration
RerankTopK: s.cfg.Conversation.RerankTopK, // Use default configuration
RerankThreshold: s.cfg.Conversation.RerankThreshold, // Use default configuration
Query: query,
RewriteQuery: query,
KnowledgeBaseID: knowledgeBaseID,
VectorThreshold: s.cfg.Conversation.VectorThreshold, // Use default configuration
KeywordThreshold: s.cfg.Conversation.KeywordThreshold, // Use default configuration
EmbeddingTopK: s.cfg.Conversation.EmbeddingTopK, // Use default configuration
RerankTopK: s.cfg.Conversation.RerankTopK, // Use default configuration
RerankThreshold: s.cfg.Conversation.RerankThreshold, // Use default configuration
MaxRounds: s.cfg.Conversation.MaxRounds,
RewritePromptSystem: s.cfg.Conversation.RewritePromptSystem,
RewritePromptUser: s.cfg.Conversation.RewritePromptUser,
}
// Get default models