feat: enhance multimodal support and image handling
- Updated configuration to improve handling of multimodal queries, including clearer guidelines for when to use [NO_SEARCH] in responses. - Enhanced the user interface to include dedicated sections for multimodal configuration, allowing users to manage image uploads and related settings more effectively. - Added localization support for new multimodal features in both English and Chinese, improving accessibility for a broader audience. - Implemented backend changes to support image analysis and extraction of OCR text, enriching the interaction experience for users. These enhancements significantly improve the application's capability to process and respond to multimodal queries, fostering a more interactive and user-friendly environment.
This commit is contained in:
@@ -233,10 +233,19 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
|
||||
|
||||
logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval found %d results", len(embeddingDBList))
|
||||
results := make([]*types.IndexWithScore, len(embeddingDBList))
|
||||
const maxKeywordResultLog = 8
|
||||
for i := range embeddingDBList {
|
||||
results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeKeywords)
|
||||
logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f",
|
||||
i, results[i].ChunkID, results[i].Score)
|
||||
if i < maxKeywordResultLog {
|
||||
logger.GetLogger(ctx).Debugf("[Postgres] Keywords result %d: chunk=%s, score=%f",
|
||||
i, results[i].ChunkID, results[i].Score)
|
||||
}
|
||||
}
|
||||
if len(results) > maxKeywordResultLog {
|
||||
logger.GetLogger(ctx).Debugf(
|
||||
"[Postgres] Keywords result summary: total=%d logged=%d truncated=%d",
|
||||
len(results), maxKeywordResultLog, len(results)-maxKeywordResultLog,
|
||||
)
|
||||
}
|
||||
return []*types.RetrieveResult{
|
||||
{
|
||||
@@ -387,10 +396,19 @@ func (g *pgRepository) VectorRetrieve(ctx context.Context,
|
||||
|
||||
logger.GetLogger(ctx).Infof("[Postgres] Vector retrieval found %d results", len(embeddingDBList))
|
||||
results := make([]*types.IndexWithScore, len(embeddingDBList))
|
||||
const maxVectorResultLog = 8
|
||||
for i := range embeddingDBList {
|
||||
results[i] = fromDBVectorEmbeddingWithScore(&embeddingDBList[i], types.MatchTypeEmbedding)
|
||||
logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f",
|
||||
i, results[i].ChunkID, results[i].Score)
|
||||
if i < maxVectorResultLog {
|
||||
logger.GetLogger(ctx).Debugf("[Postgres] Vector search result %d: chunk_id %s, score %.4f",
|
||||
i, results[i].ChunkID, results[i].Score)
|
||||
}
|
||||
}
|
||||
if len(results) > maxVectorResultLog {
|
||||
logger.GetLogger(ctx).Debugf(
|
||||
"[Postgres] Vector search result summary: total=%d logged=%d truncated=%d",
|
||||
len(results), maxVectorResultLog, len(results)-maxVectorResultLog,
|
||||
)
|
||||
}
|
||||
return []*types.RetrieveResult{
|
||||
{
|
||||
@@ -494,7 +512,7 @@ func (g *pgRepository) CopyIndices(ctx context.Context,
|
||||
// Create new vector index, copy the content and vector of the source index
|
||||
targetVector := &pgVector{
|
||||
Content: sourceVector.Content,
|
||||
SourceID: targetSourceID, // Handle SourceID transformation properly
|
||||
SourceID: targetSourceID, // Handle SourceID transformation properly
|
||||
SourceType: sourceVector.SourceType,
|
||||
ChunkID: targetChunkID, // Update to target chunk ID
|
||||
KnowledgeID: targetKnowledgeID, // Update to target knowledge ID
|
||||
|
||||
@@ -77,6 +77,30 @@ func (p *PluginIntoChatMessage) OnEvent(ctx context.Context,
|
||||
// Prepare weekday names
|
||||
weekdayName := []string{"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}
|
||||
|
||||
// Intent-based no-search path: bypass "reference materials" template entirely.
|
||||
if chatManage.SkipKBSearch {
|
||||
// Prefer rewritten query in no-search mode; fallback to original query.
|
||||
userContent := safeQuery
|
||||
if rewrite := strings.TrimSpace(chatManage.RewriteQuery); rewrite != "" {
|
||||
if safeRewrite, ok := utils.ValidateInput(rewrite); ok {
|
||||
userContent = safeRewrite
|
||||
} else {
|
||||
pipelineWarn(ctx, "IntoChatMessage", "invalid_rewrite_query_fallback", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
})
|
||||
}
|
||||
}
|
||||
if chatManage.ImageOCRText != "" && !chatManage.ChatModelSupportsVision {
|
||||
userContent += "\n\n[用户上传图片内容]\n" + chatManage.ImageOCRText
|
||||
}
|
||||
chatManage.UserContent = userContent
|
||||
pipelineInfo(ctx, "IntoChatMessage", "skip_template_no_search", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"user_content_len": len(chatManage.UserContent),
|
||||
})
|
||||
return next()
|
||||
}
|
||||
|
||||
var contextsBuilder strings.Builder
|
||||
|
||||
// Build contexts string based on FAQ priority strategy
|
||||
|
||||
@@ -121,15 +121,7 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
|
||||
"result_cnt": len(rerankResp),
|
||||
})
|
||||
|
||||
// Log input scores before reranking for debugging
|
||||
for i, sr := range chatManage.SearchResult {
|
||||
pipelineInfo(ctx, "Rerank", "input_score", map[string]interface{}{
|
||||
"index": i,
|
||||
"chunk_id": sr.ID,
|
||||
"score": fmt.Sprintf("%.4f", sr.Score),
|
||||
"match_type": sr.MatchType,
|
||||
})
|
||||
}
|
||||
logRerankInputScoreSample(ctx, chatManage.SearchResult)
|
||||
|
||||
for i := range chatManage.SearchResult {
|
||||
chatManage.SearchResult[i].Metadata = ensureMetadata(chatManage.SearchResult[i].Metadata)
|
||||
@@ -162,13 +154,6 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
|
||||
})
|
||||
}
|
||||
|
||||
pipelineInfo(ctx, "Rerank", "composite_calc", map[string]interface{}{
|
||||
"chunk_id": sr.ID,
|
||||
"base_score": fmt.Sprintf("%.4f", base),
|
||||
"model_score": fmt.Sprintf("%.4f", modelScore),
|
||||
"final_score": fmt.Sprintf("%.4f", sr.Score),
|
||||
"match_type": sr.MatchType,
|
||||
})
|
||||
reranked = append(reranked, sr)
|
||||
}
|
||||
|
||||
@@ -179,13 +164,6 @@ func (p *PluginRerank) OnEvent(ctx context.Context,
|
||||
// Assign high model score for direct load items
|
||||
modelScore := 1.0
|
||||
sr.Score = compositeScore(sr, modelScore, base)
|
||||
pipelineInfo(ctx, "Rerank", "composite_calc_direct", map[string]interface{}{
|
||||
"chunk_id": sr.ID,
|
||||
"base_score": fmt.Sprintf("%.4f", base),
|
||||
"model_score": fmt.Sprintf("%.4f", modelScore),
|
||||
"final_score": fmt.Sprintf("%.4f", sr.Score),
|
||||
"match_type": sr.MatchType,
|
||||
})
|
||||
reranked = append(reranked, sr)
|
||||
}
|
||||
final := applyMMR(ctx, reranked, chatManage, min(len(reranked), max(1, chatManage.RerankTopK)), 0.7)
|
||||
@@ -237,18 +215,26 @@ func (p *PluginRerank) rerank(ctx context.Context,
|
||||
pipelineInfo(ctx, "Rerank", "threshold", map[string]interface{}{
|
||||
"threshold": chatManage.RerankThreshold,
|
||||
})
|
||||
for i := range min(5, len(rerankResp)) {
|
||||
logged := min(5, len(rerankResp))
|
||||
for i := range logged {
|
||||
if rerankResp[i].Index < len(candidates) {
|
||||
pipelineInfo(ctx, "Rerank", "top_score", map[string]interface{}{
|
||||
"rank": i + 1,
|
||||
"score": rerankResp[i].RelevanceScore,
|
||||
"chunk_id": candidates[rerankResp[i].Index].ID,
|
||||
"match_type": candidates[rerankResp[i].Index].MatchType,
|
||||
"chunk_type": candidates[rerankResp[i].Index].ChunkType,
|
||||
"content": candidates[rerankResp[i].Index].Content,
|
||||
"rank": i + 1,
|
||||
"score": rerankResp[i].RelevanceScore,
|
||||
"chunk_id": candidates[rerankResp[i].Index].ID,
|
||||
"match_type": candidates[rerankResp[i].Index].MatchType,
|
||||
"chunk_type": candidates[rerankResp[i].Index].ChunkType,
|
||||
"content_len": len(candidates[rerankResp[i].Index].Content),
|
||||
})
|
||||
}
|
||||
}
|
||||
if len(rerankResp) > logged {
|
||||
pipelineInfo(ctx, "Rerank", "top_score_summary", map[string]interface{}{
|
||||
"total": len(rerankResp),
|
||||
"logged": logged,
|
||||
"truncated": len(rerankResp) - logged,
|
||||
})
|
||||
}
|
||||
|
||||
// Filter results based on threshold
|
||||
rankFilter := []rerank.RankResult{}
|
||||
@@ -529,11 +515,26 @@ func getEnrichedPassage(ctx context.Context, result *types.SearchResult) string
|
||||
}
|
||||
combinedText += strings.Join(enrichments, "\n")
|
||||
|
||||
pipelineInfo(ctx, "Rerank", "passage_enrich", map[string]interface{}{
|
||||
"content_len": len(result.Content),
|
||||
"cleaned_len": len(cleanPassageForRerank(result.Content)),
|
||||
"enrichment_len": len(strings.Join(enrichments, "\n")),
|
||||
})
|
||||
|
||||
return combinedText
|
||||
}
|
||||
|
||||
func logRerankInputScoreSample(ctx context.Context, results []*types.SearchResult) {
|
||||
const maxLogRows = 8
|
||||
limit := min(maxLogRows, len(results))
|
||||
for i := 0; i < limit; i++ {
|
||||
sr := results[i]
|
||||
pipelineInfo(ctx, "Rerank", "input_score", map[string]interface{}{
|
||||
"index": i,
|
||||
"chunk_id": sr.ID,
|
||||
"score": fmt.Sprintf("%.4f", sr.Score),
|
||||
"match_type": sr.MatchType,
|
||||
})
|
||||
}
|
||||
if len(results) > limit {
|
||||
pipelineInfo(ctx, "Rerank", "input_score_summary", map[string]interface{}{
|
||||
"total": len(results),
|
||||
"logged": limit,
|
||||
"truncated": len(results) - limit,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,9 +11,11 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
"github.com/Tencent/WeKnora/internal/event"
|
||||
"github.com/Tencent/WeKnora/internal/models/chat"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// PluginRewrite is a plugin for rewriting user queries
|
||||
@@ -26,10 +28,10 @@ type PluginRewrite struct {
|
||||
|
||||
// reg is a regular expression used to match and remove content between <think></think> tags
|
||||
var reg = regexp.MustCompile(`(?s)<think>.*?</think>`)
|
||||
var rewriteImageSepPattern = regexp.MustCompile(`(?s)^(.*?)\s*\n?---\n(.*)$`)
|
||||
|
||||
const (
|
||||
noSearchPrefix = "[NO_SEARCH]"
|
||||
imageDescSeparator = "\n---\n"
|
||||
noSearchPrefix = "[NO_SEARCH]"
|
||||
)
|
||||
|
||||
// NewPluginRewrite creates a new query rewriting plugin instance
|
||||
@@ -118,8 +120,23 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
maxTokens = 500
|
||||
}
|
||||
|
||||
// --- Emit progress event for image analysis ---
|
||||
var toolCallID string
|
||||
if useImages && chatManage.EventBus != nil {
|
||||
toolCallID = uuid.New().String()
|
||||
chatManage.EventBus.Emit(ctx, types.Event{
|
||||
Type: types.EventType(event.EventAgentToolCall),
|
||||
SessionID: chatManage.SessionID,
|
||||
Data: event.AgentToolCallData{
|
||||
ToolCallID: toolCallID,
|
||||
ToolName: "image_analysis",
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// --- Call model ---
|
||||
thinking := false
|
||||
vlmStart := time.Now()
|
||||
response, err := rewriteModel.Chat(ctx, []chat.Message{
|
||||
{Role: "system", Content: systemContent},
|
||||
userMsg,
|
||||
@@ -129,6 +146,19 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
Thinking: &thinking,
|
||||
})
|
||||
if err != nil {
|
||||
if toolCallID != "" && chatManage.EventBus != nil {
|
||||
chatManage.EventBus.Emit(ctx, types.Event{
|
||||
Type: types.EventType(event.EventAgentToolResult),
|
||||
SessionID: chatManage.SessionID,
|
||||
Data: event.AgentToolResultData{
|
||||
ToolCallID: toolCallID,
|
||||
ToolName: "image_analysis",
|
||||
Output: "图片分析失败",
|
||||
Success: false,
|
||||
Duration: time.Since(vlmStart).Milliseconds(),
|
||||
},
|
||||
})
|
||||
}
|
||||
pipelineError(ctx, "Rewrite", "model_call", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"error": err.Error(),
|
||||
@@ -136,9 +166,23 @@ func (p *PluginRewrite) OnEvent(ctx context.Context,
|
||||
return next()
|
||||
}
|
||||
|
||||
// --- Emit completion event for image analysis ---
|
||||
if toolCallID != "" && chatManage.EventBus != nil {
|
||||
chatManage.EventBus.Emit(ctx, types.Event{
|
||||
Type: types.EventType(event.EventAgentToolResult),
|
||||
SessionID: chatManage.SessionID,
|
||||
Data: event.AgentToolResultData{
|
||||
ToolCallID: toolCallID,
|
||||
ToolName: "image_analysis",
|
||||
Output: "已分析图片内容",
|
||||
Success: true,
|
||||
Duration: time.Since(vlmStart).Milliseconds(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// --- Parse structured output ---
|
||||
p.parseRewriteOutput(chatManage, response.Content)
|
||||
|
||||
pipelineInfo(ctx, "Rewrite", "output", map[string]interface{}{
|
||||
"session_id": chatManage.SessionID,
|
||||
"rewrite_query": chatManage.RewriteQuery,
|
||||
@@ -261,6 +305,13 @@ func (p *PluginRewrite) buildPrompts(chatManage *types.ChatManage, historyList [
|
||||
if chatManage.RewritePromptSystem != "" {
|
||||
systemPrompt = chatManage.RewritePromptSystem
|
||||
}
|
||||
// Strengthen context inheritance in multi-turn conversation:
|
||||
// for follow-up questions that clearly refer to previous turns (especially
|
||||
// uploaded-image understanding), prefer NO_SEARCH over KB retrieval.
|
||||
systemPrompt += "\n\n## Additional Context Inheritance Guidance\n" +
|
||||
"- If the current question is a follow-up to previous conversation content (especially previously uploaded images) and can be answered by that context, you MUST classify it as NO_SEARCH.\n" +
|
||||
"- Examples: “第一张图再详细描述一下”, “第二张门上的字是什么意思”, “这个再展开讲讲”.\n" +
|
||||
"- In these follow-up cases, output with [NO_SEARCH] prefix."
|
||||
|
||||
conversationText := formatConversationHistory(historyList)
|
||||
currentTime := time.Now().Format("2006-01-02 15:04:05")
|
||||
@@ -296,11 +347,16 @@ func (p *PluginRewrite) parseRewriteOutput(chatManage *types.ChatManage, raw str
|
||||
content = strings.TrimSpace(strings.TrimPrefix(content, noSearchPrefix))
|
||||
}
|
||||
|
||||
// 2. Split rewritten query and image description
|
||||
if idx := strings.Index(content, imageDescSeparator); idx >= 0 {
|
||||
chatManage.RewriteQuery = strings.TrimSpace(content[:idx])
|
||||
chatManage.ImageOCRText = strings.TrimSpace(content[idx+len(imageDescSeparator):])
|
||||
} else if content != "" {
|
||||
// 2. Split rewritten query and image description.
|
||||
// Be tolerant to model output variants like:
|
||||
// - "query\n---\nimage_desc" (expected)
|
||||
// - "query---\nimage_desc" (missing leading newline before separator)
|
||||
if m := rewriteImageSepPattern.FindStringSubmatch(content); len(m) == 3 {
|
||||
chatManage.RewriteQuery = strings.TrimSpace(m[1])
|
||||
chatManage.ImageOCRText = strings.TrimSpace(m[2])
|
||||
return
|
||||
}
|
||||
if content != "" {
|
||||
chatManage.RewriteQuery = content
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,15 +113,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
|
||||
chatManage.SearchResult = allResults
|
||||
|
||||
// Log all search results with scores before any processing
|
||||
for i, r := range chatManage.SearchResult {
|
||||
pipelineInfo(ctx, "Search", "result_score_before_normalize", map[string]interface{}{
|
||||
"index": i,
|
||||
"chunk_id": r.ID,
|
||||
"score": fmt.Sprintf("%.4f", r.Score),
|
||||
"match_type": r.MatchType,
|
||||
})
|
||||
}
|
||||
logSearchScoreSample(ctx, "result_score_before_normalize", chatManage.SearchResult)
|
||||
|
||||
// If recall is low, attempt query expansion with keyword-focused search
|
||||
if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK) {
|
||||
@@ -131,15 +123,7 @@ func (p *PluginSearch) OnEvent(ctx context.Context,
|
||||
}
|
||||
}
|
||||
|
||||
// Log final scores after all processing
|
||||
for i, r := range chatManage.SearchResult {
|
||||
pipelineInfo(ctx, "Search", "final_score", map[string]interface{}{
|
||||
"index": i,
|
||||
"chunk_id": r.ID,
|
||||
"score": fmt.Sprintf("%.4f", r.Score),
|
||||
"match_type": r.MatchType,
|
||||
})
|
||||
}
|
||||
logSearchScoreSample(ctx, "final_score", chatManage.SearchResult)
|
||||
|
||||
// Return if we have results
|
||||
if len(chatManage.SearchResult) != 0 {
|
||||
@@ -204,6 +188,27 @@ func buildContentSignature(content string) string {
|
||||
return searchutil.BuildContentSignature(content)
|
||||
}
|
||||
|
||||
func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) {
|
||||
const maxLogRows = 8
|
||||
limit := min(maxLogRows, len(results))
|
||||
for i := 0; i < limit; i++ {
|
||||
r := results[i]
|
||||
pipelineInfo(ctx, "Search", action, map[string]interface{}{
|
||||
"index": i,
|
||||
"chunk_id": r.ID,
|
||||
"score": fmt.Sprintf("%.4f", r.Score),
|
||||
"match_type": r.MatchType,
|
||||
})
|
||||
}
|
||||
if len(results) > limit {
|
||||
pipelineInfo(ctx, "Search", action+"_summary", map[string]interface{}{
|
||||
"total": len(results),
|
||||
"logged": limit,
|
||||
"truncated": len(results) - limit,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// searchByTargets performs KB searches using pre-computed SearchTargets
|
||||
// This is the main search method that uses the unified search targets
|
||||
func (p *PluginSearch) searchByTargets(
|
||||
|
||||
@@ -1231,6 +1231,9 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
|
||||
// Build final search results - preserve original order from input chunks
|
||||
var searchResults []*types.SearchResult
|
||||
addedChunkIDs := make(map[string]bool)
|
||||
const maxInvalidChunkLog = 8
|
||||
invalidChunkCnt := 0
|
||||
invalidChunkSamples := make([]string, 0, maxInvalidChunkLog)
|
||||
|
||||
// First pass: Add results in the original order from input chunks
|
||||
for _, inputChunk := range chunks {
|
||||
@@ -1240,7 +1243,10 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
|
||||
continue
|
||||
}
|
||||
if !s.isValidTextChunk(chunk) {
|
||||
logger.Debugf(ctx, "Chunk is not valid text chunk: %s, type: %s", chunk.ID, chunk.ChunkType)
|
||||
invalidChunkCnt++
|
||||
if len(invalidChunkSamples) < maxInvalidChunkLog {
|
||||
invalidChunkSamples = append(invalidChunkSamples, chunk.ID+":"+chunk.ChunkType)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if addedChunkIDs[chunk.ID] {
|
||||
@@ -1257,6 +1263,12 @@ func (s *knowledgeBaseService) processSearchResults(ctx context.Context,
|
||||
logger.Warnf(ctx, "Knowledge not found for chunk: %s, knowledge_id: %s", chunk.ID, chunk.KnowledgeID)
|
||||
}
|
||||
}
|
||||
if invalidChunkCnt > 0 {
|
||||
logger.Debugf(ctx,
|
||||
"Skip non-text chunks in search results: total=%d sampled=%d samples=%v",
|
||||
invalidChunkCnt, len(invalidChunkSamples), invalidChunkSamples,
|
||||
)
|
||||
}
|
||||
|
||||
// Second pass: Add additional chunks (parent, nearby, relation) that weren't in original input
|
||||
if !skipEnrichment {
|
||||
|
||||
Reference in New Issue
Block a user