5fe1ccd4f9
- Updated configuration to improve handling of multimodal queries, including clearer guidelines for when to use [NO_SEARCH] in responses. - Enhanced the user interface to include dedicated sections for multimodal configuration, allowing users to manage image uploads and related settings more effectively. - Added localization support for new multimodal features in both English and Chinese, improving accessibility for a broader audience. - Implemented backend changes to support image analysis and extraction of OCR text, enriching the interaction experience for users. These enhancements significantly improve the application's capability to process and respond to multimodal queries, fostering a more interactive and user-friendly environment.
447 lines
14 KiB
Go
447 lines
14 KiB
Go
package chatpipline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/Tencent/WeKnora/internal/config"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/searchutil"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
)
|
|
|
|
// PluginSearch implements search functionality for chat pipeline
|
|
type PluginSearch struct {
|
|
knowledgeBaseService interfaces.KnowledgeBaseService
|
|
knowledgeService interfaces.KnowledgeService
|
|
chunkService interfaces.ChunkService
|
|
config *config.Config
|
|
webSearchService interfaces.WebSearchService
|
|
tenantService interfaces.TenantService
|
|
sessionService interfaces.SessionService
|
|
webSearchStateService interfaces.WebSearchStateService
|
|
}
|
|
|
|
func NewPluginSearch(eventManager *EventManager,
|
|
knowledgeBaseService interfaces.KnowledgeBaseService,
|
|
knowledgeService interfaces.KnowledgeService,
|
|
chunkService interfaces.ChunkService,
|
|
config *config.Config,
|
|
webSearchService interfaces.WebSearchService,
|
|
tenantService interfaces.TenantService,
|
|
sessionService interfaces.SessionService,
|
|
webSearchStateService interfaces.WebSearchStateService,
|
|
) *PluginSearch {
|
|
res := &PluginSearch{
|
|
knowledgeBaseService: knowledgeBaseService,
|
|
knowledgeService: knowledgeService,
|
|
chunkService: chunkService,
|
|
config: config,
|
|
webSearchService: webSearchService,
|
|
tenantService: tenantService,
|
|
sessionService: sessionService,
|
|
webSearchStateService: webSearchStateService,
|
|
}
|
|
eventManager.Register(res)
|
|
return res
|
|
}
|
|
|
|
// ActivationEvents returns the event types this plugin handles
|
|
func (p *PluginSearch) ActivationEvents() []types.EventType {
|
|
return []types.EventType{types.CHUNK_SEARCH}
|
|
}
|
|
|
|
// OnEvent handles search events in the chat pipeline
|
|
func (p *PluginSearch) OnEvent(ctx context.Context,
|
|
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
|
) *PluginError {
|
|
// Check if we have search targets or web search enabled
|
|
hasKBTargets := len(chatManage.SearchTargets) > 0 || len(chatManage.KnowledgeBaseIDs) > 0 || len(chatManage.KnowledgeIDs) > 0
|
|
if !hasKBTargets && !chatManage.WebSearchEnabled {
|
|
pipelineError(ctx, "Search", "kb_not_found", map[string]interface{}{
|
|
"session_id": chatManage.SessionID,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
pipelineInfo(ctx, "Search", "input", map[string]interface{}{
|
|
"session_id": chatManage.SessionID,
|
|
"rewrite_query": chatManage.RewriteQuery,
|
|
"search_targets": len(chatManage.SearchTargets),
|
|
"tenant_id": chatManage.TenantID,
|
|
"web_enabled": chatManage.WebSearchEnabled,
|
|
})
|
|
|
|
// Run KB search and web search concurrently
|
|
pipelineInfo(ctx, "Search", "plan", map[string]interface{}{
|
|
"search_targets": len(chatManage.SearchTargets),
|
|
"embedding_top_k": chatManage.EmbeddingTopK,
|
|
"vector_threshold": chatManage.VectorThreshold,
|
|
"keyword_threshold": chatManage.KeywordThreshold,
|
|
})
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex
|
|
allResults := make([]*types.SearchResult, 0)
|
|
|
|
wg.Add(2)
|
|
// Goroutine 1: Knowledge base search using SearchTargets
|
|
go func() {
|
|
defer wg.Done()
|
|
kbResults := p.searchByTargets(ctx, chatManage)
|
|
if len(kbResults) > 0 {
|
|
mu.Lock()
|
|
allResults = append(allResults, kbResults...)
|
|
mu.Unlock()
|
|
}
|
|
}()
|
|
|
|
// Goroutine 2: Web search (if enabled)
|
|
go func() {
|
|
defer wg.Done()
|
|
webResults := p.searchWebIfEnabled(ctx, chatManage)
|
|
if len(webResults) > 0 {
|
|
mu.Lock()
|
|
allResults = append(allResults, webResults...)
|
|
mu.Unlock()
|
|
}
|
|
}()
|
|
|
|
wg.Wait()
|
|
|
|
chatManage.SearchResult = allResults
|
|
|
|
logSearchScoreSample(ctx, "result_score_before_normalize", chatManage.SearchResult)
|
|
|
|
// If recall is low, attempt query expansion with keyword-focused search
|
|
if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK) {
|
|
expResults := p.runQueryExpansion(ctx, chatManage)
|
|
if len(expResults) > 0 {
|
|
chatManage.SearchResult = append(chatManage.SearchResult, expResults...)
|
|
}
|
|
}
|
|
|
|
logSearchScoreSample(ctx, "final_score", chatManage.SearchResult)
|
|
|
|
// Return if we have results
|
|
if len(chatManage.SearchResult) != 0 {
|
|
pipelineInfo(ctx, "Search", "output", map[string]interface{}{
|
|
"session_id": chatManage.SessionID,
|
|
"result_count": len(chatManage.SearchResult),
|
|
})
|
|
return next()
|
|
}
|
|
pipelineWarn(ctx, "Search", "output", map[string]interface{}{
|
|
"session_id": chatManage.SessionID,
|
|
"result_count": 0,
|
|
})
|
|
return ErrSearchNothing
|
|
}
|
|
|
|
// getSearchResultFromHistory retrieves relevant knowledge references from chat history
|
|
func getSearchResultFromHistory(chatManage *types.ChatManage) []*types.SearchResult {
|
|
if len(chatManage.History) == 0 {
|
|
return nil
|
|
}
|
|
// Search history in reverse chronological order
|
|
for i := len(chatManage.History) - 1; i >= 0; i-- {
|
|
if len(chatManage.History[i].KnowledgeReferences) > 0 {
|
|
// Mark all references as history matches
|
|
for _, reference := range chatManage.History[i].KnowledgeReferences {
|
|
reference.MatchType = types.MatchTypeHistory
|
|
}
|
|
return chatManage.History[i].KnowledgeReferences
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult {
|
|
seen := make(map[string]bool)
|
|
contentSig := make(map[string]string) // sig -> first chunk ID
|
|
var uniqueResults []*types.SearchResult
|
|
for _, r := range results {
|
|
// Only deduplicate by exact chunk ID — do NOT treat shared ParentChunkID
|
|
// as duplicates, because different child chunks of the same parent carry
|
|
// different content segments that may all be relevant.
|
|
if seen[r.ID] {
|
|
logger.Debugf(context.Background(), "Dedup: chunk %s removed due to duplicate ID", r.ID)
|
|
continue
|
|
}
|
|
sig := buildContentSignature(r.Content)
|
|
if sig != "" {
|
|
if firstChunk, exists := contentSig[sig]; exists {
|
|
logger.Debugf(context.Background(), "Dedup: chunk %s removed due to content signature (dup of %s, sig prefix: %.50s...)", r.ID, firstChunk, sig)
|
|
continue
|
|
}
|
|
contentSig[sig] = r.ID
|
|
}
|
|
seen[r.ID] = true
|
|
uniqueResults = append(uniqueResults, r)
|
|
}
|
|
return uniqueResults
|
|
}
|
|
|
|
func buildContentSignature(content string) string {
|
|
return searchutil.BuildContentSignature(content)
|
|
}
|
|
|
|
func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) {
|
|
const maxLogRows = 8
|
|
limit := min(maxLogRows, len(results))
|
|
for i := 0; i < limit; i++ {
|
|
r := results[i]
|
|
pipelineInfo(ctx, "Search", action, map[string]interface{}{
|
|
"index": i,
|
|
"chunk_id": r.ID,
|
|
"score": fmt.Sprintf("%.4f", r.Score),
|
|
"match_type": r.MatchType,
|
|
})
|
|
}
|
|
if len(results) > limit {
|
|
pipelineInfo(ctx, "Search", action+"_summary", map[string]interface{}{
|
|
"total": len(results),
|
|
"logged": limit,
|
|
"truncated": len(results) - limit,
|
|
})
|
|
}
|
|
}
|
|
|
|
// searchByTargets performs KB searches using pre-computed SearchTargets
|
|
// This is the main search method that uses the unified search targets
|
|
func (p *PluginSearch) searchByTargets(
|
|
ctx context.Context,
|
|
chatManage *types.ChatManage,
|
|
) []*types.SearchResult {
|
|
if len(chatManage.SearchTargets) == 0 {
|
|
return nil
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex
|
|
var results []*types.SearchResult
|
|
|
|
// Search each target concurrently
|
|
for _, target := range chatManage.SearchTargets {
|
|
wg.Add(1)
|
|
go func(t *types.SearchTarget) {
|
|
defer wg.Done()
|
|
|
|
// List of knowledge IDs to perform vector search on
|
|
// Default to all IDs in the target
|
|
searchKnowledgeIDs := t.KnowledgeIDs
|
|
|
|
// Try direct loading for specific knowledge targets
|
|
if t.Type == types.SearchTargetTypeKnowledge {
|
|
directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs)
|
|
|
|
if len(directResults) > 0 {
|
|
for _, r := range directResults {
|
|
r.KnowledgeBaseID = t.KnowledgeBaseID
|
|
}
|
|
pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{
|
|
"kb_id": t.KnowledgeBaseID,
|
|
"loaded_count": len(directResults),
|
|
"skipped_ids": len(skippedIDs),
|
|
})
|
|
mu.Lock()
|
|
results = append(results, directResults...)
|
|
mu.Unlock()
|
|
}
|
|
|
|
// If all files were loaded directly, we don't need to search anything
|
|
if len(skippedIDs) == 0 && len(t.KnowledgeIDs) > 0 {
|
|
return
|
|
}
|
|
|
|
// Otherwise, only search the files that were skipped (too large)
|
|
searchKnowledgeIDs = skippedIDs
|
|
}
|
|
|
|
// If no IDs left to search (and we are in Knowledge mode), we are done
|
|
if t.Type == types.SearchTargetTypeKnowledge && len(searchKnowledgeIDs) == 0 {
|
|
return
|
|
}
|
|
|
|
// Build params for rewrite query
|
|
params := types.SearchParams{
|
|
QueryText: strings.TrimSpace(chatManage.RewriteQuery),
|
|
VectorThreshold: chatManage.VectorThreshold,
|
|
KeywordThreshold: chatManage.KeywordThreshold,
|
|
MatchCount: chatManage.EmbeddingTopK,
|
|
SkipContextEnrichment: true, // Pipeline handles context assembly in merge stage
|
|
}
|
|
// Apply knowledge ID filter if this is a partial KB search
|
|
if t.Type == types.SearchTargetTypeKnowledge {
|
|
params.KnowledgeIDs = searchKnowledgeIDs
|
|
}
|
|
res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, params)
|
|
if err != nil {
|
|
pipelineWarn(ctx, "Search", "kb_search_error", map[string]interface{}{
|
|
"kb_id": t.KnowledgeBaseID,
|
|
"target_type": t.Type,
|
|
"query": params.QueryText,
|
|
"error": err.Error(),
|
|
})
|
|
return
|
|
}
|
|
for _, r := range res {
|
|
r.KnowledgeBaseID = t.KnowledgeBaseID
|
|
}
|
|
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
|
|
"kb_id": t.KnowledgeBaseID,
|
|
"target_type": t.Type,
|
|
"hit_count": len(res),
|
|
})
|
|
mu.Lock()
|
|
results = append(results, res...)
|
|
mu.Unlock()
|
|
}(target)
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
pipelineInfo(ctx, "Search", "kb_result_summary", map[string]interface{}{
|
|
"total_hits": len(results),
|
|
})
|
|
return results
|
|
}
|
|
|
|
// tryDirectChunkLoading attempts to load chunks for given knowledge IDs directly
|
|
// Returns loaded results and a list of knowledge IDs that were skipped (e.g. due to size limits)
|
|
func (p *PluginSearch) tryDirectChunkLoading(ctx context.Context, tenantID uint64, knowledgeIDs []string) ([]*types.SearchResult, []string) {
|
|
if len(knowledgeIDs) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
// Limit direct loading to avoid OOM or context overflow
|
|
// 50 chunks * ~500 chars/chunk ~= 25k chars
|
|
const maxTotalChunks = 50
|
|
|
|
var allChunks []*types.Chunk
|
|
var skippedIDs []string
|
|
loadedKnowledgeIDs := make(map[string]bool)
|
|
|
|
for _, kid := range knowledgeIDs {
|
|
// Optimization: Check chunk count first if possible?
|
|
chunks, err := p.chunkService.ListChunksByKnowledgeID(ctx, kid)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "DirectLoad: Failed to list chunks for knowledge %s: %v", kid, err)
|
|
skippedIDs = append(skippedIDs, kid)
|
|
continue
|
|
}
|
|
|
|
if len(allChunks)+len(chunks) > maxTotalChunks {
|
|
logger.Infof(ctx, "DirectLoad: Skipped knowledge %s due to size limit (%d + %d > %d)",
|
|
kid, len(allChunks), len(chunks), maxTotalChunks)
|
|
skippedIDs = append(skippedIDs, kid)
|
|
continue
|
|
}
|
|
allChunks = append(allChunks, chunks...)
|
|
loadedKnowledgeIDs[kid] = true
|
|
}
|
|
|
|
if len(allChunks) == 0 {
|
|
return nil, skippedIDs
|
|
}
|
|
|
|
// Fetch Knowledge metadata
|
|
var uniqueKIDs []string
|
|
for kid := range loadedKnowledgeIDs {
|
|
uniqueKIDs = append(uniqueKIDs, kid)
|
|
}
|
|
|
|
knowledgeMap := make(map[string]*types.Knowledge)
|
|
if len(uniqueKIDs) > 0 {
|
|
knowledges, err := p.knowledgeService.GetKnowledgeBatchWithSharedAccess(ctx, tenantID, uniqueKIDs)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "DirectLoad: Failed to fetch knowledge batch: %v", err)
|
|
// Continue without metadata
|
|
} else {
|
|
for _, k := range knowledges {
|
|
knowledgeMap[k.ID] = k
|
|
}
|
|
}
|
|
}
|
|
|
|
var results []*types.SearchResult
|
|
for _, chunk := range allChunks {
|
|
res := &types.SearchResult{
|
|
ID: chunk.ID,
|
|
Content: chunk.Content,
|
|
Score: 1.0, // Maximum score for direct matches
|
|
KnowledgeID: chunk.KnowledgeID,
|
|
ChunkIndex: chunk.ChunkIndex,
|
|
MatchType: types.MatchTypeDirectLoad,
|
|
ChunkType: string(chunk.ChunkType),
|
|
ParentChunkID: chunk.ParentChunkID,
|
|
ImageInfo: chunk.ImageInfo,
|
|
ChunkMetadata: chunk.Metadata,
|
|
StartAt: chunk.StartAt,
|
|
EndAt: chunk.EndAt,
|
|
}
|
|
|
|
if k, ok := knowledgeMap[chunk.KnowledgeID]; ok {
|
|
res.KnowledgeTitle = k.Title
|
|
res.KnowledgeFilename = k.FileName
|
|
res.KnowledgeSource = k.Source
|
|
res.Metadata = k.GetMetadata()
|
|
}
|
|
|
|
results = append(results, res)
|
|
}
|
|
|
|
return results, skippedIDs
|
|
}
|
|
|
|
// searchWebIfEnabled executes web search when enabled and returns converted results
|
|
func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
|
|
if !chatManage.WebSearchEnabled || p.webSearchService == nil || p.tenantService == nil {
|
|
return nil
|
|
}
|
|
tenant, _ := types.TenantInfoFromContext(ctx)
|
|
if tenant == nil || tenant.WebSearchConfig == nil || tenant.WebSearchConfig.Provider == "" {
|
|
pipelineWarn(ctx, "Search", "web_config_missing", map[string]interface{}{
|
|
"tenant_id": chatManage.TenantID,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
pipelineInfo(ctx, "Search", "web_request", map[string]interface{}{
|
|
"tenant_id": chatManage.TenantID,
|
|
"provider": tenant.WebSearchConfig.Provider,
|
|
})
|
|
webResults, err := p.webSearchService.Search(ctx, tenant.WebSearchConfig, chatManage.RewriteQuery)
|
|
if err != nil {
|
|
pipelineWarn(ctx, "Search", "web_search_error", map[string]interface{}{
|
|
"tenant_id": chatManage.TenantID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil
|
|
}
|
|
// Build questions using RewriteQuery only
|
|
questions := []string{strings.TrimSpace(chatManage.RewriteQuery)}
|
|
// Load session-scoped temp KB state from Redis using WebSearchStateRepository
|
|
tempKBID, seen, ids := p.webSearchStateService.GetWebSearchTempKBState(ctx, chatManage.SessionID)
|
|
compressed, kbID, newSeen, newIDs, err := p.webSearchService.CompressWithRAG(
|
|
ctx, chatManage.SessionID, tempKBID, questions, webResults, tenant.WebSearchConfig,
|
|
p.knowledgeBaseService, p.knowledgeService, seen, ids,
|
|
)
|
|
if err != nil {
|
|
pipelineWarn(ctx, "Search", "web_compress_error", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
} else {
|
|
webResults = compressed
|
|
// Persist temp KB state back into Redis using WebSearchStateRepository
|
|
p.webSearchStateService.SaveWebSearchTempKBState(ctx, chatManage.SessionID, kbID, newSeen, newIDs)
|
|
}
|
|
res := searchutil.ConvertWebSearchResults(webResults)
|
|
pipelineInfo(ctx, "Search", "web_hits", map[string]interface{}{
|
|
"hit_count": len(res),
|
|
})
|
|
return res
|
|
}
|