Files
Enterprise-Knowledge-Base/internal/application/service/chat_pipline/search.go
T
wizardchen 5fe1ccd4f9 feat: enhance multimodal support and image handling
- Updated configuration to improve handling of multimodal queries, including clearer guidelines for when to use [NO_SEARCH] in responses.
- Enhanced the user interface to include dedicated sections for multimodal configuration, allowing users to manage image uploads and related settings more effectively.
- Added localization support for new multimodal features in both English and Chinese, improving accessibility for a broader audience.
- Implemented backend changes to support image analysis and extraction of OCR text, enriching the interaction experience for users.

These enhancements significantly improve the application's capability to process and respond to multimodal queries, fostering a more interactive and user-friendly environment.
2026-03-16 02:26:47 +08:00

447 lines
14 KiB
Go

package chatpipline
import (
"context"
"fmt"
"strings"
"sync"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/searchutil"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// PluginSearch implements search functionality for chat pipeline
type PluginSearch struct {
knowledgeBaseService interfaces.KnowledgeBaseService
knowledgeService interfaces.KnowledgeService
chunkService interfaces.ChunkService
config *config.Config
webSearchService interfaces.WebSearchService
tenantService interfaces.TenantService
sessionService interfaces.SessionService
webSearchStateService interfaces.WebSearchStateService
}
func NewPluginSearch(eventManager *EventManager,
knowledgeBaseService interfaces.KnowledgeBaseService,
knowledgeService interfaces.KnowledgeService,
chunkService interfaces.ChunkService,
config *config.Config,
webSearchService interfaces.WebSearchService,
tenantService interfaces.TenantService,
sessionService interfaces.SessionService,
webSearchStateService interfaces.WebSearchStateService,
) *PluginSearch {
res := &PluginSearch{
knowledgeBaseService: knowledgeBaseService,
knowledgeService: knowledgeService,
chunkService: chunkService,
config: config,
webSearchService: webSearchService,
tenantService: tenantService,
sessionService: sessionService,
webSearchStateService: webSearchStateService,
}
eventManager.Register(res)
return res
}
// ActivationEvents returns the event types this plugin handles
func (p *PluginSearch) ActivationEvents() []types.EventType {
return []types.EventType{types.CHUNK_SEARCH}
}
// OnEvent handles search events in the chat pipeline
func (p *PluginSearch) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
// Check if we have search targets or web search enabled
hasKBTargets := len(chatManage.SearchTargets) > 0 || len(chatManage.KnowledgeBaseIDs) > 0 || len(chatManage.KnowledgeIDs) > 0
if !hasKBTargets && !chatManage.WebSearchEnabled {
pipelineError(ctx, "Search", "kb_not_found", map[string]interface{}{
"session_id": chatManage.SessionID,
})
return nil
}
pipelineInfo(ctx, "Search", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"rewrite_query": chatManage.RewriteQuery,
"search_targets": len(chatManage.SearchTargets),
"tenant_id": chatManage.TenantID,
"web_enabled": chatManage.WebSearchEnabled,
})
// Run KB search and web search concurrently
pipelineInfo(ctx, "Search", "plan", map[string]interface{}{
"search_targets": len(chatManage.SearchTargets),
"embedding_top_k": chatManage.EmbeddingTopK,
"vector_threshold": chatManage.VectorThreshold,
"keyword_threshold": chatManage.KeywordThreshold,
})
var wg sync.WaitGroup
var mu sync.Mutex
allResults := make([]*types.SearchResult, 0)
wg.Add(2)
// Goroutine 1: Knowledge base search using SearchTargets
go func() {
defer wg.Done()
kbResults := p.searchByTargets(ctx, chatManage)
if len(kbResults) > 0 {
mu.Lock()
allResults = append(allResults, kbResults...)
mu.Unlock()
}
}()
// Goroutine 2: Web search (if enabled)
go func() {
defer wg.Done()
webResults := p.searchWebIfEnabled(ctx, chatManage)
if len(webResults) > 0 {
mu.Lock()
allResults = append(allResults, webResults...)
mu.Unlock()
}
}()
wg.Wait()
chatManage.SearchResult = allResults
logSearchScoreSample(ctx, "result_score_before_normalize", chatManage.SearchResult)
// If recall is low, attempt query expansion with keyword-focused search
if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK) {
expResults := p.runQueryExpansion(ctx, chatManage)
if len(expResults) > 0 {
chatManage.SearchResult = append(chatManage.SearchResult, expResults...)
}
}
logSearchScoreSample(ctx, "final_score", chatManage.SearchResult)
// Return if we have results
if len(chatManage.SearchResult) != 0 {
pipelineInfo(ctx, "Search", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"result_count": len(chatManage.SearchResult),
})
return next()
}
pipelineWarn(ctx, "Search", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"result_count": 0,
})
return ErrSearchNothing
}
// getSearchResultFromHistory retrieves relevant knowledge references from chat history
func getSearchResultFromHistory(chatManage *types.ChatManage) []*types.SearchResult {
if len(chatManage.History) == 0 {
return nil
}
// Search history in reverse chronological order
for i := len(chatManage.History) - 1; i >= 0; i-- {
if len(chatManage.History[i].KnowledgeReferences) > 0 {
// Mark all references as history matches
for _, reference := range chatManage.History[i].KnowledgeReferences {
reference.MatchType = types.MatchTypeHistory
}
return chatManage.History[i].KnowledgeReferences
}
}
return nil
}
func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult {
seen := make(map[string]bool)
contentSig := make(map[string]string) // sig -> first chunk ID
var uniqueResults []*types.SearchResult
for _, r := range results {
// Only deduplicate by exact chunk ID — do NOT treat shared ParentChunkID
// as duplicates, because different child chunks of the same parent carry
// different content segments that may all be relevant.
if seen[r.ID] {
logger.Debugf(context.Background(), "Dedup: chunk %s removed due to duplicate ID", r.ID)
continue
}
sig := buildContentSignature(r.Content)
if sig != "" {
if firstChunk, exists := contentSig[sig]; exists {
logger.Debugf(context.Background(), "Dedup: chunk %s removed due to content signature (dup of %s, sig prefix: %.50s...)", r.ID, firstChunk, sig)
continue
}
contentSig[sig] = r.ID
}
seen[r.ID] = true
uniqueResults = append(uniqueResults, r)
}
return uniqueResults
}
func buildContentSignature(content string) string {
return searchutil.BuildContentSignature(content)
}
func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) {
const maxLogRows = 8
limit := min(maxLogRows, len(results))
for i := 0; i < limit; i++ {
r := results[i]
pipelineInfo(ctx, "Search", action, map[string]interface{}{
"index": i,
"chunk_id": r.ID,
"score": fmt.Sprintf("%.4f", r.Score),
"match_type": r.MatchType,
})
}
if len(results) > limit {
pipelineInfo(ctx, "Search", action+"_summary", map[string]interface{}{
"total": len(results),
"logged": limit,
"truncated": len(results) - limit,
})
}
}
// searchByTargets performs KB searches using pre-computed SearchTargets
// This is the main search method that uses the unified search targets
func (p *PluginSearch) searchByTargets(
ctx context.Context,
chatManage *types.ChatManage,
) []*types.SearchResult {
if len(chatManage.SearchTargets) == 0 {
return nil
}
var wg sync.WaitGroup
var mu sync.Mutex
var results []*types.SearchResult
// Search each target concurrently
for _, target := range chatManage.SearchTargets {
wg.Add(1)
go func(t *types.SearchTarget) {
defer wg.Done()
// List of knowledge IDs to perform vector search on
// Default to all IDs in the target
searchKnowledgeIDs := t.KnowledgeIDs
// Try direct loading for specific knowledge targets
if t.Type == types.SearchTargetTypeKnowledge {
directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs)
if len(directResults) > 0 {
for _, r := range directResults {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"loaded_count": len(directResults),
"skipped_ids": len(skippedIDs),
})
mu.Lock()
results = append(results, directResults...)
mu.Unlock()
}
// If all files were loaded directly, we don't need to search anything
if len(skippedIDs) == 0 && len(t.KnowledgeIDs) > 0 {
return
}
// Otherwise, only search the files that were skipped (too large)
searchKnowledgeIDs = skippedIDs
}
// If no IDs left to search (and we are in Knowledge mode), we are done
if t.Type == types.SearchTargetTypeKnowledge && len(searchKnowledgeIDs) == 0 {
return
}
// Build params for rewrite query
params := types.SearchParams{
QueryText: strings.TrimSpace(chatManage.RewriteQuery),
VectorThreshold: chatManage.VectorThreshold,
KeywordThreshold: chatManage.KeywordThreshold,
MatchCount: chatManage.EmbeddingTopK,
SkipContextEnrichment: true, // Pipeline handles context assembly in merge stage
}
// Apply knowledge ID filter if this is a partial KB search
if t.Type == types.SearchTargetTypeKnowledge {
params.KnowledgeIDs = searchKnowledgeIDs
}
res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, params)
if err != nil {
pipelineWarn(ctx, "Search", "kb_search_error", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"target_type": t.Type,
"query": params.QueryText,
"error": err.Error(),
})
return
}
for _, r := range res {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"target_type": t.Type,
"hit_count": len(res),
})
mu.Lock()
results = append(results, res...)
mu.Unlock()
}(target)
}
wg.Wait()
pipelineInfo(ctx, "Search", "kb_result_summary", map[string]interface{}{
"total_hits": len(results),
})
return results
}
// tryDirectChunkLoading attempts to load chunks for given knowledge IDs directly
// Returns loaded results and a list of knowledge IDs that were skipped (e.g. due to size limits)
func (p *PluginSearch) tryDirectChunkLoading(ctx context.Context, tenantID uint64, knowledgeIDs []string) ([]*types.SearchResult, []string) {
if len(knowledgeIDs) == 0 {
return nil, nil
}
// Limit direct loading to avoid OOM or context overflow
// 50 chunks * ~500 chars/chunk ~= 25k chars
const maxTotalChunks = 50
var allChunks []*types.Chunk
var skippedIDs []string
loadedKnowledgeIDs := make(map[string]bool)
for _, kid := range knowledgeIDs {
// Optimization: Check chunk count first if possible?
chunks, err := p.chunkService.ListChunksByKnowledgeID(ctx, kid)
if err != nil {
logger.Warnf(ctx, "DirectLoad: Failed to list chunks for knowledge %s: %v", kid, err)
skippedIDs = append(skippedIDs, kid)
continue
}
if len(allChunks)+len(chunks) > maxTotalChunks {
logger.Infof(ctx, "DirectLoad: Skipped knowledge %s due to size limit (%d + %d > %d)",
kid, len(allChunks), len(chunks), maxTotalChunks)
skippedIDs = append(skippedIDs, kid)
continue
}
allChunks = append(allChunks, chunks...)
loadedKnowledgeIDs[kid] = true
}
if len(allChunks) == 0 {
return nil, skippedIDs
}
// Fetch Knowledge metadata
var uniqueKIDs []string
for kid := range loadedKnowledgeIDs {
uniqueKIDs = append(uniqueKIDs, kid)
}
knowledgeMap := make(map[string]*types.Knowledge)
if len(uniqueKIDs) > 0 {
knowledges, err := p.knowledgeService.GetKnowledgeBatchWithSharedAccess(ctx, tenantID, uniqueKIDs)
if err != nil {
logger.Warnf(ctx, "DirectLoad: Failed to fetch knowledge batch: %v", err)
// Continue without metadata
} else {
for _, k := range knowledges {
knowledgeMap[k.ID] = k
}
}
}
var results []*types.SearchResult
for _, chunk := range allChunks {
res := &types.SearchResult{
ID: chunk.ID,
Content: chunk.Content,
Score: 1.0, // Maximum score for direct matches
KnowledgeID: chunk.KnowledgeID,
ChunkIndex: chunk.ChunkIndex,
MatchType: types.MatchTypeDirectLoad,
ChunkType: string(chunk.ChunkType),
ParentChunkID: chunk.ParentChunkID,
ImageInfo: chunk.ImageInfo,
ChunkMetadata: chunk.Metadata,
StartAt: chunk.StartAt,
EndAt: chunk.EndAt,
}
if k, ok := knowledgeMap[chunk.KnowledgeID]; ok {
res.KnowledgeTitle = k.Title
res.KnowledgeFilename = k.FileName
res.KnowledgeSource = k.Source
res.Metadata = k.GetMetadata()
}
results = append(results, res)
}
return results, skippedIDs
}
// searchWebIfEnabled executes web search when enabled and returns converted results
func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
if !chatManage.WebSearchEnabled || p.webSearchService == nil || p.tenantService == nil {
return nil
}
tenant, _ := types.TenantInfoFromContext(ctx)
if tenant == nil || tenant.WebSearchConfig == nil || tenant.WebSearchConfig.Provider == "" {
pipelineWarn(ctx, "Search", "web_config_missing", map[string]interface{}{
"tenant_id": chatManage.TenantID,
})
return nil
}
pipelineInfo(ctx, "Search", "web_request", map[string]interface{}{
"tenant_id": chatManage.TenantID,
"provider": tenant.WebSearchConfig.Provider,
})
webResults, err := p.webSearchService.Search(ctx, tenant.WebSearchConfig, chatManage.RewriteQuery)
if err != nil {
pipelineWarn(ctx, "Search", "web_search_error", map[string]interface{}{
"tenant_id": chatManage.TenantID,
"error": err.Error(),
})
return nil
}
// Build questions using RewriteQuery only
questions := []string{strings.TrimSpace(chatManage.RewriteQuery)}
// Load session-scoped temp KB state from Redis using WebSearchStateRepository
tempKBID, seen, ids := p.webSearchStateService.GetWebSearchTempKBState(ctx, chatManage.SessionID)
compressed, kbID, newSeen, newIDs, err := p.webSearchService.CompressWithRAG(
ctx, chatManage.SessionID, tempKBID, questions, webResults, tenant.WebSearchConfig,
p.knowledgeBaseService, p.knowledgeService, seen, ids,
)
if err != nil {
pipelineWarn(ctx, "Search", "web_compress_error", map[string]interface{}{
"error": err.Error(),
})
} else {
webResults = compressed
// Persist temp KB state back into Redis using WebSearchStateRepository
p.webSearchStateService.SaveWebSearchTempKBState(ctx, chatManage.SessionID, kbID, newSeen, newIDs)
}
res := searchutil.ConvertWebSearchResults(webResults)
pipelineInfo(ctx, "Search", "web_hits", map[string]interface{}{
"hit_count": len(res),
})
return res
}