Files
Enterprise-Knowledge-Base/internal/application/service/chat_pipline/search.go
T
wizardchen 4c66be1be0 feat: enhance knowledge search by resolving embedding model keys
- Introduced a new method to resolve embedding model IDs to their actual identity keys (name + endpoint), enabling optimal grouping of knowledge bases across different tenants.
- Updated the concurrent search functionality to utilize the resolved model keys, improving the efficiency of query embedding computations.
- Refactored search target grouping to reduce redundant API calls and enhance performance during hybrid searches.

These changes significantly improve the knowledge search process by ensuring that embeddings are computed efficiently and reused across multiple knowledge bases sharing the same underlying model.
2026-03-13 20:54:25 +08:00

569 lines
18 KiB
Go

package chatpipline
import (
"context"
"fmt"
"strings"
"sync"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/searchutil"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// PluginSearch implements search functionality for chat pipeline
type PluginSearch struct {
knowledgeBaseService interfaces.KnowledgeBaseService
knowledgeService interfaces.KnowledgeService
chunkService interfaces.ChunkService
config *config.Config
webSearchService interfaces.WebSearchService
tenantService interfaces.TenantService
sessionService interfaces.SessionService
webSearchStateService interfaces.WebSearchStateService
}
func NewPluginSearch(eventManager *EventManager,
knowledgeBaseService interfaces.KnowledgeBaseService,
knowledgeService interfaces.KnowledgeService,
chunkService interfaces.ChunkService,
config *config.Config,
webSearchService interfaces.WebSearchService,
tenantService interfaces.TenantService,
sessionService interfaces.SessionService,
webSearchStateService interfaces.WebSearchStateService,
) *PluginSearch {
res := &PluginSearch{
knowledgeBaseService: knowledgeBaseService,
knowledgeService: knowledgeService,
chunkService: chunkService,
config: config,
webSearchService: webSearchService,
tenantService: tenantService,
sessionService: sessionService,
webSearchStateService: webSearchStateService,
}
eventManager.Register(res)
return res
}
// ActivationEvents returns the event types this plugin handles
func (p *PluginSearch) ActivationEvents() []types.EventType {
return []types.EventType{types.CHUNK_SEARCH}
}
// OnEvent handles search events in the chat pipeline
func (p *PluginSearch) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
// Check if we have search targets or web search enabled
hasKBTargets := len(chatManage.SearchTargets) > 0 || len(chatManage.KnowledgeBaseIDs) > 0 || len(chatManage.KnowledgeIDs) > 0
if !hasKBTargets && !chatManage.WebSearchEnabled {
pipelineError(ctx, "Search", "kb_not_found", map[string]interface{}{
"session_id": chatManage.SessionID,
})
return nil
}
pipelineInfo(ctx, "Search", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"rewrite_query": chatManage.RewriteQuery,
"search_targets": len(chatManage.SearchTargets),
"tenant_id": chatManage.TenantID,
"web_enabled": chatManage.WebSearchEnabled,
})
// Run KB search and web search concurrently
pipelineInfo(ctx, "Search", "plan", map[string]interface{}{
"search_targets": len(chatManage.SearchTargets),
"embedding_top_k": chatManage.EmbeddingTopK,
"vector_threshold": chatManage.VectorThreshold,
"keyword_threshold": chatManage.KeywordThreshold,
})
var wg sync.WaitGroup
var mu sync.Mutex
allResults := make([]*types.SearchResult, 0)
wg.Add(2)
// Goroutine 1: Knowledge base search using SearchTargets
go func() {
defer wg.Done()
kbResults := p.searchByTargets(ctx, chatManage)
if len(kbResults) > 0 {
mu.Lock()
allResults = append(allResults, kbResults...)
mu.Unlock()
}
}()
// Goroutine 2: Web search (if enabled)
go func() {
defer wg.Done()
webResults := p.searchWebIfEnabled(ctx, chatManage)
if len(webResults) > 0 {
mu.Lock()
allResults = append(allResults, webResults...)
mu.Unlock()
}
}()
wg.Wait()
chatManage.SearchResult = allResults
logSearchScoreSample(ctx, "result_score_before_normalize", chatManage.SearchResult)
// If recall is low, attempt query expansion with keyword-focused search
if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK) {
expResults := p.runQueryExpansion(ctx, chatManage)
if len(expResults) > 0 {
chatManage.SearchResult = append(chatManage.SearchResult, expResults...)
}
}
logSearchScoreSample(ctx, "final_score", chatManage.SearchResult)
// Return if we have results
if len(chatManage.SearchResult) != 0 {
pipelineInfo(ctx, "Search", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"result_count": len(chatManage.SearchResult),
})
return next()
}
pipelineWarn(ctx, "Search", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"result_count": 0,
})
return ErrSearchNothing
}
// getSearchResultFromHistory retrieves relevant knowledge references from chat history
func getSearchResultFromHistory(chatManage *types.ChatManage) []*types.SearchResult {
if len(chatManage.History) == 0 {
return nil
}
// Search history in reverse chronological order
for i := len(chatManage.History) - 1; i >= 0; i-- {
if len(chatManage.History[i].KnowledgeReferences) > 0 {
// Mark all references as history matches
for _, reference := range chatManage.History[i].KnowledgeReferences {
reference.MatchType = types.MatchTypeHistory
}
return chatManage.History[i].KnowledgeReferences
}
}
return nil
}
func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult {
seen := make(map[string]bool)
contentSig := make(map[string]string) // sig -> first chunk ID
var uniqueResults []*types.SearchResult
for _, r := range results {
// Only deduplicate by exact chunk ID — do NOT treat shared ParentChunkID
// as duplicates, because different child chunks of the same parent carry
// different content segments that may all be relevant.
if seen[r.ID] {
logger.Debugf(context.Background(), "Dedup: chunk %s removed due to duplicate ID", r.ID)
continue
}
sig := buildContentSignature(r.Content)
if sig != "" {
if firstChunk, exists := contentSig[sig]; exists {
logger.Debugf(context.Background(), "Dedup: chunk %s removed due to content signature (dup of %s, sig prefix: %.50s...)", r.ID, firstChunk, sig)
continue
}
contentSig[sig] = r.ID
}
seen[r.ID] = true
uniqueResults = append(uniqueResults, r)
}
return uniqueResults
}
func buildContentSignature(content string) string {
return searchutil.BuildContentSignature(content)
}
func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) {
const maxLogRows = 8
limit := min(maxLogRows, len(results))
for i := 0; i < limit; i++ {
r := results[i]
pipelineInfo(ctx, "Search", action, map[string]interface{}{
"index": i,
"chunk_id": r.ID,
"score": fmt.Sprintf("%.4f", r.Score),
"match_type": r.MatchType,
})
}
if len(results) > limit {
pipelineInfo(ctx, "Search", action+"_summary", map[string]interface{}{
"total": len(results),
"logged": limit,
"truncated": len(results) - limit,
})
}
}
// searchByTargets performs KB searches using pre-computed SearchTargets.
// Targets sharing the same underlying embedding model (identified by model
// name + endpoint, not just model ID) are grouped so the query embedding is
// computed once per model AND all full-KB targets in a group are combined into
// a single retrieval call, reducing both embedding API calls and DB round-trips.
func (p *PluginSearch) searchByTargets(
ctx context.Context,
chatManage *types.ChatManage,
) []*types.SearchResult {
if len(chatManage.SearchTargets) == 0 {
return nil
}
queryText := strings.TrimSpace(chatManage.RewriteQuery)
// Batch-fetch KB records to determine embedding model grouping.
// On failure, all targets fall into an empty-key group and HybridSearch
// computes the embedding per-KB (graceful degradation).
kbIDs := make([]string, 0, len(chatManage.SearchTargets))
for _, t := range chatManage.SearchTargets {
kbIDs = append(kbIDs, t.KnowledgeBaseID)
}
var kbList []*types.KnowledgeBase
kbMap := make(map[string]*types.KnowledgeBase)
if kbs, err := p.knowledgeBaseService.GetKnowledgeBasesByIDsOnly(ctx, kbIDs); err == nil {
kbList = kbs
for _, kb := range kbs {
if kb != nil {
kbMap[kb.ID] = kb
}
}
} else {
pipelineWarn(ctx, "Search", "batch_kb_fetch_error", map[string]interface{}{
"error": err.Error(),
})
}
// Resolve actual model identities (name + endpoint) so that cross-tenant
// KBs backed by the same physical model share one embedding computation.
modelKeyMap := p.knowledgeBaseService.ResolveEmbeddingModelKeys(ctx, kbList)
groups := make(map[string][]*types.SearchTarget)
for _, t := range chatManage.SearchTargets {
key := modelKeyMap[t.KnowledgeBaseID] // empty string if unresolved
groups[key] = append(groups[key], t)
}
pipelineInfo(ctx, "Search", "embedding_groups", map[string]interface{}{
"total_targets": len(chatManage.SearchTargets),
"unique_models": len(groups),
})
var wg sync.WaitGroup
var mu sync.Mutex
var results []*types.SearchResult
for modelKey, targets := range groups {
wg.Add(1)
go func(modelKey string, targets []*types.SearchTarget) {
defer wg.Done()
// Compute embedding once for this model group.
var queryEmbedding []float32
if modelKey != "" {
emb, err := p.knowledgeBaseService.GetQueryEmbedding(ctx, targets[0].KnowledgeBaseID, queryText)
if err != nil {
pipelineWarn(ctx, "Search", "group_embed_error", map[string]interface{}{
"model_key": modelKey,
"kb_id": targets[0].KnowledgeBaseID,
"error": err.Error(),
})
} else {
queryEmbedding = emb
}
}
// Separate full-KB targets (can be combined into one retrieval)
// from specific-knowledge targets (need per-target direct loading).
var fullKBIDs []string
var knowledgeTargets []*types.SearchTarget
for _, t := range targets {
if t.Type == types.SearchTargetTypeKnowledgeBase {
fullKBIDs = append(fullKBIDs, t.KnowledgeBaseID)
} else {
knowledgeTargets = append(knowledgeTargets, t)
}
}
pipelineInfo(ctx, "Search", "group_plan", map[string]interface{}{
"model_key": modelKey,
"combined_kb_count": len(fullKBIDs),
"individual_targets": len(knowledgeTargets),
"vector_len": len(queryEmbedding),
})
var innerWg sync.WaitGroup
// Combined search: one HybridSearch call spanning all full-KB targets
if len(fullKBIDs) > 0 {
innerWg.Add(1)
go func() {
defer innerWg.Done()
params := types.SearchParams{
QueryText: queryText,
QueryEmbedding: queryEmbedding,
KnowledgeBaseIDs: fullKBIDs,
VectorThreshold: chatManage.VectorThreshold,
KeywordThreshold: chatManage.KeywordThreshold,
MatchCount: chatManage.EmbeddingTopK,
SkipContextEnrichment: true,
}
res, err := p.knowledgeBaseService.HybridSearch(ctx, fullKBIDs[0], params)
if err != nil {
pipelineWarn(ctx, "Search", "combined_kb_search_error", map[string]interface{}{
"kb_ids": fullKBIDs,
"error": err.Error(),
})
return
}
pipelineInfo(ctx, "Search", "combined_kb_result", map[string]interface{}{
"kb_ids": fullKBIDs,
"hit_count": len(res),
})
mu.Lock()
results = append(results, res...)
mu.Unlock()
}()
}
// Individual search: per-target handling for specific-knowledge targets
for _, target := range knowledgeTargets {
innerWg.Add(1)
go func(t *types.SearchTarget) {
defer innerWg.Done()
p.searchSingleTarget(ctx, chatManage, t, queryText, queryEmbedding, &mu, &results)
}(target)
}
innerWg.Wait()
}(modelKey, targets)
}
wg.Wait()
pipelineInfo(ctx, "Search", "kb_result_summary", map[string]interface{}{
"total_hits": len(results),
})
return results
}
// searchSingleTarget handles the search logic for a single SearchTarget
// with specific knowledge IDs, including direct chunk loading and HybridSearch.
func (p *PluginSearch) searchSingleTarget(
ctx context.Context,
chatManage *types.ChatManage,
t *types.SearchTarget,
queryText string,
queryEmbedding []float32,
mu *sync.Mutex,
results *[]*types.SearchResult,
) {
searchKnowledgeIDs := t.KnowledgeIDs
if t.Type == types.SearchTargetTypeKnowledge {
directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs)
if len(directResults) > 0 {
for _, r := range directResults {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"loaded_count": len(directResults),
"skipped_ids": len(skippedIDs),
})
mu.Lock()
*results = append(*results, directResults...)
mu.Unlock()
}
if len(skippedIDs) == 0 && len(t.KnowledgeIDs) > 0 {
return
}
searchKnowledgeIDs = skippedIDs
}
if t.Type == types.SearchTargetTypeKnowledge && len(searchKnowledgeIDs) == 0 {
return
}
params := types.SearchParams{
QueryText: queryText,
QueryEmbedding: queryEmbedding,
VectorThreshold: chatManage.VectorThreshold,
KeywordThreshold: chatManage.KeywordThreshold,
MatchCount: chatManage.EmbeddingTopK,
SkipContextEnrichment: true,
}
if t.Type == types.SearchTargetTypeKnowledge {
params.KnowledgeIDs = searchKnowledgeIDs
}
res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, params)
if err != nil {
pipelineWarn(ctx, "Search", "kb_search_error", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"target_type": t.Type,
"query": params.QueryText,
"error": err.Error(),
})
return
}
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"target_type": t.Type,
"hit_count": len(res),
})
mu.Lock()
*results = append(*results, res...)
mu.Unlock()
}
// tryDirectChunkLoading attempts to load chunks for given knowledge IDs directly
// Returns loaded results and a list of knowledge IDs that were skipped (e.g. due to size limits)
func (p *PluginSearch) tryDirectChunkLoading(ctx context.Context, tenantID uint64, knowledgeIDs []string) ([]*types.SearchResult, []string) {
if len(knowledgeIDs) == 0 {
return nil, nil
}
// Limit direct loading to avoid OOM or context overflow
// 50 chunks * ~500 chars/chunk ~= 25k chars
const maxTotalChunks = 50
var allChunks []*types.Chunk
var skippedIDs []string
loadedKnowledgeIDs := make(map[string]bool)
for _, kid := range knowledgeIDs {
// Optimization: Check chunk count first if possible?
chunks, err := p.chunkService.ListChunksByKnowledgeID(ctx, kid)
if err != nil {
logger.Warnf(ctx, "DirectLoad: Failed to list chunks for knowledge %s: %v", kid, err)
skippedIDs = append(skippedIDs, kid)
continue
}
if len(allChunks)+len(chunks) > maxTotalChunks {
logger.Infof(ctx, "DirectLoad: Skipped knowledge %s due to size limit (%d + %d > %d)",
kid, len(allChunks), len(chunks), maxTotalChunks)
skippedIDs = append(skippedIDs, kid)
continue
}
allChunks = append(allChunks, chunks...)
loadedKnowledgeIDs[kid] = true
}
if len(allChunks) == 0 {
return nil, skippedIDs
}
// Fetch Knowledge metadata
var uniqueKIDs []string
for kid := range loadedKnowledgeIDs {
uniqueKIDs = append(uniqueKIDs, kid)
}
knowledgeMap := make(map[string]*types.Knowledge)
if len(uniqueKIDs) > 0 {
knowledges, err := p.knowledgeService.GetKnowledgeBatchWithSharedAccess(ctx, tenantID, uniqueKIDs)
if err != nil {
logger.Warnf(ctx, "DirectLoad: Failed to fetch knowledge batch: %v", err)
// Continue without metadata
} else {
for _, k := range knowledges {
knowledgeMap[k.ID] = k
}
}
}
var results []*types.SearchResult
for _, chunk := range allChunks {
res := &types.SearchResult{
ID: chunk.ID,
Content: chunk.Content,
Score: 1.0, // Maximum score for direct matches
KnowledgeID: chunk.KnowledgeID,
ChunkIndex: chunk.ChunkIndex,
MatchType: types.MatchTypeDirectLoad,
ChunkType: string(chunk.ChunkType),
ParentChunkID: chunk.ParentChunkID,
ImageInfo: chunk.ImageInfo,
ChunkMetadata: chunk.Metadata,
StartAt: chunk.StartAt,
EndAt: chunk.EndAt,
}
if k, ok := knowledgeMap[chunk.KnowledgeID]; ok {
res.KnowledgeTitle = k.Title
res.KnowledgeFilename = k.FileName
res.KnowledgeSource = k.Source
res.Metadata = k.GetMetadata()
}
results = append(results, res)
}
return results, skippedIDs
}
// searchWebIfEnabled executes web search when enabled and returns converted results
func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
if !chatManage.WebSearchEnabled || p.webSearchService == nil || p.tenantService == nil {
return nil
}
tenant, _ := types.TenantInfoFromContext(ctx)
if tenant == nil || tenant.WebSearchConfig == nil || tenant.WebSearchConfig.Provider == "" {
pipelineWarn(ctx, "Search", "web_config_missing", map[string]interface{}{
"tenant_id": chatManage.TenantID,
})
return nil
}
pipelineInfo(ctx, "Search", "web_request", map[string]interface{}{
"tenant_id": chatManage.TenantID,
"provider": tenant.WebSearchConfig.Provider,
})
webResults, err := p.webSearchService.Search(ctx, tenant.WebSearchConfig, chatManage.RewriteQuery)
if err != nil {
pipelineWarn(ctx, "Search", "web_search_error", map[string]interface{}{
"tenant_id": chatManage.TenantID,
"error": err.Error(),
})
return nil
}
// Build questions using RewriteQuery only
questions := []string{strings.TrimSpace(chatManage.RewriteQuery)}
// Load session-scoped temp KB state from Redis using WebSearchStateRepository
tempKBID, seen, ids := p.webSearchStateService.GetWebSearchTempKBState(ctx, chatManage.SessionID)
compressed, kbID, newSeen, newIDs, err := p.webSearchService.CompressWithRAG(
ctx, chatManage.SessionID, tempKBID, questions, webResults, tenant.WebSearchConfig,
p.knowledgeBaseService, p.knowledgeService, seen, ids,
)
if err != nil {
pipelineWarn(ctx, "Search", "web_compress_error", map[string]interface{}{
"error": err.Error(),
})
} else {
webResults = compressed
// Persist temp KB state back into Redis using WebSearchStateRepository
p.webSearchStateService.SaveWebSearchTempKBState(ctx, chatManage.SessionID, kbID, newSeen, newIDs)
}
res := searchutil.ConvertWebSearchResults(webResults)
pipelineInfo(ctx, "Search", "web_hits", map[string]interface{}{
"hit_count": len(res),
})
return res
}