Files
Enterprise-Knowledge-Base/internal/application/service/chat_pipeline/search.go
T

681 lines
21 KiB
Go

package chatpipeline
import (
"context"
"fmt"
"strings"
"sync"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/searchutil"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// PluginSearch implements search functionality for chat pipeline
type PluginSearch struct {
knowledgeBaseService interfaces.KnowledgeBaseService
knowledgeService interfaces.KnowledgeService
chunkService interfaces.ChunkService
config *config.Config
webSearchService interfaces.WebSearchService
tenantService interfaces.TenantService
sessionService interfaces.SessionService
webSearchStateService interfaces.WebSearchStateService
webSearchProviderRepo interfaces.WebSearchProviderRepository
}
func NewPluginSearch(eventManager *EventManager,
knowledgeBaseService interfaces.KnowledgeBaseService,
knowledgeService interfaces.KnowledgeService,
chunkService interfaces.ChunkService,
config *config.Config,
webSearchService interfaces.WebSearchService,
tenantService interfaces.TenantService,
sessionService interfaces.SessionService,
webSearchStateService interfaces.WebSearchStateService,
webSearchProviderRepo interfaces.WebSearchProviderRepository,
) *PluginSearch {
res := &PluginSearch{
knowledgeBaseService: knowledgeBaseService,
knowledgeService: knowledgeService,
chunkService: chunkService,
config: config,
webSearchService: webSearchService,
tenantService: tenantService,
sessionService: sessionService,
webSearchStateService: webSearchStateService,
webSearchProviderRepo: webSearchProviderRepo,
}
eventManager.Register(res)
return res
}
// ActivationEvents returns the event types this plugin handles
func (p *PluginSearch) ActivationEvents() []types.EventType {
return []types.EventType{types.CHUNK_SEARCH}
}
// OnEvent handles search events in the chat pipeline
func (p *PluginSearch) OnEvent(ctx context.Context,
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
) *PluginError {
// Check if we have search targets or web search enabled
hasKBTargets := len(chatManage.SearchTargets) > 0 || len(chatManage.KnowledgeBaseIDs) > 0 || len(chatManage.KnowledgeIDs) > 0
if !hasKBTargets && !chatManage.WebSearchEnabled {
pipelineError(ctx, "Search", "kb_not_found", map[string]interface{}{
"session_id": chatManage.SessionID,
})
return nil
}
pipelineInfo(ctx, "Search", "input", map[string]interface{}{
"session_id": chatManage.SessionID,
"rewrite_query": chatManage.RewriteQuery,
"search_targets": len(chatManage.SearchTargets),
"tenant_id": chatManage.TenantID,
"web_enabled": chatManage.WebSearchEnabled,
})
// Run KB search and web search concurrently
pipelineInfo(ctx, "Search", "plan", map[string]interface{}{
"search_targets": len(chatManage.SearchTargets),
"embedding_top_k": chatManage.EmbeddingTopK,
"vector_threshold": chatManage.VectorThreshold,
"keyword_threshold": chatManage.KeywordThreshold,
})
var wg sync.WaitGroup
var mu sync.Mutex
allResults := make([]*types.SearchResult, 0)
wg.Add(2)
// Goroutine 1: Knowledge base search using SearchTargets
go func() {
defer wg.Done()
kbResults := p.searchByTargets(ctx, chatManage)
if len(kbResults) > 0 {
mu.Lock()
allResults = append(allResults, kbResults...)
mu.Unlock()
}
}()
// Goroutine 2: Web search (if enabled)
go func() {
defer wg.Done()
webResults := p.searchWebIfEnabled(ctx, chatManage)
if len(webResults) > 0 {
mu.Lock()
allResults = append(allResults, webResults...)
mu.Unlock()
}
}()
wg.Wait()
chatManage.SearchResult = allResults
logSearchScoreSample(ctx, "result_score_before_normalize", chatManage.SearchResult)
// If recall is low, attempt query expansion with keyword-focused search
if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK) {
expResults := p.runQueryExpansion(ctx, chatManage)
if len(expResults) > 0 {
chatManage.SearchResult = append(chatManage.SearchResult, expResults...)
}
}
logSearchScoreSample(ctx, "final_score", chatManage.SearchResult)
// Return if we have results
if len(chatManage.SearchResult) != 0 {
pipelineInfo(ctx, "Search", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"result_count": len(chatManage.SearchResult),
})
return next()
}
pipelineWarn(ctx, "Search", "output", map[string]interface{}{
"session_id": chatManage.SessionID,
"result_count": 0,
})
return ErrSearchNothing
}
// getSearchResultFromHistory retrieves relevant knowledge references from chat history
func getSearchResultFromHistory(chatManage *types.ChatManage) []*types.SearchResult {
if len(chatManage.History) == 0 {
return nil
}
// Search history in reverse chronological order
for i := len(chatManage.History) - 1; i >= 0; i-- {
if len(chatManage.History[i].KnowledgeReferences) > 0 {
// Mark all references as history matches
for _, reference := range chatManage.History[i].KnowledgeReferences {
reference.MatchType = types.MatchTypeHistory
}
return chatManage.History[i].KnowledgeReferences
}
}
return nil
}
func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult {
seen := make(map[string]bool)
contentSig := make(map[string]string) // sig -> first chunk ID
var uniqueResults []*types.SearchResult
for _, r := range results {
// Only deduplicate by exact chunk ID — do NOT treat shared ParentChunkID
// as duplicates, because different child chunks of the same parent carry
// different content segments that may all be relevant.
if seen[r.ID] {
logger.Debugf(context.Background(), "Dedup: chunk %s removed due to duplicate ID", r.ID)
continue
}
sig := buildContentSignature(r.Content)
if sig != "" {
if firstChunk, exists := contentSig[sig]; exists {
logger.Debugf(context.Background(), "Dedup: chunk %s removed due to content signature (dup of %s, sig prefix: %.50s...)", r.ID, firstChunk, sig)
continue
}
contentSig[sig] = r.ID
}
seen[r.ID] = true
uniqueResults = append(uniqueResults, r)
}
return uniqueResults
}
func buildContentSignature(content string) string {
return searchutil.BuildContentSignature(content)
}
// removePartialOverlaps drops chunks whose content is largely contained within
// a higher-scored chunk, even across different knowledge sources. This catches
// cross-KB duplicates and near-duplicates that exact-signature dedup misses.
//
// Two thresholds are used:
// - Substring containment: if the normalized short text is a literal substring
// of the normalized long text, the shorter chunk is removed.
// - Token overlap coefficient >= 0.85: if 85%+ of the smaller chunk's tokens
// appear in the larger chunk, the smaller one is redundant.
//
// The input slice MUST already be deduplicated by ID/signature. Within each
// pair the chunk with the lower score is the candidate for removal; ties are
// broken by content length (longer wins).
func removePartialOverlaps(ctx context.Context, results []*types.SearchResult) []*types.SearchResult {
const overlapThreshold = 0.85
if len(results) <= 1 {
return results
}
type normEntry struct {
norm string
result *types.SearchResult
}
entries := make([]normEntry, 0, len(results))
for _, r := range results {
entries = append(entries, normEntry{
norm: searchutil.NormalizeContent(r.Content),
result: r,
})
}
removed := make(map[int]bool)
for i := 0; i < len(entries); i++ {
if removed[i] {
continue
}
for j := i + 1; j < len(entries); j++ {
if removed[j] {
continue
}
a, b := entries[i], entries[j]
shortIdx, longIdx := i, j
if len(a.norm) > len(b.norm) {
shortIdx, longIdx = j, i
}
contained := searchutil.IsContentContained(
entries[shortIdx].norm, entries[longIdx].norm,
)
if !contained {
ratio := searchutil.ContentOverlapRatio(
entries[shortIdx].result.Content,
entries[longIdx].result.Content,
)
if ratio < overlapThreshold {
continue
}
}
victim := shortIdx
if entries[shortIdx].result.Score > entries[longIdx].result.Score {
victim = longIdx
}
removed[victim] = true
keptIdx := i
if victim == i {
keptIdx = j
}
pipelineInfo(ctx, "Merge", "partial_overlap_drop", map[string]interface{}{
"kept_id": entries[keptIdx].result.ID,
"dropped_id": entries[victim].result.ID,
"contained": contained,
})
}
}
out := make([]*types.SearchResult, 0, len(results)-len(removed))
for i, e := range entries {
if !removed[i] {
out = append(out, e.result)
}
}
return out
}
func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) {
const maxLogRows = 8
limit := min(maxLogRows, len(results))
for i := 0; i < limit; i++ {
r := results[i]
pipelineInfo(ctx, "Search", action, map[string]interface{}{
"index": i,
"chunk_id": r.ID,
"score": fmt.Sprintf("%.4f", r.Score),
"match_type": r.MatchType,
})
}
if len(results) > limit {
pipelineInfo(ctx, "Search", action+"_summary", map[string]interface{}{
"total": len(results),
"logged": limit,
"truncated": len(results) - limit,
})
}
}
// searchByTargets performs KB searches using pre-computed SearchTargets.
// Targets sharing the same underlying embedding model (identified by model
// name + endpoint, not just model ID) are grouped so the query embedding is
// computed once per model AND all full-KB targets in a group are combined into
// a single retrieval call, reducing both embedding API calls and DB round-trips.
func (p *PluginSearch) searchByTargets(
ctx context.Context,
chatManage *types.ChatManage,
) []*types.SearchResult {
if len(chatManage.SearchTargets) == 0 {
return nil
}
queryText := strings.TrimSpace(chatManage.RewriteQuery)
// Batch-fetch KB records to determine embedding model grouping.
// On failure, all targets fall into an empty-key group and HybridSearch
// computes the embedding per-KB (graceful degradation).
kbIDs := make([]string, 0, len(chatManage.SearchTargets))
for _, t := range chatManage.SearchTargets {
kbIDs = append(kbIDs, t.KnowledgeBaseID)
}
var kbList []*types.KnowledgeBase
kbMap := make(map[string]*types.KnowledgeBase)
if kbs, err := p.knowledgeBaseService.GetKnowledgeBasesByIDsOnly(ctx, kbIDs); err == nil {
kbList = kbs
for _, kb := range kbs {
if kb != nil {
kbMap[kb.ID] = kb
}
}
} else {
pipelineWarn(ctx, "Search", "batch_kb_fetch_error", map[string]interface{}{
"error": err.Error(),
})
}
// Resolve actual model identities (name + endpoint) so that cross-tenant
// KBs backed by the same physical model share one embedding computation.
modelKeyMap := p.knowledgeBaseService.ResolveEmbeddingModelKeys(ctx, kbList)
groups := make(map[string][]*types.SearchTarget)
for _, t := range chatManage.SearchTargets {
key := modelKeyMap[t.KnowledgeBaseID] // empty string if unresolved
groups[key] = append(groups[key], t)
}
pipelineInfo(ctx, "Search", "embedding_groups", map[string]interface{}{
"total_targets": len(chatManage.SearchTargets),
"unique_models": len(groups),
})
var wg sync.WaitGroup
var mu sync.Mutex
var results []*types.SearchResult
for modelKey, targets := range groups {
wg.Add(1)
go func(modelKey string, targets []*types.SearchTarget) {
defer wg.Done()
// Compute embedding once for this model group.
var queryEmbedding []float32
if modelKey != "" {
emb, err := p.knowledgeBaseService.GetQueryEmbedding(ctx, targets[0].KnowledgeBaseID, queryText)
if err != nil {
pipelineWarn(ctx, "Search", "group_embed_error", map[string]interface{}{
"model_key": modelKey,
"kb_id": targets[0].KnowledgeBaseID,
"error": err.Error(),
})
} else {
queryEmbedding = emb
}
}
// Separate full-KB targets (can be combined into one retrieval)
// from specific-knowledge targets (need per-target direct loading).
var fullKBIDs []string
var knowledgeTargets []*types.SearchTarget
for _, t := range targets {
if t.Type == types.SearchTargetTypeKnowledgeBase {
fullKBIDs = append(fullKBIDs, t.KnowledgeBaseID)
} else {
knowledgeTargets = append(knowledgeTargets, t)
}
}
pipelineInfo(ctx, "Search", "group_plan", map[string]interface{}{
"model_key": modelKey,
"combined_kb_count": len(fullKBIDs),
"individual_targets": len(knowledgeTargets),
"vector_len": len(queryEmbedding),
})
var innerWg sync.WaitGroup
// Combined search: one HybridSearch call spanning all full-KB targets
if len(fullKBIDs) > 0 {
innerWg.Add(1)
go func() {
defer innerWg.Done()
params := types.SearchParams{
QueryText: queryText,
QueryEmbedding: queryEmbedding,
KnowledgeBaseIDs: fullKBIDs,
VectorThreshold: chatManage.VectorThreshold,
KeywordThreshold: chatManage.KeywordThreshold,
MatchCount: chatManage.EmbeddingTopK,
SkipContextEnrichment: true,
}
res, err := p.knowledgeBaseService.HybridSearch(ctx, fullKBIDs[0], params)
if err != nil {
pipelineWarn(ctx, "Search", "combined_kb_search_error", map[string]interface{}{
"kb_ids": fullKBIDs,
"error": err.Error(),
})
return
}
pipelineInfo(ctx, "Search", "combined_kb_result", map[string]interface{}{
"kb_ids": fullKBIDs,
"hit_count": len(res),
})
mu.Lock()
results = append(results, res...)
mu.Unlock()
}()
}
// Individual search: per-target handling for specific-knowledge targets
for _, target := range knowledgeTargets {
innerWg.Add(1)
go func(t *types.SearchTarget) {
defer innerWg.Done()
p.searchSingleTarget(ctx, chatManage, t, queryText, queryEmbedding, &mu, &results)
}(target)
}
innerWg.Wait()
}(modelKey, targets)
}
wg.Wait()
pipelineInfo(ctx, "Search", "kb_result_summary", map[string]interface{}{
"total_hits": len(results),
})
return results
}
// searchSingleTarget handles the search logic for a single SearchTarget
// with specific knowledge IDs, including direct chunk loading and HybridSearch.
func (p *PluginSearch) searchSingleTarget(
ctx context.Context,
chatManage *types.ChatManage,
t *types.SearchTarget,
queryText string,
queryEmbedding []float32,
mu *sync.Mutex,
results *[]*types.SearchResult,
) {
searchKnowledgeIDs := t.KnowledgeIDs
if t.Type == types.SearchTargetTypeKnowledge {
directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs)
if len(directResults) > 0 {
for _, r := range directResults {
r.KnowledgeBaseID = t.KnowledgeBaseID
}
pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"loaded_count": len(directResults),
"skipped_ids": len(skippedIDs),
})
mu.Lock()
*results = append(*results, directResults...)
mu.Unlock()
}
if len(skippedIDs) == 0 && len(t.KnowledgeIDs) > 0 {
return
}
searchKnowledgeIDs = skippedIDs
}
if t.Type == types.SearchTargetTypeKnowledge && len(searchKnowledgeIDs) == 0 {
return
}
params := types.SearchParams{
QueryText: queryText,
QueryEmbedding: queryEmbedding,
VectorThreshold: chatManage.VectorThreshold,
KeywordThreshold: chatManage.KeywordThreshold,
MatchCount: chatManage.EmbeddingTopK,
SkipContextEnrichment: true,
}
if t.Type == types.SearchTargetTypeKnowledge {
params.KnowledgeIDs = searchKnowledgeIDs
}
res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, params)
if err != nil {
pipelineWarn(ctx, "Search", "kb_search_error", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"target_type": t.Type,
"query": params.QueryText,
"error": err.Error(),
})
return
}
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
"kb_id": t.KnowledgeBaseID,
"target_type": t.Type,
"hit_count": len(res),
})
mu.Lock()
*results = append(*results, res...)
mu.Unlock()
}
// tryDirectChunkLoading attempts to load chunks for given knowledge IDs directly
// Returns loaded results and a list of knowledge IDs that were skipped (e.g. due to size limits)
func (p *PluginSearch) tryDirectChunkLoading(ctx context.Context, tenantID uint64, knowledgeIDs []string) ([]*types.SearchResult, []string) {
if len(knowledgeIDs) == 0 {
return nil, nil
}
// Limit direct loading to avoid OOM or context overflow
// 50 chunks * ~500 chars/chunk ~= 25k chars
const maxTotalChunks = 50
var allChunks []*types.Chunk
var skippedIDs []string
loadedKnowledgeIDs := make(map[string]bool)
for _, kid := range knowledgeIDs {
// Optimization: Check chunk count first if possible?
chunks, err := p.chunkService.ListChunksByKnowledgeID(ctx, kid)
if err != nil {
logger.Warnf(ctx, "DirectLoad: Failed to list chunks for knowledge %s: %v", kid, err)
skippedIDs = append(skippedIDs, kid)
continue
}
if len(allChunks)+len(chunks) > maxTotalChunks {
logger.Infof(ctx, "DirectLoad: Skipped knowledge %s due to size limit (%d + %d > %d)",
kid, len(allChunks), len(chunks), maxTotalChunks)
skippedIDs = append(skippedIDs, kid)
continue
}
allChunks = append(allChunks, chunks...)
loadedKnowledgeIDs[kid] = true
}
if len(allChunks) == 0 {
return nil, skippedIDs
}
// Fetch Knowledge metadata
var uniqueKIDs []string
for kid := range loadedKnowledgeIDs {
uniqueKIDs = append(uniqueKIDs, kid)
}
knowledgeMap := make(map[string]*types.Knowledge)
if len(uniqueKIDs) > 0 {
knowledges, err := p.knowledgeService.GetKnowledgeBatchWithSharedAccess(ctx, tenantID, uniqueKIDs)
if err != nil {
logger.Warnf(ctx, "DirectLoad: Failed to fetch knowledge batch: %v", err)
// Continue without metadata
} else {
for _, k := range knowledges {
knowledgeMap[k.ID] = k
}
}
}
var results []*types.SearchResult
for _, chunk := range allChunks {
res := &types.SearchResult{
ID: chunk.ID,
Content: chunk.Content,
Score: 1.0, // Maximum score for direct matches
KnowledgeID: chunk.KnowledgeID,
ChunkIndex: chunk.ChunkIndex,
MatchType: types.MatchTypeDirectLoad,
ChunkType: string(chunk.ChunkType),
ParentChunkID: chunk.ParentChunkID,
ImageInfo: chunk.ImageInfo,
ChunkMetadata: chunk.Metadata,
StartAt: chunk.StartAt,
EndAt: chunk.EndAt,
}
if k, ok := knowledgeMap[chunk.KnowledgeID]; ok {
res.KnowledgeTitle = k.Title
res.KnowledgeFilename = k.FileName
res.KnowledgeSource = k.Source
res.KnowledgeChannel = k.Channel
res.Metadata = k.GetMetadata()
}
results = append(results, res)
}
return results, skippedIDs
}
// searchWebIfEnabled executes web search when enabled and returns converted results
func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
if !chatManage.WebSearchEnabled || p.webSearchService == nil || p.tenantService == nil {
return nil
}
tenant, _ := types.TenantInfoFromContext(ctx)
providerID := chatManage.WebSearchProviderID
var webConfig *types.WebSearchConfig
if tenant != nil && tenant.WebSearchConfig != nil {
// Clone tenant config so we can safely override MaxResults
cfg := *tenant.WebSearchConfig
webConfig = &cfg
} else if providerID != "" {
webConfig = &types.WebSearchConfig{
MaxResults: 10,
}
} else {
pipelineWarn(ctx, "Search", "web_config_missing", map[string]interface{}{
"tenant_id": chatManage.TenantID,
})
return nil
}
// Apply agent-level web search overrides
if chatManage.WebSearchMaxResults > 0 {
webConfig.MaxResults = chatManage.WebSearchMaxResults
}
pipelineInfo(ctx, "Search", "web_request", map[string]interface{}{
"tenant_id": chatManage.TenantID,
"provider_id": providerID,
})
webResults, err := p.webSearchService.Search(ctx, providerID, webConfig, chatManage.RewriteQuery)
if err != nil {
pipelineWarn(ctx, "Search", "web_search_error", map[string]interface{}{
"tenant_id": chatManage.TenantID,
"error": err.Error(),
})
return nil
}
// Build questions using RewriteQuery only
// questions := []string{strings.TrimSpace(chatManage.RewriteQuery)}
// Load session-scoped temp KB state from Redis using WebSearchStateRepository
// tempKBID, seen, ids := p.webSearchStateService.GetWebSearchTempKBState(ctx, chatManage.SessionID)
// compressed, kbID, newSeen, newIDs, err := p.webSearchService.CompressWithRAG(
// ctx, chatManage.SessionID, tempKBID, questions, webResults, webConfig,
// p.knowledgeBaseService, p.knowledgeService, seen, ids,
// )
// if err != nil {
// pipelineWarn(ctx, "Search", "web_compress_error", map[string]interface{}{
// "error": err.Error(),
// })
// } else {
// webResults = compressed
// // Persist temp KB state back into Redis using WebSearchStateRepository
// p.webSearchStateService.SaveWebSearchTempKBState(ctx, chatManage.SessionID, kbID, newSeen, newIDs)
// }
res := searchutil.ConvertWebSearchResults(webResults)
pipelineInfo(ctx, "Search", "web_hits", map[string]interface{}{
"hit_count": len(res),
})
return res
}