681 lines
21 KiB
Go
681 lines
21 KiB
Go
package chatpipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/Tencent/WeKnora/internal/config"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/searchutil"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
)
|
|
|
|
// PluginSearch implements search functionality for chat pipeline
|
|
type PluginSearch struct {
|
|
knowledgeBaseService interfaces.KnowledgeBaseService
|
|
knowledgeService interfaces.KnowledgeService
|
|
chunkService interfaces.ChunkService
|
|
config *config.Config
|
|
webSearchService interfaces.WebSearchService
|
|
tenantService interfaces.TenantService
|
|
sessionService interfaces.SessionService
|
|
webSearchStateService interfaces.WebSearchStateService
|
|
webSearchProviderRepo interfaces.WebSearchProviderRepository
|
|
}
|
|
|
|
func NewPluginSearch(eventManager *EventManager,
|
|
knowledgeBaseService interfaces.KnowledgeBaseService,
|
|
knowledgeService interfaces.KnowledgeService,
|
|
chunkService interfaces.ChunkService,
|
|
config *config.Config,
|
|
webSearchService interfaces.WebSearchService,
|
|
tenantService interfaces.TenantService,
|
|
sessionService interfaces.SessionService,
|
|
webSearchStateService interfaces.WebSearchStateService,
|
|
webSearchProviderRepo interfaces.WebSearchProviderRepository,
|
|
) *PluginSearch {
|
|
res := &PluginSearch{
|
|
knowledgeBaseService: knowledgeBaseService,
|
|
knowledgeService: knowledgeService,
|
|
chunkService: chunkService,
|
|
config: config,
|
|
webSearchService: webSearchService,
|
|
tenantService: tenantService,
|
|
sessionService: sessionService,
|
|
webSearchStateService: webSearchStateService,
|
|
webSearchProviderRepo: webSearchProviderRepo,
|
|
}
|
|
eventManager.Register(res)
|
|
return res
|
|
}
|
|
|
|
// ActivationEvents returns the event types this plugin handles
|
|
func (p *PluginSearch) ActivationEvents() []types.EventType {
|
|
return []types.EventType{types.CHUNK_SEARCH}
|
|
}
|
|
|
|
// OnEvent handles search events in the chat pipeline
|
|
func (p *PluginSearch) OnEvent(ctx context.Context,
|
|
eventType types.EventType, chatManage *types.ChatManage, next func() *PluginError,
|
|
) *PluginError {
|
|
// Check if we have search targets or web search enabled
|
|
hasKBTargets := len(chatManage.SearchTargets) > 0 || len(chatManage.KnowledgeBaseIDs) > 0 || len(chatManage.KnowledgeIDs) > 0
|
|
if !hasKBTargets && !chatManage.WebSearchEnabled {
|
|
pipelineError(ctx, "Search", "kb_not_found", map[string]interface{}{
|
|
"session_id": chatManage.SessionID,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
pipelineInfo(ctx, "Search", "input", map[string]interface{}{
|
|
"session_id": chatManage.SessionID,
|
|
"rewrite_query": chatManage.RewriteQuery,
|
|
"search_targets": len(chatManage.SearchTargets),
|
|
"tenant_id": chatManage.TenantID,
|
|
"web_enabled": chatManage.WebSearchEnabled,
|
|
})
|
|
|
|
// Run KB search and web search concurrently
|
|
pipelineInfo(ctx, "Search", "plan", map[string]interface{}{
|
|
"search_targets": len(chatManage.SearchTargets),
|
|
"embedding_top_k": chatManage.EmbeddingTopK,
|
|
"vector_threshold": chatManage.VectorThreshold,
|
|
"keyword_threshold": chatManage.KeywordThreshold,
|
|
})
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex
|
|
allResults := make([]*types.SearchResult, 0)
|
|
|
|
wg.Add(2)
|
|
// Goroutine 1: Knowledge base search using SearchTargets
|
|
go func() {
|
|
defer wg.Done()
|
|
kbResults := p.searchByTargets(ctx, chatManage)
|
|
if len(kbResults) > 0 {
|
|
mu.Lock()
|
|
allResults = append(allResults, kbResults...)
|
|
mu.Unlock()
|
|
}
|
|
}()
|
|
|
|
// Goroutine 2: Web search (if enabled)
|
|
go func() {
|
|
defer wg.Done()
|
|
webResults := p.searchWebIfEnabled(ctx, chatManage)
|
|
if len(webResults) > 0 {
|
|
mu.Lock()
|
|
allResults = append(allResults, webResults...)
|
|
mu.Unlock()
|
|
}
|
|
}()
|
|
|
|
wg.Wait()
|
|
|
|
chatManage.SearchResult = allResults
|
|
|
|
logSearchScoreSample(ctx, "result_score_before_normalize", chatManage.SearchResult)
|
|
|
|
// If recall is low, attempt query expansion with keyword-focused search
|
|
if chatManage.EnableQueryExpansion && len(chatManage.SearchResult) < max(1, chatManage.EmbeddingTopK) {
|
|
expResults := p.runQueryExpansion(ctx, chatManage)
|
|
if len(expResults) > 0 {
|
|
chatManage.SearchResult = append(chatManage.SearchResult, expResults...)
|
|
}
|
|
}
|
|
|
|
logSearchScoreSample(ctx, "final_score", chatManage.SearchResult)
|
|
|
|
// Return if we have results
|
|
if len(chatManage.SearchResult) != 0 {
|
|
pipelineInfo(ctx, "Search", "output", map[string]interface{}{
|
|
"session_id": chatManage.SessionID,
|
|
"result_count": len(chatManage.SearchResult),
|
|
})
|
|
return next()
|
|
}
|
|
pipelineWarn(ctx, "Search", "output", map[string]interface{}{
|
|
"session_id": chatManage.SessionID,
|
|
"result_count": 0,
|
|
})
|
|
return ErrSearchNothing
|
|
}
|
|
|
|
// getSearchResultFromHistory retrieves relevant knowledge references from chat history
|
|
func getSearchResultFromHistory(chatManage *types.ChatManage) []*types.SearchResult {
|
|
if len(chatManage.History) == 0 {
|
|
return nil
|
|
}
|
|
// Search history in reverse chronological order
|
|
for i := len(chatManage.History) - 1; i >= 0; i-- {
|
|
if len(chatManage.History[i].KnowledgeReferences) > 0 {
|
|
// Mark all references as history matches
|
|
for _, reference := range chatManage.History[i].KnowledgeReferences {
|
|
reference.MatchType = types.MatchTypeHistory
|
|
}
|
|
return chatManage.History[i].KnowledgeReferences
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func removeDuplicateResults(results []*types.SearchResult) []*types.SearchResult {
|
|
seen := make(map[string]bool)
|
|
contentSig := make(map[string]string) // sig -> first chunk ID
|
|
var uniqueResults []*types.SearchResult
|
|
for _, r := range results {
|
|
// Only deduplicate by exact chunk ID — do NOT treat shared ParentChunkID
|
|
// as duplicates, because different child chunks of the same parent carry
|
|
// different content segments that may all be relevant.
|
|
if seen[r.ID] {
|
|
logger.Debugf(context.Background(), "Dedup: chunk %s removed due to duplicate ID", r.ID)
|
|
continue
|
|
}
|
|
sig := buildContentSignature(r.Content)
|
|
if sig != "" {
|
|
if firstChunk, exists := contentSig[sig]; exists {
|
|
logger.Debugf(context.Background(), "Dedup: chunk %s removed due to content signature (dup of %s, sig prefix: %.50s...)", r.ID, firstChunk, sig)
|
|
continue
|
|
}
|
|
contentSig[sig] = r.ID
|
|
}
|
|
seen[r.ID] = true
|
|
uniqueResults = append(uniqueResults, r)
|
|
}
|
|
return uniqueResults
|
|
}
|
|
|
|
func buildContentSignature(content string) string {
|
|
return searchutil.BuildContentSignature(content)
|
|
}
|
|
|
|
// removePartialOverlaps drops chunks whose content is largely contained within
|
|
// a higher-scored chunk, even across different knowledge sources. This catches
|
|
// cross-KB duplicates and near-duplicates that exact-signature dedup misses.
|
|
//
|
|
// Two thresholds are used:
|
|
// - Substring containment: if the normalized short text is a literal substring
|
|
// of the normalized long text, the shorter chunk is removed.
|
|
// - Token overlap coefficient >= 0.85: if 85%+ of the smaller chunk's tokens
|
|
// appear in the larger chunk, the smaller one is redundant.
|
|
//
|
|
// The input slice MUST already be deduplicated by ID/signature. Within each
|
|
// pair the chunk with the lower score is the candidate for removal; ties are
|
|
// broken by content length (longer wins).
|
|
func removePartialOverlaps(ctx context.Context, results []*types.SearchResult) []*types.SearchResult {
|
|
const overlapThreshold = 0.85
|
|
|
|
if len(results) <= 1 {
|
|
return results
|
|
}
|
|
|
|
type normEntry struct {
|
|
norm string
|
|
result *types.SearchResult
|
|
}
|
|
|
|
entries := make([]normEntry, 0, len(results))
|
|
for _, r := range results {
|
|
entries = append(entries, normEntry{
|
|
norm: searchutil.NormalizeContent(r.Content),
|
|
result: r,
|
|
})
|
|
}
|
|
|
|
removed := make(map[int]bool)
|
|
|
|
for i := 0; i < len(entries); i++ {
|
|
if removed[i] {
|
|
continue
|
|
}
|
|
for j := i + 1; j < len(entries); j++ {
|
|
if removed[j] {
|
|
continue
|
|
}
|
|
|
|
a, b := entries[i], entries[j]
|
|
|
|
shortIdx, longIdx := i, j
|
|
if len(a.norm) > len(b.norm) {
|
|
shortIdx, longIdx = j, i
|
|
}
|
|
|
|
contained := searchutil.IsContentContained(
|
|
entries[shortIdx].norm, entries[longIdx].norm,
|
|
)
|
|
|
|
if !contained {
|
|
ratio := searchutil.ContentOverlapRatio(
|
|
entries[shortIdx].result.Content,
|
|
entries[longIdx].result.Content,
|
|
)
|
|
if ratio < overlapThreshold {
|
|
continue
|
|
}
|
|
}
|
|
|
|
victim := shortIdx
|
|
if entries[shortIdx].result.Score > entries[longIdx].result.Score {
|
|
victim = longIdx
|
|
}
|
|
removed[victim] = true
|
|
|
|
keptIdx := i
|
|
if victim == i {
|
|
keptIdx = j
|
|
}
|
|
pipelineInfo(ctx, "Merge", "partial_overlap_drop", map[string]interface{}{
|
|
"kept_id": entries[keptIdx].result.ID,
|
|
"dropped_id": entries[victim].result.ID,
|
|
"contained": contained,
|
|
})
|
|
}
|
|
}
|
|
|
|
out := make([]*types.SearchResult, 0, len(results)-len(removed))
|
|
for i, e := range entries {
|
|
if !removed[i] {
|
|
out = append(out, e.result)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func logSearchScoreSample(ctx context.Context, action string, results []*types.SearchResult) {
|
|
const maxLogRows = 8
|
|
limit := min(maxLogRows, len(results))
|
|
for i := 0; i < limit; i++ {
|
|
r := results[i]
|
|
pipelineInfo(ctx, "Search", action, map[string]interface{}{
|
|
"index": i,
|
|
"chunk_id": r.ID,
|
|
"score": fmt.Sprintf("%.4f", r.Score),
|
|
"match_type": r.MatchType,
|
|
})
|
|
}
|
|
if len(results) > limit {
|
|
pipelineInfo(ctx, "Search", action+"_summary", map[string]interface{}{
|
|
"total": len(results),
|
|
"logged": limit,
|
|
"truncated": len(results) - limit,
|
|
})
|
|
}
|
|
}
|
|
|
|
// searchByTargets performs KB searches using pre-computed SearchTargets.
|
|
// Targets sharing the same underlying embedding model (identified by model
|
|
// name + endpoint, not just model ID) are grouped so the query embedding is
|
|
// computed once per model AND all full-KB targets in a group are combined into
|
|
// a single retrieval call, reducing both embedding API calls and DB round-trips.
|
|
func (p *PluginSearch) searchByTargets(
|
|
ctx context.Context,
|
|
chatManage *types.ChatManage,
|
|
) []*types.SearchResult {
|
|
if len(chatManage.SearchTargets) == 0 {
|
|
return nil
|
|
}
|
|
|
|
queryText := strings.TrimSpace(chatManage.RewriteQuery)
|
|
|
|
// Batch-fetch KB records to determine embedding model grouping.
|
|
// On failure, all targets fall into an empty-key group and HybridSearch
|
|
// computes the embedding per-KB (graceful degradation).
|
|
kbIDs := make([]string, 0, len(chatManage.SearchTargets))
|
|
for _, t := range chatManage.SearchTargets {
|
|
kbIDs = append(kbIDs, t.KnowledgeBaseID)
|
|
}
|
|
var kbList []*types.KnowledgeBase
|
|
kbMap := make(map[string]*types.KnowledgeBase)
|
|
if kbs, err := p.knowledgeBaseService.GetKnowledgeBasesByIDsOnly(ctx, kbIDs); err == nil {
|
|
kbList = kbs
|
|
for _, kb := range kbs {
|
|
if kb != nil {
|
|
kbMap[kb.ID] = kb
|
|
}
|
|
}
|
|
} else {
|
|
pipelineWarn(ctx, "Search", "batch_kb_fetch_error", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
}
|
|
|
|
// Resolve actual model identities (name + endpoint) so that cross-tenant
|
|
// KBs backed by the same physical model share one embedding computation.
|
|
modelKeyMap := p.knowledgeBaseService.ResolveEmbeddingModelKeys(ctx, kbList)
|
|
|
|
groups := make(map[string][]*types.SearchTarget)
|
|
for _, t := range chatManage.SearchTargets {
|
|
key := modelKeyMap[t.KnowledgeBaseID] // empty string if unresolved
|
|
groups[key] = append(groups[key], t)
|
|
}
|
|
|
|
pipelineInfo(ctx, "Search", "embedding_groups", map[string]interface{}{
|
|
"total_targets": len(chatManage.SearchTargets),
|
|
"unique_models": len(groups),
|
|
})
|
|
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex
|
|
var results []*types.SearchResult
|
|
|
|
for modelKey, targets := range groups {
|
|
wg.Add(1)
|
|
go func(modelKey string, targets []*types.SearchTarget) {
|
|
defer wg.Done()
|
|
|
|
// Compute embedding once for this model group.
|
|
var queryEmbedding []float32
|
|
if modelKey != "" {
|
|
emb, err := p.knowledgeBaseService.GetQueryEmbedding(ctx, targets[0].KnowledgeBaseID, queryText)
|
|
if err != nil {
|
|
pipelineWarn(ctx, "Search", "group_embed_error", map[string]interface{}{
|
|
"model_key": modelKey,
|
|
"kb_id": targets[0].KnowledgeBaseID,
|
|
"error": err.Error(),
|
|
})
|
|
} else {
|
|
queryEmbedding = emb
|
|
}
|
|
}
|
|
|
|
// Separate full-KB targets (can be combined into one retrieval)
|
|
// from specific-knowledge targets (need per-target direct loading).
|
|
var fullKBIDs []string
|
|
var knowledgeTargets []*types.SearchTarget
|
|
for _, t := range targets {
|
|
if t.Type == types.SearchTargetTypeKnowledgeBase {
|
|
fullKBIDs = append(fullKBIDs, t.KnowledgeBaseID)
|
|
} else {
|
|
knowledgeTargets = append(knowledgeTargets, t)
|
|
}
|
|
}
|
|
|
|
pipelineInfo(ctx, "Search", "group_plan", map[string]interface{}{
|
|
"model_key": modelKey,
|
|
"combined_kb_count": len(fullKBIDs),
|
|
"individual_targets": len(knowledgeTargets),
|
|
"vector_len": len(queryEmbedding),
|
|
})
|
|
|
|
var innerWg sync.WaitGroup
|
|
|
|
// Combined search: one HybridSearch call spanning all full-KB targets
|
|
if len(fullKBIDs) > 0 {
|
|
innerWg.Add(1)
|
|
go func() {
|
|
defer innerWg.Done()
|
|
|
|
params := types.SearchParams{
|
|
QueryText: queryText,
|
|
QueryEmbedding: queryEmbedding,
|
|
KnowledgeBaseIDs: fullKBIDs,
|
|
VectorThreshold: chatManage.VectorThreshold,
|
|
KeywordThreshold: chatManage.KeywordThreshold,
|
|
MatchCount: chatManage.EmbeddingTopK,
|
|
SkipContextEnrichment: true,
|
|
}
|
|
res, err := p.knowledgeBaseService.HybridSearch(ctx, fullKBIDs[0], params)
|
|
if err != nil {
|
|
pipelineWarn(ctx, "Search", "combined_kb_search_error", map[string]interface{}{
|
|
"kb_ids": fullKBIDs,
|
|
"error": err.Error(),
|
|
})
|
|
return
|
|
}
|
|
pipelineInfo(ctx, "Search", "combined_kb_result", map[string]interface{}{
|
|
"kb_ids": fullKBIDs,
|
|
"hit_count": len(res),
|
|
})
|
|
mu.Lock()
|
|
results = append(results, res...)
|
|
mu.Unlock()
|
|
}()
|
|
}
|
|
|
|
// Individual search: per-target handling for specific-knowledge targets
|
|
for _, target := range knowledgeTargets {
|
|
innerWg.Add(1)
|
|
go func(t *types.SearchTarget) {
|
|
defer innerWg.Done()
|
|
p.searchSingleTarget(ctx, chatManage, t, queryText, queryEmbedding, &mu, &results)
|
|
}(target)
|
|
}
|
|
|
|
innerWg.Wait()
|
|
}(modelKey, targets)
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
pipelineInfo(ctx, "Search", "kb_result_summary", map[string]interface{}{
|
|
"total_hits": len(results),
|
|
})
|
|
return results
|
|
}
|
|
|
|
// searchSingleTarget handles the search logic for a single SearchTarget
|
|
// with specific knowledge IDs, including direct chunk loading and HybridSearch.
|
|
func (p *PluginSearch) searchSingleTarget(
|
|
ctx context.Context,
|
|
chatManage *types.ChatManage,
|
|
t *types.SearchTarget,
|
|
queryText string,
|
|
queryEmbedding []float32,
|
|
mu *sync.Mutex,
|
|
results *[]*types.SearchResult,
|
|
) {
|
|
searchKnowledgeIDs := t.KnowledgeIDs
|
|
|
|
if t.Type == types.SearchTargetTypeKnowledge {
|
|
directResults, skippedIDs := p.tryDirectChunkLoading(ctx, chatManage.TenantID, t.KnowledgeIDs)
|
|
|
|
if len(directResults) > 0 {
|
|
for _, r := range directResults {
|
|
r.KnowledgeBaseID = t.KnowledgeBaseID
|
|
}
|
|
pipelineInfo(ctx, "Search", "direct_load", map[string]interface{}{
|
|
"kb_id": t.KnowledgeBaseID,
|
|
"loaded_count": len(directResults),
|
|
"skipped_ids": len(skippedIDs),
|
|
})
|
|
mu.Lock()
|
|
*results = append(*results, directResults...)
|
|
mu.Unlock()
|
|
}
|
|
|
|
if len(skippedIDs) == 0 && len(t.KnowledgeIDs) > 0 {
|
|
return
|
|
}
|
|
searchKnowledgeIDs = skippedIDs
|
|
}
|
|
|
|
if t.Type == types.SearchTargetTypeKnowledge && len(searchKnowledgeIDs) == 0 {
|
|
return
|
|
}
|
|
|
|
params := types.SearchParams{
|
|
QueryText: queryText,
|
|
QueryEmbedding: queryEmbedding,
|
|
VectorThreshold: chatManage.VectorThreshold,
|
|
KeywordThreshold: chatManage.KeywordThreshold,
|
|
MatchCount: chatManage.EmbeddingTopK,
|
|
SkipContextEnrichment: true,
|
|
}
|
|
if t.Type == types.SearchTargetTypeKnowledge {
|
|
params.KnowledgeIDs = searchKnowledgeIDs
|
|
}
|
|
res, err := p.knowledgeBaseService.HybridSearch(ctx, t.KnowledgeBaseID, params)
|
|
if err != nil {
|
|
pipelineWarn(ctx, "Search", "kb_search_error", map[string]interface{}{
|
|
"kb_id": t.KnowledgeBaseID,
|
|
"target_type": t.Type,
|
|
"query": params.QueryText,
|
|
"error": err.Error(),
|
|
})
|
|
return
|
|
}
|
|
pipelineInfo(ctx, "Search", "kb_result", map[string]interface{}{
|
|
"kb_id": t.KnowledgeBaseID,
|
|
"target_type": t.Type,
|
|
"hit_count": len(res),
|
|
})
|
|
mu.Lock()
|
|
*results = append(*results, res...)
|
|
mu.Unlock()
|
|
}
|
|
|
|
// tryDirectChunkLoading attempts to load chunks for given knowledge IDs directly
|
|
// Returns loaded results and a list of knowledge IDs that were skipped (e.g. due to size limits)
|
|
func (p *PluginSearch) tryDirectChunkLoading(ctx context.Context, tenantID uint64, knowledgeIDs []string) ([]*types.SearchResult, []string) {
|
|
if len(knowledgeIDs) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
// Limit direct loading to avoid OOM or context overflow
|
|
// 50 chunks * ~500 chars/chunk ~= 25k chars
|
|
const maxTotalChunks = 50
|
|
|
|
var allChunks []*types.Chunk
|
|
var skippedIDs []string
|
|
loadedKnowledgeIDs := make(map[string]bool)
|
|
|
|
for _, kid := range knowledgeIDs {
|
|
// Optimization: Check chunk count first if possible?
|
|
chunks, err := p.chunkService.ListChunksByKnowledgeID(ctx, kid)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "DirectLoad: Failed to list chunks for knowledge %s: %v", kid, err)
|
|
skippedIDs = append(skippedIDs, kid)
|
|
continue
|
|
}
|
|
|
|
if len(allChunks)+len(chunks) > maxTotalChunks {
|
|
logger.Infof(ctx, "DirectLoad: Skipped knowledge %s due to size limit (%d + %d > %d)",
|
|
kid, len(allChunks), len(chunks), maxTotalChunks)
|
|
skippedIDs = append(skippedIDs, kid)
|
|
continue
|
|
}
|
|
allChunks = append(allChunks, chunks...)
|
|
loadedKnowledgeIDs[kid] = true
|
|
}
|
|
|
|
if len(allChunks) == 0 {
|
|
return nil, skippedIDs
|
|
}
|
|
|
|
// Fetch Knowledge metadata
|
|
var uniqueKIDs []string
|
|
for kid := range loadedKnowledgeIDs {
|
|
uniqueKIDs = append(uniqueKIDs, kid)
|
|
}
|
|
|
|
knowledgeMap := make(map[string]*types.Knowledge)
|
|
if len(uniqueKIDs) > 0 {
|
|
knowledges, err := p.knowledgeService.GetKnowledgeBatchWithSharedAccess(ctx, tenantID, uniqueKIDs)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "DirectLoad: Failed to fetch knowledge batch: %v", err)
|
|
// Continue without metadata
|
|
} else {
|
|
for _, k := range knowledges {
|
|
knowledgeMap[k.ID] = k
|
|
}
|
|
}
|
|
}
|
|
|
|
var results []*types.SearchResult
|
|
for _, chunk := range allChunks {
|
|
res := &types.SearchResult{
|
|
ID: chunk.ID,
|
|
Content: chunk.Content,
|
|
Score: 1.0, // Maximum score for direct matches
|
|
KnowledgeID: chunk.KnowledgeID,
|
|
ChunkIndex: chunk.ChunkIndex,
|
|
MatchType: types.MatchTypeDirectLoad,
|
|
ChunkType: string(chunk.ChunkType),
|
|
ParentChunkID: chunk.ParentChunkID,
|
|
ImageInfo: chunk.ImageInfo,
|
|
ChunkMetadata: chunk.Metadata,
|
|
StartAt: chunk.StartAt,
|
|
EndAt: chunk.EndAt,
|
|
}
|
|
|
|
if k, ok := knowledgeMap[chunk.KnowledgeID]; ok {
|
|
res.KnowledgeTitle = k.Title
|
|
res.KnowledgeFilename = k.FileName
|
|
res.KnowledgeSource = k.Source
|
|
res.KnowledgeChannel = k.Channel
|
|
res.Metadata = k.GetMetadata()
|
|
}
|
|
|
|
results = append(results, res)
|
|
}
|
|
|
|
return results, skippedIDs
|
|
}
|
|
|
|
// searchWebIfEnabled executes web search when enabled and returns converted results
|
|
func (p *PluginSearch) searchWebIfEnabled(ctx context.Context, chatManage *types.ChatManage) []*types.SearchResult {
|
|
if !chatManage.WebSearchEnabled || p.webSearchService == nil || p.tenantService == nil {
|
|
return nil
|
|
}
|
|
tenant, _ := types.TenantInfoFromContext(ctx)
|
|
providerID := chatManage.WebSearchProviderID
|
|
|
|
var webConfig *types.WebSearchConfig
|
|
if tenant != nil && tenant.WebSearchConfig != nil {
|
|
// Clone tenant config so we can safely override MaxResults
|
|
cfg := *tenant.WebSearchConfig
|
|
webConfig = &cfg
|
|
} else if providerID != "" {
|
|
webConfig = &types.WebSearchConfig{
|
|
MaxResults: 10,
|
|
}
|
|
} else {
|
|
pipelineWarn(ctx, "Search", "web_config_missing", map[string]interface{}{
|
|
"tenant_id": chatManage.TenantID,
|
|
})
|
|
return nil
|
|
}
|
|
|
|
// Apply agent-level web search overrides
|
|
if chatManage.WebSearchMaxResults > 0 {
|
|
webConfig.MaxResults = chatManage.WebSearchMaxResults
|
|
}
|
|
|
|
pipelineInfo(ctx, "Search", "web_request", map[string]interface{}{
|
|
"tenant_id": chatManage.TenantID,
|
|
"provider_id": providerID,
|
|
})
|
|
webResults, err := p.webSearchService.Search(ctx, providerID, webConfig, chatManage.RewriteQuery)
|
|
if err != nil {
|
|
pipelineWarn(ctx, "Search", "web_search_error", map[string]interface{}{
|
|
"tenant_id": chatManage.TenantID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil
|
|
}
|
|
// Build questions using RewriteQuery only
|
|
// questions := []string{strings.TrimSpace(chatManage.RewriteQuery)}
|
|
// Load session-scoped temp KB state from Redis using WebSearchStateRepository
|
|
// tempKBID, seen, ids := p.webSearchStateService.GetWebSearchTempKBState(ctx, chatManage.SessionID)
|
|
// compressed, kbID, newSeen, newIDs, err := p.webSearchService.CompressWithRAG(
|
|
// ctx, chatManage.SessionID, tempKBID, questions, webResults, webConfig,
|
|
// p.knowledgeBaseService, p.knowledgeService, seen, ids,
|
|
// )
|
|
// if err != nil {
|
|
// pipelineWarn(ctx, "Search", "web_compress_error", map[string]interface{}{
|
|
// "error": err.Error(),
|
|
// })
|
|
// } else {
|
|
// webResults = compressed
|
|
// // Persist temp KB state back into Redis using WebSearchStateRepository
|
|
// p.webSearchStateService.SaveWebSearchTempKBState(ctx, chatManage.SessionID, kbID, newSeen, newIDs)
|
|
// }
|
|
res := searchutil.ConvertWebSearchResults(webResults)
|
|
pipelineInfo(ctx, "Search", "web_hits", map[string]interface{}{
|
|
"hit_count": len(res),
|
|
})
|
|
return res
|
|
}
|