feat(search): fan-out KB retrieval across bound vector stores
Multi-KB hybrid search now groups KBs by their bound VectorStore (partition key (storeID, owner_tenant_id)), retrieves in parallel via errgroup with a SetLimit(4) cap and a per-group timeout (MULTI_STORE_RETRIEVE_TIMEOUT_SEC, default 30s), and merges results. When the collected results span more than one engine type, an EngineAwareNormalizer rescales vector scores to [0, 1]; keyword (BM25) scores pass through to the existing RRF fusion. Single-group calls take the fast path with zero fan-out overhead, preserving today's behavior for deployments where every KB has vector_store_id = NULL. Embedding-model consistency is now enforced explicitly via ResolveEmbeddingModelKeys. Multi-KB searches across KBs whose resolved model identities differ return BadRequest instead of silently producing incomparable scores. Cross-tenant Organization-shared KBs are preserved by partitioning on KB.TenantID so the factory's ownership lookup runs against the source tenant. Foreign-tenant KB UUIDs injected via the request body are rejected via kbShareService.HasTenantKBPermission (Plan 3 of #1303, 3-D capped) before any retrieval; rejected scopes surface as 404 to avoid leaking foreign KB existence. Service-layer typed AppErrors (ErrVectorStoreBindingInvalid 2200 / ErrVectorStoreUnavailable 2201) are mapped from PR2 sentinel hierarchy and preserved end-to-end: the iterative FAQ path returns them rather than swallowing, and the HybridSearch handler routes typed AppErrors to the client unchanged instead of downgrading to 500. Part of #993 (Phase 2: Per-KB VectorStore Binding). Phase 2 roadmap item: PR 4 (Multi-store fan-out search). Depends on #994, #1310, #1372.
This commit is contained in:
@@ -4,10 +4,12 @@ import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
apperrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// GetQueryEmbedding computes the query embedding using the embedding model
|
||||
@@ -86,86 +88,117 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
id string,
|
||||
params types.SearchParams,
|
||||
) ([]*types.SearchResult, error) {
|
||||
// Determine the set of KB IDs to search
|
||||
// Determine the set of KB IDs to search.
|
||||
searchKBIDs := params.KnowledgeBaseIDs
|
||||
if len(searchKBIDs) == 0 {
|
||||
searchKBIDs = []string{id}
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Hybrid search parameters, knowledge base IDs: %v, query text: %s", searchKBIDs, params.QueryText)
|
||||
// QueryText is user-controlled; sanitize before logging to prevent
|
||||
// CR/LF/tab log injection. Matches the handler-layer sanitization at
|
||||
// handler/knowledgebase.go.
|
||||
logger.Infof(ctx, "Hybrid search parameters, knowledge base IDs: %v, query text: %s",
|
||||
searchKBIDs, secutils.SanitizeForLog(params.QueryText))
|
||||
|
||||
// tenantInfo is consumed below for retrieval config (post-fusion step).
|
||||
tenantInfo, _ := types.TenantInfoFromContext(ctx)
|
||||
requestTenantID := types.MustTenantIDFromContext(ctx)
|
||||
|
||||
// Resolve the primary KB first so the factory can route to the bound
|
||||
// VectorStore (if any). When the KB has no binding the factory falls
|
||||
// back to the tenant's effective engines.
|
||||
kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
|
||||
// Batch-load every KB in scope. Required for store grouping,
|
||||
// embedding-model consistency validation, and FAQ type detection.
|
||||
// GetKnowledgeBaseByIDs is intentionally tenant-agnostic at the
|
||||
// repository layer so that Organization-shared KBs (owned by a
|
||||
// different tenant) can be loaded here; authorization for each
|
||||
// returned row is enforced explicitly below.
|
||||
kbs, err := s.repo.GetKnowledgeBaseByIDs(ctx, searchKBIDs)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"knowledge_base_id": id,
|
||||
"knowledge_base_ids": searchKBIDs,
|
||||
})
|
||||
return nil, err
|
||||
}
|
||||
if len(kbs) == 0 {
|
||||
return nil, apperrors.NewNotFoundError("knowledge base not found")
|
||||
}
|
||||
|
||||
tenantID := types.MustTenantIDFromContext(ctx)
|
||||
|
||||
// Create a composite retrieval engine. When the KB is bound to a store,
|
||||
// the factory verifies tenant ownership and returns that store's engine;
|
||||
// otherwise it falls back to the tenant's configured retrievers.
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to create retrieval engine: %v", err)
|
||||
// Authorize every KB the caller asked for. Same-tenant KBs are
|
||||
// always accessible; foreign-tenant KBs (Organization-shared) must
|
||||
// pass an explicit per-KB permission check. Without this guard, a
|
||||
// caller could pass arbitrary KB UUIDs in params.KnowledgeBaseIDs
|
||||
// and reach foreign tenants' bound vector stores via the per-group
|
||||
// engine resolution downstream.
|
||||
if err := s.authorizeKBAccess(ctx, kbs, requestTenantID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Use 5x over-retrieval to ensure sufficient candidates for RRF fusion and reranking.
|
||||
// Scale proportionally when searching multiple KBs to maintain per-KB recall quality.
|
||||
// Explicit embedding-model consistency check. Multi-KB searches that
|
||||
// span different embedding spaces would otherwise silently produce
|
||||
// meaningless cross-model scores. Same-model wiki/graph KBs are
|
||||
// tolerated — see validateSameEmbeddingModel for the carve-out.
|
||||
if err := s.validateSameEmbeddingModel(ctx, kbs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Resolve the primary KB — embedding model + FAQ type come from this
|
||||
// one. Miss → 404 (no kbs[0] fallback; a silent pivot to an arbitrary
|
||||
// KB would hide caller bugs and reveal foreign KB metadata).
|
||||
kb := pickPrimary(kbs, id)
|
||||
if kb == nil {
|
||||
return nil, apperrors.NewNotFoundError("knowledge base not found")
|
||||
}
|
||||
|
||||
// Over-retrieval (existing rule, preserved): 5x per-KB matchCount,
|
||||
// floor of 50, capped at 500 across the whole search.
|
||||
matchCount := max(params.MatchCount*5, 50) * len(searchKBIDs)
|
||||
if matchCount > 500 {
|
||||
matchCount = 500
|
||||
}
|
||||
|
||||
// Build retrieval parameters for vector and keyword engines
|
||||
retrieveParams, err := s.buildRetrievalParams(ctx, retrieveEngine, kb, params, searchKBIDs, matchCount)
|
||||
// Compute the query embedding once before fan-out and propagate via
|
||||
// params.QueryEmbedding. Without this, each storeGroup's
|
||||
// buildRetrievalParams would re-embed the same query text — for N
|
||||
// stores that means N API calls of identical input.
|
||||
//
|
||||
// Skip when params already carries an embedding (e.g. the agent
|
||||
// pre-computed it) or when the primary KB has no vector indexing
|
||||
// configured.
|
||||
if len(params.QueryEmbedding) == 0 &&
|
||||
kb.IsVectorEnabled() && kb.EmbeddingModelID != "" &&
|
||||
!params.DisableVectorMatch {
|
||||
emb, embErr := s.GetQueryEmbedding(ctx, kb.ID, params.QueryText)
|
||||
if embErr != nil {
|
||||
return nil, embErr
|
||||
}
|
||||
params.QueryEmbedding = emb
|
||||
}
|
||||
|
||||
// Group KBs by (storeID, owner tenant), resolve the bound engine for
|
||||
// each group, and build the per-group base RetrieveParams once.
|
||||
groups, err := s.resolveStoreGroups(ctx, kb, kbs, params, matchCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(retrieveParams) == 0 {
|
||||
// No retrievable pipelines for this KB (e.g. a wiki-only or graph-only
|
||||
// KB that has neither vector nor keyword indexing). Return empty
|
||||
// results rather than erroring so callers that combine multiple KB
|
||||
// scopes (agent knowledge_search tool, chat pipeline, etc.) degrade
|
||||
// gracefully when one of the scopes is non-searchable.
|
||||
logger.Infof(ctx, "No retrievable indexing pipelines for KB %s (vector=%v, keyword=%v), returning empty results",
|
||||
kb.ID, kb.IsVectorEnabled(), kb.IsKeywordEnabled())
|
||||
if len(groups) == 0 || allBaseParamsEmpty(groups) {
|
||||
// Wiki-only / graph-only fan-out: every KB is non-retrievable.
|
||||
// Preserve the existing "return empty rather than error" contract
|
||||
// so agent tools that combine multiple KB scopes degrade gracefully.
|
||||
logger.Infof(ctx, "No retrievable indexing pipelines across %d KBs", len(kbs))
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Execute retrieval using the configured engines.
|
||||
// A dedicated span captures the actual vector/keyword DB round-trip
|
||||
// — this is the time previously visible in Langfuse only as the gap
|
||||
// between embedding generations and the rerank call.
|
||||
logger.Infof(ctx, "Starting retrieval, parameter count: %d", len(retrieveParams))
|
||||
retrieverTypes := make([]string, 0, len(retrieveParams))
|
||||
for _, rp := range retrieveParams {
|
||||
retrieverTypes = append(retrieverTypes, string(rp.RetrieverType))
|
||||
}
|
||||
// Execute retrieval with fan-out + score normalization (multi-store
|
||||
// only) and a langfuse span around the entire retrieve step.
|
||||
logger.Infof(ctx, "Starting multi-store retrieval, group count: %d", len(groups))
|
||||
retrieveCtx, retrieveSpan := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{
|
||||
Name: "retrieve",
|
||||
Input: map[string]interface{}{
|
||||
"kb_ids": searchKBIDs,
|
||||
"group_count": len(groups),
|
||||
"match_count": matchCount,
|
||||
"retrievers": retrieverTypes,
|
||||
},
|
||||
Metadata: map[string]interface{}{
|
||||
"param_count": len(retrieveParams),
|
||||
},
|
||||
})
|
||||
retrieveResults, err := retrieveEngine.Retrieve(retrieveCtx, retrieveParams)
|
||||
retrieveResults, err := s.retrieveFromStores(retrieveCtx, groups, retriever.EngineAwareNormalizer{})
|
||||
retrieveSpan.Finish(map[string]interface{}{
|
||||
"result_count": len(retrieveResults),
|
||||
"result_count": totalHits(retrieveResults),
|
||||
}, nil, err)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
@@ -175,13 +208,14 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Separate and fuse retrieval results
|
||||
// Separate and fuse retrieval results.
|
||||
vectorResults, keywordResults := classifyRetrievalResults(ctx, retrieveResults)
|
||||
if len(vectorResults) == 0 && len(keywordResults) == 0 {
|
||||
logger.Info(ctx, "No search results found")
|
||||
return nil, nil
|
||||
}
|
||||
logger.Infof(ctx, "Result count before fusion: vector=%d, keyword=%d", len(vectorResults), len(keywordResults))
|
||||
logger.Infof(ctx, "Result count before fusion: vector=%d, keyword=%d",
|
||||
len(vectorResults), len(keywordResults))
|
||||
|
||||
var retrievalCfg *types.RetrievalConfig
|
||||
if tenantInfo != nil {
|
||||
@@ -191,10 +225,17 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
|
||||
kb.EnsureDefaults()
|
||||
|
||||
// FAQ-specific post-processing: iterative retrieval or negative question filtering
|
||||
deduplicatedChunks = s.applyFAQPostProcessing(ctx, kb, deduplicatedChunks, vectorResults, retrieveEngine, retrieveParams, params, matchCount)
|
||||
// FAQ-specific post-processing now operates on storeGroups so the
|
||||
// iterative TopK growth applies uniformly across the fan-out. An
|
||||
// AppError from inside the iterative fan-out path (e.g. a per-group
|
||||
// timeout surfaced as ErrVectorStoreUnavailable) must surface to the
|
||||
// caller rather than be silently converted to a truncated chunk list.
|
||||
deduplicatedChunks, err = s.applyFAQPostProcessing(
|
||||
ctx, kb, deduplicatedChunks, vectorResults, groups, params, matchCount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Limit to MatchCount
|
||||
if len(deduplicatedChunks) > params.MatchCount {
|
||||
deduplicatedChunks = deduplicatedChunks[:params.MatchCount]
|
||||
}
|
||||
@@ -202,6 +243,50 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
return s.processSearchResults(ctx, deduplicatedChunks, params.SkipContextEnrichment)
|
||||
}
|
||||
|
||||
// pickPrimary returns the KB whose ID matches id, or nil if id is not in
|
||||
// scope. Callers map a nil result to NotFound; there is intentionally no
|
||||
// kbs[0] fallback because it would mask caller bugs and could leak an
|
||||
// unintended KB's embedding-model identity into the search path.
|
||||
//
|
||||
// The primary KB drives the embedding model and FAQ-type decisions for
|
||||
// buildRetrievalParams. If the caller selects a wiki-only / graph-only
|
||||
// KB as primary, the multi-KB search is implicitly demoted to
|
||||
// keyword-only retrieval — vector retrieval is skipped because
|
||||
// primary.IsVectorEnabled() is false. Callers that mix vector-enabled
|
||||
// and non-vector KBs should pass a vector-enabled KB as id.
|
||||
func pickPrimary(kbs []*types.KnowledgeBase, id string) *types.KnowledgeBase {
|
||||
for _, kb := range kbs {
|
||||
if kb.ID == id {
|
||||
return kb
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// allBaseParamsEmpty reports whether every store group has an empty
|
||||
// BaseParams slice. True only when every KB in scope is wiki-only or
|
||||
// graph-only with neither vector nor keyword indexing — HybridSearch then
|
||||
// returns nil so callers that combine searchable + non-searchable KBs
|
||||
// (agent tools, chat pipeline) degrade gracefully.
|
||||
func allBaseParamsEmpty(groups []*storeGroup) bool {
|
||||
for _, g := range groups {
|
||||
if len(g.BaseParams) > 0 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// totalHits counts the IndexWithScore entries across a slice of retrieve
|
||||
// results. Used only for langfuse span metadata.
|
||||
func totalHits(rrs []*types.RetrieveResult) int {
|
||||
n := 0
|
||||
for _, r := range rrs {
|
||||
n += len(r.Results)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// buildRetrievalParams constructs the vector and keyword retrieval parameters
|
||||
// based on the knowledge base type, engine capabilities, and search params.
|
||||
func (s *knowledgeBaseService) buildRetrievalParams(
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
apperrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// Default fan-out limits. Tuned for current production sizing: at most 4
|
||||
// stores per request and 30s per group, matching engine_factory.go defaults.
|
||||
const (
|
||||
defaultMultiStoreRetrieveTimeout = 30 * time.Second
|
||||
defaultMultiStoreFanoutLimit = 4
|
||||
)
|
||||
|
||||
// retrieveFromStores executes Retrieve per storeGroup with bounded
|
||||
// concurrency and a per-group timeout. When >1 group is present AND the
|
||||
// collected results span >1 engine type, the normalizer rescales vector
|
||||
// scores into [0, 1] before fusion.
|
||||
//
|
||||
// Failure policy: all-or-nothing. The first group error fails the whole
|
||||
// search and cancels siblings via errgroup, matching the existing single-
|
||||
// store behavior — chat/agent callers already treat "search failed" as
|
||||
// abort. A future PR can extract a MergePolicy interface (open/closed)
|
||||
// to add partial-result support without changing this function's callers.
|
||||
//
|
||||
// Fast path: len(groups) == 1 → returns the single engine's Retrieve
|
||||
// directly with zero fan-out overhead. This is the dominant case today
|
||||
// because every existing KB has vector_store_id = NULL.
|
||||
//
|
||||
// Concurrency cap: g.SetLimit(defaultMultiStoreFanoutLimit) bounds
|
||||
// per-request fan-out concurrency. NOTE: this caps application-level
|
||||
// goroutine count but does NOT set MaxOpenConns on the shared gorm pool
|
||||
// (which currently defaults to unlimited for Postgres in
|
||||
// container.go:541). Operators should still configure SetMaxOpenConns
|
||||
// per their PG max_connections budget.
|
||||
func (s *knowledgeBaseService) retrieveFromStores(
|
||||
ctx context.Context,
|
||||
groups []*storeGroup,
|
||||
normalizer retriever.ScoreNormalizer,
|
||||
) ([]*types.RetrieveResult, error) {
|
||||
if len(groups) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if len(groups) == 1 {
|
||||
return groups[0].Engine.Retrieve(ctx, paramsWithTopK(groups[0]))
|
||||
}
|
||||
|
||||
timeout := multiStoreRetrieveTimeout()
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
all []*types.RetrieveResult
|
||||
)
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
g.SetLimit(defaultMultiStoreFanoutLimit)
|
||||
|
||||
for i := range groups {
|
||||
grp := groups[i]
|
||||
g.Go(func() error {
|
||||
gcCtx, cancel := context.WithTimeout(gctx, timeout)
|
||||
defer cancel()
|
||||
res, err := grp.Engine.Retrieve(gcCtx, paramsWithTopK(grp))
|
||||
if err != nil {
|
||||
logger.WarnWithFields(gctx, logger.Fields{
|
||||
"tenant_id": grp.OwnerTenantID,
|
||||
"kb_count": len(grp.KBIDs),
|
||||
"store_kind": storeKindLabel(grp.StoreID),
|
||||
}, fmt.Sprintf("multi-store retrieve failed: %v", err))
|
||||
return fmt.Errorf("store group retrieve: %w", err)
|
||||
}
|
||||
mu.Lock()
|
||||
all = append(all, res...)
|
||||
mu.Unlock()
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := g.Wait(); err != nil {
|
||||
// Only treat an explicit client cancellation (context.Canceled)
|
||||
// as "the client gave up". A parent context whose deadline has
|
||||
// expired must still surface as a typed unavailable error so the
|
||||
// handler returns a clean 4xx instead of leaking the raw stdlib
|
||||
// DeadlineExceeded.
|
||||
if isParentCancelled(ctx) {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
// Any retrieve failure (per-group timeout, transport error,
|
||||
// upstream rejection) is collapsed into a single typed
|
||||
// unavailable error. The underlying cause is recorded in
|
||||
// structured logs above; the response body intentionally exposes
|
||||
// no internal detail.
|
||||
return nil, apperrors.NewVectorStoreUnavailableError(
|
||||
"vector retrieval failed for one or more bound stores")
|
||||
}
|
||||
|
||||
// Apply normalizer only when results span >1 distinct engine type.
|
||||
// Same-engine results keep their native scale — raw scores produced by
|
||||
// the same engine are directly comparable.
|
||||
if hasMixedEngineTypes(all) {
|
||||
seenUnknown := make(map[types.RetrieverEngineType]struct{})
|
||||
for _, rr := range all {
|
||||
for _, hit := range rr.Results {
|
||||
hit.Score = normalizer.Normalize(
|
||||
ctx, hit.Score, rr.RetrieverType, rr.RetrieverEngineType)
|
||||
}
|
||||
if !isKnownEngineType(rr.RetrieverEngineType) {
|
||||
if _, dup := seenUnknown[rr.RetrieverEngineType]; !dup {
|
||||
seenUnknown[rr.RetrieverEngineType] = struct{}{}
|
||||
// Engine type strings can originate from operator
|
||||
// configuration; sanitize before logging to defeat
|
||||
// CR/LF/tab log injection.
|
||||
logger.WarnWithFields(ctx, logger.Fields{
|
||||
"engine_type": secutils.SanitizeForLog(string(rr.RetrieverEngineType)),
|
||||
}, "score normalizer: unknown engine type, applying clamp01 fallback")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// paramsWithTopK builds a fresh []RetrieveParams for a group. BaseParams
|
||||
// stay immutable; only TopK is overridden. This isolates iterative FAQ's
|
||||
// TopK mutation from the goroutines that read params inside Retrieve,
|
||||
// so no aliasing assumption is required for the outer slice (and -race
|
||||
// will not flag this path).
|
||||
//
|
||||
// The value-copy of each RetrieveParams shallow-copies its reference
|
||||
// fields (Embedding, KnowledgeBaseIDs, TagIDs, KnowledgeIDs,
|
||||
// AdditionalParams). All fan-out goroutines from a single
|
||||
// retrieveFromStores call therefore share the same backing arrays for
|
||||
// those fields. Every retriever implementation today treats them as
|
||||
// read-only, so this is safe — but the immutability of reference
|
||||
// fields is a caller contract. If a future retriever ever mutates
|
||||
// Embedding in-place (e.g. to L2-normalize), -race will surface the
|
||||
// regression.
|
||||
func paramsWithTopK(g *storeGroup) []types.RetrieveParams {
|
||||
out := make([]types.RetrieveParams, len(g.BaseParams))
|
||||
for i, p := range g.BaseParams {
|
||||
p.TopK = g.TopK
|
||||
out[i] = p
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hasMixedEngineTypes returns true if results span 2+ distinct
|
||||
// RetrieverEngineType values. Empty/zero values count as their own bucket.
|
||||
func hasMixedEngineTypes(results []*types.RetrieveResult) bool {
|
||||
if len(results) < 2 {
|
||||
return false
|
||||
}
|
||||
first := results[0].RetrieverEngineType
|
||||
for _, r := range results[1:] {
|
||||
if r.RetrieverEngineType != first {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isKnownEngineType reports whether the given engine type has a
|
||||
// hard-coded normalization entry in EngineAwareNormalizer. Used by the
|
||||
// caller of Normalize to deduplicate "unknown engine" WARN logs per
|
||||
// request.
|
||||
func isKnownEngineType(t types.RetrieverEngineType) bool {
|
||||
switch t {
|
||||
case types.ElasticsearchRetrieverEngineType,
|
||||
types.ElasticFaissRetrieverEngineType,
|
||||
types.MilvusRetrieverEngineType,
|
||||
types.PostgresRetrieverEngineType,
|
||||
types.QdrantRetrieverEngineType,
|
||||
types.WeaviateRetrieverEngineType,
|
||||
types.SQLiteRetrieverEngineType,
|
||||
types.InfinityRetrieverEngineType,
|
||||
types.TencentVectorDBRetrieverEngineType,
|
||||
types.DorisRetrieverEngineType:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// multiStoreRetrieveTimeout reads MULTI_STORE_RETRIEVE_TIMEOUT_SEC; falls
|
||||
// back to defaultMultiStoreRetrieveTimeout (30s) on absence, parse error,
|
||||
// or non-positive values.
|
||||
func multiStoreRetrieveTimeout() time.Duration {
|
||||
raw := os.Getenv("MULTI_STORE_RETRIEVE_TIMEOUT_SEC")
|
||||
if raw == "" {
|
||||
return defaultMultiStoreRetrieveTimeout
|
||||
}
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil || n <= 0 {
|
||||
return defaultMultiStoreRetrieveTimeout
|
||||
}
|
||||
return time.Duration(n) * time.Second
|
||||
}
|
||||
|
||||
// storeKindLabel returns "env" or "bound" for log fields. Never echoes the
|
||||
// raw store UUID.
|
||||
func storeKindLabel(storeID string) string {
|
||||
if storeID == "" {
|
||||
return "env"
|
||||
}
|
||||
return "bound"
|
||||
}
|
||||
|
||||
// isParentCancelled reports whether the parent ctx was explicitly
|
||||
// cancelled by the caller (client hang-up). A parent ctx whose deadline
|
||||
// merely expired is NOT treated as a client cancel — that path falls
|
||||
// through to the typed 2201 AppError so middleware emits a clean 4xx
|
||||
// instead of leaking context.DeadlineExceeded.
|
||||
func isParentCancelled(ctx context.Context) bool {
|
||||
return errors.Is(ctx.Err(), context.Canceled)
|
||||
}
|
||||
@@ -0,0 +1,931 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
stderrors "errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
apperrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fan-out helper unit tests — no service dependencies
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestParamsWithTopK_RebuildsFresh(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
g := &storeGroup{
|
||||
BaseParams: []types.RetrieveParams{
|
||||
{Query: "q1", TopK: 10, RetrieverType: types.VectorRetrieverType},
|
||||
{Query: "q2", TopK: 20, RetrieverType: types.KeywordsRetrieverType},
|
||||
},
|
||||
TopK: 999,
|
||||
}
|
||||
out := paramsWithTopK(g)
|
||||
require.Len(t, out, 2)
|
||||
|
||||
// TopK is overridden from group.TopK.
|
||||
assert.Equal(t, 999, out[0].TopK)
|
||||
assert.Equal(t, 999, out[1].TopK)
|
||||
|
||||
// BaseParams is unchanged (immutable invariant).
|
||||
assert.Equal(t, 10, g.BaseParams[0].TopK)
|
||||
assert.Equal(t, 20, g.BaseParams[1].TopK)
|
||||
|
||||
// Output slice does not alias BaseParams (mutating out must not touch base).
|
||||
out[0].Query = "mutated"
|
||||
assert.Equal(t, "q1", g.BaseParams[0].Query)
|
||||
}
|
||||
|
||||
func TestHasMixedEngineTypes(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
name string
|
||||
in []*types.RetrieveResult
|
||||
want bool
|
||||
}{
|
||||
{"empty", nil, false},
|
||||
{"single", []*types.RetrieveResult{
|
||||
{RetrieverEngineType: types.PostgresRetrieverEngineType},
|
||||
}, false},
|
||||
{"two same", []*types.RetrieveResult{
|
||||
{RetrieverEngineType: types.PostgresRetrieverEngineType},
|
||||
{RetrieverEngineType: types.PostgresRetrieverEngineType},
|
||||
}, false},
|
||||
{"two different", []*types.RetrieveResult{
|
||||
{RetrieverEngineType: types.PostgresRetrieverEngineType},
|
||||
{RetrieverEngineType: types.ElasticsearchRetrieverEngineType},
|
||||
}, true},
|
||||
{"empty + nonempty engine", []*types.RetrieveResult{
|
||||
{RetrieverEngineType: ""},
|
||||
{RetrieverEngineType: types.PostgresRetrieverEngineType},
|
||||
}, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := hasMixedEngineTypes(tc.in)
|
||||
if got != tc.want {
|
||||
t.Errorf("%s: want %v, got %v", tc.name, tc.want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsKnownEngineType(t *testing.T) {
|
||||
t.Parallel()
|
||||
known := []types.RetrieverEngineType{
|
||||
types.PostgresRetrieverEngineType,
|
||||
types.ElasticsearchRetrieverEngineType,
|
||||
types.ElasticFaissRetrieverEngineType,
|
||||
types.MilvusRetrieverEngineType,
|
||||
types.QdrantRetrieverEngineType,
|
||||
types.WeaviateRetrieverEngineType,
|
||||
types.SQLiteRetrieverEngineType,
|
||||
types.InfinityRetrieverEngineType,
|
||||
types.TencentVectorDBRetrieverEngineType,
|
||||
types.DorisRetrieverEngineType,
|
||||
}
|
||||
for _, k := range known {
|
||||
if !isKnownEngineType(k) {
|
||||
t.Errorf("expected %s to be known", k)
|
||||
}
|
||||
}
|
||||
if isKnownEngineType("") || isKnownEngineType("nosuch") {
|
||||
t.Error("unknown engine misclassified as known")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreKindLabel(t *testing.T) {
|
||||
t.Parallel()
|
||||
if storeKindLabel("") != "env" {
|
||||
t.Errorf("empty storeID should be env")
|
||||
}
|
||||
if storeKindLabel("0193b8a0-1111-7000-8000-000000000001") != "bound" {
|
||||
t.Errorf("non-empty storeID should be bound")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiStoreRetrieveTimeout(t *testing.T) {
|
||||
// Not Parallel — mutates a process-global env var.
|
||||
t.Setenv("MULTI_STORE_RETRIEVE_TIMEOUT_SEC", "")
|
||||
os.Unsetenv("MULTI_STORE_RETRIEVE_TIMEOUT_SEC")
|
||||
if got := multiStoreRetrieveTimeout(); got != defaultMultiStoreRetrieveTimeout {
|
||||
t.Errorf("default: want %v, got %v", defaultMultiStoreRetrieveTimeout, got)
|
||||
}
|
||||
|
||||
t.Setenv("MULTI_STORE_RETRIEVE_TIMEOUT_SEC", "7")
|
||||
if got := multiStoreRetrieveTimeout(); got != 7*time.Second {
|
||||
t.Errorf("env=7: want 7s, got %v", got)
|
||||
}
|
||||
|
||||
t.Setenv("MULTI_STORE_RETRIEVE_TIMEOUT_SEC", "garbage")
|
||||
if got := multiStoreRetrieveTimeout(); got != defaultMultiStoreRetrieveTimeout {
|
||||
t.Errorf("parse-fail fallback: want default, got %v", got)
|
||||
}
|
||||
|
||||
t.Setenv("MULTI_STORE_RETRIEVE_TIMEOUT_SEC", "-3")
|
||||
if got := multiStoreRetrieveTimeout(); got != defaultMultiStoreRetrieveTimeout {
|
||||
t.Errorf("negative fallback: want default, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPickPrimary(t *testing.T) {
|
||||
t.Parallel()
|
||||
kbs := []*types.KnowledgeBase{
|
||||
{ID: "kb-a"}, {ID: "kb-b"}, {ID: "kb-c"},
|
||||
}
|
||||
if pickPrimary(kbs, "kb-b").ID != "kb-b" {
|
||||
t.Errorf("hit case failed")
|
||||
}
|
||||
// Miss returns nil rather than falling back to kbs[0]; the helper's
|
||||
// godoc explains the rationale (surface caller bugs, avoid leaking
|
||||
// unintended KB metadata).
|
||||
if pickPrimary(kbs, "kb-missing") != nil {
|
||||
t.Errorf("miss should return nil")
|
||||
}
|
||||
if pickPrimary(nil, "any") != nil {
|
||||
t.Errorf("nil slice should return nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllBaseParamsEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
empty := []*storeGroup{{}, {BaseParams: nil}}
|
||||
if !allBaseParamsEmpty(empty) {
|
||||
t.Errorf("all-empty case failed")
|
||||
}
|
||||
hasSome := []*storeGroup{{BaseParams: nil}, {BaseParams: []types.RetrieveParams{{Query: "x"}}}}
|
||||
if allBaseParamsEmpty(hasSome) {
|
||||
t.Errorf("partial-nonempty case must return false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTotalHits(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := totalHits([]*types.RetrieveResult{
|
||||
{Results: []*types.IndexWithScore{{}, {}}},
|
||||
{Results: []*types.IndexWithScore{{}}},
|
||||
{Results: nil},
|
||||
})
|
||||
if got != 3 {
|
||||
t.Errorf("want 3, got %d", got)
|
||||
}
|
||||
if totalHits(nil) != 0 {
|
||||
t.Errorf("nil → 0")
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// classifyFactoryError — sentinel→typed AppError mapping
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func TestClassifyFactoryError_ForbiddenMapsTo2200(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := classifyFactoryError(context.Background(),
|
||||
retriever.ErrVectorStoreForbidden, 99, "0193-store")
|
||||
app, ok := apperrors.IsAppError(err)
|
||||
require.True(t, ok, "expected AppError, got %T", err)
|
||||
assert.Equal(t, apperrors.ErrVectorStoreBindingInvalid, app.Code)
|
||||
// User-facing message MUST NOT echo the store UUID.
|
||||
if strings.Contains(app.Message, "0193-store") {
|
||||
t.Fatalf("AppError message leaked store UUID: %q", app.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyFactoryError_NotFoundMapsTo2201(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := classifyFactoryError(context.Background(),
|
||||
retriever.ErrVectorStoreNotFound, 7, "store-id-x")
|
||||
app, ok := apperrors.IsAppError(err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, apperrors.ErrVectorStoreUnavailable, app.Code)
|
||||
if strings.Contains(app.Message, "store-id-x") {
|
||||
t.Fatalf("AppError message leaked store UUID: %q", app.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyFactoryError_TenantInfoMissingMaps(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := classifyFactoryError(context.Background(),
|
||||
retriever.ErrTenantInfoMissing, 0, "")
|
||||
app, ok := apperrors.IsAppError(err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, apperrors.ErrVectorStoreBindingInvalid, app.Code)
|
||||
}
|
||||
|
||||
func TestClassifyFactoryError_GenericErrorPassesThrough(t *testing.T) {
|
||||
t.Parallel()
|
||||
raw := stderrors.New("generic infra failure")
|
||||
err := classifyFactoryError(context.Background(), raw, 1, "x")
|
||||
// Generic errors are returned as-is; the handler decides how to wrap.
|
||||
if !stderrors.Is(err, raw) {
|
||||
t.Fatalf("expected raw error passthrough, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// validateSameEmbeddingModel — uses a narrowly faked model service
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// fakeModelSvcForKeys is the smallest ModelService subset needed by
|
||||
// ResolveEmbeddingModelKeys → validateSameEmbeddingModel.
|
||||
//
|
||||
// ResolveEmbeddingModelKeys calls GetModelByID(ctx, modelID); the rest are
|
||||
// panic-only — none are exercised by these tests.
|
||||
type fakeModelSvcForKeys struct {
|
||||
byID map[string]*types.Model // modelID → model
|
||||
interfaces.ModelService
|
||||
}
|
||||
|
||||
func (f *fakeModelSvcForKeys) GetModelByID(ctx context.Context, id string) (*types.Model, error) {
|
||||
if m, ok := f.byID[id]; ok {
|
||||
return m, nil
|
||||
}
|
||||
return nil, fmt.Errorf("model %s not found", id)
|
||||
}
|
||||
|
||||
func buildModel(id, name, baseURL string) *types.Model {
|
||||
return &types.Model{
|
||||
ID: id,
|
||||
Name: name,
|
||||
Parameters: types.ModelParameters{
|
||||
BaseURL: baseURL,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newKBSvcForValidate(modelService interfaces.ModelService) *knowledgeBaseService {
|
||||
return &knowledgeBaseService{
|
||||
modelService: modelService,
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSameEmbeddingModel_SingleKB_NoOp(t *testing.T) {
|
||||
t.Parallel()
|
||||
svc := newKBSvcForValidate(&fakeModelSvcForKeys{})
|
||||
err := svc.validateSameEmbeddingModel(context.Background(), []*types.KnowledgeBase{
|
||||
{ID: "kb-1", EmbeddingModelID: "m-1", TenantID: 1},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestValidateSameEmbeddingModel_SameIdentity_OK(t *testing.T) {
|
||||
t.Parallel()
|
||||
models := map[string]*types.Model{
|
||||
"m-1": buildModel("m-1", "bge-m3", "http://embed-a"),
|
||||
"m-2": buildModel("m-2", "bge-m3", "http://embed-a"), // same identity key
|
||||
}
|
||||
svc := newKBSvcForValidate(&fakeModelSvcForKeys{byID: models})
|
||||
err := svc.validateSameEmbeddingModel(context.Background(), []*types.KnowledgeBase{
|
||||
{ID: "kb-1", EmbeddingModelID: "m-1", TenantID: 1},
|
||||
{ID: "kb-2", EmbeddingModelID: "m-2", TenantID: 2}, // cross-tenant
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestValidateSameEmbeddingModel_DifferentIdentity_400(t *testing.T) {
|
||||
t.Parallel()
|
||||
models := map[string]*types.Model{
|
||||
"m-1": buildModel("m-1", "bge-m3", "http://embed-a"),
|
||||
"m-2": buildModel("m-2", "openai-3", "http://embed-b"), // different
|
||||
}
|
||||
svc := newKBSvcForValidate(&fakeModelSvcForKeys{byID: models})
|
||||
err := svc.validateSameEmbeddingModel(context.Background(), []*types.KnowledgeBase{
|
||||
{ID: "kb-1", EmbeddingModelID: "m-1", TenantID: 1},
|
||||
{ID: "kb-2", EmbeddingModelID: "m-2", TenantID: 1},
|
||||
})
|
||||
app, ok := apperrors.IsAppError(err)
|
||||
require.True(t, ok, "expected AppError, got %T", err)
|
||||
assert.Equal(t, apperrors.ErrBadRequest, app.Code)
|
||||
}
|
||||
|
||||
func TestValidateSameEmbeddingModel_WikiOnly_Tolerated(t *testing.T) {
|
||||
t.Parallel()
|
||||
models := map[string]*types.Model{
|
||||
"m-1": buildModel("m-1", "bge-m3", "http://embed-a"),
|
||||
}
|
||||
svc := newKBSvcForValidate(&fakeModelSvcForKeys{byID: models})
|
||||
// One vector KB + one wiki-only (empty EmbeddingModelID) → tolerated.
|
||||
err := svc.validateSameEmbeddingModel(context.Background(), []*types.KnowledgeBase{
|
||||
{ID: "kb-vec", EmbeddingModelID: "m-1", TenantID: 1},
|
||||
{ID: "kb-wiki", EmbeddingModelID: "", TenantID: 1},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestValidateSameEmbeddingModel_LogInjection_Sanitized(t *testing.T) {
|
||||
t.Parallel()
|
||||
// BaseURL with CRLF — SanitizeForLog should keep us safe.
|
||||
models := map[string]*types.Model{
|
||||
"m-1": buildModel("m-1", "bge-m3", "http://a\r\nfoo"),
|
||||
"m-2": buildModel("m-2", "openai", "http://b\r\nbar"),
|
||||
}
|
||||
svc := newKBSvcForValidate(&fakeModelSvcForKeys{byID: models})
|
||||
err := svc.validateSameEmbeddingModel(context.Background(), []*types.KnowledgeBase{
|
||||
{ID: "kb-1", EmbeddingModelID: "m-1", TenantID: 1},
|
||||
{ID: "kb-2", EmbeddingModelID: "m-2", TenantID: 1},
|
||||
})
|
||||
// We only verify the 400 still happens; sanitize is exercised in the
|
||||
// hot path of WarnWithFields and is unit-tested in internal/utils.
|
||||
app, ok := apperrors.IsAppError(err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, apperrors.ErrBadRequest, app.Code)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// retrieveFromStores fan-out behavior
|
||||
//
|
||||
// We construct CompositeRetrieveEngines through the PR2 factory so we
|
||||
// exercise the real wiring rather than reaching into unexported fields.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// fakeRetrieveEngineService implements interfaces.RetrieveEngineService. It
|
||||
// returns canned RetrieveResults, optionally sleeps to simulate a hung
|
||||
// engine, or returns an error. Only Retrieve, EngineType, and Support are
|
||||
// exercised; the rest panic.
|
||||
type fakeRetrieveEngineService struct {
|
||||
engineType types.RetrieverEngineType
|
||||
support []types.RetrieverType
|
||||
canned []*types.IndexWithScore
|
||||
cannedErr error
|
||||
sleep time.Duration
|
||||
retrieveCalls atomic.Int64
|
||||
}
|
||||
|
||||
func (f *fakeRetrieveEngineService) EngineType() types.RetrieverEngineType {
|
||||
return f.engineType
|
||||
}
|
||||
|
||||
func (f *fakeRetrieveEngineService) Support() []types.RetrieverType { return f.support }
|
||||
|
||||
func (f *fakeRetrieveEngineService) Retrieve(ctx context.Context, p types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
||||
f.retrieveCalls.Add(1)
|
||||
if f.sleep > 0 {
|
||||
select {
|
||||
case <-time.After(f.sleep):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
if f.cannedErr != nil {
|
||||
return nil, f.cannedErr
|
||||
}
|
||||
return []*types.RetrieveResult{{
|
||||
Results: f.canned,
|
||||
RetrieverEngineType: f.engineType,
|
||||
RetrieverType: p.RetrieverType,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func (f *fakeRetrieveEngineService) Index(context.Context, embedding.Embedder, *types.IndexInfo, []types.RetrieverType) error {
|
||||
panic("unused")
|
||||
}
|
||||
func (f *fakeRetrieveEngineService) BatchIndex(context.Context, embedding.Embedder, []*types.IndexInfo, []types.RetrieverType) error {
|
||||
panic("unused")
|
||||
}
|
||||
func (f *fakeRetrieveEngineService) EstimateStorageSize(context.Context, embedding.Embedder, []*types.IndexInfo, []types.RetrieverType) int64 {
|
||||
panic("unused")
|
||||
}
|
||||
func (f *fakeRetrieveEngineService) CopyIndices(context.Context, string, map[string]string, map[string]string, string, int, string) error {
|
||||
panic("unused")
|
||||
}
|
||||
func (f *fakeRetrieveEngineService) DeleteByChunkIDList(context.Context, []string, int, string) error {
|
||||
panic("unused")
|
||||
}
|
||||
func (f *fakeRetrieveEngineService) DeleteBySourceIDList(context.Context, []string, int, string) error {
|
||||
panic("unused")
|
||||
}
|
||||
func (f *fakeRetrieveEngineService) DeleteByKnowledgeIDList(context.Context, []string, int, string) error {
|
||||
panic("unused")
|
||||
}
|
||||
func (f *fakeRetrieveEngineService) BatchUpdateChunkEnabledStatus(context.Context, map[string]bool) error {
|
||||
panic("unused")
|
||||
}
|
||||
func (f *fakeRetrieveEngineService) BatchUpdateChunkTagID(context.Context, map[string]string) error {
|
||||
panic("unused")
|
||||
}
|
||||
|
||||
var _ interfaces.RetrieveEngineService = (*fakeRetrieveEngineService)(nil)
|
||||
|
||||
// fakeFanoutRegistry returns non-nil RetrieveEngineService instances so
|
||||
// that resolveBoundEngine produces a real CompositeRetrieveEngine for
|
||||
// fan-out tests. The simpler fakeRegistry used by other test files
|
||||
// returns (nil, nil) from GetByStoreID, which is only useful for tests
|
||||
// that never reach the composite construction step.
|
||||
type fakeFanoutRegistry struct {
|
||||
byStore map[string]interfaces.RetrieveEngineService
|
||||
}
|
||||
|
||||
func (r *fakeFanoutRegistry) Register(interfaces.RetrieveEngineService) error { return nil }
|
||||
func (r *fakeFanoutRegistry) GetRetrieveEngineService(types.RetrieverEngineType) (interfaces.RetrieveEngineService, error) {
|
||||
return nil, stderrors.New("not used in fan-out tests")
|
||||
}
|
||||
func (r *fakeFanoutRegistry) GetAllRetrieveEngineServices() []interfaces.RetrieveEngineService {
|
||||
return nil
|
||||
}
|
||||
func (r *fakeFanoutRegistry) GetByStoreID(id string) (interfaces.RetrieveEngineService, error) {
|
||||
if svc, ok := r.byStore[id]; ok {
|
||||
return svc, nil
|
||||
}
|
||||
return nil, stderrors.New("store not registered")
|
||||
}
|
||||
|
||||
func buildBoundComposite(t *testing.T, svc interfaces.RetrieveEngineService) *retriever.CompositeRetrieveEngine {
|
||||
t.Helper()
|
||||
const storeID = "00000000-0000-0000-0000-000000000001"
|
||||
reg := &fakeFanoutRegistry{byStore: map[string]interfaces.RetrieveEngineService{storeID: svc}}
|
||||
own := &fakeOwnership{owned: map[string]uint64{storeID: 1}}
|
||||
sid := storeID
|
||||
composite, err := retriever.CreateRetrieveEngineForKB(
|
||||
context.Background(), reg, own, 1, &sid)
|
||||
require.NoError(t, err)
|
||||
return composite
|
||||
}
|
||||
|
||||
func vectorParams(query string) []types.RetrieveParams {
|
||||
return []types.RetrieveParams{{
|
||||
Query: query,
|
||||
TopK: 50,
|
||||
RetrieverType: types.VectorRetrieverType,
|
||||
}}
|
||||
}
|
||||
|
||||
func TestRetrieveFromStores_Empty(t *testing.T) {
|
||||
t.Parallel()
|
||||
res, err := (&knowledgeBaseService{}).retrieveFromStores(
|
||||
context.Background(), nil, retriever.EngineAwareNormalizer{})
|
||||
assert.NoError(t, err)
|
||||
assert.Nil(t, res)
|
||||
}
|
||||
|
||||
func TestRetrieveFromStores_SingleGroupFastPath(t *testing.T) {
|
||||
t.Parallel()
|
||||
fake := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{
|
||||
{ChunkID: "c1", Score: 0.8},
|
||||
},
|
||||
}
|
||||
composite := buildBoundComposite(t, fake)
|
||||
g := &storeGroup{
|
||||
StoreID: "store-x",
|
||||
KBIDs: []string{"kb-1"},
|
||||
Engine: composite,
|
||||
BaseParams: vectorParams("q"),
|
||||
TopK: 50,
|
||||
}
|
||||
|
||||
s := &knowledgeBaseService{}
|
||||
res, err := s.retrieveFromStores(context.Background(),
|
||||
[]*storeGroup{g}, retriever.EngineAwareNormalizer{})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, res, 1)
|
||||
assert.Equal(t, types.PostgresRetrieverEngineType, res[0].RetrieverEngineType)
|
||||
assert.Equal(t, 1, int(fake.retrieveCalls.Load()))
|
||||
|
||||
// Single-group raw scores must NOT be normalized (passthrough native scale).
|
||||
if len(res[0].Results) > 0 {
|
||||
assert.InDelta(t, 0.8, res[0].Results[0].Score, 1e-9)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetrieveFromStores_MultiGroupParallel_Concat(t *testing.T) {
|
||||
t.Parallel()
|
||||
fakeA := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "a1", Score: 0.6}},
|
||||
}
|
||||
fakeB := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType, // same engine type
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "b1", Score: 0.4}},
|
||||
}
|
||||
groups := []*storeGroup{
|
||||
{Engine: buildBoundComposite(t, fakeA), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-a"}},
|
||||
{Engine: buildBoundComposite(t, fakeB), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-b"}},
|
||||
}
|
||||
|
||||
s := &knowledgeBaseService{}
|
||||
res, err := s.retrieveFromStores(context.Background(),
|
||||
groups, retriever.EngineAwareNormalizer{})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Combined hit count = 2 (a1 + b1).
|
||||
gotIDs := map[string]bool{}
|
||||
for _, rr := range res {
|
||||
for _, hit := range rr.Results {
|
||||
gotIDs[hit.ChunkID] = true
|
||||
}
|
||||
}
|
||||
assert.True(t, gotIDs["a1"])
|
||||
assert.True(t, gotIDs["b1"])
|
||||
|
||||
// Same engine type across groups → normalizer NOT applied; raw scores kept.
|
||||
scoresByChunk := map[string]float64{}
|
||||
for _, rr := range res {
|
||||
for _, hit := range rr.Results {
|
||||
scoresByChunk[hit.ChunkID] = hit.Score
|
||||
}
|
||||
}
|
||||
assert.InDelta(t, 0.6, scoresByChunk["a1"], 1e-9)
|
||||
assert.InDelta(t, 0.4, scoresByChunk["b1"], 1e-9)
|
||||
}
|
||||
|
||||
func TestRetrieveFromStores_MixedEngine_Normalizes(t *testing.T) {
|
||||
t.Parallel()
|
||||
// ES returns raw cosine in [-1, 1]. Without normalization, mixing
|
||||
// with PG (already [0, 1]) ranks ES too high. The normalizer rescales
|
||||
// ES scores to [0, 1] before fusion.
|
||||
fakeES := &fakeRetrieveEngineService{
|
||||
engineType: types.ElasticsearchRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "es1", Score: 1.0}}, // top cosine
|
||||
}
|
||||
fakePG := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "pg1", Score: 0.9}},
|
||||
}
|
||||
groups := []*storeGroup{
|
||||
{Engine: buildBoundComposite(t, fakeES), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-es"}},
|
||||
{Engine: buildBoundComposite(t, fakePG), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-pg"}},
|
||||
}
|
||||
|
||||
s := &knowledgeBaseService{}
|
||||
res, err := s.retrieveFromStores(context.Background(),
|
||||
groups, retriever.EngineAwareNormalizer{})
|
||||
require.NoError(t, err)
|
||||
|
||||
scoresByChunk := map[string]float64{}
|
||||
for _, rr := range res {
|
||||
for _, hit := range rr.Results {
|
||||
scoresByChunk[hit.ChunkID] = hit.Score
|
||||
}
|
||||
}
|
||||
// ES 1.0 → (1+1)/2 = 1.0 (top), PG 0.9 → 0.9 unchanged.
|
||||
assert.InDelta(t, 1.0, scoresByChunk["es1"], 1e-9)
|
||||
assert.InDelta(t, 0.9, scoresByChunk["pg1"], 1e-9)
|
||||
|
||||
// Now check that an ES cosine of -0.4 normalizes to 0.3 (below PG 0.9).
|
||||
fakeES2 := &fakeRetrieveEngineService{
|
||||
engineType: types.ElasticsearchRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "es2", Score: -0.4}},
|
||||
}
|
||||
fakePG2 := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "pg2", Score: 0.8}},
|
||||
}
|
||||
groups2 := []*storeGroup{
|
||||
{Engine: buildBoundComposite(t, fakeES2), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-es2"}},
|
||||
{Engine: buildBoundComposite(t, fakePG2), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-pg2"}},
|
||||
}
|
||||
res2, err := s.retrieveFromStores(context.Background(), groups2, retriever.EngineAwareNormalizer{})
|
||||
require.NoError(t, err)
|
||||
scoresByChunk2 := map[string]float64{}
|
||||
for _, rr := range res2 {
|
||||
for _, hit := range rr.Results {
|
||||
scoresByChunk2[hit.ChunkID] = hit.Score
|
||||
}
|
||||
}
|
||||
assert.InDelta(t, 0.3, scoresByChunk2["es2"], 1e-9)
|
||||
assert.InDelta(t, 0.8, scoresByChunk2["pg2"], 1e-9)
|
||||
}
|
||||
|
||||
func TestRetrieveFromStores_KeywordPassthroughOnMixed(t *testing.T) {
|
||||
t.Parallel()
|
||||
// One ES vector group + one PG keyword group. Even with mixed engine
|
||||
// types, the keyword score is NOT rescaled (BM25 unbounded — RRF
|
||||
// fusion handles via rank).
|
||||
fakeES := &fakeRetrieveEngineService{
|
||||
engineType: types.ElasticsearchRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "v1", Score: 1.0}},
|
||||
}
|
||||
fakePGKW := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.KeywordsRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "k1", Score: 14.3}}, // BM25
|
||||
}
|
||||
groups := []*storeGroup{
|
||||
{Engine: buildBoundComposite(t, fakeES), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-es"}},
|
||||
{Engine: buildBoundComposite(t, fakePGKW), BaseParams: []types.RetrieveParams{{Query: "q", TopK: 50, RetrieverType: types.KeywordsRetrieverType}}, TopK: 50, KBIDs: []string{"kb-pg"}},
|
||||
}
|
||||
s := &knowledgeBaseService{}
|
||||
res, err := s.retrieveFromStores(context.Background(), groups, retriever.EngineAwareNormalizer{})
|
||||
require.NoError(t, err)
|
||||
|
||||
scoresByChunk := map[string]float64{}
|
||||
for _, rr := range res {
|
||||
for _, hit := range rr.Results {
|
||||
scoresByChunk[hit.ChunkID] = hit.Score
|
||||
}
|
||||
}
|
||||
assert.InDelta(t, 1.0, scoresByChunk["v1"], 1e-9)
|
||||
// Keyword score MUST survive untouched.
|
||||
assert.InDelta(t, 14.3, scoresByChunk["k1"], 1e-9)
|
||||
}
|
||||
|
||||
func TestRetrieveFromStores_OneGroupFails_AllFail(t *testing.T) {
|
||||
t.Parallel()
|
||||
fakeOK := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "ok1", Score: 0.5}},
|
||||
}
|
||||
fakeBad := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
cannedErr: stderrors.New("simulated retrieve failure"),
|
||||
}
|
||||
groups := []*storeGroup{
|
||||
{Engine: buildBoundComposite(t, fakeOK), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-ok"}},
|
||||
{Engine: buildBoundComposite(t, fakeBad), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-bad"}},
|
||||
}
|
||||
s := &knowledgeBaseService{}
|
||||
_, err := s.retrieveFromStores(context.Background(), groups, retriever.EngineAwareNormalizer{})
|
||||
require.Error(t, err)
|
||||
|
||||
// Generic infra failure → 2201 (Unavailable). Message MUST NOT echo
|
||||
// the raw error or any store UUID.
|
||||
app, ok := apperrors.IsAppError(err)
|
||||
require.True(t, ok, "expected typed AppError, got %T", err)
|
||||
assert.Equal(t, apperrors.ErrVectorStoreUnavailable, app.Code)
|
||||
if strings.Contains(app.Message, "simulated") {
|
||||
t.Fatalf("AppError message leaked raw err string: %q", app.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetrieveFromStores_PerGroupTimeout(t *testing.T) {
|
||||
// Not Parallel — mutates env. t.Setenv restores cleanly after the test.
|
||||
t.Setenv("MULTI_STORE_RETRIEVE_TIMEOUT_SEC", "1")
|
||||
|
||||
fakeSlow := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
sleep: 3 * time.Second, // exceeds 1s timeout
|
||||
}
|
||||
fakeFast := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "fast1", Score: 0.7}},
|
||||
}
|
||||
groups := []*storeGroup{
|
||||
{Engine: buildBoundComposite(t, fakeSlow), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-slow"}},
|
||||
{Engine: buildBoundComposite(t, fakeFast), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-fast"}},
|
||||
}
|
||||
|
||||
s := &knowledgeBaseService{}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := s.retrieveFromStores(ctx, groups, retriever.EngineAwareNormalizer{})
|
||||
require.Error(t, err)
|
||||
app, ok := apperrors.IsAppError(err)
|
||||
require.True(t, ok, "expected typed AppError on timeout, got %T", err)
|
||||
assert.Equal(t, apperrors.ErrVectorStoreUnavailable, app.Code)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// authorizeKBAccess — per-KB authorization on multi-KB search scope
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// fakeKBShareForAuth implements just enough of KBShareService for the
|
||||
// authorizeKBAccess test matrix. Only HasTenantKBPermission is exercised;
|
||||
// the embedded interface keeps the type assignable.
|
||||
type fakeKBShareForAuth struct {
|
||||
// allowed maps kbID → tenantID → allowed. Mirrors the Plan 3 (#1303)
|
||||
// per-tenant permission model.
|
||||
allowed map[string]map[uint64]bool
|
||||
err error
|
||||
interfaces.KBShareService
|
||||
}
|
||||
|
||||
func (f *fakeKBShareForAuth) HasTenantKBPermission(
|
||||
_ context.Context, kbID string, callerTenantID uint64,
|
||||
_ types.TenantRole, _ types.OrgMemberRole,
|
||||
) (bool, error) {
|
||||
if f.err != nil {
|
||||
return false, f.err
|
||||
}
|
||||
if perTenant, ok := f.allowed[kbID]; ok {
|
||||
return perTenant[callerTenantID], nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func ctxWithTenantForAuth(tenantID uint64) context.Context {
|
||||
return context.WithValue(context.Background(), types.TenantIDContextKey, tenantID)
|
||||
}
|
||||
|
||||
func TestAuthorizeKBAccess_SameTenantAllPass(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := &knowledgeBaseService{kbShareService: &fakeKBShareForAuth{}}
|
||||
kbs := []*types.KnowledgeBase{
|
||||
{ID: "kb-1", TenantID: 7},
|
||||
{ID: "kb-2", TenantID: 7},
|
||||
}
|
||||
err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs, 7)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestAuthorizeKBAccess_ForeignTenantWithShare_OK(t *testing.T) {
|
||||
t.Parallel()
|
||||
share := &fakeKBShareForAuth{
|
||||
allowed: map[string]map[uint64]bool{
|
||||
"kb-foreign": {7: true},
|
||||
},
|
||||
}
|
||||
s := &knowledgeBaseService{kbShareService: share}
|
||||
kbs := []*types.KnowledgeBase{
|
||||
{ID: "kb-own", TenantID: 7},
|
||||
{ID: "kb-foreign", TenantID: 99},
|
||||
}
|
||||
err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs, 7)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestAuthorizeKBAccess_ForeignTenantNoShare_NotFound(t *testing.T) {
|
||||
t.Parallel()
|
||||
share := &fakeKBShareForAuth{
|
||||
allowed: map[string]map[uint64]bool{
|
||||
// kb-foreign explicitly NOT in allowed map
|
||||
},
|
||||
}
|
||||
s := &knowledgeBaseService{kbShareService: share}
|
||||
kbs := []*types.KnowledgeBase{
|
||||
{ID: "kb-foreign", TenantID: 99},
|
||||
}
|
||||
err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs, 7)
|
||||
require.Error(t, err)
|
||||
app, ok := apperrors.IsAppError(err)
|
||||
require.True(t, ok, "expected typed AppError, got %T", err)
|
||||
assert.Equal(t, apperrors.ErrNotFound, app.Code,
|
||||
"reject must surface as NotFound to avoid leaking foreign KB existence")
|
||||
}
|
||||
|
||||
func TestAuthorizeKBAccess_PermissionLookupError_500(t *testing.T) {
|
||||
t.Parallel()
|
||||
share := &fakeKBShareForAuth{err: stderrors.New("share infra down")}
|
||||
s := &knowledgeBaseService{kbShareService: share}
|
||||
kbs := []*types.KnowledgeBase{{ID: "kb-foreign", TenantID: 99}}
|
||||
err := s.authorizeKBAccess(ctxWithTenantForAuth(7), kbs, 7)
|
||||
require.Error(t, err)
|
||||
app, ok := apperrors.IsAppError(err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, apperrors.ErrInternalServer, app.Code)
|
||||
}
|
||||
|
||||
func TestAuthorizeKBAccess_EmptyKBs_OK(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := &knowledgeBaseService{kbShareService: &fakeKBShareForAuth{}}
|
||||
err := s.authorizeKBAccess(ctxWithTenantForAuth(7), nil, 7)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestIterativeRetrieve_PropagatesTypedAppError pins the contract that
|
||||
// a typed AppError raised inside the iterative FAQ fan-out is
|
||||
// propagated upward to HybridSearch rather than being silently
|
||||
// converted into a truncated chunk list. Without this guarantee, a
|
||||
// per-group timeout or binding-invalid error during iteration N would
|
||||
// return whatever partial chunks landed in iterations 0..N-1 with no
|
||||
// error signal to the client.
|
||||
func TestIterativeRetrieve_PropagatesTypedAppError(t *testing.T) {
|
||||
t.Parallel()
|
||||
bad := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
cannedErr: stderrors.New("simulated retrieve failure"),
|
||||
}
|
||||
groups := []*storeGroup{
|
||||
{Engine: buildBoundComposite(t, bad), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-bad"}},
|
||||
{Engine: buildBoundComposite(t, bad), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-bad-2"}},
|
||||
}
|
||||
s := &knowledgeBaseService{}
|
||||
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))
|
||||
results, err := s.iterativeRetrieveWithDeduplication(ctx, groups, 10, "q")
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, results, "results must be nil so HybridSearch returns a clean error response")
|
||||
app, ok := apperrors.IsAppError(err)
|
||||
require.True(t, ok, "expected typed AppError to propagate, got %T", err)
|
||||
assert.Equal(t, apperrors.ErrVectorStoreUnavailable, app.Code)
|
||||
}
|
||||
|
||||
// TestApplyFAQPostProcessing_PropagatesError is the matching test at the
|
||||
// applyFAQPostProcessing layer — the error path must surface upward.
|
||||
func TestApplyFAQPostProcessing_PropagatesError(t *testing.T) {
|
||||
t.Parallel()
|
||||
bad := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
cannedErr: stderrors.New("simulated retrieve failure"),
|
||||
}
|
||||
groups := []*storeGroup{
|
||||
{Engine: buildBoundComposite(t, bad), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-bad"}},
|
||||
{Engine: buildBoundComposite(t, bad), BaseParams: vectorParams("q"), TopK: 50, KBIDs: []string{"kb-bad-2"}},
|
||||
}
|
||||
s := &knowledgeBaseService{}
|
||||
ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))
|
||||
|
||||
// FAQ KB + iterative-retrieval trigger (chunks < MatchCount AND
|
||||
// vectorResults == matchCount): with 1 chunk in `chunks` and 5 in
|
||||
// `vectorResults` matching matchCount=5, MatchCount=10, the iterative
|
||||
// path is taken and the propagated error must surface.
|
||||
kb := &types.KnowledgeBase{
|
||||
ID: "kb-bad", Type: types.KnowledgeBaseTypeFAQ,
|
||||
EmbeddingModelID: "m-1", TenantID: 1,
|
||||
}
|
||||
chunks := []*types.IndexWithScore{{ChunkID: "c1", Score: 0.5}}
|
||||
vectorResults := make([]*types.IndexWithScore, 5)
|
||||
for i := range vectorResults {
|
||||
vectorResults[i] = &types.IndexWithScore{ChunkID: fmt.Sprintf("v%d", i)}
|
||||
}
|
||||
params := types.SearchParams{QueryText: "q", MatchCount: 10}
|
||||
out, err := s.applyFAQPostProcessing(ctx, kb, chunks, vectorResults, groups, params, 5)
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, out)
|
||||
_, ok := apperrors.IsAppError(err)
|
||||
require.True(t, ok)
|
||||
}
|
||||
|
||||
// TestIsParentCancelled distinguishes a true client cancel
|
||||
// (context.Canceled) from a parent-deadline expiry. Only the former is
|
||||
// treated as "client gave up"; the latter must fall through to the
|
||||
// typed unavailable error so the handler returns a stable 4xx instead
|
||||
// of the raw stdlib DeadlineExceeded.
|
||||
func TestIsParentCancelled(t *testing.T) {
|
||||
t.Parallel()
|
||||
bg := context.Background()
|
||||
if isParentCancelled(bg) {
|
||||
t.Errorf("background ctx must not be reported cancelled")
|
||||
}
|
||||
|
||||
canc, cancel := context.WithCancel(bg)
|
||||
cancel()
|
||||
if !isParentCancelled(canc) {
|
||||
t.Errorf("explicit cancel must be reported cancelled")
|
||||
}
|
||||
|
||||
dead, cancelD := context.WithTimeout(bg, 1*time.Millisecond)
|
||||
defer cancelD()
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
if isParentCancelled(dead) {
|
||||
t.Errorf("parent-deadline expiry must NOT be reported as client cancel")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetrieveFromStores_IterativePattern_NoInternalRace mirrors the
|
||||
// iterative-FAQ usage contract: ONE driver goroutine mutates group.TopK
|
||||
// between calls, retrieveFromStores spawns parallel internal goroutines
|
||||
// each call. The race detector confirms BaseParams is never observed
|
||||
// mid-mutation because paramsWithTopK builds a fresh slice per call.
|
||||
//
|
||||
// We use two groups to actually trigger the multi-group fan-out path
|
||||
// (the single-group fast path doesn't exercise the goroutine spawn).
|
||||
func TestRetrieveFromStores_IterativePattern_NoInternalRace(t *testing.T) {
|
||||
t.Parallel()
|
||||
fakeA := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "a", Score: 0.5}},
|
||||
}
|
||||
fakeB := &fakeRetrieveEngineService{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
canned: []*types.IndexWithScore{{ChunkID: "b", Score: 0.3}},
|
||||
}
|
||||
groups := []*storeGroup{
|
||||
{Engine: buildBoundComposite(t, fakeA), BaseParams: vectorParams("q"), TopK: 10, KBIDs: []string{"kb-a"}},
|
||||
{Engine: buildBoundComposite(t, fakeB), BaseParams: vectorParams("q"), TopK: 10, KBIDs: []string{"kb-b"}},
|
||||
}
|
||||
s := &knowledgeBaseService{}
|
||||
|
||||
// Sequential driver: bump TopK then call. -race confirms the internal
|
||||
// concurrentRetrieve goroutines see a stable BaseParams slice.
|
||||
for k := 0; k < 8; k++ {
|
||||
for _, g := range groups {
|
||||
g.TopK = 10 + k
|
||||
}
|
||||
_, err := s.retrieveFromStores(context.Background(),
|
||||
groups, retriever.EngineAwareNormalizer{})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Verify BaseParams remained at its original TopK (mutation only on
|
||||
// storeGroup.TopK; paramsWithTopK does the override on a fresh slice).
|
||||
for _, g := range groups {
|
||||
assert.Equal(t, 50, g.BaseParams[0].TopK, "BaseParams TopK must stay immutable")
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
apperrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"slices"
|
||||
@@ -13,49 +14,67 @@ import (
|
||||
// applyFAQPostProcessing handles FAQ-specific post-processing: iterative retrieval
|
||||
// when not enough unique chunks are found, or negative question filtering otherwise.
|
||||
// For non-FAQ knowledge bases, returns the input unchanged.
|
||||
//
|
||||
// The iterative retrieval path fans out across the supplied storeGroups so
|
||||
// multi-store FAQ searches grow TopK uniformly across every bound vector
|
||||
// store. A typed AppError raised inside that path (e.g.
|
||||
// ErrVectorStoreUnavailable from a per-group timeout) is propagated to the
|
||||
// caller so the user receives a faithful failure response rather than a
|
||||
// silently truncated chunk list.
|
||||
func (s *knowledgeBaseService) applyFAQPostProcessing(
|
||||
ctx context.Context,
|
||||
kb *types.KnowledgeBase,
|
||||
chunks []*types.IndexWithScore,
|
||||
vectorResults []*types.IndexWithScore,
|
||||
retrieveEngine *retriever.CompositeRetrieveEngine,
|
||||
retrieveParams []types.RetrieveParams,
|
||||
groups []*storeGroup,
|
||||
params types.SearchParams,
|
||||
matchCount int,
|
||||
) []*types.IndexWithScore {
|
||||
) ([]*types.IndexWithScore, error) {
|
||||
if kb.Type != types.KnowledgeBaseTypeFAQ {
|
||||
return chunks
|
||||
return chunks, nil
|
||||
}
|
||||
|
||||
// Check if we need iterative retrieval for FAQ with separate indexing
|
||||
// Only use iterative retrieval if we don't have enough unique chunks after first deduplication
|
||||
// Check if we need iterative retrieval for FAQ with separate indexing.
|
||||
// Only use iterative retrieval if we don't have enough unique chunks
|
||||
// after first deduplication.
|
||||
needsIterativeRetrieval := len(chunks) < params.MatchCount && len(vectorResults) == matchCount
|
||||
if needsIterativeRetrieval {
|
||||
logger.Info(ctx, "Not enough unique chunks, using iterative retrieval for FAQ")
|
||||
return s.iterativeRetrieveWithDeduplication(
|
||||
ctx,
|
||||
retrieveEngine,
|
||||
retrieveParams,
|
||||
groups,
|
||||
params.MatchCount,
|
||||
params.QueryText,
|
||||
)
|
||||
}
|
||||
|
||||
// Filter by negative questions if not using iterative retrieval
|
||||
// Filter by negative questions if not using iterative retrieval.
|
||||
result := s.filterByNegativeQuestions(ctx, chunks, params.QueryText)
|
||||
logger.Infof(ctx, "Result count after negative question filtering: %d", len(result))
|
||||
return result
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// iterativeRetrieveWithDeduplication performs iterative retrieval until enough unique chunks are found.
|
||||
// This is used for FAQ knowledge bases with separate indexing mode.
|
||||
// Negative question filtering is applied after each iteration with chunk data caching.
|
||||
//
|
||||
// Each iteration only updates group.TopK; the underlying BaseParams stays
|
||||
// immutable so the fan-out goroutines inside retrieveFromStores never
|
||||
// observe a mid-mutation slice. Engines and grouping are computed once
|
||||
// upstream and reused across iterations.
|
||||
//
|
||||
// Returns (results, error). A typed AppError raised inside
|
||||
// retrieveFromStores (e.g. per-group timeout, or vector-store binding
|
||||
// invalid) is propagated to the caller so the user sees an honest failure
|
||||
// instead of a silently truncated result set. Non-AppError failures
|
||||
// continue to break the loop with a warning and return whatever partial
|
||||
// uniqueChunks have been accumulated — preserving the existing behavior
|
||||
// for transient retrieve errors.
|
||||
func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Context,
|
||||
retrieveEngine *retriever.CompositeRetrieveEngine,
|
||||
retrieveParams []types.RetrieveParams,
|
||||
groups []*storeGroup,
|
||||
matchCount int,
|
||||
queryText string,
|
||||
) []*types.IndexWithScore {
|
||||
) ([]*types.IndexWithScore, error) {
|
||||
maxIterations := 5
|
||||
// Start with a larger TopK since we're called when first retrieval wasn't enough
|
||||
// The first retrieval already used matchCount*3, so start from there
|
||||
@@ -70,16 +89,29 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co
|
||||
tenantID := types.MustTenantIDFromContext(ctx)
|
||||
|
||||
for i := 0; i < maxIterations; i++ {
|
||||
// Update TopK in retrieve params
|
||||
updatedParams := make([]types.RetrieveParams, len(retrieveParams))
|
||||
for j := range retrieveParams {
|
||||
updatedParams[j] = retrieveParams[j]
|
||||
updatedParams[j].TopK = currentTopK
|
||||
// Bump only the per-group TopK. BaseParams is immutable and read
|
||||
// concurrently inside retrieveFromStores; paramsWithTopK rebuilds
|
||||
// a fresh slice per call so no goroutine ever sees a half-mutated
|
||||
// value.
|
||||
for _, grp := range groups {
|
||||
grp.TopK = currentTopK
|
||||
}
|
||||
|
||||
// Execute retrieval
|
||||
retrieveResults, err := retrieveEngine.Retrieve(ctx, updatedParams)
|
||||
retrieveResults, err := s.retrieveFromStores(
|
||||
ctx, groups, retriever.EngineAwareNormalizer{})
|
||||
if err != nil {
|
||||
// Typed AppErrors must surface to HybridSearch so the user
|
||||
// sees the failure rather than a silently truncated chunk
|
||||
// list. Non-AppError failures (e.g. transient infra hiccups)
|
||||
// preserve the existing "warn and break" behavior so the
|
||||
// caller still gets partial results from the iterations that
|
||||
// succeeded.
|
||||
if _, ok := apperrors.IsAppError(err); ok {
|
||||
logger.WarnWithFields(ctx, logger.Fields{
|
||||
"iteration": i + 1,
|
||||
}, "Iterative retrieval surfaced typed failure")
|
||||
return nil, err
|
||||
}
|
||||
logger.Warnf(ctx, "Iterative retrieval failed at iteration %d: %v", i+1, err)
|
||||
break
|
||||
}
|
||||
@@ -179,7 +211,7 @@ func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Co
|
||||
slices.SortFunc(result, sortByScoreDesc)
|
||||
|
||||
logger.Infof(ctx, "Iterative retrieval completed: %d unique chunks found after filtering", len(result))
|
||||
return result
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// filterByNegativeQuestions filters out chunks that match negative questions for FAQ knowledge bases.
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
apperrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
secutils "github.com/Tencent/WeKnora/internal/utils"
|
||||
)
|
||||
|
||||
// storeGroup is one fan-out unit of HybridSearch: a set of KB IDs that share
|
||||
// the same (VectorStore, owning tenant) pair.
|
||||
//
|
||||
// Partition key is (VectorStoreID, OwnerTenantID), not VectorStoreID alone,
|
||||
// because Organization-shared KBs (kb.TenantID != requestTenantID) need
|
||||
// their own group whose ownership lookup runs against kb.TenantID — the
|
||||
// store is owned by the source tenant, not the caller.
|
||||
//
|
||||
// BaseParams are immutable across iterations and goroutines. TopK is the
|
||||
// only mutable per-iteration value; paramsWithTopK builds a fresh
|
||||
// []RetrieveParams per call so no goroutine sees a slice being mutated by
|
||||
// another. Callers MUST NOT mutate BaseParams after resolveStoreGroups
|
||||
// returns; doing so would race with the fan-out goroutines.
|
||||
type storeGroup struct {
|
||||
// StoreID is the bound VectorStore UUID, or "" for the env-store group
|
||||
// (KBs with VectorStoreID = NULL). Never echo this in user-facing
|
||||
// errors; use secutils.SanitizeForLog when emitting in structured logs.
|
||||
StoreID string
|
||||
|
||||
// OwnerTenantID is the tenant that owns the KBs and the store for this
|
||||
// group. For Organization-shared KBs this differs from the request's
|
||||
// tenant; the factory's StoreOwnedBy must be called with this value.
|
||||
OwnerTenantID uint64
|
||||
|
||||
// KBIDs are the knowledge base IDs in this group. The caller MUST have
|
||||
// authorized the request to access every ID here (the trust boundary
|
||||
// is the HTTP handler / session layer, matching the pre-existing
|
||||
// chat_pipeline pattern).
|
||||
KBIDs []string
|
||||
|
||||
// Engine is the resolved CompositeRetrieveEngine for this group.
|
||||
// Reused across iterative FAQ retries — never re-resolved.
|
||||
Engine *retriever.CompositeRetrieveEngine
|
||||
|
||||
// BaseParams is the immutable per-group retrieval parameter list.
|
||||
// TopK is filled in at retrieve time via paramsWithTopK.
|
||||
BaseParams []types.RetrieveParams
|
||||
|
||||
// TopK is the requested over-retrieval count. The iterative FAQ path
|
||||
// (knowledgebase_search_faq.go) updates this between calls to
|
||||
// retrieveFromStores. Single-shot HybridSearch sets it once.
|
||||
TopK int
|
||||
}
|
||||
|
||||
// resolveStoreGroups partitions kbs by (VectorStoreID, KB.TenantID),
|
||||
// resolves the engine per group via the PR2 factory using the OWNING
|
||||
// tenant for ownership lookup, and builds the per-store base RetrieveParams
|
||||
// once. Returns groups in non-deterministic order (caller must not rely on
|
||||
// iteration order).
|
||||
//
|
||||
// The primary KB supplies the embedding model and FAQ type for params; the
|
||||
// caller MUST invoke validateSameEmbeddingModel first to guarantee a
|
||||
// single embedding model identity across kbs.
|
||||
//
|
||||
// Errors are translated from sentinel to typed AppError so that the
|
||||
// upstream handler reports a stable error code without leaking storeIDs:
|
||||
//
|
||||
// - retriever.ErrVectorStoreForbidden →
|
||||
// apperrors.NewVectorStoreBindingInvalidError (2200)
|
||||
// - retriever.ErrVectorStoreNotFound →
|
||||
// apperrors.NewVectorStoreUnavailableError (2201)
|
||||
// - retriever.ErrTenantInfoMissing →
|
||||
// apperrors.NewVectorStoreBindingInvalidError (2200)
|
||||
// - any other error → returned with %w wrap (no UUID embedded).
|
||||
func (s *knowledgeBaseService) resolveStoreGroups(
|
||||
ctx context.Context,
|
||||
primary *types.KnowledgeBase,
|
||||
kbs []*types.KnowledgeBase,
|
||||
params types.SearchParams,
|
||||
matchCount int,
|
||||
) ([]*storeGroup, error) {
|
||||
type partitionKey struct {
|
||||
storeID string
|
||||
tenantID uint64
|
||||
}
|
||||
buckets := make(map[partitionKey][]string)
|
||||
for _, kb := range kbs {
|
||||
sid := ""
|
||||
if kb.HasVectorStore() {
|
||||
sid = *kb.VectorStoreID
|
||||
}
|
||||
key := partitionKey{storeID: sid, tenantID: kb.TenantID}
|
||||
buckets[key] = append(buckets[key], kb.ID)
|
||||
}
|
||||
|
||||
groups := make([]*storeGroup, 0, len(buckets))
|
||||
for key, ids := range buckets {
|
||||
var storeIDPtr *string
|
||||
if key.storeID != "" {
|
||||
sid := key.storeID
|
||||
storeIDPtr = &sid
|
||||
}
|
||||
engine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, key.tenantID, storeIDPtr)
|
||||
if err != nil {
|
||||
return nil, classifyFactoryError(ctx, err, key.tenantID, key.storeID)
|
||||
}
|
||||
baseParams, err := s.buildRetrievalParams(
|
||||
ctx, engine, primary, params, ids, matchCount)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build store-group params: %w", err)
|
||||
}
|
||||
groups = append(groups, &storeGroup{
|
||||
StoreID: key.storeID,
|
||||
OwnerTenantID: key.tenantID,
|
||||
KBIDs: ids,
|
||||
Engine: engine,
|
||||
BaseParams: baseParams,
|
||||
TopK: matchCount,
|
||||
})
|
||||
}
|
||||
return groups, nil
|
||||
}
|
||||
|
||||
// classifyFactoryError translates retriever sentinels into typed AppErrors
|
||||
// without leaking the store UUID into the user-facing message. The UUID is
|
||||
// recorded in the structured log only, sanitized via SanitizeForLog to
|
||||
// defeat log-injection through CR/LF in store IDs.
|
||||
func classifyFactoryError(
|
||||
ctx context.Context, err error, tenantID uint64, storeID string,
|
||||
) error {
|
||||
logger.WarnWithFields(ctx, logger.Fields{
|
||||
"tenant_id": tenantID,
|
||||
"store_id": secutils.SanitizeForLog(storeID),
|
||||
"reason": "resolve store engine",
|
||||
}, err.Error())
|
||||
switch {
|
||||
case errors.Is(err, retriever.ErrVectorStoreForbidden):
|
||||
return apperrors.NewVectorStoreBindingInvalidError(
|
||||
"vector store bound to the knowledge base is not available")
|
||||
case errors.Is(err, retriever.ErrVectorStoreNotFound):
|
||||
return apperrors.NewVectorStoreUnavailableError(
|
||||
"vector store is currently unavailable")
|
||||
case errors.Is(err, retriever.ErrTenantInfoMissing):
|
||||
return apperrors.NewVectorStoreBindingInvalidError(
|
||||
"tenant information missing in context")
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// authorizeKBAccess rejects multi-KB searches whose scope includes a KB
|
||||
// that the caller is not entitled to read. Same-tenant KBs always pass.
|
||||
// Foreign-tenant KBs (Organization-shared) must pass an explicit
|
||||
// tenant-scoped permission check via kbShareService.HasTenantKBPermission,
|
||||
// applying the 3-D cap (share role + caller's tenant-org role + tenant
|
||||
// Viewer cap) introduced in Plan 3 of #1303.
|
||||
//
|
||||
// Returning NotFound rather than Forbidden avoids leaking the existence
|
||||
// of unauthorized KB IDs that the caller could not otherwise observe.
|
||||
// Structured logs record the rejection with the offending kb_id (always
|
||||
// safe — KB IDs are UUIDs without sensitive content) and the requesting
|
||||
// tenant for audit.
|
||||
func (s *knowledgeBaseService) authorizeKBAccess(
|
||||
ctx context.Context,
|
||||
kbs []*types.KnowledgeBase,
|
||||
requestTenantID uint64,
|
||||
) error {
|
||||
if len(kbs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
callerTenantRole := types.TenantRoleFromContext(ctx)
|
||||
|
||||
for _, kb := range kbs {
|
||||
if kb.TenantID == requestTenantID {
|
||||
continue
|
||||
}
|
||||
hasPermission, permErr := s.kbShareService.HasTenantKBPermission(
|
||||
ctx, kb.ID, requestTenantID, callerTenantRole, types.OrgRoleViewer)
|
||||
if permErr != nil {
|
||||
logger.ErrorWithFields(ctx, permErr, map[string]interface{}{
|
||||
"caller_tenant_id": requestTenantID,
|
||||
"kb_tenant_id": kb.TenantID,
|
||||
"kb_id": kb.ID,
|
||||
"reason": "shared-KB permission lookup failed",
|
||||
})
|
||||
return apperrors.NewInternalServerError("failed to verify knowledge base access")
|
||||
}
|
||||
if !hasPermission {
|
||||
logger.WarnWithFields(ctx, logger.Fields{
|
||||
"caller_tenant_id": requestTenantID,
|
||||
"kb_tenant_id": kb.TenantID,
|
||||
"kb_id": kb.ID,
|
||||
"reason": "tenant lacks viewer permission for foreign-tenant KB",
|
||||
}, "search scope rejected: unauthorized foreign-tenant KB")
|
||||
return apperrors.NewNotFoundError("knowledge base not found")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateSameEmbeddingModel rejects multi-KB searches that span more than
|
||||
// one resolved embedding-model identity key. Single-KB calls no-op.
|
||||
//
|
||||
// Wiki-only / graph-only KBs (empty resolved key) are tolerated: if every
|
||||
// KB lacks an embedding model, validation passes and HybridSearch returns
|
||||
// an empty result set via the allBaseParamsEmpty fast path.
|
||||
//
|
||||
// Log fields are sanitized via secutils.SanitizeForLog because resolved
|
||||
// keys are derived from model.Parameters.BaseURL, which is tenant-
|
||||
// configured and can contain CR/LF or other control characters.
|
||||
func (s *knowledgeBaseService) validateSameEmbeddingModel(
|
||||
ctx context.Context,
|
||||
kbs []*types.KnowledgeBase,
|
||||
) error {
|
||||
if len(kbs) <= 1 {
|
||||
return nil
|
||||
}
|
||||
keys := s.ResolveEmbeddingModelKeys(ctx, kbs)
|
||||
var seen string
|
||||
for _, kb := range kbs {
|
||||
k, ok := keys[kb.ID]
|
||||
if !ok || k == "" {
|
||||
// Wiki-only / graph-only carve-out: KB has no embedding model.
|
||||
continue
|
||||
}
|
||||
if seen == "" {
|
||||
seen = k
|
||||
continue
|
||||
}
|
||||
if k != seen {
|
||||
logger.WarnWithFields(ctx, logger.Fields{
|
||||
"primary_key": secutils.SanitizeForLog(seen),
|
||||
"diverging": secutils.SanitizeForLog(k),
|
||||
"kb_id": kb.ID,
|
||||
"kb_count": len(kbs),
|
||||
}, "multi-KB search rejected: embedding models differ")
|
||||
return apperrors.NewBadRequestError(
|
||||
"selected knowledge bases use different embedding models; " +
|
||||
"multi-KB search requires every knowledge base to share a single embedding model")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package retriever
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
// ScoreNormalizer maps raw retriever scores to a common [0, 1] scale so that
|
||||
// vector scores produced by different engines can be compared in a single
|
||||
// ranked list. Implementations MUST be safe for concurrent use and MUST be
|
||||
// IO-free (Normalize is called inside a hot loop and may not log or block).
|
||||
//
|
||||
// Only vector scores are normalized. Keyword (BM25) scores have an unbounded
|
||||
// positive range; rescaling them would collapse the long tail. Downstream
|
||||
// RRF fusion is rank-based and immune to scale, so keyword scores pass
|
||||
// through unchanged.
|
||||
type ScoreNormalizer interface {
|
||||
Normalize(
|
||||
ctx context.Context,
|
||||
score float64,
|
||||
retrieverType types.RetrieverType,
|
||||
engineType types.RetrieverEngineType,
|
||||
) float64
|
||||
}
|
||||
|
||||
// EngineAwareNormalizer applies the documented per-engine cosine-score
|
||||
// formula. The caller (HybridSearch) enforces a same-embedding-model
|
||||
// precondition via ResolveEmbeddingModelKeys, so post-normalization values
|
||||
// are semantically comparable across engines.
|
||||
//
|
||||
// Source formulas (verified against repository implementations on
|
||||
// upstream/main 3214e3d9):
|
||||
//
|
||||
// - Elasticsearch v8 / ElasticFaiss / Milvus (cosine mode):
|
||||
// raw cosine in [-1, 1]
|
||||
// - Postgres pgvector: (1 - cosine_distance) in [0, 1]
|
||||
// - SQLite sqlite-vec: (1 - cosine_distance) in [0, 1]
|
||||
// - Weaviate: raw score in [0, 1]
|
||||
// - Qdrant / Infinity / TencentVectorDB / Doris: raw score in [0, 1]
|
||||
//
|
||||
// Unknown engines clamp to [0, 1]; the fan-out caller emits a single WARN
|
||||
// per request via warnIfUnknownEngine so Normalize itself stays lock-free
|
||||
// and panic-free even on nil ctx.
|
||||
type EngineAwareNormalizer struct{}
|
||||
|
||||
// Compile-time interface satisfaction assertion.
|
||||
var _ ScoreNormalizer = EngineAwareNormalizer{}
|
||||
|
||||
// Normalize implements ScoreNormalizer.
|
||||
func (EngineAwareNormalizer) Normalize(
|
||||
_ context.Context,
|
||||
score float64,
|
||||
retrieverType types.RetrieverType,
|
||||
engineType types.RetrieverEngineType,
|
||||
) float64 {
|
||||
if retrieverType != types.VectorRetrieverType {
|
||||
// BM25 and other non-vector retrievers: passthrough. RRF rank-based
|
||||
// fusion handles scale-mixed input correctly.
|
||||
return score
|
||||
}
|
||||
|
||||
switch engineType {
|
||||
case types.ElasticsearchRetrieverEngineType,
|
||||
types.ElasticFaissRetrieverEngineType,
|
||||
types.MilvusRetrieverEngineType:
|
||||
// Raw cosine in [-1, 1] → [0, 1]. Pass through clamp01 once more so
|
||||
// that a misbehaving engine returning 1.0000002 does not leak past
|
||||
// the [0, 1] envelope (caller sorts by score afterwards).
|
||||
return clamp01((score + 1) / 2)
|
||||
case types.PostgresRetrieverEngineType,
|
||||
types.QdrantRetrieverEngineType,
|
||||
types.WeaviateRetrieverEngineType,
|
||||
types.SQLiteRetrieverEngineType,
|
||||
types.InfinityRetrieverEngineType,
|
||||
types.TencentVectorDBRetrieverEngineType,
|
||||
types.DorisRetrieverEngineType:
|
||||
// Already in [0, 1] by repository contract.
|
||||
return clamp01(score)
|
||||
default:
|
||||
// Unknown engine. Clamp defensively; the caller emits WARN with ctx.
|
||||
return clamp01(score)
|
||||
}
|
||||
}
|
||||
|
||||
// clamp01 maps any float64 into [0, 1] safely, including NaN/Inf inputs that
|
||||
// could otherwise break slices.SortFunc's strict-weak-ordering invariant
|
||||
// downstream (NaN compares neither greater nor less than anything).
|
||||
func clamp01(s float64) float64 {
|
||||
if math.IsNaN(s) {
|
||||
return 0
|
||||
}
|
||||
if s <= 0 || math.IsInf(s, -1) {
|
||||
return 0
|
||||
}
|
||||
if s >= 1 || math.IsInf(s, 1) {
|
||||
return 1
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package retriever
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
|
||||
func TestEngineAwareNormalizer_KeywordPassthrough(t *testing.T) {
|
||||
t.Parallel()
|
||||
n := EngineAwareNormalizer{}
|
||||
cases := []float64{-12.5, 0, 0.7, 1, 27.3, math.Inf(1)}
|
||||
for _, s := range cases {
|
||||
// Any keyword retriever, any engine — must be identity.
|
||||
got := n.Normalize(context.Background(), s, types.KeywordsRetrieverType,
|
||||
types.ElasticsearchRetrieverEngineType)
|
||||
if got != s {
|
||||
// math.IsNaN special handling for the Inf case.
|
||||
if math.IsInf(s, 1) && math.IsInf(got, 1) {
|
||||
continue
|
||||
}
|
||||
t.Fatalf("keyword passthrough expected %v, got %v", s, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineAwareNormalizer_CosineRange(t *testing.T) {
|
||||
t.Parallel()
|
||||
n := EngineAwareNormalizer{}
|
||||
cases := []struct {
|
||||
score float64
|
||||
want float64
|
||||
}{
|
||||
{-1.0, 0},
|
||||
{-0.5, 0.25},
|
||||
{0, 0.5},
|
||||
{0.5, 0.75},
|
||||
{1.0, 1.0},
|
||||
// Drift beyond [-1, 1] is clamped.
|
||||
{-1.5, 0},
|
||||
{1.5, 1.0},
|
||||
}
|
||||
for _, engine := range []types.RetrieverEngineType{
|
||||
types.ElasticsearchRetrieverEngineType,
|
||||
types.ElasticFaissRetrieverEngineType,
|
||||
types.MilvusRetrieverEngineType,
|
||||
} {
|
||||
for _, tc := range cases {
|
||||
got := n.Normalize(context.Background(), tc.score,
|
||||
types.VectorRetrieverType, engine)
|
||||
if math.Abs(got-tc.want) > 1e-9 {
|
||||
t.Fatalf("cosine[%s] score=%v: want %v, got %v",
|
||||
engine, tc.score, tc.want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineAwareNormalizer_UnitInterval(t *testing.T) {
|
||||
t.Parallel()
|
||||
n := EngineAwareNormalizer{}
|
||||
cases := []struct {
|
||||
score float64
|
||||
want float64
|
||||
}{
|
||||
{-0.1, 0},
|
||||
{0, 0},
|
||||
{0.25, 0.25},
|
||||
{0.999, 0.999},
|
||||
{1, 1},
|
||||
{1.5, 1},
|
||||
}
|
||||
for _, engine := range []types.RetrieverEngineType{
|
||||
types.PostgresRetrieverEngineType,
|
||||
types.QdrantRetrieverEngineType,
|
||||
types.WeaviateRetrieverEngineType,
|
||||
types.SQLiteRetrieverEngineType,
|
||||
types.InfinityRetrieverEngineType,
|
||||
types.TencentVectorDBRetrieverEngineType,
|
||||
types.DorisRetrieverEngineType,
|
||||
} {
|
||||
for _, tc := range cases {
|
||||
got := n.Normalize(context.Background(), tc.score,
|
||||
types.VectorRetrieverType, engine)
|
||||
if math.Abs(got-tc.want) > 1e-9 {
|
||||
t.Fatalf("unit[%s] score=%v: want %v, got %v",
|
||||
engine, tc.score, tc.want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineAwareNormalizer_Unknown_ClampsAndDoesNotPanic(t *testing.T) {
|
||||
t.Parallel()
|
||||
n := EngineAwareNormalizer{}
|
||||
got := n.Normalize(context.Background(), 0.42,
|
||||
types.VectorRetrieverType, types.RetrieverEngineType("nosuch"))
|
||||
if got != 0.42 {
|
||||
t.Fatalf("unknown engine passthrough-clamp: want 0.42, got %v", got)
|
||||
}
|
||||
got = n.Normalize(context.Background(), 5.0,
|
||||
types.VectorRetrieverType, types.RetrieverEngineType("nosuch"))
|
||||
if got != 1.0 {
|
||||
t.Fatalf("unknown engine clamp on >1: want 1, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineAwareNormalizer_NilCtx_DoesNotPanic(t *testing.T) {
|
||||
// nil ctx must not panic. Normalize is IO-free by contract — it never
|
||||
// reads from ctx and never logs. The unknown-engine warning is
|
||||
// emitted by the caller (retrieveFromStores), where ctx is always
|
||||
// live.
|
||||
t.Parallel()
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
t.Fatalf("Normalize panicked on nil ctx: %v", r)
|
||||
}
|
||||
}()
|
||||
_ = EngineAwareNormalizer{}.Normalize(
|
||||
nil, 0.5, types.VectorRetrieverType,
|
||||
types.RetrieverEngineType("nosuch"),
|
||||
)
|
||||
}
|
||||
|
||||
func TestEngineAwareNormalizer_NaNAndInf(t *testing.T) {
|
||||
t.Parallel()
|
||||
n := EngineAwareNormalizer{}
|
||||
// NaN → 0 (so SortFunc does not get a comparator-poisoning value).
|
||||
got := n.Normalize(context.Background(), math.NaN(),
|
||||
types.VectorRetrieverType, types.PostgresRetrieverEngineType)
|
||||
if got != 0 {
|
||||
t.Fatalf("NaN clamp: want 0, got %v", got)
|
||||
}
|
||||
// +Inf → 1.
|
||||
got = n.Normalize(context.Background(), math.Inf(1),
|
||||
types.VectorRetrieverType, types.PostgresRetrieverEngineType)
|
||||
if got != 1 {
|
||||
t.Fatalf("+Inf clamp: want 1, got %v", got)
|
||||
}
|
||||
// -Inf → 0.
|
||||
got = n.Normalize(context.Background(), math.Inf(-1),
|
||||
types.VectorRetrieverType, types.PostgresRetrieverEngineType)
|
||||
if got != 0 {
|
||||
t.Fatalf("-Inf clamp: want 0, got %v", got)
|
||||
}
|
||||
// NaN through cosine formula too.
|
||||
got = n.Normalize(context.Background(), math.NaN(),
|
||||
types.VectorRetrieverType, types.ElasticsearchRetrieverEngineType)
|
||||
if got != 0 {
|
||||
t.Fatalf("NaN cosine: want 0, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClamp01(t *testing.T) {
|
||||
t.Parallel()
|
||||
cases := []struct {
|
||||
in, want float64
|
||||
}{
|
||||
{-1, 0},
|
||||
{0, 0},
|
||||
{0.5, 0.5},
|
||||
{1, 1},
|
||||
{2, 1},
|
||||
{math.NaN(), 0},
|
||||
{math.Inf(1), 1},
|
||||
{math.Inf(-1), 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
got := clamp01(tc.in)
|
||||
if math.Abs(got-tc.want) > 1e-9 {
|
||||
// math.IsNaN comparison would have already failed via the
|
||||
// equality above; this branch covers the rest.
|
||||
t.Fatalf("clamp01(%v): want %v, got %v", tc.in, tc.want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEngineAwareNormalizer_InterfaceSatisfied is a compile-time check that
|
||||
// catches accidental breakage of the ScoreNormalizer interface via the
|
||||
// package-scope var assertion in normalizer.go. The runtime body is a
|
||||
// no-op; failure would surface at build time.
|
||||
func TestEngineAwareNormalizer_InterfaceSatisfied(t *testing.T) {
|
||||
var _ ScoreNormalizer = EngineAwareNormalizer{}
|
||||
}
|
||||
@@ -183,6 +183,15 @@ func (h *KnowledgeBaseHandler) HybridSearch(c *gin.Context) {
|
||||
// Note: For shared KBs, the service uses effectiveTenantID internally via context
|
||||
results, err := h.service.HybridSearch(ctx, id, req)
|
||||
if err != nil {
|
||||
// Service-layer typed AppErrors (e.g. ErrVectorStoreBindingInvalid,
|
||||
// ErrVectorStoreUnavailable, BadRequest from multi-store fan-out)
|
||||
// must reach the client with their original code rather than be
|
||||
// downgraded to InternalServerError. Mirrors the pattern used in
|
||||
// CreateKnowledgeBase.
|
||||
if appErr, ok := apperrors.IsAppError(err); ok {
|
||||
c.Error(appErr)
|
||||
return
|
||||
}
|
||||
logger.ErrorWithFields(ctx, err, nil)
|
||||
c.Error(apperrors.NewInternalServerError(err.Error()))
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user