refactor(retriever): introduce factory for KB-scoped retrieve engine resolution
Extracts the 25 repeated NewCompositeRetrieveEngine call sites across seven services into two factory functions with tenant-ownership verification: CreateRetrieveEngineForKB (synchronous) and CreateRetrieveEngineFromPayload (async task handlers). Promotes GetByStoreID from concrete-only to interfaces.RetrieveEngineRegistry. Extends KBDeletePayload / IndexDeletePayload with an omitempty VectorStoreID snapshot. Design highlights - Factory verifies tenant ownership of the resolved store (defense-in-depth) so a gap in PR3 validation or a tampered Asynq payload cannot cross tenants. Cross-tenant attempts return ErrVectorStoreForbidden with a structured log entry. - Sentinel errors (ErrTenantInfoMissing, ErrVectorStoreNotFound, ErrVectorStoreForbidden) let async handlers classify non-retryable failures as asynq.SkipRetry. - nil and empty-string pointers for vectorStoreID normalize to "unbound" so callers never send an empty UUID to GetByStoreID. - resolveBoundEngine constructs engineInfos directly (with slices.Clone of Support()) instead of going through NewCompositeRetrieveEngine. The latter only reads from the byEngineType map and cannot reach DB stores in byStoreID -- multiple stores can share the same engine type. Unbound fallbacks still reuse NewCompositeRetrieveEngine so the tenant effective-engines path is byte-for-byte unchanged. - DeleteKnowledgeBase now reads the KB before soft-delete so the enqueued KBDeletePayload carries a VectorStoreID snapshot; once the row is soft-deleted GORM's default scope hides it. Cost: +1 SELECT per KB-delete request (a rare admin operation). - enqueueIndexDeleteTask gains a vectorStoreID *string parameter that the single caller (tag.go) populates from the owning KB. The worker (ProcessIndexDelete) validates ownership via the factory. - cleanupKnowledgeResources loads the KB so bound-store cleanup routes correctly. If the load fails it warns and falls back to tenant effective engines; orphan vectors become observable in logs instead of disappearing silently. - DeleteKnowledges (batch) stays on the tenant effective-engines path -- batches can span stores and multi-store fan-out is PR4 scope. Noted inline. Merge safety All knowledge bases have VectorStoreID = NULL at merge time (introduced nullable in PR1, Tencent/WeKnora#994). Every factory call falls back to NewCompositeRetrieveEngine(registry, tenantInfo.GetEffectiveEngines()), which is the existing code path for these KBs. Async payloads from before this change decode to VectorStoreID = nil (omitempty) and take the same fallback. Testing - go build ./... - go vet ./... - go test -race -count=1 ./internal/application/service/retriever/... covers unbound (nil/empty), bound, cross-tenant, unregistered store, ownership infra error, legacy payload shapes (missing field/null/ empty string/nil), tampered payload, and parallel invocation. - grep -r NewCompositeRetrieveEngine internal/application/service returns zero results outside factory.go/composite.go. Refs: Part of Tencent/WeKnora#993. Depends on Tencent/WeKnora#994.
This commit is contained in:
@@ -21,6 +21,7 @@ type chunkService struct {
|
||||
kbRepository interfaces.KnowledgeBaseRepository
|
||||
modelService interfaces.ModelService
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ownership retriever.TenantStoreOwnership
|
||||
}
|
||||
|
||||
// NewChunkService creates a new chunk service
|
||||
@@ -35,12 +36,14 @@ func NewChunkService(
|
||||
kbRepository interfaces.KnowledgeBaseRepository,
|
||||
modelService interfaces.ModelService,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
ownership retriever.TenantStoreOwnership,
|
||||
) interfaces.ChunkService {
|
||||
return &chunkService{
|
||||
chunkRepository: chunkRepository,
|
||||
kbRepository: kbRepository,
|
||||
modelService: modelService,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ownership: ownership,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -404,8 +407,8 @@ func (s *chunkService) DeleteGeneratedQuestion(ctx context.Context, chunkID stri
|
||||
// The source_id format is: {chunk_id}-{question_id}
|
||||
sourceID := fmt.Sprintf("%s-%s", chunkID, questionID)
|
||||
|
||||
tenantInfo, _ := types.TenantInfoFromContext(ctx)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"chunk_id": chunkID,
|
||||
|
||||
@@ -259,6 +259,7 @@ type DataTableSummaryService struct {
|
||||
chunkService interfaces.ChunkService
|
||||
tenantService interfaces.TenantService
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ownership retriever.TenantStoreOwnership
|
||||
sqlDB *sql.DB
|
||||
}
|
||||
|
||||
@@ -271,6 +272,7 @@ func NewDataTableSummaryService(
|
||||
chunkService interfaces.ChunkService,
|
||||
tenantService interfaces.TenantService,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
ownership retriever.TenantStoreOwnership,
|
||||
sqlDB *sql.DB,
|
||||
) interfaces.TaskHandler {
|
||||
return &DataTableSummaryService{
|
||||
@@ -281,6 +283,7 @@ func NewDataTableSummaryService(
|
||||
chunkService: chunkService,
|
||||
tenantService: tenantService,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ownership: ownership,
|
||||
sqlDB: sqlDB,
|
||||
}
|
||||
}
|
||||
@@ -370,8 +373,25 @@ func (s *DataTableSummaryService) prepareResources(ctx context.Context, payload
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 获取检索引擎
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
// Load the KB to discover its VectorStoreID binding so the factory can
|
||||
// route to the bound store (or fall back to tenant engines if unbound).
|
||||
kb, err := s.knowledgeBaseService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get knowledge base for vector store lookup: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
var vectorStoreID *string
|
||||
if kb != nil {
|
||||
vectorStoreID = kb.VectorStoreID
|
||||
}
|
||||
|
||||
// The factory's unbound path reads TenantInfo from ctx.
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
|
||||
|
||||
// Resolve the engine via the factory using the KB's VectorStore binding
|
||||
// (nil -> tenant effective engines fallback; verified tenant ownership otherwise).
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, payload.TenantID, vectorStoreID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "failed to get retrieve engine: %v", err)
|
||||
return nil, err
|
||||
|
||||
@@ -60,6 +60,7 @@ type ImageMultimodalService struct {
|
||||
knowledgeRepo interfaces.KnowledgeRepository
|
||||
tenantRepo interfaces.TenantRepository
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ownership retriever.TenantStoreOwnership
|
||||
ollamaService *ollama.OllamaService
|
||||
taskEnqueuer interfaces.TaskEnqueuer
|
||||
redisClient *redis.Client
|
||||
@@ -78,6 +79,7 @@ func NewImageMultimodalService(
|
||||
knowledgeRepo interfaces.KnowledgeRepository,
|
||||
tenantRepo interfaces.TenantRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
ownership retriever.TenantStoreOwnership,
|
||||
ollamaService *ollama.OllamaService,
|
||||
taskEnqueuer interfaces.TaskEnqueuer,
|
||||
redisClient *redis.Client,
|
||||
@@ -90,6 +92,7 @@ func NewImageMultimodalService(
|
||||
knowledgeRepo: knowledgeRepo,
|
||||
tenantRepo: tenantRepo,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ownership: ownership,
|
||||
ollamaService: ollamaService,
|
||||
taskEnqueuer: taskEnqueuer,
|
||||
redisClient: redisClient,
|
||||
@@ -269,8 +272,13 @@ func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to get tenant for indexing: %v", err)
|
||||
return
|
||||
}
|
||||
// The factory's unbound path reads TenantInfo from ctx; make sure it's there.
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
|
||||
|
||||
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
// Resolve engine via the factory using the KB's VectorStore binding
|
||||
// (nil → tenant effective engines fallback; verified tenant ownership otherwise).
|
||||
engine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, payload.TenantID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[ImageMultimodal] Failed to init retrieve engine: %v", err)
|
||||
return
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
||||
"github.com/Tencent/WeKnora/internal/config"
|
||||
werrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
|
||||
@@ -38,6 +39,7 @@ var (
|
||||
type knowledgeService struct {
|
||||
config *config.Config
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ownership retriever.TenantStoreOwnership
|
||||
repo interfaces.KnowledgeRepository
|
||||
kbService interfaces.KnowledgeBaseService
|
||||
tenantRepo interfaces.TenantRepository
|
||||
@@ -86,6 +88,7 @@ func NewKnowledgeService(
|
||||
task interfaces.TaskEnqueuer,
|
||||
graphEngine interfaces.RetrieveGraphRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
ownership retriever.TenantStoreOwnership,
|
||||
redisClient *redis.Client,
|
||||
kbShareService interfaces.KBShareService,
|
||||
imageResolver *docparser.ImageResolver,
|
||||
@@ -109,6 +112,7 @@ func NewKnowledgeService(
|
||||
task: task,
|
||||
graphEngine: graphEngine,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ownership: ownership,
|
||||
redisClient: redisClient,
|
||||
kbShareService: kbShareService,
|
||||
imageResolver: imageResolver,
|
||||
|
||||
@@ -194,8 +194,18 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
|
||||
}
|
||||
}
|
||||
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
tenantID := types.MustTenantIDFromContext(ctx)
|
||||
// Route CopyIndices via the source KB's bound store. This function does
|
||||
// not handle cross-store copies — embeddings written by different
|
||||
// VectorStore backends are not bit-compatible, so callers that allow
|
||||
// source/target KBs to bind to different stores must perform their own
|
||||
// cross-store migration before invoking this.
|
||||
var sourceStoreID *string
|
||||
if srcKB, loadErr := s.kbService.GetKnowledgeBaseByID(ctx, src.KnowledgeBaseID); loadErr == nil && srcKB != nil {
|
||||
sourceStoreID = srcKB.VectorStoreID
|
||||
}
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -441,9 +451,15 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get tenant info and initialize retrieve engine
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
// Route the FAQ clone through the source KB's bound store. Same
|
||||
// constraint as CloneChunk: callers must ensure source and target share
|
||||
// the same VectorStore (cross-store FAQ clone is not handled here).
|
||||
var sourceStoreID *string
|
||||
if srcKB != nil {
|
||||
sourceStoreID = srcKB.VectorStoreID
|
||||
}
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), sourceStoreID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
|
||||
handleError(progress, err, "Failed to initialize retrieve engine")
|
||||
@@ -856,7 +872,6 @@ func (s *knowledgeService) moveKnowledgeReuseVectors(
|
||||
sourceKB, targetKB *types.KnowledgeBase,
|
||||
) error {
|
||||
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
|
||||
// 1. Get old chunk IDs for vector index copy mapping
|
||||
oldChunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, tenantID, knowledge.ID)
|
||||
@@ -872,7 +887,16 @@ func (s *knowledgeService) moveKnowledgeReuseVectors(
|
||||
|
||||
// 2. Copy vector indices from source KB to target KB
|
||||
if len(chunkIDMapping) > 0 && knowledge.EmbeddingModelID != "" {
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
// reuse_vectors mode copies index entries directly between KBs, which
|
||||
// only works inside the same VectorStore backend. Route through the
|
||||
// source KB's binding; callers must reject cross-store moves before
|
||||
// landing here.
|
||||
var sourceStoreID *string
|
||||
if sourceKB != nil {
|
||||
sourceStoreID = sourceKB.VectorStoreID
|
||||
}
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to init retrieve engine: %w", err)
|
||||
}
|
||||
|
||||
@@ -98,10 +98,18 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
|
||||
// and GetEmbeddingModel would fail with "model ID cannot be empty".
|
||||
if strings.TrimSpace(knowledge.EmbeddingModelID) != "" {
|
||||
wg.Go(func() error {
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
// kb was already loaded above for resolveFileService — reuse its
|
||||
// VectorStoreID for engine routing.
|
||||
var boundStoreID *string
|
||||
if kb != nil {
|
||||
boundStoreID = kb.VectorStoreID
|
||||
}
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx,
|
||||
s.retrieveEngine,
|
||||
tenantInfo.GetEffectiveEngines(),
|
||||
s.ownership,
|
||||
tenantID,
|
||||
boundStoreID,
|
||||
)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
|
||||
@@ -427,11 +435,14 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
|
||||
wg := errgroup.Group{}
|
||||
// 2. Delete knowledge embeddings from vector store
|
||||
wg.Go(func() error {
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.GetEffectiveEngines(),
|
||||
)
|
||||
tenantID := types.MustTenantIDFromContext(ctx)
|
||||
// Batch cleanup spans multiple KBs that may be bound to different
|
||||
// VectorStores; routing this batch through tenant effective engines
|
||||
// keeps the legacy behavior intact.
|
||||
// TODO: fan out the batch per-store using each KB's own
|
||||
// VectorStoreID so cleanup hits the right backend for bound KBs.
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantID, nil)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
|
||||
return err
|
||||
@@ -556,10 +567,21 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle
|
||||
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if knowledge.EmbeddingModelID != "" {
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.GetEffectiveEngines(),
|
||||
)
|
||||
// Load KB to discover its VectorStoreID binding. Falls back to tenant
|
||||
// effective engines if the KB has no binding or the load fails.
|
||||
//
|
||||
// Silent fallback risk: if a bound KB fails to load here due to a
|
||||
// transient DB error, the cleanup will delete from env engines and
|
||||
// leave orphan vectors in the bound store. Warn so operators can spot it.
|
||||
var boundStoreID *string
|
||||
if kb, loadErr := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID); loadErr == nil && kb != nil {
|
||||
boundStoreID = kb.VectorStoreID
|
||||
} else if loadErr != nil {
|
||||
logger.GetLogger(ctx).WithField("error", loadErr).WithField("knowledge_base_id", knowledge.KnowledgeBaseID).
|
||||
Warnf("cleanupKnowledgeResources: failed to load KB for vector store resolution; falling back to tenant effective engines")
|
||||
}
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, boundStoreID)
|
||||
if err != nil {
|
||||
logger.GetLogger(ctx).WithField("error", err).Error("Failed to init retrieve engine during cleanup")
|
||||
cleanupErr = errors.Join(cleanupErr, err)
|
||||
|
||||
@@ -421,8 +421,8 @@ func (s *knowledgeService) UpdateFAQEntry(ctx context.Context,
|
||||
oldSimilarQuestionCount := len(oldSimilarQuestions)
|
||||
newSimilarQuestionCount := len(meta.SimilarQuestions)
|
||||
if questionIndexMode == types.FAQQuestionIndexModeSeparate && oldSimilarQuestionCount > newSimilarQuestionCount {
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, engineErr := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
retrieveEngine, engineErr := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), kb.VectorStoreID)
|
||||
if engineErr == nil {
|
||||
sourceIDsToDelete := make([]string, 0, oldSimilarQuestionCount-newSimilarQuestionCount)
|
||||
for i := newSimilarQuestionCount; i < oldSimilarQuestionCount; i++ {
|
||||
@@ -646,8 +646,8 @@ func (s *knowledgeService) UpdateFAQEntryStatus(ctx context.Context,
|
||||
|
||||
// Sync update to retriever engines
|
||||
chunkStatusMap := map[string]bool{chunk.ID: isEnabled}
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -840,11 +840,8 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
|
||||
|
||||
// Sync to retriever engines
|
||||
if len(enabledUpdates) > 0 || len(tagUpdates) > 0 {
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.GetEffectiveEngines(),
|
||||
)
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -902,11 +899,8 @@ func (s *knowledgeService) UpdateFAQEntryTag(ctx context.Context, kbID string, e
|
||||
}
|
||||
|
||||
// Sync tag update to retriever engines
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.GetEffectiveEngines(),
|
||||
)
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1003,11 +997,8 @@ func (s *knowledgeService) UpdateFAQEntryTagBatch(ctx context.Context, kbID stri
|
||||
for _, chunk := range chunksToUpdate {
|
||||
tagUpdates[chunk.ID] = chunk.TagID
|
||||
}
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.GetEffectiveEngines(),
|
||||
)
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1213,8 +1213,8 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
|
||||
) error {
|
||||
indexStartTime := time.Now()
|
||||
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), kb.VectorStoreID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1360,7 +1360,8 @@ func (s *knowledgeService) indexFAQChunks(ctx context.Context,
|
||||
logger.Debugf(ctx, "indexFAQChunks: starting to index %d chunks", len(chunks))
|
||||
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1467,7 +1468,8 @@ func (s *knowledgeService) deleteFAQChunkVectors(ctx context.Context,
|
||||
return err
|
||||
}
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1688,10 +1690,8 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
|
||||
// 删除索引数据
|
||||
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
|
||||
if err == nil {
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
s.retrieveEngine,
|
||||
tenantInfo.GetEffectiveEngines(),
|
||||
)
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
|
||||
if err == nil {
|
||||
chunkIDs := make([]string, 0, len(chunksDeleted))
|
||||
for _, chunk := range chunksDeleted {
|
||||
|
||||
@@ -244,7 +244,8 @@ func (s *knowledgeService) processChunks(ctx context.Context,
|
||||
|
||||
// 删除旧的索引数据 — only when vector/keyword indexing is enabled
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
|
||||
if err == nil && embeddingModel != nil {
|
||||
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil {
|
||||
logger.Warnf(ctx, "Failed to delete existing index data (may not exist): %v", err)
|
||||
@@ -953,7 +954,8 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
|
||||
}
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
|
||||
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
|
||||
return fmt.Errorf("failed to init retrieve engine: %w", err)
|
||||
@@ -1123,7 +1125,8 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
|
||||
}
|
||||
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
|
||||
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
exitStatus = "init_retrieve_engine_failed"
|
||||
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
|
||||
@@ -1577,8 +1580,8 @@ func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, c
|
||||
ids = append(ids, chunk.ID)
|
||||
}
|
||||
|
||||
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), sourceKB.VectorStoreID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ type knowledgeBaseService struct {
|
||||
kbShareService interfaces.KBShareService
|
||||
modelService interfaces.ModelService
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ownership retriever.TenantStoreOwnership
|
||||
tenantRepo interfaces.TenantRepository
|
||||
fileSvc interfaces.FileService
|
||||
graphEngine interfaces.RetrieveGraphRepository
|
||||
@@ -41,6 +42,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
|
||||
kbShareService interfaces.KBShareService,
|
||||
modelService interfaces.ModelService,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
ownership retriever.TenantStoreOwnership,
|
||||
tenantRepo interfaces.TenantRepository,
|
||||
fileSvc interfaces.FileService,
|
||||
graphEngine interfaces.RetrieveGraphRepository,
|
||||
@@ -54,6 +56,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
|
||||
kbShareService: kbShareService,
|
||||
modelService: modelService,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ownership: ownership,
|
||||
tenantRepo: tenantRepo,
|
||||
fileSvc: fileSvc,
|
||||
graphEngine: graphEngine,
|
||||
@@ -364,9 +367,24 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
|
||||
tenantID := types.MustTenantIDFromContext(ctx)
|
||||
tenantInfo, _ := types.TenantInfoFromContext(ctx)
|
||||
|
||||
// Load the KB before soft-delete so we can snapshot its VectorStoreID
|
||||
// into the async cleanup payload. GORM's soft-delete filter hides the
|
||||
// row from subsequent reads, so this read must happen first.
|
||||
kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"knowledge_base_id": id,
|
||||
})
|
||||
return err
|
||||
}
|
||||
var vectorStoreIDSnapshot *string
|
||||
if kb != nil {
|
||||
vectorStoreIDSnapshot = kb.VectorStoreID
|
||||
}
|
||||
|
||||
// Step 1: Delete the knowledge base record first (mark as deleted)
|
||||
logger.Infof(ctx, "Deleting knowledge base from database")
|
||||
err := s.repo.DeleteKnowledgeBase(ctx, id)
|
||||
err = s.repo.DeleteKnowledgeBase(ctx, id)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"knowledge_base_id": id,
|
||||
@@ -384,6 +402,7 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
|
||||
TenantID: tenantID,
|
||||
KnowledgeBaseID: id,
|
||||
EffectiveEngines: tenantInfo.GetEffectiveEngines(),
|
||||
VectorStoreID: vectorStoreIDSnapshot, // snapshot taken before soft-delete
|
||||
}
|
||||
langfuse.InjectTracing(ctx, &payload)
|
||||
|
||||
@@ -444,12 +463,27 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
|
||||
|
||||
logger.Infof(ctx, "Deleting all knowledge entries and their resources")
|
||||
|
||||
// Delete embeddings from vector store
|
||||
// Delete embeddings from vector store.
|
||||
// Resolve the engine via the factory, using the VectorStoreID captured
|
||||
// at enqueue time (may be nil → falls back to payload.EffectiveEngines).
|
||||
// If the payload references a store no longer owned/registered
|
||||
// (e.g. tampered queue entry or a store that was deleted while the
|
||||
// task sat in the queue), the factory returns a sentinel and we
|
||||
// SkipRetry to avoid burning retries on an unrecoverable situation.
|
||||
logger.Infof(ctx, "Deleting embeddings from vector store")
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineFromPayload(
|
||||
ctx,
|
||||
s.retrieveEngine,
|
||||
s.ownership,
|
||||
payload.TenantID,
|
||||
payload.EffectiveEngines,
|
||||
payload.VectorStoreID,
|
||||
)
|
||||
if errors.Is(err, retriever.ErrVectorStoreForbidden) ||
|
||||
errors.Is(err, retriever.ErrVectorStoreNotFound) {
|
||||
logger.Errorf(ctx, "KB delete task aborted: %v (tenant=%d, kb=%s)", err, payload.TenantID, payload.KnowledgeBaseID)
|
||||
return asynq.SkipRetry
|
||||
}
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to create retrieve engine: %v", err)
|
||||
} else {
|
||||
|
||||
@@ -94,15 +94,12 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
|
||||
logger.Infof(ctx, "Hybrid search parameters, knowledge base IDs: %v, query text: %s", searchKBIDs, params.QueryText)
|
||||
|
||||
// tenantInfo is consumed below for retrieval config (post-fusion step).
|
||||
tenantInfo, _ := types.TenantInfoFromContext(ctx)
|
||||
|
||||
// Create a composite retrieval engine with tenant's configured retrievers
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to create retrieval engine: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Resolve the primary KB first so the factory can route to the bound
|
||||
// VectorStore (if any). When the KB has no binding the factory falls
|
||||
// back to the tenant's effective engines.
|
||||
kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
@@ -111,6 +108,18 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tenantID := types.MustTenantIDFromContext(ctx)
|
||||
|
||||
// Create a composite retrieval engine. When the KB is bound to a store,
|
||||
// the factory verifies tenant ownership and returns that store's engine;
|
||||
// otherwise it falls back to the tenant's configured retrievers.
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
|
||||
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
|
||||
if err != nil {
|
||||
logger.Errorf(ctx, "Failed to create retrieval engine: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Use 5x over-retrieval to ensure sufficient candidates for RRF fusion and reranking.
|
||||
// Scale proportionally when searching multiple KBs to maintain per-KB recall quality.
|
||||
matchCount := max(params.MatchCount*5, 50) * len(searchKBIDs)
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package retriever
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/logger"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// Sentinel errors returned by factory functions. Callers may use errors.Is to
|
||||
// classify. User-facing responses MUST wrap or replace these with generic
|
||||
// messages — the sentinels intentionally omit store UUIDs to avoid enumeration
|
||||
// leaks. Structured logs inside the factory record the tenant/store IDs.
|
||||
var (
|
||||
// ErrTenantInfoMissing is returned when the factory needs a tenant from
|
||||
// context (synchronous, unbound KB path) and none is present.
|
||||
ErrTenantInfoMissing = errors.New("tenant info not found in context")
|
||||
|
||||
// ErrVectorStoreNotFound is returned when the store ID is not registered
|
||||
// (or an internal lookup for ownership failed). Async workers should treat
|
||||
// this as non-retryable.
|
||||
ErrVectorStoreNotFound = errors.New("vector store not available")
|
||||
|
||||
// ErrVectorStoreForbidden is returned when the resolved store is not
|
||||
// owned by the given tenant. This guards against cross-tenant access
|
||||
// in case the upstream validation layer has a gap. Async workers should
|
||||
// treat this as non-retryable.
|
||||
ErrVectorStoreForbidden = errors.New("vector store access denied")
|
||||
)
|
||||
|
||||
// TenantStoreOwnership abstracts the lookup used by factory functions to
|
||||
// verify that a given vector store ID is owned by the given tenant ID.
|
||||
//
|
||||
// Production implementations wrap the VectorStoreRepository; tests inject
|
||||
// in-memory fakes so they can cover the ownership branches without touching
|
||||
// a database.
|
||||
type TenantStoreOwnership interface {
|
||||
// StoreOwnedBy reports whether the store with the given ID is owned
|
||||
// by the given tenant. When the store does not exist, it returns
|
||||
// (false, nil). Errors are reserved for infrastructure failures such as
|
||||
// database connectivity issues.
|
||||
StoreOwnedBy(ctx context.Context, storeID string, tenantID uint64) (bool, error)
|
||||
}
|
||||
|
||||
// CreateRetrieveEngineForKB returns a CompositeRetrieveEngine resolved from
|
||||
// a KB's VectorStore binding.
|
||||
//
|
||||
// Resolution rules:
|
||||
//
|
||||
// - vectorStoreID == nil || *vectorStoreID == "" →
|
||||
// falls back to the tenant's effective engines (env-store flow driven
|
||||
// by RETRIEVE_DRIVER). TenantInfo is read from ctx.
|
||||
// - otherwise →
|
||||
// 1) ownership.StoreOwnedBy(*storeID, tenantID) must return true;
|
||||
// cross-tenant attempts yield ErrVectorStoreForbidden.
|
||||
// 2) registry.GetByStoreID(*storeID) must succeed;
|
||||
// unregistered stores yield ErrVectorStoreNotFound.
|
||||
// 3) the single engine is wrapped by NewCompositeRetrieveEngine so
|
||||
// that its Support()-based retriever-type matching is preserved.
|
||||
//
|
||||
// Use this for 23 synchronous call sites across the application services.
|
||||
// Async task handlers that cannot rely on ctx-based TenantInfo (currently:
|
||||
// ProcessKBDeleteTask, ProcessIndexDelete) must use
|
||||
// CreateRetrieveEngineFromPayload instead.
|
||||
func CreateRetrieveEngineForKB(
|
||||
ctx context.Context,
|
||||
registry interfaces.RetrieveEngineRegistry,
|
||||
ownership TenantStoreOwnership,
|
||||
tenantID uint64,
|
||||
vectorStoreID *string,
|
||||
) (*CompositeRetrieveEngine, error) {
|
||||
// Normalize nil and empty-string pointer to "unbound" so that callers
|
||||
// cannot accidentally route an empty UUID into GetByStoreID.
|
||||
if vectorStoreID == nil || *vectorStoreID == "" {
|
||||
tenantInfo, ok := types.TenantInfoFromContext(ctx)
|
||||
if !ok {
|
||||
return nil, ErrTenantInfoMissing
|
||||
}
|
||||
return NewCompositeRetrieveEngine(registry, tenantInfo.GetEffectiveEngines())
|
||||
}
|
||||
|
||||
return resolveBoundEngine(ctx, registry, ownership, tenantID, *vectorStoreID)
|
||||
}
|
||||
|
||||
// CreateRetrieveEngineFromPayload is the async-task variant. It does not
|
||||
// read TenantInfo from ctx because async handlers do not populate it.
|
||||
// Instead, tenantID is passed explicitly from the deserialized payload and
|
||||
// is verified against the store's tenant when vectorStoreID is non-empty.
|
||||
//
|
||||
// Tasks enqueued before vectorStoreID was added to the payload decode it as
|
||||
// nil and transparently fall back to the pre-serialized effectiveEngines
|
||||
// path — no in-flight task is lost across upgrades.
|
||||
func CreateRetrieveEngineFromPayload(
|
||||
ctx context.Context,
|
||||
registry interfaces.RetrieveEngineRegistry,
|
||||
ownership TenantStoreOwnership,
|
||||
tenantID uint64,
|
||||
effectiveEngines []types.RetrieverEngineParams,
|
||||
vectorStoreID *string,
|
||||
) (*CompositeRetrieveEngine, error) {
|
||||
if vectorStoreID == nil || *vectorStoreID == "" {
|
||||
return NewCompositeRetrieveEngine(registry, effectiveEngines)
|
||||
}
|
||||
|
||||
return resolveBoundEngine(ctx, registry, ownership, tenantID, *vectorStoreID)
|
||||
}
|
||||
|
||||
// resolveBoundEngine is the shared ownership-verified lookup path used by
|
||||
// both CreateRetrieveEngineForKB and CreateRetrieveEngineFromPayload. It
|
||||
// returns sentinel errors so that handlers can classify them (for example,
|
||||
// async workers convert Forbidden/NotFound into asynq.SkipRetry).
|
||||
func resolveBoundEngine(
|
||||
ctx context.Context,
|
||||
registry interfaces.RetrieveEngineRegistry,
|
||||
ownership TenantStoreOwnership,
|
||||
tenantID uint64,
|
||||
storeID string,
|
||||
) (*CompositeRetrieveEngine, error) {
|
||||
owned, err := ownership.StoreOwnedBy(ctx, storeID, tenantID)
|
||||
if err != nil {
|
||||
// Infrastructure failure — record the raw error for operators but
|
||||
// do not leak internals to the caller.
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"tenant_id": tenantID,
|
||||
"store_id": storeID,
|
||||
"reason": "ownership lookup failed",
|
||||
})
|
||||
return nil, ErrVectorStoreNotFound
|
||||
}
|
||||
if !owned {
|
||||
// Cross-tenant attempt (or the store has been deleted in the
|
||||
// meantime). Log with WARN so that audits can surface probing.
|
||||
logger.Warnf(ctx,
|
||||
"[retriever.factory] cross-tenant store access attempted: tenant=%d store=%s",
|
||||
tenantID, storeID)
|
||||
return nil, ErrVectorStoreForbidden
|
||||
}
|
||||
|
||||
svc, err := registry.GetByStoreID(storeID)
|
||||
if err != nil {
|
||||
logger.ErrorWithFields(ctx, err, map[string]interface{}{
|
||||
"tenant_id": tenantID,
|
||||
"store_id": storeID,
|
||||
"reason": "store not registered",
|
||||
})
|
||||
return nil, ErrVectorStoreNotFound
|
||||
}
|
||||
|
||||
// Build the composite directly from the resolved service.
|
||||
//
|
||||
// We cannot delegate to NewCompositeRetrieveEngine here because that
|
||||
// function resolves engines through registry.GetRetrieveEngineService,
|
||||
// which reads from the byEngineType map (env stores). DB stores live
|
||||
// in the byStoreID map and are not reachable via engine type alone —
|
||||
// multiple stores can share the same engine type.
|
||||
//
|
||||
// Semantics: a KB bound to a DB store uses every retriever type that
|
||||
// store supports. This intentionally overrides the tenant-level
|
||||
// effective-engines filter, because binding a KB to a specific store
|
||||
// is an explicit opt-out of tenant-default routing.
|
||||
return &CompositeRetrieveEngine{
|
||||
engineInfos: []*engineInfo{{
|
||||
retrieveEngine: svc,
|
||||
retrieverType: slices.Clone(svc.Support()),
|
||||
}},
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
package retriever
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/models/embedding"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// ----- test fakes -----
|
||||
|
||||
// fakeOwnership is an in-memory TenantStoreOwnership implementation.
|
||||
// All mutable state is guarded by mu so that the race test exercises a
|
||||
// safe fake and surfaces only genuine races inside the factory code.
|
||||
type fakeOwnership struct {
|
||||
mu sync.Mutex
|
||||
// owned maps storeID → tenantID that owns it. Absence means "not owned".
|
||||
owned map[string]uint64
|
||||
// err, when non-nil, is returned from every StoreOwnedBy call
|
||||
// (used to simulate infrastructure failures).
|
||||
err error
|
||||
// calls records every (storeID, tenantID) pair asked. Tests use this
|
||||
// to assert that the factory short-circuits to the fallback path
|
||||
// without calling ownership when vectorStoreID is nil/empty.
|
||||
calls []ownershipCall
|
||||
}
|
||||
|
||||
type ownershipCall struct {
|
||||
storeID string
|
||||
tenantID uint64
|
||||
}
|
||||
|
||||
func (f *fakeOwnership) StoreOwnedBy(ctx context.Context, storeID string, tenantID uint64) (bool, error) {
|
||||
f.mu.Lock()
|
||||
f.calls = append(f.calls, ownershipCall{storeID: storeID, tenantID: tenantID})
|
||||
err := f.err
|
||||
ownedTenant, ok := f.owned[storeID]
|
||||
f.mu.Unlock()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return ok && ownedTenant == tenantID, nil
|
||||
}
|
||||
|
||||
// callCount returns the number of StoreOwnedBy calls recorded so far.
|
||||
// Safe for concurrent use alongside StoreOwnedBy.
|
||||
func (f *fakeOwnership) callCount() int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return len(f.calls)
|
||||
}
|
||||
|
||||
// fakeEngine implements interfaces.RetrieveEngineService with only the
|
||||
// methods that factory paths exercise. All other methods panic so that
|
||||
// accidental use during testing is loud.
|
||||
type fakeEngine struct {
|
||||
engineType types.RetrieverEngineType
|
||||
support []types.RetrieverType
|
||||
}
|
||||
|
||||
func (f *fakeEngine) EngineType() types.RetrieverEngineType { return f.engineType }
|
||||
|
||||
func (f *fakeEngine) Support() []types.RetrieverType { return f.support }
|
||||
|
||||
func (f *fakeEngine) Retrieve(ctx context.Context, _ types.RetrieveParams) ([]*types.RetrieveResult, error) {
|
||||
panic("fakeEngine.Retrieve: not used in factory tests")
|
||||
}
|
||||
|
||||
func (f *fakeEngine) Index(ctx context.Context, _ embedding.Embedder, _ *types.IndexInfo, _ []types.RetrieverType) error {
|
||||
panic("fakeEngine.Index: not used in factory tests")
|
||||
}
|
||||
|
||||
func (f *fakeEngine) BatchIndex(ctx context.Context, _ embedding.Embedder, _ []*types.IndexInfo, _ []types.RetrieverType) error {
|
||||
panic("fakeEngine.BatchIndex: not used in factory tests")
|
||||
}
|
||||
|
||||
func (f *fakeEngine) EstimateStorageSize(ctx context.Context, _ embedding.Embedder, _ []*types.IndexInfo, _ []types.RetrieverType) int64 {
|
||||
panic("fakeEngine.EstimateStorageSize: not used in factory tests")
|
||||
}
|
||||
|
||||
func (f *fakeEngine) CopyIndices(ctx context.Context, _ string, _ map[string]string, _ map[string]string, _ string, _ int, _ string) error {
|
||||
panic("fakeEngine.CopyIndices: not used in factory tests")
|
||||
}
|
||||
|
||||
func (f *fakeEngine) DeleteByChunkIDList(ctx context.Context, _ []string, _ int, _ string) error {
|
||||
panic("fakeEngine.DeleteByChunkIDList: not used in factory tests")
|
||||
}
|
||||
|
||||
func (f *fakeEngine) DeleteBySourceIDList(ctx context.Context, _ []string, _ int, _ string) error {
|
||||
panic("fakeEngine.DeleteBySourceIDList: not used in factory tests")
|
||||
}
|
||||
|
||||
func (f *fakeEngine) DeleteByKnowledgeIDList(ctx context.Context, _ []string, _ int, _ string) error {
|
||||
panic("fakeEngine.DeleteByKnowledgeIDList: not used in factory tests")
|
||||
}
|
||||
|
||||
func (f *fakeEngine) BatchUpdateChunkEnabledStatus(ctx context.Context, _ map[string]bool) error {
|
||||
panic("fakeEngine.BatchUpdateChunkEnabledStatus: not used in factory tests")
|
||||
}
|
||||
|
||||
func (f *fakeEngine) BatchUpdateChunkTagID(ctx context.Context, _ map[string]string) error {
|
||||
panic("fakeEngine.BatchUpdateChunkTagID: not used in factory tests")
|
||||
}
|
||||
|
||||
// registryWithStores builds a registry with the given storeID → service
|
||||
// pairs populated in the byStoreID map. Engine-type entries are also
|
||||
// registered so that the unbound path can resolve engines via the
|
||||
// tenant's effective engines.
|
||||
func registryWithStores(t *testing.T, stores map[string]*fakeEngine, engineTypes map[types.RetrieverEngineType]*fakeEngine) *RetrieveEngineRegistry {
|
||||
t.Helper()
|
||||
r := &RetrieveEngineRegistry{
|
||||
byEngineType: map[types.RetrieverEngineType]interfaces.RetrieveEngineService{},
|
||||
byStoreID: map[string]interfaces.RetrieveEngineService{},
|
||||
}
|
||||
for id, svc := range stores {
|
||||
r.byStoreID[id] = svc
|
||||
}
|
||||
for et, svc := range engineTypes {
|
||||
r.byEngineType[et] = svc
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// newTenantCtx returns a context carrying a Tenant with the given
|
||||
// EffectiveEngines. Factory's unbound path consumes this via
|
||||
// types.TenantInfoFromContext.
|
||||
func newTenantCtx(engines []types.RetrieverEngineParams) context.Context {
|
||||
tenant := &types.Tenant{
|
||||
RetrieverEngines: types.RetrieverEngines{Engines: engines},
|
||||
}
|
||||
return context.WithValue(context.Background(), types.TenantInfoContextKey, tenant)
|
||||
}
|
||||
|
||||
// ----- CreateRetrieveEngineForKB -----
|
||||
|
||||
func TestCreateRetrieveEngineForKB_Unbound(t *testing.T) {
|
||||
postgresEngine := &fakeEngine{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType},
|
||||
}
|
||||
registry := registryWithStores(t, nil, map[types.RetrieverEngineType]*fakeEngine{
|
||||
types.PostgresRetrieverEngineType: postgresEngine,
|
||||
})
|
||||
ownership := &fakeOwnership{}
|
||||
|
||||
tenantCtx := newTenantCtx([]types.RetrieverEngineParams{
|
||||
{RetrieverEngineType: types.PostgresRetrieverEngineType, RetrieverType: types.VectorRetrieverType},
|
||||
})
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
store *string
|
||||
}{
|
||||
{"nil pointer", nil},
|
||||
{"empty string pointer", strPtr("")},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine, err := CreateRetrieveEngineForKB(tenantCtx, registry, ownership, 1, tc.store)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, engine)
|
||||
assert.Len(t, engine.engineInfos, 1, "unbound path uses tenant effective engines")
|
||||
assert.Same(t, postgresEngine, engine.engineInfos[0].retrieveEngine)
|
||||
assert.Zero(t, ownership.callCount(),
|
||||
"ownership must not be called on the unbound path")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateRetrieveEngineForKB_UnboundMissingTenant(t *testing.T) {
|
||||
registry := registryWithStores(t, nil, nil)
|
||||
ownership := &fakeOwnership{}
|
||||
|
||||
_, err := CreateRetrieveEngineForKB(context.Background(), registry, ownership, 1, nil)
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Is(err, ErrTenantInfoMissing),
|
||||
"err %q must wrap ErrTenantInfoMissing", err)
|
||||
}
|
||||
|
||||
func TestCreateRetrieveEngineForKB_StoreBound(t *testing.T) {
|
||||
esEngine := &fakeEngine{
|
||||
engineType: types.ElasticsearchRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType},
|
||||
}
|
||||
registry := registryWithStores(t,
|
||||
map[string]*fakeEngine{"store-A": esEngine},
|
||||
nil,
|
||||
)
|
||||
ownership := &fakeOwnership{owned: map[string]uint64{"store-A": 1}}
|
||||
|
||||
storeID := "store-A"
|
||||
engine, err := CreateRetrieveEngineForKB(context.Background(), registry, ownership, 1, &storeID)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, engine)
|
||||
require.Len(t, engine.engineInfos, 1)
|
||||
assert.Same(t, esEngine, engine.engineInfos[0].retrieveEngine)
|
||||
assert.Equal(t,
|
||||
[]types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType},
|
||||
engine.engineInfos[0].retrieverType,
|
||||
"store-bound KB uses every retriever type the store supports")
|
||||
}
|
||||
|
||||
func TestCreateRetrieveEngineForKB_CrossTenant(t *testing.T) {
|
||||
esEngine := &fakeEngine{
|
||||
engineType: types.ElasticsearchRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
}
|
||||
registry := registryWithStores(t,
|
||||
map[string]*fakeEngine{"store-A": esEngine},
|
||||
nil,
|
||||
)
|
||||
// store-A is owned by tenant 2, not tenant 1
|
||||
ownership := &fakeOwnership{owned: map[string]uint64{"store-A": 2}}
|
||||
|
||||
storeID := "store-A"
|
||||
_, err := CreateRetrieveEngineForKB(context.Background(), registry, ownership, 1, &storeID)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Is(err, ErrVectorStoreForbidden),
|
||||
"cross-tenant access must yield ErrVectorStoreForbidden; got %q", err)
|
||||
// Sanity: no store UUID in the sentinel's message — the structured log
|
||||
// emitted inside the factory is where UUIDs live. The user-visible
|
||||
// error is intentionally opaque.
|
||||
assert.NotContains(t, err.Error(), "store-A",
|
||||
"sentinel error must not expose store UUIDs to callers")
|
||||
}
|
||||
|
||||
func TestCreateRetrieveEngineForKB_StoreNotRegistered(t *testing.T) {
|
||||
// Ownership says the tenant owns the store, but registry has not
|
||||
// loaded it. This happens when a VectorStore row exists in the DB
|
||||
// but its engine initialization failed at app start.
|
||||
registry := registryWithStores(t, nil, nil)
|
||||
ownership := &fakeOwnership{owned: map[string]uint64{"store-A": 1}}
|
||||
|
||||
storeID := "store-A"
|
||||
_, err := CreateRetrieveEngineForKB(context.Background(), registry, ownership, 1, &storeID)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Is(err, ErrVectorStoreNotFound),
|
||||
"unregistered store must yield ErrVectorStoreNotFound; got %q", err)
|
||||
}
|
||||
|
||||
func TestCreateRetrieveEngineForKB_OwnershipLookupError(t *testing.T) {
|
||||
registry := registryWithStores(t,
|
||||
map[string]*fakeEngine{"store-A": {engineType: types.PostgresRetrieverEngineType}},
|
||||
nil,
|
||||
)
|
||||
ownership := &fakeOwnership{err: errors.New("db connection refused")}
|
||||
|
||||
storeID := "store-A"
|
||||
_, err := CreateRetrieveEngineForKB(context.Background(), registry, ownership, 1, &storeID)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Is(err, ErrVectorStoreNotFound),
|
||||
"ownership infrastructure error must collapse to ErrVectorStoreNotFound; got %q", err)
|
||||
}
|
||||
|
||||
// ----- CreateRetrieveEngineFromPayload -----
|
||||
|
||||
func TestCreateRetrieveEngineFromPayload_LegacyUnbound(t *testing.T) {
|
||||
// Four ways a payload can express "no binding": missing field,
|
||||
// explicit null, empty string, and programmatically nil. All four
|
||||
// should take the fallback path using the payload's effectiveEngines.
|
||||
postgresEngine := &fakeEngine{
|
||||
engineType: types.PostgresRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
}
|
||||
registry := registryWithStores(t, nil, map[types.RetrieverEngineType]*fakeEngine{
|
||||
types.PostgresRetrieverEngineType: postgresEngine,
|
||||
})
|
||||
ownership := &fakeOwnership{}
|
||||
engines := []types.RetrieverEngineParams{{
|
||||
RetrieverEngineType: types.PostgresRetrieverEngineType,
|
||||
RetrieverType: types.VectorRetrieverType,
|
||||
}}
|
||||
|
||||
// simulate payload decoding from JSON for each legacy shape
|
||||
decode := func(t *testing.T, body string) *string {
|
||||
t.Helper()
|
||||
var p types.KBDeletePayload
|
||||
require.NoError(t, json.Unmarshal([]byte(body), &p))
|
||||
return p.VectorStoreID
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
ptr *string
|
||||
}{
|
||||
{"missing field", decode(t, `{"tenant_id":1}`)},
|
||||
{"explicit null", decode(t, `{"tenant_id":1,"vector_store_id":null}`)},
|
||||
{"empty string", decode(t, `{"tenant_id":1,"vector_store_id":""}`)},
|
||||
{"programmatic nil", nil},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
engine, err := CreateRetrieveEngineFromPayload(
|
||||
context.Background(), registry, ownership, 1, engines, tc.ptr)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, engine)
|
||||
require.Len(t, engine.engineInfos, 1)
|
||||
assert.Same(t, postgresEngine, engine.engineInfos[0].retrieveEngine)
|
||||
assert.Zero(t, ownership.callCount(),
|
||||
"unbound payload must not trigger ownership lookup")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateRetrieveEngineFromPayload_Bound(t *testing.T) {
|
||||
qdrantEngine := &fakeEngine{
|
||||
engineType: types.QdrantRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
}
|
||||
registry := registryWithStores(t,
|
||||
map[string]*fakeEngine{"qd-1": qdrantEngine},
|
||||
nil,
|
||||
)
|
||||
ownership := &fakeOwnership{owned: map[string]uint64{"qd-1": 42}}
|
||||
|
||||
storeID := "qd-1"
|
||||
engine, err := CreateRetrieveEngineFromPayload(
|
||||
context.Background(), registry, ownership, 42, nil, &storeID)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, engine)
|
||||
require.Len(t, engine.engineInfos, 1)
|
||||
assert.Same(t, qdrantEngine, engine.engineInfos[0].retrieveEngine)
|
||||
}
|
||||
|
||||
func TestCreateRetrieveEngineFromPayload_TamperedCrossTenant(t *testing.T) {
|
||||
esEngine := &fakeEngine{engineType: types.ElasticsearchRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType}}
|
||||
registry := registryWithStores(t,
|
||||
map[string]*fakeEngine{"store-A": esEngine}, nil)
|
||||
// Store is owned by tenant 99, but the (possibly tampered) payload
|
||||
// claims tenant 1. Factory must reject.
|
||||
ownership := &fakeOwnership{owned: map[string]uint64{"store-A": 99}}
|
||||
|
||||
storeID := "store-A"
|
||||
_, err := CreateRetrieveEngineFromPayload(
|
||||
context.Background(), registry, ownership, 1,
|
||||
[]types.RetrieverEngineParams{}, &storeID)
|
||||
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Is(err, ErrVectorStoreForbidden))
|
||||
}
|
||||
|
||||
// ----- Race -----
|
||||
|
||||
// TestFactoryParallelInvocation is primarily meaningful when run under
|
||||
// `go test -race`. Factory functions themselves are stateless; the test
|
||||
// guards against accidental shared state being introduced later.
|
||||
func TestFactoryParallelInvocation(t *testing.T) {
|
||||
esEngine := &fakeEngine{
|
||||
engineType: types.ElasticsearchRetrieverEngineType,
|
||||
support: []types.RetrieverType{types.VectorRetrieverType},
|
||||
}
|
||||
registry := registryWithStores(t,
|
||||
map[string]*fakeEngine{"store-A": esEngine},
|
||||
nil,
|
||||
)
|
||||
ownership := &fakeOwnership{owned: map[string]uint64{"store-A": 1}}
|
||||
|
||||
done := make(chan error, 16)
|
||||
for i := 0; i < 16; i++ {
|
||||
go func() {
|
||||
storeID := "store-A"
|
||||
_, err := CreateRetrieveEngineForKB(
|
||||
context.Background(), registry, ownership, 1, &storeID)
|
||||
done <- err
|
||||
}()
|
||||
}
|
||||
for i := 0; i < 16; i++ {
|
||||
if err := <-done; err != nil {
|
||||
t.Errorf("concurrent call %d failed: %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ----- helpers -----
|
||||
|
||||
func strPtr(s string) *string { return &s }
|
||||
@@ -0,0 +1,35 @@
|
||||
package retriever
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
)
|
||||
|
||||
// vectorStoreRepoOwnership adapts a VectorStoreRepository to the
|
||||
// TenantStoreOwnership interface consumed by the factory functions.
|
||||
//
|
||||
// The repository's GetByID already scopes to the given tenantID
|
||||
// (WHERE id = ? AND tenant_id = ?), so "exists under this tenant"
|
||||
// is equivalent to "owned by this tenant" — no additional tenant
|
||||
// comparison is needed on top of the returned row.
|
||||
type vectorStoreRepoOwnership struct {
|
||||
repo interfaces.VectorStoreRepository
|
||||
}
|
||||
|
||||
// NewVectorStoreRepoOwnership returns the production
|
||||
// TenantStoreOwnership implementation backed by VectorStoreRepository.
|
||||
func NewVectorStoreRepoOwnership(repo interfaces.VectorStoreRepository) TenantStoreOwnership {
|
||||
return &vectorStoreRepoOwnership{repo: repo}
|
||||
}
|
||||
|
||||
// StoreOwnedBy returns true iff a vector store with the given ID exists
|
||||
// under the given tenant. Errors are reserved for infrastructure failures;
|
||||
// a non-existent (but well-formed) store ID returns (false, nil).
|
||||
func (o *vectorStoreRepoOwnership) StoreOwnedBy(ctx context.Context, storeID string, tenantID uint64) (bool, error) {
|
||||
store, err := o.repo.GetByID(ctx, tenantID, storeID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return store != nil, nil
|
||||
}
|
||||
@@ -112,3 +112,7 @@ func (r *RetrieveEngineRegistry) UnregisterByStoreID(storeID string) {
|
||||
|
||||
delete(r.byStoreID, storeID)
|
||||
}
|
||||
|
||||
// Compile-time assertion: *RetrieveEngineRegistry satisfies the
|
||||
// interfaces.RetrieveEngineRegistry contract, including GetByStoreID.
|
||||
var _ interfaces.RetrieveEngineRegistry = (*RetrieveEngineRegistry)(nil)
|
||||
|
||||
@@ -25,6 +25,7 @@ type knowledgeTagService struct {
|
||||
knowledgeRepo interfaces.KnowledgeRepository
|
||||
chunkRepo interfaces.ChunkRepository
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry
|
||||
ownership retriever.TenantStoreOwnership
|
||||
modelService interfaces.ModelService
|
||||
task interfaces.TaskEnqueuer
|
||||
kbShareService interfaces.KBShareService
|
||||
@@ -37,6 +38,7 @@ func NewKnowledgeTagService(
|
||||
knowledgeRepo interfaces.KnowledgeRepository,
|
||||
chunkRepo interfaces.ChunkRepository,
|
||||
retrieveEngine interfaces.RetrieveEngineRegistry,
|
||||
ownership retriever.TenantStoreOwnership,
|
||||
modelService interfaces.ModelService,
|
||||
task interfaces.TaskEnqueuer,
|
||||
kbShareService interfaces.KBShareService,
|
||||
@@ -47,6 +49,7 @@ func NewKnowledgeTagService(
|
||||
knowledgeRepo: knowledgeRepo,
|
||||
chunkRepo: chunkRepo,
|
||||
retrieveEngine: retrieveEngine,
|
||||
ownership: ownership,
|
||||
modelService: modelService,
|
||||
task: task,
|
||||
kbShareService: kbShareService,
|
||||
@@ -257,7 +260,7 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
|
||||
|
||||
// Enqueue async index deletion task for the deleted chunks
|
||||
if len(deletedIDs) > 0 {
|
||||
s.enqueueIndexDeleteTask(ctx, tenantID, kb.ID, kb.EmbeddingModelID, string(kb.Type), deletedIDs, tenantInfo.GetEffectiveEngines())
|
||||
s.enqueueIndexDeleteTask(ctx, tenantID, kb.ID, kb.EmbeddingModelID, string(kb.Type), deletedIDs, tenantInfo.GetEffectiveEngines(), kb.VectorStoreID)
|
||||
}
|
||||
|
||||
logger.Infof(ctx, "Deleted %d chunks under tag %s", len(deletedIDs), tag.ID)
|
||||
@@ -341,9 +344,14 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
|
||||
return s.repo.Delete(ctx, tenantID, id)
|
||||
}
|
||||
|
||||
// enqueueIndexDeleteTask enqueues an async task for index deletion (low priority)
|
||||
// enqueueIndexDeleteTask enqueues an async task for index deletion (low priority).
|
||||
//
|
||||
// vectorStoreID is captured from the owning KB at enqueue time and snapshotted
|
||||
// into the payload so the worker can route to the correct store even if the
|
||||
// KB is deleted or rebound before the task runs.
|
||||
func (s *knowledgeTagService) enqueueIndexDeleteTask(ctx context.Context,
|
||||
tenantID uint64, kbID, embeddingModelID, kbType string, chunkIDs []string, effectiveEngines []types.RetrieverEngineParams,
|
||||
tenantID uint64, kbID, embeddingModelID, kbType string, chunkIDs []string,
|
||||
effectiveEngines []types.RetrieverEngineParams, vectorStoreID *string,
|
||||
) {
|
||||
payload := types.IndexDeletePayload{
|
||||
TenantID: tenantID,
|
||||
@@ -352,6 +360,7 @@ func (s *knowledgeTagService) enqueueIndexDeleteTask(ctx context.Context,
|
||||
KBType: kbType,
|
||||
ChunkIDs: chunkIDs,
|
||||
EffectiveEngines: effectiveEngines,
|
||||
VectorStoreID: vectorStoreID,
|
||||
}
|
||||
langfuse.InjectTracing(ctx, &payload)
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
@@ -382,8 +391,20 @@ func (s *knowledgeTagService) ProcessIndexDelete(ctx context.Context, t *asynq.T
|
||||
|
||||
logger.Infof(ctx, "Processing index delete task for %d chunks in KB %s", len(payload.ChunkIDs), payload.KnowledgeBaseID)
|
||||
|
||||
// Create retrieve engine
|
||||
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, payload.EffectiveEngines)
|
||||
// Create retrieve engine.
|
||||
// The factory verifies the payload's tenant owns the bound store, so a
|
||||
// tampered queue entry cannot reach a cross-tenant store.
|
||||
// Forbidden/NotFound are non-retryable — SkipRetry prevents burning the
|
||||
// full retry budget on an unrecoverable task.
|
||||
retrieveEngine, err := retriever.CreateRetrieveEngineFromPayload(
|
||||
ctx, s.retrieveEngine, s.ownership,
|
||||
payload.TenantID, payload.EffectiveEngines, payload.VectorStoreID,
|
||||
)
|
||||
if errors.Is(err, retriever.ErrVectorStoreForbidden) ||
|
||||
errors.Is(err, retriever.ErrVectorStoreNotFound) {
|
||||
logger.Errorf(ctx, "Index delete task aborted: %v (tenant=%d, kb=%s)", err, payload.TenantID, payload.KnowledgeBaseID)
|
||||
return asynq.SkipRetry
|
||||
}
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "Failed to create retrieve engine for index cleanup: %v", err)
|
||||
return err
|
||||
|
||||
@@ -204,6 +204,9 @@ func BuildContainer(container *dig.Container) *dig.Container {
|
||||
must(container.Invoke(registerWebSearchProviders))
|
||||
must(container.Provide(repository.NewWebSearchProviderRepository))
|
||||
must(container.Provide(repository.NewVectorStoreRepository))
|
||||
// TenantStoreOwnership adapter used by the retriever factory functions
|
||||
// to verify that a resolved VectorStore belongs to the caller's tenant.
|
||||
must(container.Provide(retriever.NewVectorStoreRepoOwnership))
|
||||
must(container.Provide(service.NewWebSearchService))
|
||||
must(container.Provide(service.NewWebSearchProviderService))
|
||||
must(container.Provide(NewEngineFactory))
|
||||
|
||||
@@ -72,6 +72,15 @@ type RetrieveEngineRegistry interface {
|
||||
GetRetrieveEngineService(engineType types.RetrieverEngineType) (RetrieveEngineService, error)
|
||||
// GetAllRetrieveEngineServices gets all retrieve engine services
|
||||
GetAllRetrieveEngineServices() []RetrieveEngineService
|
||||
|
||||
// GetByStoreID returns the engine service registered for a specific DB store ID.
|
||||
//
|
||||
// IMPORTANT: This method does NOT verify tenant ownership of the returned
|
||||
// store. Callers MUST use the CreateRetrieveEngineForKB /
|
||||
// CreateRetrieveEngineFromPayload factory functions in the retriever package
|
||||
// rather than calling this directly. The factories wrap GetByStoreID with
|
||||
// tenant ownership verification (defense-in-depth against cross-tenant IDOR).
|
||||
GetByStoreID(storeID string) (RetrieveEngineService, error)
|
||||
}
|
||||
|
||||
// RetrieveEngineService defines the retrieve engine service interface
|
||||
|
||||
@@ -99,6 +99,10 @@ type IndexDeletePayload struct {
|
||||
KBType string `json:"kb_type"`
|
||||
ChunkIDs []string `json:"chunk_ids"`
|
||||
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
|
||||
// VectorStoreID is the bound store snapshot taken at enqueue time so the
|
||||
// async worker can resolve the same store the KB was bound to.
|
||||
// nil means the KB had no binding — falls back to EffectiveEngines.
|
||||
VectorStoreID *string `json:"vector_store_id,omitempty"`
|
||||
}
|
||||
|
||||
// KBDeletePayload represents the knowledge base delete task payload
|
||||
@@ -107,6 +111,10 @@ type KBDeletePayload struct {
|
||||
TenantID uint64 `json:"tenant_id"`
|
||||
KnowledgeBaseID string `json:"knowledge_base_id"`
|
||||
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
|
||||
// VectorStoreID is the bound store snapshot taken at enqueue time (before
|
||||
// soft-delete) so the async worker can resolve the right store. nil means
|
||||
// the KB had no binding — falls back to EffectiveEngines.
|
||||
VectorStoreID *string `json:"vector_store_id,omitempty"`
|
||||
}
|
||||
|
||||
// KnowledgeListDeletePayload represents the batch knowledge delete task payload
|
||||
|
||||
Reference in New Issue
Block a user