refactor(retriever): introduce factory for KB-scoped retrieve engine resolution

Extracts the 25 repeated NewCompositeRetrieveEngine call sites across
seven services into two factory functions with tenant-ownership
verification: CreateRetrieveEngineForKB (synchronous) and
CreateRetrieveEngineFromPayload (async task handlers). Promotes
GetByStoreID from concrete-only to interfaces.RetrieveEngineRegistry.
Extends KBDeletePayload / IndexDeletePayload with an omitempty
VectorStoreID snapshot.

Design highlights
- Factory verifies tenant ownership of the resolved store
  (defense-in-depth) so a gap in PR3 validation or a tampered Asynq
  payload cannot cross tenants. Cross-tenant attempts return
  ErrVectorStoreForbidden with a structured log entry.
- Sentinel errors (ErrTenantInfoMissing, ErrVectorStoreNotFound,
  ErrVectorStoreForbidden) let async handlers classify non-retryable
  failures as asynq.SkipRetry.
- nil and empty-string pointers for vectorStoreID normalize to
  "unbound" so callers never send an empty UUID to GetByStoreID.
- resolveBoundEngine constructs engineInfos directly (with
  slices.Clone of Support()) instead of going through
  NewCompositeRetrieveEngine. The latter only reads from the
  byEngineType map and cannot reach DB stores in byStoreID -- multiple
  stores can share the same engine type. Unbound fallbacks still reuse
  NewCompositeRetrieveEngine so the tenant effective-engines path is
  byte-for-byte unchanged.
- DeleteKnowledgeBase now reads the KB before soft-delete so the
  enqueued KBDeletePayload carries a VectorStoreID snapshot; once the
  row is soft-deleted GORM's default scope hides it. Cost: +1 SELECT
  per KB-delete request (a rare admin operation).
- enqueueIndexDeleteTask gains a vectorStoreID *string parameter that
  the single caller (tag.go) populates from the owning KB. The worker
  (ProcessIndexDelete) validates ownership via the factory.
- cleanupKnowledgeResources loads the KB so bound-store cleanup routes
  correctly. If the load fails it warns and falls back to tenant
  effective engines; orphan vectors become observable in logs instead
  of disappearing silently.
- DeleteKnowledges (batch) stays on the tenant effective-engines path
  -- batches can span stores and multi-store fan-out is PR4 scope.
  Noted inline.

Merge safety

All knowledge bases have VectorStoreID = NULL at merge time (introduced
nullable in PR1, Tencent/WeKnora#994). Every factory call falls back to
NewCompositeRetrieveEngine(registry, tenantInfo.GetEffectiveEngines()),
which is the existing code path for these KBs. Async payloads from
before this change decode to VectorStoreID = nil (omitempty) and take
the same fallback.

Testing

- go build ./...
- go vet ./...
- go test -race -count=1 ./internal/application/service/retriever/...
  covers unbound (nil/empty), bound, cross-tenant, unregistered store,
  ownership infra error, legacy payload shapes (missing field/null/
  empty string/nil), tampered payload, and parallel invocation.
- grep -r NewCompositeRetrieveEngine internal/application/service
  returns zero results outside factory.go/composite.go.

Refs: Part of Tencent/WeKnora#993. Depends on Tencent/WeKnora#994.
This commit is contained in:
ochan.kwon
2026-05-13 17:24:17 +09:00
committed by lyingbug
parent 35c79281c8
commit 7478e3cddb
19 changed files with 828 additions and 71 deletions
+5 -2
View File
@@ -21,6 +21,7 @@ type chunkService struct {
kbRepository interfaces.KnowledgeBaseRepository
modelService interfaces.ModelService
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
}
// NewChunkService creates a new chunk service
@@ -35,12 +36,14 @@ func NewChunkService(
kbRepository interfaces.KnowledgeBaseRepository,
modelService interfaces.ModelService,
retrieveEngine interfaces.RetrieveEngineRegistry,
ownership retriever.TenantStoreOwnership,
) interfaces.ChunkService {
return &chunkService{
chunkRepository: chunkRepository,
kbRepository: kbRepository,
modelService: modelService,
retrieveEngine: retrieveEngine,
ownership: ownership,
}
}
@@ -404,8 +407,8 @@ func (s *chunkService) DeleteGeneratedQuestion(ctx context.Context, chunkID stri
// The source_id format is: {chunk_id}-{question_id}
sourceID := fmt.Sprintf("%s-%s", chunkID, questionID)
tenantInfo, _ := types.TenantInfoFromContext(ctx)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"chunk_id": chunkID,
+22 -2
View File
@@ -259,6 +259,7 @@ type DataTableSummaryService struct {
chunkService interfaces.ChunkService
tenantService interfaces.TenantService
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
sqlDB *sql.DB
}
@@ -271,6 +272,7 @@ func NewDataTableSummaryService(
chunkService interfaces.ChunkService,
tenantService interfaces.TenantService,
retrieveEngine interfaces.RetrieveEngineRegistry,
ownership retriever.TenantStoreOwnership,
sqlDB *sql.DB,
) interfaces.TaskHandler {
return &DataTableSummaryService{
@@ -281,6 +283,7 @@ func NewDataTableSummaryService(
chunkService: chunkService,
tenantService: tenantService,
retrieveEngine: retrieveEngine,
ownership: ownership,
sqlDB: sqlDB,
}
}
@@ -370,8 +373,25 @@ func (s *DataTableSummaryService) prepareResources(ctx context.Context, payload
return nil, err
}
// 获取检索引擎
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
// Load the KB to discover its VectorStoreID binding so the factory can
// route to the bound store (or fall back to tenant engines if unbound).
kb, err := s.knowledgeBaseService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID)
if err != nil {
logger.Errorf(ctx, "failed to get knowledge base for vector store lookup: %v", err)
return nil, err
}
var vectorStoreID *string
if kb != nil {
vectorStoreID = kb.VectorStoreID
}
// The factory's unbound path reads TenantInfo from ctx.
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
// Resolve the engine via the factory using the KB's VectorStore binding
// (nil -> tenant effective engines fallback; verified tenant ownership otherwise).
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, payload.TenantID, vectorStoreID)
if err != nil {
logger.Errorf(ctx, "failed to get retrieve engine: %v", err)
return nil, err
@@ -60,6 +60,7 @@ type ImageMultimodalService struct {
knowledgeRepo interfaces.KnowledgeRepository
tenantRepo interfaces.TenantRepository
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
ollamaService *ollama.OllamaService
taskEnqueuer interfaces.TaskEnqueuer
redisClient *redis.Client
@@ -78,6 +79,7 @@ func NewImageMultimodalService(
knowledgeRepo interfaces.KnowledgeRepository,
tenantRepo interfaces.TenantRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ownership retriever.TenantStoreOwnership,
ollamaService *ollama.OllamaService,
taskEnqueuer interfaces.TaskEnqueuer,
redisClient *redis.Client,
@@ -90,6 +92,7 @@ func NewImageMultimodalService(
knowledgeRepo: knowledgeRepo,
tenantRepo: tenantRepo,
retrieveEngine: retrieveEngine,
ownership: ownership,
ollamaService: ollamaService,
taskEnqueuer: taskEnqueuer,
redisClient: redisClient,
@@ -269,8 +272,13 @@ func (s *ImageMultimodalService) indexChunks(ctx context.Context, payload types.
logger.Warnf(ctx, "[ImageMultimodal] Failed to get tenant for indexing: %v", err)
return
}
// The factory's unbound path reads TenantInfo from ctx; make sure it's there.
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
engine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
// Resolve engine via the factory using the KB's VectorStore binding
// (nil → tenant effective engines fallback; verified tenant ownership otherwise).
engine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, payload.TenantID, kb.VectorStoreID)
if err != nil {
logger.Warnf(ctx, "[ImageMultimodal] Failed to init retrieve engine: %v", err)
return
@@ -8,6 +8,7 @@ import (
"strings"
"sync"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/config"
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
@@ -38,6 +39,7 @@ var (
type knowledgeService struct {
config *config.Config
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
repo interfaces.KnowledgeRepository
kbService interfaces.KnowledgeBaseService
tenantRepo interfaces.TenantRepository
@@ -86,6 +88,7 @@ func NewKnowledgeService(
task interfaces.TaskEnqueuer,
graphEngine interfaces.RetrieveGraphRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ownership retriever.TenantStoreOwnership,
redisClient *redis.Client,
kbShareService interfaces.KBShareService,
imageResolver *docparser.ImageResolver,
@@ -109,6 +112,7 @@ func NewKnowledgeService(
task: task,
graphEngine: graphEngine,
retrieveEngine: retrieveEngine,
ownership: ownership,
redisClient: redisClient,
kbShareService: kbShareService,
imageResolver: imageResolver,
@@ -194,8 +194,18 @@ func (s *knowledgeService) CloneChunk(ctx context.Context, src, dst *types.Knowl
}
}
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
tenantID := types.MustTenantIDFromContext(ctx)
// Route CopyIndices via the source KB's bound store. This function does
// not handle cross-store copies — embeddings written by different
// VectorStore backends are not bit-compatible, so callers that allow
// source/target KBs to bind to different stores must perform their own
// cross-store migration before invoking this.
var sourceStoreID *string
if srcKB, loadErr := s.kbService.GetKnowledgeBaseByID(ctx, src.KnowledgeBaseID); loadErr == nil && srcKB != nil {
sourceStoreID = srcKB.VectorStoreID
}
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID)
if err != nil {
return err
}
@@ -441,9 +451,15 @@ func (s *knowledgeService) cloneFAQKnowledgeBase(
return nil
}
// Get tenant info and initialize retrieve engine
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
// Route the FAQ clone through the source KB's bound store. Same
// constraint as CloneChunk: callers must ensure source and target share
// the same VectorStore (cross-store FAQ clone is not handled here).
var sourceStoreID *string
if srcKB != nil {
sourceStoreID = srcKB.VectorStoreID
}
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), sourceStoreID)
if err != nil {
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
handleError(progress, err, "Failed to initialize retrieve engine")
@@ -856,7 +872,6 @@ func (s *knowledgeService) moveKnowledgeReuseVectors(
sourceKB, targetKB *types.KnowledgeBase,
) error {
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
// 1. Get old chunk IDs for vector index copy mapping
oldChunks, err := s.chunkRepo.ListChunksByKnowledgeID(ctx, tenantID, knowledge.ID)
@@ -872,7 +887,16 @@ func (s *knowledgeService) moveKnowledgeReuseVectors(
// 2. Copy vector indices from source KB to target KB
if len(chunkIDMapping) > 0 && knowledge.EmbeddingModelID != "" {
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
// reuse_vectors mode copies index entries directly between KBs, which
// only works inside the same VectorStore backend. Route through the
// source KB's binding; callers must reject cross-store moves before
// landing here.
var sourceStoreID *string
if sourceKB != nil {
sourceStoreID = sourceKB.VectorStoreID
}
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, sourceStoreID)
if err != nil {
return fmt.Errorf("failed to init retrieve engine: %w", err)
}
@@ -98,10 +98,18 @@ func (s *knowledgeService) DeleteKnowledge(ctx context.Context, id string) error
// and GetEmbeddingModel would fail with "model ID cannot be empty".
if strings.TrimSpace(knowledge.EmbeddingModelID) != "" {
wg.Go(func() error {
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
// kb was already loaded above for resolveFileService — reuse its
// VectorStoreID for engine routing.
var boundStoreID *string
if kb != nil {
boundStoreID = kb.VectorStoreID
}
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx,
s.retrieveEngine,
tenantInfo.GetEffectiveEngines(),
s.ownership,
tenantID,
boundStoreID,
)
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
@@ -427,11 +435,14 @@ func (s *knowledgeService) DeleteKnowledgeList(ctx context.Context, ids []string
wg := errgroup.Group{}
// 2. Delete knowledge embeddings from vector store
wg.Go(func() error {
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
s.retrieveEngine,
tenantInfo.GetEffectiveEngines(),
)
tenantID := types.MustTenantIDFromContext(ctx)
// Batch cleanup spans multiple KBs that may be bound to different
// VectorStores; routing this batch through tenant effective engines
// keeps the legacy behavior intact.
// TODO: fan out the batch per-store using each KB's own
// VectorStoreID so cleanup hits the right backend for bound KBs.
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, nil)
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Errorf("DeleteKnowledge delete knowledge embedding failed")
return err
@@ -556,10 +567,21 @@ func (s *knowledgeService) cleanupKnowledgeResources(ctx context.Context, knowle
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if knowledge.EmbeddingModelID != "" {
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
s.retrieveEngine,
tenantInfo.GetEffectiveEngines(),
)
// Load KB to discover its VectorStoreID binding. Falls back to tenant
// effective engines if the KB has no binding or the load fails.
//
// Silent fallback risk: if a bound KB fails to load here due to a
// transient DB error, the cleanup will delete from env engines and
// leave orphan vectors in the bound store. Warn so operators can spot it.
var boundStoreID *string
if kb, loadErr := s.kbService.GetKnowledgeBaseByID(ctx, knowledge.KnowledgeBaseID); loadErr == nil && kb != nil {
boundStoreID = kb.VectorStoreID
} else if loadErr != nil {
logger.GetLogger(ctx).WithField("error", loadErr).WithField("knowledge_base_id", knowledge.KnowledgeBaseID).
Warnf("cleanupKnowledgeResources: failed to load KB for vector store resolution; falling back to tenant effective engines")
}
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, boundStoreID)
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Error("Failed to init retrieve engine during cleanup")
cleanupErr = errors.Join(cleanupErr, err)
+10 -19
View File
@@ -421,8 +421,8 @@ func (s *knowledgeService) UpdateFAQEntry(ctx context.Context,
oldSimilarQuestionCount := len(oldSimilarQuestions)
newSimilarQuestionCount := len(meta.SimilarQuestions)
if questionIndexMode == types.FAQQuestionIndexModeSeparate && oldSimilarQuestionCount > newSimilarQuestionCount {
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, engineErr := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
retrieveEngine, engineErr := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), kb.VectorStoreID)
if engineErr == nil {
sourceIDsToDelete := make([]string, 0, oldSimilarQuestionCount-newSimilarQuestionCount)
for i := newSimilarQuestionCount; i < oldSimilarQuestionCount; i++ {
@@ -646,8 +646,8 @@ func (s *knowledgeService) UpdateFAQEntryStatus(ctx context.Context,
// Sync update to retriever engines
chunkStatusMap := map[string]bool{chunk.ID: isEnabled}
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
if err != nil {
return err
}
@@ -840,11 +840,8 @@ func (s *knowledgeService) UpdateFAQEntryFieldsBatch(ctx context.Context,
// Sync to retriever engines
if len(enabledUpdates) > 0 || len(tagUpdates) > 0 {
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
s.retrieveEngine,
tenantInfo.GetEffectiveEngines(),
)
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
if err != nil {
return err
}
@@ -902,11 +899,8 @@ func (s *knowledgeService) UpdateFAQEntryTag(ctx context.Context, kbID string, e
}
// Sync tag update to retriever engines
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
s.retrieveEngine,
tenantInfo.GetEffectiveEngines(),
)
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
if err != nil {
return err
}
@@ -1003,11 +997,8 @@ func (s *knowledgeService) UpdateFAQEntryTagBatch(ctx context.Context, kbID stri
for _, chunk := range chunksToUpdate {
tagUpdates[chunk.ID] = chunk.TagID
}
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
s.retrieveEngine,
tenantInfo.GetEffectiveEngines(),
)
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
if err != nil {
return err
}
@@ -1213,8 +1213,8 @@ func (s *knowledgeService) incrementalIndexFAQEntry(
) error {
indexStartTime := time.Now()
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), kb.VectorStoreID)
if err != nil {
return err
}
@@ -1360,7 +1360,8 @@ func (s *knowledgeService) indexFAQChunks(ctx context.Context,
logger.Debugf(ctx, "indexFAQChunks: starting to index %d chunks", len(chunks))
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
if err != nil {
return err
}
@@ -1467,7 +1468,8 @@ func (s *knowledgeService) deleteFAQChunkVectors(ctx context.Context,
return err
}
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
if err != nil {
return err
}
@@ -1688,10 +1690,8 @@ func (s *knowledgeService) ProcessFAQImport(ctx context.Context, t *asynq.Task)
// 删除索引数据
embeddingModel, err := s.modelService.GetEmbeddingModel(ctx, kb.EmbeddingModelID)
if err == nil {
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
s.retrieveEngine,
tenantInfo.GetEffectiveEngines(),
)
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
if err == nil {
chunkIDs := make([]string, 0, len(chunksDeleted))
for _, chunk := range chunksDeleted {
@@ -244,7 +244,8 @@ func (s *knowledgeService) processChunks(ctx context.Context,
// 删除旧的索引数据 — only when vector/keyword indexing is enabled
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
if err == nil && embeddingModel != nil {
if err := retrieveEngine.DeleteByKnowledgeIDList(ctx, []string{knowledge.ID}, embeddingModel.GetDimensions(), knowledge.Type); err != nil {
logger.Warnf(ctx, "Failed to delete existing index data (may not exist): %v", err)
@@ -953,7 +954,8 @@ func (s *knowledgeService) ProcessSummaryGeneration(ctx context.Context, t *asyn
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
if err != nil {
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
return fmt.Errorf("failed to init retrieve engine: %w", err)
@@ -1123,7 +1125,8 @@ func (s *knowledgeService) ProcessQuestionGeneration(ctx context.Context, t *asy
}
ctx = context.WithValue(ctx, types.TenantInfoContextKey, tenantInfo)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantInfo.ID, kb.VectorStoreID)
if err != nil {
exitStatus = "init_retrieve_engine_failed"
logger.Errorf(ctx, "Failed to init retrieve engine: %v", err)
@@ -1577,8 +1580,8 @@ func (s *knowledgeService) updateChunkVector(ctx context.Context, kbID string, c
ids = append(ids, chunk.ID)
}
tenantInfo := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, types.MustTenantIDFromContext(ctx), sourceKB.VectorStoreID)
if err != nil {
return err
}
+37 -3
View File
@@ -27,6 +27,7 @@ type knowledgeBaseService struct {
kbShareService interfaces.KBShareService
modelService interfaces.ModelService
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
tenantRepo interfaces.TenantRepository
fileSvc interfaces.FileService
graphEngine interfaces.RetrieveGraphRepository
@@ -41,6 +42,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
kbShareService interfaces.KBShareService,
modelService interfaces.ModelService,
retrieveEngine interfaces.RetrieveEngineRegistry,
ownership retriever.TenantStoreOwnership,
tenantRepo interfaces.TenantRepository,
fileSvc interfaces.FileService,
graphEngine interfaces.RetrieveGraphRepository,
@@ -54,6 +56,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
kbShareService: kbShareService,
modelService: modelService,
retrieveEngine: retrieveEngine,
ownership: ownership,
tenantRepo: tenantRepo,
fileSvc: fileSvc,
graphEngine: graphEngine,
@@ -364,9 +367,24 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
tenantID := types.MustTenantIDFromContext(ctx)
tenantInfo, _ := types.TenantInfoFromContext(ctx)
// Load the KB before soft-delete so we can snapshot its VectorStoreID
// into the async cleanup payload. GORM's soft-delete filter hides the
// row from subsequent reads, so this read must happen first.
kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"knowledge_base_id": id,
})
return err
}
var vectorStoreIDSnapshot *string
if kb != nil {
vectorStoreIDSnapshot = kb.VectorStoreID
}
// Step 1: Delete the knowledge base record first (mark as deleted)
logger.Infof(ctx, "Deleting knowledge base from database")
err := s.repo.DeleteKnowledgeBase(ctx, id)
err = s.repo.DeleteKnowledgeBase(ctx, id)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"knowledge_base_id": id,
@@ -384,6 +402,7 @@ func (s *knowledgeBaseService) DeleteKnowledgeBase(ctx context.Context, id strin
TenantID: tenantID,
KnowledgeBaseID: id,
EffectiveEngines: tenantInfo.GetEffectiveEngines(),
VectorStoreID: vectorStoreIDSnapshot, // snapshot taken before soft-delete
}
langfuse.InjectTracing(ctx, &payload)
@@ -444,12 +463,27 @@ func (s *knowledgeBaseService) ProcessKBDelete(ctx context.Context, t *asynq.Tas
logger.Infof(ctx, "Deleting all knowledge entries and their resources")
// Delete embeddings from vector store
// Delete embeddings from vector store.
// Resolve the engine via the factory, using the VectorStoreID captured
// at enqueue time (may be nil → falls back to payload.EffectiveEngines).
// If the payload references a store no longer owned/registered
// (e.g. tampered queue entry or a store that was deleted while the
// task sat in the queue), the factory returns a sentinel and we
// SkipRetry to avoid burning retries on an unrecoverable situation.
logger.Infof(ctx, "Deleting embeddings from vector store")
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(
retrieveEngine, err := retriever.CreateRetrieveEngineFromPayload(
ctx,
s.retrieveEngine,
s.ownership,
payload.TenantID,
payload.EffectiveEngines,
payload.VectorStoreID,
)
if errors.Is(err, retriever.ErrVectorStoreForbidden) ||
errors.Is(err, retriever.ErrVectorStoreNotFound) {
logger.Errorf(ctx, "KB delete task aborted: %v (tenant=%d, kb=%s)", err, payload.TenantID, payload.KnowledgeBaseID)
return asynq.SkipRetry
}
if err != nil {
logger.Warnf(ctx, "Failed to create retrieve engine: %v", err)
} else {
@@ -94,15 +94,12 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
logger.Infof(ctx, "Hybrid search parameters, knowledge base IDs: %v, query text: %s", searchKBIDs, params.QueryText)
// tenantInfo is consumed below for retrieval config (post-fusion step).
tenantInfo, _ := types.TenantInfoFromContext(ctx)
// Create a composite retrieval engine with tenant's configured retrievers
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, tenantInfo.GetEffectiveEngines())
if err != nil {
logger.Errorf(ctx, "Failed to create retrieval engine: %v", err)
return nil, err
}
// Resolve the primary KB first so the factory can route to the bound
// VectorStore (if any). When the KB has no binding the factory falls
// back to the tenant's effective engines.
kb, err := s.repo.GetKnowledgeBaseByID(ctx, id)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
@@ -111,6 +108,18 @@ func (s *knowledgeBaseService) HybridSearch(ctx context.Context,
return nil, err
}
tenantID := types.MustTenantIDFromContext(ctx)
// Create a composite retrieval engine. When the KB is bound to a store,
// the factory verifies tenant ownership and returns that store's engine;
// otherwise it falls back to the tenant's configured retrievers.
retrieveEngine, err := retriever.CreateRetrieveEngineForKB(
ctx, s.retrieveEngine, s.ownership, tenantID, kb.VectorStoreID)
if err != nil {
logger.Errorf(ctx, "Failed to create retrieval engine: %v", err)
return nil, err
}
// Use 5x over-retrieval to ensure sufficient candidates for RRF fusion and reranking.
// Scale proportionally when searching multiple KBs to maintain per-KB recall quality.
matchCount := max(params.MatchCount*5, 50) * len(searchKBIDs)
@@ -0,0 +1,170 @@
package retriever
import (
"context"
"errors"
"slices"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// Sentinel errors returned by factory functions. Callers may use errors.Is to
// classify. User-facing responses MUST wrap or replace these with generic
// messages — the sentinels intentionally omit store UUIDs to avoid enumeration
// leaks. Structured logs inside the factory record the tenant/store IDs.
var (
// ErrTenantInfoMissing is returned when the factory needs a tenant from
// context (synchronous, unbound KB path) and none is present.
ErrTenantInfoMissing = errors.New("tenant info not found in context")
// ErrVectorStoreNotFound is returned when the store ID is not registered
// (or an internal lookup for ownership failed). Async workers should treat
// this as non-retryable.
ErrVectorStoreNotFound = errors.New("vector store not available")
// ErrVectorStoreForbidden is returned when the resolved store is not
// owned by the given tenant. This guards against cross-tenant access
// in case the upstream validation layer has a gap. Async workers should
// treat this as non-retryable.
ErrVectorStoreForbidden = errors.New("vector store access denied")
)
// TenantStoreOwnership abstracts the lookup used by factory functions to
// verify that a given vector store ID is owned by the given tenant ID.
//
// Production implementations wrap the VectorStoreRepository; tests inject
// in-memory fakes so they can cover the ownership branches without touching
// a database.
type TenantStoreOwnership interface {
// StoreOwnedBy reports whether the store with the given ID is owned
// by the given tenant. When the store does not exist, it returns
// (false, nil). Errors are reserved for infrastructure failures such as
// database connectivity issues.
StoreOwnedBy(ctx context.Context, storeID string, tenantID uint64) (bool, error)
}
// CreateRetrieveEngineForKB returns a CompositeRetrieveEngine resolved from
// a KB's VectorStore binding.
//
// Resolution rules:
//
// - vectorStoreID == nil || *vectorStoreID == "" →
// falls back to the tenant's effective engines (env-store flow driven
// by RETRIEVE_DRIVER). TenantInfo is read from ctx.
// - otherwise →
// 1) ownership.StoreOwnedBy(*storeID, tenantID) must return true;
// cross-tenant attempts yield ErrVectorStoreForbidden.
// 2) registry.GetByStoreID(*storeID) must succeed;
// unregistered stores yield ErrVectorStoreNotFound.
// 3) the single engine is wrapped by NewCompositeRetrieveEngine so
// that its Support()-based retriever-type matching is preserved.
//
// Use this for 23 synchronous call sites across the application services.
// Async task handlers that cannot rely on ctx-based TenantInfo (currently:
// ProcessKBDeleteTask, ProcessIndexDelete) must use
// CreateRetrieveEngineFromPayload instead.
func CreateRetrieveEngineForKB(
ctx context.Context,
registry interfaces.RetrieveEngineRegistry,
ownership TenantStoreOwnership,
tenantID uint64,
vectorStoreID *string,
) (*CompositeRetrieveEngine, error) {
// Normalize nil and empty-string pointer to "unbound" so that callers
// cannot accidentally route an empty UUID into GetByStoreID.
if vectorStoreID == nil || *vectorStoreID == "" {
tenantInfo, ok := types.TenantInfoFromContext(ctx)
if !ok {
return nil, ErrTenantInfoMissing
}
return NewCompositeRetrieveEngine(registry, tenantInfo.GetEffectiveEngines())
}
return resolveBoundEngine(ctx, registry, ownership, tenantID, *vectorStoreID)
}
// CreateRetrieveEngineFromPayload is the async-task variant. It does not
// read TenantInfo from ctx because async handlers do not populate it.
// Instead, tenantID is passed explicitly from the deserialized payload and
// is verified against the store's tenant when vectorStoreID is non-empty.
//
// Tasks enqueued before vectorStoreID was added to the payload decode it as
// nil and transparently fall back to the pre-serialized effectiveEngines
// path — no in-flight task is lost across upgrades.
func CreateRetrieveEngineFromPayload(
ctx context.Context,
registry interfaces.RetrieveEngineRegistry,
ownership TenantStoreOwnership,
tenantID uint64,
effectiveEngines []types.RetrieverEngineParams,
vectorStoreID *string,
) (*CompositeRetrieveEngine, error) {
if vectorStoreID == nil || *vectorStoreID == "" {
return NewCompositeRetrieveEngine(registry, effectiveEngines)
}
return resolveBoundEngine(ctx, registry, ownership, tenantID, *vectorStoreID)
}
// resolveBoundEngine is the shared ownership-verified lookup path used by
// both CreateRetrieveEngineForKB and CreateRetrieveEngineFromPayload. It
// returns sentinel errors so that handlers can classify them (for example,
// async workers convert Forbidden/NotFound into asynq.SkipRetry).
func resolveBoundEngine(
ctx context.Context,
registry interfaces.RetrieveEngineRegistry,
ownership TenantStoreOwnership,
tenantID uint64,
storeID string,
) (*CompositeRetrieveEngine, error) {
owned, err := ownership.StoreOwnedBy(ctx, storeID, tenantID)
if err != nil {
// Infrastructure failure — record the raw error for operators but
// do not leak internals to the caller.
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"tenant_id": tenantID,
"store_id": storeID,
"reason": "ownership lookup failed",
})
return nil, ErrVectorStoreNotFound
}
if !owned {
// Cross-tenant attempt (or the store has been deleted in the
// meantime). Log with WARN so that audits can surface probing.
logger.Warnf(ctx,
"[retriever.factory] cross-tenant store access attempted: tenant=%d store=%s",
tenantID, storeID)
return nil, ErrVectorStoreForbidden
}
svc, err := registry.GetByStoreID(storeID)
if err != nil {
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"tenant_id": tenantID,
"store_id": storeID,
"reason": "store not registered",
})
return nil, ErrVectorStoreNotFound
}
// Build the composite directly from the resolved service.
//
// We cannot delegate to NewCompositeRetrieveEngine here because that
// function resolves engines through registry.GetRetrieveEngineService,
// which reads from the byEngineType map (env stores). DB stores live
// in the byStoreID map and are not reachable via engine type alone —
// multiple stores can share the same engine type.
//
// Semantics: a KB bound to a DB store uses every retriever type that
// store supports. This intentionally overrides the tenant-level
// effective-engines filter, because binding a KB to a specific store
// is an explicit opt-out of tenant-default routing.
return &CompositeRetrieveEngine{
engineInfos: []*engineInfo{{
retrieveEngine: svc,
retrieverType: slices.Clone(svc.Support()),
}},
}, nil
}
@@ -0,0 +1,389 @@
package retriever
import (
"context"
"encoding/json"
"errors"
"sync"
"testing"
"github.com/Tencent/WeKnora/internal/models/embedding"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// ----- test fakes -----
// fakeOwnership is an in-memory TenantStoreOwnership implementation.
// All mutable state is guarded by mu so that the race test exercises a
// safe fake and surfaces only genuine races inside the factory code.
type fakeOwnership struct {
mu sync.Mutex
// owned maps storeID → tenantID that owns it. Absence means "not owned".
owned map[string]uint64
// err, when non-nil, is returned from every StoreOwnedBy call
// (used to simulate infrastructure failures).
err error
// calls records every (storeID, tenantID) pair asked. Tests use this
// to assert that the factory short-circuits to the fallback path
// without calling ownership when vectorStoreID is nil/empty.
calls []ownershipCall
}
type ownershipCall struct {
storeID string
tenantID uint64
}
func (f *fakeOwnership) StoreOwnedBy(ctx context.Context, storeID string, tenantID uint64) (bool, error) {
f.mu.Lock()
f.calls = append(f.calls, ownershipCall{storeID: storeID, tenantID: tenantID})
err := f.err
ownedTenant, ok := f.owned[storeID]
f.mu.Unlock()
if err != nil {
return false, err
}
return ok && ownedTenant == tenantID, nil
}
// callCount returns the number of StoreOwnedBy calls recorded so far.
// Safe for concurrent use alongside StoreOwnedBy.
func (f *fakeOwnership) callCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.calls)
}
// fakeEngine implements interfaces.RetrieveEngineService with only the
// methods that factory paths exercise. All other methods panic so that
// accidental use during testing is loud.
type fakeEngine struct {
engineType types.RetrieverEngineType
support []types.RetrieverType
}
func (f *fakeEngine) EngineType() types.RetrieverEngineType { return f.engineType }
func (f *fakeEngine) Support() []types.RetrieverType { return f.support }
func (f *fakeEngine) Retrieve(ctx context.Context, _ types.RetrieveParams) ([]*types.RetrieveResult, error) {
panic("fakeEngine.Retrieve: not used in factory tests")
}
func (f *fakeEngine) Index(ctx context.Context, _ embedding.Embedder, _ *types.IndexInfo, _ []types.RetrieverType) error {
panic("fakeEngine.Index: not used in factory tests")
}
func (f *fakeEngine) BatchIndex(ctx context.Context, _ embedding.Embedder, _ []*types.IndexInfo, _ []types.RetrieverType) error {
panic("fakeEngine.BatchIndex: not used in factory tests")
}
func (f *fakeEngine) EstimateStorageSize(ctx context.Context, _ embedding.Embedder, _ []*types.IndexInfo, _ []types.RetrieverType) int64 {
panic("fakeEngine.EstimateStorageSize: not used in factory tests")
}
func (f *fakeEngine) CopyIndices(ctx context.Context, _ string, _ map[string]string, _ map[string]string, _ string, _ int, _ string) error {
panic("fakeEngine.CopyIndices: not used in factory tests")
}
func (f *fakeEngine) DeleteByChunkIDList(ctx context.Context, _ []string, _ int, _ string) error {
panic("fakeEngine.DeleteByChunkIDList: not used in factory tests")
}
func (f *fakeEngine) DeleteBySourceIDList(ctx context.Context, _ []string, _ int, _ string) error {
panic("fakeEngine.DeleteBySourceIDList: not used in factory tests")
}
func (f *fakeEngine) DeleteByKnowledgeIDList(ctx context.Context, _ []string, _ int, _ string) error {
panic("fakeEngine.DeleteByKnowledgeIDList: not used in factory tests")
}
func (f *fakeEngine) BatchUpdateChunkEnabledStatus(ctx context.Context, _ map[string]bool) error {
panic("fakeEngine.BatchUpdateChunkEnabledStatus: not used in factory tests")
}
func (f *fakeEngine) BatchUpdateChunkTagID(ctx context.Context, _ map[string]string) error {
panic("fakeEngine.BatchUpdateChunkTagID: not used in factory tests")
}
// registryWithStores builds a registry with the given storeID → service
// pairs populated in the byStoreID map. Engine-type entries are also
// registered so that the unbound path can resolve engines via the
// tenant's effective engines.
func registryWithStores(t *testing.T, stores map[string]*fakeEngine, engineTypes map[types.RetrieverEngineType]*fakeEngine) *RetrieveEngineRegistry {
t.Helper()
r := &RetrieveEngineRegistry{
byEngineType: map[types.RetrieverEngineType]interfaces.RetrieveEngineService{},
byStoreID: map[string]interfaces.RetrieveEngineService{},
}
for id, svc := range stores {
r.byStoreID[id] = svc
}
for et, svc := range engineTypes {
r.byEngineType[et] = svc
}
return r
}
// newTenantCtx returns a context carrying a Tenant with the given
// EffectiveEngines. Factory's unbound path consumes this via
// types.TenantInfoFromContext.
func newTenantCtx(engines []types.RetrieverEngineParams) context.Context {
tenant := &types.Tenant{
RetrieverEngines: types.RetrieverEngines{Engines: engines},
}
return context.WithValue(context.Background(), types.TenantInfoContextKey, tenant)
}
// ----- CreateRetrieveEngineForKB -----
func TestCreateRetrieveEngineForKB_Unbound(t *testing.T) {
postgresEngine := &fakeEngine{
engineType: types.PostgresRetrieverEngineType,
support: []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType},
}
registry := registryWithStores(t, nil, map[types.RetrieverEngineType]*fakeEngine{
types.PostgresRetrieverEngineType: postgresEngine,
})
ownership := &fakeOwnership{}
tenantCtx := newTenantCtx([]types.RetrieverEngineParams{
{RetrieverEngineType: types.PostgresRetrieverEngineType, RetrieverType: types.VectorRetrieverType},
})
cases := []struct {
name string
store *string
}{
{"nil pointer", nil},
{"empty string pointer", strPtr("")},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
engine, err := CreateRetrieveEngineForKB(tenantCtx, registry, ownership, 1, tc.store)
require.NoError(t, err)
require.NotNil(t, engine)
assert.Len(t, engine.engineInfos, 1, "unbound path uses tenant effective engines")
assert.Same(t, postgresEngine, engine.engineInfos[0].retrieveEngine)
assert.Zero(t, ownership.callCount(),
"ownership must not be called on the unbound path")
})
}
}
func TestCreateRetrieveEngineForKB_UnboundMissingTenant(t *testing.T) {
registry := registryWithStores(t, nil, nil)
ownership := &fakeOwnership{}
_, err := CreateRetrieveEngineForKB(context.Background(), registry, ownership, 1, nil)
require.Error(t, err)
assert.True(t, errors.Is(err, ErrTenantInfoMissing),
"err %q must wrap ErrTenantInfoMissing", err)
}
func TestCreateRetrieveEngineForKB_StoreBound(t *testing.T) {
esEngine := &fakeEngine{
engineType: types.ElasticsearchRetrieverEngineType,
support: []types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType},
}
registry := registryWithStores(t,
map[string]*fakeEngine{"store-A": esEngine},
nil,
)
ownership := &fakeOwnership{owned: map[string]uint64{"store-A": 1}}
storeID := "store-A"
engine, err := CreateRetrieveEngineForKB(context.Background(), registry, ownership, 1, &storeID)
require.NoError(t, err)
require.NotNil(t, engine)
require.Len(t, engine.engineInfos, 1)
assert.Same(t, esEngine, engine.engineInfos[0].retrieveEngine)
assert.Equal(t,
[]types.RetrieverType{types.KeywordsRetrieverType, types.VectorRetrieverType},
engine.engineInfos[0].retrieverType,
"store-bound KB uses every retriever type the store supports")
}
func TestCreateRetrieveEngineForKB_CrossTenant(t *testing.T) {
esEngine := &fakeEngine{
engineType: types.ElasticsearchRetrieverEngineType,
support: []types.RetrieverType{types.VectorRetrieverType},
}
registry := registryWithStores(t,
map[string]*fakeEngine{"store-A": esEngine},
nil,
)
// store-A is owned by tenant 2, not tenant 1
ownership := &fakeOwnership{owned: map[string]uint64{"store-A": 2}}
storeID := "store-A"
_, err := CreateRetrieveEngineForKB(context.Background(), registry, ownership, 1, &storeID)
require.Error(t, err)
assert.True(t, errors.Is(err, ErrVectorStoreForbidden),
"cross-tenant access must yield ErrVectorStoreForbidden; got %q", err)
// Sanity: no store UUID in the sentinel's message — the structured log
// emitted inside the factory is where UUIDs live. The user-visible
// error is intentionally opaque.
assert.NotContains(t, err.Error(), "store-A",
"sentinel error must not expose store UUIDs to callers")
}
func TestCreateRetrieveEngineForKB_StoreNotRegistered(t *testing.T) {
// Ownership says the tenant owns the store, but registry has not
// loaded it. This happens when a VectorStore row exists in the DB
// but its engine initialization failed at app start.
registry := registryWithStores(t, nil, nil)
ownership := &fakeOwnership{owned: map[string]uint64{"store-A": 1}}
storeID := "store-A"
_, err := CreateRetrieveEngineForKB(context.Background(), registry, ownership, 1, &storeID)
require.Error(t, err)
assert.True(t, errors.Is(err, ErrVectorStoreNotFound),
"unregistered store must yield ErrVectorStoreNotFound; got %q", err)
}
func TestCreateRetrieveEngineForKB_OwnershipLookupError(t *testing.T) {
registry := registryWithStores(t,
map[string]*fakeEngine{"store-A": {engineType: types.PostgresRetrieverEngineType}},
nil,
)
ownership := &fakeOwnership{err: errors.New("db connection refused")}
storeID := "store-A"
_, err := CreateRetrieveEngineForKB(context.Background(), registry, ownership, 1, &storeID)
require.Error(t, err)
assert.True(t, errors.Is(err, ErrVectorStoreNotFound),
"ownership infrastructure error must collapse to ErrVectorStoreNotFound; got %q", err)
}
// ----- CreateRetrieveEngineFromPayload -----
func TestCreateRetrieveEngineFromPayload_LegacyUnbound(t *testing.T) {
// Four ways a payload can express "no binding": missing field,
// explicit null, empty string, and programmatically nil. All four
// should take the fallback path using the payload's effectiveEngines.
postgresEngine := &fakeEngine{
engineType: types.PostgresRetrieverEngineType,
support: []types.RetrieverType{types.VectorRetrieverType},
}
registry := registryWithStores(t, nil, map[types.RetrieverEngineType]*fakeEngine{
types.PostgresRetrieverEngineType: postgresEngine,
})
ownership := &fakeOwnership{}
engines := []types.RetrieverEngineParams{{
RetrieverEngineType: types.PostgresRetrieverEngineType,
RetrieverType: types.VectorRetrieverType,
}}
// simulate payload decoding from JSON for each legacy shape
decode := func(t *testing.T, body string) *string {
t.Helper()
var p types.KBDeletePayload
require.NoError(t, json.Unmarshal([]byte(body), &p))
return p.VectorStoreID
}
cases := []struct {
name string
ptr *string
}{
{"missing field", decode(t, `{"tenant_id":1}`)},
{"explicit null", decode(t, `{"tenant_id":1,"vector_store_id":null}`)},
{"empty string", decode(t, `{"tenant_id":1,"vector_store_id":""}`)},
{"programmatic nil", nil},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
engine, err := CreateRetrieveEngineFromPayload(
context.Background(), registry, ownership, 1, engines, tc.ptr)
require.NoError(t, err)
require.NotNil(t, engine)
require.Len(t, engine.engineInfos, 1)
assert.Same(t, postgresEngine, engine.engineInfos[0].retrieveEngine)
assert.Zero(t, ownership.callCount(),
"unbound payload must not trigger ownership lookup")
})
}
}
func TestCreateRetrieveEngineFromPayload_Bound(t *testing.T) {
qdrantEngine := &fakeEngine{
engineType: types.QdrantRetrieverEngineType,
support: []types.RetrieverType{types.VectorRetrieverType},
}
registry := registryWithStores(t,
map[string]*fakeEngine{"qd-1": qdrantEngine},
nil,
)
ownership := &fakeOwnership{owned: map[string]uint64{"qd-1": 42}}
storeID := "qd-1"
engine, err := CreateRetrieveEngineFromPayload(
context.Background(), registry, ownership, 42, nil, &storeID)
require.NoError(t, err)
require.NotNil(t, engine)
require.Len(t, engine.engineInfos, 1)
assert.Same(t, qdrantEngine, engine.engineInfos[0].retrieveEngine)
}
func TestCreateRetrieveEngineFromPayload_TamperedCrossTenant(t *testing.T) {
esEngine := &fakeEngine{engineType: types.ElasticsearchRetrieverEngineType,
support: []types.RetrieverType{types.VectorRetrieverType}}
registry := registryWithStores(t,
map[string]*fakeEngine{"store-A": esEngine}, nil)
// Store is owned by tenant 99, but the (possibly tampered) payload
// claims tenant 1. Factory must reject.
ownership := &fakeOwnership{owned: map[string]uint64{"store-A": 99}}
storeID := "store-A"
_, err := CreateRetrieveEngineFromPayload(
context.Background(), registry, ownership, 1,
[]types.RetrieverEngineParams{}, &storeID)
require.Error(t, err)
assert.True(t, errors.Is(err, ErrVectorStoreForbidden))
}
// ----- Race -----
// TestFactoryParallelInvocation is primarily meaningful when run under
// `go test -race`. Factory functions themselves are stateless; the test
// guards against accidental shared state being introduced later.
func TestFactoryParallelInvocation(t *testing.T) {
esEngine := &fakeEngine{
engineType: types.ElasticsearchRetrieverEngineType,
support: []types.RetrieverType{types.VectorRetrieverType},
}
registry := registryWithStores(t,
map[string]*fakeEngine{"store-A": esEngine},
nil,
)
ownership := &fakeOwnership{owned: map[string]uint64{"store-A": 1}}
done := make(chan error, 16)
for i := 0; i < 16; i++ {
go func() {
storeID := "store-A"
_, err := CreateRetrieveEngineForKB(
context.Background(), registry, ownership, 1, &storeID)
done <- err
}()
}
for i := 0; i < 16; i++ {
if err := <-done; err != nil {
t.Errorf("concurrent call %d failed: %v", i, err)
}
}
}
// ----- helpers -----
func strPtr(s string) *string { return &s }
@@ -0,0 +1,35 @@
package retriever
import (
"context"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// vectorStoreRepoOwnership adapts a VectorStoreRepository to the
// TenantStoreOwnership interface consumed by the factory functions.
//
// The repository's GetByID already scopes to the given tenantID
// (WHERE id = ? AND tenant_id = ?), so "exists under this tenant"
// is equivalent to "owned by this tenant" — no additional tenant
// comparison is needed on top of the returned row.
type vectorStoreRepoOwnership struct {
repo interfaces.VectorStoreRepository
}
// NewVectorStoreRepoOwnership returns the production
// TenantStoreOwnership implementation backed by VectorStoreRepository.
func NewVectorStoreRepoOwnership(repo interfaces.VectorStoreRepository) TenantStoreOwnership {
return &vectorStoreRepoOwnership{repo: repo}
}
// StoreOwnedBy returns true iff a vector store with the given ID exists
// under the given tenant. Errors are reserved for infrastructure failures;
// a non-existent (but well-formed) store ID returns (false, nil).
func (o *vectorStoreRepoOwnership) StoreOwnedBy(ctx context.Context, storeID string, tenantID uint64) (bool, error) {
store, err := o.repo.GetByID(ctx, tenantID, storeID)
if err != nil {
return false, err
}
return store != nil, nil
}
@@ -112,3 +112,7 @@ func (r *RetrieveEngineRegistry) UnregisterByStoreID(storeID string) {
delete(r.byStoreID, storeID)
}
// Compile-time assertion: *RetrieveEngineRegistry satisfies the
// interfaces.RetrieveEngineRegistry contract, including GetByStoreID.
var _ interfaces.RetrieveEngineRegistry = (*RetrieveEngineRegistry)(nil)
+26 -5
View File
@@ -25,6 +25,7 @@ type knowledgeTagService struct {
knowledgeRepo interfaces.KnowledgeRepository
chunkRepo interfaces.ChunkRepository
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
modelService interfaces.ModelService
task interfaces.TaskEnqueuer
kbShareService interfaces.KBShareService
@@ -37,6 +38,7 @@ func NewKnowledgeTagService(
knowledgeRepo interfaces.KnowledgeRepository,
chunkRepo interfaces.ChunkRepository,
retrieveEngine interfaces.RetrieveEngineRegistry,
ownership retriever.TenantStoreOwnership,
modelService interfaces.ModelService,
task interfaces.TaskEnqueuer,
kbShareService interfaces.KBShareService,
@@ -47,6 +49,7 @@ func NewKnowledgeTagService(
knowledgeRepo: knowledgeRepo,
chunkRepo: chunkRepo,
retrieveEngine: retrieveEngine,
ownership: ownership,
modelService: modelService,
task: task,
kbShareService: kbShareService,
@@ -257,7 +260,7 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
// Enqueue async index deletion task for the deleted chunks
if len(deletedIDs) > 0 {
s.enqueueIndexDeleteTask(ctx, tenantID, kb.ID, kb.EmbeddingModelID, string(kb.Type), deletedIDs, tenantInfo.GetEffectiveEngines())
s.enqueueIndexDeleteTask(ctx, tenantID, kb.ID, kb.EmbeddingModelID, string(kb.Type), deletedIDs, tenantInfo.GetEffectiveEngines(), kb.VectorStoreID)
}
logger.Infof(ctx, "Deleted %d chunks under tag %s", len(deletedIDs), tag.ID)
@@ -341,9 +344,14 @@ func (s *knowledgeTagService) DeleteTag(ctx context.Context, id string, force bo
return s.repo.Delete(ctx, tenantID, id)
}
// enqueueIndexDeleteTask enqueues an async task for index deletion (low priority)
// enqueueIndexDeleteTask enqueues an async task for index deletion (low priority).
//
// vectorStoreID is captured from the owning KB at enqueue time and snapshotted
// into the payload so the worker can route to the correct store even if the
// KB is deleted or rebound before the task runs.
func (s *knowledgeTagService) enqueueIndexDeleteTask(ctx context.Context,
tenantID uint64, kbID, embeddingModelID, kbType string, chunkIDs []string, effectiveEngines []types.RetrieverEngineParams,
tenantID uint64, kbID, embeddingModelID, kbType string, chunkIDs []string,
effectiveEngines []types.RetrieverEngineParams, vectorStoreID *string,
) {
payload := types.IndexDeletePayload{
TenantID: tenantID,
@@ -352,6 +360,7 @@ func (s *knowledgeTagService) enqueueIndexDeleteTask(ctx context.Context,
KBType: kbType,
ChunkIDs: chunkIDs,
EffectiveEngines: effectiveEngines,
VectorStoreID: vectorStoreID,
}
langfuse.InjectTracing(ctx, &payload)
payloadBytes, err := json.Marshal(payload)
@@ -382,8 +391,20 @@ func (s *knowledgeTagService) ProcessIndexDelete(ctx context.Context, t *asynq.T
logger.Infof(ctx, "Processing index delete task for %d chunks in KB %s", len(payload.ChunkIDs), payload.KnowledgeBaseID)
// Create retrieve engine
retrieveEngine, err := retriever.NewCompositeRetrieveEngine(s.retrieveEngine, payload.EffectiveEngines)
// Create retrieve engine.
// The factory verifies the payload's tenant owns the bound store, so a
// tampered queue entry cannot reach a cross-tenant store.
// Forbidden/NotFound are non-retryable — SkipRetry prevents burning the
// full retry budget on an unrecoverable task.
retrieveEngine, err := retriever.CreateRetrieveEngineFromPayload(
ctx, s.retrieveEngine, s.ownership,
payload.TenantID, payload.EffectiveEngines, payload.VectorStoreID,
)
if errors.Is(err, retriever.ErrVectorStoreForbidden) ||
errors.Is(err, retriever.ErrVectorStoreNotFound) {
logger.Errorf(ctx, "Index delete task aborted: %v (tenant=%d, kb=%s)", err, payload.TenantID, payload.KnowledgeBaseID)
return asynq.SkipRetry
}
if err != nil {
logger.Warnf(ctx, "Failed to create retrieve engine for index cleanup: %v", err)
return err
+3
View File
@@ -204,6 +204,9 @@ func BuildContainer(container *dig.Container) *dig.Container {
must(container.Invoke(registerWebSearchProviders))
must(container.Provide(repository.NewWebSearchProviderRepository))
must(container.Provide(repository.NewVectorStoreRepository))
// TenantStoreOwnership adapter used by the retriever factory functions
// to verify that a resolved VectorStore belongs to the caller's tenant.
must(container.Provide(retriever.NewVectorStoreRepoOwnership))
must(container.Provide(service.NewWebSearchService))
must(container.Provide(service.NewWebSearchProviderService))
must(container.Provide(NewEngineFactory))
+9
View File
@@ -72,6 +72,15 @@ type RetrieveEngineRegistry interface {
GetRetrieveEngineService(engineType types.RetrieverEngineType) (RetrieveEngineService, error)
// GetAllRetrieveEngineServices gets all retrieve engine services
GetAllRetrieveEngineServices() []RetrieveEngineService
// GetByStoreID returns the engine service registered for a specific DB store ID.
//
// IMPORTANT: This method does NOT verify tenant ownership of the returned
// store. Callers MUST use the CreateRetrieveEngineForKB /
// CreateRetrieveEngineFromPayload factory functions in the retriever package
// rather than calling this directly. The factories wrap GetByStoreID with
// tenant ownership verification (defense-in-depth against cross-tenant IDOR).
GetByStoreID(storeID string) (RetrieveEngineService, error)
}
// RetrieveEngineService defines the retrieve engine service interface
+8
View File
@@ -99,6 +99,10 @@ type IndexDeletePayload struct {
KBType string `json:"kb_type"`
ChunkIDs []string `json:"chunk_ids"`
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
// VectorStoreID is the bound store snapshot taken at enqueue time so the
// async worker can resolve the same store the KB was bound to.
// nil means the KB had no binding — falls back to EffectiveEngines.
VectorStoreID *string `json:"vector_store_id,omitempty"`
}
// KBDeletePayload represents the knowledge base delete task payload
@@ -107,6 +111,10 @@ type KBDeletePayload struct {
TenantID uint64 `json:"tenant_id"`
KnowledgeBaseID string `json:"knowledge_base_id"`
EffectiveEngines []RetrieverEngineParams `json:"effective_engines"`
// VectorStoreID is the bound store snapshot taken at enqueue time (before
// soft-delete) so the async worker can resolve the right store. nil means
// the KB had no binding — falls back to EffectiveEngines.
VectorStoreID *string `json:"vector_store_id,omitempty"`
}
// KnowledgeListDeletePayload represents the batch knowledge delete task payload