7478e3cddb
Extracts the 25 repeated NewCompositeRetrieveEngine call sites across seven services into two factory functions with tenant-ownership verification: CreateRetrieveEngineForKB (synchronous) and CreateRetrieveEngineFromPayload (async task handlers). Promotes GetByStoreID from concrete-only to interfaces.RetrieveEngineRegistry. Extends KBDeletePayload / IndexDeletePayload with an omitempty VectorStoreID snapshot. Design highlights - Factory verifies tenant ownership of the resolved store (defense-in-depth) so a gap in PR3 validation or a tampered Asynq payload cannot cross tenants. Cross-tenant attempts return ErrVectorStoreForbidden with a structured log entry. - Sentinel errors (ErrTenantInfoMissing, ErrVectorStoreNotFound, ErrVectorStoreForbidden) let async handlers classify non-retryable failures as asynq.SkipRetry. - nil and empty-string pointers for vectorStoreID normalize to "unbound" so callers never send an empty UUID to GetByStoreID. - resolveBoundEngine constructs engineInfos directly (with slices.Clone of Support()) instead of going through NewCompositeRetrieveEngine. The latter only reads from the byEngineType map and cannot reach DB stores in byStoreID -- multiple stores can share the same engine type. Unbound fallbacks still reuse NewCompositeRetrieveEngine so the tenant effective-engines path is byte-for-byte unchanged. - DeleteKnowledgeBase now reads the KB before soft-delete so the enqueued KBDeletePayload carries a VectorStoreID snapshot; once the row is soft-deleted GORM's default scope hides it. Cost: +1 SELECT per KB-delete request (a rare admin operation). - enqueueIndexDeleteTask gains a vectorStoreID *string parameter that the single caller (tag.go) populates from the owning KB. The worker (ProcessIndexDelete) validates ownership via the factory. - cleanupKnowledgeResources loads the KB so bound-store cleanup routes correctly. If the load fails it warns and falls back to tenant effective engines; orphan vectors become observable in logs instead of disappearing silently. - DeleteKnowledges (batch) stays on the tenant effective-engines path -- batches can span stores and multi-store fan-out is PR4 scope. Noted inline. Merge safety All knowledge bases have VectorStoreID = NULL at merge time (introduced nullable in PR1, Tencent/WeKnora#994). Every factory call falls back to NewCompositeRetrieveEngine(registry, tenantInfo.GetEffectiveEngines()), which is the existing code path for these KBs. Async payloads from before this change decode to VectorStoreID = nil (omitempty) and take the same fallback. Testing - go build ./... - go vet ./... - go test -race -count=1 ./internal/application/service/retriever/... covers unbound (nil/empty), bound, cross-tenant, unregistered store, ownership infra error, legacy payload shapes (missing field/null/ empty string/nil), tampered payload, and parallel invocation. - grep -r NewCompositeRetrieveEngine internal/application/service returns zero results outside factory.go/composite.go. Refs: Part of Tencent/WeKnora#993. Depends on Tencent/WeKnora#994.
119 lines
4.1 KiB
Go
119 lines
4.1 KiB
Go
package retriever
|
|
|
|
import (
|
|
"fmt"
|
|
"sync"
|
|
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
)
|
|
|
|
// RetrieveEngineRegistry implements the retrieval engine registry.
|
|
// It maintains two maps:
|
|
// - byEngineType: env stores registered via RETRIEVE_DRIVER (backward compatible)
|
|
// - byStoreID: DB stores registered via VectorStore table (instance-based)
|
|
//
|
|
// Implements both interfaces.RetrieveEngineRegistry and interfaces.StoreRegistry.
|
|
type RetrieveEngineRegistry struct {
|
|
byEngineType map[types.RetrieverEngineType]interfaces.RetrieveEngineService
|
|
byStoreID map[string]interfaces.RetrieveEngineService
|
|
mu sync.RWMutex
|
|
}
|
|
|
|
// NewRetrieveEngineRegistry creates a new retrieval engine registry
|
|
func NewRetrieveEngineRegistry() interfaces.RetrieveEngineRegistry {
|
|
return &RetrieveEngineRegistry{
|
|
byEngineType: make(map[types.RetrieverEngineType]interfaces.RetrieveEngineService),
|
|
byStoreID: make(map[string]interfaces.RetrieveEngineService),
|
|
}
|
|
}
|
|
|
|
// --- interfaces.RetrieveEngineRegistry methods (unchanged behavior) ---
|
|
|
|
// Register registers a retrieval engine service by engine type.
|
|
// Returns an error if the engine type is already registered.
|
|
func (r *RetrieveEngineRegistry) Register(repo interfaces.RetrieveEngineService) error {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
if _, exists := r.byEngineType[repo.EngineType()]; exists {
|
|
return fmt.Errorf("repository type %s already registered", repo.EngineType())
|
|
}
|
|
|
|
r.byEngineType[repo.EngineType()] = repo
|
|
return nil
|
|
}
|
|
|
|
// GetRetrieveEngineService retrieves a retrieval engine service by type.
|
|
// Only searches the byEngineType map (env stores).
|
|
func (r *RetrieveEngineRegistry) GetRetrieveEngineService(repoType types.RetrieverEngineType) (
|
|
interfaces.RetrieveEngineService, error,
|
|
) {
|
|
r.mu.RLock()
|
|
defer r.mu.RUnlock()
|
|
|
|
repo, exists := r.byEngineType[repoType]
|
|
if !exists {
|
|
return nil, fmt.Errorf("repository of type %s not found", repoType)
|
|
}
|
|
|
|
return repo, nil
|
|
}
|
|
|
|
// GetAllRetrieveEngineServices retrieves all registered retrieval engine services.
|
|
// Only returns byEngineType entries (env stores) for backward compatibility.
|
|
func (r *RetrieveEngineRegistry) GetAllRetrieveEngineServices() []interfaces.RetrieveEngineService {
|
|
r.mu.RLock()
|
|
defer r.mu.RUnlock()
|
|
|
|
result := make([]interfaces.RetrieveEngineService, 0, len(r.byEngineType))
|
|
for _, v := range r.byEngineType {
|
|
result = append(result, v)
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// --- interfaces.StoreRegistry methods (new, for VectorStore-based engines) ---
|
|
|
|
// RegisterWithStoreID registers an engine service by VectorStore ID.
|
|
// Unlike Register(), the same EngineType can be registered multiple times
|
|
// with different StoreIDs (e.g., two Elasticsearch clusters).
|
|
// Upsert semantics: existing entry is overwritten silently.
|
|
func (r *RetrieveEngineRegistry) RegisterWithStoreID(storeID string, svc interfaces.RetrieveEngineService) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
r.byStoreID[storeID] = svc
|
|
}
|
|
|
|
// GetByStoreID retrieves an engine service by VectorStore ID.
|
|
// Callers must verify tenant ownership before using the returned service.
|
|
func (r *RetrieveEngineRegistry) GetByStoreID(storeID string) (interfaces.RetrieveEngineService, error) {
|
|
r.mu.RLock()
|
|
defer r.mu.RUnlock()
|
|
|
|
svc, exists := r.byStoreID[storeID]
|
|
if !exists {
|
|
return nil, fmt.Errorf("store %s not found in registry", storeID)
|
|
}
|
|
return svc, nil
|
|
}
|
|
|
|
// UnregisterByStoreID removes an engine service from the byStoreID map.
|
|
// Idempotent: returns silently if the storeID is not found.
|
|
//
|
|
// NOTE: gRPC-based clients (Qdrant, Milvus) hold connections that are not closed here.
|
|
// Known Phase 1 limitation — store deletion is rare, connections cleaned up on process exit.
|
|
// Phase 2 should add Close() to RetrieveEngineService interface and call it here.
|
|
func (r *RetrieveEngineRegistry) UnregisterByStoreID(storeID string) {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
|
|
delete(r.byStoreID, storeID)
|
|
}
|
|
|
|
// Compile-time assertion: *RetrieveEngineRegistry satisfies the
|
|
// interfaces.RetrieveEngineRegistry contract, including GetByStoreID.
|
|
var _ interfaces.RetrieveEngineRegistry = (*RetrieveEngineRegistry)(nil)
|