feat(storage): add multi-instance storage backends per workspace

Introduce tenant-scoped storage backend registry so workspaces can register
multiple storage instances and bind each knowledge base to a specific backend,
with migration from legacy tenant-level storage config and new settings UI.
This commit is contained in:
wizardchen
2026-07-15 13:26:29 +08:00
committed by lyingbug
parent 139ec1e6b8
commit f8b3e5f148
58 changed files with 2863 additions and 470 deletions
+81 -46
View File
@@ -27,21 +27,22 @@ var ErrInvalidTenantID = errors.New("invalid tenant ID")
// knowledgeBaseService implements the knowledge base service interface
type knowledgeBaseService struct {
repo interfaces.KnowledgeBaseRepository
kgRepo interfaces.KnowledgeRepository
chunkRepo interfaces.ChunkRepository
shareRepo interfaces.KBShareRepository
kbShareService interfaces.KBShareService
modelService interfaces.ModelService
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
tenantRepo interfaces.TenantRepository
fileSvc interfaces.FileService
graphEngine interfaces.RetrieveGraphRepository
asynqClient interfaces.TaskEnqueuer
dsRepo interfaces.DataSourceRepository
syncLogRepo interfaces.SyncLogRepository
dsScheduler *datasource.Scheduler
repo interfaces.KnowledgeBaseRepository
kgRepo interfaces.KnowledgeRepository
chunkRepo interfaces.ChunkRepository
shareRepo interfaces.KBShareRepository
kbShareService interfaces.KBShareService
modelService interfaces.ModelService
retrieveEngine interfaces.RetrieveEngineRegistry
ownership retriever.TenantStoreOwnership
tenantRepo interfaces.TenantRepository
fileSvc interfaces.FileService
storageResolver interfaces.StorageBackendResolver
graphEngine interfaces.RetrieveGraphRepository
asynqClient interfaces.TaskEnqueuer
dsRepo interfaces.DataSourceRepository
syncLogRepo interfaces.SyncLogRepository
dsScheduler *datasource.Scheduler
}
// NewKnowledgeBaseService creates a new knowledge base service
@@ -55,6 +56,7 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
ownership retriever.TenantStoreOwnership,
tenantRepo interfaces.TenantRepository,
fileSvc interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
graphEngine interfaces.RetrieveGraphRepository,
asynqClient interfaces.TaskEnqueuer,
dsRepo interfaces.DataSourceRepository,
@@ -62,21 +64,22 @@ func NewKnowledgeBaseService(repo interfaces.KnowledgeBaseRepository,
dsScheduler *datasource.Scheduler,
) interfaces.KnowledgeBaseService {
return &knowledgeBaseService{
repo: repo,
kgRepo: kgRepo,
chunkRepo: chunkRepo,
shareRepo: shareRepo,
kbShareService: kbShareService,
modelService: modelService,
retrieveEngine: retrieveEngine,
ownership: ownership,
tenantRepo: tenantRepo,
fileSvc: fileSvc,
graphEngine: graphEngine,
asynqClient: asynqClient,
dsRepo: dsRepo,
syncLogRepo: syncLogRepo,
dsScheduler: dsScheduler,
repo: repo,
kgRepo: kgRepo,
chunkRepo: chunkRepo,
shareRepo: shareRepo,
kbShareService: kbShareService,
modelService: modelService,
retrieveEngine: retrieveEngine,
ownership: ownership,
tenantRepo: tenantRepo,
fileSvc: fileSvc,
storageResolver: storageResolver,
graphEngine: graphEngine,
asynqClient: asynqClient,
dsRepo: dsRepo,
syncLogRepo: syncLogRepo,
dsScheduler: dsScheduler,
}
}
@@ -119,6 +122,9 @@ func (s *knowledgeBaseService) CreateKnowledgeBase(ctx context.Context,
}
kb.EnsureDefaults()
applyTenantDefaultStorageProvider(ctx, kb)
if err := s.applyAndValidateStorageBackend(ctx, kb); err != nil {
return nil, err
}
// Fold empty-string vector_store_id into nil so this path and the
// retrieve-engine factory's pre-condition share a single representation.
@@ -150,6 +156,37 @@ func (s *knowledgeBaseService) CreateKnowledgeBase(ctx context.Context,
return kb, nil
}
func (s *knowledgeBaseService) applyAndValidateStorageBackend(ctx context.Context, kb *types.KnowledgeBase) error {
if s.storageResolver == nil || kb == nil {
return nil
}
tenant, _ := types.TenantInfoFromContext(ctx)
if tenant == nil {
return apperrors.NewBadRequestError("workspace context missing")
}
id := ""
if kb.StorageBackendID != nil {
id = strings.TrimSpace(*kb.StorageBackendID)
}
provider := kb.GetStorageProvider()
// A newly created KB without an explicit instance follows the concrete
// tenant default. The legacy provider is only a fallback for workspaces
// that have not been migrated yet.
if id == "" && tenant.DefaultStorageBackendID != nil && strings.TrimSpace(*tenant.DefaultStorageBackendID) != "" {
provider = ""
}
backend, err := s.storageResolver.ResolveBackend(ctx, tenant, id, provider)
if err != nil {
return apperrors.NewBadRequestError("storage backend is unavailable").WithDetails(err.Error())
}
if backend == nil {
return nil
}
kb.StorageBackendID = &backend.ID
kb.SetStorageProvider(backend.Provider)
return nil
}
// applyTenantDefaultStorageProvider fills an empty KB storage provider from the
// tenant's global default (Settings → Storage engine). Frontend should send the
// same value; this keeps API clients and legacy UIs consistent.
@@ -978,22 +1015,19 @@ func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context,
"cross-store cloning is not yet supported")
}
// Defense 3: storage backend must match — only meaningful when the
// tenant has a StorageEngineConfig. Without it, resolveFileService
// ignores per-KB provider pins and routes ALL KBs to the global
// storage service, so a clone can never span two real backends and
// the pins must NOT be used to reject (that would be a false positive).
// When a tenant config exists, pins are honored, so compare effective
// providers and reject a genuine cross-backend clone up front (it would
// otherwise fail mid-clone with ErrCrossBackendCopy).
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil && tenant.StorageEngineConfig != nil {
tenantDefault := tenant.StorageEngineConfig.DefaultProvider
srcProvider := sourceKB.EffectiveStorageProvider(tenantDefault)
dstProvider := targetKB.EffectiveStorageProvider(tenantDefault)
if srcProvider != "" && dstProvider != "" && srcProvider != dstProvider {
return nil, nil, apperrors.NewBadRequestError(fmt.Sprintf(
"source and target knowledge bases use different storage backends (%s vs %s); "+
"cross-storage-backend cloning is not supported", srcProvider, dstProvider))
// Defense 3: the concrete storage instance must match. Comparing only
// provider names would incorrectly allow COS-A -> COS-B clones.
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil {
defaultID, defaultProvider := "", ""
if tenant.DefaultStorageBackendID != nil {
defaultID = *tenant.DefaultStorageBackendID
}
if tenant.StorageEngineConfig != nil {
defaultProvider = tenant.StorageEngineConfig.DefaultProvider
}
if !sourceKB.SharesStorageBackendWith(targetKB, defaultID, defaultProvider) {
return nil, nil, apperrors.NewBadRequestError(
"source and target knowledge bases use different storage instances; cross-storage-backend cloning is not supported")
}
}
} else {
@@ -1016,6 +1050,7 @@ func (s *knowledgeBaseService) CopyKnowledgeBase(ctx context.Context,
SummaryModelID: sourceKB.SummaryModelID,
VLMConfig: sourceKB.VLMConfig,
StorageProviderConfig: sourceKB.StorageProviderConfig,
StorageBackendID: sourceKB.StorageBackendID,
StorageConfig: sourceKB.StorageConfig,
FAQConfig: faqConfig,
VectorStoreID: sourceKB.VectorStoreID,