feat(storage): add multi-instance storage backends per workspace
Introduce tenant-scoped storage backend registry so workspaces can register multiple storage instances and bind each knowledge base to a specific backend, with migration from legacy tenant-level storage config and new settings UI.
This commit is contained in:
@@ -63,6 +63,7 @@ type InitializationHandler struct {
|
||||
ollamaService *ollama.OllamaService
|
||||
documentReader interfaces.DocumentReader
|
||||
pooler embedding.EmbedderPooler
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
}
|
||||
|
||||
// NewInitializationHandler 创建初始化处理器
|
||||
@@ -76,6 +77,7 @@ func NewInitializationHandler(
|
||||
ollamaService *ollama.OllamaService,
|
||||
documentReader interfaces.DocumentReader,
|
||||
pooler embedding.EmbedderPooler,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
) *InitializationHandler {
|
||||
return &InitializationHandler{
|
||||
config: config,
|
||||
@@ -87,6 +89,7 @@ func NewInitializationHandler(
|
||||
ollamaService: ollamaService,
|
||||
documentReader: documentReader,
|
||||
pooler: pooler,
|
||||
storageResolver: storageResolver,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,7 +126,8 @@ type KBModelConfigRequest struct {
|
||||
} `json:"multimodal"`
|
||||
|
||||
// 存储引擎选择("local" | "minio" | "cos"),影响文档上传与文档内图片存储,参数从全局设置读取
|
||||
StorageProvider string `json:"storageProvider"`
|
||||
StorageProvider string `json:"storageProvider"`
|
||||
StorageBackendID string `json:"storageBackendId"`
|
||||
|
||||
// 知识图谱配置
|
||||
NodeExtract struct {
|
||||
@@ -362,7 +366,30 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
|
||||
kb.VLMConfig.CustomInstructions = strings.TrimSpace(req.VLMConfig.CustomInstructions)
|
||||
}
|
||||
|
||||
// 存储引擎:仅写入 provider 到新字段,参数从空间全局 StorageEngineConfig 读取
|
||||
// Bind the concrete storage instance. Provider remains a compatibility
|
||||
// projection for older clients and historical rows.
|
||||
if strings.TrimSpace(req.StorageBackendID) != "" {
|
||||
tenant, _ := types.TenantInfoFromContext(ctx)
|
||||
backend, resolveErr := h.storageResolver.ResolveBackend(ctx, tenant, req.StorageBackendID, "")
|
||||
if resolveErr != nil || backend == nil {
|
||||
c.Error(errors.NewBadRequestError("Storage backend is unavailable"))
|
||||
return
|
||||
}
|
||||
oldID := ""
|
||||
if kb.StorageBackendID != nil {
|
||||
oldID = *kb.StorageBackendID
|
||||
}
|
||||
if oldID != "" && oldID != backend.ID {
|
||||
knowledgeList, listErr := h.knowledgeService.ListPagedKnowledgeByKnowledgeBaseID(ctx, kbIdStr, &types.Pagination{Page: 1, PageSize: 1}, types.KnowledgeListFilter{})
|
||||
if listErr == nil && knowledgeList != nil && knowledgeList.Total > 0 {
|
||||
c.Error(errors.NewBadRequestError("Storage backend cannot be changed while the knowledge base contains files; migrate storage first"))
|
||||
return
|
||||
}
|
||||
}
|
||||
kb.StorageBackendID = &backend.ID
|
||||
req.StorageProvider = backend.Provider
|
||||
}
|
||||
// Legacy provider projection.
|
||||
provider := strings.ToLower(strings.TrimSpace(req.StorageProvider))
|
||||
if provider == "" {
|
||||
provider = "local"
|
||||
|
||||
@@ -1055,21 +1055,19 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
|
||||
"cross-store cloning is not yet supported"))
|
||||
return
|
||||
}
|
||||
// Pre-flight defense 3: storage backend must match — only meaningful
|
||||
// when the tenant has a StorageEngineConfig. Without it,
|
||||
// resolveFileService ignores per-KB provider pins and routes ALL KBs to
|
||||
// the global storage service, so a clone can never span two real
|
||||
// backends and the pins must NOT be used to reject (false positive).
|
||||
// When a tenant config exists, pins are honored, so reject a genuine
|
||||
// cross-backend clone before enqueueing.
|
||||
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil && tenant.StorageEngineConfig != nil {
|
||||
tenantDefault := tenant.StorageEngineConfig.DefaultProvider
|
||||
srcProvider := sourceKB.EffectiveStorageProvider(tenantDefault)
|
||||
dstProvider := targetKB.EffectiveStorageProvider(tenantDefault)
|
||||
if srcProvider != "" && dstProvider != "" && srcProvider != dstProvider {
|
||||
// Pre-flight defense 3: compare concrete instance IDs, not just the
|
||||
// provider type (COS-A and COS-B are different physical stores).
|
||||
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil {
|
||||
defaultID, defaultProvider := "", ""
|
||||
if tenant.DefaultStorageBackendID != nil {
|
||||
defaultID = *tenant.DefaultStorageBackendID
|
||||
}
|
||||
if tenant.StorageEngineConfig != nil {
|
||||
defaultProvider = tenant.StorageEngineConfig.DefaultProvider
|
||||
}
|
||||
if !sourceKB.SharesStorageBackendWith(targetKB, defaultID, defaultProvider) {
|
||||
c.Error(apperrors.NewBadRequestError(
|
||||
"source and target knowledge bases use different storage backends (" +
|
||||
srcProvider + " vs " + dstProvider + "); cross-storage-backend cloning is not supported"))
|
||||
"source and target knowledge bases use different storage instances; cross-storage-backend cloning is not supported"))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,9 +27,10 @@ type Handler struct {
|
||||
agentShareService interfaces.AgentShareService // Service for resolving shared agents (KB scope in retrieval)
|
||||
kbShareService interfaces.KBShareService // Service for resolving shared KB permissions
|
||||
fileService interfaces.FileService // Service for file storage (image uploads)
|
||||
modelService interfaces.ModelService // Service for model management (VLM access)
|
||||
userService interfaces.UserService // Service for resolving per-user preferences (e.g. enable_memory default)
|
||||
attachmentProcessor *AttachmentProcessor // Processor for file attachments
|
||||
storageResolver interfaces.StorageBackendResolver
|
||||
modelService interfaces.ModelService // Service for model management (VLM access)
|
||||
userService interfaces.UserService // Service for resolving per-user preferences (e.g. enable_memory default)
|
||||
attachmentProcessor *AttachmentProcessor // Processor for file attachments
|
||||
}
|
||||
|
||||
// NewHandler creates a new instance of Handler with all necessary dependencies
|
||||
@@ -45,6 +46,7 @@ func NewHandler(
|
||||
agentShareService interfaces.AgentShareService,
|
||||
kbShareService interfaces.KBShareService,
|
||||
fileService interfaces.FileService,
|
||||
storageResolver interfaces.StorageBackendResolver,
|
||||
modelService interfaces.ModelService,
|
||||
userService interfaces.UserService,
|
||||
documentReader interfaces.DocumentReader,
|
||||
@@ -62,6 +64,7 @@ func NewHandler(
|
||||
agentShareService: agentShareService,
|
||||
kbShareService: kbShareService,
|
||||
fileService: fileService,
|
||||
storageResolver: storageResolver,
|
||||
modelService: modelService,
|
||||
userService: userService,
|
||||
attachmentProcessor: NewAttachmentProcessor(
|
||||
|
||||
@@ -141,12 +141,21 @@ func mimeToExt(mime string) string {
|
||||
}
|
||||
|
||||
func (h *Handler) resolveImageFileService(ctx context.Context, storageProvider string) interfaces.FileService {
|
||||
if strings.TrimSpace(storageProvider) == "" {
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if tenant == nil {
|
||||
return h.fileService
|
||||
}
|
||||
|
||||
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
|
||||
if tenant == nil || tenant.StorageEngineConfig == nil {
|
||||
if h.storageResolver != nil {
|
||||
svc, resolvedProvider, err := h.storageResolver.ResolveFileService(ctx, tenant, "", storageProvider, "")
|
||||
if err == nil && svc != nil {
|
||||
logger.Infof(ctx, "[image-storage] using storage instance provider=%s for image uploads", resolvedProvider)
|
||||
return svc
|
||||
}
|
||||
if err != nil {
|
||||
logger.Warnf(ctx, "[image-storage] failed to resolve storage instance for provider=%s: %v", storageProvider, err)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(storageProvider) == "" || tenant.StorageEngineConfig == nil {
|
||||
return h.fileService
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
apperrors "github.com/Tencent/WeKnora/internal/errors"
|
||||
"github.com/Tencent/WeKnora/internal/storageallowlist"
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type StorageBackendHandler struct {
|
||||
repo interfaces.StorageBackendRepository
|
||||
service interfaces.StorageBackendService
|
||||
}
|
||||
|
||||
func NewStorageBackendHandler(repo interfaces.StorageBackendRepository, service interfaces.StorageBackendService) *StorageBackendHandler {
|
||||
return &StorageBackendHandler{repo: repo, service: service}
|
||||
}
|
||||
|
||||
type storageBackendRequest struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
Provider string `json:"provider" binding:"required"`
|
||||
Config types.StorageBackendConfig `json:"config"`
|
||||
Status string `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
func storageTenantID(c *gin.Context) uint64 { return c.GetUint64(types.TenantIDContextKey.String()) }
|
||||
|
||||
func (h *StorageBackendHandler) List(c *gin.Context) {
|
||||
tenantID := storageTenantID(c)
|
||||
backends, err := h.repo.List(c.Request.Context(), tenantID)
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
result := make([]types.StorageBackend, 0, len(backends))
|
||||
for _, backend := range backends {
|
||||
result = append(result, types.NewStorageBackendResponse(backend))
|
||||
}
|
||||
tenant, _ := types.TenantInfoFromContext(c.Request.Context())
|
||||
var defaultID *string
|
||||
if tenant != nil {
|
||||
defaultID = tenant.DefaultStorageBackendID
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true, "data": result, "default_storage_backend_id": defaultID})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) Get(c *gin.Context) {
|
||||
backend, err := h.repo.GetByID(c.Request.Context(), storageTenantID(c), c.Param("id"))
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
if backend == nil {
|
||||
c.Error(apperrors.NewNotFoundError("storage backend not found"))
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true, "data": types.NewStorageBackendResponse(backend)})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) Create(c *gin.Context) {
|
||||
var req storageBackendRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.Error(apperrors.NewBadRequestError(err.Error()))
|
||||
return
|
||||
}
|
||||
backend := &types.StorageBackend{TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config, Status: req.Status}
|
||||
if err := h.service.Create(c.Request.Context(), backend); err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusCreated, gin.H{"success": true, "data": types.NewStorageBackendResponse(backend)})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) Update(c *gin.Context) {
|
||||
var req storageBackendRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.Error(apperrors.NewBadRequestError(err.Error()))
|
||||
return
|
||||
}
|
||||
backend := &types.StorageBackend{ID: c.Param("id"), TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config, Status: req.Status}
|
||||
if err := h.service.Update(c.Request.Context(), backend); err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
updated, err := h.repo.GetByID(c.Request.Context(), backend.TenantID, backend.ID)
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true, "data": types.NewStorageBackendResponse(updated)})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) Delete(c *gin.Context) {
|
||||
if err := h.service.Delete(c.Request.Context(), storageTenantID(c), c.Param("id")); err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) SetDefault(c *gin.Context) {
|
||||
if err := h.service.SetDefault(c.Request.Context(), storageTenantID(c), c.Param("id")); err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) TestRaw(c *gin.Context) {
|
||||
var req storageBackendRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.Error(apperrors.NewBadRequestError(err.Error()))
|
||||
return
|
||||
}
|
||||
backend := &types.StorageBackend{TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config}
|
||||
if err := backend.Validate(); err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
if err := h.service.Test(c.Request.Context(), backend); err != nil {
|
||||
c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) TestByID(c *gin.Context) {
|
||||
backend, err := h.repo.GetByID(c.Request.Context(), storageTenantID(c), c.Param("id"))
|
||||
if err != nil {
|
||||
c.Error(err)
|
||||
return
|
||||
}
|
||||
if backend == nil {
|
||||
c.Error(apperrors.NewNotFoundError("storage backend not found"))
|
||||
return
|
||||
}
|
||||
if err := h.service.Test(c.Request.Context(), backend); err != nil {
|
||||
c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"success": true})
|
||||
}
|
||||
|
||||
func (h *StorageBackendHandler) Types(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{"success": true, "data": storageallowlist.AllowedList()})
|
||||
}
|
||||
Reference in New Issue
Block a user