feat(storage): add multi-instance storage backends per workspace

Introduce tenant-scoped storage backend registry so workspaces can register
multiple storage instances and bind each knowledge base to a specific backend,
with migration from legacy tenant-level storage config and new settings UI.
This commit is contained in:
wizardchen
2026-07-15 13:26:29 +08:00
committed by lyingbug
parent 139ec1e6b8
commit f8b3e5f148
58 changed files with 2863 additions and 470 deletions
+29 -2
View File
@@ -63,6 +63,7 @@ type InitializationHandler struct {
ollamaService *ollama.OllamaService
documentReader interfaces.DocumentReader
pooler embedding.EmbedderPooler
storageResolver interfaces.StorageBackendResolver
}
// NewInitializationHandler 创建初始化处理器
@@ -76,6 +77,7 @@ func NewInitializationHandler(
ollamaService *ollama.OllamaService,
documentReader interfaces.DocumentReader,
pooler embedding.EmbedderPooler,
storageResolver interfaces.StorageBackendResolver,
) *InitializationHandler {
return &InitializationHandler{
config: config,
@@ -87,6 +89,7 @@ func NewInitializationHandler(
ollamaService: ollamaService,
documentReader: documentReader,
pooler: pooler,
storageResolver: storageResolver,
}
}
@@ -123,7 +126,8 @@ type KBModelConfigRequest struct {
} `json:"multimodal"`
// 存储引擎选择("local" | "minio" | "cos"),影响文档上传与文档内图片存储,参数从全局设置读取
StorageProvider string `json:"storageProvider"`
StorageProvider string `json:"storageProvider"`
StorageBackendID string `json:"storageBackendId"`
// 知识图谱配置
NodeExtract struct {
@@ -362,7 +366,30 @@ func (h *InitializationHandler) UpdateKBConfig(c *gin.Context) {
kb.VLMConfig.CustomInstructions = strings.TrimSpace(req.VLMConfig.CustomInstructions)
}
// 存储引擎:仅写入 provider 到新字段,参数从空间全局 StorageEngineConfig 读取
// Bind the concrete storage instance. Provider remains a compatibility
// projection for older clients and historical rows.
if strings.TrimSpace(req.StorageBackendID) != "" {
tenant, _ := types.TenantInfoFromContext(ctx)
backend, resolveErr := h.storageResolver.ResolveBackend(ctx, tenant, req.StorageBackendID, "")
if resolveErr != nil || backend == nil {
c.Error(errors.NewBadRequestError("Storage backend is unavailable"))
return
}
oldID := ""
if kb.StorageBackendID != nil {
oldID = *kb.StorageBackendID
}
if oldID != "" && oldID != backend.ID {
knowledgeList, listErr := h.knowledgeService.ListPagedKnowledgeByKnowledgeBaseID(ctx, kbIdStr, &types.Pagination{Page: 1, PageSize: 1}, types.KnowledgeListFilter{})
if listErr == nil && knowledgeList != nil && knowledgeList.Total > 0 {
c.Error(errors.NewBadRequestError("Storage backend cannot be changed while the knowledge base contains files; migrate storage first"))
return
}
}
kb.StorageBackendID = &backend.ID
req.StorageProvider = backend.Provider
}
// Legacy provider projection.
provider := strings.ToLower(strings.TrimSpace(req.StorageProvider))
if provider == "" {
provider = "local"
+12 -14
View File
@@ -1055,21 +1055,19 @@ func (h *KnowledgeBaseHandler) CopyKnowledgeBase(c *gin.Context) {
"cross-store cloning is not yet supported"))
return
}
// Pre-flight defense 3: storage backend must match — only meaningful
// when the tenant has a StorageEngineConfig. Without it,
// resolveFileService ignores per-KB provider pins and routes ALL KBs to
// the global storage service, so a clone can never span two real
// backends and the pins must NOT be used to reject (false positive).
// When a tenant config exists, pins are honored, so reject a genuine
// cross-backend clone before enqueueing.
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil && tenant.StorageEngineConfig != nil {
tenantDefault := tenant.StorageEngineConfig.DefaultProvider
srcProvider := sourceKB.EffectiveStorageProvider(tenantDefault)
dstProvider := targetKB.EffectiveStorageProvider(tenantDefault)
if srcProvider != "" && dstProvider != "" && srcProvider != dstProvider {
// Pre-flight defense 3: compare concrete instance IDs, not just the
// provider type (COS-A and COS-B are different physical stores).
if tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant); tenant != nil {
defaultID, defaultProvider := "", ""
if tenant.DefaultStorageBackendID != nil {
defaultID = *tenant.DefaultStorageBackendID
}
if tenant.StorageEngineConfig != nil {
defaultProvider = tenant.StorageEngineConfig.DefaultProvider
}
if !sourceKB.SharesStorageBackendWith(targetKB, defaultID, defaultProvider) {
c.Error(apperrors.NewBadRequestError(
"source and target knowledge bases use different storage backends (" +
srcProvider + " vs " + dstProvider + "); cross-storage-backend cloning is not supported"))
"source and target knowledge bases use different storage instances; cross-storage-backend cloning is not supported"))
return
}
}
+6 -3
View File
@@ -27,9 +27,10 @@ type Handler struct {
agentShareService interfaces.AgentShareService // Service for resolving shared agents (KB scope in retrieval)
kbShareService interfaces.KBShareService // Service for resolving shared KB permissions
fileService interfaces.FileService // Service for file storage (image uploads)
modelService interfaces.ModelService // Service for model management (VLM access)
userService interfaces.UserService // Service for resolving per-user preferences (e.g. enable_memory default)
attachmentProcessor *AttachmentProcessor // Processor for file attachments
storageResolver interfaces.StorageBackendResolver
modelService interfaces.ModelService // Service for model management (VLM access)
userService interfaces.UserService // Service for resolving per-user preferences (e.g. enable_memory default)
attachmentProcessor *AttachmentProcessor // Processor for file attachments
}
// NewHandler creates a new instance of Handler with all necessary dependencies
@@ -45,6 +46,7 @@ func NewHandler(
agentShareService interfaces.AgentShareService,
kbShareService interfaces.KBShareService,
fileService interfaces.FileService,
storageResolver interfaces.StorageBackendResolver,
modelService interfaces.ModelService,
userService interfaces.UserService,
documentReader interfaces.DocumentReader,
@@ -62,6 +64,7 @@ func NewHandler(
agentShareService: agentShareService,
kbShareService: kbShareService,
fileService: fileService,
storageResolver: storageResolver,
modelService: modelService,
userService: userService,
attachmentProcessor: NewAttachmentProcessor(
+13 -4
View File
@@ -141,12 +141,21 @@ func mimeToExt(mime string) string {
}
func (h *Handler) resolveImageFileService(ctx context.Context, storageProvider string) interfaces.FileService {
if strings.TrimSpace(storageProvider) == "" {
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil {
return h.fileService
}
tenant, _ := ctx.Value(types.TenantInfoContextKey).(*types.Tenant)
if tenant == nil || tenant.StorageEngineConfig == nil {
if h.storageResolver != nil {
svc, resolvedProvider, err := h.storageResolver.ResolveFileService(ctx, tenant, "", storageProvider, "")
if err == nil && svc != nil {
logger.Infof(ctx, "[image-storage] using storage instance provider=%s for image uploads", resolvedProvider)
return svc
}
if err != nil {
logger.Warnf(ctx, "[image-storage] failed to resolve storage instance for provider=%s: %v", storageProvider, err)
}
}
if strings.TrimSpace(storageProvider) == "" || tenant.StorageEngineConfig == nil {
return h.fileService
}
+149
View File
@@ -0,0 +1,149 @@
package handler
import (
"net/http"
apperrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/storageallowlist"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/gin-gonic/gin"
)
type StorageBackendHandler struct {
repo interfaces.StorageBackendRepository
service interfaces.StorageBackendService
}
func NewStorageBackendHandler(repo interfaces.StorageBackendRepository, service interfaces.StorageBackendService) *StorageBackendHandler {
return &StorageBackendHandler{repo: repo, service: service}
}
type storageBackendRequest struct {
Name string `json:"name" binding:"required"`
Provider string `json:"provider" binding:"required"`
Config types.StorageBackendConfig `json:"config"`
Status string `json:"status,omitempty"`
}
func storageTenantID(c *gin.Context) uint64 { return c.GetUint64(types.TenantIDContextKey.String()) }
func (h *StorageBackendHandler) List(c *gin.Context) {
tenantID := storageTenantID(c)
backends, err := h.repo.List(c.Request.Context(), tenantID)
if err != nil {
c.Error(err)
return
}
result := make([]types.StorageBackend, 0, len(backends))
for _, backend := range backends {
result = append(result, types.NewStorageBackendResponse(backend))
}
tenant, _ := types.TenantInfoFromContext(c.Request.Context())
var defaultID *string
if tenant != nil {
defaultID = tenant.DefaultStorageBackendID
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": result, "default_storage_backend_id": defaultID})
}
func (h *StorageBackendHandler) Get(c *gin.Context) {
backend, err := h.repo.GetByID(c.Request.Context(), storageTenantID(c), c.Param("id"))
if err != nil {
c.Error(err)
return
}
if backend == nil {
c.Error(apperrors.NewNotFoundError("storage backend not found"))
return
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": types.NewStorageBackendResponse(backend)})
}
func (h *StorageBackendHandler) Create(c *gin.Context) {
var req storageBackendRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.Error(apperrors.NewBadRequestError(err.Error()))
return
}
backend := &types.StorageBackend{TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config, Status: req.Status}
if err := h.service.Create(c.Request.Context(), backend); err != nil {
c.Error(err)
return
}
c.JSON(http.StatusCreated, gin.H{"success": true, "data": types.NewStorageBackendResponse(backend)})
}
func (h *StorageBackendHandler) Update(c *gin.Context) {
var req storageBackendRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.Error(apperrors.NewBadRequestError(err.Error()))
return
}
backend := &types.StorageBackend{ID: c.Param("id"), TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config, Status: req.Status}
if err := h.service.Update(c.Request.Context(), backend); err != nil {
c.Error(err)
return
}
updated, err := h.repo.GetByID(c.Request.Context(), backend.TenantID, backend.ID)
if err != nil {
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": types.NewStorageBackendResponse(updated)})
}
func (h *StorageBackendHandler) Delete(c *gin.Context) {
if err := h.service.Delete(c.Request.Context(), storageTenantID(c), c.Param("id")); err != nil {
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{"success": true})
}
func (h *StorageBackendHandler) SetDefault(c *gin.Context) {
if err := h.service.SetDefault(c.Request.Context(), storageTenantID(c), c.Param("id")); err != nil {
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{"success": true})
}
func (h *StorageBackendHandler) TestRaw(c *gin.Context) {
var req storageBackendRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.Error(apperrors.NewBadRequestError(err.Error()))
return
}
backend := &types.StorageBackend{TenantID: storageTenantID(c), Name: req.Name, Provider: req.Provider, Config: req.Config}
if err := backend.Validate(); err != nil {
c.Error(err)
return
}
if err := h.service.Test(c.Request.Context(), backend); err != nil {
c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"success": true})
}
func (h *StorageBackendHandler) TestByID(c *gin.Context) {
backend, err := h.repo.GetByID(c.Request.Context(), storageTenantID(c), c.Param("id"))
if err != nil {
c.Error(err)
return
}
if backend == nil {
c.Error(apperrors.NewNotFoundError("storage backend not found"))
return
}
if err := h.service.Test(c.Request.Context(), backend); err != nil {
c.JSON(http.StatusOK, gin.H{"success": false, "error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"success": true})
}
func (h *StorageBackendHandler) Types(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"success": true, "data": storageallowlist.AllowedList()})
}